From d261b36460a73728cea9adf01844cd9e71c9c7a3 Mon Sep 17 00:00:00 2001 From: kroutony Date: Wed, 18 Mar 2026 13:45:52 +0000 Subject: [PATCH] Add hourly Slack trend report, log all HOLD reasons, whale correlation analysis MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - hourly_trend_report.py: standalone cron script (XX:00:30) sends 1h bullish/bearish status - slack_notifier.py: add send_market_trend_report() — simple bullish/bearish only, no entry signals - main.py: log all 15 HOLD reasons (not just first 3) for debugging all-HOLD cycles - backtest/whale_correlation.py: blockchain.com on-chain correlation analysis (result: no signal) - memory/: update project memory with architecture split, cron layout, feedback Co-Authored-By: Claude Opus 4.6 (1M context) --- backtest/whale_correlation.py | 237 +++++++++++++++++++++++ hourly_trend_report.py | 31 +++ main.py | 7 +- memory/MEMORY.md | 21 +- memory/feedback_no_misleading_signals.md | 11 ++ memory/project_architecture_split.md | 20 ++ memory/project_backtest_v3.md | 13 ++ memory/project_cron_timing.md | 12 +- memory/project_whale_correlation.md | 12 ++ slack_notifier.py | 47 +++++ 10 files changed, 397 insertions(+), 14 deletions(-) create mode 100644 backtest/whale_correlation.py create mode 100644 hourly_trend_report.py create mode 100644 memory/feedback_no_misleading_signals.md create mode 100644 memory/project_architecture_split.md create mode 100644 memory/project_backtest_v3.md create mode 100644 memory/project_whale_correlation.md diff --git a/backtest/whale_correlation.py b/backtest/whale_correlation.py new file mode 100644 index 0000000..7a98212 --- /dev/null +++ b/backtest/whale_correlation.py @@ -0,0 +1,237 @@ +#!/usr/bin/env python3 +"""Fetch blockchain.com on-chain metrics and analyze correlation with BTC price changes. + +Whale proxy metrics (all free, daily granularity): +- estimated-transaction-volume (BTC): total estimated tx volume +- n-transactions: daily confirmed transaction count +- Derived: avg_tx_size = volume / n_transactions (whale activity proxy) +- output-volume (BTC): total output value + +Correlation targets: +- BTC next-day return +- BTC next-3-day return +""" + +import os +import sys +import time + +import pandas as pd +import numpy as np +import requests + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +CACHE_DIR = os.path.join(os.path.dirname(__file__), "..", "cache", "backtest") +BTC_1H_CACHE = os.path.join(CACHE_DIR, "tBTCUST_1h.csv") + +BLOCKCHAIN_API = "https://api.blockchain.info/charts" +METRICS = [ + "estimated-transaction-volume", # BTC total est. tx volume + "estimated-transaction-volume-usd", # USD total est. tx volume + "n-transactions", # daily confirmed tx count + "output-volume", # total output value (BTC) + "n-unique-addresses", # unique addresses per day +] + + +def fetch_blockchain_metric(name: str, start: str, end: str) -> pd.DataFrame: + """Fetch a single blockchain.com chart metric.""" + start_ts = int(pd.Timestamp(start).timestamp()) + end_ts = int(pd.Timestamp(end).timestamp()) + # timespan is calculated from end; we use start param to set beginning + url = f"{BLOCKCHAIN_API}/{name}" + params = { + "format": "json", + "start": start_ts, + "timespan": "1year", # large enough window + } + resp = requests.get(url, params=params, timeout=30) + resp.raise_for_status() + data = resp.json() + + values = data.get("values", []) + if not values: + return pd.DataFrame() + + df = pd.DataFrame(values) + df.columns = ["timestamp", name] + df["date"] = pd.to_datetime(df["timestamp"], unit="s").dt.date + df = df[["date", name]] + + # Filter to requested range + start_date = pd.Timestamp(start).date() + end_date = pd.Timestamp(end).date() + df = df[(df["date"] >= start_date) & (df["date"] <= end_date)] + + return df + + +def load_btc_daily_prices() -> pd.DataFrame: + """Load BTC 1h cache and resample to daily OHLC.""" + if not os.path.exists(BTC_1H_CACHE): + print(f"ERROR: BTC 1h cache not found at {BTC_1H_CACHE}") + print("Run backtest first to populate the cache.") + sys.exit(1) + + df = pd.read_csv(BTC_1H_CACHE, parse_dates=["timestamp"]) + df["date"] = df["timestamp"].dt.date + daily = df.groupby("date").agg( + open=("open", "first"), + high=("high", "max"), + low=("low", "min"), + close=("close", "last"), + volume=("volume", "sum"), + ).reset_index() + return daily + + +def main(): + start = "2025-07-01" + end = "2026-03-17" + + print("=== Whale Activity ↔ BTC Price Correlation Analysis ===\n") + + # Step 1: Fetch on-chain metrics + print("Fetching blockchain.com metrics...") + metrics_dfs = [] + for metric in METRICS: + print(f" {metric}...", end=" ", flush=True) + try: + df = fetch_blockchain_metric(metric, start, end) + print(f"{len(df)} days") + metrics_dfs.append(df) + except Exception as e: + print(f"FAILED: {e}") + time.sleep(2) # rate limit: 1 req / 10 sec (be conservative) + + if not metrics_dfs: + print("ERROR: No metrics fetched") + return + + # Merge all metrics on date + onchain = metrics_dfs[0] + for df in metrics_dfs[1:]: + onchain = onchain.merge(df, on="date", how="outer") + onchain = onchain.sort_values("date").reset_index(drop=True) + + # Derived metrics + if "estimated-transaction-volume" in onchain.columns and "n-transactions" in onchain.columns: + onchain["avg_tx_size_btc"] = onchain["estimated-transaction-volume"] / onchain["n-transactions"] + if "estimated-transaction-volume-usd" in onchain.columns and "n-transactions" in onchain.columns: + onchain["avg_tx_size_usd"] = onchain["estimated-transaction-volume-usd"] / onchain["n-transactions"] + + print(f"\nOn-chain data: {len(onchain)} days") + + # Step 2: Load BTC prices + print("Loading BTC daily prices from cache...") + btc = load_btc_daily_prices() + print(f"BTC daily data: {len(btc)} days") + + # Step 3: Merge and compute returns + merged = onchain.merge(btc[["date", "close", "volume"]], on="date", how="inner") + merged = merged.rename(columns={"close": "btc_close", "volume": "btc_volume"}) + merged = merged.sort_values("date").reset_index(drop=True) + + # Price returns (forward-looking) + merged["ret_1d"] = merged["btc_close"].pct_change().shift(-1) # next-day return + merged["ret_3d"] = merged["btc_close"].pct_change(3).shift(-3) # next-3-day return + merged["ret_5d"] = merged["btc_close"].pct_change(5).shift(-5) # next-5-day return + + # Z-score normalization for on-chain metrics (rolling 30-day) + onchain_cols = [c for c in merged.columns if c not in + ["date", "btc_close", "btc_volume", "ret_1d", "ret_3d", "ret_5d"]] + + for col in onchain_cols: + roll_mean = merged[col].rolling(30, min_periods=10).mean() + roll_std = merged[col].rolling(30, min_periods=10).std() + merged[f"{col}_zscore"] = (merged[col] - roll_mean) / roll_std.replace(0, np.nan) + + # Step 4: Correlation analysis + print(f"\nMerged dataset: {len(merged)} days") + print(f"Date range: {merged['date'].iloc[0]} to {merged['date'].iloc[-1]}") + + # Raw correlations + zscore_cols = [c for c in merged.columns if c.endswith("_zscore")] + target_cols = ["ret_1d", "ret_3d", "ret_5d"] + + print("\n" + "=" * 70) + print(" PEARSON CORRELATION: On-Chain Metrics ↔ BTC Forward Returns") + print("=" * 70) + + valid = merged.dropna(subset=target_cols + zscore_cols) + print(f" (Using {len(valid)} complete observations)\n") + + results = [] + for oc_col in zscore_cols: + for target in target_cols: + corr = valid[oc_col].corr(valid[target]) + results.append({"metric": oc_col, "target": target, "corr": corr}) + + results_df = pd.DataFrame(results) + + # Print as pivot table + pivot = results_df.pivot(index="metric", columns="target", values="corr") + pivot = pivot[target_cols] # order columns + + # Sort by absolute correlation with ret_1d + pivot["abs_ret_1d"] = pivot["ret_1d"].abs() + pivot = pivot.sort_values("abs_ret_1d", ascending=False) + pivot = pivot.drop(columns="abs_ret_1d") + + for metric in pivot.index: + name = metric.replace("_zscore", "") + vals = " ".join(f"{pivot.loc[metric, t]:+.4f}" for t in target_cols) + print(f" {name:<35s} {vals}") + + print(f"\n {'':35s} {'ret_1d':>8s} {'ret_3d':>8s} {'ret_5d':>8s}") + + # Step 5: Highlight significant correlations + print("\n" + "=" * 70) + print(" NOTABLE CORRELATIONS (|r| > 0.10)") + print("=" * 70) + + notable = results_df[results_df["corr"].abs() > 0.10].sort_values("corr", key=abs, ascending=False) + if notable.empty: + print(" None found — on-chain metrics show weak correlation with BTC returns.") + else: + for _, row in notable.iterrows(): + direction = "↑↑" if row["corr"] > 0 else "↓↑" if row["corr"] < 0 else " " + name = row["metric"].replace("_zscore", "") + print(f" {direction} {name:<35s} → {row['target']}: r={row['corr']:+.4f}") + + # Step 6: Extreme value analysis (whale spikes) + print("\n" + "=" * 70) + print(" EXTREME VALUE ANALYSIS (Top/Bottom 10% Days)") + print("=" * 70) + + for col_name in ["avg_tx_size_btc", "estimated-transaction-volume", "avg_tx_size_usd"]: + zscore_col = f"{col_name}_zscore" + if zscore_col not in merged.columns: + continue + + valid_ext = merged.dropna(subset=[zscore_col, "ret_1d"]) + if len(valid_ext) < 20: + continue + + q10 = valid_ext[zscore_col].quantile(0.10) + q90 = valid_ext[zscore_col].quantile(0.90) + + low_days = valid_ext[valid_ext[zscore_col] <= q10] + high_days = valid_ext[valid_ext[zscore_col] >= q90] + all_avg = valid_ext["ret_1d"].mean() + + print(f"\n {col_name}:") + print(f" Low activity days (bottom 10%): avg next-day ret = {low_days['ret_1d'].mean():+.4f} (n={len(low_days)})") + print(f" High activity days (top 10%): avg next-day ret = {high_days['ret_1d'].mean():+.4f} (n={len(high_days)})") + print(f" All days average: avg next-day ret = {all_avg:+.4f} (n={len(valid_ext)})") + + # Save merged data for further analysis + out_path = os.path.join(CACHE_DIR, "whale_correlation_data.csv") + merged.to_csv(out_path, index=False) + print(f"\nSaved merged dataset to {out_path}") + print("Done.") + + +if __name__ == "__main__": + main() diff --git a/hourly_trend_report.py b/hourly_trend_report.py new file mode 100644 index 0000000..aba3573 --- /dev/null +++ b/hourly_trend_report.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +"""Send hourly 1h market trend report to Slack.""" + +import sys +import os + +sys.path.insert(0, os.path.dirname(__file__)) + +import data_fetcher +import indicators +import slack_notifier + + +def main(): + market_data = data_fetcher.fetch_all_market_data() + + htf_by_symbol = {} + current_prices = {} + for sym, md in market_data.items(): + ticker = md.get("ticker", {}) + if ticker: + current_prices[sym] = ticker.get("last_price", 0) + candles_htf = md.get("candles_htf") + if candles_htf is not None and not candles_htf.empty: + htf_by_symbol[sym] = indicators.calculate_htf_indicators(candles_htf) + + slack_notifier.send_market_trend_report(htf_by_symbol, current_prices) + + +if __name__ == "__main__": + main() diff --git a/main.py b/main.py index ab12413..a4dfa40 100644 --- a/main.py +++ b/main.py @@ -358,10 +358,9 @@ def run_cycle(): logger.info("LLM signal: %s %s conf=%.2f reason=%s", s.get("action"), s.get("symbol"), s.get("confidence", 0), s.get("reason", "")) else: - # Debug: log a sample of HOLD reasons to diagnose all-HOLD cycles - samples = signals[:3] - for s in samples: - logger.info("LLM HOLD sample: %s conf=%.2f reason=%s", + # Debug: log ALL hold reasons (not just first 3) to diagnose all-HOLD cycles + for s in signals: + logger.info("LLM HOLD: %s conf=%.2f reason=%s", s.get("symbol"), s.get("confidence", 0), s.get("reason", "")) except Exception as e: logger.error("LLM analysis failed: %s", e) diff --git a/memory/MEMORY.md b/memory/MEMORY.md index 526fefc..1cc5967 100644 --- a/memory/MEMORY.md +++ b/memory/MEMORY.md @@ -1,8 +1,17 @@ # Memory Index -- [user_profile.md](user_profile.md) — User role: crypto trader running Bitfinex bot, communicates in Traditional Chinese -- [project_trading_bot.md](project_trading_bot.md) — Key architecture: stop-loss sync, sell logic, order sizing, exposure, post-trade refresh, report format -- [project_cost_basis_sync.md](project_cost_basis_sync.md) — sync_cost_basis.py: order history cost calculation, wallet sync, Bitfinex API quirks -- [project_cron_timing.md](project_cron_timing.md) — Crontab timing: main.py :01/:06, sync :02/:32, offset from candle close -- [feedback_trading.md](feedback_trading.md) — User feedback: real-time stop-loss, no exposure limit, cost-basis order sizing -- [feedback_api_errors.md](feedback_api_errors.md) — Bitfinex 500 error patterns: stale stop IDs, min order size, balance locking, cancel not-found +## User +- [user_profile.md](user_profile.md) — 繁中溝通、Bitfinex 現貨交易者 + +## Feedback +- [feedback_trading.md](feedback_trading.md) — 止損用即時資料、無總曝險上限、成本基礎下單 +- [feedback_api_errors.md](feedback_api_errors.md) — Bitfinex 500 錯誤模式與修正 +- [feedback_no_misleading_signals.md](feedback_no_misleading_signals.md) — 報告不要暗示可進場，只報市場狀態 + +## Project +- [project_trading_bot.md](project_trading_bot.md) — 核心架構：止損、SELL、下單、曝險、報告格式 +- [project_architecture_split.md](project_architecture_split.md) — Production 用 LLM，Backtest 用規則引擎，兩者獨立 +- [project_cost_basis_sync.md](project_cost_basis_sync.md) — sync_cost_basis.py：訂單歷史成本計算 +- [project_cron_timing.md](project_cron_timing.md) — Cron 排程：交易 cycle、趨勢報告、成本同步、錯誤監控 +- [project_backtest_v3.md](project_backtest_v3.md) — V3 回測：加 context filters，return -19%→-13% +- [project_whale_correlation.md](project_whale_correlation.md) — 免費鏈上數據與 BTC 無顯著相關性 diff --git a/memory/feedback_no_misleading_signals.md b/memory/feedback_no_misleading_signals.md new file mode 100644 index 0000000..7dbf331 --- /dev/null +++ b/memory/feedback_no_misleading_signals.md @@ -0,0 +1,11 @@ +--- +name: No misleading entry signals in reports +description: Trend report should not imply entry readiness — only show market state (bullish/bearish) +type: feedback +--- + +趨勢報告不要顯示「可進場」之類的判斷字眼，只報告多頭/空頭。 + +**Why:** 趨勢報告顯示「可進場」但 LLM 沒進場，造成混淆。Production 進場完全由 LLM 判斷，程式邏輯判斷與 LLM 不一致。 + +**How to apply:** Slack 報告只呈現客觀市場數據，不做進場/出場建議。 diff --git a/memory/project_architecture_split.md b/memory/project_architecture_split.md new file mode 100644 index 0000000..04a14d2 --- /dev/null +++ b/memory/project_architecture_split.md @@ -0,0 +1,20 @@ +--- +name: Production vs Backtest architecture +description: Production uses LLM for signals, backtest uses rule-based signal_generator — they are independent +type: project +--- + +Production 和 Backtest 是兩條獨立路線： + +**Production (main.py):** +- 進場/出場完全由 LLM (Claude CLI) 判斷 +- LLM prompt 包含策略規則，但 LLM 自行決定是否遵守 +- risk_manager 只做風控驗證（倉位大小、最大持倉數） + +**Backtest (backtest/):** +- 用硬編碼規則的 signal_generator.py 判斷 +- 確定性、可重複，不跑 LLM +- V3 加入 context 參數（BTC 趨勢、buy_pressure、funding sentiment） +- `--no-context` flag 可關閉做 A/B 比較 + +**How to apply:** 改 backtest 不影響 production。改 LLM prompt 才影響 production 行為。 diff --git a/memory/project_backtest_v3.md b/memory/project_backtest_v3.md new file mode 100644 index 0000000..924930e --- /dev/null +++ b/memory/project_backtest_v3.md @@ -0,0 +1,13 @@ +--- +name: Backtest V3 context filters +description: V3 added BTC trend, buy pressure, funding sentiment — return improved from -19% to -13% +type: project +--- + +V3 回測 (2025-07-01 ~ 2026-03-17, $10k): +- Return: -19.07% → -13.48% +- Max DD: -27.19% → -18.25% +- BUYs: 385 → 189 (-51%) + +新增 context 參數：BTC 趨勢過濾、buy_pressure (OHLCV proxy)、funding sentiment (perp basis)。 +只影響 backtest，不影響 production。 diff --git a/memory/project_cron_timing.md b/memory/project_cron_timing.md index 12a0bb4..3f960f3 100644 --- a/memory/project_cron_timing.md +++ b/memory/project_cron_timing.md @@ -6,9 +6,13 @@ type: project ## Crontab 排程 -- `main.py`：`*/5 * * * * sleep 30 && ...`（:00:30, :05:30, :10:30...） -- `sync_cost_basis.py`：`2,32 * * * *`（:02, :32） +| 排程 | 腳本 | 用途 | +|------|------|------| +| `*/5 * * * *` (sleep 30) | main.py | 交易 cycle（LLM 分析 + 執行） | +| `0 * * * *` (sleep 30) | hourly_trend_report.py | 每小時 Slack 1h 趨勢報告（多頭/空頭） | +| `2,32 * * * *` | sync_cost_basis.py | 成本基礎同步 | +| `7 * * * *` | check_errors.py | 錯誤監控 | -**Why:** Bitfinex 5 分鐘 K 線在整點收盤（:00, :05, :10...），延遲 30 秒確保數據到位。sync_cost_basis 在 :02/:32 避免衝突。 +**Why:** Bitfinex 5 分鐘 K 線在整點收盤（:00, :05, :10...），延遲 30 秒確保數據到位。sync_cost_basis 在 :02/:32 避免衝突。趨勢報告在 :00:30 發送。 -**How to apply:** crontab 不支援秒，用 `sleep 30 &&` 實現。修改排程時維持此偏移策略。 +**How to apply:** crontab 不支援秒，用 `sleep 30 &&` 實現。修改排程時維持此偏移策略，注意避免 Bitfinex API rate limit。 diff --git a/memory/project_whale_correlation.md b/memory/project_whale_correlation.md new file mode 100644 index 0000000..715fed0 --- /dev/null +++ b/memory/project_whale_correlation.md @@ -0,0 +1,12 @@ +--- +name: Whale data correlation analysis +description: Free on-chain metrics show near-zero correlation with BTC price — not useful for signals +type: project +--- + +2026-03-18 用 backtest/whale_correlation.py 分析 blockchain.com 免費鏈上指標。 + +結果：所有指標跟 BTC 回報相關性 < 0.11（噪音）。 +真正有用的 whale 指標（exchange inflow/outflow）需要 CryptoQuant ($99/月) 或 Glassnode ($799/月)。 + +**How to apply:** 不要再花時間在免費鏈上數據做交易信號。 diff --git a/slack_notifier.py b/slack_notifier.py index fc76277..2183a1f 100644 --- a/slack_notifier.py +++ b/slack_notifier.py @@ -151,6 +151,53 @@ def send_cycle_report( _send({"text": text}) +def send_market_trend_report(htf_by_symbol: dict, current_prices: dict, + indicators_5m: dict | None = None): + """Send hourly market trend summary to Slack.""" + import pandas as pd + + lines = ["📈 *每小時 1h 趨勢報告*\n"] + + bullish = [] + bearish = [] + + for sym in sorted(htf_by_symbol): + df = htf_by_symbol[sym] + if df.empty or len(df) < 2: + continue + last = df.iloc[-1] + name = config.SYMBOL_NAMES.get(sym, sym) + price = current_prices.get(sym, 0) + + ema9 = last.get("ema9", 0) + ema21 = last.get("ema21", 0) + adx_val = last.get("adx", 0) + rsi_1h = last.get("rsi", 50) + + is_bullish = ema9 > ema21 if pd.notna(ema9) and pd.notna(ema21) else False + adx_val = adx_val if pd.notna(adx_val) else 0 + rsi_1h = rsi_1h if pd.notna(rsi_1h) else 50 + + price_str = f"{price:.6g}" if price > 0 else "N/A" + info = f"{name}: ADX={adx_val:.0f} | RSI={rsi_1h:.0f} | ${price_str}" + + if is_bullish: + bullish.append(f"🟢 {info}") + else: + bearish.append(f"🔴 {info}") + + if bullish: + lines.append(f"*多頭 ({len(bullish)}):*") + lines.extend(f" {s}" for s in bullish) + lines.append("") + + if bearish: + lines.append(f"*空頭 ({len(bearish)}):*") + lines.extend(f" {s}" for s in bearish) + + _send({"text": "\n".join(lines)}) + + def send_startup_message(): """Notify that the bot has started.""" mode = "PAPER" if config.PAPER_TRADING else "LIVE"