Create mltechnicalscanner.py
Browse files- mltechnicalscanner.py +273 -0
mltechnicalscanner.py
ADDED
@@ -0,0 +1,273 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import sys
|
2 |
+
import os
|
3 |
+
import ccxt
|
4 |
+
import pandas as pd
|
5 |
+
import numpy as np
|
6 |
+
from datetime import datetime
|
7 |
+
import ta
|
8 |
+
import argparse
|
9 |
+
from sklearn.ensemble import RandomForestClassifier
|
10 |
+
from sklearn.model_selection import train_test_split
|
11 |
+
from sklearn.metrics import accuracy_score
|
12 |
+
import pickle
|
13 |
+
import warnings
|
14 |
+
|
15 |
+
# Suppress warnings
|
16 |
+
warnings.filterwarnings('ignore')
|
17 |
+
|
18 |
+
# Configuration
|
19 |
+
pd.set_option('display.max_columns', None)
|
20 |
+
pd.set_option('display.max_rows', None)
|
21 |
+
pd.set_option('display.expand_frame_repr', True)
|
22 |
+
|
23 |
+
class MLTechnicalScanner:
|
24 |
+
def __init__(self, training_mode=False):
|
25 |
+
self.training_mode = training_mode
|
26 |
+
self.model = None
|
27 |
+
self.model_file = "technical_ml_model.pkl"
|
28 |
+
self.training_data_file = "training_data.csv"
|
29 |
+
self.min_training_samples = 100
|
30 |
+
self.load_ml_model()
|
31 |
+
|
32 |
+
# Initialize exchanges
|
33 |
+
self.exchanges = {}
|
34 |
+
for id in ccxt.exchanges:
|
35 |
+
exchange = getattr(ccxt, id)
|
36 |
+
self.exchanges[id] = exchange()
|
37 |
+
|
38 |
+
# ML features configuration
|
39 |
+
self.feature_columns = [
|
40 |
+
'rsi', 'macd', 'bollinger_upper', 'bollinger_lower',
|
41 |
+
'volume_ma', 'ema_20', 'ema_50', 'adx'
|
42 |
+
]
|
43 |
+
|
44 |
+
# Performance tracking
|
45 |
+
self.performance_history = pd.DataFrame(columns=[
|
46 |
+
'timestamp', 'symbol', 'prediction', 'actual', 'profit'
|
47 |
+
])
|
48 |
+
|
49 |
+
# Training data collection
|
50 |
+
self.training_data = pd.DataFrame(columns=self.feature_columns + ['target'])
|
51 |
+
|
52 |
+
def load_ml_model(self):
|
53 |
+
"""Load trained ML model if exists"""
|
54 |
+
if os.path.exists(self.model_file):
|
55 |
+
with open(self.model_file, 'rb') as f:
|
56 |
+
self.model = pickle.load(f)
|
57 |
+
print("Loaded trained model from file")
|
58 |
+
else:
|
59 |
+
print("Initializing new model")
|
60 |
+
self.model = RandomForestClassifier(n_estimators=100, random_state=42)
|
61 |
+
|
62 |
+
def save_ml_model(self):
|
63 |
+
"""Save trained ML model"""
|
64 |
+
with open(self.model_file, 'wb') as f:
|
65 |
+
pickle.dump(self.model, f)
|
66 |
+
print("Saved model to file")
|
67 |
+
|
68 |
+
def load_training_data(self):
|
69 |
+
"""Load existing training data if available"""
|
70 |
+
if os.path.exists(self.training_data_file):
|
71 |
+
self.training_data = pd.read_csv(self.training_data_file)
|
72 |
+
print(f"Loaded {len(self.training_data)} training samples")
|
73 |
+
|
74 |
+
def save_training_data(self):
|
75 |
+
"""Save training data to file"""
|
76 |
+
self.training_data.to_csv(self.training_data_file, index=False)
|
77 |
+
print(f"Saved {len(self.training_data)} training samples")
|
78 |
+
|
79 |
+
def calculate_features(self, df):
|
80 |
+
"""Calculate technical indicators"""
|
81 |
+
try:
|
82 |
+
close = df['close'].astype(float)
|
83 |
+
high = df['high'].astype(float)
|
84 |
+
low = df['low'].astype(float)
|
85 |
+
volume = df['volume'].astype(float)
|
86 |
+
|
87 |
+
# Momentum Indicators
|
88 |
+
df['rsi'] = ta.momentum.rsi(close, window=14)
|
89 |
+
df['macd'] = ta.trend.macd_diff(close)
|
90 |
+
|
91 |
+
# Volatility Indicators
|
92 |
+
bollinger = ta.volatility.BollingerBands(close)
|
93 |
+
df['bollinger_upper'] = bollinger.bollinger_hband()
|
94 |
+
df['bollinger_lower'] = bollinger.bollinger_lband()
|
95 |
+
|
96 |
+
# Volume Indicators
|
97 |
+
df['volume_ma'] = volume.rolling(window=20).mean()
|
98 |
+
|
99 |
+
# Trend Indicators
|
100 |
+
df['ema_20'] = ta.trend.ema_indicator(close, window=20)
|
101 |
+
df['ema_50'] = ta.trend.ema_indicator(close, window=50)
|
102 |
+
df['adx'] = ta.trend.adx(high, low, close, window=14)
|
103 |
+
|
104 |
+
return df
|
105 |
+
except Exception as e:
|
106 |
+
print(f"Error calculating features: {str(e)}")
|
107 |
+
return None
|
108 |
+
|
109 |
+
def train_initial_model(self):
|
110 |
+
"""Train initial model if we have enough data"""
|
111 |
+
self.load_training_data()
|
112 |
+
|
113 |
+
if len(self.training_data) >= self.min_training_samples:
|
114 |
+
X = self.training_data[self.feature_columns]
|
115 |
+
y = self.training_data['target']
|
116 |
+
|
117 |
+
X_train, X_test, y_train, y_test = train_test_split(
|
118 |
+
X, y, test_size=0.2, random_state=42
|
119 |
+
)
|
120 |
+
|
121 |
+
self.model.fit(X_train, y_train)
|
122 |
+
|
123 |
+
# Evaluate model
|
124 |
+
preds = self.model.predict(X_test)
|
125 |
+
accuracy = accuracy_score(y_test, preds)
|
126 |
+
print(f"Initial model trained with accuracy: {accuracy:.2f}")
|
127 |
+
|
128 |
+
self.save_ml_model()
|
129 |
+
return True
|
130 |
+
else:
|
131 |
+
print(f"Not enough training data ({len(self.training_data)} samples). Need at least {self.min_training_samples}.")
|
132 |
+
return False
|
133 |
+
|
134 |
+
def predict_direction(self, features):
|
135 |
+
"""Predict price direction using ML model"""
|
136 |
+
try:
|
137 |
+
if self.model is None or not hasattr(self.model, 'classes_'):
|
138 |
+
return 0 # Neutral if no model
|
139 |
+
|
140 |
+
features = features[self.feature_columns].values.reshape(1, -1)
|
141 |
+
return self.model.predict(features)[0]
|
142 |
+
except Exception as e:
|
143 |
+
print(f"Prediction error: {str(e)}")
|
144 |
+
return 0
|
145 |
+
|
146 |
+
def collect_training_sample(self, symbol, exchange, timeframe='1h'):
|
147 |
+
"""Collect data sample for training"""
|
148 |
+
try:
|
149 |
+
ohlcv = exchange.fetch_ohlcv(symbol, timeframe, limit=100)
|
150 |
+
if len(ohlcv) < 50:
|
151 |
+
return
|
152 |
+
|
153 |
+
df = pd.DataFrame(ohlcv, columns=['timestamp', 'open', 'high', 'low', 'close', 'volume'])
|
154 |
+
df = self.calculate_features(df)
|
155 |
+
if df is None:
|
156 |
+
return
|
157 |
+
|
158 |
+
current_price = df['close'].iloc[-1]
|
159 |
+
future_price = df['close'].iloc[-1] # Should be forward-looking in production
|
160 |
+
|
161 |
+
price_change = future_price - current_price
|
162 |
+
target = 1 if price_change > 0 else (-1 if price_change < 0 else 0)
|
163 |
+
|
164 |
+
features = df.iloc[-2].copy()
|
165 |
+
features['target'] = target
|
166 |
+
|
167 |
+
new_row = pd.DataFrame([features])
|
168 |
+
self.training_data = pd.concat([self.training_data, new_row], ignore_index=True)
|
169 |
+
print(f"Collected training sample for {symbol}")
|
170 |
+
|
171 |
+
if len(self.training_data) % 10 == 0:
|
172 |
+
self.save_training_data()
|
173 |
+
|
174 |
+
except Exception as e:
|
175 |
+
print(f"Error collecting training sample: {str(e)}")
|
176 |
+
|
177 |
+
def scan_symbol(self, symbol, exchange, timeframes):
|
178 |
+
"""Scan symbol for trading opportunities"""
|
179 |
+
try:
|
180 |
+
primary_tf = timeframes[0]
|
181 |
+
ohlcv = exchange.fetch_ohlcv(symbol, primary_tf, limit=100)
|
182 |
+
if len(ohlcv) < 50:
|
183 |
+
return
|
184 |
+
|
185 |
+
df = pd.DataFrame(ohlcv, columns=['timestamp', 'open', 'high', 'low', 'close', 'volume'])
|
186 |
+
df = self.calculate_features(df)
|
187 |
+
if df is None:
|
188 |
+
return
|
189 |
+
|
190 |
+
latest = df.iloc[-1].copy()
|
191 |
+
features = pd.DataFrame([latest[self.feature_columns]])
|
192 |
+
|
193 |
+
if self.training_mode:
|
194 |
+
self.collect_training_sample(symbol, exchange, primary_tf)
|
195 |
+
return
|
196 |
+
|
197 |
+
prediction = self.predict_direction(features)
|
198 |
+
|
199 |
+
# Simplified trend detection using EMA crossover
|
200 |
+
ema_20 = df['ema_20'].iloc[-1]
|
201 |
+
ema_50 = df['ema_50'].iloc[-1]
|
202 |
+
price = df['close'].iloc[-1]
|
203 |
+
|
204 |
+
uptrend = (ema_20 > ema_50) and (price > ema_20)
|
205 |
+
downtrend = (ema_20 < ema_50) and (price < ema_20)
|
206 |
+
|
207 |
+
if uptrend and prediction == 1:
|
208 |
+
self.alert(symbol, "STRONG UPTREND", timeframes)
|
209 |
+
elif downtrend and prediction == -1:
|
210 |
+
self.alert(symbol, "STRONG DOWNTREND", timeframes)
|
211 |
+
elif uptrend:
|
212 |
+
self.alert(symbol, "UPTREND", timeframes)
|
213 |
+
elif downtrend:
|
214 |
+
self.alert(symbol, "DOWNTREND", timeframes)
|
215 |
+
|
216 |
+
except Exception as e:
|
217 |
+
print(f"Error scanning {symbol}: {str(e)}")
|
218 |
+
|
219 |
+
def alert(self, symbol, trend_type, timeframes):
|
220 |
+
"""Generate alert for detected trend"""
|
221 |
+
message = f"({trend_type}) detected for {symbol} on {timeframes} at {datetime.now()}"
|
222 |
+
print(message)
|
223 |
+
|
224 |
+
# Main execution
|
225 |
+
if __name__ == "__main__":
|
226 |
+
parser = argparse.ArgumentParser()
|
227 |
+
parser.add_argument("-e", "--exchange", help="Exchange name", required=True)
|
228 |
+
parser.add_argument("-f", "--filter", help="Asset filter", required=True)
|
229 |
+
parser.add_argument("-tf", "--timeframes", help="Timeframes to scan (comma separated)", required=True)
|
230 |
+
parser.add_argument("--train", help="Run in training mode", action="store_true")
|
231 |
+
args = parser.parse_args()
|
232 |
+
|
233 |
+
scanner = MLTechnicalScanner(training_mode=args.train)
|
234 |
+
|
235 |
+
exchange = scanner.exchanges.get(args.exchange.lower())
|
236 |
+
if not exchange:
|
237 |
+
print(f"Exchange {args.exchange} not supported")
|
238 |
+
sys.exit(1)
|
239 |
+
|
240 |
+
try:
|
241 |
+
markets = exchange.fetch_markets()
|
242 |
+
except Exception as e:
|
243 |
+
print(f"Error fetching markets: {str(e)}")
|
244 |
+
sys.exit(1)
|
245 |
+
|
246 |
+
symbols = [
|
247 |
+
m['id'] for m in markets
|
248 |
+
if m['active'] and args.filter in m['id']
|
249 |
+
]
|
250 |
+
|
251 |
+
if not symbols:
|
252 |
+
print(f"No symbols found matching filter {args.filter}")
|
253 |
+
sys.exit(1)
|
254 |
+
|
255 |
+
if args.train:
|
256 |
+
print(f"Running in training mode for {len(symbols)} symbols")
|
257 |
+
for symbol in symbols:
|
258 |
+
scanner.collect_training_sample(symbol, exchange)
|
259 |
+
|
260 |
+
if scanner.train_initial_model():
|
261 |
+
print("Training completed successfully")
|
262 |
+
else:
|
263 |
+
print("Not enough data collected for training")
|
264 |
+
sys.exit(0)
|
265 |
+
|
266 |
+
if not hasattr(scanner.model, 'classes_'):
|
267 |
+
print("Warning: No trained model available. Running with basic scanning only.")
|
268 |
+
|
269 |
+
timeframes = args.timeframes.split(',')
|
270 |
+
print(f"Scanning {len(symbols)} symbols on timeframes {timeframes}")
|
271 |
+
|
272 |
+
for symbol in symbols:
|
273 |
+
scanner.scan_symbol(symbol, exchange, timeframes)
|