base version only KD indicator with jpm, ft yahoo as sources

This commit is contained in:
2026-01-27 04:15:19 +08:00
commit 61b32bbdd7
12 changed files with 15992 additions and 0 deletions
Binary file not shown.
+87
View File
@@ -0,0 +1,87 @@
from flask import Flask, render_template, jsonify
from datetime import datetime, timedelta
from engine import DataEngine
import concurrent.futures
from flask_caching import Cache
import csv, os, logging
app = Flask(__name__)
cache = Cache(app, config={'CACHE_TYPE': 'SimpleCache'})
import os
import csv
def load_instruments_from_csv(file_path):
instruments = []
# Standard static templates
# For AGI, we use the ISIN-based Tearsheet URL
TEMPLATES = {
'jpm': "https://am.jpmorgan.com/FundsMarketingHandler/historicalData?cusip={cusip}&country=hk&role=per",
'yahoo': "https://query1.finance.yahoo.com/v8/finance/chart/{cusip}?range=5y&interval=1d",
'agi': "https://markets.ft.com/data/funds/tearsheet/historical?s={cusip}"
}
try:
abs_path = os.path.join(os.path.dirname(__file__), file_path)
if not os.path.exists(abs_path):
print(f"Error: {file_path} not found.")
return []
with open(abs_path, mode='r', encoding='utf-8-sig') as csvfile:
reader = csv.DictReader(csvfile)
# Standardize header names to lowercase
reader.fieldnames = [name.strip().lower() for name in reader.fieldnames]
for row in reader:
symbol = row.get('symbol', '').strip()
cusip = row.get('cusip', '').strip()
provider = row.get('provider', 'jpm').strip().lower()
if symbol and cusip:
# Fetch correct template; default to JPM if provider is unknown
template = TEMPLATES.get(provider, TEMPLATES['jpm'])
url = template.format(cusip=cusip)
instruments.append({
"symbol": symbol,
"url": url,
"provider": provider
})
except Exception as e:
print(f"CSV Loading Error: {e}")
return instruments
# Usage
URL_CONFIG = load_instruments_from_csv('instruments.csv')
@cache.memoize(timeout=3600)
def fetch_and_calculate(config):
engine = DataEngine(config['symbol'], config['url'], config['provider'])
df = engine.fetch_data()
if df is not None:
metrics = engine.calculate_table_metrics(df)
if metrics:
metrics['symbol'] = config['symbol']
return metrics
return None
@app.route('/')
def index():
return render_template('index.html')
@app.route('/api/summary')
def get_summary():
results = []
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
futures = [executor.submit(fetch_and_calculate, cfg) for cfg in URL_CONFIG]
for f in concurrent.futures.as_completed(futures):
if f.result(): results.append(f.result())
results.sort(key=lambda x: x['symbol'])
return jsonify(results)
if __name__ == '__main__':
app.run(debug=True, host='0.0.0.0', port=5000)
+88
View File
@@ -0,0 +1,88 @@
from flask import Flask, render_template, jsonify
from engine import DataEngine
import concurrent.futures
from flask_caching import Cache
import csv
import os
import logging
app = Flask(__name__)
# Configure caching
cache = Cache(app, config={'CACHE_TYPE': 'SimpleCache'})
# Configure logging
logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s')
# Function to load instruments from a CSV file
def load_instruments_from_csv(file_path):
instruments = []
try:
abs_path = os.path.join(os.path.dirname(__file__), file_path)
# 'utf-8-sig' handles the hidden characters Excel often adds
with open(abs_path, mode='r', encoding='utf-8-sig') as csvfile:
# Clean spaces from column names automatically
reader = csv.DictReader(csvfile)
reader.fieldnames = [name.strip().lower() for name in reader.fieldnames]
for row in reader:
symbol = row.get('symbol', '').strip()
cusip = row.get('cusip', '').strip()
if symbol and cusip:
url = f"https://am.jpmorgan.com/FundsMarketingHandler/historicalData?cusip={cusip}&country=hk&role=per&userLoggedIn=false&language=en&version=6.9.0_1684"
instruments.append({"symbol": symbol, "url": url})
logging.info(f"Successfully loaded {len(instruments)} instruments.")
except Exception as e:
logging.error(f"Error reading CSV file: {e}")
return instruments
# Load instruments from CSV
URL_CONFIG = load_instruments_from_csv('instruments.csv')
# Log the contents of URL_CONFIG to verify instruments are loaded
logging.debug(f"Loaded instruments: {URL_CONFIG}")
@cache.memoize(timeout=86400) # Cache results for 24 hours
def fetch_and_calculate(config):
try:
logging.debug(f"Fetching data for: {config['symbol']}")
engine = DataEngine(config['url'])
df = engine.fetch_data()
if df is None or df.empty:
logging.warning(f"No data fetched for: {config['symbol']}")
return None
metrics = engine.calculate_table_metrics(df)
if metrics:
metrics['symbol'] = config['symbol']
logging.debug(f"Metrics calculated for {config['symbol']}: {metrics}")
return metrics
logging.warning(f"Metrics calculation failed for: {config['symbol']}")
return None
except Exception as e:
logging.error(f"Error processing {config['symbol']}: {e}")
return None
@app.route('/')
def index():
return render_template('index.html')
@app.route('/api/summary')
def get_summary():
results = []
# Use ThreadPoolExecutor for faster parallel fetching
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
future_to_config = {executor.submit(fetch_and_calculate, cfg): cfg for cfg in URL_CONFIG}
for future in concurrent.futures.as_completed(future_to_config):
res = future.result()
if res:
results.append(res)
# Sort by symbol name
results.sort(key=lambda x: x['symbol'])
return jsonify(results)
if __name__ == '__main__':
app.run(debug=True, host='0.0.0.0', port=5000) # Changed port back to 5000
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+202
View File
@@ -0,0 +1,202 @@
import pandas as pd
import requests
import os
from datetime import datetime
from ta.trend import EMAIndicator
from ta.momentum import StochasticOscillator
class DataEngine:
def __init__(self, symbol, url, provider):
self.symbol = symbol
self.url = url
self.provider = provider
# 1. Get the folder where engine.py lives
base_path = os.path.dirname(os.path.abspath(__file__))
# 2. Define the cache directory path
self.cache_dir = os.path.join(base_path, "data_cache")
# 3. Create the folder if it doesn't exist (safety-first)
os.makedirs(self.cache_dir, exist_ok=True)
# 4. Set the full path for this specific instrument's CSV
self.file_path = os.path.join(self.cache_dir, f"{self.symbol}.csv")
def global_sync(self):
"""The 'One-Click' background loop."""
# 1. Get the latest list of instruments from your CSV
all_instruments = self.load_instruments_from_csv()
for item in all_instruments:
# 2. Update the 'Current' target for the engine
self.symbol = item['symbol']
self.cusip = item['cusip']
self.provider = item['provider']
# 3. Regenerate the URL and File Path for THIS specific instrument
self.url = self.generate_url()
self.file_path = os.path.join(self.data_dir, f"{self.symbol}.csv")
# 4. Run the robust fetch/merge logic we built
print(f"Syncing {self.symbol}...")
self.fetch_data()
print("Global Sync Complete.")
def _parse_jpm(self, json_data):
if isinstance(json_data, dict) and "historicalNAVList" in json_data:
df = pd.DataFrame(json_data["historicalNAVList"])
return df.rename(columns={'navPrice': 'close', 'date': 'date'})
return None
def _parse_ft_html(self, html_text):
try:
# 1. Use BeautifulSoup to handle the nested spans in the Date column
from bs4 import BeautifulSoup
soup = BeautifulSoup(html_text, 'html.parser')
# Find the specific results table
table = soup.find('table', class_='mod-tearsheet-historical-prices__results')
if not table:
print(f"❌ Could not find the results table in the HTML for {self.symbol}")
return None
data = []
rows = table.find('tbody').find_all('tr')
for row in rows:
cols = row.find_all('td')
if len(cols) >= 5:
# The Date cell has two spans. We'll take the first one (Full date).
date_cell = cols[0].find('span', class_='mod-ui-hide-small-below')
date_str = date_cell.get_text(strip=True) if date_cell else cols[0].get_text(strip=True)
# The Close price is usually the 5th column (index 4)
close_str = cols[4].get_text(strip=True).replace(',', '')
data.append({
'date': date_str,
'close': close_str
})
# 2. Convert to DataFrame
df = pd.DataFrame(data)
if df.empty:
return None
# 3. Final Type Conversion
df['date'] = pd.to_datetime(df['date'], errors='coerce')
df['close'] = pd.to_numeric(df['close'], errors='coerce')
return df.dropna().sort_values('date').reset_index(drop=True)
except Exception as e:
print(f"❌ Failed to parse FT HTML structure: {e}")
return None
def _parse_yahoo(self, json_data):
"""Parses Yahoo Finance v8 Chart JSON"""
try:
chart = json_data['chart']['result'][0]
timestamps = chart['timestamp']
indicators = chart['indicators']['quote'][0]
# Use adjclose if available, otherwise close
closes = indicators.get('close', [])
df = pd.DataFrame({
'date': pd.to_datetime(timestamps, unit='s'),
'close': closes
})
return df
except:
return None
def fetch_data(self):
local_df = pd.DataFrame()
new_df = None
# 1. Load Local Cache & Force Date Type
if os.path.exists(self.file_path):
try:
local_df = pd.read_csv(self.file_path)
local_df = local_df.loc[:, ~local_df.columns.duplicated()].copy()
local_df.columns = [c.lower().strip() for c in local_df.columns]
local_df = local_df.rename(columns={'price': 'close', 'nav': 'close'})
# FORCE CONVERSION: This fixes the '<' error
# errors='coerce' turns bad text into NaT (Not a Time), which we then drop
local_df['date'] = pd.to_datetime(local_df['date'], errors='coerce')
local_df = local_df.dropna(subset=['date']).reset_index(drop=True)
except Exception as e:
print(f"Local Load Error: {e}")
# 2. Network Fetch
try:
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)'}
response = requests.get(self.url, headers=headers, timeout=15)
response.raise_for_status()
if self.provider == 'agi':
new_df = self._parse_ft_html(response.text)
elif self.provider == 'jpm':
new_df = self._parse_jpm(response.json())
elif self.provider == 'yahoo':
new_df = self._parse_yahoo(response.json())
# 3. Safe Merge & Sort
if new_df is not None and not new_df.empty:
# Force new_df dates to match local_df format
new_df['date'] = pd.to_datetime(new_df['date'], errors='coerce')
combined_df = pd.concat([local_df, new_df], ignore_index=True)
combined_df = combined_df.drop_duplicates(subset=['date'], keep='last')
# SORTING: Now safe because all types are Timestamps
combined_df = combined_df.sort_values('date').reset_index(drop=True)
if 'close' in combined_df.columns:
final_df = combined_df[['date', 'close']].dropna()
final_df.to_csv(self.file_path, index=False)
return final_df
return local_df
except Exception as e:
print(f"Network error for {self.symbol}: {e}")
return local_df
def calculate_table_metrics(self, df):
if df is None or df.empty or len(df) < 2:
return None
last_close = float(df.iloc[-1]['close'])
prev_close = float(df.iloc[-2]['close'])
change_pct = ((last_close - prev_close) / prev_close) * 100
count = len(df)
def get_ema_offset(window):
if count >= window:
ema = EMAIndicator(close=df['close'], window=window).ema_indicator().iloc[-1]
return round(((last_close / ema) * 100) - 100, 1)
return "N/A"
k_val = d_val = "N/A"
if count >= 14:
high_14 = df['close'].rolling(window=14).max()
low_14 = df['close'].rolling(window=14).min()
stoch = StochasticOscillator(high=high_14, low=low_14, close=df['close'], window=14)
k_val = round(stoch.stoch().iloc[-1], 0)
d_val = round(stoch.stoch_signal().iloc[-1], 0)
return {
"last_close": round(last_close, 2),
"change_pct": round(change_pct, 2),
"low_52": round(float(df.tail(252)['close'].min()), 2),
"high_52": round(float(df.tail(252)['close'].max()), 2),
"last_ema20": get_ema_offset(20),
"last_ema50": get_ema_offset(50),
"last_ema100": get_ema_offset(100),
"last_ema200": get_ema_offset(200),
"kd_values": f"{k_val}/{d_val}" if k_val != "N/A" else "N/A"
}
+98
View File
@@ -0,0 +1,98 @@
import pandas as pd
import requests
import os
from datetime import datetime
from ta.trend import EMAIndicator
from ta.momentum import StochasticOscillator
class DataEngine:
def __init__(self, symbol, url):
self.symbol = symbol
self.url = url
self.cache_dir = "data_cache"
if not os.path.exists(self.cache_dir):
os.makedirs(self.cache_dir)
self.file_path = os.path.join(self.cache_dir, f"{self.symbol}.csv")
def fetch_data(self):
# 1. Try to load existing local data
local_df = pd.DataFrame()
if os.path.exists(self.file_path):
try:
local_df = pd.read_csv(self.file_path, parse_dates=['date'])
# If data is fresh (from today), return it immediately
if not local_df.empty and local_df['date'].max().date() >= datetime.today().date():
return local_df
except Exception as e:
print(f"Cache read error for {self.symbol}: {e}")
# 2. Fetch new data from URL
try:
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'}
response = requests.get(self.url, headers=headers, timeout=10)
response.raise_for_status()
json_data = response.json()
# Parsing Logic (Handles JPM and generic JSON)
if isinstance(json_data, dict) and "historicalNAVList" in json_data:
new_df = pd.DataFrame(json_data["historicalNAVList"])
new_df = new_df.rename(columns={'navPrice': 'close'})
else:
new_df = pd.DataFrame(json_data)
new_df['date'] = pd.to_datetime(new_df['date'])
# 3. Merge and Save
if not local_df.empty:
# Combine, drop duplicates (keep newest), and sort
df = pd.concat([local_df, new_df]).drop_duplicates(subset=['date'], keep='last')
else:
df = new_df
df = df.sort_values('date').dropna(subset=['close'])
df.to_csv(self.file_path, index=False)
return df
except Exception as e:
print(f"Network error for {self.symbol}: {e}")
return local_df if not local_df.empty else None
def calculate_table_metrics(self, df):
if df is None or df.empty or len(df) < 2:
return None
last_close = float(df.iloc[-1]['close'])
prev_close = float(df.iloc[-2]['close'])
change_pct = ((last_close - prev_close) / prev_close) * 100
count = len(df)
df_52 = df.tail(min(count, 252))
# EMA Calculations (Returns % offset from Price)
def get_ema_offset(window):
if count >= window:
ema = EMAIndicator(close=df['close'], window=window).ema_indicator().iloc[-1]
return round(((last_close / ema) * 100) - 100, 1)
return "N/A"
# Stochastic Logic
k_val = d_val = "N/A"
if count >= 14:
# Note: Using rolling close as proxy for High/Low since many URLs only provide Close
rolling_high = df['close'].rolling(window=14).max()
rolling_low = df['close'].rolling(window=14).min()
stoch = StochasticOscillator(high=rolling_high, low=rolling_low, close=df['close'], window=14, smooth_window=3)
k_val = round(stoch.stoch().iloc[-1], 0)
d_val = round(stoch.stoch_signal().iloc[-1], 0)
return {
"last_close": round(last_close, 2),
"change_pct": round(change_pct, 2),
"low_52": round(float(df_52['close'].min()), 2),
"high_52": round(float(df_52['close'].max()), 2),
"last_ema20": get_ema_offset(20),
"last_ema50": get_ema_offset(50),
"last_ema100": get_ema_offset(100),
"last_ema200": get_ema_offset(200),
"kd_values": f"{k_val}/{d_val}" if k_val != "N/A" else "N/A"
}
+41
View File
@@ -0,0 +1,41 @@
import pandas as pd
import os
def import_history(source_file, symbol):
"""
source_file: path to your historical data (Excel or CSV)
symbol: The symbol name matching your instruments.csv (e.g., 'AGI_USD')
"""
cache_dir = "data_cache"
os.makedirs(cache_dir, exist_ok=True)
target_path = os.path.join(cache_dir, f"{symbol}.csv")
try:
# Load your data
if source_file.endswith('.xlsx'):
df = pd.read_excel(source_file)
else:
df = pd.read_csv(source_file)
# 1. Standardize columns: We need 'date' and 'close'
# Adjust these strings if your Excel uses different names like 'Price' or 'Date'
df.columns = [c.lower().strip() for c in df.columns]
# If your Excel has columns named 'nav' or 'price', rename them
rename_map = {'price': 'close', 'nav': 'close', 'valuation date': 'date'}
df = df.rename(columns=rename_map)
# 2. Convert to proper format
df['date'] = pd.to_datetime(df['date']).dt.strftime('%Y-%m-%d')
df = df[['date', 'close']].dropna().sort_values('date')
# 3. Save to the cache folder
df.to_csv(target_path, index=False)
print(f"✅ Successfully created cache for {symbol} at {target_path}")
print(f"📊 Rows imported: {len(df)}")
except Exception as e:
print(f"❌ Error importing {symbol}: {e}")
# EXAMPLE USAGE:
# import_history('my_old_data.xlsx', 'AGI_USD')
+5
View File
@@ -0,0 +1,5 @@
symbol,cusip,provider
JPMorgan Evergreen Fund,HK0000055829,jpm
Allianz Oriental Income Cl A,LU0348783233:USD,agi
SPMO ETF - USD,SPMO,yahoo
JPM Korea,LU0301634860,jpm
1 symbol cusip provider
2 JPMorgan Evergreen Fund HK0000055829 jpm
3 Allianz Oriental Income Cl A LU0348783233:USD agi
4 SPMO ETF - USD SPMO yahoo
5 JPM Korea LU0301634860 jpm
+194
View File
@@ -0,0 +1,194 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no">
<title>Financial Dashboard</title>
<link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/css/bootstrap.min.css" rel="stylesheet">
<style>
:root {
--header-bg: #4a5568;
--text-up: #28a745;
--text-down: #dc3545;
}
body {
background-color: #f4f7f6;
font-family: -apple-system, system-ui, BlinkMacSystemFont, "Segoe UI", Roboto;
margin: 0;
padding: 10px;
}
.card {
border: none;
border-radius: 12px;
box-shadow: 0 4px 15px rgba(0,0,0,0.1);
overflow: hidden;
}
.card-header {
background-color: white !important;
border-bottom: 1px solid #eee;
padding: 15px;
}
/* Zebra Striping Logic */
.table-striped tbody tr:nth-of-type(odd) {
background-color: rgba(0, 0, 0, 0.03);
}
/* Sticky Column for Mobile Scrolling */
.table-responsive { border-radius: 8px; }
.table thead th {
background-color: var(--header-bg) !important;
color: white !important;
font-size: 0.7rem;
text-transform: uppercase;
letter-spacing: 0.5px;
text-align: center;
white-space: nowrap;
padding: 12px 8px;
border: none;
}
.table td {
text-align: center;
vertical-align: middle;
font-size: 0.85rem;
white-space: nowrap;
border-color: #f1f1f1;
}
/* Keep Instrument Name fixed on the left while swiping */
.table td:first-child, .table th:first-child {
position: sticky;
left: 0;
z-index: 10;
text-align: left;
min-width: 140px;
background-color: white;
font-weight: 700;
}
/* Ensure zebra stripe works on sticky column */
.table-striped tbody tr:nth-of-type(odd) td:first-child {
background-color: #f9f9f9;
}
/* Custom UI Elements */
.btn-refresh {
background-color: #0d6efd;
color: white !important;
font-weight: 600;
border-radius: 6px;
padding: 6px 16px;
transition: 0.2s;
}
.text-up { color: var(--text-up); font-weight: 600; }
.text-down { color: var(--text-down); font-weight: 600; }
.badge-kd {
background-color: #6c757d;
font-size: 0.75rem;
padding: 5px 8px;
}
#loading { display: none; margin-left: 10px; font-size: 0.8rem; color: #666; }
</style>
</head>
<body>
<div class="container-fluid p-0">
<div class="card">
<div class="card-header d-flex justify-content-between align-items-center">
<h5 class="mb-0 text-dark">Portfolio Signals</h5>
<div>
<span id="loading">Updating...</span>
<button class="btn btn-refresh btn-sm" onclick="loadData()">Refresh</button>
</div>
</div>
<div class="card-body p-0">
<div class="table-responsive">
<table class="table table-hover table-striped mb-0">
<thead>
<tr>
<th>Instrument</th>
<th>Close</th>
<th>Chg%</th>
<th>52W Range</th>
<th>v20 EMA</th>
<th>v50 EMA</th>
<th>v100 EMA</th>
<th>v200 EMA</th>
<th>K/D</th>
</tr>
</thead>
<tbody id="tableBody">
<tr><td colspan="9" class="p-4">Initializing data engine...</td></tr>
</tbody>
</table>
</div>
</div>
</div>
</div>
<script>
async function loadData() {
const loading = document.getElementById('loading');
const tbody = document.getElementById('tableBody');
loading.style.display = 'inline';
try {
const response = await fetch('/api/summary');
const data = await response.json();
tbody.innerHTML = '';
if (data.length === 0) {
tbody.innerHTML = '<tr><td colspan="9" class="p-4">No instruments found in CSV.</td></tr>';
return;
}
data.forEach(item => {
// Helper to format EMA offsets with +/- and Colors
const formatEma = (val) => {
if (val === "N/A" || val === null) return `<span class="text-muted">N/A</span>`;
const sign = val > 0 ? "+" : "";
const colorClass = val >= 0 ? 'text-up' : 'text-down';
return `<span class="${colorClass}">${sign}${val}%</span>`;
};
const row = `
<tr>
<td>${item.symbol}</td>
<td class="fw-bold">${item.last_close}</td>
<td class="${item.change_pct >= 0 ? 'text-up' : 'text-down'}">
${item.change_pct >= 0 ? '+' : ''}${item.change_pct}%
</td>
<td class="text-muted small">${item.low_52} - ${item.high_52}</td>
<td>${formatEma(item.last_ema20)}</td>
<td>${formatEma(item.last_ema50)}</td>
<td>${formatEma(item.last_ema100)}</td>
<td>${formatEma(item.last_ema200)}</td>
<td><span class="badge badge-kd">${item.kd_values}</span></td>
</tr>
`;
tbody.innerHTML += row;
});
} catch (error) {
console.error("Fetch error:", error);
tbody.innerHTML = '<tr><td colspan="9" class="text-danger p-4">Error connecting to server.</td></tr>';
} finally {
loading.style.display = 'none';
}
}
// Initial Load
document.addEventListener('DOMContentLoaded', loadData);
</script>
</body>
</html>