Refactor: Separate data sync from UI rendering, change colunm width

app.py

@@ -4,6 +4,7 @@ from engine import DataEngine
 import concurrent.futures
 from flask_caching import Cache
 import csv, os, logging
+from concurrent.futures import ThreadPoolExecutor
 
 app = Flask(__name__)
 cache = Cache(app, config={'CACHE_TYPE': 'SimpleCache'})
@@ -11,52 +12,6 @@ cache = Cache(app, config={'CACHE_TYPE': 'SimpleCache'})
 import os
 import csv
 
-def load_instruments_from_csv(file_path):
-    instruments = []
-    
-    # Standard static templates
-    # For AGI, we use the ISIN-based Tearsheet URL
-    TEMPLATES = {
-        'jpm': "https://am.jpmorgan.com/FundsMarketingHandler/historicalData?cusip={cusip}&country=hk&role=per",
-        'yahoo': "https://query1.finance.yahoo.com/v8/finance/chart/{cusip}?range=5y&interval=1d",
-        'agi': "https://markets.ft.com/data/funds/tearsheet/historical?s={cusip}"
-    }
-
-    try:
-        abs_path = os.path.join(os.path.dirname(__file__), file_path)
-        
-        if not os.path.exists(abs_path):
-            print(f"Error: {file_path} not found.")
-            return []
-
-        with open(abs_path, mode='r', encoding='utf-8-sig') as csvfile:
-            reader = csv.DictReader(csvfile)
-            # Standardize header names to lowercase
-            reader.fieldnames = [name.strip().lower() for name in reader.fieldnames]
-            
-            for row in reader:
-                symbol = row.get('symbol', '').strip()
-                cusip = row.get('cusip', '').strip()
-                provider = row.get('provider', 'jpm').strip().lower()
-                
-                if symbol and cusip:
-                    # Fetch correct template; default to JPM if provider is unknown
-                    template = TEMPLATES.get(provider, TEMPLATES['jpm'])
-                    url = template.format(cusip=cusip)
-                    
-                    instruments.append({
-                        "symbol": symbol, 
-                        "url": url, 
-                        "provider": provider
-                    })
-                
-    except Exception as e:
-        print(f"CSV Loading Error: {e}")
-        
-    return instruments
-
-# Usage
-URL_CONFIG = load_instruments_from_csv('instruments.csv')
 
 @cache.memoize(timeout=3600)
 def fetch_and_calculate(config):
@@ -75,13 +30,38 @@ def index():
 
 @app.route('/api/summary')
 def get_summary():
+    engine_base = DataEngine() 
+    instruments = engine_base.load_instruments_from_csv('instruments.csv')
+    
     results = []
-    with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
+    for item in instruments:
-        futures = [executor.submit(fetch_and_calculate, cfg) for cfg in URL_CONFIG]
+        engine = DataEngine(
-        for f in concurrent.futures.as_completed(futures):
+            symbol=item['symbol'], 
-            if f.result(): results.append(f.result())
+            url=item['url'], 
-    results.sort(key=lambda x: x['symbol'])
+            provider=item['provider']
+        )
+        
+        metrics = engine.get_local_metrics()
+        
+        if metrics and isinstance(metrics, dict):
+            metrics['symbol'] = item['symbol']
+            results.append(metrics)
+        else:
+            # 🔥 FIX: Include the 'error' key so JavaScript hits the Gatekeeper
+            results.append({
+                "symbol": item['symbol'], 
+                "last_close": None, # Ensure this is null, not the string "No Data"
+                "error": True 
+            })
+            
     return jsonify(results)
 
+@app.route('/api/sync', methods=['POST'])
+def run_sync():
+    # ✅ THIS RUNS THE FULL fetch_data() WITH NETWORK ACCESS
+    engine = DataEngine()
+    report = engine.global_sync()
+    return jsonify(report)
+
 if __name__ == '__main__':
     app.run(debug=True, host='0.0.0.0', port=5000)

engine.py

@@ -1,48 +1,179 @@
 import pandas as pd
 import requests
 import os
-from datetime import datetime
+import shutil
+from datetime import datetime, time
 from ta.trend import EMAIndicator
 from ta.momentum import StochasticOscillator
 
 class DataEngine:
-    def __init__(self, symbol, url, provider):
+    def __init__(self, symbol=None, url=None, provider=None, data_dir='data_cache'):
         self.symbol = symbol
         self.url = url
         self.provider = provider
         
-        # 1. Get the folder where engine.py lives
+        # Use your robust path logic
         base_path = os.path.dirname(os.path.abspath(__file__))
-        
+        self.cache_dir = os.path.join(base_path, data_dir) # Use data_dir variable
-        # 2. Define the cache directory path
-        self.cache_dir = os.path.join(base_path, "data_cache")
-        
-        # 3. Create the folder if it doesn't exist (safety-first)
         os.makedirs(self.cache_dir, exist_ok=True)
         
-        # 4. Set the full path for this specific instrument's CSV
+        # 4. Only set file_path if we actually have a symbol
-        self.file_path = os.path.join(self.cache_dir, f"{self.symbol}.csv")
+        if self.symbol:
+            self.file_path = os.path.join(self.cache_dir, f"{self.symbol}.csv")
+        else:
+            self.file_path = None
     
-    def global_sync(self):
+    def load_instruments_from_csv(self, file_path):
-        """The 'One-Click' background loop."""
+        import csv
-        # 1. Get the latest list of instruments from your CSV
+        instruments = []
-        all_instruments = self.load_instruments_from_csv() 
         
-        for item in all_instruments:
+        # Updated templates for maximum historical reach
-            # 2. Update the 'Current' target for the engine
+        TEMPLATES = {
-            self.symbol = item['symbol']
+            'jpm': "https://am.jpmorgan.com/FundsMarketingHandler/historicalData?cusip={cusip}&country=hk&role=per",
-            self.cusip = item['cusip']
+            # period1=0 fetches from the earliest available date; interval=1d is daily
-            self.provider = item['provider']
+            'yahoo': "https://query1.finance.yahoo.com/v8/finance/chart/{cusip}?period1=0&period2=9999999999&interval=1d&events=history",
+            # FT remains 30-day window; Smart Append logic in fetch_data handles the history
+            'agi': "https://markets.ft.com/data/funds/tearsheet/historical?s={cusip}"
+        }
+
+        try:
+            abs_path = os.path.join(os.path.dirname(__file__), file_path)
             
-            # 3. Regenerate the URL and File Path for THIS specific instrument
+            if not os.path.exists(abs_path):
-            self.url = self.generate_url() 
+                print(f"Error: {file_path} not found.")
-            self.file_path = os.path.join(self.data_dir, f"{self.symbol}.csv")
+                return []
-            
+
-            # 4. Run the robust fetch/merge logic we built
+            with open(abs_path, mode='r', encoding='utf-8-sig') as csvfile:
-            print(f"Syncing {self.symbol}...")
+                reader = csv.DictReader(csvfile)
-            self.fetch_data() 
+                reader.fieldnames = [name.strip().lower() for name in reader.fieldnames]
-            
+                
-        print("Global Sync Complete.")
+                for row in reader:
+                    symbol = row.get('symbol', '').strip()
+                    cusip = row.get('cusip', '').strip()
+                    provider = row.get('provider', 'jpm').strip().lower()
+                    
+                    if symbol and cusip:
+                        template = TEMPLATES.get(provider, TEMPLATES['jpm'])
+                        url = template.format(cusip=cusip)
+                        
+                        instruments.append({
+                            "symbol": symbol, 
+                            "url": url, 
+                            "provider": provider,
+                            "cusip": cusip # Added this so sync_all can use it if needed
+                        })
+                    
+        except Exception as e:
+            print(f"CSV Loading Error: {e}")        
+        return instruments
+    # URL_CONFIG = load_instruments_from_csv('instruments.csv')
+        
+    def global_sync(self):
+        """Backup, Sync all instruments, and return a summary report."""
+        # 1. Run Maintenance/Backup
+        self.run_pre_sync_maintenance()
+        
+        # FIX 1: Add 'self.' so it calls the method inside this class
+        instruments = self.load_instruments_from_csv('instruments.csv')
+        
+        report = {
+            "total": len(instruments),
+            "updated": 0,
+            "failed": 0,
+            "details": []
+        }
+        
+        for item in instruments:
+            try:
+                self.symbol = item['symbol']
+                self.provider = item['provider']
+                self.url = item['url']
+                
+                # FIX 2: Use 'self.cache_dir' to match your __init__ logic
+                self.file_path = os.path.join(self.cache_dir, f"{self.symbol}.csv")
+                
+                print(f"Updating {self.symbol}...")
+                
+                # fetch_data now returns the updated DataFrame or None
+                result_df = self.fetch_data()
+
+                time.sleep(1)
+                
+                if result_df is not None and not result_df.empty:
+                    report["updated"] += 1
+                    last_price = result_df['close'].iloc[-1]
+                    report["details"].append(f"✅ {self.symbol}: Updated (Price: {last_price})")
+                else:
+                    report["failed"] += 1
+                    report["details"].append(f"❌ {self.symbol}: No new data found")
+                    
+            except Exception as e:
+                report["failed"] += 1
+                report["details"].append(f"⚠️ {self.symbol}: Error ({str(e)})")
+                
+        return report
+    
+    def run_pre_sync_maintenance(self):
+        """Backs up files and reports current data health."""
+        import os
+        import shutil
+        import pandas as pd
+        from datetime import datetime
+
+        # 1. Setup paths correctly
+        base_dir = os.path.dirname(os.path.abspath(__file__))
+        backup_dir = os.path.join(base_dir, 'backups')
+        
+        # 2. Create the timestamped folder path FIRST
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M")
+        current_backup_path = os.path.join(backup_dir, f"sync_backup_{timestamp}")
+        
+        # 3. Create the directories (safety-first)
+        os.makedirs(current_backup_path, exist_ok=True)
+        
+        print(f"\n--- Pre-Sync Health Check ({timestamp}) ---")
+        stats = []
+
+        # 4. Check if cache exists to avoid errors
+        if not os.path.exists(self.cache_dir):
+            print(f"⚠️ Cache directory not found at {self.cache_dir}")
+            return pd.DataFrame()
+
+        # 5. Backup loop
+        for filename in os.listdir(self.cache_dir): 
+            if filename.endswith(".csv"):
+                src = os.path.join(self.cache_dir, filename)
+                dst = os.path.join(current_backup_path, filename)
+                
+                try:
+                    # Perform copy
+                    shutil.copy2(src, dst)
+                    
+                    # Read data for health check
+                    df = pd.read_csv(src)
+                    
+                    # Store stats
+                    stats.append({
+                        "Fund": filename.replace(".csv", ""),
+                        "Rows": len(df),
+                        "Start": df['date'].min() if 'date' in df.columns else "N/A",
+                        "End": df['date'].max() if 'date' in df.columns else "N/A"
+                    })
+                    print(f"📦 Backed up: {filename} ({len(df)} rows)")
+                    
+                except Exception as e:
+                    print(f"⚠️ Could not backup {filename}: {e}")
+                    continue
+
+        # 6. Display and return report
+        if stats:
+            stats_df = pd.DataFrame(stats)
+            print("\n" + stats_df.to_string(index=False))
+            print(f"\n✅ All backups saved to: {current_backup_path}")
+            return stats_df
+        else:
+            print("📭 No CSV files found to backup.")
+            return pd.DataFrame()
 
     def _parse_jpm(self, json_data):
         if isinstance(json_data, dict) and "historicalNAVList" in json_data:
@@ -165,6 +296,24 @@ class DataEngine:
             print(f"Network error for {self.symbol}: {e}")
             return local_df
     
+    def get_local_metrics(self):
+        """Reads ONLY from local CSV and returns metrics immediately."""
+        if not os.path.exists(self.file_path):
+            return {"error": "Missing Local Data", "status": "needs_sync"}
+        
+        try:
+            df = pd.read_csv(self.file_path)
+            # Ensure columns are clean
+            df.columns = [c.lower().strip() for c in df.columns]
+            df['date'] = pd.to_datetime(df['date'], errors='coerce')
+            df = df.dropna(subset=['date', 'close']).sort_values('date')
+            
+            # Pass this local dataframe to your existing calculation function
+            return self.calculate_table_metrics(df)
+        except Exception as e:
+            print(f"Error reading local data for {self.symbol}: {e}")
+            return None
+    
         
     def calculate_table_metrics(self, df):
         if df is None or df.empty or len(df) < 2:

templates/index.html

@@ -5,6 +5,7 @@
     <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no">
     <title>Financial Dashboard</title>
     <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/css/bootstrap.min.css" rel="stylesheet">
+    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap-icons@1.11.0/font/bootstrap-icons.css">
     <style>
         :root {
             --header-bg: #4a5568;
@@ -96,6 +97,21 @@
         }
 
         #loading { display: none; margin-left: 10px; font-size: 0.8rem; color: #666; }
+        /* 1. Ensure the 52W column is wide enough for the progress bar */
+        .table td:nth-child(4), .table th:nth-child(4) {
+            min-width: 140px; 
+            text-align: left;
+        }
+
+        /* 2. Narrow the EMA and K/D columns since they only have small numbers */
+        .table td:nth-child(n+5), .table th:nth-child(n+5) {
+            min-width: 75px;
+        }
+
+        /* 3. Give the Instrument column a bit more breathing room if names are long */
+        .table td:first-child, .table th:first-child {
+            min-width: 180px; /* Increased from 140px */
+}
     </style>
 </head>
 <body>
@@ -104,9 +120,16 @@
     <div class="card">
         <div class="card-header d-flex justify-content-between align-items-center">
             <h5 class="mb-0 text-dark">Portfolio Signals</h5>
-            <div>
+            <div class="d-flex align-items-center gap-2">
-                <span id="loading">Updating...</span>
+                <span id="loading" class="me-2" style="display:none;">Updating...</span>
-                <button class="btn btn-refresh btn-sm" onclick="loadData()">Refresh</button>
+                
+                <button class="btn btn-outline-secondary btn-sm" onclick="loadData()">
+                    <i class="bi bi-arrow-clockwise"></i> Refresh Table
+                </button>
+                
+                <button id="syncBtn" class="btn btn-primary btn-sm" onclick="runGlobalSync()">
+                    <i class="bi bi-cloud-download"></i> Sync New Data
+                </button>
             </div>
         </div>
         <div class="card-body p-0">
@@ -114,15 +137,14 @@
                 <table class="table table-hover table-striped mb-0">
                     <thead>
                         <tr>
-                            <th>Instrument</th>
+                            <th style="width: 25%;">Instrument</th>
-                            <th>Close</th>
+                            <th style="width: 10%;">Close</th>
-                            <th>Chg%</th>
+                            <th style="width: 10%;">Chg%</th>
-                            <th>52W Range</th>
+                            <th style="width: 10%;">52W Range</th> <th style="width: 8%;">20 EMA</th>
-                            <th>v20 EMA</th>
+                            <th style="width: 10%;">50 EMA</th>
-                            <th>v50 EMA</th>
+                            <th style="width: 10%;">100 EMA</th>
-                            <th>v100 EMA</th>
+                            <th style="width: 10%;">200 EMA</th>
-                            <th>v200 EMA</th>
+                            <th style="width: 5%;">K/D</th>
-                            <th>K/D</th>
                         </tr>
                     </thead>
                     <tbody id="tableBody">
@@ -133,13 +155,30 @@
         </div>
     </div>
 </div>
-
 <script>
+    console.log("Script block loaded successfully.");
+
+    // --- 1. Helper Function for K/D Styling ---
+    // Defined at the top level so it's ready before loadData runs
+    function formatKD(val) {
+        if (!val || val === "N/A" || !val.includes('/')) return `<span class="text-muted">N/A</span>`;
+        
+        const [k, d] = val.split('/').map(v => parseFloat(v));
+        
+        let colorClass = 'text-dark'; // Default
+        if (k >= 80) colorClass = 'text-danger fw-bold';      // Overbought
+        else if (k <= 20) colorClass = 'text-success fw-bold'; // Oversold
+        
+        return `<span class="${colorClass}">${val}</span>`;
+    }
+
+    // --- 2. Load Table Data (Fast) ---
     async function loadData() {
+        console.log("Starting loadData...");
         const loading = document.getElementById('loading');
         const tbody = document.getElementById('tableBody');
         
-        loading.style.display = 'inline';
+        if (loading) loading.style.display = 'inline';
         
         try {
             const response = await fetch('/api/summary');
@@ -147,47 +186,117 @@
             
             tbody.innerHTML = '';
 
-            if (data.length === 0) {
+            if (!data || data.length === 0) {
-                tbody.innerHTML = '<tr><td colspan="9" class="p-4">No instruments found in CSV.</td></tr>';
+                tbody.innerHTML = '<tr><td colspan="9" class="p-4">No data found. Please run Sync.</td></tr>';
                 return;
             }
 
-            data.forEach(item => {
+        data.forEach(item => {
-                // Helper to format EMA offsets with +/- and Colors
+            // 1. GATEKEEPER: Check if the item has an error or is missing data
-                const formatEma = (val) => {
+            if (item.error || !item.last_close) {
-                    if (val === "N/A" || val === null) return `<span class="text-muted">N/A</span>`;
+                const errorRow = `
-                    const sign = val > 0 ? "+" : "";
-                    const colorClass = val >= 0 ? 'text-up' : 'text-down';
-                    return `<span class="${colorClass}">${sign}${val}%</span>`;
-                };
-
-                const row = `
                     <tr>
-                        <td>${item.symbol}</td>
+                        <td>${item.symbol || 'Unknown'}</td>
-                        <td class="fw-bold">${item.last_close}</td>
+                        <td colspan="8" class="text-center p-3">
-                        <td class="${item.change_pct >= 0 ? 'text-up' : 'text-down'}">
+                            <span class="badge bg-warning text-dark">
-                            ${item.change_pct >= 0 ? '+' : ''}${item.change_pct}%
+                                <i class="bi bi-exclamation-triangle"></i> Needs Sync
+                            </span>
+                            <small class="text-muted ms-2">Local CSV not found or corrupted.</small>
                         </td>
-                        <td class="text-muted small">${item.low_52} - ${item.high_52}</td>
-                        <td>${formatEma(item.last_ema20)}</td>
-                        <td>${formatEma(item.last_ema50)}</td>
-                        <td>${formatEma(item.last_ema100)}</td>
-                        <td>${formatEma(item.last_ema200)}</td>
-                        <td><span class="badge badge-kd">${item.kd_values}</span></td>
                     </tr>
                 `;
-                tbody.innerHTML += row;
+                tbody.innerHTML += errorRow;
-            });
+                return; // This skips the rest of the math and goes to the next fund
+            }
+
+            // --- A. Helper for EMA colors (Your existing code) ---
+            const formatEma = (val) => {
+                if (val === "N/A" || val === null || val === undefined) return `<span class="text-muted">N/A</span>`;
+                const num = parseFloat(val);
+                const sign = num > 0 ? "+" : "";
+                const colorClass = num >= 0 ? 'text-up' : 'text-down';
+                return `<span class="${colorClass}">${sign}${num.toFixed(1)}%</span>`;
+            };
+
+            // --- B. Calculate 52W Range logic ---
+            const current = parseFloat(item.last_close) || 0;
+            const low = parseFloat(item.low_52) || 0;
+            const high = parseFloat(item.high_52) || 0;
+            
+            let rangePct = 0;
+            if (high > low) {
+                rangePct = ((current - low) / (high - low)) * 100;
+                rangePct = Math.min(Math.max(rangePct, 0), 100);
+            }
+
+            const rangeColor = rangePct > 80 ? 'text-danger' : (rangePct < 20 ? 'text-success' : 'text-muted');
+
+            // --- C. Build the Row (Your existing code) ---
+            const row = `
+                <tr>
+                    <td>${item.symbol}</td>
+                    <td class="fw-bold">${item.last_close}</td>
+                    <td class="${item.change_pct >= 0 ? 'text-up' : 'text-down'}">
+                        ${item.change_pct >= 0 ? '+' : ''}${item.change_pct}%
+                    </td>
+                    <td class="${rangeColor} small">
+                        <div class="d-flex justify-content-between mb-1" style="min-width: 100px;">
+                            <span>${item.low_52}</span>
+                            <span>${item.high_52}</span>
+                        </div>
+                        <div class="progress" style="height: 5px;">
+                            <div class="progress-bar bg-primary" style="width: ${rangePct}%"></div>
+                        </div>
+                    </td>
+                    <td>${formatEma(item.last_ema20)}</td>
+                    <td>${formatEma(item.last_ema50)}</td>
+                    <td>${formatEma(item.last_ema100)}</td>
+                    <td>${formatEma(item.last_ema200)}</td>
+                    <td>${formatKD(item.kd_values)}</td>
+                </tr>
+            `;
+            tbody.innerHTML += row;
+        });    
         } catch (error) {
             console.error("Fetch error:", error);
-            tbody.innerHTML = '<tr><td colspan="9" class="text-danger p-4">Error connecting to server.</td></tr>';
+            tbody.innerHTML = '<tr><td colspan="9" class="text-danger p-4">Error loading local data. Check Console.</td></tr>';
         } finally {
-            loading.style.display = 'none';
+            if (loading) loading.style.display = 'none';
         }
     }
 
-    // Initial Load
+    // --- 3. Run Global Sync (Slow) ---
-    document.addEventListener('DOMContentLoaded', loadData);
+    async function runGlobalSync() {
+        const syncBtn = document.getElementById('syncBtn');
+        const loading = document.getElementById('loading');
+        
+        if (!syncBtn) return;
+
+        syncBtn.disabled = true;
+        const originalText = syncBtn.innerHTML;
+        syncBtn.innerHTML = `<span class="spinner-border spinner-border-sm"></span> Syncing...`;
+        if (loading) loading.style.display = 'inline';
+
+        try {
+            const response = await fetch('/api/sync', { method: 'POST' });
+            if (!response.ok) throw new Error("Server error during sync");
+            
+            await loadData(); // Reload table after sync
+            alert("Sync Complete! Data updated.");
+        } catch (error) {
+            console.error("Sync error:", error);
+            alert("Sync failed. Check terminal for Python errors.");
+        } finally {
+            syncBtn.disabled = false;
+            syncBtn.innerHTML = originalText;
+            if (loading) loading.style.display = 'none';
+        }
+    }
+
+    // --- 4. Initial Trigger ---
+    window.onload = function() {
+        loadData();
+    };
 </script>
 
 </body>