add last sync time in index, fix csv file caching time in fetching new data

2026-02-06 03:56:55 +08:00
parent 2d9fa9a47b
commit c0f158684f
29 changed files with 53379 additions and 704 deletions
@@ -5293,3 +5293,7 @@ date,close
 2026-01-27,37.18
 2026-01-28,37.23
 2026-01-29,37.2
+2026-01-30,37.11
+2026-02-02,36.99
+2026-02-03,37.17
+2026-02-04,37.12
@@ -0,0 +1,24 @@
+date,close
+2026-01-05,22.87
+2026-01-06,22.77
+2026-01-07,22.61
+2026-01-08,22.56
+2026-01-09,22.49
+2026-01-12,22.54
+2026-01-13,22.66
+2026-01-14,22.66
+2026-01-15,22.93
+2026-01-16,22.9
+2026-01-19,23.01
+2026-01-20,23.0
+2026-01-21,22.66
+2026-01-22,23.02
+2026-01-23,23.25
+2026-01-26,23.34
+2026-01-27,23.04
+2026-01-28,21.99
+2026-01-29,21.97
+2026-01-30,22.46
+2026-02-02,22.51
+2026-02-03,22.78
+2026-02-04,22.85
@@ -4365,3 +4365,6 @@ date,close
 2026-01-28,28.72
 2026-01-29,29.1
 2026-01-30,28.99
+2026-02-02,27.73
+2026-02-03,28.9
+2026-02-04,29.0
@@ -4343,3 +4343,11 @@ date,close
 2026-01-22,279.27
 2026-01-23,279.99
 2026-01-26,283.53
+2026-01-27,285.51
+2026-01-28,290.19
+2026-01-29,290.78
+2026-01-30,288.15
+2026-02-02,281.4
+2026-02-03,290.03
+2026-02-04,289.6
+2026-02-05,283.7
@@ -1,5 +1,4 @@
 date,close
-2026-01-02,258.96
 2026-01-05,263.47
 2026-01-06,268.56
 2026-01-07,267.9
@@ -20,3 +19,6 @@ date,close
 2026-01-28,290.19
 2026-01-29,290.78
 2026-01-30,288.15
+2026-02-02,281.4
+2026-02-03,290.03
+2026-02-04,289.6
@@ -16,6 +16,9 @@ class DataEngine:
        self.symbol = symbol.strip().upper() if symbol else None
        self.name = name
        
+        # --- Fetch frequently ---
+        self.cache_expiry = 1 * 3600  # Set to 0 for forced refresh, 24 for normal
+
        # 2. Setup the directory variable FIRST 
        # (This was likely below the file_path line, causing the crash)
        base_path = os.path.dirname(os.path.abspath(__file__))
@@ -50,21 +53,17 @@ class DataEngine:
        
    def ensure_data(self):
        """Checks if file exists and is fresh (less than 24h old)."""
-        CACHE_EXPIRY = 24 * 3600  # 24 hours
+        # Force expiry to 0
+        CACHE_EXPIRY = 0 
        
        if os.path.exists(self.file_path):
-            # NEW: Check how old the file is
-            file_age = time.time() - os.path.getmtime(self.file_path)
-            if file_age < CACHE_EXPIRY:
-                return True  # Data is actually fresh
+            # By changing this to 'if False', we force it to ignore the cache every time
+            if False: # file_age < CACHE_EXPIRY:
+                return True 
            else:
-                print(f"DEBUG: {self.symbol} cache is stale ({round(file_age/3600)}h old). Refreshing...")
-        else:
-            print(f"DEBUG: {self.symbol} not found in cache. Attempting download...")
+                print(f"DEBUG: {self.symbol} refreshing now...")
        
-        # If we reached here, it means we either have NO file or a STALE file
-        # Instead of just yfinance, call your specialized fetch_data() 
-        # which uses the URLs from your TEMPLATES
+        # This calls your actual downloader
        return self.fetch_data()

    def load_instruments_from_csv(self, file_path='instruments.csv'):
@@ -276,7 +275,7 @@ class DataEngine:
    
    def fetch_data(self):
        local_df = pd.DataFrame()
-        CACHE_EXPIRY = 24 * 3600  
+        CACHE_EXPIRY = 0 
        file_exists = os.path.exists(self.file_path)
        
        # 1. Load Local Cache & Check Age
@@ -1,5 +1,17 @@
 name,cusip,provider
-JPMorgan Evergreen,HK0000055829,jpm
-Allianz Oriental Income Cl A,LU0348783233:USD,agi
-SPMO ETF - USD,SPMO,yahoo
-JPM Korea,LU0301634860,jpm
+JPM Vietnam,HK0000055811,jpm
+JPM Pacific ,HK0000055746,jpm
+JPM Korea ,LU0301634860,jpm
+JPM India ,MU0129U00005,jpm
+JPM Evergreen,HK0000055829,jpm
+JPM ASEAN,HK0000055555,jpm
+JPM Japan,LU0129465034,jpm
+JPM G HYB,LU0356780857,jpm
+JPM Europe,LU0119078227,jpm
+INDA INDIA,INDA,yahoo
+Fidelity Indonesia,LU0055114457,ft
+AGI Oriental Income,LU0348783233,ft
+VT,VT,yahoo
+VHYD,VHYD.L,yahoo
+SPMO,SPMO,yahoo
+EWJV,EWJV,yahoo
@@ -0,0 +1,36 @@
+beautifulsoup4==4.14.3
+blinker==1.9.0
+cachelib==0.13.0
+certifi==2026.1.4
+cffi==2.0.0
+charset-normalizer==3.4.4
+click==8.3.1
+curl_cffi==0.13.0
+Flask==3.1.2
+Flask-Caching==2.3.1
+frozendict==2.4.7
+html5lib==1.1
+idna==3.11
+itsdangerous==2.2.0
+Jinja2==3.1.6
+lxml==6.0.2
+MarkupSafe==3.0.3
+multitasking==0.0.12
+numpy==2.4.1
+pandas==3.0.0
+peewee==3.19.0
+platformdirs==4.5.1
+protobuf==6.33.4
+pycparser==3.0
+python-dateutil==2.9.0.post0
+pytz==2025.2
+requests==2.32.5
+six==1.17.0
+soupsieve==2.8.3
+ta==0.11.0
+typing_extensions==4.15.0
+urllib3==2.6.3
+webencodings==0.5.1
+websockets==16.0
+Werkzeug==3.1.5
+yfinance==1.1.0
@@ -88,7 +88,8 @@
        .table th:first-child {
            position: sticky;
            text-align: left !important; /* Force left alignment */
-            padding-left: 15px;         /* Add space so text doesn't touch the edge */
+            padding-left: 15px;
+            color: #2d3748/* Add space so text doesn't touch the edge */
            left: 0;
            z-index: 10;
            background-color: var(--sticky-bg);
@@ -101,6 +102,23 @@
            /* Clean border to define the edge */
            border-right: 1px solid #ddd;
        }
+        /* Specific fix for Close and Chg% columns readability */
+		.table td:nth-child(3), 
+		.table td:nth-child(4) {
+		    font-weight: 600;
+		    color: #1a202c !important; /* Extra bold and dark */
+		}
+		
+		/* Navbar fix: Ensure buttons don't disappear on tiny screens */
+		.navbar-nav {
+		    flex-direction: row !important; /* Keep links side-by-side on mobile */
+		    gap: 10px;
+		}
+		
+		.nav-link {
+		    padding: 0.5rem !important;
+		    font-size: 0.85rem;
+		}

        /* 4. Zebra Striping + Sticky Fix */
        .table-striped tbody tr:nth-of-type(odd) {
@@ -115,6 +133,39 @@
        .text-up { color: var(--text-up); font-weight: 600; }
        .text-down { color: var(--text-down); font-weight: 600; }
        
+        .table-date {
+		    font-size: 0.8rem !important;   /* Increased slightly to make it obvious */
+		    font-weight: 800 !important;    /* Extra bold */
+		    font-family: 'Courier New', monospace !important;
+		    display: inline-block;          /* Sometimes helps with sizing */
+		    white-space: nowrap;              /* Prevents the date from snapping to 2 lines */
+		}
+        .status-dot {
+            width: 8px;
+            height: 8px;
+            background-color: #39FF14; /* Neon green for better visibility */
+            border-radius: 50%;
+            display: inline-block;
+            box-shadow: 0 0 10px rgba(57, 255, 20, 0.4);
+            animation: status-pulse 2s infinite;
+        }
+
+        /* Custom text class if Bootstrap classes aren't bright enough */
+        .text-white-50 {
+            color: rgba(255, 255, 255, 0.7) !important; /* Increased from 0.5 to 0.7 for clarity */
+        }
+
+        @keyframes status-pulse {
+            0% { transform: scale(0.95); opacity: 1; }
+            50% { transform: scale(1.1); opacity: 0.7; }
+            100% { transform: scale(0.95); opacity: 1; }
+        }
+
+        #lastSyncTime {
+            font-size: 0.85rem;
+            letter-spacing: 0.5px;
+        }
+
        @media (prefers-color-scheme: dark) {
            :root {
                --header-bg: #1a202c; /* Darker header */
@@ -135,24 +186,27 @@
    </style>
 </head>
 <body>
-
-<nav class="navbar navbar-expand-lg navbar-dark bg-dark mb-4">
-    <div class="container-fluid">
-        <a class="navbar-brand" href="/">
+<nav class="navbar navbar-dark mb-4" style="background-color: #1a202c !important;">
+    <div class="container-fluid d-flex justify-content-between align-items-center">
+        <div class="d-flex align-items-center">
+            <a class="navbar-brand me-4" href="/">
                <i class="bi bi-graph-up-arrow me-2"></i>Finance Suite
            </a>
-        <div class="collapse navbar-collapse" id="navbarNav">
-            <ul class="navbar-nav">
-                <li class="nav-item"><a class="nav-link" href="/">Dashboard</a></li>
-                <li class="nav-item"><a class="nav-link" href="/backtest">Backtester</a></li>
-            </ul>
+            <div class="d-flex gap-2">
+                <a class="btn btn-sm btn-outline-light border-secondary" href="/">Dashboard</a>
+                <a class="btn btn-sm btn-outline-light border-secondary" href="/backtest">Backtest</a>
+            </div>
+        </div>
+
+        <div class="d-flex align-items-center" style="font-size: 0.8rem; letter-spacing: 0.3px;">
+            <div class="status-dot me-2"></div>
+            <span class="text-white-50">System Online</span>
+            <span class="mx-2 text-white-50 opacity-25">|</span>
+            <span class="text-white-50">Last Sync:</span>
+            <span id="lastSyncTime" class="ms-1 fw-bold text-white font-monospace">--:--:--</span>
        </div>
-        <span class="navbar-text text-light small d-none d-md-inline">
-            System Status: <span class="text-success">● Online</span>
-        </span>
    </div>
 </nav>
-
 <div class="container-fluid p-0">
    <div class="card">
        <div class="card-header d-flex justify-content-between align-items-center">
@@ -219,20 +273,39 @@
        console.log("Starting loadData...");
        const loading = document.getElementById('loading');
        const tbody = document.getElementById('tableBody');
+        const syncDisplay = document.getElementById('lastSyncTime'); // Get our new element
+        
        if (loading) loading.style.display = 'inline';
        
        try {
            const response = await fetch('/api/summary');
            const data = await response.json();

+            const syncDisplay = document.getElementById('lastSyncTime');
+            if (syncDisplay) {
+                const now = new Date();
+                syncDisplay.innerText = now.toLocaleTimeString([], { hour12: false });
+                
+                // Brief highlight effect
+                syncDisplay.classList.add('text-success');
+                setTimeout(() => syncDisplay.classList.remove('text-success'), 2000);
+        }   
+            // --- Update the Sync Time Display ---
+            const now = new Date();
+            const timeStr = now.toLocaleTimeString([], { hour12: false }); // e.g. 02:45:10
+            const dateStr = now.toLocaleDateString();
+            if (syncDisplay) {
+                syncDisplay.innerHTML = `${dateStr} ${timeStr}`;
+            }
+             // -----------------------------------------            
+            
            if (!data || data.length === 0) {
-                tbody.innerHTML = '<tr><td colspan="10" class="p-4">No data found. Please run Sync.</td></tr>';
+                tbody.innerHTML = '<tr><td colspan="10" class="p-4 text-center">No data found. Please run Sync.</td></tr>';
                return;
            }

-            // Get today's date string (YYYY-MM-DD) to check freshness
-            const todayStr = new Date().toISOString().split('T')[0];
            let htmlContent = ''; 
+            const todayStr = new Date().toISOString().split('T')[0];
            
            data.forEach(item => {
                // --- 1. Error Row Handling ---
@@ -241,37 +314,34 @@
                        <tr>
                            <td>${item.name || 'Unknown'}</td>
                            <td colspan="9" class="text-center p-3">
-                                <span class="badge bg-warning text-dark"><i class="bi bi-exclamation-triangle"></i> Needs Sync</span>
+                                <span class="badge bg-warning text-dark">Needs Sync</span>
                                <small class="text-muted ms-2">Local CSV not found or corrupted.</small>
                            </td>
                        </tr>`;
-                    return;
+                    return; // Skip to next item
                }
+
+                // --- 2. Date & Style Calculations ---
                let displayDate = "N/A";
+                let dateColor = '#dc3545'; // Default red
+                
                if (item.last_date && item.last_date.includes('-')) {
                    const parts = item.last_date.split('-');
-                    const formatted = `${parts[2]}/${parts[1]}`;
-                        
-                    // Get today's date in YYYY-MM-DD format to compare
-                    const today = new Date().toISOString().split('T')[0];
-                    
-                    // Pick color based on freshness
-                    const color = (item.last_date === today) ? '#28a745' : '#dc3545'; 
-                    
-                    displayDate = `<span style="color: ${color};">${formatted}</span>`;
+                    displayDate = `${parts[2]}/${parts[1]}`;
+                    dateColor = (item.last_date === todayStr) ? '#28a745' : '#dc3545';
                }
-                // --- 2. Calculations & Styling ---
+
                const current = parseFloat(item.last_close) || 0;
                const low = parseFloat(item.low_52) || 0;
                const high = parseFloat(item.high_52) || 0;
                
-                // 52W Range Progress Bar
+                // 52W Range Progress Bar calculation
                let rangePct = high > low ? Math.min(Math.max(((current - low) / (high - low)) * 100, 0), 100) : 0;
                const rangeColor = rangePct > 80 ? 'text-danger' : (rangePct < 20 ? 'text-success' : 'text-muted');

-                // Date Freshness check (highlight if date matches today)
+                // Freshness class for the badge
                const isFresh = item.last_date === todayStr;
-                const dateStyle = isFresh ? 'badge bg-success-subtle text-success border border-success-subtle' : 'text-muted';
+                const dateBadgeClass = isFresh ? 'badge bg-success-subtle text-success border border-success-subtle' : 'text-muted';

                // --- 3. Construct Row ---
                htmlContent += `
@@ -279,7 +349,11 @@
                        <td>
                            <div class="fw-bold">${item.name || item.symbol}</div>                            
                        </td>
-                        <td><span class="${dateStyle}" style="font-size: 0.75rem; padding: 2px 5px; border-radius: 4px;">${displayDate}</span></td>
+                        <td>
+                            <span class="${dateBadgeClass} table-date" style="color: ${dateColor} !important; font-size: 0.75rem; padding: 2px 5px; border-radius: 4px;">
+                                ${displayDate}
+                            </span>
+                        </td>
                        <td class="fw-bold">${item.last_close}</td>
                        <td class="${item.change_pct >= 0 ? 'text-up' : 'text-down'}">
                            ${item.change_pct >= 0 ? '+' : ''}${item.change_pct}%
@@ -298,14 +372,16 @@
                        <td>${formatEma(item.last_ema200)}</td>
                        <td>${formatKD(item.kd_values)}</td>
                    </tr>`;
-            });
+            }); // End forEach

-            // Batch update the DOM once
+            // 4. Final Injection
            tbody.innerHTML = htmlContent;

        } catch (error) {
            console.error("Fetch error:", error);
-            tbody.innerHTML = '<tr><td colspan="10" class="text-danger p-4">Error loading summary. Check server logs.</td></tr>';
+            if (tbody) {
+                tbody.innerHTML = '<tr><td colspan="10" class="text-danger p-4 text-center">Error loading summary. Check server logs.</td></tr>';
+            }
        } finally {
            if (loading) loading.style.display = 'none';
        }