feat: implement map view for lake visualization and automate data scraping pipeline

2026-06-05 22:03:38 +02:00
parent a5bd4985d1
commit 61a8af109c
35 changed files with 3894 additions and 930 deletions
@@ -0,0 +1,62 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import { lakesConfig } from './lakesConfig';
+
+interface DataRecord {
+  timestamp: string;
+  level: number;
+  flow: number;
+}
+
+const lakes = lakesConfig.map(lake => {
+  const [internalId, oid] = lake.id.split('|');
+  const DATA_FILE = path.resolve(`public/data/${internalId}.json`);
+  
+  let currentLevel = 0;
+  let currentFlow = 0;
+  let sparkline: number[] = Array(12).fill(0);
+
+  if (fs.existsSync(DATA_FILE)) {
+    try {
+      const data: DataRecord[] = JSON.parse(fs.readFileSync(DATA_FILE, 'utf-8'));
+      if (data.length > 0) {
+        // Find latest valid record or just the last record
+        const lastValidLevelData = [...data].reverse().find(d => d.level !== null && !isNaN(d.level));
+        const lastValidFlowData = [...data].reverse().find(d => d.flow !== null && !isNaN(d.flow) && d.flow > 0);
+        
+        currentLevel = lastValidLevelData ? lastValidLevelData.level : 0;
+        currentFlow = lastValidFlowData ? lastValidFlowData.flow : 0;
+
+        // Take up to 12 last records for sparkline
+        const recentData = data.slice(-12);
+        sparkline = recentData.map(d => (d.flow === null || isNaN(d.flow) ? 0 : d.flow));
+        
+        // Pad with zeros if less than 12
+        while (sparkline.length < 12) {
+          sparkline.unshift(0);
+        }
+      }
+    } catch (e) {
+      console.error(`Error reading data for ${internalId}`, e);
+    }
+  }
+
+  return {
+    id: lake.id,
+    name: lake.text.replace('VD ', '').split('-')[0].trim(),
+    river: lake.text.includes('-') ? lake.text.split('-')[1].trim() : '',
+    priority: lake.priority || false,
+    level: currentLevel.toFixed(2),
+    capacity: 0, // Removed fake capacity
+    inflow: currentFlow.toFixed(1),
+    outflow: currentFlow.toFixed(1),
+    volume: lake.maxVolume || 0, // Using real maxVolume if known
+    lat: lake.coords[0],
+    lng: lake.coords[1],
+    sparkline
+  };
+});
+
+const outputPath = path.resolve(process.cwd(), 'public/data/lakes_index.json');
+fs.writeFileSync(outputPath, JSON.stringify(lakes, null, 2));
+console.log('Real lakes index generated:', lakes.length);
@@ -1,55 +0,0 @@
-import * as fs from 'fs';
-import * as path from 'path';
-
-interface LakeRaw {
-  id: string;
-  text: string;
-  priority?: boolean;
-}
-
-const lakesRaw: LakeRaw[] = [
-  { id: "VLL1|1", text: "VD Lipno 1 - Vltava", priority: true },
-  { id: "VLOR|1", text: "VD Orlík - Vltava", priority: true },
-  { id: "VLSL|1", text: "VD Slapy - Vltava", priority: false },
-  { id: "BLHU|1", text: "VD Husinec - Blanice (PI)" },
-  { id: "BIBI|1", text: "VD Bílsko - Bílský potok" },
-  { id: "KLDP|3", text: "VD Dolejší Padrťský rybník" },
-  { id: "KLHP|3", text: "VD Hořejší Padrťský rybník" },
-  { id: "KLKL|3", text: "VD Klabava - Klabava" },
-  { id: "KCKC|3", "text": "VD Klíčava - Klíčava" },
-  { id: "LILA|3", "text": "VD Láz - Litavka" },
-  { id: "MARI|1", "text": "VD Římov - Malše" },
-  { id: "MZHR|3", "text": "VD Hracholusky - Mže" },
-  { id: "MZLU|3", "text": "VD Lučina - Mže" },
-  { id: "MZSS|3", "text": "VD Plzeň-Štruncovy sady" },
-  { id: "OPOB|3", "text": "VD Obecnice - Obecnický potok" },
-  { id: "PPPI|3", "text": "VD Pilská - Pilský potok" },
-  { id: "RACU|3", "text": "VD České Údolí - Radbuza" },
-  { id: "SPNE|2", "text": "VD Němčice - Sedlický potok" },
-  { id: "SVKR|1", "text": "VD Švihov - Želivka" },
-  { id: "UHKA|1", "text": "VD Kamýk - Vltava" },
-  { id: "VRSN|1", "text": "VD Vrané - Vltava" },
-  { id: "ZLUT|3", "text": "VD Žlutice - Střela" },
-  // Adding dummies to reach ~40
-  ...Array.from({length: 18}).map((_, i) => ({ id: `DUMMY${i}`, text: `VD Dummy Lake ${i+1}` }))
-];
-
-const lakes = lakesRaw.map(lake => {
-  const sparkline = Array.from({length: 12}).map(() => 50 + Math.random() * 20);
-  return {
-    id: lake.id,
-    name: lake.text.replace('VD ', '').split('-')[0].trim(),
-    river: lake.text.includes('-') ? lake.text.split('-')[1].trim() : '',
-    priority: lake.priority || false,
-    level: (200 + Math.random() * 500).toFixed(2),
-    capacity: Math.floor(20 + Math.random() * 80), // 20% to 100%
-    inflow: (Math.random() * 20).toFixed(1),
-    outflow: (Math.random() * 20).toFixed(1),
-    volume: (Math.random() * 300).toFixed(1),
-    sparkline
-  };
-});
-
-const outputPath = path.resolve(process.cwd(), 'public/data/lakes_index.json');
-fs.writeFileSync(outputPath, JSON.stringify(lakes, null, 2));
-console.log('Mock lakes generated:', lakes.length);
@@ -0,0 +1,31 @@
+export interface LakeConfig {
+  id: string;
+  text: string;
+  priority?: boolean;
+  coords: [number, number];
+}
+
+export const lakesConfig: LakeConfig[] = [
+  { id: "VLL1|1", text: "VD Lipno 1 - Vltava", priority: true, coords: [48.6322, 14.2215], maxVolume: 306.0 },
+  { id: "VLOR|2", text: "VD Orlík - Vltava", priority: true, coords: [49.6060, 14.1700], maxVolume: 716.5 },
+  { id: "VLSL|2", text: "VD Slapy - Vltava", priority: false, coords: [49.8220, 14.4360], maxVolume: 269.3 },
+  { id: "BLHU|1", text: "VD Husinec - Blanice (PI)", coords: [49.0520, 13.9830], maxVolume: 6.9 },
+  { id: "BIBI|1", text: "VD Bílsko - Bílský potok", coords: [49.1910, 14.0530] },
+  { id: "KLDP|3", text: "VD Dolejší Padrťský rybník", coords: [49.6640, 13.7660] },
+  { id: "KLHP|3", text: "VD Hořejší Padrťský rybník", coords: [49.6540, 13.7840] },
+  { id: "KLKL|3", text: "VD Klabava - Klabava", coords: [49.7560, 13.5650] },
+  { id: "KCKC|3", "text": "VD Klíčava - Klíčava", coords: [50.0650, 13.9290], maxVolume: 8.3 },
+  { id: "LILA|3", text: "VD Láz - Litavka", coords: [49.6670, 13.8820] },
+  { id: "MARI|1", text: "VD Římov - Malše", coords: [48.8470, 14.4870], maxVolume: 33.8 },
+  { id: "MZHR|3", text: "VD Hracholusky - Mže", coords: [49.7890, 13.1550], maxVolume: 56.7 },
+  { id: "MZLU|3", text: "VD Lučina - Mže", coords: [49.8000, 12.6100] },
+  { id: "MZSS|3", text: "VD Plzeň-Štruncovy sady", coords: [49.7510, 13.3850] },
+  { id: "OPOB|3", "text": "VD Obecnice - Obecnický potok", coords: [49.7210, 13.9450] },
+  { id: "PPPI|3", "text": "VD Pilská - Pilský potok", coords: [49.6760, 13.8960] },
+  { id: "RACU|3", "text": "VD České Údolí - Radbuza", coords: [49.7110, 13.3610] },
+  { id: "SPNE|2", "text": "VD Němčice - Sedlický potok", coords: [49.6050, 15.2280] },
+  { id: "SVKR|2", "text": "VD Švihov - Želivka", coords: [49.7180, 15.1060], maxVolume: 266.6 },
+  { id: "UHKA|2", "text": "VD Kamýk - Vltava", coords: [49.6360, 14.2540], maxVolume: 12.8 },
+  { id: "VRSN|2", "text": "VD Vrané - Vltava", coords: [49.9340, 14.3850], maxVolume: 11.1 },
+  { id: "ZLUT|3", "text": "VD Žlutice - Střela", coords: [50.0930, 13.1590] }
+];
@@ -3,9 +3,7 @@ import * as path from 'path';
 import * as cheerio from 'cheerio';
 import axios from 'axios';
 import https from 'https';
-
-const URL = 'https://www.pvl.cz/portal/nadrze/cz/pc/Mereni.aspx?oid=1&id=VLL1';
-const DATA_FILE = path.resolve('public/data/lipno.json');
+import { lakesConfig } from './lakesConfig';

 interface DataRecord {
  timestamp: string;
@@ -23,12 +21,12 @@ function parseDateString(dateStr: string): string {
  return d.toISOString();
 }

-async function scrape(): Promise<void> {
-  try {
-    const agent = new https.Agent({  
-      rejectUnauthorized: false
-    });
+async function scrapeLake(lakeId: string, oid: string, internalId: string) {
+  const URL = `https://www.pvl.cz/portal/nadrze/cz/pc/Mereni.aspx?oid=${oid}&id=${internalId}`;
+  const DATA_FILE = path.resolve(`public/data/${internalId}.json`);

+  try {
+    const agent = new https.Agent({ rejectUnauthorized: false });
    const response = await axios.get(URL, {
      httpsAgent: agent,
      headers: {
@@ -38,7 +36,6 @@ async function scrape(): Promise<void> {

    const html = response.data;
    const $ = cheerio.load(html);
-    
    const rows = $('table tr');
    const newData: DataRecord[] = [];
    
@@ -46,7 +43,6 @@ async function scrape(): Promise<void> {
      const tds = $(row).find('td');
      if (tds.length >= 3) {
        const datetimeText = $(tds[0]).text().trim();
-        // Check if it's a valid date string matching DD.MM.YYYY HH:MM
        if (/^\d{2}\.\d{2}\.\d{4}\s\d{2}:\d{2}$/.test(datetimeText)) {
          const timestamp = parseDateString(datetimeText);
          const levelText = $(tds[1]).text().trim().replace(',', '.');
@@ -61,33 +57,42 @@ async function scrape(): Promise<void> {
      }
    });

-    // Load existing data
    let existingData: DataRecord[] = [];
    if (fs.existsSync(DATA_FILE)) {
      const fileContent = fs.readFileSync(DATA_FILE, 'utf-8');
      existingData = JSON.parse(fileContent);
    }

-    // Merge and deduplicate by timestamp
    const dataMap = new Map<string, DataRecord>();
    existingData.forEach(item => dataMap.set(item.timestamp, item));
    newData.forEach(item => dataMap.set(item.timestamp, item));

-    // Sort chronologically
    const mergedData = Array.from(dataMap.values()).sort((a, b) => {
      return new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime();
    });

-    // Save back
    fs.mkdirSync(path.dirname(DATA_FILE), { recursive: true });
    fs.writeFileSync(DATA_FILE, JSON.stringify(mergedData, null, 2), 'utf-8');
    
-    console.log(`Scraped ${newData.length} records. Total records in DB: ${mergedData.length}`);
+    console.log(`[${internalId}] Scraped ${newData.length} records. DB total: ${mergedData.length}`);

  } catch (error: any) {
-    console.error('Error scraping data:', error.message);
-    process.exit(1);
+    console.error(`[${internalId}] Error scraping data:`, error.message);
  }
 }

-scrape();
+async function runScraper() {
+  console.log(`Starting bulk scraper for ${lakesConfig.length} lakes...`);
+  
+  for (const lake of lakesConfig) {
+    // ID format: VLL1|1 -> internalId=VLL1, oid=1
+    const [internalId, oid] = lake.id.split('|');
+    await scrapeLake(lake.id, oid, internalId);
+    // Add small delay to not hammer the server
+    await new Promise(resolve => setTimeout(resolve, 500));
+  }
+  
+  console.log('Bulk scraping finished.');
+}
+
+runScraper();