fencl/davisfe.cz
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: update lake index, sync scraping scripts, and prune unused data files

Browse Source
This commit is contained in:
David Fencl
2026-06-05 22:24:47 +02:00
parent 61a8af109c
commit 5411bd16ff
33 changed files with 1793 additions and 3127 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/buildIndex.ts
+27 -4
View File
@@ -16,9 +16,13 @@ const lakes = lakesConfig.map(lake => {
let currentFlow = 0;
let sparkline: number[] = Array(12).fill(0);
let capacity = 0;
let volume = 0;
let inflow = 0;
if (fs.existsSync(DATA_FILE)) {
try {
const data: DataRecord[] = JSON.parse(fs.readFileSync(DATA_FILE, 'utf-8'));
const data: any[] = JSON.parse(fs.readFileSync(DATA_FILE, 'utf-8'));
if (data.length > 0) {
// Find latest valid record or just the last record
const lastValidLevelData = [...data].reverse().find(d => d.level !== null && !isNaN(d.level));
@@ -35,11 +39,29 @@ const lakes = lakesConfig.map(lake => {
while (sparkline.length < 12) {
sparkline.unshift(0);
}
const latest = data[data.length - 1];
if (latest.volume && latest.volume > 0) {
volume = latest.volume;
}
if (latest.inflow !== undefined) {
inflow = latest.inflow;
}
}
} catch (e) {
console.error(`Error reading data for ${internalId}`, e);
}
}
if (lake.minLevel && lake.maxLevel && currentLevel > 0) {
const percentage = ((currentLevel - lake.minLevel) / (lake.maxLevel - lake.minLevel)) * 100;
capacity = Math.max(0, Math.min(100, Math.round(percentage * 10) / 10)); // Round to 1 decimal place
if (volume === 0) {
volume = Number(((capacity / 100) * (lake.maxVolume || 0)).toFixed(1));
}
} else {
if (volume === 0) volume = lake.maxVolume || 0;
}
return {
id: lake.id,
@@ -47,10 +69,11 @@ const lakes = lakesConfig.map(lake => {
river: lake.text.includes('-') ? lake.text.split('-')[1].trim() : '',
priority: lake.priority || false,
level: currentLevel.toFixed(2),
capacity: 0, // Removed fake capacity
inflow: currentFlow.toFixed(1),
capacity: capacity,
inflow: inflow.toFixed(1),
outflow: currentFlow.toFixed(1),
volume: lake.maxVolume || 0, // Using real maxVolume if known
volume: volume,
maxVolume: lake.maxVolume || 0,
lat: lake.coords[0],
lng: lake.coords[1],
sparkline
scripts/lakesConfig.ts
+12 -22
View File
@@ -6,26 +6,16 @@ export interface LakeConfig {
}
export const lakesConfig: LakeConfig[] = [
{ id: "VLL1|1", text: "VD Lipno 1 - Vltava", priority: true, coords: [48.6322, 14.2215], maxVolume: 306.0 },
{ id: "VLOR|2", text: "VD Orlík - Vltava", priority: true, coords: [49.6060, 14.1700], maxVolume: 716.5 },
{ id: "VLSL|2", text: "VD Slapy - Vltava", priority: false, coords: [49.8220, 14.4360], maxVolume: 269.3 },
{ id: "BLHU|1", text: "VD Husinec - Blanice (PI)", coords: [49.0520, 13.9830], maxVolume: 6.9 },
{ id: "BIBI|1", text: "VD Bílsko - Bílský potok", coords: [49.1910, 14.0530] },
{ id: "KLDP|3", text: "VD Dolejší Padrťský rybník", coords: [49.6640, 13.7660] },
{ id: "KLHP|3", text: "VD Hořejší Padrťský rybník", coords: [49.6540, 13.7840] },
{ id: "KLKL|3", text: "VD Klabava - Klabava", coords: [49.7560, 13.5650] },
{ id: "KCKC|3", "text": "VD Klíčava - Klíčava", coords: [50.0650, 13.9290], maxVolume: 8.3 },
{ id: "LILA|3", text: "VD Láz - Litavka", coords: [49.6670, 13.8820] },
{ id: "MARI|1", text: "VD Římov - Malše", coords: [48.8470, 14.4870], maxVolume: 33.8 },
{ id: "MZHR|3", text: "VD Hracholusky - Mže", coords: [49.7890, 13.1550], maxVolume: 56.7 },
{ id: "MZLU|3", text: "VD Lučina - Mže", coords: [49.8000, 12.6100] },
{ id: "MZSS|3", text: "VD Plzeň-Štruncovy sady", coords: [49.7510, 13.3850] },
{ id: "OPOB|3", "text": "VD Obecnice - Obecnický potok", coords: [49.7210, 13.9450] },
{ id: "PPPI|3", "text": "VD Pilská - Pilský potok", coords: [49.6760, 13.8960] },
{ id: "RACU|3", "text": "VD České Údolí - Radbuza", coords: [49.7110, 13.3610] },
{ id: "SPNE|2", "text": "VD Němčice - Sedlický potok", coords: [49.6050, 15.2280] },
{ id: "SVKR|2", "text": "VD Švihov - Želivka", coords: [49.7180, 15.1060], maxVolume: 266.6 },
{ id: "UHKA|2", "text": "VD Kamýk - Vltava", coords: [49.6360, 14.2540], maxVolume: 12.8 },
{ id: "VRSN|2", "text": "VD Vrané - Vltava", coords: [49.9340, 14.3850], maxVolume: 11.1 },
{ id: "ZLUT|3", "text": "VD Žlutice - Střela", coords: [50.0930, 13.1590] }
{ id: "VLL1|1", text: "VD Lipno 1 - Vltava", priority: true, coords: [48.6322, 14.2215], maxVolume: 306.0, minLevel: 715.00, maxLevel: 725.60 },
{ id: "VLL2|1", text: "VD Lipno II - Vltava", coords: [48.6250, 14.3180], maxVolume: 1.5, minLevel: 510.0, maxLevel: 511.5 },
{ id: "VLHN|1", text: "VD Hněvkovice - Vltava", priority: true, coords: [49.1830, 14.4440], maxVolume: 21.1, minLevel: 365.0, maxLevel: 370.5 },
{ id: "VLKO|1", text: "VD Kořensko - Vltava", coords: [49.2550, 14.3980], maxVolume: 2.8, minLevel: 352.0, maxLevel: 353.5 },
{ id: "VLOR|2", text: "VD Orlík - Vltava", priority: true, coords: [49.6060, 14.1700], maxVolume: 716.5, minLevel: 330.00, maxLevel: 354.00 },
{ id: "UHKA|2", text: "VD Kamýk - Vltava", coords: [49.6360, 14.2540], maxVolume: 12.8, minLevel: 283.0, maxLevel: 285.6 },
{ id: "VLSL|2", text: "VD Slapy - Vltava", priority: true, coords: [49.8220, 14.4360], maxVolume: 269.3, minLevel: 265.50, maxLevel: 271.10 },
{ id: "VLST|2", text: "VD Štěchovice - Vltava", coords: [49.8450, 14.4120], maxVolume: 11.2, minLevel: 217.0, maxLevel: 219.5 },
{ id: "VRSN|2", text: "VD Vrané - Vltava", coords: [49.9340, 14.3850], maxVolume: 11.1, minLevel: 198.5, maxLevel: 200.5 },
{ id: "SVKR|2", text: "VD Švihov - Želivka", priority: true, coords: [49.7180, 15.1060], maxVolume: 266.6, minLevel: 343.0, maxLevel: 377.0 },
{ id: "MARI|1", text: "VD Římov - Malše", priority: true, coords: [48.8470, 14.4870], maxVolume: 33.8, minLevel: 458.0, maxLevel: 471.0 },
{ id: "MZHR|3", text: "VD Hracholusky - Mže", priority: true, coords: [49.7890, 13.1550], maxVolume: 56.7, minLevel: 360.0, maxLevel: 373.0 }
];
scripts/scrapeLakes.ts
+70 -28
View File
@@ -12,13 +12,21 @@ interface DataRecord {
}
// Parse date from DD.MM.YYYY HH:MM to ISO
function parseDateString(dateStr: string): string {
const [datePart, timePart] = dateStr.trim().split(' ');
const [day, month, year] = datePart.split('.');
const [hours, minutes] = timePart.split(':');
const d = new Date(parseInt(year), parseInt(month) - 1, parseInt(day), parseInt(hours), parseInt(minutes));
return d.toISOString();
function parseDateString(dateStr: string): string | null {
try {
if (!dateStr || !dateStr.includes(' ')) return null;
const [datePart, timePart] = dateStr.trim().split(' ');
const [day, month, year] = datePart.split('.');
const [hours, minutes] = timePart.split(':');
if (!year || !hours) return null;
const d = new Date(parseInt(year), parseInt(month) - 1, parseInt(day), parseInt(hours), parseInt(minutes));
if (isNaN(d.getTime())) return null;
return d.toISOString();
} catch (e) {
return null;
}
}
async function scrapeLake(lakeId: string, oid: string, internalId: string) {
@@ -34,29 +42,63 @@ async function scrapeLake(lakeId: string, oid: string, internalId: string) {
}
});
const html = response.data;
const $ = cheerio.load(html);
const rows = $('table tr');
const newData: DataRecord[] = [];
const $ = cheerio.load(response.data);
let currentInflow = 0;
let currentVolume = 0;
rows.each((i, row) => {
const tds = $(row).find('td');
if (tds.length >= 3) {
const datetimeText = $(tds[0]).text().trim();
if (/^\d{2}\.\d{2}\.\d{4}\s\d{2}:\d{2}$/.test(datetimeText)) {
const timestamp = parseDateString(datetimeText);
const levelText = $(tds[1]).text().trim().replace(',', '.');
const flowText = $(tds[2]).text().trim().replace(',', '.');
newData.push({
timestamp,
level: parseFloat(levelText),
flow: parseFloat(flowText)
});
}
$('table').each((i, tbl) => {
const text = $(tbl).text();
if (text.includes('Aktuální hodnoty') && text.includes('Přítok')) {
$(tbl).find('tr').each((j, r) => {
const label = $(r).find('td').eq(0).text().trim();
const valStr = $(r).find('td').eq(1).text().trim().replace(/\s/g, '').replace(',', '.');
if (label.includes('Přítok')) currentInflow = parseFloat(valStr) || 0;
if (label.includes('Objem')) currentVolume = parseFloat(valStr) || 0;
});
}
});
const records: DataRecord[] = [];
let dataTable = null;
$('table').each((i, tbl) => {
if ($(tbl).text().includes('Datum') && $(tbl).text().includes('Odtok')) {
dataTable = $(tbl);
}
});
if (dataTable) {
dataTable.find('tr').each((i, row) => {
if (i === 0) return; // skip header
const cols = $(row).find('td');
if (cols.length >= 3) {
const rawDate = $(cols[0]).text().trim();
const levelStr = $(cols[1]).text().trim().replace(',', '.');
let flowStr = $(cols[2]).text().trim().replace(',', '.');
if (flowStr === '' && cols.length >= 4) {
flowStr = $(cols[3]).text().trim().replace(',', '.');
}
const parsedDateStr = parseDateString(rawDate);
if (parsedDateStr) {
records.push({
timestamp: parsedDateStr,
level: parseFloat(levelStr) || 0,
flow: parseFloat(flowStr) || 0,
inflow: 0,
volume: 0
});
}
}
});
}
if (records.length > 0) {
// Apply current values to the latest record
records[0].inflow = currentInflow;
records[0].volume = currentVolume;
}
let existingData: DataRecord[] = [];
if (fs.existsSync(DATA_FILE)) {
const fileContent = fs.readFileSync(DATA_FILE, 'utf-8');
@@ -65,7 +107,7 @@ async function scrapeLake(lakeId: string, oid: string, internalId: string) {
const dataMap = new Map<string, DataRecord>();
existingData.forEach(item => dataMap.set(item.timestamp, item));
newData.forEach(item => dataMap.set(item.timestamp, item));
records.forEach(item => dataMap.set(item.timestamp, item));
const mergedData = Array.from(dataMap.values()).sort((a, b) => {
return new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime();
@@ -74,7 +116,7 @@ async function scrapeLake(lakeId: string, oid: string, internalId: string) {
fs.mkdirSync(path.dirname(DATA_FILE), { recursive: true });
fs.writeFileSync(DATA_FILE, JSON.stringify(mergedData, null, 2), 'utf-8');
console.log(`[${internalId}] Scraped ${newData.length} records. DB total: ${mergedData.length}`);
console.log(`[${internalId}] Scraped ${records.length} records. DB total: ${mergedData.length}`);
} catch (error: any) {
console.error(`[${internalId}] Error scraping data:`, error.message);