fencl/davisfe.cz
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: update lake index, sync scraping scripts, and prune unused data files

Browse Source
This commit is contained in:
David Fencl
2026-06-05 22:24:47 +02:00
parent 61a8af109c
commit 5411bd16ff
33 changed files with 1793 additions and 3127 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/scrapeLakes.ts
+70 -28
View File
@@ -12,13 +12,21 @@ interface DataRecord {
}
// Parse date from DD.MM.YYYY HH:MM to ISO
function parseDateString(dateStr: string): string {
const [datePart, timePart] = dateStr.trim().split(' ');
const [day, month, year] = datePart.split('.');
const [hours, minutes] = timePart.split(':');
const d = new Date(parseInt(year), parseInt(month) - 1, parseInt(day), parseInt(hours), parseInt(minutes));
return d.toISOString();
function parseDateString(dateStr: string): string | null {
try {
if (!dateStr || !dateStr.includes(' ')) return null;
const [datePart, timePart] = dateStr.trim().split(' ');
const [day, month, year] = datePart.split('.');
const [hours, minutes] = timePart.split(':');
if (!year || !hours) return null;
const d = new Date(parseInt(year), parseInt(month) - 1, parseInt(day), parseInt(hours), parseInt(minutes));
if (isNaN(d.getTime())) return null;
return d.toISOString();
} catch (e) {
return null;
}
}
async function scrapeLake(lakeId: string, oid: string, internalId: string) {
@@ -34,29 +42,63 @@ async function scrapeLake(lakeId: string, oid: string, internalId: string) {
}
});
const html = response.data;
const $ = cheerio.load(html);
const rows = $('table tr');
const newData: DataRecord[] = [];
const $ = cheerio.load(response.data);
let currentInflow = 0;
let currentVolume = 0;
rows.each((i, row) => {
const tds = $(row).find('td');
if (tds.length >= 3) {
const datetimeText = $(tds[0]).text().trim();
if (/^\d{2}\.\d{2}\.\d{4}\s\d{2}:\d{2}$/.test(datetimeText)) {
const timestamp = parseDateString(datetimeText);
const levelText = $(tds[1]).text().trim().replace(',', '.');
const flowText = $(tds[2]).text().trim().replace(',', '.');
newData.push({
timestamp,
level: parseFloat(levelText),
flow: parseFloat(flowText)
});
}
$('table').each((i, tbl) => {
const text = $(tbl).text();
if (text.includes('Aktuální hodnoty') && text.includes('Přítok')) {
$(tbl).find('tr').each((j, r) => {
const label = $(r).find('td').eq(0).text().trim();
const valStr = $(r).find('td').eq(1).text().trim().replace(/\s/g, '').replace(',', '.');
if (label.includes('Přítok')) currentInflow = parseFloat(valStr) || 0;
if (label.includes('Objem')) currentVolume = parseFloat(valStr) || 0;
});
}
});
const records: DataRecord[] = [];
let dataTable = null;
$('table').each((i, tbl) => {
if ($(tbl).text().includes('Datum') && $(tbl).text().includes('Odtok')) {
dataTable = $(tbl);
}
});
if (dataTable) {
dataTable.find('tr').each((i, row) => {
if (i === 0) return; // skip header
const cols = $(row).find('td');
if (cols.length >= 3) {
const rawDate = $(cols[0]).text().trim();
const levelStr = $(cols[1]).text().trim().replace(',', '.');
let flowStr = $(cols[2]).text().trim().replace(',', '.');
if (flowStr === '' && cols.length >= 4) {
flowStr = $(cols[3]).text().trim().replace(',', '.');
}
const parsedDateStr = parseDateString(rawDate);
if (parsedDateStr) {
records.push({
timestamp: parsedDateStr,
level: parseFloat(levelStr) || 0,
flow: parseFloat(flowStr) || 0,
inflow: 0,
volume: 0
});
}
}
});
}
if (records.length > 0) {
// Apply current values to the latest record
records[0].inflow = currentInflow;
records[0].volume = currentVolume;
}
let existingData: DataRecord[] = [];
if (fs.existsSync(DATA_FILE)) {
const fileContent = fs.readFileSync(DATA_FILE, 'utf-8');
@@ -65,7 +107,7 @@ async function scrapeLake(lakeId: string, oid: string, internalId: string) {
const dataMap = new Map<string, DataRecord>();
existingData.forEach(item => dataMap.set(item.timestamp, item));
newData.forEach(item => dataMap.set(item.timestamp, item));
records.forEach(item => dataMap.set(item.timestamp, item));
const mergedData = Array.from(dataMap.values()).sort((a, b) => {
return new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime();
@@ -74,7 +116,7 @@ async function scrapeLake(lakeId: string, oid: string, internalId: string) {
fs.mkdirSync(path.dirname(DATA_FILE), { recursive: true });
fs.writeFileSync(DATA_FILE, JSON.stringify(mergedData, null, 2), 'utf-8');
console.log(`[${internalId}] Scraped ${newData.length} records. DB total: ${mergedData.length}`);
console.log(`[${internalId}] Scraped ${records.length} records. DB total: ${mergedData.length}`);
} catch (error: any) {
console.error(`[${internalId}] Error scraping data:`, error.message);