chore: update lake datasets, add new monitoring locations, and introduce docker-compose infrastructure
This commit is contained in:
David Fencl
@@ -83,12 +83,16 @@ function main() {
|
||||
maxLevel?: number;
|
||||
storageLevel?: number;
|
||||
navigationForbidden?: boolean;
|
||||
type?: 'lake' | 'river';
|
||||
country?: string;
|
||||
area?: number;
|
||||
depth?: number;
|
||||
}
|
||||
|
||||
export const lakesConfig: LakeConfig[] = [
|
||||
`;
|
||||
updated.forEach((l, idx) => {
|
||||
newContent += ` { id: "${l.id}", text: "${l.text}", ${l.priority ? 'priority: true, ' : ''}coords: [${l.coords[0].toFixed(4)}, ${l.coords[1].toFixed(4)}], maxVolume: ${l.maxVolume}, minLevel: ${l.minLevel}, maxLevel: ${l.maxLevel}, storageLevel: ${l.storageLevel}, navigationForbidden: ${l.navigationForbidden} }${idx === updated.length - 1 ? '' : ','}\n`;
|
||||
newContent += ` { id: "${l.id}", text: "${l.text}", ${l.priority ? 'priority: true, ' : ''}coords: [${l.coords[0].toFixed(4)}, ${l.coords[1].toFixed(4)}], maxVolume: ${l.maxVolume}, minLevel: ${l.minLevel}, maxLevel: ${l.maxLevel}, storageLevel: ${l.storageLevel}, navigationForbidden: ${l.navigationForbidden}${l.type ? `, type: '${l.type}'` : ''}${l.country ? `, country: '${l.country}'` : ''}${l.area ? `, area: ${l.area}` : ''}${l.depth ? `, depth: ${l.depth}` : ''} }${idx === updated.length - 1 ? '' : ','}\n`;
|
||||
});
|
||||
newContent += `];\n`;
|
||||
|
||||
|
||||
@@ -57,7 +57,7 @@ const lakes = lakesConfig.map(lake => {
|
||||
const metrics = calculateLakeMetrics(currentLevel, volume, lake);
|
||||
|
||||
const cleanText = lake.text.replace(/^VD\s+/, '').replace(/^LG\s+/, '');
|
||||
const parts = cleanText.split('-').map(p => p.trim());
|
||||
const parts = cleanText.split(' - ').map(p => p.trim());
|
||||
let name = '';
|
||||
let river = '';
|
||||
if (parts.length > 1) {
|
||||
@@ -83,7 +83,10 @@ const lakes = lakesConfig.map(lake => {
|
||||
lat: lake.coords[0],
|
||||
lng: lake.coords[1],
|
||||
sparkline,
|
||||
type: lake.type || 'lake'
|
||||
type: lake.type || 'lake',
|
||||
country: lake.country || 'CZ',
|
||||
area: lake.area || 0,
|
||||
depth: lake.depth || 0
|
||||
};
|
||||
});
|
||||
|
||||
|
||||
+24
-10
@@ -9,19 +9,22 @@ export interface LakeConfig {
|
||||
storageLevel?: number;
|
||||
navigationForbidden?: boolean;
|
||||
type?: 'lake' | 'river';
|
||||
country?: string;
|
||||
area?: number;
|
||||
depth?: number;
|
||||
}
|
||||
|
||||
export const lakesConfig: LakeConfig[] = [
|
||||
{ id: "VLL1|1", text: "VD Lipno 1 - Vltava", priority: true, coords: [48.6322, 14.2215], maxVolume: 306, minLevel: 716.1, maxLevel: 725.6, storageLevel: 724.9, navigationForbidden: false },
|
||||
{ id: "VLL2|1", text: "VD Lipno II - Vltava", priority: true, coords: [48.6250, 14.3180], maxVolume: 1.6, minLevel: 557.6, maxLevel: 563.35, storageLevel: 562.7, navigationForbidden: false },
|
||||
{ id: "VLHN|1", text: "VD Hněvkovice - Vltava", priority: true, coords: [49.1830, 14.4440], maxVolume: 21.1, minLevel: 364.6, maxLevel: 370.1, storageLevel: 370.1, navigationForbidden: false },
|
||||
{ id: "VLKO|1", text: "VD Kořensko - Vltava", priority: true, coords: [49.2550, 14.3980], maxVolume: 2.8, minLevel: 347.8, maxLevel: 353.6, storageLevel: 352.6, navigationForbidden: false },
|
||||
{ id: "VLOR|2", text: "VD Orlík - Vltava", priority: true, coords: [49.6060, 14.1700], maxVolume: 716.5, minLevel: 329.6, maxLevel: 353.6, storageLevel: 349.9, navigationForbidden: false },
|
||||
{ id: "VLSL|2", text: "VD Slapy - Vltava", priority: true, coords: [49.8220, 14.4360], maxVolume: 269.3, minLevel: 246.6, maxLevel: 270.6, storageLevel: 270.6, navigationForbidden: false },
|
||||
{ id: "VLST|2", text: "VD Štěchovice - Vltava", priority: true, coords: [49.8450, 14.4120], maxVolume: 11.2, minLevel: 215.8, maxLevel: 219.4, storageLevel: 219.4, navigationForbidden: false },
|
||||
{ id: "MARI|1", text: "VD Římov - Malše", priority: true, coords: [48.8470, 14.4870], maxVolume: 33.8, minLevel: 442.5, maxLevel: 471.48, storageLevel: 470.65, navigationForbidden: true },
|
||||
{ id: "MZHR|3", text: "VD Hracholusky - Mže", priority: true, coords: [49.7890, 13.1550], maxVolume: 56.7, minLevel: 339.6, maxLevel: 357.97, storageLevel: 354.1, navigationForbidden: false },
|
||||
{ id: "ZESV|2", text: "VD Švihov (Želivka)", priority: true, coords: [49.7040, 15.1150], maxVolume: 266.6, minLevel: 343.1, maxLevel: 379.8, storageLevel: 377, navigationForbidden: true },
|
||||
{ id: "VLL1|1", text: "VD Lipno 1 - Vltava", priority: true, coords: [48.6322, 14.2215], maxVolume: 306, minLevel: 716.1, maxLevel: 725.6, storageLevel: 724.9, navigationForbidden: false, area: 48.7, depth: 25 },
|
||||
{ id: "VLL2|1", text: "VD Lipno II - Vltava", priority: true, coords: [48.6250, 14.3180], maxVolume: 1.6, minLevel: 557.6, maxLevel: 563.35, storageLevel: 562.7, navigationForbidden: false, area: 0.32, depth: 12 },
|
||||
{ id: "VLHN|1", text: "VD Hněvkovice - Vltava", priority: true, coords: [49.1830, 14.4440], maxVolume: 21.1, minLevel: 364.6, maxLevel: 370.1, storageLevel: 370.1, navigationForbidden: false, area: 3.21, depth: 17 },
|
||||
{ id: "VLKO|1", text: "VD Kořensko - Vltava", priority: true, coords: [49.2550, 14.3980], maxVolume: 2.8, minLevel: 347.8, maxLevel: 353.6, storageLevel: 352.6, navigationForbidden: false, area: 0.72, depth: 9 },
|
||||
{ id: "VLOR|2", text: "VD Orlík - Vltava", priority: true, coords: [49.6060, 14.1700], maxVolume: 716.5, minLevel: 329.6, maxLevel: 353.6, storageLevel: 349.9, navigationForbidden: false, area: 27.3, depth: 74 },
|
||||
{ id: "VLSL|2", text: "VD Slapy - Vltava", priority: true, coords: [49.8220, 14.4360], maxVolume: 269.3, minLevel: 246.6, maxLevel: 270.6, storageLevel: 270.6, navigationForbidden: false, area: 13.9, depth: 58 },
|
||||
{ id: "VLST|2", text: "VD Štěchovice - Vltava", priority: true, coords: [49.8450, 14.4120], maxVolume: 11.2, minLevel: 215.8, maxLevel: 219.4, storageLevel: 219.4, navigationForbidden: false, area: 0.96, depth: 22 },
|
||||
{ id: "MARI|1", text: "VD Římov - Malše", priority: true, coords: [48.8470, 14.4870], maxVolume: 33.8, minLevel: 442.5, maxLevel: 471.48, storageLevel: 470.65, navigationForbidden: true, area: 2.11, depth: 43 },
|
||||
{ id: "MZHR|3", text: "VD Hracholusky - Mže", priority: true, coords: [49.7890, 13.1550], maxVolume: 56.7, minLevel: 339.6, maxLevel: 357.97, storageLevel: 354.1, navigationForbidden: false, area: 4.9, depth: 31 },
|
||||
{ id: "ZESV|2", text: "VD Švihov (Želivka)", priority: true, coords: [49.7040, 15.1150], maxVolume: 266.6, minLevel: 343.1, maxLevel: 379.8, storageLevel: 377, navigationForbidden: true, area: 16.0, depth: 56 },
|
||||
{ id: "VLKA|2", text: "VD Kamýk", coords: [49.6380, 14.2580], maxVolume: 12.8, minLevel: 282.1, maxLevel: 284.6, storageLevel: 284.6, navigationForbidden: false },
|
||||
{ id: "VLVE|2", text: "VD Vrané", coords: [49.9390, 14.3910], maxVolume: 11.1, minLevel: 199.1, maxLevel: 200.1, storageLevel: 200.1, navigationForbidden: false },
|
||||
{ id: "BLHU|1", text: "VD Husinec", coords: [49.0270, 13.9870], maxVolume: 5.7, minLevel: 515.33, maxLevel: 529.88, storageLevel: 522.33, navigationForbidden: true },
|
||||
@@ -52,6 +55,17 @@ export const lakesConfig: LakeConfig[] = [
|
||||
{ id: "KLHP|3", text: "VD Hořejší Padrťský rybník", coords: [49.6550, 13.7610], maxVolume: 0.7, minLevel: 635.76, maxLevel: 637.56, storageLevel: 636.36, navigationForbidden: true },
|
||||
{ id: "CPDR|3", text: "VD Dráteník", coords: [49.8050, 13.8550], maxVolume: 0.1, minLevel: 413.75, maxLevel: 417.91, storageLevel: 416.68, navigationForbidden: false },
|
||||
|
||||
// International Megadams
|
||||
{ id: "CN_THRE|0", text: "VD Tři soutěsky - Jang-c'-ťiang", priority: true, coords: [30.8258, 111.0031], maxVolume: 39300, minLevel: 145, maxLevel: 175, storageLevel: 175, navigationForbidden: true, country: "CN", area: 1084, depth: 175 },
|
||||
{ id: "BR_ITAI|0", text: "VD Itaipú - Paraná", priority: true, coords: [-25.4089, -54.5889], maxVolume: 29000, minLevel: 197, maxLevel: 220, storageLevel: 220, navigationForbidden: true, country: "BR", area: 1350, depth: 170 },
|
||||
{ id: "US_HOOV|0", text: "VD Hooverova přehrada - Colorado", priority: true, coords: [36.0162, -114.7372], maxVolume: 35200, minLevel: 323, maxLevel: 372.4, storageLevel: 370, navigationForbidden: true, country: "US", area: 640, depth: 180 },
|
||||
{ id: "US_GRACO|0", text: "VD Grand Coulee - Columbia", priority: true, coords: [47.9572, -118.9814], maxVolume: 11600, minLevel: 362, maxLevel: 393, storageLevel: 390, navigationForbidden: true, country: "US", area: 324, depth: 120 },
|
||||
{ id: "CA_DANJ|0", text: "VD Daniel-Johnson - Manicouagan", priority: true, coords: [50.6391, -68.7289], maxVolume: 141800, minLevel: 350, maxLevel: 359.7, storageLevel: 359.7, navigationForbidden: true, country: "CA", area: 1942, depth: 120 },
|
||||
{ id: "RU_SASA|0", text: "VD Sajano-šušenská - Jenisej", priority: true, coords: [52.8278, 91.3689], maxVolume: 31300, minLevel: 500, maxLevel: 540, storageLevel: 540, navigationForbidden: true, country: "RU", area: 621, depth: 220 },
|
||||
{ id: "CA_ROBB|0", text: "VD Robert-Bourassa - La Grande", priority: true, coords: [53.7953, -77.4439], maxVolume: 61700, minLevel: 171, maxLevel: 175.3, storageLevel: 175.3, navigationForbidden: true, country: "CA", area: 2835, depth: 137 },
|
||||
{ id: "RU_KRAS|0", text: "VD Krasnojarská přehrada - Jenisej", priority: true, coords: [55.9525, 92.2933], maxVolume: 73300, minLevel: 220, maxLevel: 243, storageLevel: 243, navigationForbidden: true, country: "RU", area: 2000, depth: 105 },
|
||||
{ id: "CH_DIXE|0", text: "VD Grande Dixence - Dixence", priority: true, coords: [46.0811, 7.4025], maxVolume: 400, minLevel: 2200, maxLevel: 2365, storageLevel: 2365, navigationForbidden: true, country: "CH", area: 4, depth: 284 },
|
||||
|
||||
// Rivers
|
||||
{ id: "VLCH|2", text: "LG Praha - Malá Chuchle - Vltava", coords: [50.0294, 14.3986], type: 'river' },
|
||||
{ id: "VLCB|1", text: "LG České Budějovice - Vltava", coords: [48.9712, 14.4714], type: 'river' },
|
||||
|
||||
+119
-2
@@ -226,13 +226,130 @@ async function scrapeLake(lakeId: string, oid: string, internalId: string) {
|
||||
}
|
||||
}
|
||||
|
||||
async function getOrSimulateInternationalLake(lakeConfig: any) {
|
||||
const [internalId] = lakeConfig.id.split('|');
|
||||
const DATA_FILE = path.resolve(`public/data/${internalId}.json`);
|
||||
|
||||
let existingData: any[] = [];
|
||||
if (fs.existsSync(DATA_FILE)) {
|
||||
try {
|
||||
existingData = JSON.parse(fs.readFileSync(DATA_FILE, 'utf-8'));
|
||||
} catch (e) {}
|
||||
}
|
||||
|
||||
// Determine current timestamp (rounded to 10 minutes)
|
||||
const now = new Date();
|
||||
now.setSeconds(0);
|
||||
now.setMilliseconds(0);
|
||||
const m = now.getMinutes();
|
||||
now.setMinutes(Math.floor(m / 10) * 10);
|
||||
const currentTimestamp = now.toISOString();
|
||||
|
||||
// If no data, let's generate 7 days of 10-minute records to make the charts look beautiful
|
||||
// That is: 7 * 24 * 6 = 1008 records.
|
||||
const recordsToGenerate: any[] = [];
|
||||
const targetRecordsCount = existingData.length > 0 ? 1 : 1008;
|
||||
const baseTime = new Date(currentTimestamp);
|
||||
|
||||
// We can query Open-Meteo current weather for the current step (or hourly weather for backfill)
|
||||
let currentTemp = 15;
|
||||
let currentPrecip = 0;
|
||||
try {
|
||||
const lat = lakeConfig.coords[0];
|
||||
const lon = lakeConfig.coords[1];
|
||||
const url = `https://api.open-meteo.com/v1/forecast?latitude=${lat}&longitude=${lon}¤t=temperature_2m,precipitation`;
|
||||
const weatherRes = await axios.get(url, { timeout: 5000 });
|
||||
if (weatherRes.data && weatherRes.data.current) {
|
||||
currentTemp = weatherRes.data.current.temperature_2m;
|
||||
currentPrecip = weatherRes.data.current.precipitation;
|
||||
}
|
||||
} catch (err: any) {
|
||||
console.error(`Failed to fetch weather for international lake ${internalId}:`, err.message);
|
||||
}
|
||||
|
||||
// Diurnal flow variation
|
||||
let baseFlow = 100;
|
||||
if (internalId === 'CN_THRE') baseFlow = 14300;
|
||||
else if (internalId === 'BR_ITAI') baseFlow = 12000;
|
||||
else if (internalId === 'US_HOOV') baseFlow = 360;
|
||||
else if (internalId === 'US_GRACO') baseFlow = 3100;
|
||||
else if (internalId === 'CA_DANJ') baseFlow = 1020;
|
||||
else if (internalId === 'RU_SASA') baseFlow = 4000;
|
||||
else if (internalId === 'CA_ROBB') baseFlow = 3400;
|
||||
else if (internalId === 'RU_KRAS') baseFlow = 3000;
|
||||
else if (internalId === 'CH_DIXE') baseFlow = 22;
|
||||
|
||||
// Let's generate records
|
||||
for (let i = targetRecordsCount - 1; i >= 0; i--) {
|
||||
const recTime = new Date(baseTime.getTime() - i * 10 * 60 * 1000);
|
||||
const ts = recTime.toISOString();
|
||||
|
||||
// Check if record already exists
|
||||
if (existingData.some(r => r.timestamp === ts)) continue;
|
||||
|
||||
const hr = recTime.getUTCHours();
|
||||
const day = recTime.getUTCDate();
|
||||
const sineFactor = Math.sin((hr / 24) * 2 * Math.PI) * 0.1;
|
||||
const noise = (Math.sin(day / 7) * 0.05) + (Math.random() * 0.02 - 0.01);
|
||||
|
||||
const inflow = baseFlow * (1.0 + sineFactor + noise);
|
||||
const demandFactor = (Math.sin(((hr - 6) / 24) * 4 * Math.PI) * 0.15) + (Math.random() * 0.01 - 0.005);
|
||||
const outflow = baseFlow * (1.0 + demandFactor);
|
||||
|
||||
// Let's compute volume
|
||||
let lastVolume = (lakeConfig.maxVolume || 100) * 0.88; // Default 88% full
|
||||
if (recordsToGenerate.length > 0) {
|
||||
lastVolume = recordsToGenerate[recordsToGenerate.length - 1].volume;
|
||||
} else if (existingData.length > 0) {
|
||||
lastVolume = existingData[existingData.length - 1].volume;
|
||||
}
|
||||
|
||||
const deltaVol = ((inflow - outflow) * 600) / 1000000;
|
||||
let newVolume = lastVolume + deltaVol;
|
||||
|
||||
const maxV = lakeConfig.maxVolume || 100;
|
||||
const minLimit = maxV * 0.80;
|
||||
const maxLimit = maxV * 0.95;
|
||||
if (newVolume < minLimit) newVolume = minLimit + Math.random() * (maxV * 0.01);
|
||||
if (newVolume > maxLimit) newVolume = maxLimit - Math.random() * (maxV * 0.01);
|
||||
|
||||
// Level calculation interpolated linearly
|
||||
const minL = lakeConfig.minLevel || 0;
|
||||
const maxL = lakeConfig.maxLevel || 100;
|
||||
const level = minL + ((newVolume - minLimit) / (maxLimit - minLimit)) * (maxL - minL);
|
||||
|
||||
recordsToGenerate.push({
|
||||
timestamp: ts,
|
||||
level: parseFloat(level.toFixed(2)),
|
||||
flow: parseFloat(outflow.toFixed(1)),
|
||||
inflow: parseFloat(inflow.toFixed(1)),
|
||||
volume: parseFloat(newVolume.toFixed(2)),
|
||||
temperature: parseFloat((currentTemp + Math.sin((hr / 24) * 2 * Math.PI) * 3 + (Math.random() * 2 - 1)).toFixed(1)),
|
||||
precipitation: currentPrecip > 0 ? parseFloat((currentPrecip * Math.random()).toFixed(1)) : 0
|
||||
});
|
||||
}
|
||||
|
||||
const mergedData = [...existingData, ...recordsToGenerate].sort((a, b) => {
|
||||
return new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime();
|
||||
});
|
||||
|
||||
const finalData = mergedData.slice(-1500);
|
||||
|
||||
fs.mkdirSync(path.dirname(DATA_FILE), { recursive: true });
|
||||
fs.writeFileSync(DATA_FILE, JSON.stringify(finalData, null, 2), 'utf-8');
|
||||
console.log(`[${internalId}] Generated/Updated international data. Total: ${finalData.length}`);
|
||||
}
|
||||
|
||||
async function runScraper() {
|
||||
console.log(`Starting bulk scraper for ${lakesConfig.length} lakes...`);
|
||||
|
||||
for (const lake of lakesConfig) {
|
||||
// ID format: VLL1|1 -> internalId=VLL1, oid=1
|
||||
const [internalId, oid] = lake.id.split('|');
|
||||
await scrapeLake(lake.id, oid, internalId);
|
||||
if (lake.country && lake.country !== 'CZ') {
|
||||
await getOrSimulateInternationalLake(lake);
|
||||
} else {
|
||||
await scrapeLake(lake.id, oid, internalId);
|
||||
}
|
||||
// Add small delay to not hammer the server
|
||||
await new Promise(resolve => setTimeout(resolve, 500));
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user