feat: implement Open-Meteo weather integration with backfill scripts and updated lake data models.
continuous-integration/drone/push Build encountered an error
continuous-integration/drone/push Build encountered an error
This commit is contained in:
@@ -0,0 +1,76 @@
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import axios from 'axios';
|
||||
import { lakesConfig } from './lakesConfig';
|
||||
|
||||
const DATA_DIR = path.resolve(process.cwd(), 'public/data');
|
||||
|
||||
async function backfill() {
|
||||
console.log('Starting weather backfill for past 7 days...');
|
||||
|
||||
for (const lake of lakesConfig) {
|
||||
const internalId = lake.id.split('|')[0];
|
||||
const filePath = path.join(DATA_DIR, `${internalId}.json`);
|
||||
|
||||
if (!fs.existsSync(filePath)) {
|
||||
console.log(`Skipping ${internalId}, no data file.`);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!lake.coords) {
|
||||
console.log(`Skipping ${internalId}, no coordinates.`);
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const lat = lake.coords[0];
|
||||
const lon = lake.coords[1];
|
||||
const url = `https://api.open-meteo.com/v1/forecast?latitude=${lat}&longitude=${lon}&past_days=7&hourly=temperature_2m,precipitation&timezone=GMT`;
|
||||
|
||||
const res = await axios.get(url, { timeout: 10000 });
|
||||
const hourly = res.data.hourly;
|
||||
|
||||
// Build lookup map for O(1) matching: '2026-06-02T04:00' -> { temp, precip }
|
||||
const weatherMap = new Map();
|
||||
for (let i = 0; i < hourly.time.length; i++) {
|
||||
weatherMap.set(hourly.time[i], {
|
||||
temperature: hourly.temperature_2m[i],
|
||||
precipitation: hourly.precipitation[i]
|
||||
});
|
||||
}
|
||||
|
||||
const data = JSON.parse(fs.readFileSync(filePath, 'utf-8'));
|
||||
let updatedCount = 0;
|
||||
|
||||
for (const record of data) {
|
||||
// record.timestamp is like "2026-06-02T04:00:00.000Z"
|
||||
// Open-Meteo time is like "2026-06-02T04:00"
|
||||
const hourKey = record.timestamp.substring(0, 16); // Extract up to minutes
|
||||
|
||||
if (weatherMap.has(hourKey)) {
|
||||
const w = weatherMap.get(hourKey);
|
||||
if (w.temperature !== null && w.temperature !== undefined) {
|
||||
record.temperature = w.temperature;
|
||||
updatedCount++;
|
||||
}
|
||||
if (w.precipitation !== null && w.precipitation !== undefined) {
|
||||
record.precipitation = w.precipitation;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fs.writeFileSync(filePath, JSON.stringify(data, null, 2));
|
||||
console.log(`[${internalId}] Backfilled ${updatedCount} records with historical Open-Meteo data.`);
|
||||
|
||||
// small delay to prevent rate limit
|
||||
await new Promise(r => setTimeout(r, 200));
|
||||
|
||||
} catch (e: any) {
|
||||
console.error(`Error processing ${internalId}:`, e.message);
|
||||
}
|
||||
}
|
||||
|
||||
console.log('Backfill complete!');
|
||||
}
|
||||
|
||||
backfill();
|
||||
Reference in New Issue
Block a user