singapore_rain_radar/downloader.mjs

import fs from 'fs';
import client from 'https';
import zlib from 'zlib';

function getTimeString(count) {
    let date = new Date(Date.now() - count * 60000 * 5);
    return (
        date.getFullYear().toString()
        + (date.getMonth() + 1).toString().padStart(2, "0")
        + date.getDate().toString().padStart(2, "0")
        + date.getHours().toString().padStart(2, "0")
        + (date.getMinutes() - date.getMinutes() % 5).toString().padStart(2, "0")
    );
}

function downloadImage(time_string, filepath) {
    return new Promise((resolve, reject) => {
        let options = {
            hostname: 'www.nea.gov.sg',
            port: 443,
            path: `/docs/default-source/rain-area/dpsri_70km_${time_string}0000dBR.dpsri.png`,
            headers: {
                "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:129.0) Gecko/20100101 Firefox/129." + Math.round(Math.random() * 1000),
                "Accept": "image/avif,image/webp,image/png,image/svg+xml,image/*;q=0.8,*/*;q=0.5;" + Math.round(Math.random() * 1000),
            }
        }
        client.get(options, (res) => {
            console.log(`Downloading ${time_string}`);
            if (res.statusCode >= 200 && res.statusCode < 400 && res.headers['content-type'] === 'image/png') {
                let stream = fs.createWriteStream(filepath);
                if (res.headers['content-encoding'] === 'gzip') {
                    res.pipe(zlib.createGunzip()).pipe(stream)
                        .on('error', reject)
                        .once('close', () => resolve(filepath));
                }
                else {
                    res.pipe(stream)
                        .on('error', reject)
                        .once('close', () => resolve(filepath));
                }
            } else {
                // Consume response data to free up memory
                console.log(`Failed to download /docs/default-source/rain-area/dpsri_70km_${time_string}0000dBR.dpsri.png with status code ${res.statusCode}`);
                res.resume();
                reject(new Error(res.statusCode));
            }
        });
    });
}

export async function initImages(count) {
    let files = fs.readdirSync('images/raw');
    let downloaded = 0;
    let i = 0;
    while(downloaded < count) {
        let time_string = getTimeString(i);
        let filename = `images/raw/${time_string}.png`;
        if (files.includes(`${time_string}.png`)) {
            downloaded++;
            i++;
            continue;
        }
        try {
            await downloadImage(time_string, filename);
            downloaded++;
        } catch (error) {
            //Response 302, we've reached the latest available image
            if (error.message === '302') {
                console.log('Could not download image ' + time_string);
            }
        }
        i++;
    }
}

export async function updateImages(count) {
    let time_strings = [];
    for (let i = count; i >= 0; i--) {
        time_strings.push(getTimeString(i));
    }
    let files = fs.readdirSync('images/raw');
    let image_count = files.length;

    let updated = false;

    //Try to download new images
    for (let time_string of time_strings) {
        if (!files.includes(`${time_string}.png`)) {
            try {
                await downloadImage(time_string, `images/raw/${time_string}.png`);
                updated = true;
                image_count++;
            } catch (error) {
                console.log("Skipped image " + time_string);
            }
        }
    }

    //Remove old files
    for (let file of files.sort().slice(0, image_count - count)) {
        console.log(`Deleting ${file}`);
        fs.unlinkSync(`images/raw/${file}`);
    }

    return updated;
}