diff --git a/generate.mjs b/generate.mjs index 4b0cadf..e359a92 100644 --- a/generate.mjs +++ b/generate.mjs @@ -1,6 +1,6 @@ -import { time } from 'console'; import fs from 'fs'; import client from 'https'; +import zlib from 'zlib'; function getTimeString(count) { let date = new Date(Date.now() - count * 60000 * 5); @@ -20,21 +20,39 @@ function downloadImage(time_string, filepath) { port: 443, path: `/docs/default-source/rain-area/dpsri_70km_${time_string}0000dBR.dpsri.png`, headers: { - 'Host': 'www.nea.gov.sg', - 'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:129.0) Gecko/20100101 Firefox/129.0', - 'Accept': 'image/avif,image/webp,image/png,image/svg+xml,image/*;q=0.8,*/*;q=0.5', - 'Accept-Language': 'en-US,en;q=0.5', + "Host": "www.nea.gov.sg", + "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:129.0) Gecko/20100101 Firefox/129.0", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/png,image/svg+xml,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.5", + "Accept-Encoding": "gzip, deflate, br, zstd", + "DNT": "1", + "Connection": "keep-alive", + "Upgrade-Insecure-Requests": "1", + "Sec-Fetch-Dest": "document", + "Sec-Fetch-Mode": "navigate", + "Sec-Fetch-Site": "cross-site", + "Priority": "u=0, i", + "Pragma": "no-cache", + "Cache-Control": "no-cache", } } client.get(options, (res) => { console.log(`Downloading ${time_string}`); - if (res.statusCode >= 200 && res.statusCode < 400) { - res.pipe(fs.createWriteStream(filepath)) - .on('error', reject) - .once('close', () => resolve(filepath)); + if (res.statusCode >= 200 && res.statusCode < 400 && res.headers['content-type'] === 'image/png') { + let stream = fs.createWriteStream(filepath); + if (res.headers['content-encoding'] === 'gzip') { + res.pipe(zlib.createGunzip()).pipe(stream) + .on('error', reject) + .once('close', () => resolve(filepath)); + } + else { + res.pipe(stream) + .on('error', reject) + .once('close', () => resolve(filepath)); + } } else { // Consume response data to free up memory - console.log(`Failed to download ${time_string} with status code ${res.statusCode}`); + console.log(`Failed to download /docs/default-source/rain-area/dpsri_70km_${time_string}0000dBR.dpsri.png with status code ${res.statusCode}`); res.resume(); reject(new Error(res.statusCode)); } @@ -54,7 +72,7 @@ export async function initImages(count) { await downloadImage(time_string, filename) } catch (error) { //Response 302, we've reached the latest available image - if(error.message === '302') { + if (error.message === '302') { console.log('Could not download image after ' + time_string); return; } @@ -64,17 +82,20 @@ export async function initImages(count) { export async function updateImages(count) { let time_strings = []; - for(let i = count; i >= 0 ; i--) { + for (let i = count; i >= 0; i--) { time_strings.push(getTimeString(i)); } let files = fs.readdirSync('images/'); let image_count = files.length; + let updated = false; + //Try to download new images - for(let time_string of time_strings) { + for (let time_string of time_strings) { if (!files.includes(`${time_string}.png`)) { try { await downloadImage(time_string, `images/${time_string}.png`); + updated = true; image_count++; } catch (error) { console.log("No more images available"); @@ -84,8 +105,10 @@ export async function updateImages(count) { } //Remove old files - for(let file of files.sort().slice(0, image_count-count)) { + for (let file of files.sort().slice(0, image_count - count)) { console.log(`Deleting ${file}`); fs.unlinkSync(`images/${file}`); } + + return updated; } \ No newline at end of file