public/singapore_rain_radar
1
0
Fork 0
mirror of https://git.roussel.pro/public-website/singapore_rain_radar.git synced 2026-02-09 02:20:17 +01:00
Code Issues Packages Projects Releases Wiki Activity

added gzip support

Browse Source
This commit is contained in:
Quentin Roussel
2024-08-18 02:19:14 +08:00
parent dd146e5490
commit c6fc11f276
1 changed files with 37 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
generate.mjs
View File

@@ -1,6 +1,6 @@
import { time } from 'console';
import fs from 'fs'; import fs from 'fs';
import client from 'https'; import client from 'https';
import zlib from 'zlib';
function getTimeString(count) { function getTimeString(count) {
let date = new Date(Date.now() - count * 60000 * 5); let date = new Date(Date.now() - count * 60000 * 5);
@@ -20,21 +20,39 @@ function downloadImage(time_string, filepath) {
port: 443, port: 443,
path: `/docs/default-source/rain-area/dpsri_70km_${time_string}0000dBR.dpsri.png`, path: `/docs/default-source/rain-area/dpsri_70km_${time_string}0000dBR.dpsri.png`,
headers: { headers: {
'Host': 'www.nea.gov.sg', "Host": "www.nea.gov.sg",
'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:129.0) Gecko/20100101 Firefox/129.0', "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:129.0) Gecko/20100101 Firefox/129.0",
'Accept': 'image/avif,image/webp,image/png,image/svg+xml,image/*;q=0.8,*/*;q=0.5', "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/png,image/svg+xml,*/*;q=0.8",
'Accept-Language': 'en-US,en;q=0.5', "Accept-Language": "en-US,en;q=0.5",
"Accept-Encoding": "gzip, deflate, br, zstd",
"DNT": "1",
"Connection": "keep-alive",
"Upgrade-Insecure-Requests": "1",
"Sec-Fetch-Dest": "document",
"Sec-Fetch-Mode": "navigate",
"Sec-Fetch-Site": "cross-site",
"Priority": "u=0, i",
"Pragma": "no-cache",
"Cache-Control": "no-cache",
} }
} }
client.get(options, (res) => { client.get(options, (res) => {
console.log(`Downloading ${time_string}`); console.log(`Downloading ${time_string}`);
if (res.statusCode >= 200 && res.statusCode < 400) { if (res.statusCode >= 200 && res.statusCode < 400 && res.headers['content-type'] === 'image/png') {
res.pipe(fs.createWriteStream(filepath)) let stream = fs.createWriteStream(filepath);
.on('error', reject) if (res.headers['content-encoding'] === 'gzip') {
.once('close', () => resolve(filepath)); res.pipe(zlib.createGunzip()).pipe(stream)
.on('error', reject)
.once('close', () => resolve(filepath));
}
else {
res.pipe(stream)
.on('error', reject)
.once('close', () => resolve(filepath));
}
} else { } else {
// Consume response data to free up memory // Consume response data to free up memory
console.log(`Failed to download ${time_string} with status code ${res.statusCode}`); console.log(`Failed to download /docs/default-source/rain-area/dpsri_70km_${time_string}0000dBR.dpsri.png with status code ${res.statusCode}`);
res.resume(); res.resume();
reject(new Error(res.statusCode)); reject(new Error(res.statusCode));
} }
@@ -54,7 +72,7 @@ export async function initImages(count) {
await downloadImage(time_string, filename) await downloadImage(time_string, filename)
} catch (error) { } catch (error) {
//Response 302, we've reached the latest available image //Response 302, we've reached the latest available image
if(error.message === '302') { if (error.message === '302') {
console.log('Could not download image after ' + time_string); console.log('Could not download image after ' + time_string);
return; return;
} }
@@ -64,17 +82,20 @@ export async function initImages(count) {
export async function updateImages(count) { export async function updateImages(count) {
let time_strings = []; let time_strings = [];
for(let i = count; i >= 0 ; i--) { for (let i = count; i >= 0; i--) {
time_strings.push(getTimeString(i)); time_strings.push(getTimeString(i));
} }
let files = fs.readdirSync('images/'); let files = fs.readdirSync('images/');
let image_count = files.length; let image_count = files.length;
let updated = false;
//Try to download new images //Try to download new images
for(let time_string of time_strings) { for (let time_string of time_strings) {
if (!files.includes(`${time_string}.png`)) { if (!files.includes(`${time_string}.png`)) {
try { try {
await downloadImage(time_string, `images/${time_string}.png`); await downloadImage(time_string, `images/${time_string}.png`);
updated = true;
image_count++; image_count++;
} catch (error) { } catch (error) {
console.log("No more images available"); console.log("No more images available");
@@ -84,8 +105,10 @@ export async function updateImages(count) {
} }
//Remove old files //Remove old files
for(let file of files.sort().slice(0, image_count-count)) { for (let file of files.sort().slice(0, image_count - count)) {
console.log(`Deleting ${file}`); console.log(`Deleting ${file}`);
fs.unlinkSync(`images/${file}`); fs.unlinkSync(`images/${file}`);
} }
return updated;
} }