From 443925190e7f05e8f5461fac542b0aec57ca66e2 Mon Sep 17 00:00:00 2001 From: Amy Date: Sat, 4 Jun 2022 19:31:00 -0400 Subject: [PATCH] Improve image optimization run-time and memory usage (#457) This PR aims to resolve the issues surrounding the optimize-images script, at least in the short term, in order to unblock other work. **Problems:** Our optimize-images script was taking a very long time to run (~7 minutes in CI, when successful). This led to two problems: 1. It was near impossible to run the script locally. 2. CI jobs would often get killed on the optimize-images step. **Solutions:** 1. Resize the images in `images/events` prior to optimizing them, similar to what is done for the images in `images/team`. This solution on its own reduced the run-time of the script to ~30 seconds locally on Amy's laptop, which is comparable to the run-time of the script back when it was originally written. 2. EDIT: Copy/resize/optimize the images in batches of 32 at a time. The reason why the CI job was being killed is because the script would run out of memory, however this change should resolve that while also keeping build times reasonable (~30 sec locally/when deploying on caffeine, ~3 min in CI). ~~As a temporary fix, this PR also replaces the images in `images/events` with their resized + optimized versions. (For some unknown reason, Solution 1 is not sufficient to solve Problem 2.)~~ This PR also adds some logging to the script so we can get a (slightly) better sense of where the script is getting stuck. Related issue: #456 Co-authored-by: Amy Reviewed-on: https://git.csclub.uwaterloo.ca/www/www-new/pulls/457 Reviewed-by: Shahan Neda --- scripts/optimize-images.ts | 101 ++++++++++++++++++++++++------------- 1 file changed, 65 insertions(+), 36 deletions(-) diff --git a/scripts/optimize-images.ts b/scripts/optimize-images.ts index 9151bf8e..803c54ee 100644 --- a/scripts/optimize-images.ts +++ b/scripts/optimize-images.ts @@ -16,14 +16,16 @@ import { default as getImageDimensions } from "image-size"; const SOURCE_DIRECTORY = "images"; const DESTINATION_DIRECTORY = path.join("public", "images"); -// directory where Meet the Team headshots are stored, relative to the source directory +// directories are relative to SOURCE_DIRECTORY const TEAM_IMAGES_DIRECTORY = path.join("team", ""); +const EVENTS_IMAGES_DIRECTORY = path.join("events", ""); const IMAGE_MINIMUM_SIZE = 512; const GET_ENCODER_FROM_EXTENSION: { [imageExtension: string]: string } = { jpg: "mozjpeg", jpeg: "mozjpeg", + JPG: "mozjpeg", png: "oxipng", }; @@ -35,6 +37,8 @@ const ENCODER_OPTIONS: { [encoder: string]: Record } = { void optimizeImages(); export async function optimizeImages() { + const startTime = Date.now(); + const imagePaths = await getFilePathsInDirectory(SOURCE_DIRECTORY); await fse.emptyDir(DESTINATION_DIRECTORY); @@ -42,52 +46,73 @@ export async function optimizeImages() { const numberOfWorkers = Math.min(cpus().length, 8); const imagePool = new ImagePool(numberOfWorkers); - await Promise.all( - imagePaths.map(async (imagePath) => { - const sourcePath = path.join(SOURCE_DIRECTORY, imagePath); - const destinationPath = path.join(DESTINATION_DIRECTORY, imagePath); - const fileExtension = imagePath.split(".").pop() ?? ""; - const encoder = GET_ENCODER_FROM_EXTENSION[fileExtension]; + // process smaller batches in order to reduce memory usage + const batchSize = 32; - if (!encoder) { - await fse.copy(sourcePath, destinationPath); - return; - } + for (let i = 0; i < imagePaths.length; i += batchSize) { + await Promise.all( + imagePaths.slice(i, i + batchSize).map(async (imagePath) => { + const imageStartTime = Date.now(); - const rawImageFile = await fse.readFile(sourcePath); - const ingestedImage = imagePool.ingestImage(rawImageFile); - const { width, height } = getImageDimensions(rawImageFile); + const sourcePath = path.join(SOURCE_DIRECTORY, imagePath); + const destinationPath = path.join(DESTINATION_DIRECTORY, imagePath); + const fileExtension = imagePath.split(".").pop() ?? ""; + const encoder = GET_ENCODER_FROM_EXTENSION[fileExtension]; - await ingestedImage.decoded; + if (!encoder) { + await fse.copy(sourcePath, destinationPath); + console.log( + `Copied ${imagePath} in ${getElapsedSeconds(imageStartTime)}s` + ); + return; + } - const shouldResize = - imagePath.startsWith(TEAM_IMAGES_DIRECTORY) && - (width ?? 0) > IMAGE_MINIMUM_SIZE && - (height ?? 0) > IMAGE_MINIMUM_SIZE; + const rawImageFile = await fse.readFile(sourcePath); + const ingestedImage = imagePool.ingestImage(rawImageFile); + const { width, height } = getImageDimensions(rawImageFile); - if (width && height && shouldResize) { - const smallerDimension = width < height ? "width" : "height"; + await ingestedImage.decoded; - // specifying only one dimension maintains the aspect ratio - const preprocessOptions = { - resize: { - enabled: true, - [smallerDimension]: IMAGE_MINIMUM_SIZE, - }, - }; + const shouldResize = + (imagePath.startsWith(TEAM_IMAGES_DIRECTORY) || + imagePath.startsWith(EVENTS_IMAGES_DIRECTORY)) && + (width ?? 0) > IMAGE_MINIMUM_SIZE && + (height ?? 0) > IMAGE_MINIMUM_SIZE; - await ingestedImage.preprocess(preprocessOptions); - } + if (width && height && shouldResize) { + const smallerDimension = width < height ? "width" : "height"; - const encodeOptions = { [encoder]: ENCODER_OPTIONS[encoder] }; - await ingestedImage.encode(encodeOptions); + // specifying only one dimension maintains the aspect ratio + const preprocessOptions = { + resize: { + enabled: true, + [smallerDimension]: IMAGE_MINIMUM_SIZE, + }, + }; - const encodedImage = await ingestedImage.encodedWith[encoder]; - await fse.outputFile(destinationPath, encodedImage.binary); - }) - ); + await ingestedImage.preprocess(preprocessOptions); + + console.log( + `Resized ${sourcePath} in ${getElapsedSeconds(imageStartTime)}s` + ); + } + + const encodeOptions = { [encoder]: ENCODER_OPTIONS[encoder] }; + await ingestedImage.encode(encodeOptions); + + const encodedImage = await ingestedImage.encodedWith[encoder]; + await fse.outputFile(destinationPath, encodedImage.binary); + + console.log( + `Optimized ${sourcePath} in ${getElapsedSeconds(imageStartTime)}s` + ); + }) + ); + } await imagePool.close(); + + console.log(`TOTAL DURATION: ${getElapsedSeconds(startTime)}s`); } async function getFilePathsInDirectory(directory: string): Promise { @@ -105,3 +130,7 @@ async function getFilePathsInDirectory(directory: string): Promise { ) ).flat(); } + +function getElapsedSeconds(startTime: number) { + return (Date.now() - startTime) / 1000; +}