Image Optimization - Create new ImagePool for each batch (#470)
continuous-integration/drone/push Build is passing
Details
continuous-integration/drone/push Build is passing
Details
We have been having issues with the image optimization script consuming egregious amounts of memory and failing CI as a result. This PR changes the script so that we use a new `ImagePool` to process each batch of images. Co-authored-by: Amy <a258wang@uwaterloo.ca> Reviewed-on: #470 Reviewed-by: Shahan Neda <snedadah@csclub.uwaterloo.ca>
This commit is contained in:
parent
767e32511d
commit
14ef810ad9
|
@ -44,12 +44,14 @@ export async function optimizeImages() {
|
|||
|
||||
// maximum number of workers is 8 in order to avoid running out of memory
|
||||
const numberOfWorkers = Math.min(cpus().length, 8);
|
||||
const imagePool = new ImagePool(numberOfWorkers);
|
||||
|
||||
// process smaller batches in order to reduce memory usage
|
||||
const batchSize = 32;
|
||||
|
||||
for (let i = 0; i < imagePaths.length; i += batchSize) {
|
||||
// use a new ImagePool for each batch to reduce memory usage
|
||||
const imagePool = new ImagePool(numberOfWorkers);
|
||||
|
||||
await Promise.all(
|
||||
imagePaths.slice(i, i + batchSize).map(async (imagePath) => {
|
||||
const imageStartTime = Date.now();
|
||||
|
@ -108,9 +110,9 @@ export async function optimizeImages() {
|
|||
);
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
await imagePool.close();
|
||||
await imagePool.close();
|
||||
}
|
||||
|
||||
console.log(`TOTAL DURATION: ${getElapsedSeconds(startTime)}s`);
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue