Image Optimization Script (#348)

A build-time script to optimize images. Note that going forward, all images should be placed in `images` instead of `public/images`.

Co-authored-by: Amy <a258wang@uwaterloo.ca>
Reviewed-on: #348
Reviewed-by: Aditya Thakral <a3thakra@csclub.uwaterloo.ca>
Reviewed-by: n3parikh <n3parikh@csclub.uwaterloo.ca>
Co-authored-by: Amy <a258wang@csclub.uwaterloo.ca>
Co-committed-by: Amy <a258wang@csclub.uwaterloo.ca>

.drone.yml

@@ -23,25 +23,32 @@ steps:
   commands:
   - npm run lint
 
-- name: build
+- name: optimize-images
   image: node:16
   depends_on:
   - install-deps
   commands:
-  - npm run build:web
+  - npm run build:images
 
 - name: generate-calendar
   image: node:16
   depends_on:
   - install-deps
   commands:
-  - npm run generate:calendar
+  - npm run build:calendar
+
+- name: build
+  image: node:16
+  depends_on:
+  - optimize-images
+  commands:
+  - npm run build:web
 
 - name: export
   image: node:16
   depends_on:
-  - build
   - generate-calendar
+  - build
   commands:
   - npm run export
 

.gitignore

@@ -25,4 +25,7 @@ yarn-debug.log*
 yarn-error.log*
 
 # Calendar is automatically generated
-/public/events.ics
+/public/events.ics
+
+# Images should be optimized
+/public/images

lib/team.ts

@@ -52,6 +52,7 @@ export async function getMemberImagePath(name: string) {
     (await getImage(imgPath + ".jpg")) ??
     (await getImage(imgPath + ".png")) ??
     (await getImage(imgPath + ".gif")) ??
+    (await getImage(imgPath + ".jpeg")) ??
     placeholder;
   return img;
 }

package-lock.json

@@ -10,8 +10,11 @@
         "@mdx-js/loader": "^1.6.22",
         "@mdx-js/react": "^1.6.22",
         "@next/mdx": "11.0.1",
+        "@squoosh/lib": "^0.4.0",
         "date-fns": "^2.11.1",
         "date-fns-tz": "^1.1.6",
+        "fs-extra": "^10.0.0",
+        "image-size": "^1.0.0",
         "next": "11.0.1",
         "next-mdx-remote": "3.0.4",
         "prettier": "^2.3.0",
@@ -21,6 +24,7 @@
         "remark-html": "^12.0.0"
       },
       "devDependencies": {
+        "@types/fs-extra": "^9.0.13",
         "@types/mdx-js__react": "^1.5.3",
         "@types/node": "^16.9.1",
         "@types/react": "^17.0.14",
@@ -990,6 +994,18 @@
         "node": ">= 8"
       }
     },
+    "node_modules/@squoosh/lib": {
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/@squoosh/lib/-/lib-0.4.0.tgz",
+      "integrity": "sha512-O1LyugWLZjMI4JZeZMA5vzfhfPjfMZXH5/HmVkRagP8B70wH3uoR7tjxfGNdSavey357MwL8YJDxbGwBBdHp7Q==",
+      "dependencies": {
+        "wasm-feature-detect": "^1.2.11",
+        "web-streams-polyfill": "^3.0.3"
+      },
+      "engines": {
+        "node": " ^12.5.0 || ^14.0.0 || ^16.0.0 "
+      }
+    },
     "node_modules/@tsconfig/node10": {
       "version": "1.0.8",
       "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.8.tgz",
@@ -1014,6 +1030,15 @@
       "integrity": "sha512-eZxlbI8GZscaGS7kkc/trHTT5xgrjH3/1n2JDwusC9iahPKWMRvRjJSAN5mCXviuTGQ/lHnhvv8Q1YTpnfz9gA==",
       "dev": true
     },
+    "node_modules/@types/fs-extra": {
+      "version": "9.0.13",
+      "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-9.0.13.tgz",
+      "integrity": "sha512-nEnwB++1u5lVDM2UI4c1+5R+FYaKfaAzS4OococimjVm3nQw3TuzH5UNsocrcTBbhnerblyHj4A49qXbIiZdpA==",
+      "dev": true,
+      "dependencies": {
+        "@types/node": "*"
+      }
+    },
     "node_modules/@types/hast": {
       "version": "2.3.1",
       "resolved": "https://registry.npmjs.org/@types/hast/-/hast-2.3.1.tgz",
@@ -3510,6 +3535,19 @@
       "resolved": "https://registry.npmjs.org/foreach/-/foreach-2.0.5.tgz",
       "integrity": "sha1-C+4AUBiusmDQo6865ljdATbsG5k="
     },
+    "node_modules/fs-extra": {
+      "version": "10.0.0",
+      "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.0.0.tgz",
+      "integrity": "sha512-C5owb14u9eJwizKGdchcDUQeFtlSHHthBk8pbX9Vc1PFZrLombudjDnNns88aYslCyF6IY5SUw3Roz6xShcEIQ==",
+      "dependencies": {
+        "graceful-fs": "^4.2.0",
+        "jsonfile": "^6.0.1",
+        "universalify": "^2.0.0"
+      },
+      "engines": {
+        "node": ">=12"
+      }
+    },
     "node_modules/fs.realpath": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
@@ -4549,6 +4587,17 @@
         "json5": "lib/cli.js"
       }
     },
+    "node_modules/jsonfile": {
+      "version": "6.1.0",
+      "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz",
+      "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==",
+      "dependencies": {
+        "universalify": "^2.0.0"
+      },
+      "optionalDependencies": {
+        "graceful-fs": "^4.1.6"
+      }
+    },
     "node_modules/jsx-ast-utils": {
       "version": "3.2.0",
       "resolved": "https://registry.npmjs.org/jsx-ast-utils/-/jsx-ast-utils-3.2.0.tgz",
@@ -8963,6 +9012,14 @@
         "url": "https://opencollective.com/unified"
       }
     },
+    "node_modules/universalify": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz",
+      "integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==",
+      "engines": {
+        "node": ">= 10.0.0"
+      }
+    },
     "node_modules/unpipe": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",
@@ -9104,6 +9161,11 @@
       "resolved": "https://registry.npmjs.org/vm-browserify/-/vm-browserify-1.1.2.tgz",
       "integrity": "sha512-2ham8XPWTONajOR0ohOKOHXkm3+gaBmGut3SRuu75xLd/RRaY6vqgh8NBYYk7+RW3u5AtzPQZG8F10LHkl0lAQ=="
     },
+    "node_modules/wasm-feature-detect": {
+      "version": "1.2.11",
+      "resolved": "https://registry.npmjs.org/wasm-feature-detect/-/wasm-feature-detect-1.2.11.tgz",
+      "integrity": "sha512-HUqwaodrQGaZgz1lZaNioIkog9tkeEJjrM3eq4aUL04whXOVDRc/o2EGb/8kV0QX411iAYWEqq7fMBmJ6dKS6w=="
+    },
     "node_modules/watchpack": {
       "version": "2.1.1",
       "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.1.1.tgz",
@@ -9125,6 +9187,14 @@
         "url": "https://github.com/sponsors/wooorm"
       }
     },
+    "node_modules/web-streams-polyfill": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.1.1.tgz",
+      "integrity": "sha512-Czi3fG883e96T4DLEPRvufrF2ydhOOW1+1a6c3gNjH2aIh50DNFBdfwh2AKoOf1rXvpvavAoA11Qdq9+BKjE0Q==",
+      "engines": {
+        "node": ">= 8"
+      }
+    },
     "node_modules/webidl-conversions": {
       "version": "4.0.2",
       "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-4.0.2.tgz",
@@ -10052,6 +10122,15 @@
         "fastq": "^1.6.0"
       }
     },
+    "@squoosh/lib": {
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/@squoosh/lib/-/lib-0.4.0.tgz",
+      "integrity": "sha512-O1LyugWLZjMI4JZeZMA5vzfhfPjfMZXH5/HmVkRagP8B70wH3uoR7tjxfGNdSavey357MwL8YJDxbGwBBdHp7Q==",
+      "requires": {
+        "wasm-feature-detect": "^1.2.11",
+        "web-streams-polyfill": "^3.0.3"
+      }
+    },
     "@tsconfig/node10": {
       "version": "1.0.8",
       "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.8.tgz",
@@ -10076,6 +10155,15 @@
       "integrity": "sha512-eZxlbI8GZscaGS7kkc/trHTT5xgrjH3/1n2JDwusC9iahPKWMRvRjJSAN5mCXviuTGQ/lHnhvv8Q1YTpnfz9gA==",
       "dev": true
     },
+    "@types/fs-extra": {
+      "version": "9.0.13",
+      "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-9.0.13.tgz",
+      "integrity": "sha512-nEnwB++1u5lVDM2UI4c1+5R+FYaKfaAzS4OococimjVm3nQw3TuzH5UNsocrcTBbhnerblyHj4A49qXbIiZdpA==",
+      "dev": true,
+      "requires": {
+        "@types/node": "*"
+      }
+    },
     "@types/hast": {
       "version": "2.3.1",
       "resolved": "https://registry.npmjs.org/@types/hast/-/hast-2.3.1.tgz",
@@ -11967,6 +12055,16 @@
       "resolved": "https://registry.npmjs.org/foreach/-/foreach-2.0.5.tgz",
       "integrity": "sha1-C+4AUBiusmDQo6865ljdATbsG5k="
     },
+    "fs-extra": {
+      "version": "10.0.0",
+      "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.0.0.tgz",
+      "integrity": "sha512-C5owb14u9eJwizKGdchcDUQeFtlSHHthBk8pbX9Vc1PFZrLombudjDnNns88aYslCyF6IY5SUw3Roz6xShcEIQ==",
+      "requires": {
+        "graceful-fs": "^4.2.0",
+        "jsonfile": "^6.0.1",
+        "universalify": "^2.0.0"
+      }
+    },
     "fs.realpath": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
@@ -12650,6 +12748,15 @@
         "minimist": "^1.2.0"
       }
     },
+    "jsonfile": {
+      "version": "6.1.0",
+      "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz",
+      "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==",
+      "requires": {
+        "graceful-fs": "^4.1.6",
+        "universalify": "^2.0.0"
+      }
+    },
     "jsx-ast-utils": {
       "version": "3.2.0",
       "resolved": "https://registry.npmjs.org/jsx-ast-utils/-/jsx-ast-utils-3.2.0.tgz",
@@ -15975,6 +16082,11 @@
         "unist-util-is": "^4.0.0"
       }
     },
+    "universalify": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz",
+      "integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ=="
+    },
     "unpipe": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",
@@ -16096,6 +16208,11 @@
       "resolved": "https://registry.npmjs.org/vm-browserify/-/vm-browserify-1.1.2.tgz",
       "integrity": "sha512-2ham8XPWTONajOR0ohOKOHXkm3+gaBmGut3SRuu75xLd/RRaY6vqgh8NBYYk7+RW3u5AtzPQZG8F10LHkl0lAQ=="
     },
+    "wasm-feature-detect": {
+      "version": "1.2.11",
+      "resolved": "https://registry.npmjs.org/wasm-feature-detect/-/wasm-feature-detect-1.2.11.tgz",
+      "integrity": "sha512-HUqwaodrQGaZgz1lZaNioIkog9tkeEJjrM3eq4aUL04whXOVDRc/o2EGb/8kV0QX411iAYWEqq7fMBmJ6dKS6w=="
+    },
     "watchpack": {
       "version": "2.1.1",
       "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.1.1.tgz",
@@ -16110,6 +16227,11 @@
       "resolved": "https://registry.npmjs.org/web-namespaces/-/web-namespaces-1.1.4.tgz",
       "integrity": "sha512-wYxSGajtmoP4WxfejAPIr4l0fVh+jeMXZb08wNc0tMg6xsfZXj3cECqIK0G7ZAqUq0PP8WlMDtaOGVBTAWztNw=="
     },
+    "web-streams-polyfill": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.1.1.tgz",
+      "integrity": "sha512-Czi3fG883e96T4DLEPRvufrF2ydhOOW1+1a6c3gNjH2aIh50DNFBdfwh2AKoOf1rXvpvavAoA11Qdq9+BKjE0Q=="
+    },
     "webidl-conversions": {
       "version": "4.0.2",
       "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-4.0.2.tgz",

package.json

@@ -7,21 +7,25 @@
   },
   "scripts": {
     "dev": "next dev",
-    "build": "npm run build:web && npm run generate:calendar",
+    "build": "npm run build:images && npm run build:web && npm run build:calendar",
+    "build:images": "ts-node ./scripts/optimize-images",
     "build:web": "next build",
+    "build:calendar": "ts-node ./scripts/generate-calendar",
     "start": "next start",
     "export": "next export",
     "lint": "eslint \"{pages,components,lib,hooks,scripts}/**/*.{js,ts,tsx,jsx}\" --quiet",
     "lint:fix": "eslint \"{pages,components,lib,hooks,scripts}/**/*.{js,ts,tsx,jsx}\" --quiet --fix",
-    "generate:calendar": "ts-node ./scripts/generate-calendar",
     "check-lockfile": "ts-node ./scripts/check-lockfile"
   },
   "dependencies": {
     "@mdx-js/loader": "^1.6.22",
     "@mdx-js/react": "^1.6.22",
     "@next/mdx": "11.0.1",
+    "@squoosh/lib": "^0.4.0",
     "date-fns": "^2.11.1",
     "date-fns-tz": "^1.1.6",
+    "fs-extra": "^10.0.0",
+    "image-size": "^1.0.0",
     "next": "11.0.1",
     "next-mdx-remote": "3.0.4",
     "prettier": "^2.3.0",
@@ -31,6 +35,7 @@
     "remark-html": "^12.0.0"
   },
   "devDependencies": {
+    "@types/fs-extra": "^9.0.13",
     "@types/mdx-js__react": "^1.5.3",
     "@types/node": "^16.9.1",
     "@types/react": "^17.0.14",

scripts/optimize-images.ts

@@ -0,0 +1,107 @@
+/* eslint-disable @typescript-eslint/no-unsafe-assignment */
+/* eslint-disable @typescript-eslint/no-unsafe-call */
+/* eslint-disable @typescript-eslint/no-unsafe-member-access */
+
+// TODO: upgrade libsquoosh once types are available: https://github.com/GoogleChromeLabs/squoosh/issues/1077
+
+import { cpus } from "os";
+import path from "path";
+
+// eslint-disable-next-line @typescript-eslint/ban-ts-comment
+// @ts-ignore
+import { ImagePool } from "@squoosh/lib";
+import fse from "fs-extra";
+import { default as getImageDimensions } from "image-size";
+
+const SOURCE_DIRECTORY = "images";
+const DESTINATION_DIRECTORY = path.join("public", "images");
+
+// directory where Meet the Team headshots are stored, relative to the source directory
+const TEAM_IMAGES_DIRECTORY = path.join("team", "");
+
+const IMAGE_MINIMUM_SIZE = 512;
+
+const GET_ENCODER_FROM_EXTENSION: { [imageExtension: string]: string } = {
+  jpg: "mozjpeg",
+  jpeg: "mozjpeg",
+  png: "oxipng",
+};
+
+const ENCODER_OPTIONS: { [encoder: string]: Record<string, unknown> } = {
+  mozjpeg: {},
+  oxipng: {},
+};
+
+void optimizeImages();
+
+export async function optimizeImages() {
+  const imagePaths = await getFilePathsInDirectory(SOURCE_DIRECTORY);
+  await fse.emptyDir(DESTINATION_DIRECTORY);
+
+  // maximum number of workers is 8 in order to avoid running out of memory
+  const numberOfWorkers = Math.min(cpus().length, 8);
+  const imagePool = new ImagePool(numberOfWorkers);
+
+  await Promise.all(
+    imagePaths.map(async (imagePath) => {
+      const sourcePath = path.join(SOURCE_DIRECTORY, imagePath);
+      const destinationPath = path.join(DESTINATION_DIRECTORY, imagePath);
+      const fileExtension = imagePath.split(".").pop() ?? "";
+      const encoder = GET_ENCODER_FROM_EXTENSION[fileExtension];
+
+      if (!encoder) {
+        await fse.copy(sourcePath, destinationPath);
+        return;
+      }
+
+      const rawImageFile = await fse.readFile(sourcePath);
+      const ingestedImage = imagePool.ingestImage(rawImageFile);
+      const { width, height } = getImageDimensions(rawImageFile);
+
+      await ingestedImage.decoded;
+
+      const shouldResize =
+        imagePath.startsWith(TEAM_IMAGES_DIRECTORY) &&
+        (width ?? 0) > IMAGE_MINIMUM_SIZE &&
+        (height ?? 0) > IMAGE_MINIMUM_SIZE;
+
+      if (width && height && shouldResize) {
+        const smallerDimension = width < height ? "width" : "height";
+
+        // specifying only one dimension maintains the aspect ratio
+        const preprocessOptions = {
+          resize: {
+            enabled: true,
+            [smallerDimension]: IMAGE_MINIMUM_SIZE,
+          },
+        };
+
+        await ingestedImage.preprocess(preprocessOptions);
+      }
+
+      const encodeOptions = { [encoder]: ENCODER_OPTIONS[encoder] };
+      await ingestedImage.encode(encodeOptions);
+
+      const encodedImage = await ingestedImage.encodedWith[encoder];
+      await fse.outputFile(destinationPath, encodedImage.binary);
+    })
+  );
+
+  await imagePool.close();
+}
+
+async function getFilePathsInDirectory(directory: string): Promise<string[]> {
+  const entries = await fse.readdir(directory, { withFileTypes: true });
+  return (
+    await Promise.all(
+      entries.map(async (entry) => {
+        if (entry.isDirectory()) {
+          const subdirectory = path.join(directory, entry.name);
+          const subentries = await getFilePathsInDirectory(subdirectory);
+          return subentries.map((subentry) => path.join(entry.name, subentry));
+        }
+        return entry.name;
+      })
+    )
+  ).flat();
+}