Snapshot: add parallelism

snapshot-src/get-snap.js

@@ -1,9 +1,11 @@
 "use strict";
 
 const { app, BrowserWindow } = require("electron");
-let argDir = process.argv[2];
+const fs = require("fs");
-let fs = require("fs");
+const cp = require("child_process");
-let cp = require("child_process");
+
+const argDir = process.argv[2];
+const taskFile = process.argv[3];
 
 let mainWindow = null;
 let allWindowClosed = false;
@@ -43,6 +45,13 @@ const phases = {
 	prepare: function (event, arg) {
 		console.log(arg);
 		GOTO(phases["receive-rect"]);
+
+		const tasks = JSON.parse(fs.readFileSync(taskFile));
+		event.sender.send("start", tasks);
+	},
+	"wait-screenshot": function (event, arg) {
+		console.log(arg);
+		GOTO(phases["receive-rect"]);
 	},
 	"receive-rect": function (event, rect) {
 		pendingTasks += 1;

snapshot-src/index.js

@@ -3,9 +3,36 @@
 "use strict";
 
 const ipc = require("electron").ipcRenderer;
-const packagingTasks = require("./packaging-tasks.json");
 const auxData = require("./index.data.json");
 
+ipc.on("start", function (event, snapshotTasksRaw) {
+	let snapshotTasks = [];
+	for (const task of snapshotTasksRaw) {
+		for (const theme of ["dark", "light"]) {
+			for (const bg of ["black", "white"]) {
+				snapshotTasks.push({
+					...task,
+					theme,
+					background: bg,
+					name: task.name + "." + theme + "." + bg
+				});
+			}
+		}
+	}
+	let current = 0;
+	const step = function () {
+		const doit = function () {
+			captureElement(snapshotTasks[current], function () {
+				current += 1;
+				if (current >= snapshotTasks.length) window.close();
+				else setTimeout(step, 100);
+			});
+		};
+		setTimeout(doit, 100);
+	};
+	setTimeout(step, 2000);
+});
+
 let onScroll = function () {};
 ipc.on("scroll", function () {
 	onScroll.apply(this, arguments);
@@ -13,6 +40,7 @@ ipc.on("scroll", function () {
 		ipc.send("snapshot", "scroll-done");
 	}, 100);
 });
+
 let onComplete = function () {};
 ipc.on("complete", function () {
 	onComplete.apply(this, arguments);
@@ -153,32 +181,5 @@ function captureElement(options, callback) {
 }
 
 window.onload = function () {
-	const snapshotTasksRaw = [...auxData.readmeSnapshotTasks, ...packagingTasks];
+	ipc.send("snapshot", "Browser process ready");
-	let snapshotTasks = [];
-	for (const task of snapshotTasksRaw) {
-		for (const theme of ["dark", "light"]) {
-			for (const bg of ["black", "white"]) {
-				snapshotTasks.push({
-					...task,
-					theme,
-					background: bg,
-					name: task.name + "." + theme + "." + bg
-				});
-			}
-		}
-	}
-	let current = 0;
-	const step = function () {
-		const doit = function () {
-			captureElement(snapshotTasks[current], function () {
-				current += 1;
-				if (current >= snapshotTasks.length) window.close();
-				else setTimeout(step, 100);
-			});
-		};
-		setTimeout(doit, 100);
-	};
-	ipc.send("snapshot", "i am ready");
-	console.log("I AM READY");
-	setTimeout(step, 2000);
 };

utility/generate-snapshot-page/index.js

@@ -27,102 +27,129 @@ module.exports = async function main(argv) {
 	});
 	await fs.writeFile(argv.outputPath, html);
 
-	let readmeSnapshotTasks = [
+	for (let i = 0; i < argv.parallel; i++) {
-		{ el: "#languages", name: "languages" },
+		await generateTaskFile(
-		{ el: "#matrix", name: "matrix" },
+			argv.outputTaskFilePrefix,
-		{ el: "#previews", name: "preview-all" },
+			i,
-		{ el: "#weights", name: "weights" }
+			argv.parallel,
-	];
+			variationData,
-	for (const ls of ligationData.nonMergeSets) {
+			ligationData
-		readmeSnapshotTasks.push({
+		);
-			el: "#ligation-sampler",
-			applyClass: "iosevka",
-			applyFeature: `'${ls.tag}' ${ls.rank}`,
-			name: `ligset-${ls.tag}-${ls.rank}`,
-			applyCallback: `cbAmendLigsetSamplerContents`,
-			applyCallbackArgs: ls
-		});
-	}
-	for (const ss of variationData.composites) {
-		readmeSnapshotTasks.push({
-			el: "#packaging-sampler",
-			applyClass: "scl iosevka",
-			applyFeature: `'${ss.tag}' ${ss.rank}`,
-			name: `stylistic-set-u-${ss.tag}-${ss.rank}`,
-			applyCallback: `cbAmendStylisticSetContents`,
-			applyCallbackArgs: { hotChars: ss.hotChars.sans.upright }
-		});
-		readmeSnapshotTasks.push({
-			el: "#packaging-sampler",
-			applyClass: "scl iosevka italic",
-			applyFeature: `'${ss.tag}' ${ss.rank}`,
-			name: `stylistic-set-i-${ss.tag}-${ss.rank}`,
-			applyCallback: `cbAmendStylisticSetContents`,
-			applyCallbackArgs: { hotChars: ss.hotChars.sans.italic }
-		});
-	}
-	readmeSnapshotTasks.push({
-		el: "#cv-sampler",
-		applyClass: "cv-sampler",
-		applyFeature: "'lnum' on",
-		name: "character-variant-lnum",
-		applyCallback: `cbAmendCharacterVariantContents`,
-		applyCallbackArgs: {
-			hotChars: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"],
-			slopeDependent: false
-		}
-	});
-	readmeSnapshotTasks.push({
-		el: "#cv-sampler",
-		applyClass: "cv-sampler",
-		applyFeature: "'onum' on",
-		name: "character-variant-onum",
-		applyCallback: `cbAmendCharacterVariantContents`,
-		applyCallbackArgs: {
-			hotChars: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"],
-			slopeDependent: false
-		}
-	});
-	readmeSnapshotTasks.push({
-		el: "#cv-sampler",
-		applyClass: "cv-sampler",
-		applyFeature: "'APLF' on",
-		name: "character-variant-APLF-on",
-		applyCallback: `cbAmendCharacterVariantContents`,
-		applyCallbackArgs: { hotChars: ["∆", "∇", "○", "←", "→", "↑", "↓"], slopeDependent: false }
-	});
-	readmeSnapshotTasks.push({
-		el: "#cv-sampler",
-		applyClass: "cv-sampler",
-		applyFeature: "'APLF' off",
-		name: "character-variant-APLF-off",
-		applyCallback: `cbAmendCharacterVariantContents`,
-		applyCallbackArgs: { hotChars: ["∆", "∇", "○", "←", "→", "↑", "↓"], slopeDependent: false }
-	});
-	for (const cv of variationData.primes) {
-		if (!cv.tag) continue;
-		for (const variant of cv.variants) {
-			readmeSnapshotTasks.push({
-				el: "#cv-sampler",
-				applyClass: "cv-sampler",
-				applyFeature: `'${cv.tag}' ${variant.rank}`,
-				name: `character-variant-${cv.tag}-${variant.rank}`,
-				applyCallback: `cbAmendCharacterVariantContents`,
-				applyCallbackArgs: {
-					hotChars: cv.hotChars,
-					slopeDependent: !!cv.slopeDependent
-				}
-			});
-		}
 	}
 
 	await fs.writeJson(
 		argv.outputDataPath,
-		{
+		{ ligationSamples: ligationData.samples, ligationCherry: ligationData.cherry },
-			readmeSnapshotTasks,
-			ligationSamples: ligationData.samples,
-			ligationCherry: ligationData.cherry
-		},
 		{ spaces: "  " }
 	);
 };
+
+async function generateTaskFile(prefix, ith, total, variationData, ligationData) {
+	let readmeSnapshotTasks = [];
+	{
+		readmeSnapshotTasks.push({ el: "#languages", name: "languages" });
+		readmeSnapshotTasks.push({ el: "#matrix", name: "matrix" });
+		readmeSnapshotTasks.push({ el: "#previews", name: "preview-all" });
+		readmeSnapshotTasks.push({ el: "#weights", name: "weights" });
+	}
+	{
+		for (const ls of ligationData.nonMergeSets) {
+			readmeSnapshotTasks.push({
+				el: "#ligation-sampler",
+				applyClass: "iosevka",
+				applyFeature: `'${ls.tag}' ${ls.rank}`,
+				name: `ligset-${ls.tag}-${ls.rank}`,
+				applyCallback: `cbAmendLigsetSamplerContents`,
+				applyCallbackArgs: ls
+			});
+		}
+	}
+	{
+		for (const ss of variationData.composites) {
+			readmeSnapshotTasks.push({
+				el: "#packaging-sampler",
+				applyClass: "scl iosevka",
+				applyFeature: `'${ss.tag}' ${ss.rank}`,
+				name: `stylistic-set-u-${ss.tag}-${ss.rank}`,
+				applyCallback: `cbAmendStylisticSetContents`,
+				applyCallbackArgs: { hotChars: ss.hotChars.sans.upright }
+			});
+			readmeSnapshotTasks.push({
+				el: "#packaging-sampler",
+				applyClass: "scl iosevka italic",
+				applyFeature: `'${ss.tag}' ${ss.rank}`,
+				name: `stylistic-set-i-${ss.tag}-${ss.rank}`,
+				applyCallback: `cbAmendStylisticSetContents`,
+				applyCallbackArgs: { hotChars: ss.hotChars.sans.italic }
+			});
+		}
+	}
+	{
+		readmeSnapshotTasks.push({
+			el: "#cv-sampler",
+			applyClass: "cv-sampler",
+			applyFeature: "'lnum' on",
+			name: "character-variant-lnum",
+			applyCallback: `cbAmendCharacterVariantContents`,
+			applyCallbackArgs: {
+				hotChars: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"],
+				slopeDependent: false
+			}
+		});
+		readmeSnapshotTasks.push({
+			el: "#cv-sampler",
+			applyClass: "cv-sampler",
+			applyFeature: "'onum' on",
+			name: "character-variant-onum",
+			applyCallback: `cbAmendCharacterVariantContents`,
+			applyCallbackArgs: {
+				hotChars: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"],
+				slopeDependent: false
+			}
+		});
+		readmeSnapshotTasks.push({
+			el: "#cv-sampler",
+			applyClass: "cv-sampler",
+			applyFeature: "'APLF' on",
+			name: "character-variant-APLF-on",
+			applyCallback: `cbAmendCharacterVariantContents`,
+			applyCallbackArgs: {
+				hotChars: ["∆", "∇", "○", "←", "→", "↑", "↓"],
+				slopeDependent: false
+			}
+		});
+		readmeSnapshotTasks.push({
+			el: "#cv-sampler",
+			applyClass: "cv-sampler",
+			applyFeature: "'APLF' off",
+			name: "character-variant-APLF-off",
+			applyCallback: `cbAmendCharacterVariantContents`,
+			applyCallbackArgs: {
+				hotChars: ["∆", "∇", "○", "←", "→", "↑", "↓"],
+				slopeDependent: false
+			}
+		});
+		for (const cv of variationData.primes) {
+			if (!cv.tag) continue;
+			for (const variant of cv.variants) {
+				readmeSnapshotTasks.push({
+					el: "#cv-sampler",
+					applyClass: "cv-sampler",
+					applyFeature: `'${cv.tag}' ${variant.rank}`,
+					name: `character-variant-${cv.tag}-${variant.rank}`,
+					applyCallback: `cbAmendCharacterVariantContents`,
+					applyCallbackArgs: {
+						hotChars: cv.hotChars,
+						slopeDependent: !!cv.slopeDependent
+					}
+				});
+			}
+		}
+	}
+
+	let filteredTasks = [];
+	for (let i = 0; i < readmeSnapshotTasks.length; i++) {
+		if (i % total === ith) filteredTasks.push(readmeSnapshotTasks[i]);
+	}
+	await fs.writeJson(prefix + "-" + ith + ".json", filteredTasks);
+}

verdafile.js

@@ -639,14 +639,17 @@ const PagesFastFontExport = task.group(`pages:fast-font-export`, async (target,
 ///////////////////////////////////////////////////////////
 // Sample Images
 
+const SnapshotParallel = 8;
 const SampleImages = task(`sample-images`, async target => {
 	const [cfgP, sh] = await target.need(PackageSnapshotConfig, SnapShotHtml, TakeSampleImages);
-	const de = JSON.parse(fs.readFileSync(`${sh.dir}/${sh.name}.data.json`));
+	let snapshotFiles = [...cfgP];
+	for (let i = 0; i < SnapshotParallel; i++) {
+		const de = JSON.parse(fs.readFileSync(`${sh.dir}/readme-tasks-${i}.json`));
+		for (const x of de) snapshotFiles.push(x);
+	}
 	await target.need(
-		cfgP.map(opt => ScreenShot(opt.name + ".dark")),
+		snapshotFiles.map(opt => ScreenShot(opt.name + ".dark")),
-		cfgP.map(opt => ScreenShot(opt.name + ".light")),
+		snapshotFiles.map(opt => ScreenShot(opt.name + ".light"))
-		de.readmeSnapshotTasks.map(opt => ScreenShot(opt.name + ".dark")),
-		de.readmeSnapshotTasks.map(opt => ScreenShot(opt.name + ".light"))
 	);
 });
 
@@ -696,7 +699,9 @@ const SnapShotHtml = file(`${SNAPSHOT_TMP}/index.html`, async (target, out) => {
 	await node(`utility/generate-snapshot-page/index`, {
 		inputPath: "snapshot-src/templates",
 		outputPath: out.full,
-		outputDataPath: `${out.dir}/${out.name}.data.json`
+		outputDataPath: `${out.dir}/${out.name}.data.json`,
+		outputTaskFilePrefix: `${out.dir}/readme-tasks`,
+		parallel: SnapshotParallel
 	});
 });
 
@@ -736,7 +741,18 @@ const SnapShotCSS = file(`${SNAPSHOT_TMP}/index.css`, async (target, out) => {
 });
 const TakeSampleImages = task(`sample-images:take`, async target => {
 	await target.need(SampleImagesPre);
-	await cd(SNAPSHOT_TMP).run("npx", "electron", "get-snap.js", "../../images");
+
+	await run("npm", "install", "--no-save", "electron");
+
+	let taskLists = [`packaging-tasks.json`];
+	for (let i = 0; i < SnapshotParallel; i++) taskLists.push(`readme-tasks-${i}.json`);
+	await Promise.all(
+		taskLists.map((file, i) =>
+			Delay(i * 1000).then(() =>
+				cd(SNAPSHOT_TMP).run("npx", "electron", "get-snap.js", "../../images", file)
+			)
+		)
+	);
 });
 const ScreenShot = file.make(
 	img => `images/${img}.png`,
@@ -1065,3 +1081,9 @@ const VlCssFontStretch = {
 		x == "extra-expanded" ||
 		x == "ultra-expanded"
 };
+
+// Utilities
+
+function Delay(t) {
+	return new Promise(resolve => setTimeout(resolve, t));
+}