Use Faster Hashing Library (#1021)

[INTERNAL]

tools/gulpfile.ts

@@ -19,7 +19,7 @@ import clientTasks from "./tasks/client/index.ts";
 import serverTasks from "./tasks/server/index.ts";
 import langTasks from "./tasks/lang/index.ts";
 import mmcTasks from "./tasks/mmc/index.ts";
-import modTasks from "./tasks/misc/downloadMods.ts";
+import * as modTasks from "./tasks/misc/downloadMods.ts";
 
 export const buildClient = gulp.series(
 	sharedTasks,
@@ -27,20 +27,23 @@ export const buildClient = gulp.series(
 	pruneCacheTask,
 );
 export const buildServer = gulp.series(
-	gulp.parallel(sharedTasks, modTasks),
+	gulp.parallel(sharedTasks, modTasks.downloadSharedAndServer),
 	serverTasks,
 	pruneCacheTask,
 );
 export const buildLang = gulp.series(sharedTasks, langTasks, pruneCacheTask);
 export const buildMMC = gulp.series(
-	gulp.parallel(sharedTasks, modTasks),
+	gulp.parallel(sharedTasks, modTasks.downloadSharedAndClient),
-	clientTasks,
 	mmcTasks,
 	pruneCacheTask,
 );
 export const buildAll = gulp.series(
 	sharedTasks,
-	gulp.parallel(clientTasks, langTasks, gulp.series(modTasks, serverTasks)),
+	gulp.parallel(
+		clientTasks,
+		langTasks,
+		gulp.series(modTasks.downloadSharedAndServer, serverTasks),
+	),
 	pruneCacheTask,
 );
 

tools/package-lock.json

@@ -24,11 +24,9 @@
 				"@types/gulp-zip": "^4.0.4",
 				"@types/inquirer": "^9.0.7",
 				"@types/lodash": "^4.17.1",
-				"@types/md5": "^2.3.5",
 				"@types/merge-stream": "^1.1.5",
 				"@types/mustache": "^4.2.5",
 				"@types/picomatch": "^2.3.3",
-				"@types/sha1": "^1.1.5",
 				"@types/through2": "^2.0.41",
 				"@types/unzipper": "^0.10.9",
 				"@typescript-eslint/eslint-plugin": "^7.8.0",
@@ -51,20 +49,19 @@
 				"gulp": "^5.0.0",
 				"gulp-rename": "^2.0.0",
 				"gulp-zip": "^6.0.0",
+				"hash-wasm": "^4.11.0",
 				"inquirer": "^9.2.20",
 				"javascript-stringify": "^2.1.0",
 				"just-diff": "^6.0.2",
 				"just-remove": "^3.2.0",
 				"lodash": "^4.17.21",
 				"marked": "^12.0.2",
-				"md5": "^2.3.0",
 				"merge-stream": "^2.0.0",
 				"mustache": "^4.2.0",
 				"picomatch": "^4.0.2",
 				"prettier": "^3.2.5",
 				"retry-axios": "^3.1.3",
 				"sanitize-filename": "^1.6.3",
-				"sha1": "^1.1.1",
 				"simple-git": "^3.24.0",
 				"sort-keys": "^5.0.0",
 				"sort-keys-recursive": "^2.1.10",
@@ -866,12 +863,6 @@
 			"integrity": "sha512-X+2qazGS3jxLAIz5JDXDzglAF3KpijdhFxlf/V1+hEsOUc+HnWi81L/uv/EvGuV90WY+7mPGFCUDGfQC3Gj95Q==",
 			"dev": true
 		},
-		"node_modules/@types/md5": {
-			"version": "2.3.5",
-			"resolved": "https://registry.npmjs.org/@types/md5/-/md5-2.3.5.tgz",
-			"integrity": "sha512-/i42wjYNgE6wf0j2bcTX6kuowmdL/6PE4IVitMpm2eYKBUuYCprdcWVK+xEF0gcV6ufMCRhtxmReGfc6hIK7Jw==",
-			"dev": true
-		},
 		"node_modules/@types/merge-stream": {
 			"version": "1.1.5",
 			"resolved": "https://registry.npmjs.org/@types/merge-stream/-/merge-stream-1.1.5.tgz",
@@ -923,15 +914,6 @@
 			"integrity": "sha512-I8EUhyrgfLrcTkzV3TSsGyl1tSuPrEDzr0yd5m90UgNxQkyDXULk3b6MlQqTCpZpNtWe1K0hzclnZkTcLBe2UQ==",
 			"dev": true
 		},
-		"node_modules/@types/sha1": {
-			"version": "1.1.5",
-			"resolved": "https://registry.npmjs.org/@types/sha1/-/sha1-1.1.5.tgz",
-			"integrity": "sha512-eE1PzjW7u2VfxI+bTsvjzjBfpwqvxSpgfUmnRNVY+PJU1NBsdGZlaO/qnVnPKHzzpgIl9YyBIxvrgBvt1mzt2A==",
-			"dev": true,
-			"dependencies": {
-				"@types/node": "*"
-			}
-		},
 		"node_modules/@types/streamx": {
 			"version": "2.9.5",
 			"resolved": "https://registry.npmjs.org/@types/streamx/-/streamx-2.9.5.tgz",
@@ -1667,15 +1649,6 @@
 			"integrity": "sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==",
 			"dev": true
 		},
-		"node_modules/charenc": {
-			"version": "0.0.2",
-			"resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz",
-			"integrity": "sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==",
-			"dev": true,
-			"engines": {
-				"node": "*"
-			}
-		},
 		"node_modules/chokidar": {
 			"version": "3.6.0",
 			"resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz",
@@ -1902,15 +1875,6 @@
 				"node": ">= 8"
 			}
 		},
-		"node_modules/crypt": {
-			"version": "0.0.2",
-			"resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz",
-			"integrity": "sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==",
-			"dev": true,
-			"engines": {
-				"node": "*"
-			}
-		},
 		"node_modules/debug": {
 			"version": "4.3.4",
 			"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
@@ -3325,6 +3289,13 @@
 				"url": "https://github.com/sponsors/ljharb"
 			}
 		},
+		"node_modules/hash-wasm": {
+			"version": "4.11.0",
+			"resolved": "https://registry.npmjs.org/hash-wasm/-/hash-wasm-4.11.0.tgz",
+			"integrity": "sha512-HVusNXlVqHe0fzIzdQOGolnFN6mX/fqcrSAOcTBXdvzrXVHwTz11vXeKRmkR5gTuwVpvHZEIyKoePDvuAR+XwQ==",
+			"dev": true,
+			"license": "MIT"
+		},
 		"node_modules/hasown": {
 			"version": "2.0.2",
 			"resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz",
@@ -3521,12 +3492,6 @@
 				"node": ">=8"
 			}
 		},
-		"node_modules/is-buffer": {
-			"version": "1.1.6",
-			"resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz",
-			"integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==",
-			"dev": true
-		},
 		"node_modules/is-core-module": {
 			"version": "2.13.1",
 			"resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.13.1.tgz",
@@ -3922,17 +3887,6 @@
 				"node": ">= 18"
 			}
 		},
-		"node_modules/md5": {
-			"version": "2.3.0",
-			"resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz",
-			"integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==",
-			"dev": true,
-			"dependencies": {
-				"charenc": "0.0.2",
-				"crypt": "0.0.2",
-				"is-buffer": "~1.1.6"
-			}
-		},
 		"node_modules/merge-stream": {
 			"version": "2.0.0",
 			"resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz",
@@ -4778,19 +4732,6 @@
 				"node": ">= 0.4"
 			}
 		},
-		"node_modules/sha1": {
-			"version": "1.1.1",
-			"resolved": "https://registry.npmjs.org/sha1/-/sha1-1.1.1.tgz",
-			"integrity": "sha512-dZBS6OrMjtgVkopB1Gmo4RQCDKiZsqcpAQpkV/aaj+FCrCg8r4I4qMkDPQjBgLIxlmu9k4nUbWq6ohXahOneYA==",
-			"dev": true,
-			"dependencies": {
-				"charenc": ">= 0.0.1",
-				"crypt": ">= 0.0.1"
-			},
-			"engines": {
-				"node": "*"
-			}
-		},
 		"node_modules/shebang-command": {
 			"version": "2.0.0",
 			"resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz",

tools/package.json

@@ -34,11 +34,9 @@
 		"@types/gulp-zip": "^4.0.4",
 		"@types/inquirer": "^9.0.7",
 		"@types/lodash": "^4.17.1",
-		"@types/md5": "^2.3.5",
 		"@types/merge-stream": "^1.1.5",
 		"@types/mustache": "^4.2.5",
 		"@types/picomatch": "^2.3.3",
-		"@types/sha1": "^1.1.5",
 		"@types/through2": "^2.0.41",
 		"@types/unzipper": "^0.10.9",
 		"@typescript-eslint/eslint-plugin": "^7.8.0",
@@ -61,20 +59,19 @@
 		"gulp": "^5.0.0",
 		"gulp-rename": "^2.0.0",
 		"gulp-zip": "^6.0.0",
+		"hash-wasm": "^4.11.0",
 		"inquirer": "^9.2.20",
 		"javascript-stringify": "^2.1.0",
 		"just-diff": "^6.0.2",
 		"just-remove": "^3.2.0",
 		"lodash": "^4.17.21",
 		"marked": "^12.0.2",
-		"md5": "^2.3.0",
 		"merge-stream": "^2.0.0",
 		"mustache": "^4.2.0",
 		"picomatch": "^4.0.2",
 		"prettier": "^3.2.5",
 		"retry-axios": "^3.1.3",
 		"sanitize-filename": "^1.6.3",
-		"sha1": "^1.1.1",
 		"simple-git": "^3.24.0",
 		"sort-keys": "^5.0.0",
 		"sort-keys-recursive": "^2.1.10",

tools/tasks/misc/downloadMods.ts

@@ -3,7 +3,7 @@ import { fetchMods } from "#utils/curseForgeAPI.ts";
 import upath from "upath";
 import fs from "fs";
 import { deleteAsync } from "del";
-import gulp from "gulp";
+import { parallel, series } from "gulp";
 import logInfo from "#utils/log.ts";
 
 async function modCleanUp() {
@@ -31,19 +31,29 @@ async function createModDirs() {
 /**
  * Downloads mods according to manifest.json and checks hashes.
  */
-export async function downloadMods(): Promise<void> {
+async function downloadMods(): Promise<void> {
 	logInfo("Fetching Shared Mods...");
 	await fetchMods(
 		modpackManifest.files.filter((f) => !f.sides),
 		modDestDirectory,
 	);
+}
 
+/**
+ * Downloads mods according to manifest.json and checks hashes.
+ */
+async function downloadClientMods(): Promise<void> {
 	logInfo("Fetching Client Mods...");
 	await fetchMods(
 		modpackManifest.files.filter((f) => f.sides && f.sides.includes("client")),
 		upath.join(modDestDirectory, "client"),
 	);
+}
 
+/**
+ * Downloads mods according to manifest.json and checks hashes.
+ */
+async function downloadServerMods(): Promise<void> {
 	logInfo("Fetching Server Mods...");
 	await fetchMods(
 		modpackManifest.files.filter((f) => f.sides && f.sides.includes("server")),
@@ -51,4 +61,14 @@ export async function downloadMods(): Promise<void> {
 	);
 }
 
-export default gulp.series(modCleanUp, createModDirs, downloadMods);
+export const downloadSharedAndServer = series(
+	modCleanUp,
+	createModDirs,
+	parallel(downloadMods, downloadServerMods),
+);
+
+export const downloadSharedAndClient = series(
+	modCleanUp,
+	createModDirs,
+	parallel(downloadMods, downloadClientMods),
+);

tools/tasks/misc/pruneCache.ts

@@ -2,7 +2,7 @@ import { modpackManifest } from "#globals";
 import { FORGE_MAVEN, getForgeJar, getVersionManifest } from "#utils/util.ts";
 import unzip from "unzipper";
 import { ForgeProfile } from "#types/forgeProfile.ts";
-import sha1 from "sha1";
+import { sha1 } from "hash-wasm";
 import { fetchFileInfo } from "#utils/curseForgeAPI.ts";
 import fs from "fs";
 import upath from "upath";
@@ -91,16 +91,14 @@ export default async function pruneCache(): Promise<void> {
 			.isFile(),
 	);
 
-	const shaMap: { [key: string]: boolean } = urls.reduce(
+	const shaMap: { [key: string]: boolean } = {};
-		(map: Record<string, boolean>, url) => {
+	const hashes = await Promise.all(urls.map((url) => sha1(url)));
-			map[sha1(url)] = true;
+	for (const hash of hashes) {
-			return map;
+		shaMap[hash] = true;
-		},
+	}
-		{},
-	);
 
-	let count = 0,
+	let count = 0;
-		bytes = 0;
+	let bytes = 0;
 
 	for (const sha of cache) {
 		if (!shaMap[sha]) {

tools/tasks/mmc/index.ts

@@ -1,11 +1,11 @@
 import {
-	clientDestDirectory,
 	mmcDestDirectory,
 	modDestDirectory,
 	modpackManifest,
+	sharedDestDirectory,
 } from "#globals";
-import * as upath from "upath";
+import upath from "upath";
-import * as fs from "fs";
+import fs from "fs";
 import { dest, series, src } from "gulp";
 import buildConfig from "#buildConfig";
 import { shouldSkipChangelog } from "#utils/util.ts";
@@ -56,20 +56,11 @@ async function copyMMCChangelog() {
  * Copies modpack overrides.
  */
 async function copyOverrides() {
-	return src(upath.join(clientDestDirectory, "**/*"), {
+	return src("**/*", {
-		resolveSymlinks: false,
+		resolveSymlinks: true,
-	}).pipe(dest(upath.join(mmcDestDirectory)));
+		encoding: false,
-}
+		cwd: upath.join(sharedDestDirectory, "overrides"),
-
+	}).pipe(dest(upath.join(mmcDestDirectory, ".minecraft")));
-/**
- * Renames copied overrides to '.minecraft'.
- */
-async function renameOverrides() {
-	await fs.promises.rename(
-		upath.join(mmcDestDirectory, "overrides"),
-		upath.join(mmcDestDirectory, ".minecraft"),
-	);
-	return fs.promises.rm(upath.join(mmcDestDirectory, "manifest.json"));
 }
 
 /**
@@ -149,7 +140,6 @@ export default series(
 	copyMMCLicense,
 	copyMMCUpdateNotes,
 	copyOverrides,
-	renameOverrides,
 	createMMCConfig,
 	createMMCManifest,
 	copyMMCModJars,

tools/utils/buildConfig.default.json

@@ -5,6 +5,7 @@
 	"downloaderCheckHashes": true,
 	"downloaderCacheDirectory": "../.cache",
 	"changelogCacheMaxPages": 5,
+	"changelogRequestRetryMaxSeconds": 60,
 	"launchscriptsMinRAM": "2048M",
 	"launchscriptsMaxRAM": "2048M",
 	"launchscriptsJVMArgs": "",

tools/utils/hashes.ts

@@ -1,6 +1,4 @@
-import _sha1 from "sha1";
+import { sha1, md5 } from "hash-wasm";
-import _md5 from "md5";
-
 import { HashDef } from "#types/hashDef.ts";
 
 /**
@@ -8,8 +6,8 @@ import { HashDef } from "#types/hashDef.ts";
  *
  * This is what CurseForge and Forge are using to check files.
  */
-export const sha1 = (inputBuffer: Buffer): string => {
+const performSha1 = (inputBuffer: Buffer): Promise<string> => {
-	return _sha1(inputBuffer);
+	return sha1(inputBuffer);
 };
 
 /**
@@ -17,13 +15,13 @@ export const sha1 = (inputBuffer: Buffer): string => {
  *
  * This is what CF is using to check files.
  */
-export const md5 = (inputBuffer: Buffer): string => {
+const performMd5 = (inputBuffer: Buffer): Promise<string> => {
-	return _md5(inputBuffer);
+	return md5(inputBuffer);
 };
 
-const hashFuncs: { [key: string]: (buffer: Buffer) => string } = {
+const hashFuncs: { [key: string]: (buffer: Buffer) => Promise<string> } = {
-	sha1,
+	sha1: performSha1,
-	md5,
+	md5: performMd5,
 };
 
 /**
@@ -34,17 +32,17 @@ const hashFuncs: { [key: string]: (buffer: Buffer) => string } = {
  *
  * @throws {Error} Throws a generic error if hashes don't match.
  */
-export const compareBufferToHashDef = (
+export async function compareBufferToHashDef(
 	buffer: Buffer,
 	hashDef: HashDef,
-): boolean => {
+): Promise<boolean> {
 	if (!hashFuncs[hashDef.id]) {
 		throw new Error(`No hash function found for ${hashDef.id}.`);
 	}
 
-	const sum = hashFuncs[hashDef.id](buffer);
+	const sum = await hashFuncs[hashDef.id](buffer);
 	return (
 		(Array.isArray(hashDef.hashes) && hashDef.hashes.includes(sum)) ||
 		hashDef.hashes == sum
 	);
-};
+}

tools/utils/util.ts

@@ -1,4 +1,4 @@
-import sha1 from "sha1";
+import { sha1 } from "hash-wasm";
 import { FileDef } from "#types/fileDef.ts";
 import fs from "fs";
 import buildConfig from "#buildConfig";
@@ -48,6 +48,7 @@ const LIBRARY_REG = /^(.+?):(.+?):(.+?)$/;
 export const git: SimpleGit = simpleGit(rootDirectory);
 
 const RetryOctokit = Octokit.plugin(retry, throttling);
+let shouldTryGetInfo = true;
 export const octokit = new RetryOctokit({
 	auth: process.env.GITHUB_TOKEN,
 	throttle: {
@@ -57,8 +58,15 @@ export const octokit = new RetryOctokit({
 			);
 
 			if (retryCount < buildConfig.downloaderMaxRetries) {
-				logInfo(`Retrying after ${retryAfter} seconds.`);
+				if (retryAfter < buildConfig.changelogRequestRetryMaxSeconds) {
-				return true;
+					logInfo(`Retrying after ${retryAfter} seconds.`);
+					return true;
+				}
+				logError(
+					`Reset Time of ${retryAfter} Seconds is Too Long! Not Retrying!`,
+				);
+				shouldTryGetInfo = false;
+				return false;
 			}
 		},
 		onSecondaryRateLimit: (retryAfter, options, _octokit, retryCount) => {
@@ -67,8 +75,15 @@ export const octokit = new RetryOctokit({
 			);
 
 			if (retryCount < buildConfig.downloaderMaxRetries) {
-				logInfo(`Retrying after ${retryAfter} seconds.`);
+				if (retryAfter < buildConfig.changelogRequestRetryMaxSeconds) {
-				return true;
+					logInfo(`Retrying after ${retryAfter} seconds.`);
+					return true;
+				}
+				logError(
+					`Reset Time of ${retryAfter} Seconds is Too Long! Not Retrying!`,
+				);
+				shouldTryGetInfo = false;
+				return false;
 			}
 		},
 	},
@@ -111,9 +126,12 @@ fileDownloader.interceptors.response.use(async (response) => {
 	if (!buffer)
 		throw new Error(`Failed to Download File from ${url}, no Buffer Returned!`);
 	if (nomiCfg.fileDef.hashes) {
-		const success = nomiCfg.fileDef.hashes.every((hashDef) => {
+		let success = true;
-			return compareBufferToHashDef(buffer, hashDef);
+		for (const hash of nomiCfg.fileDef.hashes) {
-		});
+			if (await compareBufferToHashDef(buffer, hash)) continue;
+			success = false;
+			break;
+		}
 		if (!success)
 			return retryOrThrow(
 				response,
@@ -197,7 +215,7 @@ export interface RetrievedFileDef {
 export async function downloadOrRetrieveFileDef(
 	fileDef: FileDef,
 ): Promise<RetrievedFileDef> {
-	const fileNameSha = sha1(fileDef.url);
+	const fileNameSha = await sha1(fileDef.url);
 
 	const cachedFilePath = upath.join(
 		buildConfig.downloaderCacheDirectory,
@@ -214,13 +232,13 @@ export async function downloadOrRetrieveFileDef(
 
 			// Check hashes.
 			if (fileDef.hashes) {
-				if (
+				let success = true;
-					fileDef.hashes.every((hashDef) => {
+				for (const hash of fileDef.hashes) {
-						return compareBufferToHashDef(file, hashDef);
+					if (await compareBufferToHashDef(file, hash)) continue;
-					})
+					success = false;
-				) {
+					break;
-					return rFileDef;
 				}
+				if (success) return rFileDef;
 			} else {
 				return rFileDef;
 			}
@@ -641,6 +659,10 @@ const issueURLCache: Map<number, string> = new Map<number, string>();
  */
 export async function getIssueURLs(): Promise<void> {
 	if (issueURLCache.size > 0) return;
+	if (!shouldTryGetInfo) {
+		logError("Skipping Get Issues because of Rate Limits!");
+		return;
+	}
 	try {
 		let page = 1;
 		const issues = await octokit.paginate(
@@ -679,6 +701,7 @@ export async function getIssueURLs(): Promise<void> {
 export async function getIssueURL(issueNumber: number): Promise<string> {
 	if (issueURLCache.has(issueNumber))
 		return issueURLCache.get(issueNumber) ?? "";
+	if (!shouldTryGetInfo) return "";
 	try {
 		// Try to retrieve, might be open
 		const issueInfo = await octokit.issues.get({
@@ -715,6 +738,10 @@ const commitAuthorCache: Map<string, string> = new Map<string, string>();
  */
 export async function getCommitAuthors(): Promise<void> {
 	if (commitAuthorCache.size > 0) return;
+	if (!shouldTryGetInfo) {
+		logError("Skipping Get Commits because of Rate Limits!");
+		return;
+	}
 	try {
 		let page = 1;
 		const commits = await octokit.paginate(
@@ -757,6 +784,8 @@ export async function formatAuthor(commit: Commit) {
 		return defaultFormat;
 	}
 
+	if (!shouldTryGetInfo) return defaultFormat;
+
 	try {
 		// Try to retrieve, just in case
 		const commitInfo = await octokit.repos.getCommit({
@@ -881,7 +910,6 @@ export function shouldSkipChangelog(): boolean {
 		throw new Error("Skip Changelog Env Variable set to Invalid Value.");
 	}
 
-	if (skip)
+	if (skip) logInfo("Skipping Changelogs...");
-		logInfo("Skipping Changelogs...");
 	return skip;
 }