From 6baff4e57a7bd46d848e0fb2965fad26236ed1d8 Mon Sep 17 00:00:00 2001 From: Integer Limit <103940576+IntegerLimit@users.noreply.github.com> Date: Thu, 3 Oct 2024 15:55:39 +1000 Subject: [PATCH] Use Faster Hashing Library (#1021) [INTERNAL] --- tools/gulpfile.ts | 13 +++-- tools/package-lock.json | 75 +++------------------------- tools/package.json | 5 +- tools/tasks/misc/downloadMods.ts | 26 ++++++++-- tools/tasks/misc/pruneCache.ts | 18 +++---- tools/tasks/mmc/index.ts | 26 +++------- tools/utils/buildConfig.default.json | 1 + tools/utils/hashes.ts | 26 +++++----- tools/utils/util.ts | 62 ++++++++++++++++------- 9 files changed, 114 insertions(+), 138 deletions(-) diff --git a/tools/gulpfile.ts b/tools/gulpfile.ts index 850563a..e0c8257 100644 --- a/tools/gulpfile.ts +++ b/tools/gulpfile.ts @@ -19,7 +19,7 @@ import clientTasks from "./tasks/client/index.ts"; import serverTasks from "./tasks/server/index.ts"; import langTasks from "./tasks/lang/index.ts"; import mmcTasks from "./tasks/mmc/index.ts"; -import modTasks from "./tasks/misc/downloadMods.ts"; +import * as modTasks from "./tasks/misc/downloadMods.ts"; export const buildClient = gulp.series( sharedTasks, @@ -27,20 +27,23 @@ export const buildClient = gulp.series( pruneCacheTask, ); export const buildServer = gulp.series( - gulp.parallel(sharedTasks, modTasks), + gulp.parallel(sharedTasks, modTasks.downloadSharedAndServer), serverTasks, pruneCacheTask, ); export const buildLang = gulp.series(sharedTasks, langTasks, pruneCacheTask); export const buildMMC = gulp.series( - gulp.parallel(sharedTasks, modTasks), - clientTasks, + gulp.parallel(sharedTasks, modTasks.downloadSharedAndClient), mmcTasks, pruneCacheTask, ); export const buildAll = gulp.series( sharedTasks, - gulp.parallel(clientTasks, langTasks, gulp.series(modTasks, serverTasks)), + gulp.parallel( + clientTasks, + langTasks, + gulp.series(modTasks.downloadSharedAndServer, serverTasks), + ), pruneCacheTask, ); diff --git a/tools/package-lock.json b/tools/package-lock.json index f5c29d8..b8206b7 100644 --- a/tools/package-lock.json +++ b/tools/package-lock.json @@ -24,11 +24,9 @@ "@types/gulp-zip": "^4.0.4", "@types/inquirer": "^9.0.7", "@types/lodash": "^4.17.1", - "@types/md5": "^2.3.5", "@types/merge-stream": "^1.1.5", "@types/mustache": "^4.2.5", "@types/picomatch": "^2.3.3", - "@types/sha1": "^1.1.5", "@types/through2": "^2.0.41", "@types/unzipper": "^0.10.9", "@typescript-eslint/eslint-plugin": "^7.8.0", @@ -51,20 +49,19 @@ "gulp": "^5.0.0", "gulp-rename": "^2.0.0", "gulp-zip": "^6.0.0", + "hash-wasm": "^4.11.0", "inquirer": "^9.2.20", "javascript-stringify": "^2.1.0", "just-diff": "^6.0.2", "just-remove": "^3.2.0", "lodash": "^4.17.21", "marked": "^12.0.2", - "md5": "^2.3.0", "merge-stream": "^2.0.0", "mustache": "^4.2.0", "picomatch": "^4.0.2", "prettier": "^3.2.5", "retry-axios": "^3.1.3", "sanitize-filename": "^1.6.3", - "sha1": "^1.1.1", "simple-git": "^3.24.0", "sort-keys": "^5.0.0", "sort-keys-recursive": "^2.1.10", @@ -866,12 +863,6 @@ "integrity": "sha512-X+2qazGS3jxLAIz5JDXDzglAF3KpijdhFxlf/V1+hEsOUc+HnWi81L/uv/EvGuV90WY+7mPGFCUDGfQC3Gj95Q==", "dev": true }, - "node_modules/@types/md5": { - "version": "2.3.5", - "resolved": "https://registry.npmjs.org/@types/md5/-/md5-2.3.5.tgz", - "integrity": "sha512-/i42wjYNgE6wf0j2bcTX6kuowmdL/6PE4IVitMpm2eYKBUuYCprdcWVK+xEF0gcV6ufMCRhtxmReGfc6hIK7Jw==", - "dev": true - }, "node_modules/@types/merge-stream": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/@types/merge-stream/-/merge-stream-1.1.5.tgz", @@ -923,15 +914,6 @@ "integrity": "sha512-I8EUhyrgfLrcTkzV3TSsGyl1tSuPrEDzr0yd5m90UgNxQkyDXULk3b6MlQqTCpZpNtWe1K0hzclnZkTcLBe2UQ==", "dev": true }, - "node_modules/@types/sha1": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@types/sha1/-/sha1-1.1.5.tgz", - "integrity": "sha512-eE1PzjW7u2VfxI+bTsvjzjBfpwqvxSpgfUmnRNVY+PJU1NBsdGZlaO/qnVnPKHzzpgIl9YyBIxvrgBvt1mzt2A==", - "dev": true, - "dependencies": { - "@types/node": "*" - } - }, "node_modules/@types/streamx": { "version": "2.9.5", "resolved": "https://registry.npmjs.org/@types/streamx/-/streamx-2.9.5.tgz", @@ -1667,15 +1649,6 @@ "integrity": "sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==", "dev": true }, - "node_modules/charenc": { - "version": "0.0.2", - "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz", - "integrity": "sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==", - "dev": true, - "engines": { - "node": "*" - } - }, "node_modules/chokidar": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz", @@ -1902,15 +1875,6 @@ "node": ">= 8" } }, - "node_modules/crypt": { - "version": "0.0.2", - "resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz", - "integrity": "sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==", - "dev": true, - "engines": { - "node": "*" - } - }, "node_modules/debug": { "version": "4.3.4", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", @@ -3325,6 +3289,13 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/hash-wasm": { + "version": "4.11.0", + "resolved": "https://registry.npmjs.org/hash-wasm/-/hash-wasm-4.11.0.tgz", + "integrity": "sha512-HVusNXlVqHe0fzIzdQOGolnFN6mX/fqcrSAOcTBXdvzrXVHwTz11vXeKRmkR5gTuwVpvHZEIyKoePDvuAR+XwQ==", + "dev": true, + "license": "MIT" + }, "node_modules/hasown": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", @@ -3521,12 +3492,6 @@ "node": ">=8" } }, - "node_modules/is-buffer": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", - "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==", - "dev": true - }, "node_modules/is-core-module": { "version": "2.13.1", "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.13.1.tgz", @@ -3922,17 +3887,6 @@ "node": ">= 18" } }, - "node_modules/md5": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz", - "integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==", - "dev": true, - "dependencies": { - "charenc": "0.0.2", - "crypt": "0.0.2", - "is-buffer": "~1.1.6" - } - }, "node_modules/merge-stream": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", @@ -4778,19 +4732,6 @@ "node": ">= 0.4" } }, - "node_modules/sha1": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/sha1/-/sha1-1.1.1.tgz", - "integrity": "sha512-dZBS6OrMjtgVkopB1Gmo4RQCDKiZsqcpAQpkV/aaj+FCrCg8r4I4qMkDPQjBgLIxlmu9k4nUbWq6ohXahOneYA==", - "dev": true, - "dependencies": { - "charenc": ">= 0.0.1", - "crypt": ">= 0.0.1" - }, - "engines": { - "node": "*" - } - }, "node_modules/shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", diff --git a/tools/package.json b/tools/package.json index 7de0c2b..e5a9948 100644 --- a/tools/package.json +++ b/tools/package.json @@ -34,11 +34,9 @@ "@types/gulp-zip": "^4.0.4", "@types/inquirer": "^9.0.7", "@types/lodash": "^4.17.1", - "@types/md5": "^2.3.5", "@types/merge-stream": "^1.1.5", "@types/mustache": "^4.2.5", "@types/picomatch": "^2.3.3", - "@types/sha1": "^1.1.5", "@types/through2": "^2.0.41", "@types/unzipper": "^0.10.9", "@typescript-eslint/eslint-plugin": "^7.8.0", @@ -61,20 +59,19 @@ "gulp": "^5.0.0", "gulp-rename": "^2.0.0", "gulp-zip": "^6.0.0", + "hash-wasm": "^4.11.0", "inquirer": "^9.2.20", "javascript-stringify": "^2.1.0", "just-diff": "^6.0.2", "just-remove": "^3.2.0", "lodash": "^4.17.21", "marked": "^12.0.2", - "md5": "^2.3.0", "merge-stream": "^2.0.0", "mustache": "^4.2.0", "picomatch": "^4.0.2", "prettier": "^3.2.5", "retry-axios": "^3.1.3", "sanitize-filename": "^1.6.3", - "sha1": "^1.1.1", "simple-git": "^3.24.0", "sort-keys": "^5.0.0", "sort-keys-recursive": "^2.1.10", diff --git a/tools/tasks/misc/downloadMods.ts b/tools/tasks/misc/downloadMods.ts index fda6eda..e7323f6 100644 --- a/tools/tasks/misc/downloadMods.ts +++ b/tools/tasks/misc/downloadMods.ts @@ -3,7 +3,7 @@ import { fetchMods } from "#utils/curseForgeAPI.ts"; import upath from "upath"; import fs from "fs"; import { deleteAsync } from "del"; -import gulp from "gulp"; +import { parallel, series } from "gulp"; import logInfo from "#utils/log.ts"; async function modCleanUp() { @@ -31,19 +31,29 @@ async function createModDirs() { /** * Downloads mods according to manifest.json and checks hashes. */ -export async function downloadMods(): Promise { +async function downloadMods(): Promise { logInfo("Fetching Shared Mods..."); await fetchMods( modpackManifest.files.filter((f) => !f.sides), modDestDirectory, ); +} +/** + * Downloads mods according to manifest.json and checks hashes. + */ +async function downloadClientMods(): Promise { logInfo("Fetching Client Mods..."); await fetchMods( modpackManifest.files.filter((f) => f.sides && f.sides.includes("client")), upath.join(modDestDirectory, "client"), ); +} +/** + * Downloads mods according to manifest.json and checks hashes. + */ +async function downloadServerMods(): Promise { logInfo("Fetching Server Mods..."); await fetchMods( modpackManifest.files.filter((f) => f.sides && f.sides.includes("server")), @@ -51,4 +61,14 @@ export async function downloadMods(): Promise { ); } -export default gulp.series(modCleanUp, createModDirs, downloadMods); +export const downloadSharedAndServer = series( + modCleanUp, + createModDirs, + parallel(downloadMods, downloadServerMods), +); + +export const downloadSharedAndClient = series( + modCleanUp, + createModDirs, + parallel(downloadMods, downloadClientMods), +); diff --git a/tools/tasks/misc/pruneCache.ts b/tools/tasks/misc/pruneCache.ts index 1d07821..ea01c88 100644 --- a/tools/tasks/misc/pruneCache.ts +++ b/tools/tasks/misc/pruneCache.ts @@ -2,7 +2,7 @@ import { modpackManifest } from "#globals"; import { FORGE_MAVEN, getForgeJar, getVersionManifest } from "#utils/util.ts"; import unzip from "unzipper"; import { ForgeProfile } from "#types/forgeProfile.ts"; -import sha1 from "sha1"; +import { sha1 } from "hash-wasm"; import { fetchFileInfo } from "#utils/curseForgeAPI.ts"; import fs from "fs"; import upath from "upath"; @@ -91,16 +91,14 @@ export default async function pruneCache(): Promise { .isFile(), ); - const shaMap: { [key: string]: boolean } = urls.reduce( - (map: Record, url) => { - map[sha1(url)] = true; - return map; - }, - {}, - ); + const shaMap: { [key: string]: boolean } = {}; + const hashes = await Promise.all(urls.map((url) => sha1(url))); + for (const hash of hashes) { + shaMap[hash] = true; + } - let count = 0, - bytes = 0; + let count = 0; + let bytes = 0; for (const sha of cache) { if (!shaMap[sha]) { diff --git a/tools/tasks/mmc/index.ts b/tools/tasks/mmc/index.ts index a7cbf98..2d60e5e 100644 --- a/tools/tasks/mmc/index.ts +++ b/tools/tasks/mmc/index.ts @@ -1,11 +1,11 @@ import { - clientDestDirectory, mmcDestDirectory, modDestDirectory, modpackManifest, + sharedDestDirectory, } from "#globals"; -import * as upath from "upath"; -import * as fs from "fs"; +import upath from "upath"; +import fs from "fs"; import { dest, series, src } from "gulp"; import buildConfig from "#buildConfig"; import { shouldSkipChangelog } from "#utils/util.ts"; @@ -56,20 +56,11 @@ async function copyMMCChangelog() { * Copies modpack overrides. */ async function copyOverrides() { - return src(upath.join(clientDestDirectory, "**/*"), { - resolveSymlinks: false, - }).pipe(dest(upath.join(mmcDestDirectory))); -} - -/** - * Renames copied overrides to '.minecraft'. - */ -async function renameOverrides() { - await fs.promises.rename( - upath.join(mmcDestDirectory, "overrides"), - upath.join(mmcDestDirectory, ".minecraft"), - ); - return fs.promises.rm(upath.join(mmcDestDirectory, "manifest.json")); + return src("**/*", { + resolveSymlinks: true, + encoding: false, + cwd: upath.join(sharedDestDirectory, "overrides"), + }).pipe(dest(upath.join(mmcDestDirectory, ".minecraft"))); } /** @@ -149,7 +140,6 @@ export default series( copyMMCLicense, copyMMCUpdateNotes, copyOverrides, - renameOverrides, createMMCConfig, createMMCManifest, copyMMCModJars, diff --git a/tools/utils/buildConfig.default.json b/tools/utils/buildConfig.default.json index cb818c3..af4c08d 100644 --- a/tools/utils/buildConfig.default.json +++ b/tools/utils/buildConfig.default.json @@ -5,6 +5,7 @@ "downloaderCheckHashes": true, "downloaderCacheDirectory": "../.cache", "changelogCacheMaxPages": 5, + "changelogRequestRetryMaxSeconds": 60, "launchscriptsMinRAM": "2048M", "launchscriptsMaxRAM": "2048M", "launchscriptsJVMArgs": "", diff --git a/tools/utils/hashes.ts b/tools/utils/hashes.ts index 74b3d80..632d1ba 100644 --- a/tools/utils/hashes.ts +++ b/tools/utils/hashes.ts @@ -1,6 +1,4 @@ -import _sha1 from "sha1"; -import _md5 from "md5"; - +import { sha1, md5 } from "hash-wasm"; import { HashDef } from "#types/hashDef.ts"; /** @@ -8,8 +6,8 @@ import { HashDef } from "#types/hashDef.ts"; * * This is what CurseForge and Forge are using to check files. */ -export const sha1 = (inputBuffer: Buffer): string => { - return _sha1(inputBuffer); +const performSha1 = (inputBuffer: Buffer): Promise => { + return sha1(inputBuffer); }; /** @@ -17,13 +15,13 @@ export const sha1 = (inputBuffer: Buffer): string => { * * This is what CF is using to check files. */ -export const md5 = (inputBuffer: Buffer): string => { - return _md5(inputBuffer); +const performMd5 = (inputBuffer: Buffer): Promise => { + return md5(inputBuffer); }; -const hashFuncs: { [key: string]: (buffer: Buffer) => string } = { - sha1, - md5, +const hashFuncs: { [key: string]: (buffer: Buffer) => Promise } = { + sha1: performSha1, + md5: performMd5, }; /** @@ -34,17 +32,17 @@ const hashFuncs: { [key: string]: (buffer: Buffer) => string } = { * * @throws {Error} Throws a generic error if hashes don't match. */ -export const compareBufferToHashDef = ( +export async function compareBufferToHashDef( buffer: Buffer, hashDef: HashDef, -): boolean => { +): Promise { if (!hashFuncs[hashDef.id]) { throw new Error(`No hash function found for ${hashDef.id}.`); } - const sum = hashFuncs[hashDef.id](buffer); + const sum = await hashFuncs[hashDef.id](buffer); return ( (Array.isArray(hashDef.hashes) && hashDef.hashes.includes(sum)) || hashDef.hashes == sum ); -}; +} diff --git a/tools/utils/util.ts b/tools/utils/util.ts index 25a9b2e..db2f020 100644 --- a/tools/utils/util.ts +++ b/tools/utils/util.ts @@ -1,4 +1,4 @@ -import sha1 from "sha1"; +import { sha1 } from "hash-wasm"; import { FileDef } from "#types/fileDef.ts"; import fs from "fs"; import buildConfig from "#buildConfig"; @@ -48,6 +48,7 @@ const LIBRARY_REG = /^(.+?):(.+?):(.+?)$/; export const git: SimpleGit = simpleGit(rootDirectory); const RetryOctokit = Octokit.plugin(retry, throttling); +let shouldTryGetInfo = true; export const octokit = new RetryOctokit({ auth: process.env.GITHUB_TOKEN, throttle: { @@ -57,8 +58,15 @@ export const octokit = new RetryOctokit({ ); if (retryCount < buildConfig.downloaderMaxRetries) { - logInfo(`Retrying after ${retryAfter} seconds.`); - return true; + if (retryAfter < buildConfig.changelogRequestRetryMaxSeconds) { + logInfo(`Retrying after ${retryAfter} seconds.`); + return true; + } + logError( + `Reset Time of ${retryAfter} Seconds is Too Long! Not Retrying!`, + ); + shouldTryGetInfo = false; + return false; } }, onSecondaryRateLimit: (retryAfter, options, _octokit, retryCount) => { @@ -67,8 +75,15 @@ export const octokit = new RetryOctokit({ ); if (retryCount < buildConfig.downloaderMaxRetries) { - logInfo(`Retrying after ${retryAfter} seconds.`); - return true; + if (retryAfter < buildConfig.changelogRequestRetryMaxSeconds) { + logInfo(`Retrying after ${retryAfter} seconds.`); + return true; + } + logError( + `Reset Time of ${retryAfter} Seconds is Too Long! Not Retrying!`, + ); + shouldTryGetInfo = false; + return false; } }, }, @@ -111,9 +126,12 @@ fileDownloader.interceptors.response.use(async (response) => { if (!buffer) throw new Error(`Failed to Download File from ${url}, no Buffer Returned!`); if (nomiCfg.fileDef.hashes) { - const success = nomiCfg.fileDef.hashes.every((hashDef) => { - return compareBufferToHashDef(buffer, hashDef); - }); + let success = true; + for (const hash of nomiCfg.fileDef.hashes) { + if (await compareBufferToHashDef(buffer, hash)) continue; + success = false; + break; + } if (!success) return retryOrThrow( response, @@ -197,7 +215,7 @@ export interface RetrievedFileDef { export async function downloadOrRetrieveFileDef( fileDef: FileDef, ): Promise { - const fileNameSha = sha1(fileDef.url); + const fileNameSha = await sha1(fileDef.url); const cachedFilePath = upath.join( buildConfig.downloaderCacheDirectory, @@ -214,13 +232,13 @@ export async function downloadOrRetrieveFileDef( // Check hashes. if (fileDef.hashes) { - if ( - fileDef.hashes.every((hashDef) => { - return compareBufferToHashDef(file, hashDef); - }) - ) { - return rFileDef; + let success = true; + for (const hash of fileDef.hashes) { + if (await compareBufferToHashDef(file, hash)) continue; + success = false; + break; } + if (success) return rFileDef; } else { return rFileDef; } @@ -641,6 +659,10 @@ const issueURLCache: Map = new Map(); */ export async function getIssueURLs(): Promise { if (issueURLCache.size > 0) return; + if (!shouldTryGetInfo) { + logError("Skipping Get Issues because of Rate Limits!"); + return; + } try { let page = 1; const issues = await octokit.paginate( @@ -679,6 +701,7 @@ export async function getIssueURLs(): Promise { export async function getIssueURL(issueNumber: number): Promise { if (issueURLCache.has(issueNumber)) return issueURLCache.get(issueNumber) ?? ""; + if (!shouldTryGetInfo) return ""; try { // Try to retrieve, might be open const issueInfo = await octokit.issues.get({ @@ -715,6 +738,10 @@ const commitAuthorCache: Map = new Map(); */ export async function getCommitAuthors(): Promise { if (commitAuthorCache.size > 0) return; + if (!shouldTryGetInfo) { + logError("Skipping Get Commits because of Rate Limits!"); + return; + } try { let page = 1; const commits = await octokit.paginate( @@ -757,6 +784,8 @@ export async function formatAuthor(commit: Commit) { return defaultFormat; } + if (!shouldTryGetInfo) return defaultFormat; + try { // Try to retrieve, just in case const commitInfo = await octokit.repos.getCommit({ @@ -881,7 +910,6 @@ export function shouldSkipChangelog(): boolean { throw new Error("Skip Changelog Env Variable set to Invalid Value."); } - if (skip) - logInfo("Skipping Changelogs..."); + if (skip) logInfo("Skipping Changelogs..."); return skip; }