Use Faster Hashing Library (#1021)

[INTERNAL]
This commit is contained in:
Integer Limit 2024-10-03 15:55:39 +10:00 committed by GitHub
parent 8388e12cc6
commit 6baff4e57a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 114 additions and 138 deletions

View File

@ -19,7 +19,7 @@ import clientTasks from "./tasks/client/index.ts";
import serverTasks from "./tasks/server/index.ts";
import langTasks from "./tasks/lang/index.ts";
import mmcTasks from "./tasks/mmc/index.ts";
import modTasks from "./tasks/misc/downloadMods.ts";
import * as modTasks from "./tasks/misc/downloadMods.ts";
export const buildClient = gulp.series(
sharedTasks,
@ -27,20 +27,23 @@ export const buildClient = gulp.series(
pruneCacheTask,
);
export const buildServer = gulp.series(
gulp.parallel(sharedTasks, modTasks),
gulp.parallel(sharedTasks, modTasks.downloadSharedAndServer),
serverTasks,
pruneCacheTask,
);
export const buildLang = gulp.series(sharedTasks, langTasks, pruneCacheTask);
export const buildMMC = gulp.series(
gulp.parallel(sharedTasks, modTasks),
clientTasks,
gulp.parallel(sharedTasks, modTasks.downloadSharedAndClient),
mmcTasks,
pruneCacheTask,
);
export const buildAll = gulp.series(
sharedTasks,
gulp.parallel(clientTasks, langTasks, gulp.series(modTasks, serverTasks)),
gulp.parallel(
clientTasks,
langTasks,
gulp.series(modTasks.downloadSharedAndServer, serverTasks),
),
pruneCacheTask,
);

View File

@ -24,11 +24,9 @@
"@types/gulp-zip": "^4.0.4",
"@types/inquirer": "^9.0.7",
"@types/lodash": "^4.17.1",
"@types/md5": "^2.3.5",
"@types/merge-stream": "^1.1.5",
"@types/mustache": "^4.2.5",
"@types/picomatch": "^2.3.3",
"@types/sha1": "^1.1.5",
"@types/through2": "^2.0.41",
"@types/unzipper": "^0.10.9",
"@typescript-eslint/eslint-plugin": "^7.8.0",
@ -51,20 +49,19 @@
"gulp": "^5.0.0",
"gulp-rename": "^2.0.0",
"gulp-zip": "^6.0.0",
"hash-wasm": "^4.11.0",
"inquirer": "^9.2.20",
"javascript-stringify": "^2.1.0",
"just-diff": "^6.0.2",
"just-remove": "^3.2.0",
"lodash": "^4.17.21",
"marked": "^12.0.2",
"md5": "^2.3.0",
"merge-stream": "^2.0.0",
"mustache": "^4.2.0",
"picomatch": "^4.0.2",
"prettier": "^3.2.5",
"retry-axios": "^3.1.3",
"sanitize-filename": "^1.6.3",
"sha1": "^1.1.1",
"simple-git": "^3.24.0",
"sort-keys": "^5.0.0",
"sort-keys-recursive": "^2.1.10",
@ -866,12 +863,6 @@
"integrity": "sha512-X+2qazGS3jxLAIz5JDXDzglAF3KpijdhFxlf/V1+hEsOUc+HnWi81L/uv/EvGuV90WY+7mPGFCUDGfQC3Gj95Q==",
"dev": true
},
"node_modules/@types/md5": {
"version": "2.3.5",
"resolved": "https://registry.npmjs.org/@types/md5/-/md5-2.3.5.tgz",
"integrity": "sha512-/i42wjYNgE6wf0j2bcTX6kuowmdL/6PE4IVitMpm2eYKBUuYCprdcWVK+xEF0gcV6ufMCRhtxmReGfc6hIK7Jw==",
"dev": true
},
"node_modules/@types/merge-stream": {
"version": "1.1.5",
"resolved": "https://registry.npmjs.org/@types/merge-stream/-/merge-stream-1.1.5.tgz",
@ -923,15 +914,6 @@
"integrity": "sha512-I8EUhyrgfLrcTkzV3TSsGyl1tSuPrEDzr0yd5m90UgNxQkyDXULk3b6MlQqTCpZpNtWe1K0hzclnZkTcLBe2UQ==",
"dev": true
},
"node_modules/@types/sha1": {
"version": "1.1.5",
"resolved": "https://registry.npmjs.org/@types/sha1/-/sha1-1.1.5.tgz",
"integrity": "sha512-eE1PzjW7u2VfxI+bTsvjzjBfpwqvxSpgfUmnRNVY+PJU1NBsdGZlaO/qnVnPKHzzpgIl9YyBIxvrgBvt1mzt2A==",
"dev": true,
"dependencies": {
"@types/node": "*"
}
},
"node_modules/@types/streamx": {
"version": "2.9.5",
"resolved": "https://registry.npmjs.org/@types/streamx/-/streamx-2.9.5.tgz",
@ -1667,15 +1649,6 @@
"integrity": "sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==",
"dev": true
},
"node_modules/charenc": {
"version": "0.0.2",
"resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz",
"integrity": "sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==",
"dev": true,
"engines": {
"node": "*"
}
},
"node_modules/chokidar": {
"version": "3.6.0",
"resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz",
@ -1902,15 +1875,6 @@
"node": ">= 8"
}
},
"node_modules/crypt": {
"version": "0.0.2",
"resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz",
"integrity": "sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==",
"dev": true,
"engines": {
"node": "*"
}
},
"node_modules/debug": {
"version": "4.3.4",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
@ -3325,6 +3289,13 @@
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/hash-wasm": {
"version": "4.11.0",
"resolved": "https://registry.npmjs.org/hash-wasm/-/hash-wasm-4.11.0.tgz",
"integrity": "sha512-HVusNXlVqHe0fzIzdQOGolnFN6mX/fqcrSAOcTBXdvzrXVHwTz11vXeKRmkR5gTuwVpvHZEIyKoePDvuAR+XwQ==",
"dev": true,
"license": "MIT"
},
"node_modules/hasown": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz",
@ -3521,12 +3492,6 @@
"node": ">=8"
}
},
"node_modules/is-buffer": {
"version": "1.1.6",
"resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz",
"integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==",
"dev": true
},
"node_modules/is-core-module": {
"version": "2.13.1",
"resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.13.1.tgz",
@ -3922,17 +3887,6 @@
"node": ">= 18"
}
},
"node_modules/md5": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz",
"integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==",
"dev": true,
"dependencies": {
"charenc": "0.0.2",
"crypt": "0.0.2",
"is-buffer": "~1.1.6"
}
},
"node_modules/merge-stream": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz",
@ -4778,19 +4732,6 @@
"node": ">= 0.4"
}
},
"node_modules/sha1": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/sha1/-/sha1-1.1.1.tgz",
"integrity": "sha512-dZBS6OrMjtgVkopB1Gmo4RQCDKiZsqcpAQpkV/aaj+FCrCg8r4I4qMkDPQjBgLIxlmu9k4nUbWq6ohXahOneYA==",
"dev": true,
"dependencies": {
"charenc": ">= 0.0.1",
"crypt": ">= 0.0.1"
},
"engines": {
"node": "*"
}
},
"node_modules/shebang-command": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz",

View File

@ -34,11 +34,9 @@
"@types/gulp-zip": "^4.0.4",
"@types/inquirer": "^9.0.7",
"@types/lodash": "^4.17.1",
"@types/md5": "^2.3.5",
"@types/merge-stream": "^1.1.5",
"@types/mustache": "^4.2.5",
"@types/picomatch": "^2.3.3",
"@types/sha1": "^1.1.5",
"@types/through2": "^2.0.41",
"@types/unzipper": "^0.10.9",
"@typescript-eslint/eslint-plugin": "^7.8.0",
@ -61,20 +59,19 @@
"gulp": "^5.0.0",
"gulp-rename": "^2.0.0",
"gulp-zip": "^6.0.0",
"hash-wasm": "^4.11.0",
"inquirer": "^9.2.20",
"javascript-stringify": "^2.1.0",
"just-diff": "^6.0.2",
"just-remove": "^3.2.0",
"lodash": "^4.17.21",
"marked": "^12.0.2",
"md5": "^2.3.0",
"merge-stream": "^2.0.0",
"mustache": "^4.2.0",
"picomatch": "^4.0.2",
"prettier": "^3.2.5",
"retry-axios": "^3.1.3",
"sanitize-filename": "^1.6.3",
"sha1": "^1.1.1",
"simple-git": "^3.24.0",
"sort-keys": "^5.0.0",
"sort-keys-recursive": "^2.1.10",

View File

@ -3,7 +3,7 @@ import { fetchMods } from "#utils/curseForgeAPI.ts";
import upath from "upath";
import fs from "fs";
import { deleteAsync } from "del";
import gulp from "gulp";
import { parallel, series } from "gulp";
import logInfo from "#utils/log.ts";
async function modCleanUp() {
@ -31,19 +31,29 @@ async function createModDirs() {
/**
* Downloads mods according to manifest.json and checks hashes.
*/
export async function downloadMods(): Promise<void> {
async function downloadMods(): Promise<void> {
logInfo("Fetching Shared Mods...");
await fetchMods(
modpackManifest.files.filter((f) => !f.sides),
modDestDirectory,
);
}
/**
* Downloads mods according to manifest.json and checks hashes.
*/
async function downloadClientMods(): Promise<void> {
logInfo("Fetching Client Mods...");
await fetchMods(
modpackManifest.files.filter((f) => f.sides && f.sides.includes("client")),
upath.join(modDestDirectory, "client"),
);
}
/**
* Downloads mods according to manifest.json and checks hashes.
*/
async function downloadServerMods(): Promise<void> {
logInfo("Fetching Server Mods...");
await fetchMods(
modpackManifest.files.filter((f) => f.sides && f.sides.includes("server")),
@ -51,4 +61,14 @@ export async function downloadMods(): Promise<void> {
);
}
export default gulp.series(modCleanUp, createModDirs, downloadMods);
export const downloadSharedAndServer = series(
modCleanUp,
createModDirs,
parallel(downloadMods, downloadServerMods),
);
export const downloadSharedAndClient = series(
modCleanUp,
createModDirs,
parallel(downloadMods, downloadClientMods),
);

View File

@ -2,7 +2,7 @@ import { modpackManifest } from "#globals";
import { FORGE_MAVEN, getForgeJar, getVersionManifest } from "#utils/util.ts";
import unzip from "unzipper";
import { ForgeProfile } from "#types/forgeProfile.ts";
import sha1 from "sha1";
import { sha1 } from "hash-wasm";
import { fetchFileInfo } from "#utils/curseForgeAPI.ts";
import fs from "fs";
import upath from "upath";
@ -91,16 +91,14 @@ export default async function pruneCache(): Promise<void> {
.isFile(),
);
const shaMap: { [key: string]: boolean } = urls.reduce(
(map: Record<string, boolean>, url) => {
map[sha1(url)] = true;
return map;
},
{},
);
const shaMap: { [key: string]: boolean } = {};
const hashes = await Promise.all(urls.map((url) => sha1(url)));
for (const hash of hashes) {
shaMap[hash] = true;
}
let count = 0,
bytes = 0;
let count = 0;
let bytes = 0;
for (const sha of cache) {
if (!shaMap[sha]) {

View File

@ -1,11 +1,11 @@
import {
clientDestDirectory,
mmcDestDirectory,
modDestDirectory,
modpackManifest,
sharedDestDirectory,
} from "#globals";
import * as upath from "upath";
import * as fs from "fs";
import upath from "upath";
import fs from "fs";
import { dest, series, src } from "gulp";
import buildConfig from "#buildConfig";
import { shouldSkipChangelog } from "#utils/util.ts";
@ -56,20 +56,11 @@ async function copyMMCChangelog() {
* Copies modpack overrides.
*/
async function copyOverrides() {
return src(upath.join(clientDestDirectory, "**/*"), {
resolveSymlinks: false,
}).pipe(dest(upath.join(mmcDestDirectory)));
}
/**
* Renames copied overrides to '.minecraft'.
*/
async function renameOverrides() {
await fs.promises.rename(
upath.join(mmcDestDirectory, "overrides"),
upath.join(mmcDestDirectory, ".minecraft"),
);
return fs.promises.rm(upath.join(mmcDestDirectory, "manifest.json"));
return src("**/*", {
resolveSymlinks: true,
encoding: false,
cwd: upath.join(sharedDestDirectory, "overrides"),
}).pipe(dest(upath.join(mmcDestDirectory, ".minecraft")));
}
/**
@ -149,7 +140,6 @@ export default series(
copyMMCLicense,
copyMMCUpdateNotes,
copyOverrides,
renameOverrides,
createMMCConfig,
createMMCManifest,
copyMMCModJars,

View File

@ -5,6 +5,7 @@
"downloaderCheckHashes": true,
"downloaderCacheDirectory": "../.cache",
"changelogCacheMaxPages": 5,
"changelogRequestRetryMaxSeconds": 60,
"launchscriptsMinRAM": "2048M",
"launchscriptsMaxRAM": "2048M",
"launchscriptsJVMArgs": "",

View File

@ -1,6 +1,4 @@
import _sha1 from "sha1";
import _md5 from "md5";
import { sha1, md5 } from "hash-wasm";
import { HashDef } from "#types/hashDef.ts";
/**
@ -8,8 +6,8 @@ import { HashDef } from "#types/hashDef.ts";
*
* This is what CurseForge and Forge are using to check files.
*/
export const sha1 = (inputBuffer: Buffer): string => {
return _sha1(inputBuffer);
const performSha1 = (inputBuffer: Buffer): Promise<string> => {
return sha1(inputBuffer);
};
/**
@ -17,13 +15,13 @@ export const sha1 = (inputBuffer: Buffer): string => {
*
* This is what CF is using to check files.
*/
export const md5 = (inputBuffer: Buffer): string => {
return _md5(inputBuffer);
const performMd5 = (inputBuffer: Buffer): Promise<string> => {
return md5(inputBuffer);
};
const hashFuncs: { [key: string]: (buffer: Buffer) => string } = {
sha1,
md5,
const hashFuncs: { [key: string]: (buffer: Buffer) => Promise<string> } = {
sha1: performSha1,
md5: performMd5,
};
/**
@ -34,17 +32,17 @@ const hashFuncs: { [key: string]: (buffer: Buffer) => string } = {
*
* @throws {Error} Throws a generic error if hashes don't match.
*/
export const compareBufferToHashDef = (
export async function compareBufferToHashDef(
buffer: Buffer,
hashDef: HashDef,
): boolean => {
): Promise<boolean> {
if (!hashFuncs[hashDef.id]) {
throw new Error(`No hash function found for ${hashDef.id}.`);
}
const sum = hashFuncs[hashDef.id](buffer);
const sum = await hashFuncs[hashDef.id](buffer);
return (
(Array.isArray(hashDef.hashes) && hashDef.hashes.includes(sum)) ||
hashDef.hashes == sum
);
};
}

View File

@ -1,4 +1,4 @@
import sha1 from "sha1";
import { sha1 } from "hash-wasm";
import { FileDef } from "#types/fileDef.ts";
import fs from "fs";
import buildConfig from "#buildConfig";
@ -48,6 +48,7 @@ const LIBRARY_REG = /^(.+?):(.+?):(.+?)$/;
export const git: SimpleGit = simpleGit(rootDirectory);
const RetryOctokit = Octokit.plugin(retry, throttling);
let shouldTryGetInfo = true;
export const octokit = new RetryOctokit({
auth: process.env.GITHUB_TOKEN,
throttle: {
@ -57,9 +58,16 @@ export const octokit = new RetryOctokit({
);
if (retryCount < buildConfig.downloaderMaxRetries) {
if (retryAfter < buildConfig.changelogRequestRetryMaxSeconds) {
logInfo(`Retrying after ${retryAfter} seconds.`);
return true;
}
logError(
`Reset Time of ${retryAfter} Seconds is Too Long! Not Retrying!`,
);
shouldTryGetInfo = false;
return false;
}
},
onSecondaryRateLimit: (retryAfter, options, _octokit, retryCount) => {
logError(
@ -67,9 +75,16 @@ export const octokit = new RetryOctokit({
);
if (retryCount < buildConfig.downloaderMaxRetries) {
if (retryAfter < buildConfig.changelogRequestRetryMaxSeconds) {
logInfo(`Retrying after ${retryAfter} seconds.`);
return true;
}
logError(
`Reset Time of ${retryAfter} Seconds is Too Long! Not Retrying!`,
);
shouldTryGetInfo = false;
return false;
}
},
},
});
@ -111,9 +126,12 @@ fileDownloader.interceptors.response.use(async (response) => {
if (!buffer)
throw new Error(`Failed to Download File from ${url}, no Buffer Returned!`);
if (nomiCfg.fileDef.hashes) {
const success = nomiCfg.fileDef.hashes.every((hashDef) => {
return compareBufferToHashDef(buffer, hashDef);
});
let success = true;
for (const hash of nomiCfg.fileDef.hashes) {
if (await compareBufferToHashDef(buffer, hash)) continue;
success = false;
break;
}
if (!success)
return retryOrThrow(
response,
@ -197,7 +215,7 @@ export interface RetrievedFileDef {
export async function downloadOrRetrieveFileDef(
fileDef: FileDef,
): Promise<RetrievedFileDef> {
const fileNameSha = sha1(fileDef.url);
const fileNameSha = await sha1(fileDef.url);
const cachedFilePath = upath.join(
buildConfig.downloaderCacheDirectory,
@ -214,13 +232,13 @@ export async function downloadOrRetrieveFileDef(
// Check hashes.
if (fileDef.hashes) {
if (
fileDef.hashes.every((hashDef) => {
return compareBufferToHashDef(file, hashDef);
})
) {
return rFileDef;
let success = true;
for (const hash of fileDef.hashes) {
if (await compareBufferToHashDef(file, hash)) continue;
success = false;
break;
}
if (success) return rFileDef;
} else {
return rFileDef;
}
@ -641,6 +659,10 @@ const issueURLCache: Map<number, string> = new Map<number, string>();
*/
export async function getIssueURLs(): Promise<void> {
if (issueURLCache.size > 0) return;
if (!shouldTryGetInfo) {
logError("Skipping Get Issues because of Rate Limits!");
return;
}
try {
let page = 1;
const issues = await octokit.paginate(
@ -679,6 +701,7 @@ export async function getIssueURLs(): Promise<void> {
export async function getIssueURL(issueNumber: number): Promise<string> {
if (issueURLCache.has(issueNumber))
return issueURLCache.get(issueNumber) ?? "";
if (!shouldTryGetInfo) return "";
try {
// Try to retrieve, might be open
const issueInfo = await octokit.issues.get({
@ -715,6 +738,10 @@ const commitAuthorCache: Map<string, string> = new Map<string, string>();
*/
export async function getCommitAuthors(): Promise<void> {
if (commitAuthorCache.size > 0) return;
if (!shouldTryGetInfo) {
logError("Skipping Get Commits because of Rate Limits!");
return;
}
try {
let page = 1;
const commits = await octokit.paginate(
@ -757,6 +784,8 @@ export async function formatAuthor(commit: Commit) {
return defaultFormat;
}
if (!shouldTryGetInfo) return defaultFormat;
try {
// Try to retrieve, just in case
const commitInfo = await octokit.repos.getCommit({
@ -881,7 +910,6 @@ export function shouldSkipChangelog(): boolean {
throw new Error("Skip Changelog Env Variable set to Invalid Value.");
}
if (skip)
logInfo("Skipping Changelogs...");
if (skip) logInfo("Skipping Changelogs...");
return skip;
}