From 322482299fb4edc8b88386be38dafa6948d3f3f1 Mon Sep 17 00:00:00 2001 From: tracer4b <61507029+tracer4b@users.noreply.github.com> Date: Thu, 7 Jul 2022 16:11:30 +0800 Subject: [PATCH] update buildscript to new cf protocol --- README.md | 8 +- buildtools/config.json | 4 +- buildtools/package-lock.json | 50 ++++++++++--- buildtools/package.json | 4 + buildtools/tasks/checks/index.ts | 1 + buildtools/tasks/deploy/curseforge.ts | 6 +- buildtools/tasks/mmc/index.ts | 2 +- buildtools/tasks/server/index.ts | 2 +- buildtools/types/curseForge.ts | 79 ++++++++++---------- buildtools/util/buildConfig.default.json | 1 + buildtools/util/curseForgeAPI.ts | 94 +++++++++++++++++++----- buildtools/util/hashes.ts | 92 ++++------------------- 12 files changed, 179 insertions(+), 164 deletions(-) diff --git a/README.md b/README.md index c405bf0..e1d9ba4 100644 --- a/README.md +++ b/README.md @@ -7,13 +7,7 @@ Port of [Nomifactory](https://github.com/Nomifactory/Nomifactory) 1.3 to the [Gr - Fixed lots of unpleasant CE-ness - no more stupid Amp system, different ore stone types don't clog up your inventory, etc. - Super-performant emissive effects on machines, coil blocks, the Fusion Reactor, and more - Updated questbook to guide you through all the new stuff -- lots more... - -Installation instructions for building the bleeding-edge version: -- Download [Node.js](https://nodejs.org/en/) -- Open a command line in `buildtools` -- Run `npm install` (only 1st time) -- Run `npx gulp buildAll` and `npx gulp zipAll` +- lots more... ## Hard mode info If you want a harder, or perhaps a more "true" GregTech experience, check out the Expert mode. This pack mode is based on the [Self-Torture Edition fork](https://github.com/NotMyWing/Omnifactory-Self-Torture-Edition) of the original pack. Highlights include: diff --git a/buildtools/config.json b/buildtools/config.json index e0553f0..d6b0995 100644 --- a/buildtools/config.json +++ b/buildtools/config.json @@ -7,6 +7,6 @@ "overrides/**/*", "!overrides/resources/minecraft/textures/gui/title/background/*" ], - "nightlyHookAvatar": "https://github.com/NomifactoryDevs.png", + "nightlyHookAvatar": "https://github.com/Nomifactory.png", "nightlyHookName": "Nightly Builds" -} \ No newline at end of file +} diff --git a/buildtools/package-lock.json b/buildtools/package-lock.json index e584c85..1e4eb02 100644 --- a/buildtools/package-lock.json +++ b/buildtools/package-lock.json @@ -8,6 +8,9 @@ "name": "nomifactory-buildtools", "version": "1.2.2", "license": "LGPL-3.0", + "dependencies": { + "md5": "^2.3.0" + }, "devDependencies": { "@octokit/rest": "^18.3.5", "@types/bluebird": "^3.5.33", @@ -16,6 +19,7 @@ "@types/gulp-imagemin": "^7.0.2", "@types/gulp-rename": "^2.0.0", "@types/gulp-zip": "^4.0.1", + "@types/md5": "^2.3.2", "@types/merge-stream": "^1.1.2", "@types/mustache": "^4.1.1", "@types/requestretry": "^1.12.7", @@ -517,6 +521,12 @@ "integrity": "sha512-qcUXuemtEu+E5wZSJHNxUXeCZhAfXKQ41D+duX+VYPde7xyEVZci+/oXKJL13tnRs9lR2pr4fod59GT6/X1/yQ==", "dev": true }, + "node_modules/@types/md5": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/@types/md5/-/md5-2.3.2.tgz", + "integrity": "sha512-v+JFDu96+UYJ3/UWzB0mEglIS//MZXgRaJ4ubUPwOM0gvLc/kcQ3TWNYwENEK7/EcXGQVrW8h/XqednSjBd/Og==", + "dev": true + }, "node_modules/@types/merge-stream": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@types/merge-stream/-/merge-stream-1.1.2.tgz", @@ -2128,7 +2138,6 @@ "version": "0.0.2", "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz", "integrity": "sha1-wKHS86cJLgN3S/qD8UwPxXkKhmc=", - "dev": true, "engines": { "node": "*" } @@ -2652,7 +2661,6 @@ "version": "0.0.2", "resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz", "integrity": "sha1-iNf/fsDfuG9xPch7u0LQRNPmxBs=", - "dev": true, "engines": { "node": "*" } @@ -6252,8 +6260,7 @@ "node_modules/is-buffer": { "version": "1.1.6", "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", - "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==", - "dev": true + "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==" }, "node_modules/is-callable": { "version": "1.2.4", @@ -7362,6 +7369,16 @@ "node": ">=0.10.0" } }, + "node_modules/md5": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz", + "integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==", + "dependencies": { + "charenc": "0.0.2", + "crypt": "0.0.2", + "is-buffer": "~1.1.6" + } + }, "node_modules/mdn-data": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.0.4.tgz", @@ -11802,6 +11819,12 @@ "integrity": "sha512-qcUXuemtEu+E5wZSJHNxUXeCZhAfXKQ41D+duX+VYPde7xyEVZci+/oXKJL13tnRs9lR2pr4fod59GT6/X1/yQ==", "dev": true }, + "@types/md5": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/@types/md5/-/md5-2.3.2.tgz", + "integrity": "sha512-v+JFDu96+UYJ3/UWzB0mEglIS//MZXgRaJ4ubUPwOM0gvLc/kcQ3TWNYwENEK7/EcXGQVrW8h/XqednSjBd/Og==", + "dev": true + }, "@types/merge-stream": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@types/merge-stream/-/merge-stream-1.1.2.tgz", @@ -13053,8 +13076,7 @@ "charenc": { "version": "0.0.2", "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz", - "integrity": "sha1-wKHS86cJLgN3S/qD8UwPxXkKhmc=", - "dev": true + "integrity": "sha1-wKHS86cJLgN3S/qD8UwPxXkKhmc=" }, "chokidar": { "version": "3.5.2", @@ -13491,8 +13513,7 @@ "crypt": { "version": "0.0.2", "resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz", - "integrity": "sha1-iNf/fsDfuG9xPch7u0LQRNPmxBs=", - "dev": true + "integrity": "sha1-iNf/fsDfuG9xPch7u0LQRNPmxBs=" }, "css-select": { "version": "2.1.0", @@ -16344,8 +16365,7 @@ "is-buffer": { "version": "1.1.6", "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", - "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==", - "dev": true + "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==" }, "is-callable": { "version": "1.2.4", @@ -17207,6 +17227,16 @@ } } }, + "md5": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz", + "integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==", + "requires": { + "charenc": "0.0.2", + "crypt": "0.0.2", + "is-buffer": "~1.1.6" + } + }, "mdn-data": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.0.4.tgz", diff --git a/buildtools/package.json b/buildtools/package.json index bd61f7f..f69f9f5 100644 --- a/buildtools/package.json +++ b/buildtools/package.json @@ -13,6 +13,7 @@ "@types/gulp-imagemin": "^7.0.2", "@types/gulp-rename": "^2.0.0", "@types/gulp-zip": "^4.0.1", + "@types/md5": "^2.3.2", "@types/merge-stream": "^1.1.2", "@types/mustache": "^4.1.1", "@types/requestretry": "^1.12.7", @@ -43,5 +44,8 @@ "typescript": "^4.2.3", "unzipper": "^0.10.11", "upath": "^2.0.1" + }, + "dependencies": { + "md5": "^2.3.0" } } diff --git a/buildtools/tasks/checks/index.ts b/buildtools/tasks/checks/index.ts index 8d0cfc9..e728c90 100644 --- a/buildtools/tasks/checks/index.ts +++ b/buildtools/tasks/checks/index.ts @@ -8,6 +8,7 @@ const vars = [ "GITHUB_REF", "CURSEFORGE_PROJECT_ID", "CURSEFORGE_API_TOKEN", + "CFCORE_API_TOKEN", ]; /** diff --git a/buildtools/tasks/deploy/curseforge.ts b/buildtools/tasks/deploy/curseforge.ts index 43f8463..9dba934 100644 --- a/buildtools/tasks/deploy/curseforge.ts +++ b/buildtools/tasks/deploy/curseforge.ts @@ -8,7 +8,7 @@ import buildConfig from "../../buildConfig"; import { makeArtifactNameBody } from "../../util/util"; import sanitize from "sanitize-filename"; -const CURSEFORGE_ENDPOINT = "https://minecraft.curseforge.com/"; +const CURSEFORGE_LEGACY_ENDPOINT = "https://minecraft.curseforge.com/"; const variablesToCheck = ["CURSEFORGE_API_TOKEN", "CURSEFORGE_PROJECT_ID", "GITHUB_TAG"]; /** @@ -63,7 +63,7 @@ async function deployCurseForge(): Promise { log("Fetching CurseForge version manifest..."); const versionsManifest = (await request({ - uri: CURSEFORGE_ENDPOINT + "api/game/versions", + uri: CURSEFORGE_LEGACY_ENDPOINT + "api/game/versions", headers: tokenHeaders, method: "GET", json: true, @@ -82,7 +82,7 @@ async function deployCurseForge(): Promise { // Upload artifacts. for (const file of files) { const options = { - uri: CURSEFORGE_ENDPOINT + `api/projects/${process.env.CURSEFORGE_PROJECT_ID}/upload-file`, + uri: CURSEFORGE_LEGACY_ENDPOINT + `api/projects/${process.env.CURSEFORGE_PROJECT_ID}/upload-file`, method: "POST", headers: { ...tokenHeaders, diff --git a/buildtools/tasks/mmc/index.ts b/buildtools/tasks/mmc/index.ts index b64c2f6..961dafb 100644 --- a/buildtools/tasks/mmc/index.ts +++ b/buildtools/tasks/mmc/index.ts @@ -27,7 +27,7 @@ async function createMMCDirs(cb) { * Copies modpack overrides. */ function copyOverrides() { - return src(upath.join(clientDestDirectory, "**/*.*"), { + return src(upath.join(clientDestDirectory, "**/*"), { nodir: true, resolveSymlinks: false, }).pipe(symlink(upath.join(mmcDestDirectory))); diff --git a/buildtools/tasks/server/index.ts b/buildtools/tasks/server/index.ts index 56e4006..0f97541 100644 --- a/buildtools/tasks/server/index.ts +++ b/buildtools/tasks/server/index.ts @@ -204,7 +204,7 @@ function copyServerfiles() { * Copies the license file. */ function copyServerLicense() { - return src("../LICENSE").pipe(dest(serverDestDirectory)); + return src("../LICENSE.md").pipe(dest(serverDestDirectory)); } /** diff --git a/buildtools/types/curseForge.ts b/buildtools/types/curseForge.ts index e63f246..7eded33 100644 --- a/buildtools/types/curseForge.ts +++ b/buildtools/types/curseForge.ts @@ -20,25 +20,6 @@ interface Attachment { status: number; } -interface Dependency { - id: number; - addonId: number; - type: number; - fileId: number; -} - -interface Module { - foldername: string; - type: number; -} - -interface SortableGameVersion { - gameVersionPadded: string; - gameVersion: string; - gameVersionReleaseDate: Date; - gameVersionName: string; -} - export interface CurseForgeFileInfo { id: number; displayName: string; @@ -129,32 +110,48 @@ export interface CurseForgeModInfo { isExperiemental: boolean; } +export interface CurseForgeFetchedFileInfo { + id: number; + gameId: number; + modId: number; + isAvailable: boolean; + displayName: string; + fileName: string; + releaseType: number; + fileStatus: number; + hashes?: Hash[]; + fileDate: Date; + fileLength: number; + downloadCount: number; + downloadUrl?: string; + gameVersions: string[]; + sortableGameVersions: SortableGameVersion[]; + dependencies: Dependency[]; + alternateFileId: number; + isServerPack: boolean; + fileFingerprint: number; + modules: Module[]; +} + interface Dependency { - addonId: number; - type: number; + modId: number; + relationType: number; +} + +interface Hash { + value: string; + algo: number; } interface Module { - foldername: string; - fingerprint: unknown; + name: string; + fingerprint: number; } -export interface CurseForgeFetchedFileInfo { - id: number; - displayName: string; - fileName: string; - fileDate: Date; - fileLength: number; - releaseType: number; - fileStatus: number; - downloadUrl: string; - isAlternate: boolean; - alternateFileId: number; - dependencies: Dependency[]; - isAvailable: boolean; - modules: Module[]; - packageFingerprint: number; - gameVersion: string[]; - hasInstallScript: boolean; - gameVersionDateReleased: Date; +interface SortableGameVersion { + gameVersionName: string; + gameVersionPadded: string; + gameVersion: string; + gameVersionReleaseDate: Date; + gameVersionTypeId: number; } diff --git a/buildtools/util/buildConfig.default.json b/buildtools/util/buildConfig.default.json index c1506fc..250cd6f 100644 --- a/buildtools/util/buildConfig.default.json +++ b/buildtools/util/buildConfig.default.json @@ -1,4 +1,5 @@ { + "cfCoreApiEndpoint": "https://curseforge.neeve.workers.dev", "downloaderMaxRetries": 5, "downloaderConcurrency": 50, "downloaderCheckHashes": true, diff --git a/buildtools/util/curseForgeAPI.ts b/buildtools/util/curseForgeAPI.ts index b3c6660..c0cacab 100644 --- a/buildtools/util/curseForgeAPI.ts +++ b/buildtools/util/curseForgeAPI.ts @@ -10,22 +10,43 @@ import fs from "fs"; import { FileDef } from "../types/fileDef"; import { downloadOrRetrieveFileDef, relative, RetrievedFileDefReason } from "./util"; +function getCurseForgeToken() { + const vari = "CFCORE_API_TOKEN"; + const val = process.env[vari]; + + if (!process.env[vari]) { + throw new Error(`Environmental variable ${vari} is unset.`); + } + + return val; +} + const curseForgeProjectCache: { [key: number]: CurseForgeProject } = {}; export async function fetchProject(toFetch: number): Promise { if (curseForgeProjectCache[toFetch]) { return curseForgeProjectCache[toFetch]; } - const project: CurseForgeProject = await request({ - uri: `https://addons-ecs.forgesvc.net/api/v2/addon/${toFetch}`, - json: true, - fullResponse: false, - maxAttempts: 5, - }); + const project: CurseForgeProject | undefined = ( + await request({ + uri: `${buildConfig.cfCoreApiEndpoint}/v1/mods/${toFetch}`, + json: true, + fullResponse: false, + maxAttempts: 5, + headers: { + "X-Api-Key": getCurseForgeToken(), + }, + }) + )?.data; + + if (!project) { + throw new Error(`Failed to fetch project ${toFetch}`); + } if (project) { curseForgeProjectCache[toFetch] = project; } + return project; } @@ -37,14 +58,29 @@ export async function fetchFileInfo(projectID: number, fileID: number): Promise< return fetchedFileInfoCache[slug]; } - const fileInfo: CurseForgeFetchedFileInfo = await request({ - uri: `https://addons-ecs.forgesvc.net/api/v2/addon/${projectID}/file/${fileID}`, - json: true, - fullResponse: false, - }); + const fileInfo: CurseForgeFetchedFileInfo = ( + await request({ + uri: `${buildConfig.cfCoreApiEndpoint}/v1/mods/${projectID}/files/${fileID}`, + json: true, + fullResponse: false, + headers: { + "X-Api-Key": getCurseForgeToken(), + }, + }) + )?.data; + + if (!fileInfo) { + throw new Error(`Failed to download file ${projectID}/file/${fileID}`); + } if (fileInfo) { fetchedFileInfoCache[slug] = fileInfo; + + if (!fileInfo.downloadUrl) { + const fid = `${Math.floor(fileInfo.id / 1000)}/${fileInfo.id % 1000}`; + + fileInfo.downloadUrl = `https://edge.forgecdn.net/files/${fid}/${fileInfo.fileName}`; + } } return fileInfo; @@ -73,12 +109,23 @@ export async function fetchProjectsBulk(toFetch: number[]): Promise 0) { // Augment the array of known projects with new info. - const fetched: CurseForgeProject[] = await request.post({ - uri: "https://addons-ecs.forgesvc.net/api/v2/addon/", - json: unfetched, - fullResponse: false, - maxAttempts: 5, - }); + const fetched: CurseForgeProject[] = ( + await request.post({ + uri: `${buildConfig.cfCoreApiEndpoint}/v1/mods`, + json: { + modIds: unfetched, + }, + fullResponse: false, + maxAttempts: 5, + headers: { + "X-Api-Key": getCurseForgeToken(), + }, + }) + )?.data; + + if (!fetched) { + throw new Error(`Failed to bulk-fetch projects ${unfetched.join(", ")}`); + } modInfos.push(...fetched); @@ -133,16 +180,23 @@ export async function fetchMods(toFetch: ModpackManifestFile[], destination: str const fileDef: FileDef = { url: fileInfo.downloadUrl, - hashes: [{ id: "murmurhash", hashes: fileInfo.packageFingerprint }], }; + // https://docs.curseforge.com/#tocS_GetModsByIdsListRequestBody + if (fileInfo.hashes) { + fileDef.hashes = fileInfo.hashes.map((hash) => ({ + hashes: hash.value, + id: hash.algo == 1 ? "sha1" : "md5", + })); + } + const modFile = await downloadOrRetrieveFileDef(fileDef); fetched += 1; if (modFile.reason == RetrievedFileDefReason.Downloaded) { - log(`Downloaded ${upath.basename(fileInfo.downloadUrl)}... (${fetched} / ${toFetch.length})`); + log(`Downloaded ${upath.basename(fileDef.url)}... (${fetched} / ${toFetch.length})`); } else if (modFile.reason == RetrievedFileDefReason.CacheHit) { - log(`Fetched ${upath.basename(fileInfo.downloadUrl)} from cache... (${fetched} / ${toFetch.length})`); + log(`Fetched ${upath.basename(fileDef.url)} from cache... (${fetched} / ${toFetch.length})`); } const dest = upath.join(destination, "mods", fileInfo.fileName); diff --git a/buildtools/util/hashes.ts b/buildtools/util/hashes.ts index 50899f9..b2428ff 100644 --- a/buildtools/util/hashes.ts +++ b/buildtools/util/hashes.ts @@ -1,95 +1,29 @@ -/** - * Bytes to exclude from hashing. - * - * Why? I dunno. - */ -const MURMUR_SKIP_BYTES: { [key: number]: boolean } = { - 9: true, - 10: true, - 13: true, - 32: true, -}; - import _sha1 from "sha1"; - -/** - * JS Implementation of MurmurHash2 - * - * @author Gary Court - * @see http://github.com/garycourt/murmurhash-js - * @author Austin Appleby - * @see http://sites.google.com/site/murmurhash/ - */ -function murmurhash2_32_gc(arr, seed, len = arr.length) { - let l = len, - h = seed ^ l, - i = 0, - k; - - while (l >= 4) { - k = (arr[i] & 0xff) | ((arr[++i] & 0xff) << 8) | ((arr[++i] & 0xff) << 16) | ((arr[++i] & 0xff) << 24); - - k = (k & 0xffff) * 0x5bd1e995 + ((((k >>> 16) * 0x5bd1e995) & 0xffff) << 16); - k ^= k >>> 24; - k = (k & 0xffff) * 0x5bd1e995 + ((((k >>> 16) * 0x5bd1e995) & 0xffff) << 16); - - h = ((h & 0xffff) * 0x5bd1e995 + ((((h >>> 16) * 0x5bd1e995) & 0xffff) << 16)) ^ k; - - l -= 4; - ++i; - } - - switch (l) { - case 3: - h ^= (arr[i + 2] & 0xff) << 16; - case 2: - h ^= (arr[i + 1] & 0xff) << 8; - case 1: - h ^= arr[i] & 0xff; - h = (h & 0xffff) * 0x5bd1e995 + ((((h >>> 16) * 0x5bd1e995) & 0xffff) << 16); - } - - h ^= h >>> 13; - h = (h & 0xffff) * 0x5bd1e995 + ((((h >>> 16) * 0x5bd1e995) & 0xffff) << 16); - h ^= h >>> 15; - - return h >>> 0; -} - -/** - * Returns the hash sum of bytes of given bytes using MurmurHash v2. - * - * This is what Twitch is using to fingerprint mod files. - */ -export const murmurhash = (inputBuffer: Buffer, seed = 1): string => { - const output = new Uint8Array(inputBuffer.length); - - let pos = 0; - for (let i = 0; i < inputBuffer.length; i++) { - const byte = inputBuffer.readUInt8(i); - - if (!MURMUR_SKIP_BYTES[byte]) { - output[pos++] = byte; - } - } - - return String(murmurhash2_32_gc(output, seed, pos)); -}; +import _md5 from "md5"; import { HashDef } from "../types/hashDef"; /** * Returns the hash sum of bytes of given bytes using SHA1. * - * This is what Forge is using to check files. + * This is what CurseForge and Forge are using to check files. */ export const sha1 = (inputBuffer: Buffer): string => { return _sha1(inputBuffer); }; +/** + * Returns the hash sum of bytes of given bytes using MD5. + * + * This is what CF is using to check files. + */ +export const md5 = (inputBuffer: Buffer): string => { + return _md5(inputBuffer); +}; + const hashFuncs: { [key: string]: (buffer: Buffer) => string } = { - murmurhash: murmurhash, - sha1: sha1, + sha1, + md5, }; /**