update buildscript to new cf protocol

README.md

@@ -7,13 +7,7 @@ Port of [Nomifactory](https://github.com/Nomifactory/Nomifactory) 1.3 to the [Gr
 - Fixed lots of unpleasant CE-ness - no more stupid Amp system, different ore stone types don't clog up your inventory, etc.
 - Super-performant emissive effects on machines, coil blocks, the Fusion Reactor, and more
 - Updated questbook to guide you through all the new stuff
-- lots more...
+- lots more...   
-
-Installation instructions for building the bleeding-edge version:  
-- Download [Node.js](https://nodejs.org/en/)  
-- Open a command line in `buildtools`  
-- Run `npm install` (only 1st time)  
-- Run `npx gulp buildAll` and `npx gulp zipAll`  
 
 ## Hard mode info
 If you want a harder, or perhaps a more "true" GregTech experience, check out the Expert mode. This pack mode is based on the [Self-Torture Edition fork](https://github.com/NotMyWing/Omnifactory-Self-Torture-Edition) of the original pack. Highlights include:

buildtools/config.json

@@ -7,6 +7,6 @@
 		"overrides/**/*",
 		"!overrides/resources/minecraft/textures/gui/title/background/*"
 	],
-	"nightlyHookAvatar": "https://github.com/NomifactoryDevs.png",
+	"nightlyHookAvatar": "https://github.com/Nomifactory.png",
 	"nightlyHookName": "Nightly Builds"
-}
+}

buildtools/package-lock.json

@@ -8,6 +8,9 @@
 			"name": "nomifactory-buildtools",
 			"version": "1.2.2",
 			"license": "LGPL-3.0",
+			"dependencies": {
+				"md5": "^2.3.0"
+			},
 			"devDependencies": {
 				"@octokit/rest": "^18.3.5",
 				"@types/bluebird": "^3.5.33",
@@ -16,6 +19,7 @@
 				"@types/gulp-imagemin": "^7.0.2",
 				"@types/gulp-rename": "^2.0.0",
 				"@types/gulp-zip": "^4.0.1",
+				"@types/md5": "^2.3.2",
 				"@types/merge-stream": "^1.1.2",
 				"@types/mustache": "^4.1.1",
 				"@types/requestretry": "^1.12.7",
@@ -517,6 +521,12 @@
 			"integrity": "sha512-qcUXuemtEu+E5wZSJHNxUXeCZhAfXKQ41D+duX+VYPde7xyEVZci+/oXKJL13tnRs9lR2pr4fod59GT6/X1/yQ==",
 			"dev": true
 		},
+		"node_modules/@types/md5": {
+			"version": "2.3.2",
+			"resolved": "https://registry.npmjs.org/@types/md5/-/md5-2.3.2.tgz",
+			"integrity": "sha512-v+JFDu96+UYJ3/UWzB0mEglIS//MZXgRaJ4ubUPwOM0gvLc/kcQ3TWNYwENEK7/EcXGQVrW8h/XqednSjBd/Og==",
+			"dev": true
+		},
 		"node_modules/@types/merge-stream": {
 			"version": "1.1.2",
 			"resolved": "https://registry.npmjs.org/@types/merge-stream/-/merge-stream-1.1.2.tgz",
@@ -2128,7 +2138,6 @@
 			"version": "0.0.2",
 			"resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz",
 			"integrity": "sha1-wKHS86cJLgN3S/qD8UwPxXkKhmc=",
-			"dev": true,
 			"engines": {
 				"node": "*"
 			}
@@ -2652,7 +2661,6 @@
 			"version": "0.0.2",
 			"resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz",
 			"integrity": "sha1-iNf/fsDfuG9xPch7u0LQRNPmxBs=",
-			"dev": true,
 			"engines": {
 				"node": "*"
 			}
@@ -6252,8 +6260,7 @@
 		"node_modules/is-buffer": {
 			"version": "1.1.6",
 			"resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz",
-			"integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==",
+			"integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w=="
-			"dev": true
 		},
 		"node_modules/is-callable": {
 			"version": "1.2.4",
@@ -7362,6 +7369,16 @@
 				"node": ">=0.10.0"
 			}
 		},
+		"node_modules/md5": {
+			"version": "2.3.0",
+			"resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz",
+			"integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==",
+			"dependencies": {
+				"charenc": "0.0.2",
+				"crypt": "0.0.2",
+				"is-buffer": "~1.1.6"
+			}
+		},
 		"node_modules/mdn-data": {
 			"version": "2.0.4",
 			"resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.0.4.tgz",
@@ -11802,6 +11819,12 @@
 			"integrity": "sha512-qcUXuemtEu+E5wZSJHNxUXeCZhAfXKQ41D+duX+VYPde7xyEVZci+/oXKJL13tnRs9lR2pr4fod59GT6/X1/yQ==",
 			"dev": true
 		},
+		"@types/md5": {
+			"version": "2.3.2",
+			"resolved": "https://registry.npmjs.org/@types/md5/-/md5-2.3.2.tgz",
+			"integrity": "sha512-v+JFDu96+UYJ3/UWzB0mEglIS//MZXgRaJ4ubUPwOM0gvLc/kcQ3TWNYwENEK7/EcXGQVrW8h/XqednSjBd/Og==",
+			"dev": true
+		},
 		"@types/merge-stream": {
 			"version": "1.1.2",
 			"resolved": "https://registry.npmjs.org/@types/merge-stream/-/merge-stream-1.1.2.tgz",
@@ -13053,8 +13076,7 @@
 		"charenc": {
 			"version": "0.0.2",
 			"resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz",
-			"integrity": "sha1-wKHS86cJLgN3S/qD8UwPxXkKhmc=",
+			"integrity": "sha1-wKHS86cJLgN3S/qD8UwPxXkKhmc="
-			"dev": true
 		},
 		"chokidar": {
 			"version": "3.5.2",
@@ -13491,8 +13513,7 @@
 		"crypt": {
 			"version": "0.0.2",
 			"resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz",
-			"integrity": "sha1-iNf/fsDfuG9xPch7u0LQRNPmxBs=",
+			"integrity": "sha1-iNf/fsDfuG9xPch7u0LQRNPmxBs="
-			"dev": true
 		},
 		"css-select": {
 			"version": "2.1.0",
@@ -16344,8 +16365,7 @@
 		"is-buffer": {
 			"version": "1.1.6",
 			"resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz",
-			"integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==",
+			"integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w=="
-			"dev": true
 		},
 		"is-callable": {
 			"version": "1.2.4",
@@ -17207,6 +17227,16 @@
 				}
 			}
 		},
+		"md5": {
+			"version": "2.3.0",
+			"resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz",
+			"integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==",
+			"requires": {
+				"charenc": "0.0.2",
+				"crypt": "0.0.2",
+				"is-buffer": "~1.1.6"
+			}
+		},
 		"mdn-data": {
 			"version": "2.0.4",
 			"resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.0.4.tgz",

buildtools/package.json

@@ -13,6 +13,7 @@
 		"@types/gulp-imagemin": "^7.0.2",
 		"@types/gulp-rename": "^2.0.0",
 		"@types/gulp-zip": "^4.0.1",
+		"@types/md5": "^2.3.2",
 		"@types/merge-stream": "^1.1.2",
 		"@types/mustache": "^4.1.1",
 		"@types/requestretry": "^1.12.7",
@@ -43,5 +44,8 @@
 		"typescript": "^4.2.3",
 		"unzipper": "^0.10.11",
 		"upath": "^2.0.1"
+	},
+	"dependencies": {
+		"md5": "^2.3.0"
 	}
 }

buildtools/tasks/checks/index.ts

@@ -8,6 +8,7 @@ const vars = [
 	"GITHUB_REF",
 	"CURSEFORGE_PROJECT_ID",
 	"CURSEFORGE_API_TOKEN",
+	"CFCORE_API_TOKEN",
 ];
 
 /**

buildtools/tasks/deploy/curseforge.ts

@@ -8,7 +8,7 @@ import buildConfig from "../../buildConfig";
 import { makeArtifactNameBody } from "../../util/util";
 import sanitize from "sanitize-filename";
 
-const CURSEFORGE_ENDPOINT = "https://minecraft.curseforge.com/";
+const CURSEFORGE_LEGACY_ENDPOINT = "https://minecraft.curseforge.com/";
 const variablesToCheck = ["CURSEFORGE_API_TOKEN", "CURSEFORGE_PROJECT_ID", "GITHUB_TAG"];
 
 /**
@@ -63,7 +63,7 @@ async function deployCurseForge(): Promise<void> {
 	log("Fetching CurseForge version manifest...");
 	const versionsManifest =
 		(await request({
-			uri: CURSEFORGE_ENDPOINT + "api/game/versions",
+			uri: CURSEFORGE_LEGACY_ENDPOINT + "api/game/versions",
 			headers: tokenHeaders,
 			method: "GET",
 			json: true,
@@ -82,7 +82,7 @@ async function deployCurseForge(): Promise<void> {
 	// Upload artifacts.
 	for (const file of files) {
 		const options = {
-			uri: CURSEFORGE_ENDPOINT + `api/projects/${process.env.CURSEFORGE_PROJECT_ID}/upload-file`,
+			uri: CURSEFORGE_LEGACY_ENDPOINT + `api/projects/${process.env.CURSEFORGE_PROJECT_ID}/upload-file`,
 			method: "POST",
 			headers: {
 				...tokenHeaders,

buildtools/tasks/mmc/index.ts

@@ -27,7 +27,7 @@ async function createMMCDirs(cb) {
  * Copies modpack overrides.
  */
 function copyOverrides() {
-	return src(upath.join(clientDestDirectory, "**/*.*"), {
+	return src(upath.join(clientDestDirectory, "**/*"), {
 		nodir: true,
 		resolveSymlinks: false,
 	}).pipe(symlink(upath.join(mmcDestDirectory)));

buildtools/tasks/server/index.ts

@@ -204,7 +204,7 @@ function copyServerfiles() {
  * Copies the license file.
  */
 function copyServerLicense() {
-	return src("../LICENSE").pipe(dest(serverDestDirectory));
+	return src("../LICENSE.md").pipe(dest(serverDestDirectory));
 }
 
 /**

buildtools/types/curseForge.ts

@@ -20,25 +20,6 @@ interface Attachment {
 	status: number;
 }
 
-interface Dependency {
-	id: number;
-	addonId: number;
-	type: number;
-	fileId: number;
-}
-
-interface Module {
-	foldername: string;
-	type: number;
-}
-
-interface SortableGameVersion {
-	gameVersionPadded: string;
-	gameVersion: string;
-	gameVersionReleaseDate: Date;
-	gameVersionName: string;
-}
-
 export interface CurseForgeFileInfo {
 	id: number;
 	displayName: string;
@@ -129,32 +110,48 @@ export interface CurseForgeModInfo {
 	isExperiemental: boolean;
 }
 
+export interface CurseForgeFetchedFileInfo {
+	id: number;
+	gameId: number;
+	modId: number;
+	isAvailable: boolean;
+	displayName: string;
+	fileName: string;
+	releaseType: number;
+	fileStatus: number;
+	hashes?: Hash[];
+	fileDate: Date;
+	fileLength: number;
+	downloadCount: number;
+	downloadUrl?: string;
+	gameVersions: string[];
+	sortableGameVersions: SortableGameVersion[];
+	dependencies: Dependency[];
+	alternateFileId: number;
+	isServerPack: boolean;
+	fileFingerprint: number;
+	modules: Module[];
+}
+
 interface Dependency {
-	addonId: number;
+	modId: number;
-	type: number;
+	relationType: number;
+}
+
+interface Hash {
+	value: string;
+	algo: number;
 }
 
 interface Module {
-	foldername: string;
+	name: string;
-	fingerprint: unknown;
+	fingerprint: number;
 }
 
-export interface CurseForgeFetchedFileInfo {
+interface SortableGameVersion {
-	id: number;
+	gameVersionName: string;
-	displayName: string;
+	gameVersionPadded: string;
-	fileName: string;
+	gameVersion: string;
-	fileDate: Date;
+	gameVersionReleaseDate: Date;
-	fileLength: number;
+	gameVersionTypeId: number;
-	releaseType: number;
-	fileStatus: number;
-	downloadUrl: string;
-	isAlternate: boolean;
-	alternateFileId: number;
-	dependencies: Dependency[];
-	isAvailable: boolean;
-	modules: Module[];
-	packageFingerprint: number;
-	gameVersion: string[];
-	hasInstallScript: boolean;
-	gameVersionDateReleased: Date;
 }

buildtools/util/buildConfig.default.json

@@ -1,4 +1,5 @@
 {
+	"cfCoreApiEndpoint": "https://curseforge.neeve.workers.dev",
 	"downloaderMaxRetries": 5,
 	"downloaderConcurrency": 50,
 	"downloaderCheckHashes": true,

buildtools/util/curseForgeAPI.ts

@@ -10,22 +10,43 @@ import fs from "fs";
 import { FileDef } from "../types/fileDef";
 import { downloadOrRetrieveFileDef, relative, RetrievedFileDefReason } from "./util";
 
+function getCurseForgeToken() {
+	const vari = "CFCORE_API_TOKEN";
+	const val = process.env[vari];
+
+	if (!process.env[vari]) {
+		throw new Error(`Environmental variable ${vari} is unset.`);
+	}
+
+	return val;
+}
+
 const curseForgeProjectCache: { [key: number]: CurseForgeProject } = {};
 export async function fetchProject(toFetch: number): Promise<CurseForgeProject> {
 	if (curseForgeProjectCache[toFetch]) {
 		return curseForgeProjectCache[toFetch];
 	}
 
-	const project: CurseForgeProject = await request({
+	const project: CurseForgeProject | undefined = (
-		uri: `https://addons-ecs.forgesvc.net/api/v2/addon/${toFetch}`,
+		await request({
-		json: true,
+			uri: `${buildConfig.cfCoreApiEndpoint}/v1/mods/${toFetch}`,
-		fullResponse: false,
+			json: true,
-		maxAttempts: 5,
+			fullResponse: false,
-	});
+			maxAttempts: 5,
+			headers: {
+				"X-Api-Key": getCurseForgeToken(),
+			},
+		})
+	)?.data;
+
+	if (!project) {
+		throw new Error(`Failed to fetch project ${toFetch}`);
+	}
 
 	if (project) {
 		curseForgeProjectCache[toFetch] = project;
 	}
+
 	return project;
 }
 
@@ -37,14 +58,29 @@ export async function fetchFileInfo(projectID: number, fileID: number): Promise<
 		return fetchedFileInfoCache[slug];
 	}
 
-	const fileInfo: CurseForgeFetchedFileInfo = await request({
+	const fileInfo: CurseForgeFetchedFileInfo = (
-		uri: `https://addons-ecs.forgesvc.net/api/v2/addon/${projectID}/file/${fileID}`,
+		await request({
-		json: true,
+			uri: `${buildConfig.cfCoreApiEndpoint}/v1/mods/${projectID}/files/${fileID}`,
-		fullResponse: false,
+			json: true,
-	});
+			fullResponse: false,
+			headers: {
+				"X-Api-Key": getCurseForgeToken(),
+			},
+		})
+	)?.data;
+
+	if (!fileInfo) {
+		throw new Error(`Failed to download file ${projectID}/file/${fileID}`);
+	}
 
 	if (fileInfo) {
 		fetchedFileInfoCache[slug] = fileInfo;
+
+		if (!fileInfo.downloadUrl) {
+			const fid = `${Math.floor(fileInfo.id / 1000)}/${fileInfo.id % 1000}`;
+
+			fileInfo.downloadUrl = `https://edge.forgecdn.net/files/${fid}/${fileInfo.fileName}`;
+		}
 	}
 
 	return fileInfo;
@@ -73,12 +109,23 @@ export async function fetchProjectsBulk(toFetch: number[]): Promise<CurseForgePr
 
 	if (unfetched.length > 0) {
 		// Augment the array of known projects with new info.
-		const fetched: CurseForgeProject[] = await request.post({
+		const fetched: CurseForgeProject[] = (
-			uri: "https://addons-ecs.forgesvc.net/api/v2/addon/",
+			await request.post({
-			json: unfetched,
+				uri: `${buildConfig.cfCoreApiEndpoint}/v1/mods`,
-			fullResponse: false,
+				json: {
-			maxAttempts: 5,
+					modIds: unfetched,
-		});
+				},
+				fullResponse: false,
+				maxAttempts: 5,
+				headers: {
+					"X-Api-Key": getCurseForgeToken(),
+				},
+			})
+		)?.data;
+
+		if (!fetched) {
+			throw new Error(`Failed to bulk-fetch projects ${unfetched.join(", ")}`);
+		}
 
 		modInfos.push(...fetched);
 
@@ -133,16 +180,23 @@ export async function fetchMods(toFetch: ModpackManifestFile[], destination: str
 
 				const fileDef: FileDef = {
 					url: fileInfo.downloadUrl,
-					hashes: [{ id: "murmurhash", hashes: fileInfo.packageFingerprint }],
 				};
 
+				// https://docs.curseforge.com/#tocS_GetModsByIdsListRequestBody
+				if (fileInfo.hashes) {
+					fileDef.hashes = fileInfo.hashes.map((hash) => ({
+						hashes: hash.value,
+						id: hash.algo == 1 ? "sha1" : "md5",
+					}));
+				}
+
 				const modFile = await downloadOrRetrieveFileDef(fileDef);
 				fetched += 1;
 
 				if (modFile.reason == RetrievedFileDefReason.Downloaded) {
-					log(`Downloaded ${upath.basename(fileInfo.downloadUrl)}... (${fetched} / ${toFetch.length})`);
+					log(`Downloaded ${upath.basename(fileDef.url)}... (${fetched} / ${toFetch.length})`);
 				} else if (modFile.reason == RetrievedFileDefReason.CacheHit) {
-					log(`Fetched ${upath.basename(fileInfo.downloadUrl)} from cache... (${fetched} / ${toFetch.length})`);
+					log(`Fetched ${upath.basename(fileDef.url)} from cache... (${fetched} / ${toFetch.length})`);
 				}
 
 				const dest = upath.join(destination, "mods", fileInfo.fileName);

buildtools/util/hashes.ts

@@ -1,95 +1,29 @@
-/**
- * Bytes to exclude from hashing.
- *
- * Why? I dunno.
- */
-const MURMUR_SKIP_BYTES: { [key: number]: boolean } = {
-	9: true,
-	10: true,
-	13: true,
-	32: true,
-};
-
 import _sha1 from "sha1";
-
+import _md5 from "md5";
-/**
- * JS Implementation of MurmurHash2
- *
- * @author <a href="mailto:gary.court@gmail.com">Gary Court</a>
- * @see http://github.com/garycourt/murmurhash-js
- * @author <a href="mailto:aappleby@gmail.com">Austin Appleby</a>
- * @see http://sites.google.com/site/murmurhash/
- */
-function murmurhash2_32_gc(arr, seed, len = arr.length) {
-	let l = len,
-		h = seed ^ l,
-		i = 0,
-		k;
-
-	while (l >= 4) {
-		k = (arr[i] & 0xff) | ((arr[++i] & 0xff) << 8) | ((arr[++i] & 0xff) << 16) | ((arr[++i] & 0xff) << 24);
-
-		k = (k & 0xffff) * 0x5bd1e995 + ((((k >>> 16) * 0x5bd1e995) & 0xffff) << 16);
-		k ^= k >>> 24;
-		k = (k & 0xffff) * 0x5bd1e995 + ((((k >>> 16) * 0x5bd1e995) & 0xffff) << 16);
-
-		h = ((h & 0xffff) * 0x5bd1e995 + ((((h >>> 16) * 0x5bd1e995) & 0xffff) << 16)) ^ k;
-
-		l -= 4;
-		++i;
-	}
-
-	switch (l) {
-		case 3:
-			h ^= (arr[i + 2] & 0xff) << 16;
-		case 2:
-			h ^= (arr[i + 1] & 0xff) << 8;
-		case 1:
-			h ^= arr[i] & 0xff;
-			h = (h & 0xffff) * 0x5bd1e995 + ((((h >>> 16) * 0x5bd1e995) & 0xffff) << 16);
-	}
-
-	h ^= h >>> 13;
-	h = (h & 0xffff) * 0x5bd1e995 + ((((h >>> 16) * 0x5bd1e995) & 0xffff) << 16);
-	h ^= h >>> 15;
-
-	return h >>> 0;
-}
-
-/**
- * Returns the hash sum of bytes of given bytes using MurmurHash v2.
- *
- * This is what Twitch is using to fingerprint mod files.
- */
-export const murmurhash = (inputBuffer: Buffer, seed = 1): string => {
-	const output = new Uint8Array(inputBuffer.length);
-
-	let pos = 0;
-	for (let i = 0; i < inputBuffer.length; i++) {
-		const byte = inputBuffer.readUInt8(i);
-
-		if (!MURMUR_SKIP_BYTES[byte]) {
-			output[pos++] = byte;
-		}
-	}
-
-	return String(murmurhash2_32_gc(output, seed, pos));
-};
 
 import { HashDef } from "../types/hashDef";
 
 /**
  * Returns the hash sum of bytes of given bytes using SHA1.
  *
- * This is what Forge is using to check files.
+ * This is what CurseForge and Forge are using to check files.
  */
 export const sha1 = (inputBuffer: Buffer): string => {
 	return _sha1(inputBuffer);
 };
 
+/**
+ * Returns the hash sum of bytes of given bytes using MD5.
+ *
+ * This is what CF is using to check files.
+ */
+export const md5 = (inputBuffer: Buffer): string => {
+	return _md5(inputBuffer);
+};
+
 const hashFuncs: { [key: string]: (buffer: Buffer) => string } = {
-	murmurhash: murmurhash,
+	sha1,
-	sha1: sha1,
+	md5,
 };
 
 /**