TMC/modded7
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 5 Wiki Activity

update buildscript to new cf protocol

Browse Source
This commit is contained in:
tracer4b 2022-07-07 16:11:30 +08:00
parent c282202cd9
commit 322482299f
12 changed files with 179 additions and 164 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
README.md
View File

@ -9,12 +9,6 @@ Port of [Nomifactory](https://github.com/Nomifactory/Nomifactory) 1.3 to the [Gr
- Updated questbook to guide you through all the new stuff
- lots more...
Installation instructions for building the bleeding-edge version:
- Download [Node.js](https://nodejs.org/en/)
- Open a command line in `buildtools`
- Run `npm install` (only 1st time)
- Run `npx gulp buildAll` and `npx gulp zipAll`
## Hard mode info
If you want a harder, or perhaps a more "true" GregTech experience, check out the Expert mode. This pack mode is based on the [Self-Torture Edition fork](https://github.com/NotMyWing/Omnifactory-Self-Torture-Edition) of the original pack. Highlights include:

2
buildtools/config.json
View File

@ -7,6 +7,6 @@
"overrides/**/*",
"!overrides/resources/minecraft/textures/gui/title/background/*"
],
"nightlyHookAvatar": "https://github.com/NomifactoryDevs.png",
"nightlyHookAvatar": "https://github.com/Nomifactory.png",
"nightlyHookName": "Nightly Builds"
}

50
buildtools/package-lock.json generated
View File

@ -8,6 +8,9 @@
"name": "nomifactory-buildtools",
"version": "1.2.2",
"license": "LGPL-3.0",
"dependencies": {
"md5": "^2.3.0"
},
"devDependencies": {
"@octokit/rest": "^18.3.5",
"@types/bluebird": "^3.5.33",
@ -16,6 +19,7 @@
"@types/gulp-imagemin": "^7.0.2",
"@types/gulp-rename": "^2.0.0",
"@types/gulp-zip": "^4.0.1",
"@types/md5": "^2.3.2",
"@types/merge-stream": "^1.1.2",
"@types/mustache": "^4.1.1",
"@types/requestretry": "^1.12.7",
@ -517,6 +521,12 @@
"integrity": "sha512-qcUXuemtEu+E5wZSJHNxUXeCZhAfXKQ41D+duX+VYPde7xyEVZci+/oXKJL13tnRs9lR2pr4fod59GT6/X1/yQ==",
"dev": true
},
"node_modules/@types/md5": {
"version": "2.3.2",
"resolved": "https://registry.npmjs.org/@types/md5/-/md5-2.3.2.tgz",
"integrity": "sha512-v+JFDu96+UYJ3/UWzB0mEglIS//MZXgRaJ4ubUPwOM0gvLc/kcQ3TWNYwENEK7/EcXGQVrW8h/XqednSjBd/Og==",
"dev": true
},
"node_modules/@types/merge-stream": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/@types/merge-stream/-/merge-stream-1.1.2.tgz",
@ -2128,7 +2138,6 @@
"version": "0.0.2",
"resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz",
"integrity": "sha1-wKHS86cJLgN3S/qD8UwPxXkKhmc=",
"dev": true,
"engines": {
"node": "*"
}
@ -2652,7 +2661,6 @@
"version": "0.0.2",
"resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz",
"integrity": "sha1-iNf/fsDfuG9xPch7u0LQRNPmxBs=",
"dev": true,
"engines": {
"node": "*"
}
@ -6252,8 +6260,7 @@
"node_modules/is-buffer": {
"version": "1.1.6",
"resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz",
"integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==",
"dev": true
"integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w=="
},
"node_modules/is-callable": {
"version": "1.2.4",
@ -7362,6 +7369,16 @@
"node": ">=0.10.0"
}
},
"node_modules/md5": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz",
"integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==",
"dependencies": {
"charenc": "0.0.2",
"crypt": "0.0.2",
"is-buffer": "~1.1.6"
}
},
"node_modules/mdn-data": {
"version": "2.0.4",
"resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.0.4.tgz",
@ -11802,6 +11819,12 @@
"integrity": "sha512-qcUXuemtEu+E5wZSJHNxUXeCZhAfXKQ41D+duX+VYPde7xyEVZci+/oXKJL13tnRs9lR2pr4fod59GT6/X1/yQ==",
"dev": true
},
"@types/md5": {
"version": "2.3.2",
"resolved": "https://registry.npmjs.org/@types/md5/-/md5-2.3.2.tgz",
"integrity": "sha512-v+JFDu96+UYJ3/UWzB0mEglIS//MZXgRaJ4ubUPwOM0gvLc/kcQ3TWNYwENEK7/EcXGQVrW8h/XqednSjBd/Og==",
"dev": true
},
"@types/merge-stream": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/@types/merge-stream/-/merge-stream-1.1.2.tgz",
@ -13053,8 +13076,7 @@
"charenc": {
"version": "0.0.2",
"resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz",
"integrity": "sha1-wKHS86cJLgN3S/qD8UwPxXkKhmc=",
"dev": true
"integrity": "sha1-wKHS86cJLgN3S/qD8UwPxXkKhmc="
},
"chokidar": {
"version": "3.5.2",
@ -13491,8 +13513,7 @@
"crypt": {
"version": "0.0.2",
"resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz",
"integrity": "sha1-iNf/fsDfuG9xPch7u0LQRNPmxBs=",
"dev": true
"integrity": "sha1-iNf/fsDfuG9xPch7u0LQRNPmxBs="
},
"css-select": {
"version": "2.1.0",
@ -16344,8 +16365,7 @@
"is-buffer": {
"version": "1.1.6",
"resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz",
"integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==",
"dev": true
"integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w=="
},
"is-callable": {
"version": "1.2.4",
@ -17207,6 +17227,16 @@
}
}
},
"md5": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz",
"integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==",
"requires": {
"charenc": "0.0.2",
"crypt": "0.0.2",
"is-buffer": "~1.1.6"
}
},
"mdn-data": {
"version": "2.0.4",
"resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.0.4.tgz",

4
buildtools/package.json
View File

@ -13,6 +13,7 @@
"@types/gulp-imagemin": "^7.0.2",
"@types/gulp-rename": "^2.0.0",
"@types/gulp-zip": "^4.0.1",
"@types/md5": "^2.3.2",
"@types/merge-stream": "^1.1.2",
"@types/mustache": "^4.1.1",
"@types/requestretry": "^1.12.7",
@ -43,5 +44,8 @@
"typescript": "^4.2.3",
"unzipper": "^0.10.11",
"upath": "^2.0.1"
},
"dependencies": {
"md5": "^2.3.0"
}
}

1
buildtools/tasks/checks/index.ts
View File

@ -8,6 +8,7 @@ const vars = [
"GITHUB_REF",
"CURSEFORGE_PROJECT_ID",
"CURSEFORGE_API_TOKEN",
"CFCORE_API_TOKEN",
];
/**

6
buildtools/tasks/deploy/curseforge.ts
View File

@ -8,7 +8,7 @@ import buildConfig from "../../buildConfig";
import { makeArtifactNameBody } from "../../util/util";
import sanitize from "sanitize-filename";
const CURSEFORGE_ENDPOINT = "https://minecraft.curseforge.com/";
const CURSEFORGE_LEGACY_ENDPOINT = "https://minecraft.curseforge.com/";
const variablesToCheck = ["CURSEFORGE_API_TOKEN", "CURSEFORGE_PROJECT_ID", "GITHUB_TAG"];
/**
@ -63,7 +63,7 @@ async function deployCurseForge(): Promise<void> {
log("Fetching CurseForge version manifest...");
const versionsManifest =
(await request({
uri: CURSEFORGE_ENDPOINT + "api/game/versions",
uri: CURSEFORGE_LEGACY_ENDPOINT + "api/game/versions",
headers: tokenHeaders,
method: "GET",
json: true,
@ -82,7 +82,7 @@ async function deployCurseForge(): Promise<void> {
// Upload artifacts.
for (const file of files) {
const options = {
uri: CURSEFORGE_ENDPOINT + `api/projects/${process.env.CURSEFORGE_PROJECT_ID}/upload-file`,
uri: CURSEFORGE_LEGACY_ENDPOINT + `api/projects/${process.env.CURSEFORGE_PROJECT_ID}/upload-file`,
method: "POST",
headers: {
...tokenHeaders,

2
buildtools/tasks/mmc/index.ts
View File

@ -27,7 +27,7 @@ async function createMMCDirs(cb) {
* Copies modpack overrides.
*/
function copyOverrides() {
return src(upath.join(clientDestDirectory, "**/*.*"), {
return src(upath.join(clientDestDirectory, "**/*"), {
nodir: true,
resolveSymlinks: false,
}).pipe(symlink(upath.join(mmcDestDirectory)));

2
buildtools/tasks/server/index.ts
View File

@ -204,7 +204,7 @@ function copyServerfiles() {
* Copies the license file.
*/
function copyServerLicense() {
return src("../LICENSE").pipe(dest(serverDestDirectory));
return src("../LICENSE.md").pipe(dest(serverDestDirectory));
}
/**

79
buildtools/types/curseForge.ts
View File

@ -20,25 +20,6 @@ interface Attachment {
status: number;
}
interface Dependency {
id: number;
addonId: number;
type: number;
fileId: number;
}
interface Module {
foldername: string;
type: number;
}
interface SortableGameVersion {
gameVersionPadded: string;
gameVersion: string;
gameVersionReleaseDate: Date;
gameVersionName: string;
}
export interface CurseForgeFileInfo {
id: number;
displayName: string;
@ -129,32 +110,48 @@ export interface CurseForgeModInfo {
isExperiemental: boolean;
}
export interface CurseForgeFetchedFileInfo {
id: number;
gameId: number;
modId: number;
isAvailable: boolean;
displayName: string;
fileName: string;
releaseType: number;
fileStatus: number;
hashes?: Hash[];
fileDate: Date;
fileLength: number;
downloadCount: number;
downloadUrl?: string;
gameVersions: string[];
sortableGameVersions: SortableGameVersion[];
dependencies: Dependency[];
alternateFileId: number;
isServerPack: boolean;
fileFingerprint: number;
modules: Module[];
}
interface Dependency {
addonId: number;
type: number;
modId: number;
relationType: number;
}
interface Hash {
value: string;
algo: number;
}
interface Module {
foldername: string;
fingerprint: unknown;
name: string;
fingerprint: number;
}
export interface CurseForgeFetchedFileInfo {
id: number;
displayName: string;
fileName: string;
fileDate: Date;
fileLength: number;
releaseType: number;
fileStatus: number;
downloadUrl: string;
isAlternate: boolean;
alternateFileId: number;
dependencies: Dependency[];
isAvailable: boolean;
modules: Module[];
packageFingerprint: number;
gameVersion: string[];
hasInstallScript: boolean;
gameVersionDateReleased: Date;
interface SortableGameVersion {
gameVersionName: string;
gameVersionPadded: string;
gameVersion: string;
gameVersionReleaseDate: Date;
gameVersionTypeId: number;
}

1
buildtools/util/buildConfig.default.json
View File

@ -1,4 +1,5 @@
{
"cfCoreApiEndpoint": "https://curseforge.neeve.workers.dev",
"downloaderMaxRetries": 5,
"downloaderConcurrency": 50,
"downloaderCheckHashes": true,

80
buildtools/util/curseForgeAPI.ts
View File

@ -10,22 +10,43 @@ import fs from "fs";
import { FileDef } from "../types/fileDef";
import { downloadOrRetrieveFileDef, relative, RetrievedFileDefReason } from "./util";
function getCurseForgeToken() {
const vari = "CFCORE_API_TOKEN";
const val = process.env[vari];
if (!process.env[vari]) {
throw new Error(`Environmental variable ${vari} is unset.`);
}
return val;
}
const curseForgeProjectCache: { [key: number]: CurseForgeProject } = {};
export async function fetchProject(toFetch: number): Promise<CurseForgeProject> {
if (curseForgeProjectCache[toFetch]) {
return curseForgeProjectCache[toFetch];
}
const project: CurseForgeProject = await request({
uri: `https://addons-ecs.forgesvc.net/api/v2/addon/${toFetch}`,
const project: CurseForgeProject | undefined = (
await request({
uri: `${buildConfig.cfCoreApiEndpoint}/v1/mods/${toFetch}`,
json: true,
fullResponse: false,
maxAttempts: 5,
});
headers: {
"X-Api-Key": getCurseForgeToken(),
},
})
)?.data;
if (!project) {
throw new Error(`Failed to fetch project ${toFetch}`);
}
if (project) {
curseForgeProjectCache[toFetch] = project;
}
return project;
}
@ -37,14 +58,29 @@ export async function fetchFileInfo(projectID: number, fileID: number): Promise<
return fetchedFileInfoCache[slug];
}
const fileInfo: CurseForgeFetchedFileInfo = await request({
uri: `https://addons-ecs.forgesvc.net/api/v2/addon/${projectID}/file/${fileID}`,
const fileInfo: CurseForgeFetchedFileInfo = (
await request({
uri: `${buildConfig.cfCoreApiEndpoint}/v1/mods/${projectID}/files/${fileID}`,
json: true,
fullResponse: false,
});
headers: {
"X-Api-Key": getCurseForgeToken(),
},
})
)?.data;
if (!fileInfo) {
throw new Error(`Failed to download file ${projectID}/file/${fileID}`);
}
if (fileInfo) {
fetchedFileInfoCache[slug] = fileInfo;
if (!fileInfo.downloadUrl) {
const fid = `${Math.floor(fileInfo.id / 1000)}/${fileInfo.id % 1000}`;
fileInfo.downloadUrl = `https://edge.forgecdn.net/files/${fid}/${fileInfo.fileName}`;
}
}
return fileInfo;
@ -73,12 +109,23 @@ export async function fetchProjectsBulk(toFetch: number[]): Promise<CurseForgePr
if (unfetched.length > 0) {
// Augment the array of known projects with new info.
const fetched: CurseForgeProject[] = await request.post({
uri: "https://addons-ecs.forgesvc.net/api/v2/addon/",
json: unfetched,
const fetched: CurseForgeProject[] = (
await request.post({
uri: `${buildConfig.cfCoreApiEndpoint}/v1/mods`,
json: {
modIds: unfetched,
},
fullResponse: false,
maxAttempts: 5,
});
headers: {
"X-Api-Key": getCurseForgeToken(),
},
})
)?.data;
if (!fetched) {
throw new Error(`Failed to bulk-fetch projects ${unfetched.join(", ")}`);
}
modInfos.push(...fetched);
@ -133,16 +180,23 @@ export async function fetchMods(toFetch: ModpackManifestFile[], destination: str
const fileDef: FileDef = {
url: fileInfo.downloadUrl,
hashes: [{ id: "murmurhash", hashes: fileInfo.packageFingerprint }],
};
// https://docs.curseforge.com/#tocS_GetModsByIdsListRequestBody
if (fileInfo.hashes) {
fileDef.hashes = fileInfo.hashes.map((hash) => ({
hashes: hash.value,
id: hash.algo == 1 ? "sha1" : "md5",
}));
}
const modFile = await downloadOrRetrieveFileDef(fileDef);
fetched += 1;
if (modFile.reason == RetrievedFileDefReason.Downloaded) {
log(`Downloaded ${upath.basename(fileInfo.downloadUrl)}... (${fetched} / ${toFetch.length})`);
log(`Downloaded ${upath.basename(fileDef.url)}... (${fetched} / ${toFetch.length})`);
} else if (modFile.reason == RetrievedFileDefReason.CacheHit) {
log(`Fetched ${upath.basename(fileInfo.downloadUrl)} from cache... (${fetched} / ${toFetch.length})`);
log(`Fetched ${upath.basename(fileDef.url)} from cache... (${fetched} / ${toFetch.length})`);
}
const dest = upath.join(destination, "mods", fileInfo.fileName);

92
buildtools/util/hashes.ts
View File

@ -1,95 +1,29 @@
/**
* Bytes to exclude from hashing.
*
* Why? I dunno.
*/
const MURMUR_SKIP_BYTES: { [key: number]: boolean } = {
9: true,
10: true,
13: true,
32: true,
};
import _sha1 from "sha1";
/**
* JS Implementation of MurmurHash2
*
* @author <a href="mailto:gary.court@gmail.com">Gary Court</a>
* @see http://github.com/garycourt/murmurhash-js
* @author <a href="mailto:aappleby@gmail.com">Austin Appleby</a>
* @see http://sites.google.com/site/murmurhash/
*/
function murmurhash2_32_gc(arr, seed, len = arr.length) {
let l = len,
h = seed ^ l,
i = 0,
k;
while (l >= 4) {
k = (arr[i] & 0xff) | ((arr[++i] & 0xff) << 8) | ((arr[++i] & 0xff) << 16) | ((arr[++i] & 0xff) << 24);
k = (k & 0xffff) * 0x5bd1e995 + ((((k >>> 16) * 0x5bd1e995) & 0xffff) << 16);
k ^= k >>> 24;
k = (k & 0xffff) * 0x5bd1e995 + ((((k >>> 16) * 0x5bd1e995) & 0xffff) << 16);
h = ((h & 0xffff) * 0x5bd1e995 + ((((h >>> 16) * 0x5bd1e995) & 0xffff) << 16)) ^ k;
l -= 4;
++i;
}
switch (l) {
case 3:
h ^= (arr[i + 2] & 0xff) << 16;
case 2:
h ^= (arr[i + 1] & 0xff) << 8;
case 1:
h ^= arr[i] & 0xff;
h = (h & 0xffff) * 0x5bd1e995 + ((((h >>> 16) * 0x5bd1e995) & 0xffff) << 16);
}
h ^= h >>> 13;
h = (h & 0xffff) * 0x5bd1e995 + ((((h >>> 16) * 0x5bd1e995) & 0xffff) << 16);
h ^= h >>> 15;
return h >>> 0;
}
/**
* Returns the hash sum of bytes of given bytes using MurmurHash v2.
*
* This is what Twitch is using to fingerprint mod files.
*/
export const murmurhash = (inputBuffer: Buffer, seed = 1): string => {
const output = new Uint8Array(inputBuffer.length);
let pos = 0;
for (let i = 0; i < inputBuffer.length; i++) {
const byte = inputBuffer.readUInt8(i);
if (!MURMUR_SKIP_BYTES[byte]) {
output[pos++] = byte;
}
}
return String(murmurhash2_32_gc(output, seed, pos));
};
import _md5 from "md5";
import { HashDef } from "../types/hashDef";
/**
* Returns the hash sum of bytes of given bytes using SHA1.
*
* This is what Forge is using to check files.
* This is what CurseForge and Forge are using to check files.
*/
export const sha1 = (inputBuffer: Buffer): string => {
return _sha1(inputBuffer);
};
/**
* Returns the hash sum of bytes of given bytes using MD5.
*
* This is what CF is using to check files.
*/
export const md5 = (inputBuffer: Buffer): string => {
return _md5(inputBuffer);
};
const hashFuncs: { [key: string]: (buffer: Buffer) => string } = {
murmurhash: murmurhash,
sha1: sha1,
sha1,
md5,
};
/**