From fcf89d1f33f6e0ad87e0e4e5ff318914e4f6b587 Mon Sep 17 00:00:00 2001 From: SPGoding <15277496+SPGoding@users.noreply.github.com> Date: Thu, 6 Feb 2025 16:39:22 -0600 Subject: [PATCH 01/13] =?UTF-8?q?=F0=9F=93=9D=20Clarify=20web=20API=20in?= =?UTF-8?q?=20docs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/developer/api.adoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/developer/api.adoc b/docs/developer/api.adoc index 9fbd5a4e5..dce4d382b 100644 --- a/docs/developer/api.adoc +++ b/docs/developer/api.adoc @@ -1,12 +1,12 @@ :page-layout: default -:page-title: API +:page-title: Web API :page-parent: Developer Guides :toc: == Introduction -The Spyglass API provides access to various information that is helpful for data pack/resource pack +The Spyglass Web API provides access to various information that is helpful for data pack/resource pack toolings. It uses https://github.com/misode/mcmeta[misode/mcmeta] and https://github.com/SpyglassMC/vanilla-mcdoc[SpyglassMC/vanilla-mcdoc] under the hood and provides a few advantages over using the GitHub API directly: From d7a56c46a55417d2209071bc555bf2a614ba4372 Mon Sep 17 00:00:00 2001 From: SPGoding Date: Thu, 6 Feb 2025 14:48:58 -0600 Subject: [PATCH 02/13] =?UTF-8?q?=E2=9C=A8=20Integrate=20web=20API=20into?= =?UTF-8?q?=20other=20packages?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/common/externals/BrowserExternals.ts | 30 +- .../src/common/externals/NodeJsExternals.ts | 276 +++++++++++------- .../core/src/common/externals/downloader.ts | 54 ---- packages/core/src/common/externals/index.ts | 10 +- packages/core/src/service/Dependency.ts | 5 +- packages/core/src/service/Downloader.ts | 184 ------------ packages/core/src/service/FileService.ts | 55 ++-- packages/core/src/service/Project.ts | 29 +- packages/core/src/service/fetcher.ts | 31 ++ packages/core/src/service/index.ts | 2 +- packages/core/test/utils.ts | 7 +- packages/java-edition/src/dependency/index.ts | 225 ++++---------- .../java-edition/src/dependency/mcmeta.ts | 36 --- packages/java-edition/src/index.ts | 28 +- .../test/dependency/mcmeta.spec.ts | 22 +- packages/language-server/src/server.ts | 12 +- 16 files changed, 339 insertions(+), 667 deletions(-) delete mode 100644 packages/core/src/common/externals/downloader.ts delete mode 100644 packages/core/src/service/Downloader.ts create mode 100644 packages/core/src/service/fetcher.ts diff --git a/packages/core/src/common/externals/BrowserExternals.ts b/packages/core/src/common/externals/BrowserExternals.ts index dd4d5ed4e..a9240d148 100644 --- a/packages/core/src/common/externals/BrowserExternals.ts +++ b/packages/core/src/common/externals/BrowserExternals.ts @@ -1,11 +1,6 @@ import { decode as arrayBufferFromBase64, encode as arrayBufferToBase64 } from 'base64-arraybuffer' import pako from 'pako' import { fileUtil } from '../../service/fileUtil.js' -import type { - ExternalDownloader, - ExternalDownloaderOptions, - RemoteUriString, -} from './downloader.js' import type { ExternalEventEmitter, ExternalFileSystem, @@ -53,24 +48,6 @@ export class BrowserEventEmitter implements ExternalEventEmitter { } } -class BrowserExternalDownloader implements ExternalDownloader { - async get(uri: RemoteUriString, options: ExternalDownloaderOptions = {}): Promise { - const headers = new Headers() - for (const [name, value] of Object.entries(options?.headers ?? {})) { - const values = typeof value === 'string' ? [value] : value - for (const v of values) { - headers.append(name, v) - } - } - const res = await fetch(uri, { headers, redirect: 'follow' }) - if (!res.ok) { - throw new Error(`Status code ${res.status}: ${res.ok}`) - } else { - return new Uint8Array(await res.arrayBuffer()) - } - } -} - class BrowserFsWatcher implements FsWatcher { on(event: string, listener: (...args: any[]) => unknown): this { if (event === 'ready') { @@ -86,7 +63,7 @@ class BrowserFsWatcher implements FsWatcher { return this } - async close(): Promise {} + async close(): Promise { } } class BrowserFileSystem implements ExternalFileSystem { @@ -188,7 +165,6 @@ export const BrowserExternals: Externals = { return uint8ArrayToHex(new Uint8Array(hash)) }, }, - downloader: new BrowserExternalDownloader(), error: { createKind(kind, message) { return new Error(`${kind}: ${message}`) @@ -199,6 +175,10 @@ export const BrowserExternals: Externals = { }, event: { EventEmitter: BrowserEventEmitter }, fs: new BrowserFileSystem(), + web: { + fetch, + getCache: () => window.caches.open('spyglassmc'), + }, } function uint8ArrayToHex(array: Uint8Array) { diff --git a/packages/core/src/common/externals/NodeJsExternals.ts b/packages/core/src/common/externals/NodeJsExternals.ts index ba211824f..d7c95bcc6 100644 --- a/packages/core/src/common/externals/NodeJsExternals.ts +++ b/packages/core/src/common/externals/NodeJsExternals.ts @@ -1,134 +1,116 @@ // https://github.com/DefinitelyTyped/DefinitelyTyped/discussions/60592 import chokidar from 'chokidar' import decompress from 'decompress' -import followRedirects from 'follow-redirects' import { Buffer } from 'node:buffer' import cp from 'node:child_process' import crypto from 'node:crypto' import { EventEmitter } from 'node:events' -import type fs from 'node:fs' -import { promises as fsp } from 'node:fs' -import type { IncomingMessage } from 'node:http' +import os from 'node:os' +import fs, { promises as fsp } from 'node:fs' import process from 'node:process' +import stream from 'node:stream' +import streamWeb from 'node:stream/web' import url from 'node:url' import { promisify } from 'node:util' import zlib from 'node:zlib' -import { promisifyAsyncIterable, Uri } from '../util.js' -import type { - ExternalDownloader, - ExternalDownloaderOptions, - RemoteUriString, -} from './downloader.js' +import { Uri } from '../util.js' import type { Externals, FsLocation, FsWatcher } from './index.js' +import type { RootUriString } from '../../index.js' -const { http, https } = followRedirects const gunzip = promisify(zlib.gunzip) const gzip = promisify(zlib.gzip) -class NodeJsExternalDownloader implements ExternalDownloader { - get(uri: RemoteUriString, options: ExternalDownloaderOptions = {}): Promise { - const protocol = new Uri(uri).protocol - return new Promise((resolve, reject) => { - const backend = protocol === 'http:' ? http : https - backend.get(uri, options, (res: IncomingMessage) => { - if (res.statusCode !== 200) { - reject(new Error(`Status code ${res.statusCode}: ${res.statusMessage}`)) - } else { - resolve(promisifyAsyncIterable(res, (chunks) => Buffer.concat(chunks))) +export function getNodeJsExternals({ cacheRoot }: { cacheRoot?: RootUriString } = {}) { + return Object.freeze({ + archive: { + decompressBall(buffer, options) { + if (!(buffer instanceof Buffer)) { + buffer = Buffer.from(buffer) } - }).on('error', (e) => { - reject(e) - }) - }) - } -} - -export const NodeJsExternals: Externals = { - archive: { - decompressBall(buffer, options) { - if (buffer instanceof Buffer) { - return decompress(buffer, { strip: options?.stripLevel }) - } - throw new TypeError( - `The 'buffer' argument for 'decompressBall' on Node.js must be an instance of 'Buffer'. Got '${buffer}' instead.`, - ) - }, - gunzip(buffer) { - return gunzip(buffer) - }, - gzip(buffer) { - return gzip(buffer) - }, - }, - crypto: { - async getSha1(data) { - const hash = crypto.createHash('sha1') - hash.update(data) - return hash.digest('hex') - }, - }, - downloader: new NodeJsExternalDownloader(), - error: { - createKind(kind, message) { - const error = new Error(message) - ;(error as NodeJS.ErrnoException).code = kind - return error - }, - isKind(e, kind) { - return e instanceof Error && (e as NodeJS.ErrnoException).code === kind + return decompress(buffer as Buffer, { strip: options?.stripLevel }) + }, + gunzip(buffer) { + return gunzip(buffer) + }, + gzip(buffer) { + return gzip(buffer) + }, }, - }, - event: { EventEmitter }, - fs: { - chmod(location, mode) { - return fsp.chmod(toFsPathLike(location), mode) + crypto: { + async getSha1(data) { + const hash = crypto.createHash('sha1') + hash.update(data) + return hash.digest('hex') + }, }, - async mkdir(location, options) { - return void (await fsp.mkdir(toFsPathLike(location), options)) + error: { + createKind(kind, message) { + const error = new Error(message) + ; (error as NodeJS.ErrnoException).code = kind + return error + }, + isKind(e, kind) { + return e instanceof Error && (e as NodeJS.ErrnoException).code === kind + }, }, - readdir(location) { - return fsp.readdir(toFsPathLike(location), { encoding: 'utf-8', withFileTypes: true }) - }, - readFile(location) { - return fsp.readFile(toFsPathLike(location)) + event: { EventEmitter }, + fs: { + chmod(location, mode) { + return fsp.chmod(toFsPathLike(location), mode) + }, + async mkdir(location, options) { + return void (await fsp.mkdir(toFsPathLike(location), options)) + }, + readdir(location) { + return fsp.readdir(toFsPathLike(location), { encoding: 'utf-8', withFileTypes: true }) + }, + readFile(location) { + return fsp.readFile(toFsPathLike(location)) + }, + async showFile(location): Promise { + const execFile = promisify(cp.execFile) + let command: string + switch (process.platform) { + case 'darwin': + command = 'open' + break + case 'win32': + command = 'explorer' + break + default: + command = 'xdg-open' + break + } + return void (await execFile(command, [toPath(location)])) + }, + stat(location) { + return fsp.stat(toFsPathLike(location)) + }, + unlink(location) { + return fsp.unlink(toFsPathLike(location)) + }, + watch(locations, { usePolling = false } = {}) { + return new ChokidarWatcherWrapper( + chokidar.watch(locations.map(toPath), { + usePolling, + disableGlobbing: true, + }), + ) + }, + writeFile(location, data, options) { + return fsp.writeFile(toFsPathLike(location), data, options) + }, }, - async showFile(location): Promise { - const execFile = promisify(cp.execFile) - let command: string - switch (process.platform) { - case 'darwin': - command = 'open' - break - case 'win32': - command = 'explorer' - break - default: - command = 'xdg-open' - break + web: { + fetch, + getCache: async () => { + return new HttpCache(cacheRoot) } - return void (await execFile(command, [toPath(location)])) - }, - stat(location) { - return fsp.stat(toFsPathLike(location)) - }, - unlink(location) { - return fsp.unlink(toFsPathLike(location)) - }, - watch(locations, { usePolling = false } = {}) { - return new ChokidarWatcherWrapper( - chokidar.watch(locations.map(toPath), { - usePolling, - disableGlobbing: true, - }), - ) - }, - writeFile(location, data, options) { - return fsp.writeFile(toFsPathLike(location), data, options) }, - }, + } satisfies Externals) } -Object.freeze(NodeJsExternals) +export const NodeJsExternals = getNodeJsExternals() /** * @returns A {@link fs.PathLike}. @@ -173,3 +155,85 @@ class ChokidarWatcherWrapper extends EventEmitter implements FsWatcher { return this.#watcher.close() } } + +/** + * A non-spec-compliant, non-complete implementation of the Cache Web API for use in Spyglass. + * This class stores the cached response on the file system under the cache root. + */ +class HttpCache implements Cache { + readonly #cacheRoot: RootUriString | undefined + + constructor(cacheRoot: RootUriString | undefined) { + if (cacheRoot) { + this.#cacheRoot = `${cacheRoot}http/` + } + } + + async match(request: RequestInfo | URL, _options?: CacheQueryOptions | undefined): Promise { + if (!this.#cacheRoot) { + return undefined + } + + const fileName = this.#getFileName(request) + try { + const etag = (await fsp.readFile(new URL(`${fileName}.etag`, this.#cacheRoot), 'utf8')).trim() + const bodyStream = fs.createReadStream(new URL(`${fileName}.bin`, this.#cacheRoot)) + return new Response( + stream.Readable.toWeb(bodyStream) as ReadableStream, + // \___/ + // stream Readable -> stream/web ReadableStream + // \_______________/ + // stream/web ReadableStream -> DOM ReadableStream + { headers: { etag } }, + ) + } catch (e) { + if ((e as NodeJS.ErrnoException)?.code === 'ENOENT') { + return undefined + } + + throw e + } + } + + async put(request: RequestInfo | URL, response: Response): Promise { + const clonedResponse = response.clone() + const etag = clonedResponse.headers.get('etag') + if (!(this.#cacheRoot && clonedResponse.body && etag)) { + return + } + + const fileName = this.#getFileName(request) + await fsp.mkdir(new URL(this.#cacheRoot), { recursive: true }) + await Promise.all([ + fsp.writeFile( + new URL(`${fileName}.bin`, this.#cacheRoot), + stream.Readable.fromWeb(clonedResponse.body as streamWeb.ReadableStream) + // \_____/ \_________________________/ + // | DOM ReadableStream -> stream/web ReadableStream + // stream/web ReadableStream -> stream Readable + ), + fsp.writeFile(new URL(`${fileName}.etag`, this.#cacheRoot), `${etag}${os.EOL}`) + ]) + } + + #getFileName(request: RequestInfo | URL) { + const uriString = request instanceof Request ? request.url : request.toString() + return Buffer.from(uriString, 'utf8').toString('base64url') + } + + async add(): Promise { + throw new Error('Method not implemented.') + } + async addAll(): Promise { + throw new Error('Method not implemented.') + } + async delete(): Promise { + throw new Error('Method not implemented.') + } + async keys(): Promise { + throw new Error('Method not implemented.') + } + async matchAll(): Promise { + throw new Error('Method not implemented.') + } +} diff --git a/packages/core/src/common/externals/downloader.ts b/packages/core/src/common/externals/downloader.ts deleted file mode 100644 index 8a7f689e2..000000000 --- a/packages/core/src/common/externals/downloader.ts +++ /dev/null @@ -1,54 +0,0 @@ -type RemoteUriProtocol = 'http:' | 'https:' -export type RemoteUriString = `${RemoteUriProtocol}${string}` -export namespace RemoteUriString { - export function is(value: string): value is RemoteUriString { - return value.startsWith('http:') || value.startsWith('https:') - } -} - -export interface ExternalDownloaderOptions { - /** - * Use an string array to set multiple values to the header. - */ - headers?: Record - timeout?: number -} - -export interface ExternalDownloader { - /** - * @throws - */ - get(uri: RemoteUriString, options?: ExternalDownloaderOptions): Promise -} -export namespace ExternalDownloader { - export function mock(options: ExternalDownloaderMockOptions): ExternalDownloader { - return new ExternalDownloaderMock(options) - } -} - -interface ExternalDownloaderMockOptions { - /** - * A record from URIs to fixture data. The {@link ExternalDownloader.get} only returns a {@link Uint8Array}, - * therefore `string` fixtures will be turned into a `Uint8Array` and `object` fixtures will be transformed - * into JSON and then turned into a `Uint8Array`. - */ - fixtures: Record -} - -class ExternalDownloaderMock implements ExternalDownloader { - constructor(private readonly options: ExternalDownloaderMockOptions) {} - - async get(uri: RemoteUriString): Promise { - if (!this.options.fixtures[uri]) { - throw new Error(`404 not found: ${uri}`) - } - const fixture = this.options.fixtures[uri] - if (fixture instanceof Uint8Array) { - return fixture - } else if (typeof fixture === 'string') { - return new TextEncoder().encode(fixture) - } else { - return new TextEncoder().encode(JSON.stringify(fixture)) - } - } -} diff --git a/packages/core/src/common/externals/index.ts b/packages/core/src/common/externals/index.ts index f3e03016c..cc8f23de6 100644 --- a/packages/core/src/common/externals/index.ts +++ b/packages/core/src/common/externals/index.ts @@ -1,7 +1,4 @@ import type { Uri } from '../util.js' -import type { ExternalDownloader } from './downloader.js' - -export * from './downloader.js' export interface Externals { archive: { @@ -18,7 +15,6 @@ export interface Externals { */ getSha1: (data: string | Uint8Array) => Promise } - downloader: ExternalDownloader error: { /** * @returns an error of the specified kind @@ -29,8 +25,12 @@ export interface Externals { */ isKind: (e: unknown, kind: ExternalErrorKind) => boolean } - event: { EventEmitter: new() => ExternalEventEmitter } + event: { EventEmitter: new () => ExternalEventEmitter } fs: ExternalFileSystem + web: { + fetch: typeof fetch, + getCache: () => Promise, + } } export interface DecompressedFile { diff --git a/packages/core/src/service/Dependency.ts b/packages/core/src/service/Dependency.ts index 4bbf42d11..ad559dbb2 100644 --- a/packages/core/src/service/Dependency.ts +++ b/packages/core/src/service/Dependency.ts @@ -1,4 +1,7 @@ -export type Dependency = { uri: string; info?: Record } +export type Dependency = + | { type: 'directory'; uri: string } + | { type: 'tarball-file'; uri: string; stripLevel?: number } + | { type: 'tarball-ram'; name: string; data: Uint8Array; stripLevel?: number } export type DependencyKey = `@${string}` export namespace DependencyKey { diff --git a/packages/core/src/service/Downloader.ts b/packages/core/src/service/Downloader.ts deleted file mode 100644 index 7c5e567cf..000000000 --- a/packages/core/src/service/Downloader.ts +++ /dev/null @@ -1,184 +0,0 @@ -import type { - ExternalDownloaderOptions, - Externals, - Logger, - RemoteUriString, -} from '../common/index.js' -import { bufferToString, Uri } from '../common/index.js' -import type { RootUriString } from './fileUtil.js' -import { fileUtil } from './fileUtil.js' - -export interface DownloaderDownloadOut { - cacheUri?: string - checksum?: string -} - -interface MemoryCacheEntry { - buffer: Uint8Array - time: number - cacheUri?: string - checksum?: string -} - -export class Downloader { - readonly #memoryCache = new Map() - - constructor( - private readonly cacheRoot: RootUriString, - private readonly externals: Externals, - private readonly logger: Logger, - ) {} - - async download(job: Job, out: DownloaderDownloadOut = {}): Promise { - const { id, cache, uri, options, transformer, ttl } = job - if (ttl && this.#memoryCache.has(uri)) { - const memoryCacheEntry = this.#memoryCache.get(uri)! - const { buffer, time, cacheUri, checksum } = memoryCacheEntry - if (performance.now() <= time + ttl) { - this.logger.info(`[Downloader] [${id}] Skipped thanks to valid cache in memory`) - out.cacheUri = cacheUri - out.checksum = checksum - return await transformer(buffer) - } else { - this.#memoryCache.delete(uri) - } - } - let checksum: string | undefined - let cacheUri: string | undefined - let cacheChecksumUri: string | undefined - if (cache) { - const { checksumJob, checksumExtension } = cache - out.cacheUri = cacheUri = new Uri(`downloader/${id}`, this.cacheRoot).toString() - cacheChecksumUri = new Uri(`downloader/${id}${checksumExtension}`, this.cacheRoot) - .toString() - try { - out.checksum = checksum = await this.download({ - ...checksumJob, - id: id + checksumExtension, - }) - try { - const cacheChecksum = bufferToString( - await fileUtil.readFile(this.externals, cacheChecksumUri), - ).slice(0, -1) // Remove ending newline - if (checksum === cacheChecksum) { - try { - const cachedBuffer = await fileUtil.readFile(this.externals, cacheUri) - if (ttl) { - this.#memoryCache.set(uri, { - buffer: cachedBuffer, - cacheUri, - checksum, - time: performance.now(), - }) - } - const deserializer = cache.deserializer ?? ((b) => b) - const ans = await transformer(deserializer(cachedBuffer)) - this.logger.info( - `[Downloader] [${id}] Skipped downloading thanks to cache ${cacheChecksum} (${cachedBuffer.length} bytes)`, - ) - return ans - } catch (e) { - this.logger.error(`[Downloader] [${id}] Loading cached file ${cacheUri}`, e) - if (this.externals.error.isKind(e, 'ENOENT')) { - // Cache checksum exists, but cached file doesn't. - // Remove the invalid cache checksum. - try { - await this.externals.fs.unlink(cacheChecksumUri) - } catch (e) { - this.logger.error( - `[Downloader] [${id}] Removing invalid cache checksum ${cacheChecksumUri}`, - e, - ) - } - } - } - } - } catch (e) { - if (!this.externals.error.isKind(e, 'ENOENT')) { - this.logger.error( - `[Downloader] [${id}] Loading cache checksum ${cacheChecksumUri}`, - e, - ) - } - } - } catch (e) { - this.logger.error( - `[Downloader] [${id}] Fetching latest checksum ${checksumJob.uri}`, - e, - ) - } - } - - try { - const buffer = await this.externals.downloader.get(uri, options) - if (ttl) { - this.#memoryCache.set(uri, { buffer, time: performance.now() }) - } - if (cache && cacheUri && cacheChecksumUri) { - if (checksum) { - try { - await fileUtil.writeFile(this.externals, cacheChecksumUri, `${checksum}\n`) - } catch (e) { - this.logger.error( - `[Downloader] [${id}] Saving cache checksum ${cacheChecksumUri}`, - e, - ) - } - } - try { - const serializer = cache.serializer ?? ((b) => b) - await fileUtil.writeFile(this.externals, cacheUri, serializer(buffer)) - } catch (e) { - this.logger.error(`[Downloader] [${id}] Caching file ${cacheUri}`, e) - } - } - this.logger.info(`[Downloader] [${id}] Downloaded from ${uri} (${buffer.length} bytes)`) - return await transformer(buffer) - } catch (e) { - this.logger.error(`[Downloader] [${id}] Downloading ${uri}`, e) - if (cache && cacheUri) { - try { - const cachedBuffer = await fileUtil.readFile(this.externals, cacheUri) - const deserializer = cache.deserializer ?? ((b) => b) - const ans = await transformer(deserializer(cachedBuffer)) - this.logger.warn( - `[Downloader] [${id}] Fell back to cached file ${cacheUri} (${cachedBuffer.length} bytes)`, - ) - return ans - } catch (e) { - this.logger.error( - `[Downloader] [${id}] Fallback: loading cached file ${cacheUri}`, - e, - ) - } - } - } - - return undefined - } -} - -interface Job { - /** - * A unique ID for the cache. - * - * It also determines where the file is cached. Use slashes (`/`) to create directories. - */ - id: string - uri: RemoteUriString - cache?: { - /** - * A download {@link Job} that will return a checksum of the latest remote data. - */ - checksumJob: Omit, 'cache' | 'id'> - checksumExtension: `.${string}` - serializer?: (data: Uint8Array) => Uint8Array - deserializer?: (cache: Uint8Array) => Uint8Array - } - transformer: (data: Uint8Array) => PromiseLike | R - options?: ExternalDownloaderOptions - /** - * If set, caches the result in memory. Time in milliseconds. - */ - ttl?: number -} diff --git a/packages/core/src/service/FileService.ts b/packages/core/src/service/FileService.ts index 29f22fbe3..242a80fb4 100644 --- a/packages/core/src/service/FileService.ts +++ b/packages/core/src/service/FileService.ts @@ -78,7 +78,7 @@ export class FileServiceImpl implements FileService { constructor( private readonly externals: Externals, private readonly virtualUrisRoot?: RootUriString, - ) {} + ) { } register(protocol: Protocol, supporter: UriProtocolSupporter, force = false): void { if (!force && this.supporters.has(protocol)) { @@ -183,7 +183,7 @@ export class FileUriSupporter implements UriProtocolSupporter { private readonly externals: Externals, private readonly roots: RootUriString[], private readonly files: Map, - ) {} + ) { } async hash(uri: string): Promise { return hashFile(this.externals, uri) @@ -215,7 +215,12 @@ export class FileUriSupporter implements UriProtocolSupporter { const roots: RootUriString[] = [] const files = new Map() - for (let { uri } of dependencies) { + for (const dependency of dependencies) { + if (dependency.type !== 'directory') { + continue + } + + let { uri } = dependency try { if (fileUtil.isFileUri(uri) && (await externals.fs.stat(uri)).isDirectory()) { uri = fileUtil.ensureEndingSlash(uri) @@ -244,7 +249,7 @@ export class ArchiveUriSupporter implements UriProtocolSupporter { private readonly externals: Externals, private readonly logger: Logger, private readonly entries: Map>, - ) {} + ) { } async hash(uri: string): Promise { const { archiveName, pathInArchive } = ArchiveUriSupporter.decodeUri(new Uri(uri)) @@ -333,29 +338,31 @@ export class ArchiveUriSupporter implements UriProtocolSupporter { ): Promise { const entries = new Map>() - for (const { uri, info } of dependencies) { + for (const dependency of dependencies) { + if (dependency.type === 'directory') { + continue + } + + const archiveName = dependency.type === 'tarball-file' + ? fileUtil.basename(dependency.uri) + : dependency.name try { - if ( - uri.startsWith('file:') - && ArchiveUriSupporter.SupportedArchiveExtnames.some((ext) => uri.endsWith(ext)) - && (await externals.fs.stat(uri)).isFile() - ) { - const archiveName = fileUtil.basename(uri) - if (entries.has(archiveName)) { - throw new Error(`A different URI with ${archiveName} already exists`) - } - const files = await externals.archive.decompressBall( - await externals.fs.readFile(uri), - { stripLevel: typeof info?.startDepth === 'number' ? info.startDepth : 0 }, - ) - /// Debug message for #1609 - logger.info( - `[ArchiveUriSupporter#create] Extracted ${files.length} files from ${archiveName}`, - ) - entries.set(archiveName, new Map(files.map((f) => [f.path.replace(/\\/g, '/'), f]))) + if (entries.has(archiveName)) { + throw new Error(`A different archive with ${archiveName} already exists`) } + const files = await externals.archive.decompressBall( + dependency.type === 'tarball-file' + ? await externals.fs.readFile(dependency.uri) + : dependency.data, + { stripLevel: dependency.stripLevel ?? 0 }, + ) + /// Debug message for #1609 + logger.info( + `[ArchiveUriSupporter#create] Extracted ${files.length} files from ${archiveName}`, + ) + entries.set(archiveName, new Map(files.map((f) => [f.path.replace(/\\/g, '/'), f]))) } catch (e) { - logger.error(`[ArchiveUriSupporter#create] Bad dependency ${uri}`, e) + logger.error(`[ArchiveUriSupporter#create] Bad dependency ${archiveName}`, e) } } diff --git a/packages/core/src/service/Project.ts b/packages/core/src/service/Project.ts index 9643ee7f9..fd0b312f0 100644 --- a/packages/core/src/service/Project.ts +++ b/packages/core/src/service/Project.ts @@ -28,7 +28,6 @@ import { } from './Context.js' import type { Dependency } from './Dependency.js' import { DependencyKey } from './Dependency.js' -import { Downloader } from './Downloader.js' import { LinterErrorReporter } from './ErrorReporter.js' import { ArchiveUriSupporter, FileService, FileUriSupporter } from './FileService.js' import type { RootUriString } from './fileUtil.js' @@ -42,7 +41,6 @@ export type ProjectInitializerContext = Pick< Project, | 'cacheRoot' | 'config' - | 'downloader' | 'externals' | 'isDebugging' | 'logger' @@ -62,7 +60,6 @@ export type ProjectInitializer = SyncProjectInitializer | AsyncProjectInitialize export interface ProjectOptions { cacheRoot: RootUriString defaultConfig?: Config - downloader?: Downloader externals: Externals fs?: FileService initializers?: readonly ProjectInitializer[] @@ -81,7 +78,7 @@ export interface DocAndNode { node: FileNode } -interface DocumentEvent extends DocAndNode {} +interface DocumentEvent extends DocAndNode { } interface DocumentErrorEvent { errors: readonly PosRangeLanguageError[] uri: string @@ -90,7 +87,7 @@ interface DocumentErrorEvent { interface FileEvent { uri: string } -interface EmptyEvent {} +interface EmptyEvent { } interface RootsEvent { roots: readonly RootUriString[] } @@ -103,7 +100,6 @@ export type ProjectData = Pick< Project, | 'cacheRoot' | 'config' - | 'downloader' | 'ensureBindingStarted' | 'externals' | 'fs' @@ -181,7 +177,6 @@ export class Project implements ExternalEventEmitter { } config!: Config - readonly downloader: Downloader readonly externals: Externals readonly fs: FileService readonly isDebugging: boolean @@ -301,7 +296,6 @@ export class Project implements ExternalEventEmitter { { cacheRoot, defaultConfig, - downloader, externals, fs = FileService.create(externals, cacheRoot), initializers = [], @@ -323,7 +317,6 @@ export class Project implements ExternalEventEmitter { this.cacheService = new CacheService(cacheRoot, this) this.#configService = new ConfigService(this, defaultConfig) - this.downloader = downloader ?? new Downloader(cacheRoot, externals, logger) this.symbols = new SymbolUtil({}, externals.event.EventEmitter) this.#ctx = {} @@ -396,7 +389,6 @@ export class Project implements ExternalEventEmitter { const initCtx: ProjectInitializerContext = { cacheRoot: this.cacheRoot, config: this.config, - downloader: this.downloader, externals: this.externals, isDebugging: this.isDebugging, logger: this.logger, @@ -437,28 +429,29 @@ export class Project implements ExternalEventEmitter { private setReadyPromise(): void { const getDependencies = async () => { const ans: Dependency[] = [] - for (const dependency of this.config.env.dependencies) { - if (DependencyKey.is(dependency)) { - const provider = this.meta.getDependencyProvider(dependency) + for (const input of this.config.env.dependencies) { + if (DependencyKey.is(input)) { + const provider = this.meta.getDependencyProvider(input) if (provider) { try { ans.push(await provider()) this.logger.info( - `[Project] [getDependencies] Executed provider “${dependency}”`, + `[Project] [getDependencies] Executed provider “${input}”`, ) } catch (e) { this.logger.error( - `[Project] [getDependencies] Bad provider “${dependency}”`, + `[Project] [getDependencies] Bad provider “${input}”`, e, ) } } else { this.logger.error( - `[Project] [getDependencies] Bad dependency “${dependency}”: no associated provider`, + `[Project] [getDependencies] Bad dependency “${input}”: no associated provider`, ) } } else { - ans.push({ uri: dependency }) + // FIXME: recognize tarball + ans.push({ type: 'directory', uri: input }) } } return ans @@ -823,7 +816,7 @@ export class Project implements ExternalEventEmitter { linter(proxy, ctx) } }) - ;(node.linterErrors as LanguageError[]).push(...ctx.err.dump()) + ; (node.linterErrors as LanguageError[]).push(...ctx.err.dump()) } } catch (e) { this.logger.error(`[Project] [lint] Failed for ${doc.uri} # ${doc.version}`, e) diff --git a/packages/core/src/service/fetcher.ts b/packages/core/src/service/fetcher.ts new file mode 100644 index 000000000..90ef4542b --- /dev/null +++ b/packages/core/src/service/fetcher.ts @@ -0,0 +1,31 @@ +import { Dev, Externals, Logger } from "../common/index.js" + +export async function fetchWithCache({ web }: Externals, logger: Logger, input: RequestInfo | URL, init?: RequestInit): Promise { + const cache = await web.getCache() + const request = new Request(input, init) + const cachedResponse = await cache.match(request) + const cachedEtag = cachedResponse?.headers.get('ETag') + if (cachedEtag) { + request.headers.set('If-None-Match', cachedEtag) + } + try { + const response = await web.fetch(request) + if (response.status === 304) { + Dev.assertDefined(cachedResponse) + return cachedResponse + } else { + try { + await cache.put(request, response) + } catch (e) { + logger.warn('[fetchWithCache] put cache', e) + } + return response + } + } catch (e) { + logger.warn('[fetchWithCache] fetch', e) + if (cachedResponse) { + return cachedResponse + } + throw e + } +} diff --git a/packages/core/src/service/index.ts b/packages/core/src/service/index.ts index 20bc83629..0393a4889 100644 --- a/packages/core/src/service/index.ts +++ b/packages/core/src/service/index.ts @@ -3,8 +3,8 @@ export * from './CacheService.js' export * from './Config.js' export * from './Context.js' +export * from './fetcher.js' export * from './Dependency.js' -export * from './Downloader.js' export * from './ErrorReporter.js' export { FileService, UriProtocolSupporter } from './FileService.js' export * from './fileUtil.js' diff --git a/packages/core/test/utils.ts b/packages/core/test/utils.ts index 6eeaae91c..55bee7770 100644 --- a/packages/core/test/utils.ts +++ b/packages/core/test/utils.ts @@ -11,7 +11,6 @@ import type { import { AstNode, BinderContext, - Downloader, Failure, file, FileService, @@ -53,12 +52,10 @@ export function mockProjectData(data: Partial = {}): ProjectData { const cacheRoot: RootUriString = data.cacheRoot ?? 'file:///cache/' const externals = data.externals ?? NodeJsExternals const logger = data.logger ?? Logger.create() - const downloader = data.downloader ?? new Downloader(cacheRoot, externals, logger) return { cacheRoot, config: data.config ?? VanillaConfig, ctx: data.ctx ?? {}, - downloader, ensureBindingStarted: data.ensureBindingStarted!, externals, fs: data.fs ?? FileService.create(externals, cacheRoot), @@ -154,7 +151,7 @@ export function testParser( * This function has a signature similar to mocha's `describe` and `it` methods. * The method passed into this function is never actually executed. */ -export function typing(_title: string, _fn: () => void): void {} +export function typing(_title: string, _fn: () => void): void { } /** * Assert the type of `_value` is `T`. @@ -168,7 +165,7 @@ export function assertType(_value: T): void { ) } -export function assertError(fn: () => void, errorCallback: (e: unknown) => void = () => {}) { +export function assertError(fn: () => void, errorCallback: (e: unknown) => void = () => { }) { try { fn() fail('Expected an error to be thrown.') diff --git a/packages/java-edition/src/dependency/index.ts b/packages/java-edition/src/dependency/index.ts index 2604133db..72aa15b62 100644 --- a/packages/java-edition/src/dependency/index.ts +++ b/packages/java-edition/src/dependency/index.ts @@ -9,10 +9,7 @@ import type { McmetaSummary, McmetaVersions, } from './mcmeta.js' -import { Fluids, getMcmetaSummaryUris } from './mcmeta.js' - -// Memory cache TTL in milliseconds -const DownloaderTtl = 15_000 +import { Fluids } from './mcmeta.js' /* istanbul ignore next */ /** @@ -22,15 +19,9 @@ const DownloaderTtl = 15_000 */ export async function getVersions( externals: core.Externals, - downloader: core.Downloader, + logger: core.Logger, ): Promise { - return downloader.download({ - id: 'mc-je/versions.json.gz', - uri: 'https://raw.githubusercontent.com/misode/mcmeta/summary/versions/data.json.gz', - transformer: (buffer) => core.parseGzippedJson(externals, buffer) as Promise, - cache: getCacheOptionsBasedOnGitHubCommitSha('misode', 'mcmeta', 'refs/heads/summary'), - ttl: DownloaderTtl, - }) + return (await core.fetchWithCache(externals, logger, 'https://api.spyglassmc.com/mcje/versions')).json() } interface GetMcmetaSummaryResult extends Partial { @@ -45,23 +36,12 @@ interface GetMcmetaSummaryResult extends Partial { */ export async function getMcmetaSummary( externals: core.Externals, - downloader: core.Downloader, logger: core.Logger, version: string, - isLatest: boolean, - source: string, overridePaths: core.EnvConfig['mcmetaSummaryOverrides'] = {}, ): Promise { type OverrideConfig = core.EnvConfig['mcmetaSummaryOverrides'][keyof core.EnvConfig['mcmetaSummaryOverrides']] - const ref = getGitRef({ - defaultBranch: 'summary', - getTag: (v) => `${v}-summary`, - isLatest, - version, - }) - const uris = getMcmetaSummaryUris(version, isLatest, source) - let checksum: string | undefined async function handleOverride(currentValue: T, overrideConfig: OverrideConfig) { if (overrideConfig) { @@ -83,179 +63,94 @@ export async function getMcmetaSummary( } const getResource = async ( - type: 'blocks' | 'commands' | 'registries', + type: 'block_states' | 'commands' | 'registries', overrideConfig: OverrideConfig, - ): Promise => { - const out: core.DownloaderDownloadOut = {} - const data = await downloader.download({ - id: `mc-je/${version}/${type}.json.gz`, - uri: uris[type], - transformer: (buffer) => core.parseGzippedJson(externals, buffer) as Promise, - cache: getCacheOptionsBasedOnGitHubCommitSha('misode', 'mcmeta', ref), - ttl: DownloaderTtl, - }, out) - checksum ||= out.checksum - return handleOverride(data, overrideConfig) + ): Promise<{ data: T | undefined; checksum: string }> => { + const response = await core.fetchWithCache( + externals, + logger, + `https://api.spyglassmc.com/mcje/versions/${encodeURIComponent(version)}/${type}` + ) + return { + data: await handleOverride(await response.json(), overrideConfig), + checksum: response.headers.get('etag') ?? '', + } } const [blocks, commands, fluids, registries] = [ - await getResource('blocks', overridePaths.blocks), + await getResource('block_states', overridePaths.blocks), await getResource('commands', overridePaths.commands), - await handleOverride(Fluids, overridePaths.fluids), + { + data: await handleOverride(Fluids, overridePaths.fluids), + checksum: 'v1', + }, await getResource('registries', overridePaths.registries), ] - return { blocks, commands, fluids, registries, checksum } -} - -type GitHubRefResponse = { message: string } | { - message?: undefined - ref: string - object: { sha: string } -} | { message?: undefined; ref: string; object: { sha: string } }[] - -function getGitRef( - { defaultBranch, getTag, isLatest, version }: { - defaultBranch: string - getTag: (version: string) => string - isLatest: boolean - version: string - }, -): string { - return isLatest ? `refs/heads/${defaultBranch}` : `refs/tags/${getTag(version)}` -} - -const GitHubApiDownloadOptions = { - headers: { Accept: 'application/vnd.github.v3+json', 'User-Agent': 'SpyglassMC' }, -} - -function getCacheOptionsBasedOnGitHubCommitSha(owner: string, repo: string, ref: string) { return { - checksumExtension: '.commit-sha' as const, - checksumJob: { - uri: `https://api.github.com/repos/${owner}/${repo}/git/${ref}` as const, - transformer: (buffer: Uint8Array) => { - const response = JSON.parse(core.bufferToString(buffer)) as GitHubRefResponse - if (Array.isArray(response)) { - return response[0].object.sha - } else if (response.message === undefined) { - return response.object.sha - } else { - throw new Error(response.message) - } - }, - options: GitHubApiDownloadOptions, - ttl: DownloaderTtl, - }, + blocks: blocks.data, + commands: commands.data, + fluids: fluids.data, + registries: registries.data, + checksum: `${blocks.checksum}-${commands.checksum}-${fluids.checksum}-${registries.checksum}` } } -/** - * Download data from a GitHub repository with tags corresponding to Minecraft versions. - * The downloaded data will be cached based on the commit SHA of the respective tag. - * - * If `isLatest` if `true`, instead of finding the tag corresponding to the given version, the default branch will be used. - * - * @returns The URI to the `.tar.gz` file. - */ -async function downloadGitHubRepo( - { defaultBranch, downloader, getTag, repo, isLatest, owner, version, suffix }: { - defaultBranch: string - downloader: core.Downloader - getTag: (version: string) => string - owner: string - repo: string - isLatest: boolean - version: string - suffix?: string - }, -): Promise { - const ref = getGitRef({ defaultBranch, getTag, isLatest, version }) - - const out: core.DownloaderDownloadOut = {} - await downloader.download({ - id: `mc-je/${version}/${repo}${suffix ?? ''}.tar.gz`, - uri: `https://api.github.com/repos/${owner}/${repo}/tarball/${ref}`, - transformer: (b) => b, - cache: getCacheOptionsBasedOnGitHubCommitSha(owner, repo, ref), - options: GitHubApiDownloadOptions, - ttl: DownloaderTtl, - }, out) - - return out.cacheUri! -} - /* istanbul ignore next */ /** * @throws Network/file system errors. - * - * @returns - * - `startDepth`: The amount of level to skip when unzipping the tarball. - * - `uri`: URI to the `.tar.gz` file. */ export async function getVanillaDatapack( - downloader: core.Downloader, + externals: core.Externals, + logger: core.Logger, version: string, - isLatest: boolean, ): Promise { - const uri = await downloadGitHubRepo({ - defaultBranch: 'data', - downloader, - getTag: (v) => `${v}-data`, - owner: 'misode', - repo: 'mcmeta', - isLatest, - version, - }) - return { info: { startDepth: 1 }, uri } + return { + type: 'tarball-ram', + name: 'vanilla-datapack', + data: new Uint8Array(await (await core.fetchWithCache( + externals, + logger, + `https://api.spyglassmc.com/mcje/versions/${encodeURIComponent(version)}/vanilla-data/tarball` + )).arrayBuffer()), + stripLevel: 1, + } } /* istanbul ignore next */ /** * @throws Network/file system errors. - * - * @returns - * - `startDepth`: The amount of level to skip when unzipping the tarball. - * - `uri`: URI to the `.tar.gz` file. */ export async function getVanillaResourcepack( - downloader: core.Downloader, + externals: core.Externals, + logger: core.Logger, version: string, - isLatest: boolean, ): Promise { - const uri = await downloadGitHubRepo({ - defaultBranch: 'assets-tiny', - downloader, - getTag: (v) => `${v}-assets-tiny`, - owner: 'misode', - repo: 'mcmeta', - isLatest, - version, - suffix: '-assets', - }) - return { info: { startDepth: 1 }, uri } + return { + type: 'tarball-ram', + name: 'vanilla-assets-tiny', + data: new Uint8Array(await (await core.fetchWithCache( + externals, + logger, + `https://api.spyglassmc.com/mcje/versions/${encodeURIComponent(version)}/vanilla-assets-tiny/tarball` + )).arrayBuffer()), + stripLevel: 1, + } } /** * @throws Network/file system errors. - * - * @returns - * - `startDepth`: The amount of level to skip when unzipping the tarball. - * - `uri`: URI to the `.tar.gz` file. */ -export async function getVanillaMcdoc(downloader: core.Downloader): Promise { - const owner = 'SpyglassMC' - const repo = 'vanilla-mcdoc' - const ref = 'refs/heads/main' - const out: core.DownloaderDownloadOut = {} - await downloader.download({ - id: 'mc-je/vanilla-mcdoc.tar.gz', - uri: `https://api.github.com/repos/${owner}/${repo}/tarball/${ref}`, - transformer: (b) => b, - cache: getCacheOptionsBasedOnGitHubCommitSha(owner, repo, ref), - options: GitHubApiDownloadOptions, - ttl: DownloaderTtl, - }, out) - - return { info: { startDepth: 1 }, uri: out.cacheUri! } +export async function getVanillaMcdoc( + externals: core.Externals, + logger: core.Logger, +): Promise { + return { + type: 'tarball-ram', + name: 'vanilla-mcdoc', + data: new Uint8Array(await (await core.fetchWithCache( + externals, logger, `https://api.spyglassmc.com/vanilla-mcdoc/tarball` + )).arrayBuffer()), + stripLevel: 1, + } } diff --git a/packages/java-edition/src/dependency/mcmeta.ts b/packages/java-edition/src/dependency/mcmeta.ts index 703545706..f4b56bb2b 100644 --- a/packages/java-edition/src/dependency/mcmeta.ts +++ b/packages/java-edition/src/dependency/mcmeta.ts @@ -111,42 +111,6 @@ export function resolveConfiguredVersion( ) } -const DataSources: Partial> = { - fastly: 'https://fastly.jsdelivr.net/gh/${user}/${repo}@${tag}/${path}', - github: 'https://raw.githubusercontent.com/${user}/${repo}/${tag}/${path}', - jsdelivr: 'https://cdn.jsdelivr.net/gh/${user}/${repo}@${tag}/${path}', -} - -export function getMcmetaSummaryUris( - version: string, - isLatest: boolean, - source: string, -): { - blocks: core.RemoteUriString - commands: core.RemoteUriString - registries: core.RemoteUriString -} { - const tag = isLatest ? 'summary' : `${version}-summary` - - function getUri(path: string): core.RemoteUriString { - const template = DataSources[source.toLowerCase()] ?? source - const ans = template.replace(/\${user}/g, 'misode').replace(/\${repo}/g, 'mcmeta').replace( - /\${tag}/g, - tag, - ).replace(/\${path}/g, path) - if (!core.RemoteUriString.is(ans)) { - throw new Error(`Expected a remote URI from data source template but got ${ans}`) - } - return ans - } - - return { - blocks: getUri('blocks/data.json.gz'), - commands: getUri('commands/data.json.gz'), - registries: getUri('registries/data.json.gz'), - } -} - export function symbolRegistrar( summary: McmetaSummary, release: ReleaseVersion, diff --git a/packages/java-edition/src/index.ts b/packages/java-edition/src/index.ts index e515401c4..30d61548d 100644 --- a/packages/java-edition/src/index.ts +++ b/packages/java-edition/src/index.ts @@ -25,7 +25,7 @@ export * from './mcdocAttributes.js' export * as mcf from './mcfunction/index.js' export const initialize: core.ProjectInitializer = async (ctx) => { - const { config, downloader, externals, logger, meta, projectRoots } = ctx + const { config, externals, logger, meta, projectRoots } = ctx async function readPackMcmeta(uri: string): Promise { try { @@ -74,7 +74,7 @@ export const initialize: core.ProjectInitializer = async (ctx) => { meta.registerUriBinder(uriBinder) registerUriBuilders(meta) - const versions = await getVersions(ctx.externals, ctx.downloader) + const versions = await getVersions(externals, logger) if (!versions) { ctx.logger.error( '[je-initialize] Failed loading game version list. Expect everything to be broken.', @@ -97,13 +97,12 @@ export const initialize: core.ProjectInitializer = async (ctx) => { const reasonMessage = pack && version.reason === 'auto' ? `using ${pack.type} pack format ${pack.packMcmeta?.pack.pack_format} to select` : version.reason === 'config' - ? `but using config override "${config.env.gameVersion}" to select` - : version.reason === 'fallback' - ? 'using fallback' - : 'impossible' // should never occur - const versionMessage = `version ${version.release}${ - version.id === version.release ? '' : ` (${version.id})` - }` + ? `but using config override "${config.env.gameVersion}" to select` + : version.reason === 'fallback' + ? 'using fallback' + : 'impossible' // should never occur + const versionMessage = `version ${version.release}${version.id === version.release ? '' : ` (${version.id})` + }` ctx.logger.info(`[je.initialize] ${packMessage}, ${reasonMessage} ${versionMessage}`) return version } @@ -112,23 +111,20 @@ export const initialize: core.ProjectInitializer = async (ctx) => { meta.registerDependencyProvider( '@vanilla-datapack', - () => getVanillaDatapack(downloader, version.id, version.isLatest), + () => getVanillaDatapack(externals, logger, version.id), ) meta.registerDependencyProvider( '@vanilla-resourcepack', - () => getVanillaResourcepack(downloader, version.id, version.isLatest), + () => getVanillaResourcepack(externals, logger, version.id), ) - meta.registerDependencyProvider('@vanilla-mcdoc', () => getVanillaMcdoc(downloader)) + meta.registerDependencyProvider('@vanilla-mcdoc', () => getVanillaMcdoc(externals, logger)) const summary = await getMcmetaSummary( ctx.externals, - downloader, logger, version.id, - version.isLatest, - config.env.dataSource, config.env.mcmetaSummaryOverrides, ) if (!summary.blocks || !summary.commands || !summary.fluids || !summary.registries) { @@ -139,7 +135,7 @@ export const initialize: core.ProjectInitializer = async (ctx) => { } meta.registerSymbolRegistrar('mcmeta-summary', { - checksum: `${summary.checksum}_v3`, + checksum: `${summary.checksum}-v4`, registrar: symbolRegistrar(summary as McmetaSummary, release), }) diff --git a/packages/java-edition/test/dependency/mcmeta.spec.ts b/packages/java-edition/test/dependency/mcmeta.spec.ts index cd890ee8c..1f0533e7f 100644 --- a/packages/java-edition/test/dependency/mcmeta.spec.ts +++ b/packages/java-edition/test/dependency/mcmeta.spec.ts @@ -7,12 +7,7 @@ import snapshot from 'snap-shot-it' import url from 'url' import type { PackMcmeta } from '../../lib/dependency/common.js' import type { McmetaRegistries, McmetaStates, McmetaVersions } from '../../lib/dependency/mcmeta.js' -import { - Fluids, - getMcmetaSummaryUris, - resolveConfiguredVersion, - symbolRegistrar, -} from '../../lib/dependency/mcmeta.js' +import { Fluids, resolveConfiguredVersion, symbolRegistrar } from '../../lib/dependency/mcmeta.js' function readJsonSync(path: string): unknown { return JSON.parse(fs.readFileSync(path, 'utf-8')) @@ -64,21 +59,6 @@ describe('mcmeta', () => { } }) - describe('getMcmetaSummaryUris()', () => { - const cases: { version: string; isLatest: boolean; source: string }[] = [ - { version: '1.17', isLatest: false, source: 'GitHub' }, - { version: '22w03a', isLatest: true, source: 'GitHub' }, - { version: '1.17', isLatest: false, source: 'jsDelivr' }, - { version: '22w03a', isLatest: true, source: 'jsDelivr' }, - ] - for (const { version, isLatest, source } of cases) { - it(`Should return correctly for "${version}" (${isLatest}) from "${source}"`, async () => { - const actual = getMcmetaSummaryUris(version, isLatest, source) - snapshot(actual) - }) - } - }) - describe('symbolRegistrar()', () => { it('Should register correctly', () => { const registrar = symbolRegistrar({ diff --git a/packages/language-server/src/server.ts b/packages/language-server/src/server.ts index 500d7b1fd..20e3909ba 100644 --- a/packages/language-server/src/server.ts +++ b/packages/language-server/src/server.ts @@ -1,6 +1,6 @@ import * as core from '@spyglassmc/core' import { fileUtil } from '@spyglassmc/core' -import { NodeJsExternals } from '@spyglassmc/core/lib/nodejs.js' +import { getNodeJsExternals } from '@spyglassmc/core/lib/nodejs.js' import * as je from '@spyglassmc/java-edition' import * as locales from '@spyglassmc/locales' import * as mcdoc from '@spyglassmc/mcdoc' @@ -24,15 +24,15 @@ if (process.argv.length === 2) { process.argv.push('--stdio') } -const { cache: cacheRoot } = envPaths('spyglassmc') +const { cache: cacheRootPath } = envPaths('spyglassmc') +const cacheRoot = fileUtil.ensureEndingSlash(url.pathToFileURL(cacheRootPath).toString()) const connection = ls.createConnection() let capabilities!: ls.ClientCapabilities let workspaceFolders!: ls.WorkspaceFolder[] let hasShutdown = false -let progressReporter: ls.WorkDoneProgressReporter | undefined -const externals = NodeJsExternals +const externals = getNodeJsExternals({ cacheRoot }) const logger: core.Logger = { error: (msg: any, ...args: any[]): void => connection.console.error(util.format(msg, ...args)), info: (msg: any, ...args: any[]): void => connection.console.info(util.format(msg, ...args)), @@ -81,7 +81,7 @@ connection.onInitialize(async (params) => { defaultConfig: core.ConfigService.merge(core.VanillaConfig, { env: { gameVersion: initializationOptions?.gameVersion }, }), - cacheRoot: fileUtil.ensureEndingSlash(url.pathToFileURL(cacheRoot).toString()), + cacheRoot, externals, initializers: [mcdoc.initialize, je.initialize], projectRoots: workspaceFolders.map(f => core.fileUtil.ensureEndingSlash(f.uri)), @@ -172,7 +172,7 @@ connection.onDidCloseTextDocument(({ textDocument: { uri } }) => { service.project.onDidClose(uri) }) -connection.workspace.onDidRenameFiles(({}) => {}) +connection.workspace.onDidRenameFiles(({ }) => { }) connection.onCodeAction(async ({ textDocument: { uri }, range }) => { const docAndNode = await service.project.ensureClientManagedChecked(uri) From 3ef54a1fb5b18d7c12b31754e0f8893dd388d8fd Mon Sep 17 00:00:00 2001 From: SPGoding Date: Fri, 7 Feb 2025 02:41:35 -0600 Subject: [PATCH 03/13] =?UTF-8?q?=F0=9F=90=9B=20Lock=20when=20updating=20g?= =?UTF-8?q?it=20repos?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix #1734 --- package-lock.json | 16 ++++++++++++++++ packages/web-api-server/package.json | 1 + packages/web-api-server/src/index.ts | 14 ++++++++++---- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/package-lock.json b/package-lock.json index 447070a5c..d8597ef28 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3147,6 +3147,21 @@ "dev": true, "license": "MIT" }, + "node_modules/async-mutex": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/async-mutex/-/async-mutex-0.5.0.tgz", + "integrity": "sha512-1A94B18jkJ3DYq284ohPxoXbfTA5HsQ7/Mf4DEhcyLx3Bz27Rh59iScbB6EPiP+B+joue6YCxcMXSbFC1tZKwA==", + "license": "MIT", + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/async-mutex/node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, "node_modules/asynckit": { "version": "0.4.0", "dev": true, @@ -13517,6 +13532,7 @@ "version": "0.1.0-PLACEHOLDER", "license": "MIT", "dependencies": { + "async-mutex": "^0.5.0", "chalk": "^5.4.1", "cors": "^2.8.5", "express": "^5.0.1", diff --git a/packages/web-api-server/package.json b/packages/web-api-server/package.json index a22e65121..e2dcb7b2d 100644 --- a/packages/web-api-server/package.json +++ b/packages/web-api-server/package.json @@ -16,6 +16,7 @@ "start": "./bin/server.js" }, "dependencies": { + "async-mutex": "^0.5.0", "chalk": "^5.4.1", "cors": "^2.8.5", "express": "^5.0.1", diff --git a/packages/web-api-server/src/index.ts b/packages/web-api-server/src/index.ts index ef39b5af9..af052703b 100644 --- a/packages/web-api-server/src/index.ts +++ b/packages/web-api-server/src/index.ts @@ -2,7 +2,8 @@ import chalk from 'chalk' import cors from 'cors' import express from 'express' import { slowDown } from 'express-slow-down' -import { fileURLToPath } from 'url' +import { Mutex } from 'async-mutex' +import { fileURLToPath } from 'node:url' import { assertRootDir, cheapRateLimiter, @@ -23,6 +24,7 @@ const { hookSecret, port, rootDir } = loadConfig() await assertRootDir(rootDir) const gits = await initGitRepos(rootDir) const cache = new MemCache(gits.mcmeta) +const gitMutex = new Mutex() const versionRoute = express.Router({ mergeParams: true }) .use(getVersionValidator(cache)) @@ -111,9 +113,13 @@ const app = express() repository: { name: string } } const git = gits[name === 'vanilla-mcdoc' ? 'mcdoc' : 'mcmeta'] - console.info(chalk.yellow(`Updating ${name}...`)) - await git.remote(['update', '--prune']) - console.info(chalk.green(`Updated ${name}`)) + + await gitMutex.runExclusive(async () => { + console.info(chalk.yellow(`Updating ${name}...`)) + await git.remote(['update', '--prune']) + cache.invalidate() + console.info(chalk.green(`Updated ${name}`)) + }) }, ) .get('/favicon.ico', cheapRateLimiter, (_req, res) => { From f05d2dfa8d5d60c950823a60ecdbec658b3e4074 Mon Sep 17 00:00:00 2001 From: SPGoding Date: Fri, 7 Feb 2025 02:42:56 -0600 Subject: [PATCH 04/13] =?UTF-8?q?=F0=9F=8E=A8=20Format?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/common/externals/BrowserExternals.ts | 2 +- .../src/common/externals/NodeJsExternals.ts | 187 +++++++++--------- packages/core/src/common/externals/index.ts | 6 +- packages/core/src/service/FileService.ts | 6 +- packages/core/src/service/Project.ts | 6 +- packages/core/src/service/fetcher.ts | 10 +- packages/core/src/service/index.ts | 2 +- packages/core/test/utils.ts | 4 +- packages/java-edition/src/dependency/index.ts | 45 +++-- packages/java-edition/src/index.ts | 13 +- packages/language-server/src/server.ts | 2 +- packages/web-api-server/src/index.ts | 2 +- 12 files changed, 157 insertions(+), 128 deletions(-) diff --git a/packages/core/src/common/externals/BrowserExternals.ts b/packages/core/src/common/externals/BrowserExternals.ts index a9240d148..c7ef92eb9 100644 --- a/packages/core/src/common/externals/BrowserExternals.ts +++ b/packages/core/src/common/externals/BrowserExternals.ts @@ -63,7 +63,7 @@ class BrowserFsWatcher implements FsWatcher { return this } - async close(): Promise { } + async close(): Promise {} } class BrowserFileSystem implements ExternalFileSystem { diff --git a/packages/core/src/common/externals/NodeJsExternals.ts b/packages/core/src/common/externals/NodeJsExternals.ts index d7c95bcc6..518c2612c 100644 --- a/packages/core/src/common/externals/NodeJsExternals.ts +++ b/packages/core/src/common/externals/NodeJsExternals.ts @@ -5,109 +5,114 @@ import { Buffer } from 'node:buffer' import cp from 'node:child_process' import crypto from 'node:crypto' import { EventEmitter } from 'node:events' -import os from 'node:os' import fs, { promises as fsp } from 'node:fs' +import os from 'node:os' import process from 'node:process' import stream from 'node:stream' -import streamWeb from 'node:stream/web' +import type streamWeb from 'node:stream/web' import url from 'node:url' import { promisify } from 'node:util' import zlib from 'node:zlib' +import type { RootUriString } from '../../index.js' import { Uri } from '../util.js' import type { Externals, FsLocation, FsWatcher } from './index.js' -import type { RootUriString } from '../../index.js' const gunzip = promisify(zlib.gunzip) const gzip = promisify(zlib.gzip) export function getNodeJsExternals({ cacheRoot }: { cacheRoot?: RootUriString } = {}) { - return Object.freeze({ - archive: { - decompressBall(buffer, options) { - if (!(buffer instanceof Buffer)) { - buffer = Buffer.from(buffer) - } - return decompress(buffer as Buffer, { strip: options?.stripLevel }) - }, - gunzip(buffer) { - return gunzip(buffer) - }, - gzip(buffer) { - return gzip(buffer) - }, - }, - crypto: { - async getSha1(data) { - const hash = crypto.createHash('sha1') - hash.update(data) - return hash.digest('hex') - }, - }, - error: { - createKind(kind, message) { - const error = new Error(message) - ; (error as NodeJS.ErrnoException).code = kind - return error + return Object.freeze( + { + archive: { + decompressBall(buffer, options) { + if (!(buffer instanceof Buffer)) { + buffer = Buffer.from(buffer) + } + return decompress(buffer as Buffer, { strip: options?.stripLevel }) + }, + gunzip(buffer) { + return gunzip(buffer) + }, + gzip(buffer) { + return gzip(buffer) + }, }, - isKind(e, kind) { - return e instanceof Error && (e as NodeJS.ErrnoException).code === kind + crypto: { + async getSha1(data) { + const hash = crypto.createHash('sha1') + hash.update(data) + return hash.digest('hex') + }, }, - }, - event: { EventEmitter }, - fs: { - chmod(location, mode) { - return fsp.chmod(toFsPathLike(location), mode) + error: { + createKind(kind, message) { + const error = new Error(message) + ;(error as NodeJS.ErrnoException).code = kind + return error + }, + isKind(e, kind) { + return e instanceof Error && (e as NodeJS.ErrnoException).code === kind + }, }, - async mkdir(location, options) { - return void (await fsp.mkdir(toFsPathLike(location), options)) + event: { EventEmitter }, + fs: { + chmod(location, mode) { + return fsp.chmod(toFsPathLike(location), mode) + }, + async mkdir(location, options) { + return void (await fsp.mkdir(toFsPathLike(location), options)) + }, + readdir(location) { + return fsp.readdir(toFsPathLike(location), { + encoding: 'utf-8', + withFileTypes: true, + }) + }, + readFile(location) { + return fsp.readFile(toFsPathLike(location)) + }, + async showFile(location): Promise { + const execFile = promisify(cp.execFile) + let command: string + switch (process.platform) { + case 'darwin': + command = 'open' + break + case 'win32': + command = 'explorer' + break + default: + command = 'xdg-open' + break + } + return void (await execFile(command, [toPath(location)])) + }, + stat(location) { + return fsp.stat(toFsPathLike(location)) + }, + unlink(location) { + return fsp.unlink(toFsPathLike(location)) + }, + watch(locations, { usePolling = false } = {}) { + return new ChokidarWatcherWrapper( + chokidar.watch(locations.map(toPath), { + usePolling, + disableGlobbing: true, + }), + ) + }, + writeFile(location, data, options) { + return fsp.writeFile(toFsPathLike(location), data, options) + }, }, - readdir(location) { - return fsp.readdir(toFsPathLike(location), { encoding: 'utf-8', withFileTypes: true }) + web: { + fetch, + getCache: async () => { + return new HttpCache(cacheRoot) + }, }, - readFile(location) { - return fsp.readFile(toFsPathLike(location)) - }, - async showFile(location): Promise { - const execFile = promisify(cp.execFile) - let command: string - switch (process.platform) { - case 'darwin': - command = 'open' - break - case 'win32': - command = 'explorer' - break - default: - command = 'xdg-open' - break - } - return void (await execFile(command, [toPath(location)])) - }, - stat(location) { - return fsp.stat(toFsPathLike(location)) - }, - unlink(location) { - return fsp.unlink(toFsPathLike(location)) - }, - watch(locations, { usePolling = false } = {}) { - return new ChokidarWatcherWrapper( - chokidar.watch(locations.map(toPath), { - usePolling, - disableGlobbing: true, - }), - ) - }, - writeFile(location, data, options) { - return fsp.writeFile(toFsPathLike(location), data, options) - }, - }, - web: { - fetch, - getCache: async () => { - return new HttpCache(cacheRoot) - } - }, - } satisfies Externals) + } satisfies Externals, + ) } export const NodeJsExternals = getNodeJsExternals() @@ -169,14 +174,18 @@ class HttpCache implements Cache { } } - async match(request: RequestInfo | URL, _options?: CacheQueryOptions | undefined): Promise { + async match( + request: RequestInfo | URL, + _options?: CacheQueryOptions | undefined, + ): Promise { if (!this.#cacheRoot) { return undefined } const fileName = this.#getFileName(request) try { - const etag = (await fsp.readFile(new URL(`${fileName}.etag`, this.#cacheRoot), 'utf8')).trim() + const etag = (await fsp.readFile(new URL(`${fileName}.etag`, this.#cacheRoot), 'utf8')) + .trim() const bodyStream = fs.createReadStream(new URL(`${fileName}.bin`, this.#cacheRoot)) return new Response( stream.Readable.toWeb(bodyStream) as ReadableStream, @@ -207,12 +216,12 @@ class HttpCache implements Cache { await Promise.all([ fsp.writeFile( new URL(`${fileName}.bin`, this.#cacheRoot), - stream.Readable.fromWeb(clonedResponse.body as streamWeb.ReadableStream) + stream.Readable.fromWeb(clonedResponse.body as streamWeb.ReadableStream), // \_____/ \_________________________/ // | DOM ReadableStream -> stream/web ReadableStream // stream/web ReadableStream -> stream Readable ), - fsp.writeFile(new URL(`${fileName}.etag`, this.#cacheRoot), `${etag}${os.EOL}`) + fsp.writeFile(new URL(`${fileName}.etag`, this.#cacheRoot), `${etag}${os.EOL}`), ]) } diff --git a/packages/core/src/common/externals/index.ts b/packages/core/src/common/externals/index.ts index cc8f23de6..c4843bd50 100644 --- a/packages/core/src/common/externals/index.ts +++ b/packages/core/src/common/externals/index.ts @@ -25,11 +25,11 @@ export interface Externals { */ isKind: (e: unknown, kind: ExternalErrorKind) => boolean } - event: { EventEmitter: new () => ExternalEventEmitter } + event: { EventEmitter: new() => ExternalEventEmitter } fs: ExternalFileSystem web: { - fetch: typeof fetch, - getCache: () => Promise, + fetch: typeof fetch + getCache: () => Promise } } diff --git a/packages/core/src/service/FileService.ts b/packages/core/src/service/FileService.ts index 242a80fb4..c5e785793 100644 --- a/packages/core/src/service/FileService.ts +++ b/packages/core/src/service/FileService.ts @@ -78,7 +78,7 @@ export class FileServiceImpl implements FileService { constructor( private readonly externals: Externals, private readonly virtualUrisRoot?: RootUriString, - ) { } + ) {} register(protocol: Protocol, supporter: UriProtocolSupporter, force = false): void { if (!force && this.supporters.has(protocol)) { @@ -183,7 +183,7 @@ export class FileUriSupporter implements UriProtocolSupporter { private readonly externals: Externals, private readonly roots: RootUriString[], private readonly files: Map, - ) { } + ) {} async hash(uri: string): Promise { return hashFile(this.externals, uri) @@ -249,7 +249,7 @@ export class ArchiveUriSupporter implements UriProtocolSupporter { private readonly externals: Externals, private readonly logger: Logger, private readonly entries: Map>, - ) { } + ) {} async hash(uri: string): Promise { const { archiveName, pathInArchive } = ArchiveUriSupporter.decodeUri(new Uri(uri)) diff --git a/packages/core/src/service/Project.ts b/packages/core/src/service/Project.ts index fd0b312f0..0db853fe8 100644 --- a/packages/core/src/service/Project.ts +++ b/packages/core/src/service/Project.ts @@ -78,7 +78,7 @@ export interface DocAndNode { node: FileNode } -interface DocumentEvent extends DocAndNode { } +interface DocumentEvent extends DocAndNode {} interface DocumentErrorEvent { errors: readonly PosRangeLanguageError[] uri: string @@ -87,7 +87,7 @@ interface DocumentErrorEvent { interface FileEvent { uri: string } -interface EmptyEvent { } +interface EmptyEvent {} interface RootsEvent { roots: readonly RootUriString[] } @@ -816,7 +816,7 @@ export class Project implements ExternalEventEmitter { linter(proxy, ctx) } }) - ; (node.linterErrors as LanguageError[]).push(...ctx.err.dump()) + ;(node.linterErrors as LanguageError[]).push(...ctx.err.dump()) } } catch (e) { this.logger.error(`[Project] [lint] Failed for ${doc.uri} # ${doc.version}`, e) diff --git a/packages/core/src/service/fetcher.ts b/packages/core/src/service/fetcher.ts index 90ef4542b..b20a4acf1 100644 --- a/packages/core/src/service/fetcher.ts +++ b/packages/core/src/service/fetcher.ts @@ -1,6 +1,12 @@ -import { Dev, Externals, Logger } from "../common/index.js" +import { Dev } from '../common/index.js' +import type { Externals, Logger } from '../common/index.js' -export async function fetchWithCache({ web }: Externals, logger: Logger, input: RequestInfo | URL, init?: RequestInit): Promise { +export async function fetchWithCache( + { web }: Externals, + logger: Logger, + input: RequestInfo | URL, + init?: RequestInit, +): Promise { const cache = await web.getCache() const request = new Request(input, init) const cachedResponse = await cache.match(request) diff --git a/packages/core/src/service/index.ts b/packages/core/src/service/index.ts index 0393a4889..1c215785c 100644 --- a/packages/core/src/service/index.ts +++ b/packages/core/src/service/index.ts @@ -3,9 +3,9 @@ export * from './CacheService.js' export * from './Config.js' export * from './Context.js' -export * from './fetcher.js' export * from './Dependency.js' export * from './ErrorReporter.js' +export * from './fetcher.js' export { FileService, UriProtocolSupporter } from './FileService.js' export * from './fileUtil.js' export * from './Hover.js' diff --git a/packages/core/test/utils.ts b/packages/core/test/utils.ts index 55bee7770..637b5e607 100644 --- a/packages/core/test/utils.ts +++ b/packages/core/test/utils.ts @@ -151,7 +151,7 @@ export function testParser( * This function has a signature similar to mocha's `describe` and `it` methods. * The method passed into this function is never actually executed. */ -export function typing(_title: string, _fn: () => void): void { } +export function typing(_title: string, _fn: () => void): void {} /** * Assert the type of `_value` is `T`. @@ -165,7 +165,7 @@ export function assertType(_value: T): void { ) } -export function assertError(fn: () => void, errorCallback: (e: unknown) => void = () => { }) { +export function assertError(fn: () => void, errorCallback: (e: unknown) => void = () => {}) { try { fn() fail('Expected an error to be thrown.') diff --git a/packages/java-edition/src/dependency/index.ts b/packages/java-edition/src/dependency/index.ts index 72aa15b62..c417c9a43 100644 --- a/packages/java-edition/src/dependency/index.ts +++ b/packages/java-edition/src/dependency/index.ts @@ -21,7 +21,8 @@ export async function getVersions( externals: core.Externals, logger: core.Logger, ): Promise { - return (await core.fetchWithCache(externals, logger, 'https://api.spyglassmc.com/mcje/versions')).json() + return (await core.fetchWithCache(externals, logger, 'https://api.spyglassmc.com/mcje/versions')) + .json() } interface GetMcmetaSummaryResult extends Partial { @@ -69,7 +70,7 @@ export async function getMcmetaSummary( const response = await core.fetchWithCache( externals, logger, - `https://api.spyglassmc.com/mcje/versions/${encodeURIComponent(version)}/${type}` + `https://api.spyglassmc.com/mcje/versions/${encodeURIComponent(version)}/${type}`, ) return { data: await handleOverride(await response.json(), overrideConfig), @@ -92,7 +93,7 @@ export async function getMcmetaSummary( commands: commands.data, fluids: fluids.data, registries: registries.data, - checksum: `${blocks.checksum}-${commands.checksum}-${fluids.checksum}-${registries.checksum}` + checksum: `${blocks.checksum}-${commands.checksum}-${fluids.checksum}-${registries.checksum}`, } } @@ -108,11 +109,15 @@ export async function getVanillaDatapack( return { type: 'tarball-ram', name: 'vanilla-datapack', - data: new Uint8Array(await (await core.fetchWithCache( - externals, - logger, - `https://api.spyglassmc.com/mcje/versions/${encodeURIComponent(version)}/vanilla-data/tarball` - )).arrayBuffer()), + data: new Uint8Array( + await (await core.fetchWithCache( + externals, + logger, + `https://api.spyglassmc.com/mcje/versions/${ + encodeURIComponent(version) + }/vanilla-data/tarball`, + )).arrayBuffer(), + ), stripLevel: 1, } } @@ -129,11 +134,15 @@ export async function getVanillaResourcepack( return { type: 'tarball-ram', name: 'vanilla-assets-tiny', - data: new Uint8Array(await (await core.fetchWithCache( - externals, - logger, - `https://api.spyglassmc.com/mcje/versions/${encodeURIComponent(version)}/vanilla-assets-tiny/tarball` - )).arrayBuffer()), + data: new Uint8Array( + await (await core.fetchWithCache( + externals, + logger, + `https://api.spyglassmc.com/mcje/versions/${ + encodeURIComponent(version) + }/vanilla-assets-tiny/tarball`, + )).arrayBuffer(), + ), stripLevel: 1, } } @@ -148,9 +157,13 @@ export async function getVanillaMcdoc( return { type: 'tarball-ram', name: 'vanilla-mcdoc', - data: new Uint8Array(await (await core.fetchWithCache( - externals, logger, `https://api.spyglassmc.com/vanilla-mcdoc/tarball` - )).arrayBuffer()), + data: new Uint8Array( + await (await core.fetchWithCache( + externals, + logger, + `https://api.spyglassmc.com/vanilla-mcdoc/tarball`, + )).arrayBuffer(), + ), stripLevel: 1, } } diff --git a/packages/java-edition/src/index.ts b/packages/java-edition/src/index.ts index 30d61548d..77d6929a1 100644 --- a/packages/java-edition/src/index.ts +++ b/packages/java-edition/src/index.ts @@ -97,12 +97,13 @@ export const initialize: core.ProjectInitializer = async (ctx) => { const reasonMessage = pack && version.reason === 'auto' ? `using ${pack.type} pack format ${pack.packMcmeta?.pack.pack_format} to select` : version.reason === 'config' - ? `but using config override "${config.env.gameVersion}" to select` - : version.reason === 'fallback' - ? 'using fallback' - : 'impossible' // should never occur - const versionMessage = `version ${version.release}${version.id === version.release ? '' : ` (${version.id})` - }` + ? `but using config override "${config.env.gameVersion}" to select` + : version.reason === 'fallback' + ? 'using fallback' + : 'impossible' // should never occur + const versionMessage = `version ${version.release}${ + version.id === version.release ? '' : ` (${version.id})` + }` ctx.logger.info(`[je.initialize] ${packMessage}, ${reasonMessage} ${versionMessage}`) return version } diff --git a/packages/language-server/src/server.ts b/packages/language-server/src/server.ts index 20e3909ba..cd8f6ba7d 100644 --- a/packages/language-server/src/server.ts +++ b/packages/language-server/src/server.ts @@ -172,7 +172,7 @@ connection.onDidCloseTextDocument(({ textDocument: { uri } }) => { service.project.onDidClose(uri) }) -connection.workspace.onDidRenameFiles(({ }) => { }) +connection.workspace.onDidRenameFiles(({}) => {}) connection.onCodeAction(async ({ textDocument: { uri }, range }) => { const docAndNode = await service.project.ensureClientManagedChecked(uri) diff --git a/packages/web-api-server/src/index.ts b/packages/web-api-server/src/index.ts index af052703b..c8d90597f 100644 --- a/packages/web-api-server/src/index.ts +++ b/packages/web-api-server/src/index.ts @@ -1,8 +1,8 @@ +import { Mutex } from 'async-mutex' import chalk from 'chalk' import cors from 'cors' import express from 'express' import { slowDown } from 'express-slow-down' -import { Mutex } from 'async-mutex' import { fileURLToPath } from 'node:url' import { assertRootDir, From d43b24b10fb4c5c1c9ba36d2f4c543fabaa44564 Mon Sep 17 00:00:00 2001 From: SPGoding Date: Fri, 7 Feb 2025 02:49:53 -0600 Subject: [PATCH 05/13] =?UTF-8?q?=F0=9F=93=B8=20Update=20snapshots?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../test-out/dependency/mcmeta.spec.js | 24 ------------------- packages/core/src/service/fetcher.ts | 2 ++ 2 files changed, 2 insertions(+), 24 deletions(-) diff --git a/__snapshots__/packages/java-edition/test-out/dependency/mcmeta.spec.js b/__snapshots__/packages/java-edition/test-out/dependency/mcmeta.spec.js index d62340356..41f07ab99 100644 --- a/__snapshots__/packages/java-edition/test-out/dependency/mcmeta.spec.js +++ b/__snapshots__/packages/java-edition/test-out/dependency/mcmeta.spec.js @@ -1,27 +1,3 @@ -exports['mcmeta getMcmetaSummaryUris() Should return correctly for "1.17" (false) from "GitHub" 1'] = { - "blocks": "https://raw.githubusercontent.com/misode/mcmeta/1.17-summary/blocks/data.json.gz", - "commands": "https://raw.githubusercontent.com/misode/mcmeta/1.17-summary/commands/data.json.gz", - "registries": "https://raw.githubusercontent.com/misode/mcmeta/1.17-summary/registries/data.json.gz" -} - -exports['mcmeta getMcmetaSummaryUris() Should return correctly for "1.17" (false) from "jsDelivr" 1'] = { - "blocks": "https://cdn.jsdelivr.net/gh/misode/mcmeta@1.17-summary/blocks/data.json.gz", - "commands": "https://cdn.jsdelivr.net/gh/misode/mcmeta@1.17-summary/commands/data.json.gz", - "registries": "https://cdn.jsdelivr.net/gh/misode/mcmeta@1.17-summary/registries/data.json.gz" -} - -exports['mcmeta getMcmetaSummaryUris() Should return correctly for "22w03a" (true) from "GitHub" 1'] = { - "blocks": "https://raw.githubusercontent.com/misode/mcmeta/summary/blocks/data.json.gz", - "commands": "https://raw.githubusercontent.com/misode/mcmeta/summary/commands/data.json.gz", - "registries": "https://raw.githubusercontent.com/misode/mcmeta/summary/registries/data.json.gz" -} - -exports['mcmeta getMcmetaSummaryUris() Should return correctly for "22w03a" (true) from "jsDelivr" 1'] = { - "blocks": "https://cdn.jsdelivr.net/gh/misode/mcmeta@summary/blocks/data.json.gz", - "commands": "https://cdn.jsdelivr.net/gh/misode/mcmeta@summary/commands/data.json.gz", - "registries": "https://cdn.jsdelivr.net/gh/misode/mcmeta@summary/registries/data.json.gz" -} - exports['mcmeta resolveConfiguredVersion() Should resolve "1.16.5" 1'] = { "id": "1.16.5", "name": "1.16.5", diff --git a/packages/core/src/service/fetcher.ts b/packages/core/src/service/fetcher.ts index b20a4acf1..83db3ee53 100644 --- a/packages/core/src/service/fetcher.ts +++ b/packages/core/src/service/fetcher.ts @@ -35,3 +35,5 @@ export async function fetchWithCache( throw e } } + +// Fetchr? I hardly know her: https://github.com/NeunEinser/bingo From 79390db6818ae366a7315cfb81ff2b88715b074e Mon Sep 17 00:00:00 2001 From: SPGoding Date: Fri, 7 Feb 2025 02:55:33 -0600 Subject: [PATCH 06/13] =?UTF-8?q?=F0=9F=90=9B=20Detect=20user=20input=20de?= =?UTF-8?q?pendency=20type?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/core/src/service/Project.ts | 43 ++++++++++++++-------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/packages/core/src/service/Project.ts b/packages/core/src/service/Project.ts index 0db853fe8..0b6075883 100644 --- a/packages/core/src/service/Project.ts +++ b/packages/core/src/service/Project.ts @@ -428,33 +428,34 @@ export class Project implements ExternalEventEmitter { private setReadyPromise(): void { const getDependencies = async () => { - const ans: Dependency[] = [] + const dependencies: Dependency[] = [] for (const input of this.config.env.dependencies) { - if (DependencyKey.is(input)) { - const provider = this.meta.getDependencyProvider(input) - if (provider) { - try { - ans.push(await provider()) - this.logger.info( - `[Project] [getDependencies] Executed provider “${input}”`, - ) - } catch (e) { - this.logger.error( - `[Project] [getDependencies] Bad provider “${input}”`, - e, - ) + try { + if (DependencyKey.is(input)) { + const provider = this.meta.getDependencyProvider(input) + if (!provider) { + throw new Error(`No provider for ${input}`) } - } else { - this.logger.error( - `[Project] [getDependencies] Bad dependency “${input}”: no associated provider`, + + dependencies.push(await provider()) + this.logger.info( + `[Project] [getDependencies] Executed provider “${input}”`, ) + } else { + const stats = await this.externals.fs.stat(input) + if (stats.isDirectory()) { + dependencies.push({ type: 'directory', uri: input }) + } else if (stats.isFile()) { + dependencies.push({ type: 'tarball-file', uri: input }) + } else { + throw new Error('Unsupported file entry type') + } } - } else { - // FIXME: recognize tarball - ans.push({ type: 'directory', uri: input }) + } catch (e) { + this.logger.error(`[Project] [getDependencies] Bad dependency “${input}”`, e) } } - return ans + return dependencies } const listDependencyFiles = async () => { const dependencies = await getDependencies() From dece85949b604ff9d6e3e3b8851b0c9bcaa2e19b Mon Sep 17 00:00:00 2001 From: SPGoding Date: Fri, 7 Feb 2025 03:02:16 -0600 Subject: [PATCH 07/13] =?UTF-8?q?=F0=9F=94=8A=20Add=20more=20logs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/core/src/service/fetcher.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/core/src/service/fetcher.ts b/packages/core/src/service/fetcher.ts index 83db3ee53..fc1b01f6b 100644 --- a/packages/core/src/service/fetcher.ts +++ b/packages/core/src/service/fetcher.ts @@ -18,10 +18,12 @@ export async function fetchWithCache( const response = await web.fetch(request) if (response.status === 304) { Dev.assertDefined(cachedResponse) + logger.info(`[fetchWithCache] reusing cache for ${request.url}`) return cachedResponse } else { try { await cache.put(request, response) + logger.info(`[fetchWithCache] updated cache for ${request.url}`) } catch (e) { logger.warn('[fetchWithCache] put cache', e) } @@ -30,6 +32,7 @@ export async function fetchWithCache( } catch (e) { logger.warn('[fetchWithCache] fetch', e) if (cachedResponse) { + logger.info(`[fetchWithCache] falling back to cache for ${request.url}`) return cachedResponse } throw e From 12768a3532bf933af90ada29087c4663fe6df477 Mon Sep 17 00:00:00 2001 From: SPGoding Date: Sun, 9 Feb 2025 20:10:59 -0600 Subject: [PATCH 08/13] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Remove=20isLatest=20?= =?UTF-8?q?from=20VersionInfo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/java-edition/src/dependency/common.ts | 1 - packages/java-edition/src/dependency/mcmeta.ts | 1 - 2 files changed, 2 deletions(-) diff --git a/packages/java-edition/src/dependency/common.ts b/packages/java-edition/src/dependency/common.ts index f6b5ba16f..61defbea6 100644 --- a/packages/java-edition/src/dependency/common.ts +++ b/packages/java-edition/src/dependency/common.ts @@ -31,7 +31,6 @@ export interface VersionInfo { release: ReleaseVersion id: string name: string - isLatest: boolean reason: VersionInfoReason } diff --git a/packages/java-edition/src/dependency/mcmeta.ts b/packages/java-edition/src/dependency/mcmeta.ts index f4b56bb2b..9ec4396e5 100644 --- a/packages/java-edition/src/dependency/mcmeta.ts +++ b/packages/java-edition/src/dependency/mcmeta.ts @@ -41,7 +41,6 @@ export function resolveConfiguredVersion( id: version.id, name: version.name, release: findReleaseTarget(version) as ReleaseVersion, - isLatest: version === versions[0], reason, } } From 6808fe97d3009a606f5baf1b784ba67342b6ee0e Mon Sep 17 00:00:00 2001 From: SPGoding Date: Mon, 10 Feb 2025 13:01:03 -0600 Subject: [PATCH 09/13] =?UTF-8?q?=F0=9F=93=B8=20Update=20snapshots?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java-edition/test-out/dependency/mcmeta.spec.js | 7 ------- 1 file changed, 7 deletions(-) diff --git a/__snapshots__/packages/java-edition/test-out/dependency/mcmeta.spec.js b/__snapshots__/packages/java-edition/test-out/dependency/mcmeta.spec.js index 41f07ab99..573e3fa70 100644 --- a/__snapshots__/packages/java-edition/test-out/dependency/mcmeta.spec.js +++ b/__snapshots__/packages/java-edition/test-out/dependency/mcmeta.spec.js @@ -2,7 +2,6 @@ exports['mcmeta resolveConfiguredVersion() Should resolve "1.16.5" 1'] = { "id": "1.16.5", "name": "1.16.5", "release": "1.16.5", - "isLatest": false, "reason": "config" } @@ -10,7 +9,6 @@ exports['mcmeta resolveConfiguredVersion() Should resolve "20w06a" 1'] = { "id": "20w06a", "name": "Snapshot 20w06a", "release": "1.16", - "isLatest": false, "reason": "config" } @@ -18,7 +16,6 @@ exports['mcmeta resolveConfiguredVersion() Should resolve "22w03a" 1'] = { "id": "22w03a", "name": "22w03a", "release": "1.18.2", - "isLatest": true, "reason": "config" } @@ -26,7 +23,6 @@ exports['mcmeta resolveConfiguredVersion() Should resolve "Auto" 1'] = { "id": "1.16.5", "name": "1.16.5", "release": "1.16.5", - "isLatest": false, "reason": "auto" } @@ -34,7 +30,6 @@ exports['mcmeta resolveConfiguredVersion() Should resolve "Latest Release" 1'] = "id": "1.18.1", "name": "1.18.1", "release": "1.18.1", - "isLatest": false, "reason": "config" } @@ -42,7 +37,6 @@ exports['mcmeta resolveConfiguredVersion() Should resolve "Latest Snapshot" 1'] "id": "22w03a", "name": "22w03a", "release": "1.18.2", - "isLatest": true, "reason": "config" } @@ -50,7 +44,6 @@ exports['mcmeta resolveConfiguredVersion() Should resolve "unknown" 1'] = { "id": "22w03a", "name": "22w03a", "release": "1.18.2", - "isLatest": true, "reason": "config" } From 93f62dd651d5bbdf9e5df971a9b59f1edcffa1ba Mon Sep 17 00:00:00 2001 From: SPGoding Date: Sun, 16 Feb 2025 00:43:18 -0600 Subject: [PATCH 10/13] =?UTF-8?q?=F0=9F=90=9B=20Fix=20stripLevel=20for=20v?= =?UTF-8?q?anilla-mcdoc?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/java-edition/src/dependency/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/java-edition/src/dependency/index.ts b/packages/java-edition/src/dependency/index.ts index c417c9a43..7701a51f1 100644 --- a/packages/java-edition/src/dependency/index.ts +++ b/packages/java-edition/src/dependency/index.ts @@ -164,6 +164,6 @@ export async function getVanillaMcdoc( `https://api.spyglassmc.com/vanilla-mcdoc/tarball`, )).arrayBuffer(), ), - stripLevel: 1, + stripLevel: 0, } } From fa4d9f3580f3de139550167cbea10a8bd1bdc626 Mon Sep 17 00:00:00 2001 From: Misode Date: Mon, 17 Feb 2025 21:58:31 +0100 Subject: [PATCH 11/13] =?UTF-8?q?=F0=9F=90=9B=20Fix=20stripLevel=20for=20v?= =?UTF-8?q?anilla=20data=20and=20resource=20packs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/java-edition/src/dependency/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/java-edition/src/dependency/index.ts b/packages/java-edition/src/dependency/index.ts index 7701a51f1..4b9db32f4 100644 --- a/packages/java-edition/src/dependency/index.ts +++ b/packages/java-edition/src/dependency/index.ts @@ -118,7 +118,7 @@ export async function getVanillaDatapack( }/vanilla-data/tarball`, )).arrayBuffer(), ), - stripLevel: 1, + stripLevel: 0, } } @@ -143,7 +143,7 @@ export async function getVanillaResourcepack( }/vanilla-assets-tiny/tarball`, )).arrayBuffer(), ), - stripLevel: 1, + stripLevel: 0, } } From d6022e819fac939de370be9c65a0df3b3717ba72 Mon Sep 17 00:00:00 2001 From: Misode Date: Mon, 17 Feb 2025 23:18:58 +0100 Subject: [PATCH 12/13] =?UTF-8?q?=F0=9F=90=9B=20Handle=20non-2xx=20respons?= =?UTF-8?q?es=20in=20fetchWithCache?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/core/src/service/fetcher.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/core/src/service/fetcher.ts b/packages/core/src/service/fetcher.ts index fc1b01f6b..7acac0889 100644 --- a/packages/core/src/service/fetcher.ts +++ b/packages/core/src/service/fetcher.ts @@ -20,6 +20,12 @@ export async function fetchWithCache( Dev.assertDefined(cachedResponse) logger.info(`[fetchWithCache] reusing cache for ${request.url}`) return cachedResponse + } else if (!response.ok) { + let message = response.statusText + try { + message = (await response.json()).message + } catch (e) {} + throw new TypeError(`${response.status} ${message}`) } else { try { await cache.put(request, response) From 6ac47a47ea0176c0690c1851070ca8772e98f5ef Mon Sep 17 00:00:00 2001 From: SPGoding Date: Thu, 20 Feb 2025 02:05:41 -0600 Subject: [PATCH 13/13] =?UTF-8?q?=F0=9F=A9=B9=20Identify=20myself=20to=20m?= =?UTF-8?q?yself?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/core/src/service/fetcher.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/core/src/service/fetcher.ts b/packages/core/src/service/fetcher.ts index 7acac0889..a94f3b507 100644 --- a/packages/core/src/service/fetcher.ts +++ b/packages/core/src/service/fetcher.ts @@ -15,7 +15,9 @@ export async function fetchWithCache( request.headers.set('If-None-Match', cachedEtag) } try { - const response = await web.fetch(request) + const response = await web.fetch(request, { + headers: { 'User-Agent': 'SpyglassMC (+https://spyglassmc.com)' }, + }) if (response.status === 304) { Dev.assertDefined(cachedResponse) logger.info(`[fetchWithCache] reusing cache for ${request.url}`)