From 423624842d84618152cffd287dd0a9db98d7ac21 Mon Sep 17 00:00:00 2001 From: SPGoding Date: Thu, 20 Feb 2025 18:03:57 -0600 Subject: [PATCH 1/3] =?UTF-8?q?=F0=9F=A7=91=E2=80=8D=F0=9F=92=BB=20Increas?= =?UTF-8?q?e=20slow=20down=20limit=20from=2050=20to=20150?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/developer/{api.adoc => web-api.adoc} | 8 ++++---- packages/web-api-server/src/index.ts | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) rename docs/developer/{api.adoc => web-api.adoc} (93%) diff --git a/docs/developer/api.adoc b/docs/developer/web-api.adoc similarity index 93% rename from docs/developer/api.adoc rename to docs/developer/web-api.adoc index 9fbd5a4e5..ae8051116 100644 --- a/docs/developer/api.adoc +++ b/docs/developer/web-api.adoc @@ -1,13 +1,13 @@ :page-layout: default -:page-title: API +:page-title: Web API :page-parent: Developer Guides :toc: == Introduction -The Spyglass API provides access to various information that is helpful for data pack/resource pack -toolings. It uses https://github.com/misode/mcmeta[misode/mcmeta] and +The Spyglass Web API provides access to various information that is helpful for data pack/resource +pack toolings. It uses https://github.com/misode/mcmeta[misode/mcmeta] and https://github.com/SpyglassMC/vanilla-mcdoc[SpyglassMC/vanilla-mcdoc] under the hood and provides a few advantages over using the GitHub API directly: @@ -42,7 +42,7 @@ the last response. Each API endpoint has a point cost depending on how expensive it is to serve the request. Each IP address can consume up to 100 points per one hour window before receiving `429 Too Many Requests` -responses. Additionally, the response time will be degraded starting from 50 requests per 15 minute +responses. Additionally, the response time will be degraded starting from 150 requests per 15 minute window. .Point Costs diff --git a/packages/web-api-server/src/index.ts b/packages/web-api-server/src/index.ts index c8d90597f..a21c5ba6a 100644 --- a/packages/web-api-server/src/index.ts +++ b/packages/web-api-server/src/index.ts @@ -49,7 +49,7 @@ const versionRoute = express.Router({ mergeParams: true }) await sendGitTarball(req, res, gits.mcmeta, `${version}-data`) }) -const DELAY_AFTER = 50 +const DELAY_AFTER = 150 const app = express() .set('trust proxy', 1) From 0336aadd0c5cd9c7095e545b7a860f2344c6586f Mon Sep 17 00:00:00 2001 From: SPGoding Date: Thu, 20 Feb 2025 18:08:16 -0600 Subject: [PATCH 2/3] =?UTF-8?q?=F0=9F=A7=B1=20Restart=20API=20server=20aut?= =?UTF-8?q?omatically=20if=20changed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ansible/roles/api-server/handlers/main.yml | 7 +++++++ ansible/roles/api-server/tasks/main.yml | 2 ++ 2 files changed, 9 insertions(+) diff --git a/ansible/roles/api-server/handlers/main.yml b/ansible/roles/api-server/handlers/main.yml index 60f5be53e..273b0956c 100644 --- a/ansible/roles/api-server/handlers/main.yml +++ b/ansible/roles/api-server/handlers/main.yml @@ -2,3 +2,10 @@ become: true ansible.builtin.systemd: daemon_reload: true + +- name: Restart API server service + become: true + ansible.builtin.service: + name: spyglassmc-api-server + enabled: true + state: restarted diff --git a/ansible/roles/api-server/tasks/main.yml b/ansible/roles/api-server/tasks/main.yml index d8d6c9a13..792e13bdb 100644 --- a/ansible/roles/api-server/tasks/main.yml +++ b/ansible/roles/api-server/tasks/main.yml @@ -4,6 +4,7 @@ name: '@spyglassmc/web-api-server' global: true state: latest + notify: Restart API server service - name: Create user become: true @@ -29,6 +30,7 @@ owner: api-server group: api-server mode: '700' + notify: Restart API server service - name: Create service become: true From d16f114a18d97cc134fa3ff00ba71e6908cc6350 Mon Sep 17 00:00:00 2001 From: SPGoding Date: Thu, 20 Feb 2025 18:18:45 -0600 Subject: [PATCH 3/3] =?UTF-8?q?=F0=9F=90=9B=20Update=20repo=20on=20startup?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/web-api-server/src/index.ts | 19 +++++++------------ packages/web-api-server/src/utils.ts | 18 +++++++++++++++--- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/packages/web-api-server/src/index.ts b/packages/web-api-server/src/index.ts index a21c5ba6a..a894b6f39 100644 --- a/packages/web-api-server/src/index.ts +++ b/packages/web-api-server/src/index.ts @@ -1,8 +1,8 @@ -import { Mutex } from 'async-mutex' import chalk from 'chalk' import cors from 'cors' import express from 'express' import { slowDown } from 'express-slow-down' +import assert from 'node:assert' import { fileURLToPath } from 'node:url' import { assertRootDir, @@ -16,6 +16,7 @@ import { MemCache, sendGitFile, sendGitTarball, + updateGitRepo, userAgentEnforcer, verifySignature, } from './utils.js' @@ -24,7 +25,6 @@ const { hookSecret, port, rootDir } = loadConfig() await assertRootDir(rootDir) const gits = await initGitRepos(rootDir) const cache = new MemCache(gits.mcmeta) -const gitMutex = new Mutex() const versionRoute = express.Router({ mergeParams: true }) .use(getVersionValidator(cache)) @@ -86,10 +86,10 @@ const app = express() }) .use('/mcje/versions/:version', versionRoute) .get('/vanilla-mcdoc/symbols', cheapRateLimiter, async (req, res) => { - await sendGitFile(req, res, gits.mcdoc, `generated`, 'symbols.json') + await sendGitFile(req, res, gits['vanilla-mcdoc'], `generated`, 'symbols.json') }) .get('/vanilla-mcdoc/tarball', expensiveRateLimiter, async (req, res) => { - await sendGitTarball(req, res, gits.mcdoc, 'main', 'vanilla-mcdoc') + await sendGitTarball(req, res, gits['vanilla-mcdoc'], 'main', 'vanilla-mcdoc') }) .post( '/hooks/github', @@ -112,14 +112,9 @@ const app = express() const { repository: { name } } = JSON.parse(req.body.toString()) as { repository: { name: string } } - const git = gits[name === 'vanilla-mcdoc' ? 'mcdoc' : 'mcmeta'] - - await gitMutex.runExclusive(async () => { - console.info(chalk.yellow(`Updating ${name}...`)) - await git.remote(['update', '--prune']) - cache.invalidate() - console.info(chalk.green(`Updated ${name}`)) - }) + assert(name === 'vanilla-mcdoc' || name === 'mcmeta') + await updateGitRepo(name, gits[name]) + cache.invalidate() }, ) .get('/favicon.ico', cheapRateLimiter, (_req, res) => { diff --git a/packages/web-api-server/src/utils.ts b/packages/web-api-server/src/utils.ts index 5bda748b0..26fbfca2c 100644 --- a/packages/web-api-server/src/utils.ts +++ b/packages/web-api-server/src/utils.ts @@ -1,3 +1,4 @@ +import { Mutex } from 'async-mutex' import chalk from 'chalk' import { createHmac, timingSafeEqual } from 'crypto' import type { NextFunction, Request, Response } from 'express' @@ -7,6 +8,8 @@ import { RateLimiterMemory, RateLimiterRes } from 'rate-limiter-flexible' import simpleGit from 'simple-git' import type { SimpleGit } from 'simple-git' +const gitMutex = new Mutex() + export function loadConfig() { if ( !(process.env.SPYGLASSMC_API_SERVER_DIR && process.env.SPYGLASSMC_API_SERVER_WEBHOOK_SECRET) @@ -63,8 +66,10 @@ export async function initGitRepos(rootDir: string) { async function initGitRepo(owner: string, repo: string) { const repoDir = path.join(rootDir, repo) + const repoGit = simpleGit({ baseDir: repoDir }).outputHandler(defaultOutputHandler) if (await doesPathExist(repoDir)) { console.info(chalk.green(`Repo ${owner}/${repo} already cloned.`)) + await updateGitRepo(repo, repoGit) } else { console.info(chalk.yellow(`Cloning ${owner}/${repo}...`)) await gitCloner.clone( @@ -74,16 +79,23 @@ export async function initGitRepos(rootDir: string) { ) console.info(chalk.green(`Repo ${owner}/${repo} cloned.`)) } - - return simpleGit({ baseDir: repoDir }).outputHandler(defaultOutputHandler) + return repoGit } return { mcmeta: await initGitRepo('misode', 'mcmeta'), - mcdoc: await initGitRepo('SpyglassMC', 'vanilla-mcdoc'), + 'vanilla-mcdoc': await initGitRepo('SpyglassMC', 'vanilla-mcdoc'), } } +export async function updateGitRepo(name: string, git: SimpleGit) { + await gitMutex.runExclusive(async () => { + console.info(chalk.yellow(`Updating ${name}...`)) + await git.remote(['update', '--prune']) + console.info(chalk.green(`Updated ${name}`)) + }) +} + export async function sendGitFile( req: Request, res: Response,