From 6a47ffdecb5a8cec0410c1375703eb1cf7211942 Mon Sep 17 00:00:00 2001 From: Kirk Scheibelhut Date: Thu, 28 Mar 2019 14:24:33 -0700 Subject: [PATCH 01/13] Introduce `dev-tools/harness.js` for testing/benchmarking --- dev-tools/HARNESS.md | 50 +++++++++++ dev-tools/harness.js | 206 +++++++++++++++++++++++++++++++++++++++++++ sim/battle-stream.ts | 6 +- 3 files changed, 260 insertions(+), 2 deletions(-) create mode 100644 dev-tools/HARNESS.md create mode 100644 dev-tools/harness.js diff --git a/dev-tools/HARNESS.md b/dev-tools/HARNESS.md new file mode 100644 index 0000000000000..84967889ef20f --- /dev/null +++ b/dev-tools/HARNESS.md @@ -0,0 +1,50 @@ +# Harness + +`harness.js` allows for running multiple random simulations of Pokemon battles +for testing or benchmarking purposes. Without any arguments, `harness.js` will +run 100 random battles and report any errors that occurred. The number of +battles run can be configured through by passing the number as the sole argument +to `harness.js`, or through the `--num` flag (see below). + +## Flags + +Using any flag will trigger [minimist](https://github.com/substack/minimist) to +be installed if it has not been already. + +### General + +- **`--num`**: play a specific number of games for a format instead of the + default 100. +- **`--seed`**: PRNG seed to use (eg. `'1234,5678,9012,3456'`). +- **`--output`**: makes the harness display the _output_ logs of each battle + it runs. +- **`--input`**: dump the battle _input_ logs of each battle it runs. + +### Format + +By default, the harness will select the format of the next game it runs randomly +based on its initial `--seed`. Alternatively, it can run games all in the same +format, cycle through the formats or run all formats. + +- **`--format`**: play the specified format for each of the games it runs. + Note that the harness only supports formats where the team can be randomly + generated. +- **`--cycle`**: cycles through the possible formats, playing one battle in + each `--num` battles in total. +- **`--all`**: plays every supported format `--num` times before moving on to + the next format. + +### Concurrency + +The harness runs games sequentially by default, but can be configured to run +games asynchronously. + +- **`--async`**: runs each game concurrently instead of waiting for the + previous game to complete before starting the next. Note that since battles + should not be blocking to begin with, this mode is not expected to have + performance benefits over the default sequential mode and may require + additional memory (run with `node --max-old-space-size= + --stack-size=` if you encounter issues). + +**TODO**: Add support for running battles in `--parallel` on muliple cores with +[`worker_threads`](https://nodejs.org/api/worker_threads.html). diff --git a/dev-tools/harness.js b/dev-tools/harness.js new file mode 100644 index 0000000000000..abb79ee3109ad --- /dev/null +++ b/dev-tools/harness.js @@ -0,0 +1,206 @@ +/** + * Random Simulation harness for testing and benchmarking purposes. + * + * Refer to `HARNESS.md` for detailed usage instructions. + * + * Pokemon Showdown - http://pokemonshowdown.com/ + * + * @license MIT + */ + +'use strict'; + +const child_process = require('child_process'); +const shell = cmd => child_process.execSync(cmd, {stdio: 'inherit', cwd: __dirname}); + +// Run the build script if we're being called from the command line. +// NOTE: `require('../build')` is not safe because `replace` is async. +if (require.main === module) shell('node ../build'); + +const BattleStreams = require('../.sim-dist/battle-stream'); +const PRNG = require('../.sim-dist/prng').PRNG; +const RandomPlayerAI = require('../.sim-dist/examples/random-player-ai').RandomPlayerAI; + +const FORMATS = [ + 'gen7randombattle', 'gen7randomdoublesbattle', 'gen7battlefactory', 'gen6randombattle', 'gen6battlefactory', + 'gen5randombattle', 'gen4randombattle', 'gen3randombattle', 'gen2randombattle', 'gen1randombattle', +]; + +// 'move' 70% of the time (ie. 'switch' 30%) and ' mega' 60% of the time that its an option. +const AI_OPTIONS = {move: 0.7, mega: 0.6}; + +class Runner { + constructor(options) { + this.totalGames = options.totalGames; + + this.prng = (options.prng && !Array.isArray(options.prng)) ? + options.prng : new PRNG(options.prng); + this.p1options = Object.assign({}, AI_OPTIONS, options.p1options); + this.p2options = Object.assign({}, AI_OPTIONS, options.p2options); + + this.format = options.format; + this.cycle = !!options.cycle; + this.all = !!options.all; + + this.async = !!options.async; + + this.input = !!options.input; + this.output = !!options.output; + + this.formatIndex = 0; + this.numGames = 0; + } + + async run() { + let games = []; + let format, lastFormat; + while ((format = this.getNextFormat())) { + if (this.all && lastFormat && format !== lastFormat) { + await Promise.all(games); + games = []; + } + + const seed = this.prng.seed; + + try { + const battleStream = new BattleStreams.BattleStream({retainBattle: this.input}); + const game = this.runGame(format, battleStream).finally(() => { + if (battleStream.battle && this.input) { + console.error(`\n${battleStream.battle.inputLog.join('\n')}\n`); + } + }); + if (!this.async) await game; + games.push(game); + } catch (e) { + console.error( + `Run \`node dev-tools/harness 1 --format=${format} --seed=${seed.join()}\` ` + + `to debug (optionally with \`--output\` and/or \`--input\` for more info):\n`, e); + } + lastFormat = format; + } + + // Calculate how many games failed (failures weren't added to `games`). + return this.totalGames - (await Promise.all(games)).length; + } + + async runGame(format, battleStream) { + const streams = BattleStreams.getPlayerStreams(battleStream); + // The seed used is the intial seed to begin with (its important that nothing + // advances the PRNG before the initial `runGame` call for repro purposes), but + // later is advanced by the four `newSeed()` calls, so each iteration should be + // 16 frames off the previous (17 if running in the default random format mode, + // as `getNextFormat` calls `PRNG.sample` which also advances the PRNG). + const spec = {formatid: format, seed: this.prng.seed}; + const p1spec = {name: "Bot 1", seed: this.newSeed()}; + const p2spec = {name: "Bot 2", seed: this.newSeed()}; + + const p1 = new RandomPlayerAI( + streams.p1, Object.assign({seed: this.newSeed()}, this.p1options)).start(); + const p2 = new RandomPlayerAI( + streams.p2, Object.assign({seed: this.newSeed()}, this.p2options)).start(); + + streams.omniscient.write(`>start ${JSON.stringify(spec)}\n` + + `>player p1 ${JSON.stringify(p1spec)}\n` + + `>player p2 ${JSON.stringify(p2spec)}`); + + let chunk; + while ((chunk = await Promise.race([streams.omniscient.read(), p1, p2]))) { + if (this.output) console.log(chunk); + } + } + + // Same as PRNG#generatedSeed, only deterministic. + // NOTE: advances this.prng's seed by 4. + newSeed() { + return [ + Math.floor(this.prng.next() * 0x10000), + Math.floor(this.prng.next() * 0x10000), + Math.floor(this.prng.next() * 0x10000), + Math.floor(this.prng.next() * 0x10000), + ]; + } + + getNextFormat() { + if (this.formatIndex > FORMATS.length) return false; + + if (this.numGames++ < this.totalGames) { + if (this.format) { + return this.format; + } else if (this.all) { + return FORMATS[this.formatIndex]; + } else if (this.cycle) { + const format = FORMATS[this.formatIndex]; + this.formatIndex = (this.formatIndex + 1) % FORMATS.length; + return format; + } else { + return this.prng.sample(FORMATS); + } + } else if (this.all) { + this.numGames = 1; + this.formatIndex++; + return FORMATS[this.formatIndex]; + } + + return false; + } +} + +module.exports = Runner; + +// Kick off the Runner if we're being called from the command line. +if (require.main === module) { + const options = {totalGames: 100}; + // If we have more than one arg, or the arg looks like a flag, we need minimist to understand it. + if (process.argv.length > 3 || process.argv.length === 3 && process.argv[2].startsWith('-')) { + const missing = dep => { + try { + require.resolve(dep); + return false; + } catch (err) { + if (err.code !== 'MODULE_NOT_FOUND') throw err; + return true; + } + }; + + if (missing('minimist')) shell('npm install minimist'); + const argv = require('minimist')(process.argv.slice(2)); + Object.assign(options, argv); + options.totalGames = Number(argv._[0] || argv.num) || options.totalGames; + if (argv.seed) options.prng = argv.seed.split(',').map(s => Number(s)); + } else if (process.argv.length === 3) { + // If we have one arg, treat it as the total number of games to play. + options.totalGames = Number(process.argv[2]) || options.totalGames; + } + + // Tracks whether some promises threw errors that weren't caught so we can log + // and exit with a non-zero status to fail any tests. This "shouldn't happen" + // because we're "great at propagating promises (TM)", but better safe than sorry. + const RejectionTracker = new class { + constructor() { + this.unhandled = []; + } + onUnhandledRejection(reason, promise) { + this.unhandled.push({reason, promise}); + } + onRejectionHandled(promise) { + this.unhandled.splice(this.unhandled.findIndex(u => u.promise === promise), 1); + } + onExit(code) { + let i = 0; + for (const u of this.unhandled) { + const error = (u.reason instanceof Error) ? u.reason : + new Error(`Promise rejected with value: ${u.reason}`); + console.error(error.stack); + i++; + } + process.exit(code + i); + } + }(); + + process.on('unhandledRejection', (r, p) => RejectionTracker.onUnhandledRejection(r, p)); + process.on('rejectionHandled', p => RejectionTracker.onRejectionHandled(p)); + process.on('exit', c => RejectionTracker.onExit(c)); + + // Run options.totalGames, exiting with the number of games with errors. + (async () => process.exit(await new Runner(options).run()))(); +} diff --git a/sim/battle-stream.ts b/sim/battle-stream.ts index d75b21f66836f..9ea37da25260f 100644 --- a/sim/battle-stream.ts +++ b/sim/battle-stream.ts @@ -41,12 +41,14 @@ function splitFirst(str: string, delimiter: string, limit: number = 1) { export class BattleStream extends Streams.ObjectReadWriteStream { readonly debug: boolean; readonly keepAlive: boolean; + readonly retainBattle: boolean; battle: Battle | null; - constructor(options: {debug?: boolean, keepAlive?: boolean} = {}) { + constructor(options: {debug?: boolean, keepAlive?: boolean, retainBattle?: boolean} = {}) { super(); this.debug = !!options.debug; this.keepAlive = !!options.keepAlive; + this.retainBattle = !!options.retainBattle; this.battle = null; } @@ -167,7 +169,7 @@ export class BattleStream extends Streams.ObjectReadWriteStream { if (this.battle) { this.battle.destroy(); } - this.battle = null; + if (!this.retainBattle) this.battle = null; } } From 07cd48d197d1692ee80ef83b9caa07fbcfc78fab Mon Sep 17 00:00:00 2001 From: Kirk Scheibelhut Date: Thu, 28 Mar 2019 19:12:44 -0700 Subject: [PATCH 02/13] Make Runner/MultiRunner split to allow for non-random formats Also fixes an issue with error accounting in async mode. --- dev-tools/HARNESS.md | 1 + dev-tools/harness.js | 133 +++++++++++++++++++++++++------------------ 2 files changed, 79 insertions(+), 55 deletions(-) diff --git a/dev-tools/HARNESS.md b/dev-tools/HARNESS.md index 84967889ef20f..73b461fcfbf22 100644 --- a/dev-tools/HARNESS.md +++ b/dev-tools/HARNESS.md @@ -19,6 +19,7 @@ be installed if it has not been already. - **`--output`**: makes the harness display the _output_ logs of each battle it runs. - **`--input`**: dump the battle _input_ logs of each battle it runs. +- **`--error`**: dump the battle _input_ logs of each battle which errors. ### Format diff --git a/dev-tools/harness.js b/dev-tools/harness.js index abb79ee3109ad..502198a75753b 100644 --- a/dev-tools/harness.js +++ b/dev-tools/harness.js @@ -21,78 +21,39 @@ const BattleStreams = require('../.sim-dist/battle-stream'); const PRNG = require('../.sim-dist/prng').PRNG; const RandomPlayerAI = require('../.sim-dist/examples/random-player-ai').RandomPlayerAI; -const FORMATS = [ - 'gen7randombattle', 'gen7randomdoublesbattle', 'gen7battlefactory', 'gen6randombattle', 'gen6battlefactory', - 'gen5randombattle', 'gen4randombattle', 'gen3randombattle', 'gen2randombattle', 'gen1randombattle', -]; - // 'move' 70% of the time (ie. 'switch' 30%) and ' mega' 60% of the time that its an option. const AI_OPTIONS = {move: 0.7, mega: 0.6}; class Runner { constructor(options) { - this.totalGames = options.totalGames; - this.prng = (options.prng && !Array.isArray(options.prng)) ? options.prng : new PRNG(options.prng); this.p1options = Object.assign({}, AI_OPTIONS, options.p1options); this.p2options = Object.assign({}, AI_OPTIONS, options.p2options); this.format = options.format; - this.cycle = !!options.cycle; - this.all = !!options.all; - - this.async = !!options.async; - this.input = !!options.input; this.output = !!options.output; - - this.formatIndex = 0; - this.numGames = 0; + this.error = !!options.error; } async run() { - let games = []; - let format, lastFormat; - while ((format = this.getNextFormat())) { - if (this.all && lastFormat && format !== lastFormat) { - await Promise.all(games); - games = []; - } - - const seed = this.prng.seed; - - try { - const battleStream = new BattleStreams.BattleStream({retainBattle: this.input}); - const game = this.runGame(format, battleStream).finally(() => { - if (battleStream.battle && this.input) { - console.error(`\n${battleStream.battle.inputLog.join('\n')}\n`); - } - }); - if (!this.async) await game; - games.push(game); - } catch (e) { - console.error( - `Run \`node dev-tools/harness 1 --format=${format} --seed=${seed.join()}\` ` + - `to debug (optionally with \`--output\` and/or \`--input\` for more info):\n`, e); - } - lastFormat = format; - } - - // Calculate how many games failed (failures weren't added to `games`). - return this.totalGames - (await Promise.all(games)).length; + const battleStream = new BattleStreams.BattleStream({retainBattle: this.input}); + const game = this.runGame(this.format, battleStream); + const log = () => battleStream.battle && console.error(`\n${battleStream.battle.inputLog.join('\n')}\n`); + if (this.input) return game.finally(log); + if (!this.error) return game; + return game.catch(err => { + log(); + throw err; + }); } async runGame(format, battleStream) { const streams = BattleStreams.getPlayerStreams(battleStream); - // The seed used is the intial seed to begin with (its important that nothing - // advances the PRNG before the initial `runGame` call for repro purposes), but - // later is advanced by the four `newSeed()` calls, so each iteration should be - // 16 frames off the previous (17 if running in the default random format mode, - // as `getNextFormat` calls `PRNG.sample` which also advances the PRNG). const spec = {formatid: format, seed: this.prng.seed}; - const p1spec = {name: "Bot 1", seed: this.newSeed()}; - const p2spec = {name: "Bot 2", seed: this.newSeed()}; + const p1spec = this.getPlayerSpec("Bot 1", this.p1options); + const p2spec = this.getPlayerSpec("Bot 2", this.p2options); const p1 = new RandomPlayerAI( streams.p1, Object.assign({seed: this.newSeed()}, this.p1options)).start(); @@ -120,6 +81,70 @@ class Runner { ]; } + getPlayerSpec(name, options) { + if (options.team) return {name, team: options.team}; + return {name, seed: this.newSeed()}; + } +} + +module.exports = Runner; + +const FORMATS = [ + 'gen7randombattle', 'gen7randomdoublesbattle', 'gen7battlefactory', 'gen6randombattle', 'gen6battlefactory', + 'gen5randombattle', 'gen4randombattle', 'gen3randombattle', 'gen2randombattle', 'gen1randombattle', +]; + +class MultiRunner { + constructor(options) { + this.options = Object.assign({}, options); + + this.totalGames = options.totalGames; + + this.prng = (options.prng && !Array.isArray(options.prng)) ? + options.prng : new PRNG(options.prng); + this.options.prng = this.prng; + + this.format = options.format; + this.cycle = !!options.cycle; + this.all = !!options.all; + + this.async = !!options.async; + + this.formatIndex = 0; + this.numGames = 0; + } + + async run() { + let games = []; + let format, lastFormat; + let failures = 0; + while ((format = this.getNextFormat())) { + if (this.all && lastFormat && format !== lastFormat) { + // If we ran async, we need to now wait for each game and determine its status. + // NOTE: Promise.all doesn't help us here because it will resolve when the first + // rejection occurs, we need to wait for *all* the rejections. + if (this.async) for (const game of games) await game; + games = []; + } + + const seed = this.prng.seed; + const game = new Runner(Object.assign({format}, this.options)).run().catch(err => { + failures++; + console.error( + `Run \`node dev-tools/harness 1 --format=${format} --seed=${seed.join()}\` ` + + `to debug (optionally with \`--output\` and/or \`--input\` for more info):\n`, err); + }); + + if (!this.async) await game; + games.push(game); + lastFormat = format; + } + + // See comment above regarding Promise.all. + if (this.async) for (const game of games) await game; + return failures; + } + getNextFormat() { if (this.formatIndex > FORMATS.length) return false; @@ -145,9 +170,7 @@ class Runner { } } -module.exports = Runner; - -// Kick off the Runner if we're being called from the command line. +// Kick off the MultiRunner if we're being called from the command line. if (require.main === module) { const options = {totalGames: 100}; // If we have more than one arg, or the arg looks like a flag, we need minimist to understand it. @@ -202,5 +225,5 @@ if (require.main === module) { process.on('exit', c => RejectionTracker.onExit(c)); // Run options.totalGames, exiting with the number of games with errors. - (async () => process.exit(await new Runner(options).run()))(); + (async () => process.exit(await new MultiRunner(options).run()))(); } From c45ff0e4bcd87d0101ea97b046551767d41e1c84 Mon Sep 17 00:00:00 2001 From: Kirk Scheibelhut Date: Fri, 29 Mar 2019 20:48:27 -0700 Subject: [PATCH 03/13] @Zarel review --- dev-tools/HARNESS.md | 6 ++++-- dev-tools/harness.js | 2 +- sim/battle-stream.ts | 9 +++------ 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/dev-tools/HARNESS.md b/dev-tools/HARNESS.md index 73b461fcfbf22..4be700445f036 100644 --- a/dev-tools/HARNESS.md +++ b/dev-tools/HARNESS.md @@ -42,8 +42,10 @@ games asynchronously. - **`--async`**: runs each game concurrently instead of waiting for the previous game to complete before starting the next. Note that since battles - should not be blocking to begin with, this mode is not expected to have - performance benefits over the default sequential mode and may require + run through the harness should not be blocking to begin with (battles + naturally wait for players to make their decisions, but the AI's should be + making decisions pretty much immediately), this mode is not expected to have + large performance benefits over the default sequential mode and may require additional memory (run with `node --max-old-space-size= --stack-size=` if you encounter issues). diff --git a/dev-tools/harness.js b/dev-tools/harness.js index 502198a75753b..4844b96598458 100644 --- a/dev-tools/harness.js +++ b/dev-tools/harness.js @@ -38,7 +38,7 @@ class Runner { } async run() { - const battleStream = new BattleStreams.BattleStream({retainBattle: this.input}); + const battleStream = new BattleStreams.BattleStream(); const game = this.runGame(this.format, battleStream); const log = () => battleStream.battle && console.error(`\n${battleStream.battle.inputLog.join('\n')}\n`); if (this.input) return game.finally(log); diff --git a/sim/battle-stream.ts b/sim/battle-stream.ts index 9ea37da25260f..75d1a5be93f12 100644 --- a/sim/battle-stream.ts +++ b/sim/battle-stream.ts @@ -39,16 +39,14 @@ function splitFirst(str: string, delimiter: string, limit: number = 1) { } export class BattleStream extends Streams.ObjectReadWriteStream { - readonly debug: boolean; - readonly keepAlive: boolean; - readonly retainBattle: boolean; + debug: boolean; + keepAlive: boolean; battle: Battle | null; - constructor(options: {debug?: boolean, keepAlive?: boolean, retainBattle?: boolean} = {}) { + constructor(options: {debug?: boolean, keepAlive?: boolean} = {}) { super(); this.debug = !!options.debug; this.keepAlive = !!options.keepAlive; - this.retainBattle = !!options.retainBattle; this.battle = null; } @@ -169,7 +167,6 @@ export class BattleStream extends Streams.ObjectReadWriteStream { if (this.battle) { this.battle.destroy(); } - if (!this.retainBattle) this.battle = null; } } From e08cd4b2ba046dfba47d661def79c92e38b0c1db Mon Sep 17 00:00:00 2001 From: Kirk Scheibelhut Date: Sat, 30 Mar 2019 07:48:15 -0700 Subject: [PATCH 04/13] Use Promise.all --- dev-tools/harness.js | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/dev-tools/harness.js b/dev-tools/harness.js index 4844b96598458..fded962aaf87a 100644 --- a/dev-tools/harness.js +++ b/dev-tools/harness.js @@ -120,10 +120,7 @@ class MultiRunner { let failures = 0; while ((format = this.getNextFormat())) { if (this.all && lastFormat && format !== lastFormat) { - // If we ran async, we need to now wait for each game and determine its status. - // NOTE: Promise.all doesn't help us here because it will resolve when the first - // rejection occurs, we need to wait for *all* the rejections. - if (this.async) for (const game of games) await game; + if (this.async) await Promise.all(games); games = []; } @@ -140,8 +137,7 @@ class MultiRunner { lastFormat = format; } - // See comment above regarding Promise.all. - if (this.async) for (const game of games) await game; + if (this.async) await Promise.all(games); return failures; } From 2f2dcd281c6b9c9f2e9872cac15f16672dfdec1c Mon Sep 17 00:00:00 2001 From: Kirk Scheibelhut Date: Sat, 30 Mar 2019 20:05:32 -0700 Subject: [PATCH 05/13] Fix input log tracking to track raw input log instead --- dev-tools/harness.js | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/dev-tools/harness.js b/dev-tools/harness.js index fded962aaf87a..bd97a1ff2284f 100644 --- a/dev-tools/harness.js +++ b/dev-tools/harness.js @@ -38,13 +38,11 @@ class Runner { } async run() { - const battleStream = new BattleStreams.BattleStream(); + const battleStream = new RawBattleStream(this.input); const game = this.runGame(this.format, battleStream); - const log = () => battleStream.battle && console.error(`\n${battleStream.battle.inputLog.join('\n')}\n`); - if (this.input) return game.finally(log); if (!this.error) return game; return game.catch(err => { - log(); + console.log(`\n${battleStream.rawInputLog.join('\n')}\n`); throw err; }); } @@ -87,6 +85,20 @@ class Runner { } } +class RawBattleStream extends BattleStreams.BattleStream { + constructor(input) { + super(); + this.input = !!input; + this.rawInputLog = []; + } + + _write(message) { + if (this.input) console.log(message); + this.rawInputLog.push(message); + super._write(message); + } +} + module.exports = Runner; const FORMATS = [ From eb9e76e9ed28ad6a67f17e6886b79702717dd4eb Mon Sep 17 00:00:00 2001 From: Kirk Scheibelhut Date: Mon, 1 Apr 2019 11:37:11 -0700 Subject: [PATCH 06/13] Remove Promise.race --- dev-tools/harness.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dev-tools/harness.js b/dev-tools/harness.js index bd97a1ff2284f..22c3e0716d886 100644 --- a/dev-tools/harness.js +++ b/dev-tools/harness.js @@ -53,6 +53,9 @@ class Runner { const p1spec = this.getPlayerSpec("Bot 1", this.p1options); const p2spec = this.getPlayerSpec("Bot 2", this.p2options); + // TODO: Use `await Promise.race([streams.omniscient.read(), p1, p2])` to avoid + // leaving these promises dangling once it no longer causes memory leaks (v8#9069). + /* eslint-disable no-unused-vars */ const p1 = new RandomPlayerAI( streams.p1, Object.assign({seed: this.newSeed()}, this.p1options)).start(); const p2 = new RandomPlayerAI( @@ -63,7 +66,7 @@ class Runner { `>player p2 ${JSON.stringify(p2spec)}`); let chunk; - while ((chunk = await Promise.race([streams.omniscient.read(), p1, p2]))) { + while ((chunk = await streams.omniscient.read())) { if (this.output) console.log(chunk); } } From f7675264bb0428e7d6057cded67c03969eb5a6b4 Mon Sep 17 00:00:00 2001 From: Kirk Scheibelhut Date: Mon, 1 Apr 2019 12:02:54 -0700 Subject: [PATCH 07/13] Add smoke test for the smoke test harness :) --- dev-tools/harness.js | 14 +++++++------- test/dev-tools/harness.js | 12 ++++++++++++ test/mocha.opts | 2 +- 3 files changed, 20 insertions(+), 8 deletions(-) create mode 100644 test/dev-tools/harness.js diff --git a/dev-tools/harness.js b/dev-tools/harness.js index 22c3e0716d886..b7829f15cb36f 100644 --- a/dev-tools/harness.js +++ b/dev-tools/harness.js @@ -24,6 +24,11 @@ const RandomPlayerAI = require('../.sim-dist/examples/random-player-ai').RandomP // 'move' 70% of the time (ie. 'switch' 30%) and ' mega' 60% of the time that its an option. const AI_OPTIONS = {move: 0.7, mega: 0.6}; +const FORMATS = [ + 'gen7randombattle', 'gen7randomdoublesbattle', 'gen7battlefactory', 'gen6randombattle', 'gen6battlefactory', + 'gen5randombattle', 'gen4randombattle', 'gen3randombattle', 'gen2randombattle', 'gen1randombattle', +]; + class Runner { constructor(options) { this.prng = (options.prng && !Array.isArray(options.prng)) ? @@ -102,13 +107,6 @@ class RawBattleStream extends BattleStreams.BattleStream { } } -module.exports = Runner; - -const FORMATS = [ - 'gen7randombattle', 'gen7randomdoublesbattle', 'gen7battlefactory', 'gen6randombattle', 'gen6battlefactory', - 'gen5randombattle', 'gen4randombattle', 'gen3randombattle', 'gen2randombattle', 'gen1randombattle', -]; - class MultiRunner { constructor(options) { this.options = Object.assign({}, options); @@ -181,6 +179,8 @@ class MultiRunner { } } +module.exports = {Runner, MultiRunner}; + // Kick off the MultiRunner if we're being called from the command line. if (require.main === module) { const options = {totalGames: 100}; diff --git a/test/dev-tools/harness.js b/test/dev-tools/harness.js new file mode 100644 index 0000000000000..371813ea94d97 --- /dev/null +++ b/test/dev-tools/harness.js @@ -0,0 +1,12 @@ +'use strict'; + +const assert = require('assert'); + +const {MultiRunner} = require('./../../dev-tools/harness'); + +describe('MultiRunner', async () => { + it('should run successfully', async () => { + const opts = {totalGames: 1, prng: [1, 2, 3, 4]}; + assert.strictEqual(await (new MultiRunner(opts).run()), 0); + }); +}); diff --git a/test/mocha.opts b/test/mocha.opts index 5a2c19eb71774..18275aa96fd0f 100644 --- a/test/mocha.opts +++ b/test/mocha.opts @@ -1,4 +1,4 @@ -test/main.js test/simulator/**/*.js test/application/*.js test/chat-plugins/*.js +test/main.js test/simulator/**/*.js test/application/*.js test/chat-plugins/*.js test/dev-tools/*.js -R dot -u bdd --exit From 64ba5e689bd10dcf4e278513911ba9d67374e3c1 Mon Sep 17 00:00:00 2001 From: Kirk Scheibelhut Date: Tue, 2 Apr 2019 14:15:21 -0700 Subject: [PATCH 08/13] Export rejection tracker, support arbitrary AI --- dev-tools/harness.js | 71 ++++++++++++++++++++++++-------------------- 1 file changed, 39 insertions(+), 32 deletions(-) diff --git a/dev-tools/harness.js b/dev-tools/harness.js index b7829f15cb36f..940253c4b248f 100644 --- a/dev-tools/harness.js +++ b/dev-tools/harness.js @@ -22,7 +22,7 @@ const PRNG = require('../.sim-dist/prng').PRNG; const RandomPlayerAI = require('../.sim-dist/examples/random-player-ai').RandomPlayerAI; // 'move' 70% of the time (ie. 'switch' 30%) and ' mega' 60% of the time that its an option. -const AI_OPTIONS = {move: 0.7, mega: 0.6}; +const AI_OPTIONS = {move: 0.7, mega: 0.6, createAI: (s, o) => new RandomPlayerAI(s, o)}; const FORMATS = [ 'gen7randombattle', 'gen7randomdoublesbattle', 'gen7battlefactory', 'gen6randombattle', 'gen6battlefactory', @@ -61,9 +61,9 @@ class Runner { // TODO: Use `await Promise.race([streams.omniscient.read(), p1, p2])` to avoid // leaving these promises dangling once it no longer causes memory leaks (v8#9069). /* eslint-disable no-unused-vars */ - const p1 = new RandomPlayerAI( + const p1 = this.p1options.createAI( streams.p1, Object.assign({seed: this.newSeed()}, this.p1options)).start(); - const p2 = new RandomPlayerAI( + const p2 = this.p2options.createAI( streams.p2, Object.assign({seed: this.newSeed()}, this.p2options)).start(); streams.omniscient.write(`>start ${JSON.stringify(spec)}\n` + @@ -179,7 +179,41 @@ class MultiRunner { } } -module.exports = {Runner, MultiRunner}; +// Tracks whether some promises threw errors that weren't caught so we can log +// and exit with a non-zero status to fail any tests. This "shouldn't happen" +// because we're "great at propagating promises (TM)", but better safe than sorry. +const RejectionTracker = new class { + constructor() { + this.unhandled = []; + } + + onUnhandledRejection(reason, promise) { + this.unhandled.push({reason, promise}); + } + + onRejectionHandled(promise) { + this.unhandled.splice(this.unhandled.findIndex(u => u.promise === promise), 1); + } + + onExit(code) { + let i = 0; + for (const u of this.unhandled) { + const error = (u.reason instanceof Error) ? u.reason : + new Error(`Promise rejected with value: ${u.reason}`); + console.error(error.stack); + i++; + } + process.exit(code + i); + } + + register() { + process.on('unhandledRejection', (r, p) => RejectionTracker.onUnhandledRejection(r, p)); + process.on('rejectionHandled', p => RejectionTracker.onRejectionHandled(p)); + process.on('exit', c => RejectionTracker.onExit(c)); + } +}(); + +module.exports = {Runner, MultiRunner, RejectionTracker}; // Kick off the MultiRunner if we're being called from the command line. if (require.main === module) { @@ -206,34 +240,7 @@ if (require.main === module) { options.totalGames = Number(process.argv[2]) || options.totalGames; } - // Tracks whether some promises threw errors that weren't caught so we can log - // and exit with a non-zero status to fail any tests. This "shouldn't happen" - // because we're "great at propagating promises (TM)", but better safe than sorry. - const RejectionTracker = new class { - constructor() { - this.unhandled = []; - } - onUnhandledRejection(reason, promise) { - this.unhandled.push({reason, promise}); - } - onRejectionHandled(promise) { - this.unhandled.splice(this.unhandled.findIndex(u => u.promise === promise), 1); - } - onExit(code) { - let i = 0; - for (const u of this.unhandled) { - const error = (u.reason instanceof Error) ? u.reason : - new Error(`Promise rejected with value: ${u.reason}`); - console.error(error.stack); - i++; - } - process.exit(code + i); - } - }(); - - process.on('unhandledRejection', (r, p) => RejectionTracker.onUnhandledRejection(r, p)); - process.on('rejectionHandled', p => RejectionTracker.onRejectionHandled(p)); - process.on('exit', c => RejectionTracker.onExit(c)); + RejectionTracker.register(); // Run options.totalGames, exiting with the number of games with errors. (async () => process.exit(await new MultiRunner(options).run()))(); From a9d93b595a3a204c20c7367c3b54230c54c89a1a Mon Sep 17 00:00:00 2001 From: Kirk Scheibelhut Date: Tue, 2 Apr 2019 14:17:01 -0700 Subject: [PATCH 09/13] Add lgtm crap --- dev-tools/harness.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-tools/harness.js b/dev-tools/harness.js index 940253c4b248f..24cde7a938213 100644 --- a/dev-tools/harness.js +++ b/dev-tools/harness.js @@ -61,9 +61,9 @@ class Runner { // TODO: Use `await Promise.race([streams.omniscient.read(), p1, p2])` to avoid // leaving these promises dangling once it no longer causes memory leaks (v8#9069). /* eslint-disable no-unused-vars */ - const p1 = this.p1options.createAI( + const p1 = this.p1options.createAI( // lgtm streams.p1, Object.assign({seed: this.newSeed()}, this.p1options)).start(); - const p2 = this.p2options.createAI( + const p2 = this.p2options.createAI( // lgtm streams.p2, Object.assign({seed: this.newSeed()}, this.p2options)).start(); streams.omniscient.write(`>start ${JSON.stringify(spec)}\n` + From f393bc7fb36a8b71b863c934071a76f13d0a8719 Mon Sep 17 00:00:00 2001 From: Kirk Scheibelhut Date: Tue, 2 Apr 2019 14:34:15 -0700 Subject: [PATCH 10/13] Make RejectionTracker a singleton --- dev-tools/harness.js | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/dev-tools/harness.js b/dev-tools/harness.js index 24cde7a938213..a9cbbba17b2ee 100644 --- a/dev-tools/harness.js +++ b/dev-tools/harness.js @@ -182,19 +182,14 @@ class MultiRunner { // Tracks whether some promises threw errors that weren't caught so we can log // and exit with a non-zero status to fail any tests. This "shouldn't happen" // because we're "great at propagating promises (TM)", but better safe than sorry. -const RejectionTracker = new class { - constructor() { - this.unhandled = []; - } - +const RejectionTracker = { + unhandled: [], onUnhandledRejection(reason, promise) { this.unhandled.push({reason, promise}); - } - + }, onRejectionHandled(promise) { this.unhandled.splice(this.unhandled.findIndex(u => u.promise === promise), 1); - } - + }, onExit(code) { let i = 0; for (const u of this.unhandled) { @@ -204,14 +199,13 @@ const RejectionTracker = new class { i++; } process.exit(code + i); - } - + }, register() { - process.on('unhandledRejection', (r, p) => RejectionTracker.onUnhandledRejection(r, p)); - process.on('rejectionHandled', p => RejectionTracker.onRejectionHandled(p)); - process.on('exit', c => RejectionTracker.onExit(c)); - } -}(); + process.on('unhandledRejection', (r, p) => this.onUnhandledRejection(r, p)); + process.on('rejectionHandled', p => this.onRejectionHandled(p)); + process.on('exit', c => this.onExit(c)); + }, +}; module.exports = {Runner, MultiRunner, RejectionTracker}; From 3669dc19ef88eb42ee93838ca8f50c83cfb138e8 Mon Sep 17 00:00:00 2001 From: Kirk Scheibelhut Date: Tue, 2 Apr 2019 15:26:30 -0700 Subject: [PATCH 11/13] Fucking LGTM --- dev-tools/harness.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-tools/harness.js b/dev-tools/harness.js index a9cbbba17b2ee..1908a13f479fd 100644 --- a/dev-tools/harness.js +++ b/dev-tools/harness.js @@ -61,9 +61,9 @@ class Runner { // TODO: Use `await Promise.race([streams.omniscient.read(), p1, p2])` to avoid // leaving these promises dangling once it no longer causes memory leaks (v8#9069). /* eslint-disable no-unused-vars */ - const p1 = this.p1options.createAI( // lgtm + const p1 = this.p1options.createAI( // lgtm [js/unused-local-variable] streams.p1, Object.assign({seed: this.newSeed()}, this.p1options)).start(); - const p2 = this.p2options.createAI( // lgtm + const p2 = this.p2options.createAI( // lgtm [js/unused-local-variable] streams.p2, Object.assign({seed: this.newSeed()}, this.p2options)).start(); streams.omniscient.write(`>start ${JSON.stringify(spec)}\n` + From f242844b4fb1abc8000d00dcc2cf2b18ac966eb1 Mon Sep 17 00:00:00 2001 From: Slayer95 Date: Wed, 3 Apr 2019 00:38:15 -0500 Subject: [PATCH 12/13] Try suppressing LGTM --- dev-tools/harness.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-tools/harness.js b/dev-tools/harness.js index 1908a13f479fd..4067d5aba4fd6 100644 --- a/dev-tools/harness.js +++ b/dev-tools/harness.js @@ -61,9 +61,9 @@ class Runner { // TODO: Use `await Promise.race([streams.omniscient.read(), p1, p2])` to avoid // leaving these promises dangling once it no longer causes memory leaks (v8#9069). /* eslint-disable no-unused-vars */ - const p1 = this.p1options.createAI( // lgtm [js/unused-local-variable] + const p1 = this.p1options.createAI( // lgtm[js/unused-local-variable] streams.p1, Object.assign({seed: this.newSeed()}, this.p1options)).start(); - const p2 = this.p2options.createAI( // lgtm [js/unused-local-variable] + const p2 = this.p2options.createAI( // lgtm[js/unused-local-variable] streams.p2, Object.assign({seed: this.newSeed()}, this.p2options)).start(); streams.omniscient.write(`>start ${JSON.stringify(spec)}\n` + From 800a0e5f7d71b72e04e17edcd52f049cc1a59c40 Mon Sep 17 00:00:00 2001 From: Kirk Scheibelhut Date: Wed, 3 Apr 2019 08:10:55 -0700 Subject: [PATCH 13/13] gg LGTM --- dev-tools/harness.js | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/dev-tools/harness.js b/dev-tools/harness.js index 4067d5aba4fd6..6a9312ee1b8b1 100644 --- a/dev-tools/harness.js +++ b/dev-tools/harness.js @@ -58,13 +58,14 @@ class Runner { const p1spec = this.getPlayerSpec("Bot 1", this.p1options); const p2spec = this.getPlayerSpec("Bot 2", this.p2options); + const p1 = this.p1options.createAI( + streams.p1, Object.assign({seed: this.newSeed()}, this.p1options)); + const p2 = this.p2options.createAI( + streams.p2, Object.assign({seed: this.newSeed()}, this.p2options)); // TODO: Use `await Promise.race([streams.omniscient.read(), p1, p2])` to avoid // leaving these promises dangling once it no longer causes memory leaks (v8#9069). - /* eslint-disable no-unused-vars */ - const p1 = this.p1options.createAI( // lgtm[js/unused-local-variable] - streams.p1, Object.assign({seed: this.newSeed()}, this.p1options)).start(); - const p2 = this.p2options.createAI( // lgtm[js/unused-local-variable] - streams.p2, Object.assign({seed: this.newSeed()}, this.p2options)).start(); + p1.start(); + p2.start(); streams.omniscient.write(`>start ${JSON.stringify(spec)}\n` + `>player p1 ${JSON.stringify(p1spec)}\n` +