Make Runner/MultiRunner split to allow for non-random formats

Also fixes an issue with error accounting in async mode.

dev-tools/HARNESS.md

@@ -19,6 +19,7 @@ be installed if it has not been already.
 -   **`--output`**: makes the harness display the _output_ logs of each battle
     it runs.
 -   **`--input`**: dump the battle _input_ logs of each battle it runs.
+-   **`--error`**: dump the battle _input_ logs of each battle which errors.
 
 ### Format
 

dev-tools/harness.js

@@ -21,78 +21,39 @@ const BattleStreams = require('../.sim-dist/battle-stream');
 const PRNG = require('../.sim-dist/prng').PRNG;
 const RandomPlayerAI = require('../.sim-dist/examples/random-player-ai').RandomPlayerAI;
 
-const FORMATS = [
-	'gen7randombattle', 'gen7randomdoublesbattle', 'gen7battlefactory', 'gen6randombattle', 'gen6battlefactory',
-	'gen5randombattle', 'gen4randombattle', 'gen3randombattle', 'gen2randombattle', 'gen1randombattle',
-];
-
 // 'move' 70% of the time (ie. 'switch' 30%) and ' mega' 60% of the time that its an option.
 const AI_OPTIONS = {move: 0.7, mega: 0.6};
 
 class Runner {
 	constructor(options) {
-		this.totalGames = options.totalGames;
-
 		this.prng = (options.prng && !Array.isArray(options.prng)) ?
 			options.prng : new PRNG(options.prng);
 		this.p1options = Object.assign({}, AI_OPTIONS, options.p1options);
 		this.p2options = Object.assign({}, AI_OPTIONS, options.p2options);
 
 		this.format = options.format;
-		this.cycle = !!options.cycle;
-		this.all = !!options.all;
-
-		this.async = !!options.async;
-
 		this.input = !!options.input;
 		this.output = !!options.output;
-
-		this.formatIndex = 0;
-		this.numGames = 0;
+		this.error = !!options.error;
 	}
 
 	async run() {
-		let games = [];
-		let format, lastFormat;
-		while ((format = this.getNextFormat())) {
-			if (this.all && lastFormat && format !== lastFormat) {
-				await Promise.all(games);
-				games = [];
-			}
-
-			const seed = this.prng.seed;
-
-			try {
-				const battleStream = new BattleStreams.BattleStream({retainBattle: this.input});
-				const game = this.runGame(format, battleStream).finally(() => {
-					if (battleStream.battle && this.input) {
-						console.error(`\n${battleStream.battle.inputLog.join('\n')}\n`);
-					}
-				});
-				if (!this.async) await game;
-				games.push(game);
-			} catch (e) {
-				console.error(
-					`Run \`node dev-tools/harness 1 --format=${format} --seed=${seed.join()}\` ` +
-					`to debug (optionally with \`--output\` and/or \`--input\` for more info):\n`, e);
-			}
-			lastFormat = format;
-		}
-
-		// Calculate how many games failed (failures weren't added to `games`).
-		return this.totalGames - (await Promise.all(games)).length;
+		const battleStream = new BattleStreams.BattleStream({retainBattle: this.input});
+		const game = this.runGame(this.format, battleStream);
+		const log = () => battleStream.battle && console.error(`\n${battleStream.battle.inputLog.join('\n')}\n`);
+		if (this.input) return game.finally(log);
+		if (!this.error) return game;
+		return game.catch(err => {
+			log();
+			throw err;
+		});
 	}
 
 	async runGame(format, battleStream) {
 		const streams = BattleStreams.getPlayerStreams(battleStream);
-		// The seed used is the intial seed to begin with (its important that nothing
-		// advances the PRNG before the initial `runGame` call for repro purposes), but
-		// later is advanced by the four `newSeed()` calls, so each iteration should be
-		// 16 frames off the previous (17 if running in the default random format mode,
-		// as `getNextFormat` calls `PRNG.sample` which also advances the PRNG).
 		const spec = {formatid: format, seed: this.prng.seed};
-		const p1spec = {name: "Bot 1", seed: this.newSeed()};
-		const p2spec = {name: "Bot 2", seed: this.newSeed()};
+		const p1spec = this.getPlayerSpec("Bot 1", this.p1options);
+		const p2spec = this.getPlayerSpec("Bot 2", this.p2options);
 
 		const p1 = new RandomPlayerAI(
 			streams.p1, Object.assign({seed: this.newSeed()}, this.p1options)).start();
@@ -120,6 +81,70 @@ class Runner {
 		];
 	}
 
+	getPlayerSpec(name, options) {
+		if (options.team) return {name, team: options.team};
+		return {name, seed: this.newSeed()};
+	}
+}
+
+module.exports = Runner;
+
+const FORMATS = [
+	'gen7randombattle', 'gen7randomdoublesbattle', 'gen7battlefactory', 'gen6randombattle', 'gen6battlefactory',
+	'gen5randombattle', 'gen4randombattle', 'gen3randombattle', 'gen2randombattle', 'gen1randombattle',
+];
+
+class MultiRunner {
+	constructor(options) {
+		this.options = Object.assign({}, options);
+
+		this.totalGames = options.totalGames;
+
+		this.prng = (options.prng && !Array.isArray(options.prng)) ?
+			options.prng : new PRNG(options.prng);
+		this.options.prng = this.prng;
+
+		this.format = options.format;
+		this.cycle = !!options.cycle;
+		this.all = !!options.all;
+
+		this.async = !!options.async;
+
+		this.formatIndex = 0;
+		this.numGames = 0;
+	}
+
+	async run() {
+		let games = [];
+		let format, lastFormat;
+		let failures = 0;
+		while ((format = this.getNextFormat())) {
+			if (this.all && lastFormat && format !== lastFormat) {
+				// If we ran async, we need to now wait for each game and determine its status.
+				// NOTE: Promise.all doesn't help us here because it will resolve when the first
+				// rejection occurs, we need to wait for *all* the rejections.
+				if (this.async) for (const game of games) await game;
+				games = [];
+			}
+
+			const seed = this.prng.seed;
+			const game = new Runner(Object.assign({format}, this.options)).run().catch(err => {
+				failures++;
+				console.error(
+					`Run \`node dev-tools/harness 1 --format=${format} --seed=${seed.join()}\` ` +
+					`to debug (optionally with \`--output\` and/or \`--input\` for more info):\n`, err);
+			});
+
+			if (!this.async) await game;
+			games.push(game);
+			lastFormat = format;
+		}
+
+		// See comment above regarding Promise.all.
+		if (this.async) for (const game of games) await game;
+		return failures;
+	}
+
 	getNextFormat() {
 		if (this.formatIndex > FORMATS.length) return false;
 
@@ -145,9 +170,7 @@ class Runner {
 	}
 }
 
-module.exports = Runner;
-
-// Kick off the Runner if we're being called from the command line.
+// Kick off the MultiRunner if we're being called from the command line.
 if (require.main === module) {
 	const options = {totalGames: 100};
 	// If we have more than one arg, or the arg looks like a flag, we need minimist to understand it.
@@ -202,5 +225,5 @@ if (require.main === module) {
 	process.on('exit', c => RejectionTracker.onExit(c));
 
 	// Run options.totalGames, exiting with the number of games with errors.
-	(async () => process.exit(await new Runner(options).run()))();
+	(async () => process.exit(await new MultiRunner(options).run()))();
 }