@@ -4,15 +4,19 @@ import {
44 logger ,
55 userInitiatedPriority ,
66 systemInitiatedPriority ,
7+ query ,
78} from '@cardstack/runtime-common' ;
89import yargs from 'yargs' ;
910import * as Sentry from '@sentry/node' ;
10- import { createServer } from 'net' ;
1111import flattenDeep from 'lodash/flattenDeep' ;
1212import { spawn } from 'child_process' ;
1313import pluralize from 'pluralize' ;
14+ import Koa from 'koa' ;
15+ import Router from '@koa/router' ;
16+ import { ecsMetadata , fullRequestURL , livenessCheck } from './middleware' ;
17+ import { PgAdapter } from '@cardstack/postgres' ;
1418
15- let log = logger ( 'worker' ) ;
19+ let log = logger ( 'worker-manager ' ) ;
1620
1721const REALM_SECRET_SEED = process . env . REALM_SECRET_SEED ;
1822if ( ! REALM_SECRET_SEED ) {
3438 . usage ( 'Start worker manager' )
3539 . options ( {
3640 port : {
37- description : 'TCP port for worker to communicate readiness (for tests)' ,
41+ description :
42+ 'HTTP port for worker manager to communicate readiness and status' ,
3843 type : 'number' ,
44+ demandOption : true ,
3945 } ,
4046 highPriorityCount : {
4147 description :
@@ -75,63 +81,93 @@ let isExiting = false;
7581process . on ( 'SIGINT' , ( ) => ( isExiting = true ) ) ;
7682process . on ( 'SIGTERM' , ( ) => ( isExiting = true ) ) ;
7783
78- if ( port != null ) {
79- // in tests we start a simple TCP server to communicate to the realm when
80- // the worker is ready to start processing jobs
81- let server = createServer ( ( socket ) => {
82- log . info ( `realm connected to worker manager` ) ;
83- socket . on ( 'data' , ( data ) => {
84- if ( data . toString ( ) === 'ready?' ) {
85- socket . write ( isReady ? 'ready' : 'not-ready' ) ;
86- }
87- } ) ;
88- socket . on ( 'close' , ( hadError ) => {
89- log . info ( `realm has disconnected${ hadError ? ' due to an error' : '' } ) ;
90- } ) ;
91- socket . on ( 'error' , ( err : any ) => {
92- console . error ( `realm disconnected from worker manager: ${ err . message } ) ;
93- } ) ;
94- } ) ;
95- server . unref ( ) ;
84+ let dbAdapter = new PgAdapter ( { } ) ;
9685
97- server . listen ( port , ( ) => {
98- log . info ( `worker manager listening for realm on port ${ port } ) ;
99- } ) ;
86+ let webServer = new Koa < Koa . DefaultState , Koa . Context > ( ) ;
87+ let router = new Router ( ) ;
88+ router . head ( '/' , livenessCheck ) ;
89+ router . get ( '/' , async ( ctxt : Koa . Context , _next : Koa . Next ) => {
90+ let result = {
91+ ready : isReady ,
92+ } as Record < string , boolean | number > ;
93+ if ( isReady ) {
94+ let [ { queue_depth } ] = ( await query ( dbAdapter , [
95+ `SELECT COUNT(*) as queue_depth FROM jobs WHERE status='unfulfilled'` ,
96+ ] ) ) as {
97+ queue_depth : string ;
98+ } [ ] ;
99+ result = {
100+ ...result ,
101+ highPriorityWorkers : highPriorityCount ,
102+ allPriorityWorkers : allPriorityCount ,
103+ queueDepth : parseInt ( queue_depth , 10 ) ,
104+ } ;
105+ }
106+ ctxt . set ( 'Content-Type' , 'application/json' ) ;
107+ ctxt . body = JSON . stringify ( result ) ;
108+ ctxt . status = isReady ? 200 : 503 ;
109+ } ) ;
110+
111+ webServer
112+ . use ( router . routes ( ) )
113+ . use ( ( ctxt : Koa . Context , next : Koa . Next ) => {
114+ log . info (
115+ `<-- ${ ctxt . method } ${ ctxt . req . headers . accept } ${
116+ fullRequestURL ( ctxt ) . href
117+ } `,
118+ ) ;
100119
101- const shutdown = ( ) => {
102- log . info ( `Shutting down server for worker manager...` ) ;
103- server . close ( ( err ) => {
104- if ( err ) {
105- log . error ( `Error while closing the server for worker manager:` , err ) ;
106- process . exit ( 1 ) ;
107- }
108- log . info ( `Server closed for worker manager.` ) ;
109- process . exit ( 0 ) ;
120+ ctxt . res . on ( 'finish' , ( ) => {
121+ log . info (
122+ `--> ${ ctxt . method } ${ ctxt . req . headers . accept } ${
123+ fullRequestURL ( ctxt ) . href
124+ } : ${ ctxt . status } ,
125+ ) ;
126+ log . debug ( JSON . stringify ( ctxt . req . headers ) ) ;
110127 } ) ;
111- } ;
128+ return next ( ) ;
129+ } )
130+ . use ( ecsMetadata ) ;
112131
113- process . on ( 'SIGINT' , shutdown ) ;
114- process . on ( 'SIGTERM' , shutdown ) ;
115- process . on ( 'uncaughtException' , ( err ) => {
116- log . error ( `Uncaught exception in worker manager:` , err ) ;
117- shutdown ( ) ;
118- } ) ;
132+ webServer . on ( 'error' , ( err : any ) => {
133+ console . error ( `worker manager HTTP server error: ${ err . message } ) ;
134+ } ) ;
119135
120- process . on ( 'message' , ( message ) => {
121- if ( message === 'stop' ) {
122- console . log ( `stopping realm server on port ${ port } ) ;
123- server . close ( ( ) => {
124- console . log ( `worker manager on port ${ port } ) ;
125- if ( process . send ) {
126- process . send ( 'stopped' ) ;
127- }
128- } ) ;
129- } else if ( message === 'kill' ) {
130- console . log ( `Ending worker manager process for ${ port } ) ;
131- process . exit ( 0 ) ;
136+ let webServerInstance = webServer . listen ( port ) ;
137+ log . info ( `worker manager HTTP listening on port ${ port } ) ;
138+
139+ const shutdown = ( onShutdown ?: ( ) => void ) => {
140+ log . info ( `Shutting down server for worker manager...` ) ;
141+ webServerInstance . closeAllConnections ( ) ;
142+ webServerInstance . close ( ( err ?: Error ) => {
143+ if ( err ) {
144+ log . error ( `Error while closing the server for worker manager HTTP:` , err ) ;
145+ process . exit ( 1 ) ;
132146 }
147+ dbAdapter . close ( ) ; // warning this is async
148+ log . info ( `worker manager HTTP on port ${ port } ) ;
149+ onShutdown ?.( ) ;
150+ process . exit ( 0 ) ;
133151 } ) ;
134- }
152+ } ;
153+
154+ process . on ( 'SIGINT' , shutdown ) ;
155+ process . on ( 'SIGTERM' , shutdown ) ;
156+ process . on ( 'uncaughtException' , ( err ) => {
157+ log . error ( `Uncaught exception in worker manager:` , err ) ;
158+ shutdown ( ) ;
159+ } ) ;
160+
161+ process . on ( 'message' , ( message ) => {
162+ if ( message === 'stop' ) {
163+ shutdown ( ( ) => {
164+ process . send ?.( 'stopped' ) ;
165+ } ) ;
166+ } else if ( message === 'kill' ) {
167+ console . log ( `Ending worker manager process for ${ port } ) ;
168+ process . exit ( 0 ) ;
169+ }
170+ } ) ;
135171
136172( async ( ) => {
137173 log . info (
0 commit comments