diff --git a/.dependencygraph/setting.json b/.dependencygraph/setting.json deleted file mode 100644 index 896e9bc..0000000 --- a/.dependencygraph/setting.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "alias": false, - "resolveExtensions": [ - ".js", - ".jsx", - ".ts", - ".tsx", - ".vue", - ".scss", - ".less" - ], - "entryFilePath": "/api" -} \ No newline at end of file diff --git a/api/.dependencygraph/setting.json b/api/.dependencygraph/setting.json deleted file mode 100644 index bd91851..0000000 --- a/api/.dependencygraph/setting.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "entryFilePath": "src/api/v2.js", - "alias": {}, - "resolveExtensions": [ - ".js", - ".jsx", - ".ts", - ".tsx", - ".vue", - ".scss", - ".less" - ] -} \ No newline at end of file diff --git a/api/src/api/v2.js b/api/src/api/v2.js index 52c56f5..3de18d4 100644 --- a/api/src/api/v2.js +++ b/api/src/api/v2.js @@ -50,7 +50,6 @@ const SIGNALS = [ ]; // ref: https://man7.org/linux/man-pages/man7/signal.7.html -// NOTE /**Job Fuctory Function * this function is used to create a job object from the request body * validates the request body and returns a promise that resolves to a job object @@ -113,6 +112,7 @@ function get_job(body) { }); } + // check if the constraints are within the configured limits for (const constraint of ['memory_limit', 'timeout']) { for (const type of ['compile', 'run']) { const constraint_name = `${type}_${constraint}`; @@ -129,8 +129,7 @@ function get_job(body) { if (configured_limit <= 0) { continue; } - - // NOTE - configured limit is specified for each runtime( in the runtime.js file ) + if (constraint_value > configured_limit) { return reject({ message: `${constraint_name} cannot exceed the configured limit of ${configured_limit}`, @@ -181,6 +180,10 @@ router.use((req, res, next) => { next(); }); +/** Websocket route + * used to create a websocket connection to the server to run code + * in a more interactive way by writing to stdin and reading from stdout and stderr + */ router.ws('/connect', async (ws, req) => { let job = null; let event_bus = new events.EventEmitter(); diff --git a/api/src/job.js b/api/src/job.js index 9f92da2..cacca7a 100644 --- a/api/src/job.js +++ b/api/src/job.js @@ -21,6 +21,11 @@ let gid = 0; let remaining_job_spaces = config.max_concurrent_jobs; let job_queue = []; +/** Every code execution is a job. This class is used to manage the job and its resources. + * @method prime Used to write the files to the job cache and transfer ownership of the files to the runner. + * @method safe_call Used to call the child process and limit its resources also used to compile and run the code. + * @method execute Used to execute the job runtime and return the result. + */ class Job { #active_timeouts; #active_parent_processes; @@ -58,6 +63,7 @@ class Job { uid++; gid++; + // generate a new uid and gid within the range of the config values (1001 , 1500) uid %= config.runner_uid_max - config.runner_uid_min + 1; gid %= config.runner_gid_max - config.runner_gid_min + 1; @@ -71,12 +77,12 @@ class Job { ); } - // NOTE - /** This function is used to prime the job, which means to write the files - * to the job cache and transfer ownership of the files to the runner. - * It also waits for a job slot if there are no available slots. (64 slots are available by default) + /** - Used to write the files (containing code to be executed) to the job cache folder + * and transfer ownership of the files to the runner. */ async prime() { + // wait for a job slot to open up (default concurrent jobs is 64) + // this is to prevent the runner from being overwhelmed with jobs if (remaining_job_spaces < 1) { this.logger.info(`Awaiting job slot`); await new Promise(resolve => { @@ -89,6 +95,7 @@ class Job { this.logger.debug(`Transfering ownership`); + // create the job cache folder and transfer ownership to the runner await fs.mkdir(this.dir, { mode: 0o700 }); await fs.chown(this.dir, this.uid, this.gid); @@ -117,6 +124,7 @@ class Job { this.logger.debug('Primed job'); } + /** Used to clear the active timeouts and processes */ exit_cleanup() { for (const timeout of this.#active_timeouts) { clear_timeout(timeout); @@ -128,6 +136,7 @@ class Job { this.logger.debug(`Finished exit cleanup`); } + /** Close the writables ( stdin, stdout, stderr ) of the active parent processes */ close_cleanup() { for (const proc of this.#active_parent_processes) { proc.stderr.destroy(); @@ -153,7 +162,7 @@ class Job { return new Promise((resolve, reject) => { const nonetwork = config.disable_networking ? ['nosocket'] : []; - // NOTE prlimit is a linux specific command + // prlimit is a linux specific command // It is used to limit the resources of the child process const prlimit = [ 'prlimit', @@ -162,7 +171,7 @@ class Job { '--fsize=' + this.runtime.max_file_size, ]; - // NOTE timeout_call is a linux specific command + // timeout is a linux specific command // It is used to limit the time of the child process const timeout_call = [ 'timeout', @@ -176,10 +185,10 @@ class Job { } const proc_call = [ - 'nice', - ...timeout_call, - ...prlimit, - ...nonetwork, + 'nice', // lower the priority of the process + ...timeout_call, // kill the process if it exceeds the time limit + ...prlimit, // limit the resources of the process + ...nonetwork, // disable networking 'bash', file, ...args, @@ -189,6 +198,7 @@ class Job { var stderr = ''; var output = ''; + // spawn the child process to execute the file with the given arguments const proc = cp.spawn(proc_call[0], proc_call.splice(1), { env: { ...this.runtime.env_vars, @@ -208,6 +218,8 @@ class Job { proc.stdin.end(); proc.stdin.destroy(); } else { + // when the event_bus receives a 'stdin' event (over websocket), write the data to the process's stdin + // used to handle interactive programs (like those that require input) event_bus.on('stdin', data => { proc.stdin.write(data); }); @@ -217,6 +229,7 @@ class Job { }); } + // set a timeout to kill the process if it exceeds the time limit const kill_timeout = (timeout >= 0 && set_timeout(async _ => { @@ -234,6 +247,7 @@ class Job { null; this.#active_timeouts.push(kill_timeout); + // when the process writes to stderr, send the data to the event_bus (over websocket later) proc.stderr.on('data', async data => { if (event_bus !== null) { event_bus.emit('stderr', data); @@ -257,6 +271,7 @@ class Job { } }); + // when the process writes to stdout, send the data to the event_bus (over websocket later) proc.stdout.on('data', async data => { if (event_bus !== null) { event_bus.emit('stdout', data); @@ -297,6 +312,10 @@ class Job { }); } + /** Used to execute the job and return the result. + * @param {EventEmitter} event_bus - The event bus to be used for communication + * @returns {Promise} - The result of the execution + */ async execute(event_bus = null) { if (this.state !== job_states.PRIMED) { throw new Error( @@ -373,6 +392,9 @@ class Job { }; } + /** Used to cleanup the processes and wait for any zombie processes to end + * - scan /proc for any processes that are owned by the runner and kill them + */ cleanup_processes(dont_wait = []) { let processes = [1]; const to_wait = []; @@ -469,6 +491,7 @@ class Job { this.logger.debug(`Cleaned up processes`); } + // used to cleanup the filesystem for any residual files async cleanup_filesystem() { for (const clean_path of globals.clean_directories) { const contents = await fs.readdir(clean_path);