chore: comment both api(v2.js) and job.js files
This commit is contained in:
parent
e46499e9b5
commit
1d42ae6434
|
@ -1,13 +0,0 @@
|
||||||
{
|
|
||||||
"alias": false,
|
|
||||||
"resolveExtensions": [
|
|
||||||
".js",
|
|
||||||
".jsx",
|
|
||||||
".ts",
|
|
||||||
".tsx",
|
|
||||||
".vue",
|
|
||||||
".scss",
|
|
||||||
".less"
|
|
||||||
],
|
|
||||||
"entryFilePath": "/api"
|
|
||||||
}
|
|
|
@ -1,13 +0,0 @@
|
||||||
{
|
|
||||||
"entryFilePath": "src/api/v2.js",
|
|
||||||
"alias": {},
|
|
||||||
"resolveExtensions": [
|
|
||||||
".js",
|
|
||||||
".jsx",
|
|
||||||
".ts",
|
|
||||||
".tsx",
|
|
||||||
".vue",
|
|
||||||
".scss",
|
|
||||||
".less"
|
|
||||||
]
|
|
||||||
}
|
|
|
@ -50,7 +50,6 @@ const SIGNALS = [
|
||||||
];
|
];
|
||||||
// ref: https://man7.org/linux/man-pages/man7/signal.7.html
|
// ref: https://man7.org/linux/man-pages/man7/signal.7.html
|
||||||
|
|
||||||
// NOTE
|
|
||||||
/**Job Fuctory Function
|
/**Job Fuctory Function
|
||||||
* this function is used to create a job object from the request body
|
* this function is used to create a job object from the request body
|
||||||
* validates the request body and returns a promise that resolves to a job object
|
* validates the request body and returns a promise that resolves to a job object
|
||||||
|
@ -113,6 +112,7 @@ function get_job(body) {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// check if the constraints are within the configured limits
|
||||||
for (const constraint of ['memory_limit', 'timeout']) {
|
for (const constraint of ['memory_limit', 'timeout']) {
|
||||||
for (const type of ['compile', 'run']) {
|
for (const type of ['compile', 'run']) {
|
||||||
const constraint_name = `${type}_${constraint}`;
|
const constraint_name = `${type}_${constraint}`;
|
||||||
|
@ -129,8 +129,7 @@ function get_job(body) {
|
||||||
if (configured_limit <= 0) {
|
if (configured_limit <= 0) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
// NOTE - configured limit is specified for each runtime( in the runtime.js file )
|
|
||||||
if (constraint_value > configured_limit) {
|
if (constraint_value > configured_limit) {
|
||||||
return reject({
|
return reject({
|
||||||
message: `${constraint_name} cannot exceed the configured limit of ${configured_limit}`,
|
message: `${constraint_name} cannot exceed the configured limit of ${configured_limit}`,
|
||||||
|
@ -181,6 +180,10 @@ router.use((req, res, next) => {
|
||||||
next();
|
next();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
/** Websocket route
|
||||||
|
* used to create a websocket connection to the server to run code
|
||||||
|
* in a more interactive way by writing to stdin and reading from stdout and stderr
|
||||||
|
*/
|
||||||
router.ws('/connect', async (ws, req) => {
|
router.ws('/connect', async (ws, req) => {
|
||||||
let job = null;
|
let job = null;
|
||||||
let event_bus = new events.EventEmitter();
|
let event_bus = new events.EventEmitter();
|
||||||
|
|
|
@ -21,6 +21,11 @@ let gid = 0;
|
||||||
let remaining_job_spaces = config.max_concurrent_jobs;
|
let remaining_job_spaces = config.max_concurrent_jobs;
|
||||||
let job_queue = [];
|
let job_queue = [];
|
||||||
|
|
||||||
|
/** Every code execution is a job. This class is used to manage the job and its resources.
|
||||||
|
* @method prime Used to write the files to the job cache and transfer ownership of the files to the runner.
|
||||||
|
* @method safe_call Used to call the child process and limit its resources also used to compile and run the code.
|
||||||
|
* @method execute Used to execute the job runtime and return the result.
|
||||||
|
*/
|
||||||
class Job {
|
class Job {
|
||||||
#active_timeouts;
|
#active_timeouts;
|
||||||
#active_parent_processes;
|
#active_parent_processes;
|
||||||
|
@ -58,6 +63,7 @@ class Job {
|
||||||
uid++;
|
uid++;
|
||||||
gid++;
|
gid++;
|
||||||
|
|
||||||
|
// generate a new uid and gid within the range of the config values (1001 , 1500)
|
||||||
uid %= config.runner_uid_max - config.runner_uid_min + 1;
|
uid %= config.runner_uid_max - config.runner_uid_min + 1;
|
||||||
gid %= config.runner_gid_max - config.runner_gid_min + 1;
|
gid %= config.runner_gid_max - config.runner_gid_min + 1;
|
||||||
|
|
||||||
|
@ -71,12 +77,12 @@ class Job {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// NOTE
|
/** - Used to write the files (containing code to be executed) to the job cache folder
|
||||||
/** This function is used to prime the job, which means to write the files
|
* and transfer ownership of the files to the runner.
|
||||||
* to the job cache and transfer ownership of the files to the runner.
|
|
||||||
* It also waits for a job slot if there are no available slots. (64 slots are available by default)
|
|
||||||
*/
|
*/
|
||||||
async prime() {
|
async prime() {
|
||||||
|
// wait for a job slot to open up (default concurrent jobs is 64)
|
||||||
|
// this is to prevent the runner from being overwhelmed with jobs
|
||||||
if (remaining_job_spaces < 1) {
|
if (remaining_job_spaces < 1) {
|
||||||
this.logger.info(`Awaiting job slot`);
|
this.logger.info(`Awaiting job slot`);
|
||||||
await new Promise(resolve => {
|
await new Promise(resolve => {
|
||||||
|
@ -89,6 +95,7 @@ class Job {
|
||||||
|
|
||||||
this.logger.debug(`Transfering ownership`);
|
this.logger.debug(`Transfering ownership`);
|
||||||
|
|
||||||
|
// create the job cache folder and transfer ownership to the runner
|
||||||
await fs.mkdir(this.dir, { mode: 0o700 });
|
await fs.mkdir(this.dir, { mode: 0o700 });
|
||||||
await fs.chown(this.dir, this.uid, this.gid);
|
await fs.chown(this.dir, this.uid, this.gid);
|
||||||
|
|
||||||
|
@ -117,6 +124,7 @@ class Job {
|
||||||
this.logger.debug('Primed job');
|
this.logger.debug('Primed job');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Used to clear the active timeouts and processes */
|
||||||
exit_cleanup() {
|
exit_cleanup() {
|
||||||
for (const timeout of this.#active_timeouts) {
|
for (const timeout of this.#active_timeouts) {
|
||||||
clear_timeout(timeout);
|
clear_timeout(timeout);
|
||||||
|
@ -128,6 +136,7 @@ class Job {
|
||||||
this.logger.debug(`Finished exit cleanup`);
|
this.logger.debug(`Finished exit cleanup`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Close the writables ( stdin, stdout, stderr ) of the active parent processes */
|
||||||
close_cleanup() {
|
close_cleanup() {
|
||||||
for (const proc of this.#active_parent_processes) {
|
for (const proc of this.#active_parent_processes) {
|
||||||
proc.stderr.destroy();
|
proc.stderr.destroy();
|
||||||
|
@ -153,7 +162,7 @@ class Job {
|
||||||
return new Promise((resolve, reject) => {
|
return new Promise((resolve, reject) => {
|
||||||
const nonetwork = config.disable_networking ? ['nosocket'] : [];
|
const nonetwork = config.disable_networking ? ['nosocket'] : [];
|
||||||
|
|
||||||
// NOTE prlimit is a linux specific command
|
// prlimit is a linux specific command
|
||||||
// It is used to limit the resources of the child process
|
// It is used to limit the resources of the child process
|
||||||
const prlimit = [
|
const prlimit = [
|
||||||
'prlimit',
|
'prlimit',
|
||||||
|
@ -162,7 +171,7 @@ class Job {
|
||||||
'--fsize=' + this.runtime.max_file_size,
|
'--fsize=' + this.runtime.max_file_size,
|
||||||
];
|
];
|
||||||
|
|
||||||
// NOTE timeout_call is a linux specific command
|
// timeout is a linux specific command
|
||||||
// It is used to limit the time of the child process
|
// It is used to limit the time of the child process
|
||||||
const timeout_call = [
|
const timeout_call = [
|
||||||
'timeout',
|
'timeout',
|
||||||
|
@ -176,10 +185,10 @@ class Job {
|
||||||
}
|
}
|
||||||
|
|
||||||
const proc_call = [
|
const proc_call = [
|
||||||
'nice',
|
'nice', // lower the priority of the process
|
||||||
...timeout_call,
|
...timeout_call, // kill the process if it exceeds the time limit
|
||||||
...prlimit,
|
...prlimit, // limit the resources of the process
|
||||||
...nonetwork,
|
...nonetwork, // disable networking
|
||||||
'bash',
|
'bash',
|
||||||
file,
|
file,
|
||||||
...args,
|
...args,
|
||||||
|
@ -189,6 +198,7 @@ class Job {
|
||||||
var stderr = '';
|
var stderr = '';
|
||||||
var output = '';
|
var output = '';
|
||||||
|
|
||||||
|
// spawn the child process to execute the file with the given arguments
|
||||||
const proc = cp.spawn(proc_call[0], proc_call.splice(1), {
|
const proc = cp.spawn(proc_call[0], proc_call.splice(1), {
|
||||||
env: {
|
env: {
|
||||||
...this.runtime.env_vars,
|
...this.runtime.env_vars,
|
||||||
|
@ -208,6 +218,8 @@ class Job {
|
||||||
proc.stdin.end();
|
proc.stdin.end();
|
||||||
proc.stdin.destroy();
|
proc.stdin.destroy();
|
||||||
} else {
|
} else {
|
||||||
|
// when the event_bus receives a 'stdin' event (over websocket), write the data to the process's stdin
|
||||||
|
// used to handle interactive programs (like those that require input)
|
||||||
event_bus.on('stdin', data => {
|
event_bus.on('stdin', data => {
|
||||||
proc.stdin.write(data);
|
proc.stdin.write(data);
|
||||||
});
|
});
|
||||||
|
@ -217,6 +229,7 @@ class Job {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// set a timeout to kill the process if it exceeds the time limit
|
||||||
const kill_timeout =
|
const kill_timeout =
|
||||||
(timeout >= 0 &&
|
(timeout >= 0 &&
|
||||||
set_timeout(async _ => {
|
set_timeout(async _ => {
|
||||||
|
@ -234,6 +247,7 @@ class Job {
|
||||||
null;
|
null;
|
||||||
this.#active_timeouts.push(kill_timeout);
|
this.#active_timeouts.push(kill_timeout);
|
||||||
|
|
||||||
|
// when the process writes to stderr, send the data to the event_bus (over websocket later)
|
||||||
proc.stderr.on('data', async data => {
|
proc.stderr.on('data', async data => {
|
||||||
if (event_bus !== null) {
|
if (event_bus !== null) {
|
||||||
event_bus.emit('stderr', data);
|
event_bus.emit('stderr', data);
|
||||||
|
@ -257,6 +271,7 @@ class Job {
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// when the process writes to stdout, send the data to the event_bus (over websocket later)
|
||||||
proc.stdout.on('data', async data => {
|
proc.stdout.on('data', async data => {
|
||||||
if (event_bus !== null) {
|
if (event_bus !== null) {
|
||||||
event_bus.emit('stdout', data);
|
event_bus.emit('stdout', data);
|
||||||
|
@ -297,6 +312,10 @@ class Job {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Used to execute the job and return the result.
|
||||||
|
* @param {EventEmitter} event_bus - The event bus to be used for communication
|
||||||
|
* @returns {Promise} - The result of the execution
|
||||||
|
*/
|
||||||
async execute(event_bus = null) {
|
async execute(event_bus = null) {
|
||||||
if (this.state !== job_states.PRIMED) {
|
if (this.state !== job_states.PRIMED) {
|
||||||
throw new Error(
|
throw new Error(
|
||||||
|
@ -373,6 +392,9 @@ class Job {
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Used to cleanup the processes and wait for any zombie processes to end
|
||||||
|
* - scan /proc for any processes that are owned by the runner and kill them
|
||||||
|
*/
|
||||||
cleanup_processes(dont_wait = []) {
|
cleanup_processes(dont_wait = []) {
|
||||||
let processes = [1];
|
let processes = [1];
|
||||||
const to_wait = [];
|
const to_wait = [];
|
||||||
|
@ -469,6 +491,7 @@ class Job {
|
||||||
this.logger.debug(`Cleaned up processes`);
|
this.logger.debug(`Cleaned up processes`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// used to cleanup the filesystem for any residual files
|
||||||
async cleanup_filesystem() {
|
async cleanup_filesystem() {
|
||||||
for (const clean_path of globals.clean_directories) {
|
for (const clean_path of globals.clean_directories) {
|
||||||
const contents = await fs.readdir(clean_path);
|
const contents = await fs.readdir(clean_path);
|
||||||
|
|
Loading…
Reference in New Issue