mirror of
https://github.com/engineer-man/piston.git
synced 2025-05-14 15:46:27 +02:00
api refactoring
This commit is contained in:
Brian Seymour
parent
5a05537a7f
commit
b3d18600cd
11 changed files with 358 additions and 327 deletions
|
|
@ -1,5 +1,5 @@
|
|||
const logger = require('logplease').create('executor/job');
|
||||
const { v4: uuidv4 } = require('uuid');
|
||||
const uuidv4 = require('uuid/v4');
|
||||
const cp = require('child_process');
|
||||
const path = require('path');
|
||||
const config = require('../config');
|
||||
|
|
@ -12,11 +12,12 @@ const job_states = {
|
|||
EXECUTED: Symbol('Executed and ready for cleanup')
|
||||
};
|
||||
|
||||
var uid=0;
|
||||
var gid=0;
|
||||
let uid = 0;
|
||||
let gid = 0;
|
||||
|
||||
class Job {
|
||||
constructor({runtime, files, args, stdin, timeouts, main}){
|
||||
|
||||
constructor({ runtime, files, args, stdin, timeouts, main }) {
|
||||
this.uuid = uuidv4();
|
||||
this.runtime = runtime;
|
||||
this.files = files;
|
||||
|
|
@ -25,8 +26,11 @@ class Job {
|
|||
this.timeouts = timeouts;
|
||||
this.main = main;
|
||||
|
||||
if(!this.files.map(f=>f.name).includes(this.main))
|
||||
let file_list = this.files.map(f => f.name);
|
||||
|
||||
if (!file_list.includes(this.main)) {
|
||||
throw new Error(`Main file "${this.main}" will not be written to disk`);
|
||||
}
|
||||
|
||||
this.uid = config.runner_uid_min + uid;
|
||||
this.gid = config.runner_gid_min + gid;
|
||||
|
|
@ -41,34 +45,30 @@ class Job {
|
|||
this.dir = path.join(config.data_directory, globals.data_directories.jobs, this.uuid);
|
||||
}
|
||||
|
||||
async prime(){
|
||||
async prime() {
|
||||
logger.info(`Priming job uuid=${this.uuid}`);
|
||||
|
||||
logger.debug('Writing files to job cache');
|
||||
|
||||
await fs.mkdir(this.dir, {mode:0o700});
|
||||
|
||||
const files = this.files.map(({name: file_name, content}) => {
|
||||
return fs.write_file(path.join(this.dir, file_name), content);
|
||||
});
|
||||
|
||||
await Promise.all(files);
|
||||
|
||||
logger.debug(`Transfering ownership uid=${this.uid} gid=${this.gid}`);
|
||||
await fs.chown(this.dir, this.uid, this.gid);
|
||||
|
||||
const chowns = this.files.map(({name:file_name}) => {
|
||||
return fs.chown(path.join(this.dir, file_name), this.uid, this.gid);
|
||||
});
|
||||
|
||||
await Promise.all(chowns);
|
||||
await fs.mkdir(this.dir, { mode:0o700 });
|
||||
await fs.chown(this.dir, this.uid, this.gid);
|
||||
|
||||
for (const file of this.files) {
|
||||
let file_path = path.join(this.dir, file.name);
|
||||
|
||||
await fs.write_file(file_path, file.content);
|
||||
await fs.chown(file_path, this.uid, this.gid);
|
||||
}
|
||||
|
||||
this.state = job_states.PRIMED;
|
||||
|
||||
logger.debug('Primed job');
|
||||
}
|
||||
|
||||
async safe_call(file, args, timeout){
|
||||
return await new Promise((resolve, reject) => {
|
||||
async safe_call(file, args, timeout) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const unshare = config.enable_unshare ? ['unshare','-n','-r'] : [];
|
||||
|
||||
const prlimit = [
|
||||
|
|
@ -94,63 +94,81 @@ class Job {
|
|||
gid: this.gid,
|
||||
detached: true //give this process its own process group
|
||||
});
|
||||
|
||||
|
||||
proc.stdin.write(this.stdin);
|
||||
proc.stdin.end();
|
||||
|
||||
|
||||
const kill_timeout = setTimeout(_ => proc.kill('SIGKILL'), timeout);
|
||||
const kill_timeout = set_timeout(_ => proc.kill('SIGKILL'), timeout);
|
||||
|
||||
proc.stderr.on('data', d=>{if(stderr.length>config.output_max_size) proc.kill('SIGKILL'); else stderr += d;});
|
||||
proc.stdout.on('data', d=>{if(stdout.length>config.output_max_size) proc.kill('SIGKILL'); else stdout += d;});
|
||||
proc.stderr.on('data', data => {
|
||||
if (stderr.length > config.output_max_size) {
|
||||
proc.kill('SIGKILL');
|
||||
} else {
|
||||
stderr += data;
|
||||
}
|
||||
);
|
||||
|
||||
proc.stdout.on('data', data => {
|
||||
if (stdout.length > config.output_max_size) {
|
||||
proc.kill('SIGKILL');
|
||||
} else {
|
||||
stdout += data;
|
||||
}
|
||||
);
|
||||
|
||||
const exit_cleanup = () => {
|
||||
clear_timeout(kill_timeout);
|
||||
|
||||
function exit_cleanup(){
|
||||
clearTimeout(kill_timeout);
|
||||
proc.stderr.destroy();
|
||||
proc.stdout.destroy();
|
||||
try{
|
||||
|
||||
try {
|
||||
process.kill(-proc.pid, 'SIGKILL');
|
||||
}catch{
|
||||
} catch {
|
||||
// Process will be dead already, so nothing to kill.
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
proc.on('exit', (code, signal)=>{
|
||||
exit_cleanup();
|
||||
|
||||
resolve({stdout, stderr, code, signal});
|
||||
|
||||
resolve({ stdout, stderr, code, signal });
|
||||
});
|
||||
|
||||
proc.on('error', (err) => {
|
||||
exit_cleanup();
|
||||
|
||||
reject({error: err, stdout, stderr});
|
||||
reject({ error: err, stdout, stderr });
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async execute(){
|
||||
if(this.state != job_states.PRIMED)
|
||||
async execute() {
|
||||
if (this.state !== job_states.PRIMED) {
|
||||
throw new Error('Job must be in primed state, current state: ' + this.state.toString());
|
||||
}
|
||||
|
||||
logger.info(`Executing job uuid=${this.uuid} uid=${this.uid} gid=${this.gid} runtime=${this.runtime.toString()}`);
|
||||
|
||||
logger.debug('Compiling');
|
||||
|
||||
var compile = undefined;
|
||||
if(this.runtime.compiled)
|
||||
let compile;
|
||||
|
||||
if (this.runtime.compiled) {
|
||||
compile = await this.safe_call(
|
||||
path.join(this.runtime.pkgdir, 'compile'),
|
||||
this.files.map(x=>x.name),
|
||||
this.timeouts.compile);
|
||||
|
||||
this.files.map(x => x.name),
|
||||
this.timeouts.compile
|
||||
);
|
||||
}
|
||||
|
||||
logger.debug('Running');
|
||||
|
||||
const run = await this.safe_call(
|
||||
path.join(this.runtime.pkgdir, 'run'),
|
||||
[this.main, ...this.args],
|
||||
this.timeouts.run);
|
||||
this.timeouts.run
|
||||
);
|
||||
|
||||
this.state = job_states.EXECUTED;
|
||||
|
||||
|
|
@ -158,13 +176,15 @@ class Job {
|
|||
compile,
|
||||
run
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
async cleanup(){
|
||||
async cleanup() {
|
||||
logger.info(`Cleaning up job uuid=${this.uuid}`);
|
||||
await fs.rm(this.dir, {recursive: true, force: true});
|
||||
await fs.rm(this.dir, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
module.exports = {Job};
|
||||
module.exports = {
|
||||
Job
|
||||
};
|
||||
|
|
|
|||
|
|
@ -6,38 +6,45 @@ const { Job } = require('./job');
|
|||
const { body } = require('express-validator');
|
||||
|
||||
module.exports = {
|
||||
|
||||
run_job_validators: [
|
||||
body('language')
|
||||
.isString(), // eslint-disable-line snakecasejs/snakecasejs
|
||||
.isString(),
|
||||
body('version')
|
||||
.isString(), // eslint-disable-line snakecasejs/snakecasejs
|
||||
.isString(),
|
||||
// isSemVer requires it to be a version, not a selector
|
||||
body('files')
|
||||
.isArray(), // eslint-disable-line snakecasejs/snakecasejs
|
||||
.isArray(),
|
||||
body('files.*.name')
|
||||
.isString() // eslint-disable-line snakecasejs/snakecasejs
|
||||
.isString()
|
||||
.bail()
|
||||
.not()
|
||||
.contains('/'),
|
||||
body('files.*.content')
|
||||
.isString(), // eslint-disable-line snakecasejs/snakecasejs
|
||||
.isString(),
|
||||
body('compile_timeout')
|
||||
.isNumeric(), // eslint-disable-line snakecasejs/snakecasejs
|
||||
.isNumeric(),
|
||||
body('run_timeout')
|
||||
.isNumeric(), // eslint-disable-line snakecasejs/snakecasejs
|
||||
.isNumeric(),
|
||||
body('stdin')
|
||||
.isString(), // eslint-disable-line snakecasejs/snakecasejs
|
||||
.isString(),
|
||||
body('args')
|
||||
.isArray(),
|
||||
body('args.*')
|
||||
.isString() // eslint-disable-line snakecasejs/snakecasejs
|
||||
.isString()
|
||||
],
|
||||
async run_job(req, res){
|
||||
// POST /jobs
|
||||
|
||||
|
||||
// POST /jobs
|
||||
async run_job(req, res) {
|
||||
const runtime = get_latest_runtime_matching_language_version(req.body.language, req.body.version);
|
||||
if(runtime == undefined) return res.json_error(`${req.body.language}-${req.body.version} runtime is unknown`, 400);
|
||||
|
||||
if (runtime === undefined) {
|
||||
return res
|
||||
.status(400)
|
||||
.send({
|
||||
message: `${req.body.language}-${req.body.version} runtime is unknown`
|
||||
});
|
||||
}
|
||||
|
||||
const job = new Job({
|
||||
runtime,
|
||||
|
|
@ -54,8 +61,12 @@ module.exports = {
|
|||
await job.prime();
|
||||
|
||||
const result = await job.execute();
|
||||
res.json_success(result);
|
||||
|
||||
await job.cleanup();
|
||||
|
||||
return res
|
||||
.status(200)
|
||||
.send(result);
|
||||
}
|
||||
};
|
||||
|
||||
};
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue