2021-04-25 11:01:16 +02:00
|
|
|
const logger = require('logplease').create('job');
|
2021-05-07 21:22:25 +02:00
|
|
|
const { v4: uuidv4 } = require('uuid');
|
2021-02-20 23:39:03 +01:00
|
|
|
const cp = require('child_process');
|
|
|
|
const path = require('path');
|
2021-04-25 10:55:20 +02:00
|
|
|
const config = require('./config');
|
|
|
|
const globals = require('./globals');
|
2021-02-20 23:39:03 +01:00
|
|
|
const fs = require('fs/promises');
|
2021-04-26 03:00:34 +02:00
|
|
|
const wait_pid = require('waitpid');
|
2021-02-20 23:39:03 +01:00
|
|
|
|
2021-02-20 15:13:56 +01:00
|
|
|
const job_states = {
|
2021-05-08 02:30:40 +02:00
|
|
|
READY: Symbol('Ready to be primed'),
|
|
|
|
PRIMED: Symbol('Primed and ready for execution'),
|
|
|
|
EXECUTED: Symbol('Executed and ready for cleanup'),
|
2021-02-20 23:39:03 +01:00
|
|
|
};
|
2021-02-20 15:13:56 +01:00
|
|
|
|
2021-03-13 06:01:04 +01:00
|
|
|
let uid = 0;
|
|
|
|
let gid = 0;
|
2021-02-20 15:13:56 +01:00
|
|
|
|
2021-10-01 09:28:54 +02:00
|
|
|
let remainingJobSpaces = config.max_concurrent_jobs;
|
|
|
|
let jobQueue = [];
|
|
|
|
|
2021-10-08 15:16:57 +02:00
|
|
|
setInterval(() => {
|
2021-10-01 09:28:54 +02:00
|
|
|
// Every 10ms try resolve a new job, if there is an available slot
|
2021-10-08 15:16:57 +02:00
|
|
|
if (jobQueue.length > 0 && remainingJobSpaces > 0) {
|
|
|
|
jobQueue.shift()();
|
2021-10-01 09:28:54 +02:00
|
|
|
}
|
2021-10-08 15:16:57 +02:00
|
|
|
}, 10);
|
2021-10-01 09:28:54 +02:00
|
|
|
|
2021-02-20 15:13:56 +01:00
|
|
|
class Job {
|
2021-10-02 14:08:36 +02:00
|
|
|
constructor({ runtime, files, args, stdin }) {
|
2021-05-08 02:30:40 +02:00
|
|
|
this.uuid = uuidv4();
|
|
|
|
this.runtime = runtime;
|
|
|
|
this.files = files.map((file, i) => ({
|
|
|
|
name: file.name || `file${i}.code`,
|
|
|
|
content: file.content,
|
|
|
|
}));
|
|
|
|
|
|
|
|
this.args = args;
|
|
|
|
this.stdin = stdin;
|
|
|
|
|
|
|
|
this.uid = config.runner_uid_min + uid;
|
|
|
|
this.gid = config.runner_gid_min + gid;
|
|
|
|
|
|
|
|
uid++;
|
|
|
|
gid++;
|
|
|
|
|
|
|
|
uid %= config.runner_uid_max - config.runner_uid_min + 1;
|
|
|
|
gid %= config.runner_gid_max - config.runner_gid_min + 1;
|
|
|
|
|
|
|
|
this.state = job_states.READY;
|
|
|
|
this.dir = path.join(
|
|
|
|
config.data_directory,
|
|
|
|
globals.data_directories.jobs,
|
|
|
|
this.uuid
|
|
|
|
);
|
|
|
|
}
|
2021-02-20 15:13:56 +01:00
|
|
|
|
2021-05-08 02:30:40 +02:00
|
|
|
async prime() {
|
2021-10-08 15:16:57 +02:00
|
|
|
if (remainingJobSpaces < 1) {
|
|
|
|
logger.info(`Awaiting job slot uuid=${this.uuid}`);
|
|
|
|
await new Promise(resolve => {
|
|
|
|
jobQueue.push(resolve);
|
|
|
|
});
|
2021-10-01 09:28:54 +02:00
|
|
|
}
|
2021-02-20 15:13:56 +01:00
|
|
|
|
2021-10-01 09:28:54 +02:00
|
|
|
logger.info(`Priming job uuid=${this.uuid}`);
|
|
|
|
remainingJobSpaces--;
|
2021-05-08 02:30:40 +02:00
|
|
|
logger.debug('Writing files to job cache');
|
2021-02-20 15:13:56 +01:00
|
|
|
|
2021-05-08 02:30:40 +02:00
|
|
|
logger.debug(`Transfering ownership uid=${this.uid} gid=${this.gid}`);
|
2021-02-20 15:13:56 +01:00
|
|
|
|
2021-05-08 02:30:40 +02:00
|
|
|
await fs.mkdir(this.dir, { mode: 0o700 });
|
|
|
|
await fs.chown(this.dir, this.uid, this.gid);
|
2021-02-20 15:13:56 +01:00
|
|
|
|
2021-05-08 02:30:40 +02:00
|
|
|
for (const file of this.files) {
|
|
|
|
let file_path = path.join(this.dir, file.name);
|
2021-07-22 04:58:41 +02:00
|
|
|
const rel = path.relative(this.dir, file_path);
|
|
|
|
|
2021-10-08 15:16:57 +02:00
|
|
|
if (rel.startsWith('..'))
|
|
|
|
throw Error(
|
|
|
|
`File path "${file.name}" tries to escape parent directory: ${rel}`
|
|
|
|
);
|
2021-07-22 04:58:41 +02:00
|
|
|
|
2021-10-08 15:16:57 +02:00
|
|
|
await fs.mkdir(path.dirname(file_path), {
|
|
|
|
recursive: true,
|
|
|
|
mode: 0o700,
|
|
|
|
});
|
2021-07-22 04:58:41 +02:00
|
|
|
await fs.chown(path.dirname(file_path), this.uid, this.gid);
|
2021-02-20 15:13:56 +01:00
|
|
|
|
2021-05-08 02:30:40 +02:00
|
|
|
await fs.write_file(file_path, file.content);
|
|
|
|
await fs.chown(file_path, this.uid, this.gid);
|
|
|
|
}
|
2021-03-13 06:01:04 +01:00
|
|
|
|
2021-05-08 02:30:40 +02:00
|
|
|
this.state = job_states.PRIMED;
|
2021-02-20 15:13:56 +01:00
|
|
|
|
2021-05-08 02:30:40 +02:00
|
|
|
logger.debug('Primed job');
|
2021-05-08 02:20:21 +02:00
|
|
|
}
|
2021-03-13 06:01:04 +01:00
|
|
|
|
2021-07-16 13:10:44 +02:00
|
|
|
async safe_call(file, args, timeout, memory_limit, eventBus = null) {
|
2021-05-08 02:30:40 +02:00
|
|
|
return new Promise((resolve, reject) => {
|
|
|
|
const nonetwork = config.disable_networking ? ['nosocket'] : [];
|
|
|
|
|
|
|
|
const prlimit = [
|
|
|
|
'prlimit',
|
2021-09-14 13:59:19 +02:00
|
|
|
'--nproc=' + this.runtime.max_process_count,
|
|
|
|
'--nofile=' + this.runtime.max_open_files,
|
|
|
|
'--fsize=' + this.runtime.max_file_size,
|
2021-05-08 02:30:40 +02:00
|
|
|
];
|
|
|
|
|
|
|
|
if (memory_limit >= 0) {
|
|
|
|
prlimit.push('--as=' + memory_limit);
|
|
|
|
}
|
|
|
|
|
|
|
|
const proc_call = [...prlimit, ...nonetwork, 'bash', file, ...args];
|
|
|
|
|
|
|
|
var stdout = '';
|
|
|
|
var stderr = '';
|
|
|
|
var output = '';
|
|
|
|
|
|
|
|
const proc = cp.spawn(proc_call[0], proc_call.splice(1), {
|
|
|
|
env: {
|
|
|
|
...this.runtime.env_vars,
|
|
|
|
PISTON_LANGUAGE: this.runtime.language,
|
|
|
|
},
|
|
|
|
stdio: 'pipe',
|
|
|
|
cwd: this.dir,
|
|
|
|
uid: this.uid,
|
|
|
|
gid: this.gid,
|
|
|
|
detached: true, //give this process its own process group
|
|
|
|
});
|
2021-02-20 15:13:56 +01:00
|
|
|
|
2021-10-08 15:16:57 +02:00
|
|
|
if (eventBus === null) {
|
2021-07-16 13:10:44 +02:00
|
|
|
proc.stdin.write(this.stdin);
|
|
|
|
proc.stdin.end();
|
|
|
|
proc.stdin.destroy();
|
2021-10-08 15:16:57 +02:00
|
|
|
} else {
|
|
|
|
eventBus.on('stdin', data => {
|
2021-07-16 13:10:44 +02:00
|
|
|
proc.stdin.write(data);
|
2021-10-08 15:16:57 +02:00
|
|
|
});
|
2021-07-16 14:22:55 +02:00
|
|
|
|
2021-10-08 15:16:57 +02:00
|
|
|
eventBus.on('kill', signal => {
|
|
|
|
proc.kill(signal);
|
|
|
|
});
|
2021-07-16 13:10:44 +02:00
|
|
|
}
|
2021-05-08 02:30:40 +02:00
|
|
|
|
2021-10-08 15:16:57 +02:00
|
|
|
const kill_timeout = set_timeout(async _ => {
|
|
|
|
logger.info(
|
|
|
|
`Timeout exceeded timeout=${timeout} uuid=${this.uuid}`
|
|
|
|
);
|
|
|
|
process.kill(proc.pid, 'SIGKILL');
|
|
|
|
}, timeout);
|
2021-05-08 02:30:40 +02:00
|
|
|
|
2021-10-01 13:07:37 +02:00
|
|
|
proc.stderr.on('data', async data => {
|
2021-10-08 15:16:57 +02:00
|
|
|
if (eventBus !== null) {
|
|
|
|
eventBus.emit('stderr', data);
|
2021-10-02 14:08:36 +02:00
|
|
|
} else if (stderr.length > this.runtime.output_max_size) {
|
2021-10-08 15:16:57 +02:00
|
|
|
logger.info(`stderr length exceeded uuid=${this.uuid}`);
|
|
|
|
process.kill(proc.pid, 'SIGKILL');
|
2021-05-08 02:30:40 +02:00
|
|
|
} else {
|
|
|
|
stderr += data;
|
|
|
|
output += data;
|
|
|
|
}
|
|
|
|
});
|
2021-02-20 15:13:56 +01:00
|
|
|
|
2021-10-01 13:07:37 +02:00
|
|
|
proc.stdout.on('data', async data => {
|
2021-10-08 15:16:57 +02:00
|
|
|
if (eventBus !== null) {
|
|
|
|
eventBus.emit('stdout', data);
|
2021-10-02 14:08:36 +02:00
|
|
|
} else if (stdout.length > this.runtime.output_max_size) {
|
2021-10-08 15:16:57 +02:00
|
|
|
logger.info(`stdout length exceeded uuid=${this.uuid}`);
|
|
|
|
process.kill(proc.pid, 'SIGKILL');
|
2021-05-08 02:30:40 +02:00
|
|
|
} else {
|
|
|
|
stdout += data;
|
|
|
|
output += data;
|
|
|
|
}
|
|
|
|
});
|
2021-03-13 06:01:04 +01:00
|
|
|
|
2021-10-01 09:28:54 +02:00
|
|
|
const exit_cleanup = async () => {
|
2021-05-08 02:30:40 +02:00
|
|
|
clear_timeout(kill_timeout);
|
2021-02-22 09:55:51 +01:00
|
|
|
|
2021-05-08 02:30:40 +02:00
|
|
|
proc.stderr.destroy();
|
|
|
|
proc.stdout.destroy();
|
2021-10-01 09:28:54 +02:00
|
|
|
|
2021-10-08 15:16:57 +02:00
|
|
|
await this.cleanup_processes();
|
|
|
|
logger.debug(`Finished exit cleanup uuid=${this.uuid}`);
|
2021-05-08 02:30:40 +02:00
|
|
|
};
|
2021-02-20 15:13:56 +01:00
|
|
|
|
2021-10-01 09:28:54 +02:00
|
|
|
proc.on('exit', async (code, signal) => {
|
|
|
|
await exit_cleanup();
|
2021-03-13 06:01:04 +01:00
|
|
|
|
2021-10-08 15:16:57 +02:00
|
|
|
resolve({ stdout, stderr, code, signal, output });
|
2021-05-08 02:30:40 +02:00
|
|
|
});
|
2021-03-13 06:01:04 +01:00
|
|
|
|
2021-10-01 09:28:54 +02:00
|
|
|
proc.on('error', async err => {
|
|
|
|
await exit_cleanup();
|
2021-02-23 07:52:49 +01:00
|
|
|
|
2021-05-08 02:30:40 +02:00
|
|
|
reject({ error: err, stdout, stderr, output });
|
|
|
|
});
|
|
|
|
});
|
|
|
|
}
|
2021-02-22 09:55:51 +01:00
|
|
|
|
2021-05-08 02:30:40 +02:00
|
|
|
async execute() {
|
|
|
|
if (this.state !== job_states.PRIMED) {
|
|
|
|
throw new Error(
|
|
|
|
'Job must be in primed state, current state: ' +
|
|
|
|
this.state.toString()
|
|
|
|
);
|
|
|
|
}
|
2021-03-13 06:01:04 +01:00
|
|
|
|
2021-05-08 02:30:40 +02:00
|
|
|
logger.info(
|
|
|
|
`Executing job uuid=${this.uuid} uid=${this.uid} gid=${
|
|
|
|
this.gid
|
|
|
|
} runtime=${this.runtime.toString()}`
|
|
|
|
);
|
2021-02-20 15:13:56 +01:00
|
|
|
|
2021-05-08 02:30:40 +02:00
|
|
|
logger.debug('Compiling');
|
2021-02-21 09:36:49 +01:00
|
|
|
|
2021-05-08 02:30:40 +02:00
|
|
|
let compile;
|
2021-02-20 15:13:56 +01:00
|
|
|
|
2021-05-08 02:30:40 +02:00
|
|
|
if (this.runtime.compiled) {
|
|
|
|
compile = await this.safe_call(
|
|
|
|
path.join(this.runtime.pkgdir, 'compile'),
|
|
|
|
this.files.map(x => x.name),
|
2021-10-02 14:08:36 +02:00
|
|
|
this.runtime.timeouts.compile,
|
|
|
|
this.runtime.memory_limits.compile
|
2021-05-08 02:30:40 +02:00
|
|
|
);
|
2021-03-13 06:01:04 +01:00
|
|
|
}
|
2021-02-20 15:13:56 +01:00
|
|
|
|
2021-05-08 02:30:40 +02:00
|
|
|
logger.debug('Running');
|
2021-04-26 00:35:34 +02:00
|
|
|
|
2021-05-08 02:30:40 +02:00
|
|
|
const run = await this.safe_call(
|
|
|
|
path.join(this.runtime.pkgdir, 'run'),
|
|
|
|
[this.files[0].name, ...this.args],
|
2021-10-02 14:08:36 +02:00
|
|
|
this.runtime.timeouts.run,
|
|
|
|
this.runtime.memory_limits.run
|
2021-05-08 02:30:40 +02:00
|
|
|
);
|
2021-04-28 06:03:35 +02:00
|
|
|
|
2021-05-08 02:30:40 +02:00
|
|
|
this.state = job_states.EXECUTED;
|
2021-05-07 21:22:25 +02:00
|
|
|
|
2021-05-08 02:30:40 +02:00
|
|
|
return {
|
|
|
|
compile,
|
|
|
|
run,
|
|
|
|
language: this.runtime.language,
|
|
|
|
version: this.runtime.version.raw,
|
|
|
|
};
|
|
|
|
}
|
2021-05-07 21:22:25 +02:00
|
|
|
|
2021-10-08 15:16:57 +02:00
|
|
|
async execute_interactive(eventBus) {
|
2021-07-16 13:10:44 +02:00
|
|
|
if (this.state !== job_states.PRIMED) {
|
|
|
|
throw new Error(
|
|
|
|
'Job must be in primed state, current state: ' +
|
|
|
|
this.state.toString()
|
|
|
|
);
|
|
|
|
}
|
|
|
|
|
|
|
|
logger.info(
|
2021-10-08 15:16:57 +02:00
|
|
|
`Interactively executing job uuid=${this.uuid} uid=${
|
|
|
|
this.uid
|
|
|
|
} gid=${this.gid} runtime=${this.runtime.toString()}`
|
2021-07-16 13:10:44 +02:00
|
|
|
);
|
|
|
|
|
2021-10-08 15:16:57 +02:00
|
|
|
if (this.runtime.compiled) {
|
|
|
|
eventBus.emit('stage', 'compile');
|
|
|
|
const { error, code, signal } = await this.safe_call(
|
2021-07-16 13:10:44 +02:00
|
|
|
path.join(this.runtime.pkgdir, 'compile'),
|
|
|
|
this.files.map(x => x.name),
|
2021-10-02 14:08:36 +02:00
|
|
|
this.runtime.timeouts.compile,
|
|
|
|
this.runtime.memory_limits.compile,
|
2021-07-16 13:10:44 +02:00
|
|
|
eventBus
|
2021-10-08 15:16:57 +02:00
|
|
|
);
|
2021-07-16 13:10:44 +02:00
|
|
|
|
2021-10-08 15:16:57 +02:00
|
|
|
eventBus.emit('exit', 'compile', { error, code, signal });
|
2021-07-16 13:10:44 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
logger.debug('Running');
|
2021-10-08 15:16:57 +02:00
|
|
|
eventBus.emit('stage', 'run');
|
|
|
|
const { error, code, signal } = await this.safe_call(
|
2021-07-16 13:10:44 +02:00
|
|
|
path.join(this.runtime.pkgdir, 'run'),
|
|
|
|
[this.files[0].name, ...this.args],
|
2021-10-02 14:08:36 +02:00
|
|
|
this.runtime.timeouts.run,
|
|
|
|
this.runtime.memory_limits.run,
|
2021-07-16 13:10:44 +02:00
|
|
|
eventBus
|
|
|
|
);
|
|
|
|
|
2021-10-08 15:16:57 +02:00
|
|
|
eventBus.emit('exit', 'run', { error, code, signal });
|
2021-10-02 14:08:36 +02:00
|
|
|
|
2021-07-16 13:10:44 +02:00
|
|
|
this.state = job_states.EXECUTED;
|
|
|
|
}
|
|
|
|
|
2021-10-01 13:07:37 +02:00
|
|
|
async cleanup_processes(dont_wait = []) {
|
2021-05-08 02:30:40 +02:00
|
|
|
let processes = [1];
|
2021-10-08 15:16:57 +02:00
|
|
|
logger.debug(`Cleaning up processes uuid=${this.uuid}`);
|
2021-05-08 02:30:40 +02:00
|
|
|
|
|
|
|
while (processes.length > 0) {
|
2021-10-08 15:16:57 +02:00
|
|
|
processes = [];
|
|
|
|
|
|
|
|
const proc_ids = await fs.readdir('/proc');
|
|
|
|
|
|
|
|
processes = await Promise.all(
|
|
|
|
proc_ids.map(async proc_id => {
|
|
|
|
if (isNaN(proc_id)) return -1;
|
|
|
|
try {
|
|
|
|
const proc_status = await fs.read_file(
|
|
|
|
path.join('/proc', proc_id, 'status')
|
|
|
|
);
|
|
|
|
const proc_lines = proc_status.to_string().split('\n');
|
|
|
|
const uid_line = proc_lines.find(line =>
|
|
|
|
line.starts_with('Uid:')
|
|
|
|
);
|
|
|
|
const [_, ruid, euid, suid, fuid] =
|
|
|
|
uid_line.split(/\s+/);
|
|
|
|
|
|
|
|
if (ruid == this.uid || euid == this.uid)
|
|
|
|
return parse_int(proc_id);
|
|
|
|
} catch {
|
|
|
|
return -1;
|
|
|
|
}
|
2021-10-01 13:07:37 +02:00
|
|
|
|
2021-10-08 15:16:57 +02:00
|
|
|
return -1;
|
|
|
|
})
|
|
|
|
);
|
2021-10-01 13:07:37 +02:00
|
|
|
|
2021-10-08 15:16:57 +02:00
|
|
|
processes = processes.filter(p => p > 0);
|
2021-05-08 02:30:40 +02:00
|
|
|
|
2021-10-08 15:16:57 +02:00
|
|
|
if (processes.length > 0)
|
|
|
|
logger.debug(
|
|
|
|
`Got processes to kill: ${processes} uuid=${this.uuid}`
|
|
|
|
);
|
2021-05-08 02:30:40 +02:00
|
|
|
|
|
|
|
for (const proc of processes) {
|
|
|
|
// First stop the processes, but keep their resources allocated so they cant re-fork
|
|
|
|
try {
|
2021-10-01 13:07:37 +02:00
|
|
|
process.kill(proc, 'SIGSTOP');
|
2021-05-08 02:30:40 +02:00
|
|
|
} catch {
|
|
|
|
// Could already be dead
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
for (const proc of processes) {
|
|
|
|
// Then clear them out of the process tree
|
|
|
|
try {
|
2021-10-01 13:07:37 +02:00
|
|
|
process.kill(proc, 'SIGKILL');
|
2021-05-08 02:30:40 +02:00
|
|
|
} catch {
|
|
|
|
// Could already be dead and just needs to be waited on
|
|
|
|
}
|
|
|
|
|
2021-10-08 15:16:57 +02:00
|
|
|
if (!dont_wait.includes(proc)) wait_pid(proc);
|
2021-05-08 02:30:40 +02:00
|
|
|
}
|
2021-04-28 06:11:49 +02:00
|
|
|
}
|
2021-10-01 13:07:37 +02:00
|
|
|
|
2021-10-08 15:16:57 +02:00
|
|
|
logger.debug(`Cleaned up processes uuid=${this.uuid}`);
|
2021-04-28 06:03:35 +02:00
|
|
|
}
|
|
|
|
|
2021-05-08 02:30:40 +02:00
|
|
|
async cleanup_filesystem() {
|
|
|
|
for (const clean_path of globals.clean_directories) {
|
|
|
|
const contents = await fs.readdir(clean_path);
|
|
|
|
|
|
|
|
for (const file of contents) {
|
|
|
|
const file_path = path.join(clean_path, file);
|
|
|
|
|
|
|
|
try {
|
|
|
|
const stat = await fs.stat(file_path);
|
|
|
|
|
|
|
|
if (stat.uid === this.uid) {
|
|
|
|
await fs.rm(file_path, {
|
|
|
|
recursive: true,
|
|
|
|
force: true,
|
|
|
|
});
|
|
|
|
}
|
|
|
|
} catch (e) {
|
|
|
|
// File was somehow deleted in the time that we read the dir to when we checked the file
|
|
|
|
logger.warn(`Error removing file ${file_path}: ${e}`);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
await fs.rm(this.dir, { recursive: true, force: true });
|
|
|
|
}
|
2021-05-07 21:22:25 +02:00
|
|
|
|
2021-05-08 02:30:40 +02:00
|
|
|
async cleanup() {
|
|
|
|
logger.info(`Cleaning up job uuid=${this.uuid}`);
|
2021-03-13 06:01:04 +01:00
|
|
|
|
2021-08-19 15:05:13 +02:00
|
|
|
await this.cleanup_filesystem();
|
2021-10-01 09:28:54 +02:00
|
|
|
|
|
|
|
remainingJobSpaces++;
|
2021-05-08 02:30:40 +02:00
|
|
|
}
|
2021-02-20 15:13:56 +01:00
|
|
|
}
|
|
|
|
|
2021-03-13 06:01:04 +01:00
|
|
|
module.exports = {
|
2021-05-08 02:30:40 +02:00
|
|
|
Job,
|
2021-03-13 06:01:04 +01:00
|
|
|
};
|