From c8b69fe5eadc70a2222dab37191ac2799ff8bfa9 Mon Sep 17 00:00:00 2001 From: Omar Brikaa Date: Sat, 24 Aug 2024 20:14:48 +0300 Subject: [PATCH 01/10] Initial: use Isolate for isolation --- api/Dockerfile | 16 +- api/src/docker-entrypoint.sh | 12 + api/src/globals.js | 1 - api/src/job.js | 566 +++++++++++++++-------------------- docker-compose.dev.yaml | 5 +- docker-compose.yaml | 1 + 6 files changed, 264 insertions(+), 337 deletions(-) create mode 100644 api/src/docker-entrypoint.sh diff --git a/api/Dockerfile b/api/Dockerfile index ec0d2a8..640fbcc 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -1,3 +1,13 @@ +FROM buildpack-deps@sha256:d56cd472000631b8faca51f40d4e3f1b20deffa588f9f207fa6c60efb62ba7c4 AS isolate +RUN apt-get update && \ + apt-get install -y --no-install-recommends git libcap-dev && \ + rm -rf /var/lib/apt/lists/* && \ + git clone https://github.com/envicutor/isolate.git /tmp/isolate/ && \ + cd /tmp/isolate && \ + git checkout af6db68042c3aa0ded80787fbb78bc0846ea2114 && \ + make -j$(nproc) install && \ + rm -rf /tmp/* + FROM node:15.10.0-buster-slim ENV DEBIAN_FRONTEND=noninteractive @@ -15,6 +25,8 @@ RUN apt-get update && \ libfftw3-dev libglpk-dev libqhull-dev libqrupdate-dev libsuitesparse-dev \ libsundials-dev libpcre2-dev && \ rm -rf /var/lib/apt/lists/* +COPY --from=isolate /usr/local/bin/isolate /usr/local/bin +COPY --from=isolate /usr/local/etc/isolate /usr/local/etc/isolate RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen @@ -23,7 +35,5 @@ COPY ["package.json", "package-lock.json", "./"] RUN npm install COPY ./src ./src -RUN make -C ./src/nosocket/ all && make -C ./src/nosocket/ install - -CMD [ "node", "src"] +CMD ["/piston_api/src/docker-entrypoint.sh"] EXPOSE 2000/tcp diff --git a/api/src/docker-entrypoint.sh b/api/src/docker-entrypoint.sh new file mode 100644 index 0000000..1160a25 --- /dev/null +++ b/api/src/docker-entrypoint.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +cd /sys/fs/cgroup && \ +mkdir isolate/ && \ +echo 1 > isolate/cgroup.procs && \ +echo '+cpuset +cpu +io +memory +pids' > cgroup.subtree_control && \ +cd isolate && \ +mkdir init && \ +echo 1 > init/cgroup.procs && \ +echo '+cpuset +memory' > cgroup.subtree_control && \ +echo "Initialized cgroup" && \ +exec su -- piston -c 'ulimit -n 65536 && node' diff --git a/api/src/globals.js b/api/src/globals.js index 933d2ca..d83d834 100644 --- a/api/src/globals.js +++ b/api/src/globals.js @@ -11,7 +11,6 @@ const platform = `${is_docker() ? 'docker' : 'baremetal'}-${fs module.exports = { data_directories: { packages: 'packages', - jobs: 'jobs', }, version: require('../package.json').version, platform, diff --git a/api/src/job.js b/api/src/job.js index a2641f9..1ca235f 100644 --- a/api/src/job.js +++ b/api/src/job.js @@ -1,13 +1,9 @@ const logplease = require('logplease'); -const logger = logplease.create('job'); const { v4: uuidv4 } = require('uuid'); const cp = require('child_process'); const path = require('path'); const config = require('./config'); -const globals = require('./globals'); const fs = require('fs/promises'); -const fss = require('fs'); -const wait_pid = require('waitpid'); const job_states = { READY: Symbol('Ready to be primed'), @@ -15,15 +11,19 @@ const job_states = { EXECUTED: Symbol('Executed and ready for cleanup'), }; -let uid = 0; -let gid = 0; +const MAX_BOX_ID = 999; +const ISOLATE_PATH = '/usr/local/bin/isolate'; +let box_id = 0; let remaining_job_spaces = config.max_concurrent_jobs; let job_queue = []; +const get_next_box_id = () => (box_id + 1) % MAX_BOX_ID; + class Job { - #active_timeouts; - #active_parent_processes; + #box_id; + #metadata_file_path; + #box_dir; constructor({ runtime, files, args, stdin, timeouts, memory_limits }) { this.uuid = uuidv4(); @@ -46,29 +46,10 @@ class Job { this.stdin += '\n'; } - this.#active_timeouts = []; - this.#active_parent_processes = []; - this.timeouts = timeouts; this.memory_limits = memory_limits; - this.uid = config.runner_uid_min + uid; - this.gid = config.runner_gid_min + gid; - - uid++; - gid++; - - uid %= config.runner_uid_max - config.runner_uid_min + 1; - gid %= config.runner_gid_max - config.runner_gid_min + 1; - - this.logger.debug(`Assigned uid=${this.uid} gid=${this.gid}`); - this.state = job_states.READY; - this.dir = path.join( - config.data_directory, - globals.data_directories.jobs, - this.uuid - ); } async prime() { @@ -80,31 +61,46 @@ class Job { } this.logger.info(`Priming job`); remaining_job_spaces--; - this.logger.debug('Writing files to job cache'); + this.logger.debug('Running isolate --init'); + this.#box_id = get_next_box_id(); + this.#metadata_file_path = `/tmp/${this.#box_id}-metadata.txt`; + await new Promise((res, rej) => { + cp.exec( + `isolate --init --cg -b${this.#box_id}`, + (error, stdout, stderr) => { + if (error) { + rej( + `Failed to run isolate --init: ${error.message}\nstdout: ${stdout}\nstderr: ${stderr}` + ); + } + if (stdout === '') { + rej('Received empty stdout from isolate --init'); + } + this.#box_dir = stdout; + res(); + } + ); + }); - this.logger.debug(`Transfering ownership`); - - await fs.mkdir(this.dir, { mode: 0o700 }); - await fs.chown(this.dir, this.uid, this.gid); + this.logger.debug(`Creating submission files in Isolate box`); + await fs.mkdir(path.join(this.#box_dir, 'submission')); for (const file of this.files) { - const file_path = path.join(this.dir, file.name); - const rel = path.relative(this.dir, file_path); - const file_content = Buffer.from(file.content, file.encoding); + const file_path = path.join(box_dir, file.name); + const rel = path.relative(box_dir, file_path); if (rel.startsWith('..')) throw Error( `File path "${file.name}" tries to escape parent directory: ${rel}` ); + const file_content = Buffer.from(file.content, file.encoding); + await fs.mkdir(path.dirname(file_path), { recursive: true, mode: 0o700, }); - await fs.chown(path.dirname(file_path), this.uid, this.gid); - await fs.write_file(file_path, file_content); - await fs.chown(file_path, this.uid, this.gid); } this.state = job_states.PRIMED; @@ -112,167 +108,204 @@ class Job { this.logger.debug('Primed job'); } - exit_cleanup() { - for (const timeout of this.#active_timeouts) { - clear_timeout(timeout); - } - this.#active_timeouts = []; - this.logger.debug('Cleared the active timeouts'); - - this.cleanup_processes(); - this.logger.debug(`Finished exit cleanup`); - } - - close_cleanup() { - for (const proc of this.#active_parent_processes) { - proc.stderr.destroy(); - if (!proc.stdin.destroyed) { - proc.stdin.end(); - proc.stdin.destroy(); - } - proc.stdout.destroy(); - } - this.#active_parent_processes = []; - this.logger.debug('Destroyed processes writables'); - } - async safe_call(file, args, timeout, memory_limit, event_bus = null) { - return new Promise((resolve, reject) => { - const nonetwork = config.disable_networking ? ['nosocket'] : []; + var stdout = ''; + var stderr = ''; + var output = ''; - const prlimit = [ - 'prlimit', - '--nproc=' + this.runtime.max_process_count, - '--nofile=' + this.runtime.max_open_files, - '--fsize=' + this.runtime.max_file_size, - ]; - - const timeout_call = [ - 'timeout', + const proc = cp.spawn( + ISOLATE_PATH, + [ + '--run', + `-b${this.#box_id}`, + `--meta=${this.#metadata_file_path}`, + '--cg', '-s', - '9', - Math.ceil(timeout / 1000), - ]; - - if (memory_limit >= 0) { - prlimit.push('--as=' + memory_limit); - } - - const proc_call = [ - 'nice', - ...timeout_call, - ...prlimit, - ...nonetwork, - 'bash', + '-c', + '/box/submission', + '-e', + `--dir=/runtime=${this.runtime.pkgdir}`, + `--processes=${this.runtime.max_process_count}`, + `--open-files=${this.runtime.max_open_files}`, + `--fsize=${this.runtime.max_file_size}`, + `--time=${timeout}`, + `--extra-time=0`, + ...(memory_limit >= 0 ? [`--cg-mem=${memory_limit}`] : []), + ...(config.disable_networking ? [] : '--share-net'), + '--', file, ...args, - ]; - - var stdout = ''; - var stderr = ''; - var output = ''; - - const proc = cp.spawn(proc_call[0], proc_call.splice(1), { + ], + { env: { ...this.runtime.env_vars, PISTON_LANGUAGE: this.runtime.language, }, stdio: 'pipe', - cwd: this.dir, - uid: this.uid, - gid: this.gid, - detached: true, //give this process its own process group - }); - - this.#active_parent_processes.push(proc); - - if (event_bus === null) { - proc.stdin.write(this.stdin); - proc.stdin.end(); - proc.stdin.destroy(); - } else { - event_bus.on('stdin', data => { - proc.stdin.write(data); - }); - - event_bus.on('kill', signal => { - proc.kill(signal); - }); } + ); - const kill_timeout = - (timeout >= 0 && - set_timeout(async _ => { - this.logger.info(`Timeout exceeded timeout=${timeout}`); - try { - process.kill(proc.pid, 'SIGKILL'); - } - catch (e) { - // Could already be dead and just needs to be waited on - this.logger.debug( - `Got error while SIGKILLing process ${proc}:`, - e - ); - } - }, timeout)) || - null; - this.#active_timeouts.push(kill_timeout); + if (event_bus === null) { + proc.stdin.write(this.stdin); + proc.stdin.end(); + proc.stdin.destroy(); + } else { + event_bus.on('stdin', data => { + proc.stdin.write(data); + }); - proc.stderr.on('data', async data => { - if (event_bus !== null) { - event_bus.emit('stderr', data); - } else if ((stderr.length + data.length) > this.runtime.output_max_size) { - this.logger.info(`stderr length exceeded`); - try { - process.kill(proc.pid, 'SIGKILL'); - } - catch (e) { - // Could already be dead and just needs to be waited on - this.logger.debug( - `Got error while SIGKILLing process ${proc}:`, - e - ); - } - } else { - stderr += data; - output += data; + event_bus.on('kill', signal => { + proc.kill(signal); + }); + } + + proc.stderr.on('data', async data => { + if (event_bus !== null) { + event_bus.emit('stderr', data); + } else if ( + stderr.length + data.length > + this.runtime.output_max_size + ) { + this.logger.info(`stderr length exceeded`); + try { + process.kill(proc.pid, 'SIGABRT'); + } catch (e) { + // Could already be dead and just needs to be waited on + this.logger.debug( + `Got error while SIGABRTing process ${proc}:`, + e + ); } - }); + } else { + stderr += data; + output += data; + } + }); - proc.stdout.on('data', async data => { - if (event_bus !== null) { - event_bus.emit('stdout', data); - } else if ((stdout.length + data.length) > this.runtime.output_max_size) { - this.logger.info(`stdout length exceeded`); - try { - process.kill(proc.pid, 'SIGKILL'); - } - catch (e) { - // Could already be dead and just needs to be waited on - this.logger.debug( - `Got error while SIGKILLing process ${proc}:`, - e - ); - } - } else { - stdout += data; - output += data; + proc.stdout.on('data', async data => { + if (event_bus !== null) { + event_bus.emit('stdout', data); + } else if ( + stdout.length + data.length > + this.runtime.output_max_size + ) { + this.logger.info(`stdout length exceeded`); + try { + process.kill(proc.pid, 'SIGABRT'); + } catch (e) { + // Could already be dead and just needs to be waited on + this.logger.debug( + `Got error while SIGABRTing process ${proc}:`, + e + ); } + } else { + stdout += data; + output += data; + } + }); + + let memory = null; + let code = null; + let signal = null; + let message = null; + let status = null; + let time = null; + + try { + const metadata_str = await fs.readFile( + self.metadata_file_path, + 'utf-8' + ); + const metadata_lines = metadata_str.split('\n'); + for (const line of metadata_lines) { + if (!line) continue; + + const [key, value] = line.split(':'); + if (key === undefined || value === undefined) { + throw new Error( + `Failed to parse metadata file, received: ${line}` + ); + } + switch (key) { + case 'cg-mem': + memory = + parseInt(value) || + (() => { + throw new Error( + `Failed to parse memory usage, received value: ${value}` + ); + })(); + break; + case 'exitcode': + code = + parseInt(value) || + (() => { + throw new Error( + `Failed to parse exit code, received value: ${value}` + ); + })(); + break; + case 'exitsig': + signal = + parseInt(value) || + (() => { + throw new Error( + `Failed to parse exit signal, received value: ${value}` + ); + })(); + break; + case 'message': + message = value; + break; + case 'status': + status = value; + break; + case 'time': + time = + parseFloat(value) || + (() => { + throw new Error( + `Failed to parse cpu time, received value: ${value}` + ); + })(); + break; + default: + break; + } + } + } catch (e) { + throw new Error( + `Error reading metadata file: ${self.metadata_file_path}\nError: ${e.message}\nIsolate run stdout: ${stdout}\nIsolate run stderr: ${stderr}` + ); + } + + proc.on('close', () => { + resolve({ + stdout, + stderr, + code, + signal, + output, + memory, + message, + status, + time, }); + }); - proc.on('exit', () => this.exit_cleanup()); - - proc.on('close', (code, signal) => { - this.close_cleanup(); - - resolve({ stdout, stderr, code, signal, output }); - }); - - proc.on('error', err => { - this.exit_cleanup(); - this.close_cleanup(); - - reject({ error: err, stdout, stderr, output }); + proc.on('error', err => { + reject({ + error: err, + stdout, + stderr, + code, + signal, + output, + memory, + message, + status, + time, }); }); } @@ -281,7 +314,7 @@ class Job { if (this.state !== job_states.PRIMED) { throw new Error( 'Job must be in primed state, current state: ' + - this.state.toString() + this.state.toString() ); } @@ -298,49 +331,49 @@ class Job { const { emit_event_bus_result, emit_event_bus_stage } = event_bus === null ? { - emit_event_bus_result: () => { }, - emit_event_bus_stage: () => { }, - } + emit_event_bus_result: () => {}, + emit_event_bus_stage: () => {}, + } : { - emit_event_bus_result: (stage, result, event_bus) => { - const { error, code, signal } = result; - event_bus.emit('exit', stage, { - error, - code, - signal, - }); - }, - emit_event_bus_stage: (stage, event_bus) => { - event_bus.emit('stage', stage); - }, - }; + emit_event_bus_result: (stage, result) => { + const { error, code, signal } = result; + event_bus.emit('exit', stage, { + error, + code, + signal, + }); + }, + emit_event_bus_stage: stage => { + event_bus.emit('stage', stage); + }, + }; if (this.runtime.compiled) { this.logger.debug('Compiling'); - emit_event_bus_stage('compile', event_bus); + emit_event_bus_stage('compile'); compile = await this.safe_call( - path.join(this.runtime.pkgdir, 'compile'), + '/runtime/compile', code_files.map(x => x.name), this.timeouts.compile, this.memory_limits.compile, event_bus ); - emit_event_bus_result('compile', compile, event_bus); + emit_event_bus_result('compile', compile); compile_errored = compile.code !== 0; } let run; if (!compile_errored) { this.logger.debug('Running'); - emit_event_bus_stage('run', event_bus); + emit_event_bus_stage('run'); run = await this.safe_call( - path.join(this.runtime.pkgdir, 'run'), + '/runtime/run', [code_files[0].name, ...this.args], this.timeouts.run, this.memory_limits.run, event_bus ); - emit_event_bus_result('run', run, event_bus); + emit_event_bus_result('run', run); } this.state = job_states.EXECUTED; @@ -353,135 +386,10 @@ class Job { }; } - cleanup_processes(dont_wait = []) { - let processes = [1]; - const to_wait = []; - this.logger.debug(`Cleaning up processes`); - - while (processes.length > 0) { - processes = []; - - const proc_ids = fss.readdir_sync('/proc'); - - processes = proc_ids.map(proc_id => { - if (isNaN(proc_id)) return -1; - try { - const proc_status = fss.read_file_sync( - path.join('/proc', proc_id, 'status') - ); - const proc_lines = proc_status.to_string().split('\n'); - const state_line = proc_lines.find(line => - line.starts_with('State:') - ); - const uid_line = proc_lines.find(line => - line.starts_with('Uid:') - ); - const [_, ruid, euid, suid, fuid] = uid_line.split(/\s+/); - - const [_1, state, user_friendly] = state_line.split(/\s+/); - - const proc_id_int = parse_int(proc_id); - - // Skip over any processes that aren't ours. - if (ruid != this.uid && euid != this.uid) return -1; - - if (state == 'Z') { - // Zombie process, just needs to be waited, regardless of the user id - if (!to_wait.includes(proc_id_int)) - to_wait.push(proc_id_int); - - return -1; - } - // We should kill in all other state (Sleep, Stopped & Running) - - return proc_id_int; - } catch { - return -1; - } - - return -1; - }); - - processes = processes.filter(p => p > 0); - - if (processes.length > 0) - this.logger.debug(`Got processes to kill: ${processes}`); - - for (const proc of processes) { - // First stop the processes, but keep their resources allocated so they cant re-fork - try { - process.kill(proc, 'SIGSTOP'); - } catch (e) { - // Could already be dead - this.logger.debug( - `Got error while SIGSTOPping process ${proc}:`, - e - ); - } - } - - for (const proc of processes) { - // Then clear them out of the process tree - try { - process.kill(proc, 'SIGKILL'); - } catch (e) { - // Could already be dead and just needs to be waited on - this.logger.debug( - `Got error while SIGKILLing process ${proc}:`, - e - ); - } - - to_wait.push(proc); - } - } - - this.logger.debug( - `Finished kill-loop, calling wait_pid to end any zombie processes` - ); - - for (const proc of to_wait) { - if (dont_wait.includes(proc)) continue; - - wait_pid(proc); - } - - this.logger.debug(`Cleaned up processes`); - } - - async cleanup_filesystem() { - for (const clean_path of globals.clean_directories) { - const contents = await fs.readdir(clean_path); - - for (const file of contents) { - const file_path = path.join(clean_path, file); - - try { - const stat = await fs.stat(file_path); - - if (stat.uid === this.uid) { - await fs.rm(file_path, { - recursive: true, - force: true, - }); - } - } catch (e) { - // File was somehow deleted in the time that we read the dir to when we checked the file - this.logger.warn(`Error removing file ${file_path}: ${e}`); - } - } - } - - await fs.rm(this.dir, { recursive: true, force: true }); - } - async cleanup() { this.logger.info(`Cleaning up job`); - this.exit_cleanup(); // Run process janitor, just incase there are any residual processes somehow - this.close_cleanup(); - await this.cleanup_filesystem(); - + await fs.rm(`${this.#metadata_file_path}`); remaining_job_spaces++; if (job_queue.length > 0) { job_queue.shift()(); diff --git a/docker-compose.dev.yaml b/docker-compose.dev.yaml index 8a0d385..a3d74cc 100644 --- a/docker-compose.dev.yaml +++ b/docker-compose.dev.yaml @@ -4,15 +4,12 @@ services: api: build: api container_name: piston_api - cap_add: - - CAP_SYS_ADMIN + privileged: true restart: always ports: - 2000:2000 volumes: - ./data/piston/packages:/piston/packages - environment: - - PISTON_REPO_URL=http://repo:8000/index tmpfs: - /piston/jobs:exec,uid=1000,gid=1000,mode=711 diff --git a/docker-compose.yaml b/docker-compose.yaml index 839b340..1d8e307 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -5,6 +5,7 @@ services: image: ghcr.io/engineer-man/piston container_name: piston_api restart: always + privileged: true ports: - 2000:2000 volumes: From 63dd925bb1015bcfdf768a183d6902c163378f18 Mon Sep 17 00:00:00 2001 From: Omar Brikaa Date: Sat, 24 Aug 2024 21:01:49 +0300 Subject: [PATCH 02/10] Continue: use Isolate for isolation --- api/src/api/v2.js | 8 ++-- api/src/job.js | 116 ++++++++++++++++++++++++++++++---------------- 2 files changed, 80 insertions(+), 44 deletions(-) diff --git a/api/src/api/v2.js b/api/src/api/v2.js index 032fd51..c7975d6 100644 --- a/api/src/api/v2.js +++ b/api/src/api/v2.js @@ -211,7 +211,7 @@ router.ws('/connect', async (ws, req) => { job = await get_job(msg); try { - await job.prime(); + const box = await job.prime(); ws.send( JSON.stringify({ @@ -221,7 +221,7 @@ router.ws('/connect', async (ws, req) => { }) ); - await job.execute(event_bus); + await job.execute(box, event_bus); } catch (error) { logger.error( `Error cleaning up job: ${job.uuid}:\n${error}` @@ -279,9 +279,9 @@ router.post('/execute', async (req, res) => { return res.status(400).json(error); } try { - await job.prime(); + const box = await job.prime(); - let result = await job.execute(); + let result = await job.execute(box); // Backward compatibility when the run stage is not started if (result.run === undefined) { result.run = result.compile; diff --git a/api/src/job.js b/api/src/job.js index 1ca235f..ceb08de 100644 --- a/api/src/job.js +++ b/api/src/job.js @@ -21,10 +21,7 @@ let job_queue = []; const get_next_box_id = () => (box_id + 1) % MAX_BOX_ID; class Job { - #box_id; - #metadata_file_path; - #box_dir; - + #dirty_boxes; constructor({ runtime, files, args, stdin, timeouts, memory_limits }) { this.uuid = uuidv4(); @@ -50,6 +47,34 @@ class Job { this.memory_limits = memory_limits; this.state = job_states.READY; + this.#dirty_boxes = []; + } + + async #create_isolate_box() { + const box_id = get_next_box_id(); + const metadata_file_path = `/tmp/${box_id}-metadata.txt`; + return new Promise((res, rej) => { + cp.exec( + `isolate --init --cg -b${box_id}`, + (error, stdout, stderr) => { + if (error) { + rej( + `Failed to run isolate --init: ${error.message}\nstdout: ${stdout}\nstderr: ${stderr}` + ); + } + if (stdout === '') { + rej('Received empty stdout from isolate --init'); + } + const box = { + id: box_id, + metadata_file_path, + dir: stdout, + }; + this.#dirty_boxes.push(box); + res(box); + } + ); + }); } async prime() { @@ -62,32 +87,14 @@ class Job { this.logger.info(`Priming job`); remaining_job_spaces--; this.logger.debug('Running isolate --init'); - this.#box_id = get_next_box_id(); - this.#metadata_file_path = `/tmp/${this.#box_id}-metadata.txt`; - await new Promise((res, rej) => { - cp.exec( - `isolate --init --cg -b${this.#box_id}`, - (error, stdout, stderr) => { - if (error) { - rej( - `Failed to run isolate --init: ${error.message}\nstdout: ${stdout}\nstderr: ${stderr}` - ); - } - if (stdout === '') { - rej('Received empty stdout from isolate --init'); - } - this.#box_dir = stdout; - res(); - } - ); - }); + const box = await this.#create_isolate_box(); this.logger.debug(`Creating submission files in Isolate box`); - - await fs.mkdir(path.join(this.#box_dir, 'submission')); + const submission_dir = path.join(box.dir, 'submission'); + await fs.mkdir(submission_dir); for (const file of this.files) { - const file_path = path.join(box_dir, file.name); - const rel = path.relative(box_dir, file_path); + const file_path = path.join(submission_dir, file.name); + const rel = path.relative(submission_dir, file_path); if (rel.startsWith('..')) throw Error( @@ -106,9 +113,10 @@ class Job { this.state = job_states.PRIMED; this.logger.debug('Primed job'); + return box; } - async safe_call(file, args, timeout, memory_limit, event_bus = null) { + async safe_call(box, file, args, timeout, memory_limit, event_bus = null) { var stdout = ''; var stderr = ''; var output = ''; @@ -117,8 +125,8 @@ class Job { ISOLATE_PATH, [ '--run', - `-b${this.#box_id}`, - `--meta=${this.#metadata_file_path}`, + `-b${box.id}`, + `--meta=${box.metadata_file_path}`, '--cg', '-s', '-c', @@ -131,7 +139,7 @@ class Job { `--time=${timeout}`, `--extra-time=0`, ...(memory_limit >= 0 ? [`--cg-mem=${memory_limit}`] : []), - ...(config.disable_networking ? [] : '--share-net'), + ...(config.disable_networking ? [] : ['--share-net']), '--', file, ...args, @@ -213,8 +221,8 @@ class Job { let time = null; try { - const metadata_str = await fs.readFile( - self.metadata_file_path, + const metadata_str = await fs.read_file( + box.metadata_file_path, 'utf-8' ); const metadata_lines = metadata_str.split('\n'); @@ -230,7 +238,7 @@ class Job { switch (key) { case 'cg-mem': memory = - parseInt(value) || + parse_int(value) || (() => { throw new Error( `Failed to parse memory usage, received value: ${value}` @@ -239,7 +247,7 @@ class Job { break; case 'exitcode': code = - parseInt(value) || + parse_int(value) || (() => { throw new Error( `Failed to parse exit code, received value: ${value}` @@ -248,7 +256,7 @@ class Job { break; case 'exitsig': signal = - parseInt(value) || + parse_int(value) || (() => { throw new Error( `Failed to parse exit signal, received value: ${value}` @@ -263,7 +271,7 @@ class Job { break; case 'time': time = - parseFloat(value) || + parse_float(value) || (() => { throw new Error( `Failed to parse cpu time, received value: ${value}` @@ -276,7 +284,7 @@ class Job { } } catch (e) { throw new Error( - `Error reading metadata file: ${self.metadata_file_path}\nError: ${e.message}\nIsolate run stdout: ${stdout}\nIsolate run stderr: ${stderr}` + `Error reading metadata file: ${box.metadata_file_path}\nError: ${e.message}\nIsolate run stdout: ${stdout}\nIsolate run stderr: ${stderr}` ); } @@ -310,7 +318,7 @@ class Job { }); } - async execute(event_bus = null) { + async execute(box, event_bus = null) { if (this.state !== job_states.PRIMED) { throw new Error( 'Job must be in primed state, current state: ' + @@ -352,6 +360,7 @@ class Job { this.logger.debug('Compiling'); emit_event_bus_stage('compile'); compile = await this.safe_call( + box, '/runtime/compile', code_files.map(x => x.name), this.timeouts.compile, @@ -365,8 +374,15 @@ class Job { let run; if (!compile_errored) { this.logger.debug('Running'); + const old_box_dir = box.dir; + const new_box = await this.#create_isolate_box(); + await fs.rename( + path.join(old_box_dir, 'submission'), + path.join(new_box, 'submission') + ); emit_event_bus_stage('run'); run = await this.safe_call( + new_box, '/runtime/run', [code_files[0].name, ...this.args], this.timeouts.run, @@ -389,11 +405,31 @@ class Job { async cleanup() { this.logger.info(`Cleaning up job`); - await fs.rm(`${this.#metadata_file_path}`); remaining_job_spaces++; if (job_queue.length > 0) { job_queue.shift()(); } + await Promise.all( + this.#dirty_boxes.map(async box => { + cp.exec( + `isolate --cleanup --cg -b${box.id}`, + (error, stdout, stderr) => { + if (error) { + this.logger.error( + `Failed to run isolate --cleanup: ${error.message} on box #${box.id}\nstdout: ${stdout}\nstderr: ${stderr}` + ); + } + } + ); + try { + await fs.rm(box.metadata_file_path); + } catch (e) { + this.logger.error( + `Failed to remove the metadata directory of box #${box.id}. Error: ${e.message}` + ); + } + }) + ); } } From 1a1236dcbe87242cfbdb970ddce694c748e4e63c Mon Sep 17 00:00:00 2001 From: Omar Brikaa Date: Fri, 30 Aug 2024 20:49:42 +0300 Subject: [PATCH 03/10] Bug fixes --- api/Dockerfile | 11 +-- api/src/api/v2.js | 16 ++-- api/src/docker-entrypoint.sh | 2 +- api/src/index.js | 4 - api/src/job.js | 173 ++++++++++++++++++----------------- 5 files changed, 100 insertions(+), 106 deletions(-) mode change 100644 => 100755 api/src/docker-entrypoint.sh diff --git a/api/Dockerfile b/api/Dockerfile index 640fbcc..51367f0 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -1,4 +1,4 @@ -FROM buildpack-deps@sha256:d56cd472000631b8faca51f40d4e3f1b20deffa588f9f207fa6c60efb62ba7c4 AS isolate +FROM buildpack-deps:bookworm AS isolate RUN apt-get update && \ apt-get install -y --no-install-recommends git libcap-dev && \ rm -rf /var/lib/apt/lists/* && \ @@ -8,23 +8,20 @@ RUN apt-get update && \ make -j$(nproc) install && \ rm -rf /tmp/* -FROM node:15.10.0-buster-slim +FROM node:20-bookworm-slim ENV DEBIAN_FRONTEND=noninteractive RUN dpkg-reconfigure -p critical dash -RUN for i in $(seq 1001 1500); do \ - groupadd -g $i runner$i && \ - useradd -M runner$i -g $i -u $i ; \ - done RUN apt-get update && \ apt-get install -y libxml2 gnupg tar coreutils util-linux libc6-dev \ binutils build-essential locales libpcre3-dev libevent-dev libgmp3-dev \ libncurses6 libncurses5 libedit-dev libseccomp-dev rename procps python3 \ libreadline-dev libblas-dev liblapack-dev libpcre3-dev libarpack2-dev \ libfftw3-dev libglpk-dev libqhull-dev libqrupdate-dev libsuitesparse-dev \ - libsundials-dev libpcre2-dev && \ + libsundials-dev libpcre2-dev libcap-dev && \ rm -rf /var/lib/apt/lists/* +RUN useradd -M piston COPY --from=isolate /usr/local/bin/isolate /usr/local/bin COPY --from=isolate /usr/local/etc/isolate /usr/local/etc/isolate diff --git a/api/src/api/v2.js b/api/src/api/v2.js index c7975d6..3a71306 100644 --- a/api/src/api/v2.js +++ b/api/src/api/v2.js @@ -135,23 +135,19 @@ function get_job(body) { } } - compile_timeout = compile_timeout || rt.timeouts.compile; - run_timeout = run_timeout || rt.timeouts.run; - compile_memory_limit = compile_memory_limit || rt.memory_limits.compile; - run_memory_limit = run_memory_limit || rt.memory_limits.run; resolve( new Job({ runtime: rt, - args: args || [], - stdin: stdin || '', + args: args ?? [], + stdin: stdin ?? '', files, timeouts: { - run: run_timeout, - compile: compile_timeout, + run: run_timeout ?? rt.timeouts.run, + compile: compile_timeout ?? rt.timeouts.compile, }, memory_limits: { - run: run_memory_limit, - compile: compile_memory_limit, + run: run_memory_limit ?? rt.memory_limits.run, + compile: compile_memory_limit ?? rt.memory_limits.compile, }, }) ); diff --git a/api/src/docker-entrypoint.sh b/api/src/docker-entrypoint.sh old mode 100644 new mode 100755 index 1160a25..8f32986 --- a/api/src/docker-entrypoint.sh +++ b/api/src/docker-entrypoint.sh @@ -9,4 +9,4 @@ mkdir init && \ echo 1 > init/cgroup.procs && \ echo '+cpuset +memory' > cgroup.subtree_control && \ echo "Initialized cgroup" && \ -exec su -- piston -c 'ulimit -n 65536 && node' +exec su -- piston -c 'ulimit -n 65536 && node /piston_api/src' diff --git a/api/src/index.js b/api/src/index.js index 6ad9390..4a6dd42 100644 --- a/api/src/index.js +++ b/api/src/index.js @@ -35,10 +35,6 @@ expressWs(app); } } }); - fss.chmodSync( - path.join(config.data_directory, globals.data_directories.jobs), - 0o711 - ); logger.info('Loading packages'); const pkgdir = path.join( diff --git a/api/src/job.js b/api/src/job.js index ceb08de..189d73b 100644 --- a/api/src/job.js +++ b/api/src/job.js @@ -18,7 +18,7 @@ let box_id = 0; let remaining_job_spaces = config.max_concurrent_jobs; let job_queue = []; -const get_next_box_id = () => (box_id + 1) % MAX_BOX_ID; +const get_next_box_id = () => ++box_id % MAX_BOX_ID; class Job { #dirty_boxes; @@ -68,7 +68,7 @@ class Job { const box = { id: box_id, metadata_file_path, - dir: stdout, + dir: `${stdout.trim()}/box`, }; this.#dirty_boxes.push(box); res(box); @@ -117,9 +117,15 @@ class Job { } async safe_call(box, file, args, timeout, memory_limit, event_bus = null) { - var stdout = ''; - var stderr = ''; - var output = ''; + let stdout = ''; + let stderr = ''; + let output = ''; + let memory = null; + let code = null; + let signal = null; + let message = null; + let status = null; + let time = null; const proc = cp.spawn( ISOLATE_PATH, @@ -133,14 +139,18 @@ class Job { '/box/submission', '-e', `--dir=/runtime=${this.runtime.pkgdir}`, + `--dir=/etc:noexec`, `--processes=${this.runtime.max_process_count}`, `--open-files=${this.runtime.max_open_files}`, - `--fsize=${this.runtime.max_file_size}`, - `--time=${timeout}`, + `--fsize=${Math.floor(this.runtime.max_file_size / 1000)}`, + `--time=${timeout / 1000}`, `--extra-time=0`, - ...(memory_limit >= 0 ? [`--cg-mem=${memory_limit}`] : []), + ...(memory_limit >= 0 + ? [`--cg-mem=${Math.floor(memory_limit / 1000)}`] + : []), ...(config.disable_networking ? [] : ['--share-net']), '--', + '/bin/bash', file, ...args, ], @@ -174,7 +184,8 @@ class Job { stderr.length + data.length > this.runtime.output_max_size ) { - this.logger.info(`stderr length exceeded`); + message = 'stderr length exceeded'; + this.logger.info(message); try { process.kill(proc.pid, 'SIGABRT'); } catch (e) { @@ -197,7 +208,8 @@ class Job { stdout.length + data.length > this.runtime.output_max_size ) { - this.logger.info(`stdout length exceeded`); + message = 'stdout length exceeded'; + this.logger.info(message); try { process.kill(proc.pid, 'SIGABRT'); } catch (e) { @@ -213,18 +225,27 @@ class Job { } }); - let memory = null; - let code = null; - let signal = null; - let message = null; - let status = null; - let time = null; + const data = await new Promise((res, rej) => { + proc.on('close', () => { + res({ + stdout, + stderr, + }); + }); + + proc.on('error', err => { + rej({ + error: err, + stdout, + stderr, + }); + }); + }); try { - const metadata_str = await fs.read_file( - box.metadata_file_path, - 'utf-8' - ); + const metadata_str = ( + await fs.read_file(box.metadata_file_path) + ).toString(); const metadata_lines = metadata_str.split('\n'); for (const line of metadata_lines) { if (!line) continue; @@ -237,46 +258,46 @@ class Job { } switch (key) { case 'cg-mem': - memory = - parse_int(value) || - (() => { - throw new Error( - `Failed to parse memory usage, received value: ${value}` - ); - })(); + try { + memory = parse_int(value); + } catch (e) { + throw new Error( + `Failed to parse memory usage, received value: ${value}` + ); + } break; case 'exitcode': - code = - parse_int(value) || - (() => { - throw new Error( - `Failed to parse exit code, received value: ${value}` - ); - })(); + try { + code = parse_int(value); + } catch (e) { + throw new Error( + `Failed to parse exit code, received value: ${value}` + ); + } break; case 'exitsig': - signal = - parse_int(value) || - (() => { - throw new Error( - `Failed to parse exit signal, received value: ${value}` - ); - })(); + try { + signal = parse_int(value); + } catch (e) { + throw new Error( + `Failed to parse exit signal, received value: ${value}` + ); + } break; case 'message': - message = value; + message = message || value; break; case 'status': status = value; break; case 'time': - time = - parse_float(value) || - (() => { - throw new Error( - `Failed to parse cpu time, received value: ${value}` - ); - })(); + try { + time = parse_float(value); + } catch (e) { + throw new Error( + `Failed to parse cpu time, received value: ${value}` + ); + } break; default: break; @@ -288,34 +309,16 @@ class Job { ); } - proc.on('close', () => { - resolve({ - stdout, - stderr, - code, - signal, - output, - memory, - message, - status, - time, - }); - }); - - proc.on('error', err => { - reject({ - error: err, - stdout, - stderr, - code, - signal, - output, - memory, - message, - status, - time, - }); - }); + return { + ...data, + code, + signal, + output, + memory, + message, + status, + time, + }; } async execute(box, event_bus = null) { @@ -369,20 +372,22 @@ class Job { ); emit_event_bus_result('compile', compile); compile_errored = compile.code !== 0; + if (!compile_errored) { + const old_box_dir = box.dir; + box = await this.#create_isolate_box(); + await fs.rename( + path.join(old_box_dir, 'submission'), + path.join(box.dir, 'submission') + ); + } } let run; if (!compile_errored) { this.logger.debug('Running'); - const old_box_dir = box.dir; - const new_box = await this.#create_isolate_box(); - await fs.rename( - path.join(old_box_dir, 'submission'), - path.join(new_box, 'submission') - ); emit_event_bus_stage('run'); run = await this.safe_call( - new_box, + box, '/runtime/run', [code_files[0].name, ...this.args], this.timeouts.run, From 2e00325163f6c5775e53f474dfc5855f726dbd2e Mon Sep 17 00:00:00 2001 From: Omar Brikaa Date: Fri, 30 Aug 2024 21:33:39 +0300 Subject: [PATCH 04/10] timeout is wall-time for backward compatibility --- api/src/api/v2.js | 9 ++++++- api/src/config.js | 16 +++++++++++- api/src/job.js | 43 ++++++++++++++++++++++++++----- api/src/runtime.js | 14 ++++++++++ docs/configuration.md | 2 +- packages/zig/0.10.1/metadata.json | 5 ++-- packages/zig/0.8.0/metadata.json | 3 ++- packages/zig/0.9.1/metadata.json | 3 ++- 8 files changed, 82 insertions(+), 13 deletions(-) diff --git a/api/src/api/v2.js b/api/src/api/v2.js index 3a71306..22c74f2 100644 --- a/api/src/api/v2.js +++ b/api/src/api/v2.js @@ -61,6 +61,8 @@ function get_job(body) { run_memory_limit, run_timeout, compile_timeout, + run_cpu_time, + compile_cpu_time, } = body; return new Promise((resolve, reject) => { @@ -106,7 +108,7 @@ function get_job(body) { }); } - for (const constraint of ['memory_limit', 'timeout']) { + for (const constraint of ['memory_limit', 'timeout', 'cpu_time']) { for (const type of ['compile', 'run']) { const constraint_name = `${type}_${constraint}`; const constraint_value = body[constraint_name]; @@ -145,6 +147,10 @@ function get_job(body) { run: run_timeout ?? rt.timeouts.run, compile: compile_timeout ?? rt.timeouts.compile, }, + cpu_times: { + run: run_cpu_time ?? rt.cpu_times.run, + compile: compile_cpu_time ?? rt.cpu_times.compile, + }, memory_limits: { run: run_memory_limit ?? rt.memory_limits.run, compile: compile_memory_limit ?? rt.memory_limits.compile, @@ -272,6 +278,7 @@ router.post('/execute', async (req, res) => { try { job = await get_job(req.body); } catch (error) { + logger.error({ error }); return res.status(400).json(error); } try { diff --git a/api/src/config.js b/api/src/config.js index b8fa97d..034e3b6 100644 --- a/api/src/config.js +++ b/api/src/config.js @@ -90,6 +90,18 @@ const options = { parser: parse_int, validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`], }, + compile_cpu_time: { + desc: 'Max CPU time allowed for compile stage in milliseconds', + default: 10000, // 10 seconds + parser: parse_int, + validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`], + }, + run_cpu_time: { + desc: 'Max CPU time allowed for run stage in milliseconds', + default: 3000, // 3 seconds + parser: parse_int, + validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`], + }, compile_memory_limit: { desc: 'Max memory usage for compile stage in bytes (set to -1 for no limit)', default: -1, // no limit @@ -117,7 +129,7 @@ const options = { limit_overrides: { desc: 'Per-language exceptions in JSON format for each of:\ max_process_count, max_open_files, max_file_size, compile_memory_limit,\ - run_memory_limit, compile_timeout, run_timeout, output_max_size', + run_memory_limit, compile_timeout, run_timeout, compile_cpu_time, run_cpu_time, output_max_size', default: {}, parser: parse_overrides, validators: [ @@ -165,6 +177,8 @@ function parse_overrides(overrides_string) { 'run_memory_limit', 'compile_timeout', 'run_timeout', + 'compile_cpu_time', + 'run_cpu_time', 'output_max_size', ].includes(key) ) { diff --git a/api/src/job.js b/api/src/job.js index 189d73b..f8d0043 100644 --- a/api/src/job.js +++ b/api/src/job.js @@ -22,7 +22,15 @@ const get_next_box_id = () => ++box_id % MAX_BOX_ID; class Job { #dirty_boxes; - constructor({ runtime, files, args, stdin, timeouts, memory_limits }) { + constructor({ + runtime, + files, + args, + stdin, + timeouts, + cpu_times, + memory_limits, + }) { this.uuid = uuidv4(); this.logger = logplease.create(`job/${this.uuid}`); @@ -44,6 +52,7 @@ class Job { } this.timeouts = timeouts; + this.cpu_times = cpu_times; this.memory_limits = memory_limits; this.state = job_states.READY; @@ -116,7 +125,15 @@ class Job { return box; } - async safe_call(box, file, args, timeout, memory_limit, event_bus = null) { + async safe_call( + box, + file, + args, + timeout, + cpu_time, + memory_limit, + event_bus = null + ) { let stdout = ''; let stderr = ''; let output = ''; @@ -125,7 +142,8 @@ class Job { let signal = null; let message = null; let status = null; - let time = null; + let cpu_time_stat = null; + let wall_time_stat = null; const proc = cp.spawn( ISOLATE_PATH, @@ -143,7 +161,8 @@ class Job { `--processes=${this.runtime.max_process_count}`, `--open-files=${this.runtime.max_open_files}`, `--fsize=${Math.floor(this.runtime.max_file_size / 1000)}`, - `--time=${timeout / 1000}`, + `--wall-time=${timeout / 1000}`, + `--time=${cpu_time / 1000}`, `--extra-time=0`, ...(memory_limit >= 0 ? [`--cg-mem=${Math.floor(memory_limit / 1000)}`] @@ -292,13 +311,22 @@ class Job { break; case 'time': try { - time = parse_float(value); + cpu_time_stat = parse_float(value); } catch (e) { throw new Error( `Failed to parse cpu time, received value: ${value}` ); } break; + case 'time-wall': + try { + wall_time_stat = parse_float(value); + } catch (e) { + throw new Error( + `Failed to parse wall time, received value: ${value}` + ); + } + break; default: break; } @@ -317,7 +345,8 @@ class Job { memory, message, status, - time, + cpu_time: cpu_time_stat, + wall_time: wall_time_stat, }; } @@ -367,6 +396,7 @@ class Job { '/runtime/compile', code_files.map(x => x.name), this.timeouts.compile, + this.cpu_times.compile, this.memory_limits.compile, event_bus ); @@ -391,6 +421,7 @@ class Job { '/runtime/run', [code_files[0].name, ...this.args], this.timeouts.run, + this.cpu_times.run, this.memory_limits.run, event_bus ); diff --git a/api/src/runtime.js b/api/src/runtime.js index 6c6f10e..9a2adf4 100644 --- a/api/src/runtime.js +++ b/api/src/runtime.js @@ -15,6 +15,7 @@ class Runtime { pkgdir, runtime, timeouts, + cpu_times, memory_limits, max_process_count, max_open_files, @@ -27,6 +28,7 @@ class Runtime { this.pkgdir = pkgdir; this.runtime = runtime; this.timeouts = timeouts; + this.cpu_times = cpu_times; this.memory_limits = memory_limits; this.max_process_count = max_process_count; this.max_open_files = max_open_files; @@ -62,6 +64,18 @@ class Runtime { language_limit_overrides ), }, + cpu_times: { + compile: this.compute_single_limit( + language_name, + 'compile_cpu_time', + language_limit_overrides + ), + run: this.compute_single_limit( + language_name, + 'run_cpu_time', + language_limit_overrides + ), + }, memory_limits: { compile: this.compute_single_limit( language_name, diff --git a/docs/configuration.md b/docs/configuration.md index 1a6f5bd..4c4aa55 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -178,7 +178,7 @@ default: {} ``` Per-language overrides/exceptions for the each of `max_process_count`, `max_open_files`, `max_file_size`, -`compile_memory_limit`, `run_memory_limit`, `compile_timeout`, `run_timeout`, `output_max_size`. Defined as follows: +`compile_memory_limit`, `run_memory_limit`, `compile_timeout`, `run_timeout`, `compile_cpu_time`, `run_cpu_time`, `output_max_size`. Defined as follows: ``` PISTON_LIMIT_OVERRIDES={"c++":{"max_process_count":128}} diff --git a/packages/zig/0.10.1/metadata.json b/packages/zig/0.10.1/metadata.json index 9ecb955..20a9963 100644 --- a/packages/zig/0.10.1/metadata.json +++ b/packages/zig/0.10.1/metadata.json @@ -3,6 +3,7 @@ "version": "0.10.1", "aliases": [], "limit_overrides": { - "compile_timeout": 15000 + "compile_timeout": 15000, + "compile_cpu_time": 15000 } -} \ No newline at end of file +} diff --git a/packages/zig/0.8.0/metadata.json b/packages/zig/0.8.0/metadata.json index 8c02d33..38bc1fc 100644 --- a/packages/zig/0.8.0/metadata.json +++ b/packages/zig/0.8.0/metadata.json @@ -3,6 +3,7 @@ "version": "0.8.0", "aliases": ["zig"], "limit_overrides": { - "compile_timeout": 15000 + "compile_timeout": 15000, + "compile_cpu_time": 15000 } } diff --git a/packages/zig/0.9.1/metadata.json b/packages/zig/0.9.1/metadata.json index e7061cd..1ad7a70 100644 --- a/packages/zig/0.9.1/metadata.json +++ b/packages/zig/0.9.1/metadata.json @@ -3,6 +3,7 @@ "version": "0.9.1", "aliases": ["zig"], "limit_overrides": { - "compile_timeout": 15000 + "compile_timeout": 15000, + "compile_cpu_time": 15000 } } From 09ff4ca79e618b5032218ba17f87c9787bb77f73 Mon Sep 17 00:00:00 2001 From: Omar Brikaa Date: Sat, 31 Aug 2024 21:45:43 +0300 Subject: [PATCH 05/10] Documentation, signal names, reported time in ms --- api/src/api/v2.js | 47 +++-------------------------- api/src/globals.js | 65 +++++++++++++++++++++++++++++++++++++++++ api/src/job.js | 50 +++++++------------------------ docker-compose.dev.yaml | 2 ++ docs/configuration.md | 17 +++++++++-- readme.md | 24 +++++++++++---- 6 files changed, 115 insertions(+), 90 deletions(-) diff --git a/api/src/api/v2.js b/api/src/api/v2.js index 22c74f2..b454272 100644 --- a/api/src/api/v2.js +++ b/api/src/api/v2.js @@ -6,50 +6,9 @@ const events = require('events'); const runtime = require('../runtime'); const { Job } = require('../job'); const package = require('../package'); +const globals = require('../globals'); const logger = require('logplease').create('api/v2'); -const SIGNALS = [ - 'SIGABRT', - 'SIGALRM', - 'SIGBUS', - 'SIGCHLD', - 'SIGCLD', - 'SIGCONT', - 'SIGEMT', - 'SIGFPE', - 'SIGHUP', - 'SIGILL', - 'SIGINFO', - 'SIGINT', - 'SIGIO', - 'SIGIOT', - 'SIGKILL', - 'SIGLOST', - 'SIGPIPE', - 'SIGPOLL', - 'SIGPROF', - 'SIGPWR', - 'SIGQUIT', - 'SIGSEGV', - 'SIGSTKFLT', - 'SIGSTOP', - 'SIGTSTP', - 'SIGSYS', - 'SIGTERM', - 'SIGTRAP', - 'SIGTTIN', - 'SIGTTOU', - 'SIGUNUSED', - 'SIGURG', - 'SIGUSR1', - 'SIGUSR2', - 'SIGVTALRM', - 'SIGXCPU', - 'SIGXFSZ', - 'SIGWINCH', -]; -// ref: https://man7.org/linux/man-pages/man7/signal.7.html - function get_job(body) { let { language, @@ -250,7 +209,9 @@ router.ws('/connect', async (ws, req) => { break; case 'signal': if (job !== null) { - if (SIGNALS.includes(msg.signal)) { + if ( + Object.values(globals.SIGNALS).includes(msg.signal) + ) { event_bus.emit('signal', msg.signal); } else { ws.close(4005, 'Invalid signal'); diff --git a/api/src/globals.js b/api/src/globals.js index d83d834..c2fef42 100644 --- a/api/src/globals.js +++ b/api/src/globals.js @@ -7,6 +7,70 @@ const platform = `${is_docker() ? 'docker' : 'baremetal'}-${fs .split('\n') .find(x => x.startsWith('ID')) .replace('ID=', '')}`; +const SIGNALS = { + 1: 'SIGHUP', + 2: 'SIGINT', + 3: 'SIGQUIT', + 4: 'SIGILL', + 5: 'SIGTRAP', + 6: 'SIGABRT', + 7: 'SIGBUS', + 8: 'SIGFPE', + 9: 'SIGKILL', + 10: 'SIGUSR1', + 11: 'SIGSEGV', + 12: 'SIGUSR2', + 13: 'SIGPIPE', + 14: 'SIGALRM', + 15: 'SIGTERM', + 16: 'SIGSTKFLT', + 17: 'SIGCHLD', + 18: 'SIGCONT', + 19: 'SIGSTOP', + 20: 'SIGTSTP', + 21: 'SIGTTIN', + 22: 'SIGTTOU', + 23: 'SIGURG', + 24: 'SIGXCPU', + 25: 'SIGXFSZ', + 26: 'SIGVTALRM', + 27: 'SIGPROF', + 28: 'SIGWINCH', + 29: 'SIGIO', + 30: 'SIGPWR', + 31: 'SIGSYS', + 34: 'SIGRTMIN', + 35: 'SIGRTMIN+1', + 36: 'SIGRTMIN+2', + 37: 'SIGRTMIN+3', + 38: 'SIGRTMIN+4', + 39: 'SIGRTMIN+5', + 40: 'SIGRTMIN+6', + 41: 'SIGRTMIN+7', + 42: 'SIGRTMIN+8', + 43: 'SIGRTMIN+9', + 44: 'SIGRTMIN+10', + 45: 'SIGRTMIN+11', + 46: 'SIGRTMIN+12', + 47: 'SIGRTMIN+13', + 48: 'SIGRTMIN+14', + 49: 'SIGRTMIN+15', + 50: 'SIGRTMAX-14', + 51: 'SIGRTMAX-13', + 52: 'SIGRTMAX-12', + 53: 'SIGRTMAX-11', + 54: 'SIGRTMAX-10', + 55: 'SIGRTMAX-9', + 56: 'SIGRTMAX-8', + 57: 'SIGRTMAX-7', + 58: 'SIGRTMAX-6', + 59: 'SIGRTMAX-5', + 60: 'SIGRTMAX-4', + 61: 'SIGRTMAX-3', + 62: 'SIGRTMAX-2', + 63: 'SIGRTMAX-1', + 64: 'SIGRTMAX', +}; module.exports = { data_directories: { @@ -16,4 +80,5 @@ module.exports = { platform, pkg_installed_file: '.ppman-installed', //Used as indication for if a package was installed clean_directories: ['/dev/shm', '/run/lock', '/tmp', '/var/tmp'], + SIGNALS, }; diff --git a/api/src/job.js b/api/src/job.js index f8d0043..e637a7e 100644 --- a/api/src/job.js +++ b/api/src/job.js @@ -4,6 +4,7 @@ const cp = require('child_process'); const path = require('path'); const config = require('./config'); const fs = require('fs/promises'); +const globals = require('./globals'); const job_states = { READY: Symbol('Ready to be primed'), @@ -245,18 +246,15 @@ class Job { }); const data = await new Promise((res, rej) => { - proc.on('close', () => { + proc.on('exit', (_, signal) => { res({ - stdout, - stderr, + signal, }); }); proc.on('error', err => { rej({ error: err, - stdout, - stderr, }); }); }); @@ -277,31 +275,13 @@ class Job { } switch (key) { case 'cg-mem': - try { - memory = parse_int(value); - } catch (e) { - throw new Error( - `Failed to parse memory usage, received value: ${value}` - ); - } + memory = parse_int(value); break; case 'exitcode': - try { - code = parse_int(value); - } catch (e) { - throw new Error( - `Failed to parse exit code, received value: ${value}` - ); - } + code = parse_int(value); break; case 'exitsig': - try { - signal = parse_int(value); - } catch (e) { - throw new Error( - `Failed to parse exit signal, received value: ${value}` - ); - } + signal = globals.SIGNALS[parse_int(value)] ?? null; break; case 'message': message = message || value; @@ -310,22 +290,10 @@ class Job { status = value; break; case 'time': - try { - cpu_time_stat = parse_float(value); - } catch (e) { - throw new Error( - `Failed to parse cpu time, received value: ${value}` - ); - } + cpu_time_stat = parse_float(value) * 1000; break; case 'time-wall': - try { - wall_time_stat = parse_float(value); - } catch (e) { - throw new Error( - `Failed to parse wall time, received value: ${value}` - ); - } + wall_time_stat = parse_float(value) * 1000; break; default: break; @@ -339,6 +307,8 @@ class Job { return { ...data, + stdout, + stderr, code, signal, output, diff --git a/docker-compose.dev.yaml b/docker-compose.dev.yaml index a3d74cc..2180344 100644 --- a/docker-compose.dev.yaml +++ b/docker-compose.dev.yaml @@ -10,6 +10,8 @@ services: - 2000:2000 volumes: - ./data/piston/packages:/piston/packages + environment: + - PISTON_REPO_URL=http://repo:8000/index tmpfs: - /piston/jobs:exec,uid=1000,gid=1000,mode=711 diff --git a/docs/configuration.md b/docs/configuration.md index 4c4aa55..163cd08 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -135,8 +135,21 @@ key: default: 3000 ``` -The maximum time that is allowed to be taken by a stage in milliseconds. -Use -1 for unlimited time. +The maximum time that is allowed to be taken by a stage in milliseconds. This is the wall-time of the stage. The time that the CPU does not spend working on the stage (e.g, due to context switches or IO) is counted. + +## Compile/Run CPU-Time + +```yaml +key: + - PISTON_COMPILE_CPU_TIME +default: 10000 + +key: + - PISTON_RUN_CPU_TIME +default: 3000 +``` + +The maximum CPU-time that is allowed to be consumed by a stage in milliseconds. The time that the CPU does not spend working on the stage (e.g, IO and context switches) is not counted. This option is typically used in algorithm contests. ## Compile/Run memory limits diff --git a/readme.md b/readme.md index 040e74d..31ccf5c 100644 --- a/readme.md +++ b/readme.md @@ -104,7 +104,8 @@ POST https://emkc.org/api/v2/piston/execute - Docker - Docker Compose -- Node JS (>= 13, preferably >= 15) +- Node JS (>= 15) +- cgroup v2 enabled, and cgroup v1 disabled ### After system dependencies are installed, clone this repository: @@ -245,8 +246,10 @@ This endpoint requests execution of some arbitrary code. - `files[].encoding` (_optional_) The encoding scheme used for the file content. One of `base64`, `hex` or `utf8`. Defaults to `utf8`. - `stdin` (_optional_) The text to pass as stdin to the program. Must be a string or left out. Defaults to blank string. - `args` (_optional_) The arguments to pass to the program. Must be an array or left out. Defaults to `[]`. -- `compile_timeout` (_optional_) The maximum time allowed for the compile stage to finish before bailing out in milliseconds. Must be a number or left out. Defaults to `10000` (10 seconds). -- `run_timeout` (_optional_) The maximum time allowed for the run stage to finish before bailing out in milliseconds. Must be a number or left out. Defaults to `3000` (3 seconds). +- `compile_timeout` (_optional_) The maximum wall-time allowed for the compile stage to finish before bailing out in milliseconds. Must be a number or left out. Defaults to `10000` (10 seconds). +- `run_timeout` (_optional_) The maximum wall-time allowed for the run stage to finish before bailing out in milliseconds. Must be a number or left out. Defaults to `3000` (3 seconds). +- `compile_cpu_time` (_optional_) The maximum CPU-time allowed for the compile stage to finish before bailing out in milliseconds. Must be a number or left out. Defaults to `10000` (10 seconds). +- `run_cpu_time` (_optional_) The maximum CPU-time allowed for the run stage to finish before bailing out in milliseconds. Must be a number or left out. Defaults to `3000` (3 seconds). - `compile_memory_limit` (_optional_) The maximum amount of memory the compile stage is allowed to use in bytes. Must be a number or left out. Defaults to `-1` (no limit) - `run_memory_limit` (_optional_) The maximum amount of memory the run stage is allowed to use in bytes. Must be a number or left out. Defaults to `-1` (no limit) @@ -264,6 +267,8 @@ This endpoint requests execution of some arbitrary code. "args": ["1", "2", "3"], "compile_timeout": 10000, "run_timeout": 3000, + "compile_cpu_time": 10000, + "run_cpu_time": 3000, "compile_memory_limit": -1, "run_memory_limit": -1 } @@ -273,7 +278,12 @@ A typical response upon successful execution will contain 1 or 2 keys `run` and `compile` will only be present if the language requested requires a compile stage. Each of these keys has an identical structure, containing both a `stdout` and `stderr` key, which is a string containing the text outputted during the stage into each buffer. -It also contains the `code` and `signal` which was returned from each process. +It also contains the `code` and `signal` which was returned from each process. It also includes a nullable human-readable `message` which is a description of why a stage has failed and a two-letter `status` that is either: + +- `RE` for runtime error +- `SG` for dying on a signal +- `TO` for timeout (either via `timeout` or `cpu_time`) +- `XX` for internal error ```json HTTP/1.1 200 OK @@ -287,7 +297,11 @@ Content-Type: application/json "stderr": "", "output": "[\n '/piston/packages/node/15.10.0/bin/node',\n '/piston/jobs/9501b09d-0105-496b-b61a-e5148cf66384/my_cool_code.js',\n '1',\n '2',\n '3'\n]\n", "code": 0, - "signal": null + "signal": null, + "message": null, + "status": null, + "cpu_time": 8, + "wall_time": 154 } } ``` From b3cc3c14e847cd97e392351e019471e5dfa71144 Mon Sep 17 00:00:00 2001 From: Omar Brikaa Date: Fri, 6 Sep 2024 16:26:43 +0300 Subject: [PATCH 06/10] Report memory usage in bytes --- api/src/job.js | 2 +- readme.md | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/api/src/job.js b/api/src/job.js index e637a7e..e573843 100644 --- a/api/src/job.js +++ b/api/src/job.js @@ -275,7 +275,7 @@ class Job { } switch (key) { case 'cg-mem': - memory = parse_int(value); + memory = parse_int(value) * 1000; break; case 'exitcode': code = parse_int(value); diff --git a/readme.md b/readme.md index 31ccf5c..b765a5b 100644 --- a/readme.md +++ b/readme.md @@ -301,7 +301,8 @@ Content-Type: application/json "message": null, "status": null, "cpu_time": 8, - "wall_time": 154 + "wall_time": 154, + "memory": 1160000 } } ``` From 6b02d120fdc48240c1eab9813b3d21e256975aff Mon Sep 17 00:00:00 2001 From: Omar Brikaa Date: Fri, 6 Sep 2024 18:43:07 +0300 Subject: [PATCH 07/10] Add privileged flags where needed --- .github/workflows/package-pr.yaml | 2 +- builder/build.sh | 7 ++++--- docker-compose.dev.yaml | 6 ++++-- readme.md | 1 + 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/package-pr.yaml b/.github/workflows/package-pr.yaml index 7a550f2..e0c5e12 100644 --- a/.github/workflows/package-pr.yaml +++ b/.github/workflows/package-pr.yaml @@ -92,7 +92,7 @@ jobs: docker run -v $(pwd)'/repo:/piston/repo' -v $(pwd)'/packages:/piston/packages' -d --name repo docker.pkg.github.com/engineer-man/piston/repo-builder --no-build docker pull docker.pkg.github.com/engineer-man/piston/api docker build -t piston-api api - docker run --network container:repo -v $(pwd)'/data:/piston' -e PISTON_LOG_LEVEL=DEBUG -e 'PISTON_REPO_URL=http://localhost:8000/index' -d --name api piston-api + docker run --privileged --network container:repo -v $(pwd)'/data:/piston' -e PISTON_LOG_LEVEL=DEBUG -e 'PISTON_REPO_URL=http://localhost:8000/index' -d --name api piston-api echo Waiting for API to start.. docker run --network container:api appropriate/curl -s --retry 10 --retry-connrefused http://localhost:2000/api/v2/runtimes diff --git a/builder/build.sh b/builder/build.sh index eaae21d..ca4d40f 100755 --- a/builder/build.sh +++ b/builder/build.sh @@ -23,20 +23,21 @@ fetch_packages(){ mkdir build # Start a piston container docker run \ + --privileged \ -v "$PWD/build":'/piston/packages' \ --tmpfs /piston/jobs \ -dit \ -p $port:2000 \ --name builder_piston_instance \ ghcr.io/engineer-man/piston - + # Ensure the CLI is installed cd ../cli npm i cd - # Evalulate the specfile - ../cli/index.js -u "http://127.0.0.1:$port" ppman spec $1 + ../cli/index.js -u "http://127.0.0.1:$port" ppman spec $1 } build_container(){ @@ -61,4 +62,4 @@ fetch_packages $SPEC_FILE build_container $TAG echo "Start your custom piston container with" -echo "$ docker run --tmpfs /piston/jobs -dit -p 2000:2000 $TAG" +echo "$ docker run --privileged --tmpfs /piston/jobs -dit -p 2000:2000 $TAG" diff --git a/docker-compose.dev.yaml b/docker-compose.dev.yaml index 2180344..2311c5b 100644 --- a/docker-compose.dev.yaml +++ b/docker-compose.dev.yaml @@ -10,10 +10,12 @@ services: - 2000:2000 volumes: - ./data/piston/packages:/piston/packages - environment: - - PISTON_REPO_URL=http://repo:8000/index tmpfs: - /piston/jobs:exec,uid=1000,gid=1000,mode=711 + environment: + - PISTON_DISABLE_NETWORKING=false + - PISTON_RUN_CPU_TIME=1000 + - PISTON_LIMIT_OVERRIDES={"c++":{"run_cpu_time":700}} repo: # Local testing of packages build: repo diff --git a/readme.md b/readme.md index b765a5b..28e552d 100644 --- a/readme.md +++ b/readme.md @@ -136,6 +136,7 @@ The API will now be online with no language runtimes installed. To install runti ```sh docker run \ + --privileged \ -v $PWD:'/piston' \ --tmpfs /piston/jobs \ -dit \ From c292e36a38f06ee87b5ede69909fe3d720a8439d Mon Sep 17 00:00:00 2001 From: Omar Brikaa Date: Fri, 6 Sep 2024 19:01:57 +0300 Subject: [PATCH 08/10] Remove tmpfs --- docker-compose.dev.yaml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/docker-compose.dev.yaml b/docker-compose.dev.yaml index 2311c5b..33f615d 100644 --- a/docker-compose.dev.yaml +++ b/docker-compose.dev.yaml @@ -10,12 +10,8 @@ services: - 2000:2000 volumes: - ./data/piston/packages:/piston/packages - tmpfs: - - /piston/jobs:exec,uid=1000,gid=1000,mode=711 environment: - - PISTON_DISABLE_NETWORKING=false - - PISTON_RUN_CPU_TIME=1000 - - PISTON_LIMIT_OVERRIDES={"c++":{"run_cpu_time":700}} + - PISTON_REPO_URL=http://repo:8000/index repo: # Local testing of packages build: repo From 409f7a1318afa14257c815c793f1b27769fcef23 Mon Sep 17 00:00:00 2001 From: Omar Brikaa Date: Fri, 6 Sep 2024 19:13:02 +0300 Subject: [PATCH 09/10] Remove tmpfs --- builder/build.sh | 3 +-- docker-compose.yaml | 1 - readme.md | 1 - 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/builder/build.sh b/builder/build.sh index ca4d40f..8087e9f 100755 --- a/builder/build.sh +++ b/builder/build.sh @@ -25,7 +25,6 @@ fetch_packages(){ docker run \ --privileged \ -v "$PWD/build":'/piston/packages' \ - --tmpfs /piston/jobs \ -dit \ -p $port:2000 \ --name builder_piston_instance \ @@ -62,4 +61,4 @@ fetch_packages $SPEC_FILE build_container $TAG echo "Start your custom piston container with" -echo "$ docker run --privileged --tmpfs /piston/jobs -dit -p 2000:2000 $TAG" +echo "$ docker run --privileged -dit -p 2000:2000 $TAG" diff --git a/docker-compose.yaml b/docker-compose.yaml index 1d8e307..ea62b06 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -11,5 +11,4 @@ services: volumes: - ./data/piston/packages:/piston/packages tmpfs: - - /piston/jobs:exec,uid=1000,gid=1000,mode=711 - /tmp:exec diff --git a/readme.md b/readme.md index 28e552d..18e9e5a 100644 --- a/readme.md +++ b/readme.md @@ -138,7 +138,6 @@ The API will now be online with no language runtimes installed. To install runti docker run \ --privileged \ -v $PWD:'/piston' \ - --tmpfs /piston/jobs \ -dit \ -p 2000:2000 \ --name piston_api \ From 9a4ea72043474859074b9a0c181681226eec914d Mon Sep 17 00:00:00 2001 From: Omar Brikaa Date: Fri, 6 Sep 2024 20:38:00 +0300 Subject: [PATCH 10/10] Fix package installation --- api/src/api/v2.js | 1 - api/src/docker-entrypoint.sh | 1 + api/src/package.js | 6 +++++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/api/src/api/v2.js b/api/src/api/v2.js index b454272..f8e2b57 100644 --- a/api/src/api/v2.js +++ b/api/src/api/v2.js @@ -239,7 +239,6 @@ router.post('/execute', async (req, res) => { try { job = await get_job(req.body); } catch (error) { - logger.error({ error }); return res.status(400).json(error); } try { diff --git a/api/src/docker-entrypoint.sh b/api/src/docker-entrypoint.sh index 8f32986..7cf37e3 100755 --- a/api/src/docker-entrypoint.sh +++ b/api/src/docker-entrypoint.sh @@ -9,4 +9,5 @@ mkdir init && \ echo 1 > init/cgroup.procs && \ echo '+cpuset +memory' > cgroup.subtree_control && \ echo "Initialized cgroup" && \ +chown -R piston:piston /piston && \ exec su -- piston -c 'ulimit -n 65536 && node /piston_api/src' diff --git a/api/src/package.js b/api/src/package.js index 11e4f34..8edb008 100644 --- a/api/src/package.js +++ b/api/src/package.js @@ -145,7 +145,11 @@ class Package { await fs.write_file(path.join(this.install_path, '.env'), filtered_env); logger.debug('Changing Ownership of package directory'); - await util.promisify(chownr)(this.install_path, 0, 0); + await util.promisify(chownr)( + this.install_path, + process.getuid(), + process.getgid() + ); logger.debug('Writing installed state to disk'); await fs.write_file(