Mascali
/
piston
mirror of https://github.com/engineer-man/piston.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add per-language constraint overrides

This commit is contained in:
Brikaa 2021-10-02 14:08:36 +02:00
parent 4870441574
commit 94af5639bf
5 changed files with 112 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

93
api/src/api/v2.js
View File

@ -3,7 +3,6 @@ const router = express.Router();
const events = require('events');
const config = require('../config');
const runtime = require('../runtime');
const { Job } = require('../job');
const package = require('../package');
@ -13,7 +12,7 @@ const SIGNALS = ["SIGABRT","SIGALRM","SIGBUS","SIGCHLD","SIGCLD","SIGCONT","SIGE
// ref: https://man7.org/linux/man-pages/man7/signal.7.html
function get_job(body){
const {
let {
language,
version,
args,
@ -31,19 +30,16 @@ function get_job(body){
message: 'language is required as a string',
});
}
if (!version || typeof version !== 'string') {
return reject({
message: 'version is required as a string',
});
}
if (!files || !Array.isArray(files)) {
return reject({
message: 'files is required as an array',
});
}
for (const [i, file] of files.entries()) {
if (typeof file.content !== 'string') {
return reject({
@ -52,73 +48,64 @@ function get_job(body){
}
}
if (compile_memory_limit) {
if (typeof compile_memory_limit !== 'number') {
return reject({
message: 'if specified, compile_memory_limit must be a number',
});
}
if (
config.compile_memory_limit >= 0 &&
(compile_memory_limit > config.compile_memory_limit ||
compile_memory_limit < 0)
) {
return reject({
message:
'compile_memory_limit cannot exceed the configured limit of ' +
config.compile_memory_limit,
});
}
}
if (run_memory_limit) {
if (typeof run_memory_limit !== 'number') {
return reject({
message: 'if specified, run_memory_limit must be a number',
});
}
if (
config.run_memory_limit >= 0 &&
(run_memory_limit > config.run_memory_limit || run_memory_limit < 0)
) {
return reject({
message:
'run_memory_limit cannot exceed the configured limit of ' +
config.run_memory_limit,
});
}
}
const rt = runtime.get_latest_runtime_matching_language_version(
language,
version
);
if (rt === undefined) {
return reject({
message: `${language}-${version} runtime is unknown`,
});
}
for (let constraint of ['memory_limit', 'timeout']) {
for (let type of ['compile', 'run']) {
let constraint_name = `${type}_${constraint}`;
let constraint_value = body[constraint_name];
let configured_limit = rt[`${constraint}s`][type];
if (!constraint_value) {
continue;
}
if (typeof constraint_value !== 'number') {
return reject({
message: `If specified, ${constraint_name} must be a number`
});
}
if (configured_limit <= 0) {
continue;
}
if (constraint_value > configured_limit) {
return reject({
message: `${constraint_name} cannot exceed the configured limit of ${configured_limit}`
});
}
if (constraint_value < 0) {
return reject({
message: `${constraint_name} must be non-negative`
});
}
}
}
compile_timeout = compile_timeout || rt.timeouts.compile;
run_timeout = run_timeout || rt.timeouts.run;
compile_memory_limit = compile_memory_limit || rt.memory_limits.compile;
run_timeout = run_timeout || rt.timeouts.run;
resolve(new Job({
runtime: rt,
alias: language,
args: args || [],
stdin: stdin || "",
files,
timeouts: {
run: run_timeout || config.run_timeout,
compile: compile_timeout || config.compile_timeout,
run: run_timeout,
compile: compile_timeout,
},
memory_limits: {
run: run_memory_limit || config.run_memory_limit,
compile: compile_memory_limit || config.compile_memory_limit,
run: run_memory_limit,
compile: compile_memory_limit,
}
}));
})
});
}
router.use((req, res, next) => {
@ -228,7 +215,7 @@ router.post('/execute', async (req, res) => {
return res.status(200).send(result);
}catch(error){
return res.status(400).json(error.to_string());
return res.status(400).json(error);
}
});

22
api/src/config.js
View File

@ -24,7 +24,7 @@ function validate_overrides(overrides, options) {
logger.error(`Invalid overridden option: ${key}`);
return false;
}
let option = options.find((o) => o.key == key);
let option = options.find((o) => o.key === key);
let parser = option.parser;
let raw = overrides[language][key];
let value = parser(raw);
@ -38,8 +38,10 @@ function validate_overrides(overrides, options) {
}
overrides[language][key] = value;
}
// Modifies the reference
options[options.index_of(options.find((o) => o.key === 'limit_overrides'))] = overrides;
}
return overrides;
return true;
}
const options = [
@ -184,8 +186,10 @@ const options = [
run_memory_limit, compile_timeout, run_timeout, output_max_size',
default: {},
parser: parse_overrides,
validators: [(x) => !!x || `Invalid JSON format for the overrides\n${x}`]
// More validation is done after the configs are loaded
validators: [
(x) => !!x || `Invalid JSON format for the overrides\n${x}`,
(overrides, _, options) => validate_overrides(overrides, options) || `Failed to validate the overrides`
]
}
];
@ -208,8 +212,8 @@ options.forEach(option => {
option.validators.for_each(validator => {
let response = null;
if (env_val) response = validator(parsed_val, env_val);
else response = validator(value, value);
if (env_val) response = validator(parsed_val, env_val, options);
else response = validator(value, value, options);
if (response !== true) {
errored = true;
@ -224,16 +228,10 @@ options.forEach(option => {
config[option.key] = value;
});
let overrides = validate_overrides(config.limit_overrides, options)
errored = errored || !overrides;
if (errored) {
process.exit(1);
}
config.limit_overrides = overrides;
console.log(config.limit_overrides);
logger.info('Configuration successfully loaded');
module.exports = config;

33
api/src/job.js
View File

@ -30,7 +30,7 @@ setInterval(()=>{
class Job {
constructor({ runtime, files, args, stdin, timeouts, memory_limits }) {
constructor({ runtime, files, args, stdin }) {
this.uuid = uuidv4();
this.runtime = runtime;
this.files = files.map((file, i) => ({
@ -40,8 +40,6 @@ class Job {
this.args = args;
this.stdin = stdin;
this.timeouts = timeouts;
this.memory_limits = memory_limits;
this.uid = config.runner_uid_min + uid;
this.gid = config.runner_gid_min + gid;
@ -102,9 +100,9 @@ class Job {
const prlimit = [
'prlimit',
'--nproc=' + config.max_process_count,
'--nofile=' + config.max_open_files,
'--fsize=' + config.max_file_size,
'--nproc=' + this.runtime.max_process_count ,
'--nofile=' + this.runtime.max_open_files ,
'--fsize=' + this.runtime.max_file_size ,
];
if (memory_limit >= 0) {
@ -143,8 +141,6 @@ class Job {
})
}
const kill_timeout = set_timeout(
async _ => {
logger.info(`Timeout exceeded timeout=${timeout} uuid=${this.uuid}`)
@ -156,7 +152,7 @@ class Job {
proc.stderr.on('data', async data => {
if(eventBus !== null) {
eventBus.emit("stderr", data);
} else if (stderr.length > config.output_max_size) {
} else if (stderr.length > this.runtime.output_max_size) {
logger.info(`stderr length exceeded uuid=${this.uuid}`)
process.kill(proc.pid, 'SIGKILL')
} else {
@ -168,7 +164,7 @@ class Job {
proc.stdout.on('data', async data => {
if(eventBus !== null){
eventBus.emit("stdout", data);
} else if (stdout.length > config.output_max_size) {
} else if (stdout.length > this.runtime.output_max_size) {
logger.info(`stdout length exceeded uuid=${this.uuid}`)
process.kill(proc.pid, 'SIGKILL')
} else {
@ -223,8 +219,8 @@ class Job {
compile = await this.safe_call(
path.join(this.runtime.pkgdir, 'compile'),
this.files.map(x => x.name),
this.timeouts.compile,
this.memory_limits.compile
this.runtime.timeouts.compile,
this.runtime.memory_limits.compile
);
}
@ -233,8 +229,8 @@ class Job {
const run = await this.safe_call(
path.join(this.runtime.pkgdir, 'run'),
[this.files[0].name, ...this.args],
this.timeouts.run,
this.memory_limits.run
this.runtime.timeouts.run,
this.runtime.memory_limits.run
);
this.state = job_states.EXECUTED;
@ -266,8 +262,8 @@ class Job {
const {error, code, signal} = await this.safe_call(
path.join(this.runtime.pkgdir, 'compile'),
this.files.map(x => x.name),
this.timeouts.compile,
this.memory_limits.compile,
this.runtime.timeouts.compile,
this.runtime.memory_limits.compile,
eventBus
)
@ -279,8 +275,8 @@ class Job {
const {error, code, signal} = await this.safe_call(
path.join(this.runtime.pkgdir, 'run'),
[this.files[0].name, ...this.args],
this.timeouts.run,
this.memory_limits.run,
this.runtime.timeouts.run,
this.runtime.memory_limits.run,
eventBus
);
@ -309,7 +305,6 @@ class Job {
const uid_line = proc_lines.find(line=>line.starts_with("Uid:"))
const [_, ruid, euid, suid, fuid] = uid_line.split(/\s+/);
if(ruid == this.uid || euid == this.uid)
return parse_int(proc_id)

48
api/src/runtime.js
View File

@ -8,12 +8,54 @@ const path = require('path');
const runtimes = [];
class Runtime {
constructor({ language, version, aliases, pkgdir, runtime }) {
constructor({
language, version, aliases, pkgdir, runtime, timeouts, memory_limits, max_process_count,
max_open_files, max_file_size, output_max_size
}) {
this.language = language;
this.version = version;
this.aliases = aliases || [];
this.pkgdir = pkgdir;
this.runtime = runtime;
this.timeouts = timeouts;
this.memory_limits = memory_limits;
this.max_process_count = max_process_count;
this.max_open_files = max_open_files;
this.max_file_size = max_file_size;
this.output_max_size = output_max_size;
}
static compute_single_limit(language_name, limit_name, language_limit_overrides) {
return (
config.limit_overrides[language_name] && config.limit_overrides[language_name][limit_name]
|| language_limit_overrides && language_limit_overrides[limit_name]
|| config[limit_name]
);
}
static compute_all_limits(language_name, language_limit_overrides) {
return {
timeouts: {
compile:
this.compute_single_limit(language_name, 'compile_timeout', language_limit_overrides),
run:
this.compute_single_limit(language_name, 'run_timeout', language_limit_overrides)
},
memory_limits: {
compile:
this.compute_single_limit(language_name, 'compile_memory_limit', language_limit_overrides),
run:
this.compute_single_limit(language_name, 'run_memory_limit', language_limit_overrides)
},
max_process_count:
this.compute_single_limit(language_name, 'max_process_count', language_limit_overrides),
max_open_files:
this.compute_single_limit(language_name, 'max_open_files', language_limit_overrides),
max_file_size:
this.compute_single_limit(language_name, 'max_file_size', language_limit_overrides),
output_max_size:
this.compute_single_limit(language_name, 'output_max_size', language_limit_overrides),
}
}
static load_package(package_dir) {
@ -21,7 +63,7 @@ class Runtime {
fss.read_file_sync(path.join(package_dir, 'pkg-info.json'))
);
let { language, version, build_platform, aliases, provides } = info;
let { language, version, build_platform, aliases, provides, limit_overrides } = info;
version = semver.parse(version);
if (build_platform !== globals.platform) {
@ -41,6 +83,7 @@ class Runtime {
version,
pkgdir: package_dir,
runtime: language,
...Runtime.compute_all_limits(lang.language, lang.limit_overrides)
})
);
});
@ -51,6 +94,7 @@ class Runtime {
version,
aliases,
pkgdir: package_dir,
...Runtime.compute_all_limits(language, limit_overrides)
})
);
}

0
packages/dotnet/5.0.201/build.sh vendored Normal file → Executable file
View File