Merge pull request #348 from Brikaa/overrides

Add per-language limit overrides

api/src/api/v2.js

@@ -3,7 +3,6 @@ const router = express.Router();
 
 const events = require('events');
 
-const config = require('../config');
 const runtime = require('../runtime');
 const { Job } = require('../job');
 const package = require('../package');
@@ -13,7 +12,7 @@ const SIGNALS = ["SIGABRT","SIGALRM","SIGBUS","SIGCHLD","SIGCLD","SIGCONT","SIGE
 // ref: https://man7.org/linux/man-pages/man7/signal.7.html
 
 function get_job(body){
-    const {
+    let {
         language,
         version,
         args,
@@ -31,19 +30,16 @@ function get_job(body){
                 message: 'language is required as a string',
             });
         }
-
         if (!version || typeof version !== 'string') {
             return reject({
                 message: 'version is required as a string',
             });
         }
-
         if (!files || !Array.isArray(files)) {
             return reject({
                 message: 'files is required as an array',
             });
         }
-
         for (const [i, file] of files.entries()) {
             if (typeof file.content !== 'string') {
                 return reject({
@@ -52,73 +48,64 @@ function get_job(body){
             }
         }
 
-        if (compile_memory_limit) {
-            if (typeof compile_memory_limit !== 'number') {
-                return reject({
-                    message: 'if specified, compile_memory_limit must be a number',
-                });
-            }
-
-            if (
-                config.compile_memory_limit >= 0 &&
-                (compile_memory_limit > config.compile_memory_limit ||
-                    compile_memory_limit < 0)
-            ) {
-                return reject({
-                    message:
-                        'compile_memory_limit cannot exceed the configured limit of ' +
-                        config.compile_memory_limit,
-                });
-            }
-        }
-
-        if (run_memory_limit) {
-            if (typeof run_memory_limit !== 'number') {
-                return reject({
-                    message: 'if specified, run_memory_limit must be a number',
-                });
-            }
-
-            if (
-                config.run_memory_limit >= 0 &&
-                (run_memory_limit > config.run_memory_limit || run_memory_limit < 0)
-            ) {
-                return reject({
-                    message:
-                        'run_memory_limit cannot exceed the configured limit of ' +
-                        config.run_memory_limit,
-                });
-            }
-        }
-
         const rt = runtime.get_latest_runtime_matching_language_version(
             language,
             version
         );
-
         if (rt === undefined) {
             return reject({
                 message: `${language}-${version} runtime is unknown`,
             });
         }
 
+        for (let constraint of ['memory_limit', 'timeout']) {
+            for (let type of ['compile', 'run']) {
+                let constraint_name = `${type}_${constraint}`;
+                let constraint_value = body[constraint_name];
+                let configured_limit = rt[`${constraint}s`][type];
+                if (!constraint_value) {
+                    continue;
+                }
+                if (typeof constraint_value !== 'number') {
+                    return reject({
+                        message: `If specified, ${constraint_name} must be a number`
+                    });
+                }
+                if (configured_limit <= 0) {
+                    continue;
+                }
+                if (constraint_value > configured_limit) {
+                    return reject({
+                        message: `${constraint_name} cannot exceed the configured limit of ${configured_limit}`
+                    });
+                }
+                if (constraint_value < 0) {
+                    return reject({
+                        message: `${constraint_name} must be non-negative`
+                    });
+                }
+            }
+        }
+
+        compile_timeout = compile_timeout || rt.timeouts.compile;
+        run_timeout = run_timeout || rt.timeouts.run;
+        compile_memory_limit = compile_memory_limit || rt.memory_limits.compile;
+        run_timeout = run_timeout || rt.timeouts.run;
         resolve(new Job({
             runtime: rt,
-            alias: language,
             args: args || [],
             stdin: stdin || "",
             files,
             timeouts: {
-                run: run_timeout || 3000,
+                run: run_timeout,
-                compile: compile_timeout || 10000,
+                compile: compile_timeout,
             },
             memory_limits: {
-                run: run_memory_limit || config.run_memory_limit,
+                run: run_memory_limit,
-                compile: compile_memory_limit || config.compile_memory_limit,
+                compile: compile_memory_limit,
             }
         }));
-    })
+    });
-
 }
 
 router.use((req, res, next) => {

api/src/config.js

@@ -2,6 +2,48 @@ const fss = require('fs');
 const Logger = require('logplease');
 const logger = Logger.create('config');
 
+function parse_overrides(overrides) {
+    try {
+        return JSON.parse(overrides);
+    }
+    catch (e) {
+        return null;
+    }
+}
+
+function validate_overrides(overrides, options) {
+    for (let language in overrides) {
+        for (let key in overrides[language]) {
+            if (
+                ![
+                    'max_process_count', 'max_open_files', 'max_file_size',
+                    'compile_memory_limit', 'run_memory_limit', 'compile_timeout',
+                    'run_timeout', 'output_max_size'
+                ].includes(key)
+            ) {
+                logger.error(`Invalid overridden option: ${key}`);
+                return false;
+            }
+            let option = options.find((o) => o.key === key);
+            let parser = option.parser;
+            let raw = overrides[language][key];
+            let value = parser(raw);
+            let validators = option.validators;
+            for (let validator of validators) {
+                let response = validator(value, raw);
+                if (response !== true) {
+                    logger.error(`Failed to validate overridden option: ${key}`, response);
+                    return false;
+                }
+            }
+            overrides[language][key] = value;
+        }
+        // Modifies the reference
+        options[options.index_of(options.find((o) => o.key === 'limit_overrides'))] = overrides;
+    }
+    return true;
+}
+
 const options = [
     {
         key: 'log_level',
@@ -91,6 +133,22 @@ const options = [
         parser: parse_int,
         validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
     },
+    {
+        key: 'compile_timeout',
+        desc:
+            'Max time allowed for compile stage in milliseconds',
+        default: 10000, // 10 seconds
+        parser: parse_int,
+        validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
+    },
+    {
+        key: 'run_timeout',
+        desc:
+            'Max time allowed for run stage in milliseconds',
+        default: 3000, // 3 seconds
+        parser: parse_int,
+        validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
+    },
     {
         key: 'compile_memory_limit',
         desc:
@@ -120,6 +178,18 @@ const options = [
         default: 64,
         parser: parse_int,
         validators: [(x) => x > 0 || `${x} cannot be negative`]
+    },
+    {
+        key: 'limit_overrides',
+        desc: 'Per-language exceptions in JSON format for each of:\
+        max_process_count, max_open_files, max_file_size, compile_memory_limit,\
+        run_memory_limit, compile_timeout, run_timeout, output_max_size',
+        default: {},
+        parser: parse_overrides,
+        validators: [
+            (x) => !!x || `Invalid JSON format for the overrides\n${x}`,
+            (overrides, _, options) => validate_overrides(overrides, options) || `Failed to validate the overrides`
+        ]
     }
 ];
 
@@ -138,12 +208,12 @@ options.forEach(option => {
 
     const parsed_val = parser(env_val);
 
-    const value = env_val || option.default;
+    const value = parsed_val || option.default;
 
     option.validators.for_each(validator => {
         let response = null;
-        if (env_val) response = validator(parsed_val, env_val);
+        if (env_val) response = validator(parsed_val, env_val, options);
-        else response = validator(value, value);
+        else response = validator(value, value, options);
 
         if (response !== true) {
             errored = true;

api/src/job.js

@@ -30,7 +30,7 @@ setInterval(()=>{
 
 
 class Job {
-    constructor({ runtime, files, args, stdin, timeouts, memory_limits }) {
+    constructor({ runtime, files, args, stdin }) {
         this.uuid = uuidv4();
         this.runtime = runtime;
         this.files = files.map((file, i) => ({
@@ -40,8 +40,6 @@ class Job {
 
         this.args = args;
         this.stdin = stdin;
-        this.timeouts = timeouts;
-        this.memory_limits = memory_limits;
 
         this.uid = config.runner_uid_min + uid;
         this.gid = config.runner_gid_min + gid;
@@ -102,9 +100,9 @@ class Job {
 
             const prlimit = [
                 'prlimit',
-                '--nproc=' + config.max_process_count,
+                '--nproc=' + this.runtime.max_process_count ,
-                '--nofile=' + config.max_open_files,
+                '--nofile=' + this.runtime.max_open_files ,
-                '--fsize=' + config.max_file_size,
+                '--fsize=' + this.runtime.max_file_size ,
             ];
 
             if (memory_limit >= 0) {
@@ -143,8 +141,6 @@ class Job {
                 })
             }
 
-            
-
             const kill_timeout = set_timeout(
                 async _ => {
                     logger.info(`Timeout exceeded timeout=${timeout} uuid=${this.uuid}`)
@@ -156,7 +152,7 @@ class Job {
             proc.stderr.on('data', async data => {
                 if(eventBus !== null) {
                     eventBus.emit("stderr", data);
-                } else if (stderr.length > config.output_max_size) {
+                } else if (stderr.length > this.runtime.output_max_size) {
                     logger.info(`stderr length exceeded uuid=${this.uuid}`)
                     process.kill(proc.pid, 'SIGKILL')
                 } else {
@@ -168,7 +164,7 @@ class Job {
             proc.stdout.on('data', async data => {
                 if(eventBus !== null){
                     eventBus.emit("stdout", data);
-                } else if (stdout.length > config.output_max_size) {
+                } else if (stdout.length > this.runtime.output_max_size) {
                     logger.info(`stdout length exceeded uuid=${this.uuid}`)
                     process.kill(proc.pid, 'SIGKILL')
                 } else {
@@ -223,8 +219,8 @@ class Job {
             compile = await this.safe_call(
                 path.join(this.runtime.pkgdir, 'compile'),
                 this.files.map(x => x.name),
-                this.timeouts.compile,
+                this.runtime.timeouts.compile,
-                this.memory_limits.compile
+                this.runtime.memory_limits.compile
             );
         }
 
@@ -233,8 +229,8 @@ class Job {
         const run = await this.safe_call(
             path.join(this.runtime.pkgdir, 'run'),
             [this.files[0].name, ...this.args],
-            this.timeouts.run,
+            this.runtime.timeouts.run,
-            this.memory_limits.run
+            this.runtime.memory_limits.run
         );
 
         this.state = job_states.EXECUTED;
@@ -266,8 +262,8 @@ class Job {
             const {error, code, signal} = await this.safe_call(
                 path.join(this.runtime.pkgdir, 'compile'),
                 this.files.map(x => x.name),
-                this.timeouts.compile,
+                this.runtime.timeouts.compile,
-                this.memory_limits.compile,
+                this.runtime.memory_limits.compile,
                 eventBus
             )
 
@@ -279,8 +275,8 @@ class Job {
         const {error, code, signal} = await this.safe_call(
             path.join(this.runtime.pkgdir, 'run'),
             [this.files[0].name, ...this.args],
-            this.timeouts.run,
+            this.runtime.timeouts.run,
-            this.memory_limits.run,
+            this.runtime.memory_limits.run,
             eventBus
         );
 
@@ -309,7 +305,6 @@ class Job {
                     const uid_line = proc_lines.find(line=>line.starts_with("Uid:"))
                     const [_, ruid, euid, suid, fuid] = uid_line.split(/\s+/);
 
-                    
                     if(ruid == this.uid || euid == this.uid)
                         return parse_int(proc_id)
 

api/src/runtime.js

@@ -8,12 +8,54 @@ const path = require('path');
 const runtimes = [];
 
 class Runtime {
-    constructor({ language, version, aliases, pkgdir, runtime }) {
+    constructor({
+        language, version, aliases, pkgdir, runtime, timeouts, memory_limits, max_process_count,
+        max_open_files, max_file_size, output_max_size
+    }) {
         this.language = language;
         this.version = version;
         this.aliases = aliases || [];
         this.pkgdir = pkgdir;
         this.runtime = runtime;
+        this.timeouts = timeouts;
+        this.memory_limits = memory_limits;
+        this.max_process_count = max_process_count;
+        this.max_open_files = max_open_files;
+        this.max_file_size = max_file_size;
+        this.output_max_size = output_max_size;
+    }
+
+    static compute_single_limit(language_name, limit_name, language_limit_overrides) {
+        return (
+            config.limit_overrides[language_name] && config.limit_overrides[language_name][limit_name]
+            || language_limit_overrides && language_limit_overrides[limit_name]
+            || config[limit_name]
+        );
+    }
+
+    static compute_all_limits(language_name, language_limit_overrides) {
+        return {
+            timeouts: {
+                compile:
+                    this.compute_single_limit(language_name, 'compile_timeout', language_limit_overrides),
+                run:
+                    this.compute_single_limit(language_name, 'run_timeout', language_limit_overrides)
+            },
+            memory_limits: {
+                compile:
+                    this.compute_single_limit(language_name, 'compile_memory_limit', language_limit_overrides),
+                run:
+                    this.compute_single_limit(language_name, 'run_memory_limit', language_limit_overrides)
+            },
+            max_process_count:
+                this.compute_single_limit(language_name, 'max_process_count', language_limit_overrides),
+            max_open_files:
+                this.compute_single_limit(language_name, 'max_open_files', language_limit_overrides),
+            max_file_size:
+                this.compute_single_limit(language_name, 'max_file_size', language_limit_overrides),
+            output_max_size:
+                this.compute_single_limit(language_name, 'output_max_size', language_limit_overrides),
+        }
     }
 
     static load_package(package_dir) {
@@ -21,7 +63,7 @@ class Runtime {
             fss.read_file_sync(path.join(package_dir, 'pkg-info.json'))
         );
 
-        let { language, version, build_platform, aliases, provides } = info;
+        let { language, version, build_platform, aliases, provides, limit_overrides } = info;
         version = semver.parse(version);
 
         if (build_platform !== globals.platform) {
@@ -41,6 +83,7 @@ class Runtime {
                         version,
                         pkgdir: package_dir,
                         runtime: language,
+                        ...Runtime.compute_all_limits(lang.language, lang.limit_overrides)
                     })
                 );
             });
@@ -51,6 +94,7 @@ class Runtime {
                     version,
                     aliases,
                     pkgdir: package_dir,
+                    ...Runtime.compute_all_limits(language, limit_overrides)
                 })
             );
         }