Merge branch 'nix-packages' of https://github.com/engineer-man/piston into add-nix-packages

.gitignore

@@ -2,3 +2,4 @@ data/
 .piston_env
 node_modules
 result
+.vscode/

Dockerfile.withset

@@ -9,4 +9,4 @@ COPY flake.nix flake.lock /piston/packages/
 ARG RUNTIMESET=all
 ENV PISTON_RUNTIME_SET=$RUNTIMESET
 
-RUN piston-install
+RUN piston-install

api/src/api/v2.js

@@ -5,7 +5,6 @@ const events = require('events');
 
 const runtime = require('../runtime');
 const { Job } = require('../job');
-const logger = require('logplease').create('api/v3');
 
 const SIGNALS = [
     'SIGABRT',
@@ -81,49 +80,9 @@ function get_job(body) {
             }
         }
 
-        if (compile_memory_limit) {
+        const rt = runtime.find(rt =>
-            if (typeof compile_memory_limit !== 'number') {
+            [...rt.aliases, rt.language].includes(rt.language)
-                return reject({
+        );
-                    message: 'if specified, compile_memory_limit must be a number',
-                });
-            }
-
-            if (
-                config.compile_memory_limit >= 0 &&
-                (compile_memory_limit > config.compile_memory_limit ||
-                    compile_memory_limit < 0)
-            ) {
-                return reject({
-                    message:
-                        'compile_memory_limit cannot exceed the configured limit of ' +
-                        config.compile_memory_limit,
-                });
-            }
-        }
-
-        if (run_memory_limit) {
-            if (typeof run_memory_limit !== 'number') {
-                return reject({
-                    message: 'if specified, run_memory_limit must be a number',
-                });
-            }
-
-            if (
-                config.run_memory_limit >= 0 &&
-                (run_memory_limit > config.run_memory_limit || run_memory_limit < 0)
-            ) {
-                return reject({
-                    message:
-                        'run_memory_limit cannot exceed the configured limit of ' +
-                        config.run_memory_limit,
-                });
-            }
-        }
-
-        const rt = runtime.find(rt => [
-            ...rt.aliases,
-            rt.language
-        ].includes(rt.language))
 
         if (rt === undefined) {
             return reject({

api/src/api/v3.js

@@ -3,15 +3,52 @@ const router = express.Router();
 
 const events = require('events');
 
-const config = require('../config');
 const runtime = require('../runtime');
 const { Job } = require('../job');
-const logger = require('logplease').create('api/v3');
 
-const SIGNALS = ["SIGABRT","SIGALRM","SIGBUS","SIGCHLD","SIGCLD","SIGCONT","SIGEMT","SIGFPE","SIGHUP","SIGILL","SIGINFO","SIGINT","SIGIO","SIGIOT","SIGKILL","SIGLOST","SIGPIPE","SIGPOLL","SIGPROF","SIGPWR","SIGQUIT","SIGSEGV","SIGSTKFLT","SIGSTOP","SIGTSTP","SIGSYS","SIGTERM","SIGTRAP","SIGTTIN","SIGTTOU","SIGUNUSED","SIGURG","SIGUSR1","SIGUSR2","SIGVTALRM","SIGXCPU","SIGXFSZ","SIGWINCH"]
+const SIGNALS = [
+    'SIGABRT',
+    'SIGALRM',
+    'SIGBUS',
+    'SIGCHLD',
+    'SIGCLD',
+    'SIGCONT',
+    'SIGEMT',
+    'SIGFPE',
+    'SIGHUP',
+    'SIGILL',
+    'SIGINFO',
+    'SIGINT',
+    'SIGIO',
+    'SIGIOT',
+    'SIGKILL',
+    'SIGLOST',
+    'SIGPIPE',
+    'SIGPOLL',
+    'SIGPROF',
+    'SIGPWR',
+    'SIGQUIT',
+    'SIGSEGV',
+    'SIGSTKFLT',
+    'SIGSTOP',
+    'SIGTSTP',
+    'SIGSYS',
+    'SIGTERM',
+    'SIGTRAP',
+    'SIGTTIN',
+    'SIGTTOU',
+    'SIGUNUSED',
+    'SIGURG',
+    'SIGUSR1',
+    'SIGUSR2',
+    'SIGVTALRM',
+    'SIGXCPU',
+    'SIGXFSZ',
+    'SIGWINCH',
+];
 // ref: https://man7.org/linux/man-pages/man7/signal.7.html
 
-function get_job(body){
+function get_job(body) {
     const {
         runtime_id,
         args,
@@ -20,93 +57,96 @@ function get_job(body){
         compile_memory_limit,
         run_memory_limit,
         run_timeout,
-        compile_timeout
+        compile_timeout,
     } = body;
 
     return new Promise((resolve, reject) => {
         if (typeof runtime_id !== 'number') {
             return reject({
-                message: 'runtime_id is required as a number'
+                message: 'runtime_id is required as a number',
             });
         }
 
-        if (!Array.isArray(files)) {
+        if (!files || !Array.isArray(files)) {
             return reject({
                 message: 'files is required as an array',
             });
         }
 
-        for (const [i, file] of files.entries()) {
-            if (typeof file.content !== 'string') {
-                return reject({
-                    message: `files[${i}].content is required as a string`,
-                });
-            }
-        }
-
-        if (compile_memory_limit) {
-            if (typeof compile_memory_limit !== 'number') {
-                return reject({
-                    message: 'if specified, compile_memory_limit must be a number',
-                });
-            }
-
-            if (
-                config.compile_memory_limit >= 0 &&
-                (compile_memory_limit > config.compile_memory_limit ||
-                    compile_memory_limit < 0)
-            ) {
-                return reject({
-                    message:
-                        'compile_memory_limit cannot exceed the configured limit of ' +
-                        config.compile_memory_limit,
-                });
-            }
-        }
-
-        if (run_memory_limit) {
-            if (typeof run_memory_limit !== 'number') {
-                return reject({
-                    message: 'if specified, run_memory_limit must be a number',
-                });
-            }
-
-            if (
-                config.run_memory_limit >= 0 &&
-                (run_memory_limit > config.run_memory_limit || run_memory_limit < 0)
-            ) {
-                return reject({
-                    message:
-                        'run_memory_limit cannot exceed the configured limit of ' +
-                        config.run_memory_limit,
-                });
-            }
-        }
         const rt = runtime[runtime_id];
 
-
         if (rt === undefined) {
             return reject({
                 message: `Runtime #${runtime_id} is unknown`,
             });
         }
 
-        resolve(new Job({
+        if (
-            runtime: rt,
+            rt.language !== 'file' &&
-            args: args || [],
+            !files.some(file => !file.encoding || file.encoding === 'utf8')
-            stdin: stdin || "",
+        ) {
-            files,
+            return reject({
-            timeouts: {
+                message: 'files must include at least one utf8 encoded file',
-                run: run_timeout || 3000,
+            });
-                compile: compile_timeout || 10000,
+        }
-            },
-            memory_limits: {
-                run: run_memory_limit || config.run_memory_limit,
-                compile: compile_memory_limit || config.compile_memory_limit,
-            }
-        }));
-    })
 
+        if (files.some(file => typeof file.content !== 'string')) {
+            return reject({
+                message: 'file.content is required as a string',
+            });
+        }
+
+        for (const constraint of ['memory_limit', 'timeout']) {
+            for (const type of ['compile', 'run']) {
+                const constraint_name = `${type}_${constraint}`;
+                const constraint_value = body[constraint_name];
+                const configured_limit = rt[`${constraint}s`][type];
+                if (!constraint_value) {
+                    continue;
+                }
+                if (typeof constraint_value !== 'number') {
+                    return reject({
+                        message: `If specified, ${constraint_name} must be a number`,
+                    });
+                }
+                if (configured_limit <= 0) {
+                    continue;
+                }
+                if (constraint_value > configured_limit) {
+                    return reject({
+                        message: `${constraint_name} cannot exceed the configured limit of ${configured_limit}`,
+                    });
+                }
+                if (constraint_value < 0) {
+                    return reject({
+                        message: `${constraint_name} must be non-negative`,
+                    });
+                }
+            }
+        }
+
+        const job_compile_timeout = compile_timeout || rt.timeouts.compile;
+        const job_run_timeout = run_timeout || rt.timeouts.run;
+        const job_compile_memory_limit =
+            compile_memory_limit || rt.memory_limits.compile;
+        const job_run_memory_limit = run_memory_limit || rt.memory_limits.run;
+        resolve(
+            new Job({
+                runtime: rt,
+                args: args || [],
+                stdin: stdin || '',
+                files,
+                timeouts: {
+                    run: job_run_timeout,
+                    compile: job_compile_timeout,
+                },
+                memory_limits: {
+                    run: job_run_memory_limit,
+                    compile: job_compile_memory_limit,
+                },
+            })
+        );
+    });
 }
 
 router.use((req, res, next) => {
@@ -124,89 +164,106 @@ router.use((req, res, next) => {
 });
 
 router.ws('/connect', async (ws, req) => {
-
     let job = null;
     let eventBus = new events.EventEmitter();
 
-    eventBus.on("stdout", (data) => ws.send(JSON.stringify({type: "data", stream: "stdout", data: data.toString()})))
+    eventBus.on('stdout', data =>
-    eventBus.on("stderr", (data) => ws.send(JSON.stringify({type: "data", stream: "stderr", data: data.toString()})))
+        ws.send(
-    eventBus.on("stage", (stage)=> ws.send(JSON.stringify({type: "stage", stage})))
+            JSON.stringify({
-    eventBus.on("exit", (stage, status) => ws.send(JSON.stringify({type: "exit", stage, ...status})))
+                type: 'data',
+                stream: 'stdout',
+                data: data.toString(),
+            })
+        )
+    );
+    eventBus.on('stderr', data =>
+        ws.send(
+            JSON.stringify({
+                type: 'data',
+                stream: 'stderr',
+                data: data.toString(),
+            })
+        )
+    );
+    eventBus.on('stage', stage =>
+        ws.send(JSON.stringify({ type: 'stage', stage }))
+    );
+    eventBus.on('exit', (stage, status) =>
+        ws.send(JSON.stringify({ type: 'exit', stage, ...status }))
+    );
 
-    ws.on("message", async (data) => {
+    ws.on('message', async data => {
-
+        try {
-        try{
             const msg = JSON.parse(data);
 
-            switch(msg.type){
+            switch (msg.type) {
-                case "init":
+                case 'init':
-                    if(job === null){
+                    if (job === null) {
                         job = await get_job(msg);
 
                         await job.prime();
 
-                        ws.send(JSON.stringify({
+                        ws.send(
-                            type: "runtime",
+                            JSON.stringify({
-                            language: job.runtime.language,
+                                type: 'runtime',
-                            version: job.runtime.version.raw
+                                language: job.runtime.language,
-                        }))
+                                version: job.runtime.version.raw,
+                            })
+                        );
 
                         await job.execute_interactive(eventBus);
 
-                        ws.close(4999, "Job Completed");
+                        ws.close(4999, 'Job Completed');
-
+                    } else {
-                    }else{
+                        ws.close(4000, 'Already Initialized');
-                        ws.close(4000, "Already Initialized");
                     }
                     break;
-            case "data":
+                case 'data':
-                if(job !== null){
+                    if (job !== null) {
-                    if(msg.stream === "stdin"){
+                        if (msg.stream === 'stdin') {
-                        eventBus.emit("stdin", msg.data)
+                            eventBus.emit('stdin', msg.data);
-                    }else{
+                        } else {
-                        ws.close(4004, "Can only write to stdin")
+                            ws.close(4004, 'Can only write to stdin');
+                        }
+                    } else {
+                        ws.close(4003, 'Not yet initialized');
                     }
-                }else{
+                    break;
-                    ws.close(4003, "Not yet initialized")
+                case 'signal':
-                }
+                    if (job !== null) {
-                break;
+                        if (SIGNALS.includes(msg.signal)) {
-            case "signal":
+                            eventBus.emit('signal', msg.signal);
-                if(job !== null){
+                        } else {
-                    if(SIGNALS.includes(msg.signal)){
+                            ws.close(4005, 'Invalid signal');
-                        eventBus.emit("signal", msg.signal)
+                        }
-                    }else{
+                    } else {
-                        ws.close(4005, "Invalid signal")
+                        ws.close(4003, 'Not yet initialized');
                     }
-                }else{
+                    break;
-                    ws.close(4003, "Not yet initialized")
-                }
-                break;
             }
-
+        } catch (error) {
-        }catch(error){
+            ws.send(JSON.stringify({ type: 'error', message: error.message }));
-            ws.send(JSON.stringify({type: "error", message: error.message}))
+            ws.close(4002, 'Notified Error');
-            ws.close(4002, "Notified Error")
             // ws.close message is limited to 123 characters, so we notify over WS then close.
         }
-    })
+    });
 
-    ws.on("close", async ()=>{
+    ws.on('close', async () => {
-        if(job !== null){
+        if (job !== null) {
-            await job.cleanup()
+            await job.cleanup();
         }
-    })
+    });
 
-    setTimeout(()=>{
+    setTimeout(() => {
         //Terminate the socket after 1 second, if not initialized.
-        if(job === null)
+        if (job === null) ws.close(4001, 'Initialization Timeout');
-            ws.close(4001, "Initialization Timeout");
+    }, 1000);
-    }, 1000)
+});
-})
 
 router.post('/execute', async (req, res) => {
-
+    try {
-    try{
         const job = await get_job(req.body);
+
         await job.prime();
 
         const result = await job.execute();
@@ -214,7 +271,7 @@ router.post('/execute', async (req, res) => {
         await job.cleanup();
 
         return res.status(200).send(result);
-    }catch(error){
+    } catch (error) {
         return res.status(400).json(error);
     }
 });
@@ -226,7 +283,7 @@ router.get('/runtimes', (req, res) => {
             version: rt.version.raw,
             aliases: rt.aliases,
             runtime: rt.runtime,
-            id: rt.id
+            id: rt.id,
         };
     });
 

api/src/config.js

@@ -2,16 +2,163 @@ const fss = require('fs');
 const Logger = require('logplease');
 const logger = Logger.create('config');
 
-function parse_overrides(overrides) {
+const options = {
-    try {
+    log_level: {
-        return JSON.parse(overrides);
+        desc: 'Level of data to log',
-    } catch (e) {
+        default: 'INFO',
-        return null;
+        validators: [
+            x =>
+                Object.values(Logger.LogLevels).includes(x) ||
+                `Log level ${x} does not exist`,
+        ],
+    },
+    bind_address: {
+        desc: 'Address to bind REST API on',
+        default: `0.0.0.0:${process.env['PORT'] || 2000}`,
+        validators: [],
+    },
+    data_directory: {
+        desc: 'Absolute path to store all piston related data at',
+        default: '/piston',
+        validators: [
+            x => fss.exists_sync(x) || `Directory ${x} does not exist`,
+        ],
+    },
+    runner_uid_min: {
+        desc: 'Minimum uid to use for runner',
+        default: 1001,
+        parser: parse_int,
+        validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
+    },
+    runner_uid_max: {
+        desc: 'Maximum uid to use for runner',
+        default: 1500,
+        parser: parse_int,
+        validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
+    },
+    runner_gid_min: {
+        desc: 'Minimum gid to use for runner',
+        default: 1001,
+        parser: parse_int,
+        validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
+    },
+    runner_gid_max: {
+        desc: 'Maximum gid to use for runner',
+        default: 1500,
+        parser: parse_int,
+        validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
+    },
+    disable_networking: {
+        desc: 'Set to true to disable networking',
+        default: true,
+        parser: x => x === 'true',
+        validators: [x => typeof x === 'boolean' || `${x} is not a boolean`],
+    },
+    output_max_size: {
+        desc: 'Max size of each stdio buffer',
+        default: 1024,
+        parser: parse_int,
+        validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
+    },
+    max_process_count: {
+        desc: 'Max number of processes per job',
+        default: 64,
+        parser: parse_int,
+        validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
+    },
+    max_open_files: {
+        desc: 'Max number of open files per job',
+        default: 2048,
+        parser: parse_int,
+        validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
+    },
+    max_file_size: {
+        desc: 'Max file size in bytes for a file',
+        default: 10000000, //10MB
+        parser: parse_int,
+        validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
+    },
+    compile_timeout: {
+        desc: 'Max time allowed for compile stage in milliseconds',
+        default: 10000, // 10 seconds
+        parser: parse_int,
+        validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
+    },
+    run_timeout: {
+        desc: 'Max time allowed for run stage in milliseconds',
+        default: 3000, // 3 seconds
+        parser: parse_int,
+        validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
+    },
+    compile_memory_limit: {
+        desc: 'Max memory usage for compile stage in bytes (set to -1 for no limit)',
+        default: -1, // no limit
+        parser: parse_int,
+        validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
+    },
+    run_memory_limit: {
+        desc: 'Max memory usage for run stage in bytes (set to -1 for no limit)',
+        default: -1, // no limit
+        parser: parse_int,
+        validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
+    },
+    flake_path: {
+        desc: 'Path to nix flake defining runtimes to install',
+        default: 'github:engineer-man/piston?directory=packages',
+        validators: [],
+    },
+    runtime_set: {
+        desc: 'Key on the flake specified by flake_path to access runtimes from',
+        default: 'all',
+        validators: [],
+    },
+    max_concurrent_jobs: {
+        desc: 'Maximum number of concurrent jobs to run at one time',
+        default: 64,
+        parser: parse_int,
+        validators: [x => x > 0 || `${x} cannot be negative`],
+    },
+    limit_overrides: {
+        desc: 'Per-language exceptions in JSON format for each of:\
+        max_process_count, max_open_files, max_file_size, compile_memory_limit,\
+        run_memory_limit, compile_timeout, run_timeout, output_max_size',
+        default: {},
+        parser: parse_overrides,
+        validators: [
+            x => !!x || `Failed to parse the overrides\n${x}`,
+            validate_overrides,
+        ],
+    },
+};
+
+Object.freeze(options);
+
+function apply_validators(validators, validator_parameters) {
+    for (const validator of validators) {
+        const validation_response = validator(...validator_parameters);
+        if (validation_response !== true) {
+            return validation_response;
+        }
     }
+    return true;
 }
 
-function validate_overrides(overrides, options) {
+function parse_overrides(overrides_string) {
+    function get_parsed_json_or_null(overrides) {
+        try {
+            return JSON.parse(overrides);
+        } catch (e) {
+            return null;
+        }
+    }
+
+    const overrides = get_parsed_json_or_null(overrides_string);
+    if (overrides === null) {
+        return null;
+    }
+    const parsed_overrides = {};
     for (const language in overrides) {
+        parsed_overrides[language] = {};
         for (const key in overrides[language]) {
             if (
                 ![
@@ -25,223 +172,62 @@ function validate_overrides(overrides, options) {
                     'output_max_size',
                 ].includes(key)
             ) {
-                logger.error(`Invalid overridden option: ${key}`);
+                return null;
-                return false;
             }
-            const option = options.find(o => o.key === key);
+            // Find the option for the override
+            const option = options[key];
             const parser = option.parser;
-            const raw = overrides[language][key];
+            const raw_value = overrides[language][key];
-            const value = parser(raw);
+            const parsed_value = parser(raw_value);
-            const validators = option.validators;
+            parsed_overrides[language][key] = parsed_value;
-            for (const validator of validators) {
+        }
-                const response = validator(value, raw);
+    }
-                if (response !== true) {
+    return parsed_overrides;
-                    logger.error(
+}
-                        `Failed to validate overridden option: ${key}`,
+
-                        response
+function validate_overrides(overrides) {
-                    );
+    for (const language in overrides) {
-                    return false;
+        for (const key in overrides[language]) {
-                }
+            const value = overrides[language][key];
-            }
+            const option = options[key];
-            overrides[language][key] = value;
+            const validators = option.validators;
+            const validation_response = apply_validators(validators, [
+                value,
+                value,
+            ]);
+            if (validation_response !== true) {
+                return `In overridden option ${key} for ${language}, ${validation_response}`;
+            }
         }
-        // Modifies the reference
-        options[
-            options.index_of(options.find(o => o.key === 'limit_overrides'))
-        ] = overrides;
     }
     return true;
 }
 
-const options = [
-    {
-        key: 'log_level',
-        desc: 'Level of data to log',
-        default: 'INFO',
-        options: Object.values(Logger.LogLevels),
-        validators: [
-            x =>
-                Object.values(Logger.LogLevels).includes(x) ||
-                `Log level ${x} does not exist`,
-        ],
-    },
-    {
-        key: 'bind_address',
-        desc: 'Address to bind REST API on',
-        default: `0.0.0.0:${process.env["PORT"] || 2000}`,
-        validators: [],
-    },
-    {
-        key: 'data_directory',
-        desc: 'Absolute path to store all piston related data at',
-        default: '/piston',
-        validators: [
-            x => fss.exists_sync(x) || `Directory ${x} does not exist`,
-        ],
-    },
-    {
-        key: 'runner_uid_min',
-        desc: 'Minimum uid to use for runner',
-        default: 1001,
-        parser: parse_int,
-        validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
-    },
-    {
-        key: 'runner_uid_max',
-        desc: 'Maximum uid to use for runner',
-        default: 1500,
-        parser: parse_int,
-        validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
-    },
-    {
-        key: 'runner_gid_min',
-        desc: 'Minimum gid to use for runner',
-        default: 1001,
-        parser: parse_int,
-        validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
-    },
-    {
-        key: 'runner_gid_max',
-        desc: 'Maximum gid to use for runner',
-        default: 1500,
-        parser: parse_int,
-        validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
-    },
-    {
-        key: 'disable_networking',
-        desc: 'Set to true to disable networking',
-        default: true,
-        parser: x => x === 'true',
-        validators: [x => typeof x === 'boolean' || `${x} is not a boolean`],
-    },
-    {
-        key: 'output_max_size',
-        desc: 'Max size of each stdio buffer',
-        default: 1024,
-        parser: parse_int,
-        validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
-    },
-    {
-        key: 'max_process_count',
-        desc: 'Max number of processes per job',
-        default: 64,
-        parser: parse_int,
-        validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
-    },
-    {
-        key: 'max_open_files',
-        desc: 'Max number of open files per job',
-        default: 2048,
-        parser: parse_int,
-        validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
-    },
-    {
-        key: 'max_file_size',
-        desc: 'Max file size in bytes for a file',
-        default: 10000000, //10MB
-        parser: parse_int,
-        validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
-    },
-    {
-        key: 'compile_timeout',
-        desc: 'Max time allowed for compile stage in milliseconds',
-        default: 10000, // 10 seconds
-        parser: parse_int,
-        validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
-    },
-    {
-        key: 'run_timeout',
-        desc: 'Max time allowed for run stage in milliseconds',
-        default: 3000, // 3 seconds
-        parser: parse_int,
-        validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
-    },
-    {
-        key: 'compile_memory_limit',
-        desc: 'Max memory usage for compile stage in bytes (set to -1 for no limit)',
-        default: -1, // no limit
-        parser: parse_int,
-        validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
-    },
-    {
-        key: 'run_memory_limit',
-        desc: 'Max memory usage for run stage in bytes (set to -1 for no limit)',
-        default: -1, // no limit
-        parser: parse_int,
-        validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
-    },
-    {
-        key: 'flake_path',
-        desc: 'Path to nix flake defining runtimes to install',
-        default: 'github:engineer-man/piston?directory=packages',
-        validators: [],
-    },
-    {
-        key: 'runtime_set',
-        desc: 'Key on the flake specified by flake_path to access runtimes from',
-        default: 'all',
-        validators: []
-    },
-    {
-        key: 'max_concurrent_jobs',
-        desc: 'Maximum number of concurrent jobs to run at one time',
-        default: 64,
-        parser: parse_int,
-        validators: [x => x > 0 || `${x} cannot be negative`],
-    },
-    {
-        key: 'limit_overrides',
-        desc: 'Per-language exceptions in JSON format for each of:\
-        max_process_count, max_open_files, max_file_size, compile_memory_limit,\
-        run_memory_limit, compile_timeout, run_timeout, output_max_size',
-        default: {},
-        parser: parse_overrides,
-        validators: [
-            x => !!x || `Invalid JSON format for the overrides\n${x}`,
-            (overrides, _, options) =>
-                validate_overrides(overrides, options) ||
-                `Failed to validate the overrides`,
-        ],
-    },
-];
-
 logger.info(`Loading Configuration from environment`);
 
-let errored = false;
-
 let config = {};
 
-options.forEach(option => {
+for (const option_name in options) {
-    const env_key = 'PISTON_' + option.key.to_upper_case();
+    const env_key = 'PISTON_' + option_name.to_upper_case();
-
+    const option = options[option_name];
     const parser = option.parser || (x => x);
-
     const env_val = process.env[env_key];
-
     const parsed_val = parser(env_val);
-
     const value = env_val === undefined ? option.default : parsed_val;
-
+    const validator_parameters =
-    option.validators.for_each(validator => {
+        env_val === undefined ? [value, value] : [parsed_val, env_val];
-        let response = null;
+    const validation_response = apply_validators(
-        if (env_val) response = validator(parsed_val, env_val, options);
+        option.validators,
-        else response = validator(value, value, options);
+        validator_parameters
-
+    );
-        if (response !== true) {
+    if (validation_response !== true) {
-            errored = true;
+        logger.error(
-            logger.error(
+            `Config option ${option_name} failed validation:`,
-                `Config option ${option.key} failed validation:`,
+            validation_response
-                response
+        );
-            );
+        process.exit(1);
-            return;
+    }
-        }
+    config[option_name] = value;
-    });
-
-    config[option.key] = value;
-});
-
-if (errored) {
-    process.exit(1);
 }
 
 logger.info('Configuration successfully loaded');

api/src/job.js

@@ -281,7 +281,7 @@ class Job {
         if (this.runtime.compiled) {
             eventBus.emit('stage', 'compile');
             const { error, code, signal } = await this.safe_call(
-                path.join(this.runtime.pkgdir, 'compile'),
+                this.runtime.compile,
                 code_files.map(x => x.name),
                 this.timeouts.compile,
                 this.memory_limits.compile,
@@ -294,7 +294,7 @@ class Job {
         this.logger.debug('Running');
         eventBus.emit('stage', 'run');
         const { error, code, signal } = await this.safe_call(
-            path.join(this.runtime.pkgdir, 'run'),
+            this.runtime.run,
             [code_files[0].name, ...this.args],
             this.timeouts.run,
             this.memory_limits.run,

api/src/runtime.js

@@ -1,9 +1,6 @@
 const logger = require('logplease').create('runtime');
 const cp = require('child_process');
 const config = require('./config');
-const globals = require('./globals');
-const fss = require('fs');
-const path = require('path');
 
 const runtimes = [];
 

packages/MATL/22.5.0/build.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+# build octave as dependency
+source ../../octave/6.2.0/build.sh
+
+# curl MATL 22.5.0
+curl -L "https://github.com/lmendo/MATL/archive/refs/tags/22.5.0.tar.gz" -o MATL.tar.xz
+tar xf MATL.tar.xz --strip-components=1
+rm MATL.tar.xz

packages/MATL/22.5.0/environment

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+# Path to MATL binary
+export PATH=$PWD/bin:$PATH
+export MATL_PATH=$PWD

packages/MATL/22.5.0/metadata.json

@@ -0,0 +1,5 @@
+{
+    "language": "matl",
+    "version": "22.5.0",
+    "aliases": []
+}

packages/MATL/22.5.0/run

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+# get file as first argument
+file="$1"
+
+# remove the file from $@
+shift
+
+# use the rest of the arguments as stdin
+stdin=`printf "%s\n" "$@"`
+
+# pass stdin into octave which will run MATL
+echo "$stdin" | octave -W -p "$MATL_PATH" --eval "matl -of '$file'"

packages/MATL/22.5.0/test.matl

@@ -0,0 +1 @@
+'OK'

piston

@@ -58,6 +58,10 @@ case "$SUBCOMMAND" in
 
     start-dev)
         build_base
+        runtime_set=all
+        if [[ ! -z "$1" ]]; then
+            runtime_set=$1
+        fi
         docker run \
             --rm \
             -p 2000:2000 \
@@ -65,6 +69,7 @@ case "$SUBCOMMAND" in
             --name $CONTAINER_NAME \
             -e PISTON_LOG_LEVEL=DEBUG \
             -e PISTON_FLAKE_PATH=/piston/packages \
+            -e PISTON_RUNTIME_SET=$runtime_set \
             -v $PWD:/piston/packages \
             -d "$IMAGE_NAME_DEV:$IMAGE_TAG"
     ;;

readme.md

@@ -239,32 +239,32 @@ Content-Type: application/json
 `POST /api/v2/execute`
 This endpoint requests execution of some arbitrary code.
 
-- `language` (**required**) The language to use for execution, must be a string and must be installed.
+-   `language` (**required**) The language to use for execution, must be a string and must be installed.
-- `files` (**required**) An array of files containing code or other data that should be used for execution. The first file in this array is considered the main file.
+-   `files` (**required**) An array of files containing code or other data that should be used for execution. The first file in this array is considered the main file.
-- `files[].name` (_optional_) The name of the file to upload, must be a string containing no path or left out.
+-   `files[].name` (_optional_) The name of the file to upload, must be a string containing no path or left out.
-- `files[].content` (**required**) The content of the files to upload, must be a string containing text to write.
+-   `files[].content` (**required**) The content of the files to upload, must be a string containing text to write.
-- `stdin` (_optional_) The text to pass as stdin to the program. Must be a string or left out. Defaults to blank string.
+-   `stdin` (_optional_) The text to pass as stdin to the program. Must be a string or left out. Defaults to blank string.
-- `args` (_optional_) The arguments to pass to the program. Must be an array or left out. Defaults to `[]`.
+-   `args` (_optional_) The arguments to pass to the program. Must be an array or left out. Defaults to `[]`.
-- `compile_timeout` (_optional_) The maximum time allowed for the compile stage to finish before bailing out in milliseconds. Must be a number or left out. Defaults to `10000` (10 seconds).
+-   `compile_timeout` (_optional_) The maximum time allowed for the compile stage to finish before bailing out in milliseconds. Must be a number or left out. Defaults to `10000` (10 seconds).
-- `run_timeout` (_optional_) The maximum time allowed for the run stage to finish before bailing out in milliseconds. Must be a number or left out. Defaults to `3000` (3 seconds).
+-   `run_timeout` (_optional_) The maximum time allowed for the run stage to finish before bailing out in milliseconds. Must be a number or left out. Defaults to `3000` (3 seconds).
-- `compile_memory_limit` (_optional_) The maximum amount of memory the compile stage is allowed to use in bytes. Must be a number or left out. Defaults to `-1` (no limit)
+-   `compile_memory_limit` (_optional_) The maximum amount of memory the compile stage is allowed to use in bytes. Must be a number or left out. Defaults to `-1` (no limit)
-- `run_memory_limit` (_optional_) The maximum amount of memory the run stage is allowed to use in bytes. Must be a number or left out. Defaults to `-1` (no limit)
+-   `run_memory_limit` (_optional_) The maximum amount of memory the run stage is allowed to use in bytes. Must be a number or left out. Defaults to `-1` (no limit)
 
 ```json
 {
-  "language": "js",
+    "language": "js",
-  "files": [
+    "files": [
-    {
+        {
-      "name": "my_cool_code.js",
+            "name": "my_cool_code.js",
-      "content": "console.log(process.argv)"
+            "content": "console.log(process.argv)"
-    }
+        }
-  ],
+    ],
-  "stdin": "",
+    "stdin": "",
-  "args": ["1", "2", "3"],
+    "args": ["1", "2", "3"],
-  "compile_timeout": 10000,
+    "compile_timeout": 10000,
-  "run_timeout": 3000,
+    "run_timeout": 3000,
-  "compile_memory_limit": -1,
+    "compile_memory_limit": -1,
-  "run_memory_limit": -1
+    "run_memory_limit": -1
 }
 ```
 
@@ -350,6 +350,7 @@ Content-Type: application/json
 `llvm_ir`,
 `lolcode`,
 `lua`,
+`matl`,
 `nasm`,
 `nasm64`,
 `nim`,