api: v3

api/src/executor/job.js

@@ -0,0 +1,146 @@
+const logger = require("logplease").create("executor/job")
+const { v4: uuidv4 } = require("uuid")
+const cp = require("child_process")
+const path = require("path")
+const config = require("../config");
+const globals = require("../globals");
+const fs = require("fs");
+const util = require("util");
+
+const job_states = {
+    READY: Symbol("Ready to be primed"),
+    PRIMED: Symbol("Primed and ready for execution"),
+    EXECUTED: Symbol("Executed and ready for cleanup")
+}
+
+var uid=0;
+var gid=0;
+
+class Job {
+    constructor(runtime, files, args, stdin, timeouts, main){
+        this.uuid =  uuidv4()
+        this.runtime = runtime
+        this.files = files
+        this.args = args
+        this.stdin = stdin
+        this.timeouts = timeouts
+        this.main = main
+
+        if(!Object.keys(this.files).includes(this.main))
+            throw new Error(`Main file "${this.main}" will not be written to disk`)
+
+        this.uid = config.runner_uid_min + uid;
+        this.gid = config.runner_gid_min + gid;
+
+        uid++
+        gid++
+
+        uid %= (config.runner_uid_max - config.runner_uid_min) + 1
+        gid %= (config.runner_gid_max - config.runner_gid_min) + 1
+
+        this.state = job_states.READY;
+        this.dir = path.join(config.data_directory, globals.data_directories.jobs, this.uuid);
+    }
+
+    async prime(){
+        logger.info(`Priming job uuid=${this.uuid}`)
+
+        logger.debug("Writing files to job cache")
+
+        await util.promisify(fs.mkdir)(this.dir, {mode:0o700})
+
+        const files = Object.keys(this.files).map(fileName => {
+            var content = this.files[fileName];
+            return util.promisify(fs.writeFile)(path.join(this.dir, fileName), content)
+        })
+
+        await Promise.all(files)
+
+        logger.debug(`Transfering ownership uid=${this.uid} gid=${this.gid}`)
+        await util.promisify(fs.chown)(this.dir, this.uid, this.gid)
+        
+        const chowns = Object.keys(this.files).map(fileName => {
+            return util.promisify(fs.chown)(path.join(this.dir, fileName), this.uid, this.gid)
+        })
+
+        await Promise.all(chowns)
+
+        this.state = job_states.PRIMED;
+        logger.debug("Primed job")
+    }
+
+    async execute(){
+        if(this.state != job_states.PRIMED) throw new Error("Job must be in primed state, current state: " + this.state.toString())
+        logger.info(`Executing job uuid=${this.uuid} uid=${this.uid} gid=${this.gid} runtime=${this.runtime.toString()}`)
+        logger.debug(`Compiling`)
+
+        const compile = this.runtime.compiled && await new Promise((resolve, reject) => {
+            var stderr, stdout = "";            
+            const proc = cp.spawn(this.runtime.pkgdir, [this.main, ...this.args] ,{
+                env: this.runtime.env_vars,
+                stdio: ['pipe', 'pipe', 'pipe'],
+                cwd: this.dir,
+                uid: this.uid,
+                gid: this.gid
+            })
+
+            const killTimeout = setTimeout(proc.kill, this.timeouts.compile, "SIGKILL")
+
+            proc.stderr.on('data', d=>stderr += d)
+            proc.stdout.on('data', d=>stdout += d)
+
+            proc.on('exit', (code, signal)=>{
+                clearTimeout(killTimeout);
+                resolve({stdout, stderr, code, signal})
+            })
+
+            proc.on('error', (code, signal) => {
+                clearTimeout(killTimeout);
+                reject({stdout, stderr, code, signal})
+            })
+        })
+
+        logger.debug("Running")
+
+        const run = await new Promise((resolve, reject) => {
+            var stderr, stdout = "";            
+            const proc = cp.spawn('bash', [path.join(this.runtime.pkgdir, "run"), this.main, ...this.args] ,{
+                env: this.runtime.env_vars,
+                stdio: ['pipe', 'pipe', 'pipe'],
+                cwd: this.dir,
+                uid: this.uid,
+                gid: this.gid
+            })
+
+            const killTimeout = setTimeout(proc.kill, this.timeouts.run, "SIGKILL")
+
+            proc.stderr.on('data', d=>stderr += d)
+            proc.stdout.on('data', d=>stdout += d)
+
+            proc.on('exit', (code, signal)=>{
+                clearTimeout(killTimeout);
+                resolve({stdout, stderr, code, signal})
+            })
+
+            proc.on('error', (code, signal) => {
+                clearTimeout(killTimeout);
+                reject({stdout, stderr, code, signal})
+            })
+        })
+
+        this.state = job_states.EXECUTED;
+
+        return {
+            compile,
+            run
+        }
+
+    }
+
+    async cleanup(){
+        logger.info(`Cleaning up job uuid=${this.uuid}`)
+        await util.promisify(fs.rm)(this.dir, {recursive: true, force: true})
+    }
+}
+
+module.exports = {Job}

api/src/executor/routes.js

@@ -0,0 +1,34 @@
+// {"language":"python","version":"3.9.1","files":{"code.py":"print('hello world')"},"args":[],"stdin":"","compile_timeout":10, "run_timeout":3, "main": "code.py"}
+// {"success":true, "run":{"stdout":"hello world", "stderr":"", "error_code":0},"compile":{"stdout":"","stderr":"","error_code":0}}
+
+const { get_latest_runtime_matching_language_version } = require("../runtime");
+const { Job } = require("./job");
+
+module.exports = {
+    async run_job(req, res){
+        // POST /jobs
+        var errored = false;
+        ["language", "version",
+         "files", "main",
+         "args", "stdin",
+         "compile_timeout", "run_timeout",
+         ].forEach(key => {
+            if(req.body[key] == undefined) errored = errored || res.json_error(`${key} is required`, 400)
+         })
+         if(errored) return errored;
+
+        const runtime = get_latest_runtime_matching_language_version(req.body.language, req.body.version);
+        if(runtime == undefined) return res.json_error(`${req.body.language}-${req.body.version} runtime is unknown`, 400)
+
+        const job = new Job(runtime, req.body.files, req.body.args, req.body.stdin, {run: req.body.run_timeout, compile: req.body.compile_timeout}, req.body.main)
+        await job.prime()
+
+        const result = await job.execute()
+        res.json_success(result)
+
+        await job.cleanup()
+
+        
+
+    }
+}