cli: correct package name

api-client/.gitignore

@@ -0,0 +1 @@
+node_modules

api-client/index.cjs

@@ -0,0 +1,110 @@
+const fetch = require('node-fetch')
+
+class APIWrapper {
+    #base;
+    constructor(base_url){
+        this.#base = base_url.toString()
+    }
+
+    async query(endpoint, options={}){
+        const url = new URL(endpoint, this.#base).href;
+        return await fetch(url, options)
+            .then(res=>res.json())
+            .then(res=>{if(res.data)return res.data; throw new Error(res.message)});
+    }
+
+    get(endpoint){
+        return this.query(endpoint);
+    }
+
+    post(endpoint, body={}){
+        return this.query(endpoint, {
+            method: 'post',
+            headers: {'Content-Type': 'application/json'},
+            body: JSON.stringify(body)
+        })
+    }
+
+    delete(endpoint, body={}){
+        return this.query(endpoint, {
+            method: 'delete',
+            headers: {'Content-Type': 'application/json'},
+            body: JSON.stringify(body)
+        })
+    }
+
+    get_child_object(endpoint, class_type){
+        return this.get(endpoint).then(x => new class_type(this, x))
+    }
+
+    get url_base(){
+        return this.#base
+    }
+}
+
+class PistonEngineRepositoryPackage extends APIWrapper {
+    constructor(repo, {language, language_version, author, buildfile, size, dependencies, installed}){
+        super(new URL(`/packages/${language}/${language_version}`,repo.url_base))
+
+        this.language = language;
+        this.language_version = language_version;
+        this.author = author;
+        this,buildfile = buildfile;
+        this.size = size;
+        this.dependencies = dependencies;
+        this.installed = installed;
+    }
+
+    install(){
+        return this.post('/', {});
+    }
+
+    uninstall(){
+        return this.delete('/', {});
+    }
+}
+
+class PistonEngineRepository extends APIWrapper {
+    
+    constructor(engine, {slug, url, packages}){
+        super(new URL(`/repos/${slug}`, engine.url_base,))
+
+        this.slug = slug;
+        this.url = url;
+        this.package_count = packages
+
+    }
+
+    list_packages(){
+        return this.get(`/packages`).then(x=>x.packages)
+    }
+
+    get_package(language, language_version){
+        return this.get_child_object(`/packages/${language}/${language_version}`, PistonEngineRepositoryPackage)
+    }
+}
+
+class PistonEngine extends APIWrapper {
+    constructor(base_url = 'http://127.0.0.1:6969'){
+        super(base_url);
+    }
+
+    list_repos(){
+        return this.get(`/repos`);
+    }
+
+    add_repo(slug, url){
+        return this.post(`/repos`, {slug, url})
+    }
+
+    get_repo(slug){
+        return this.get_child_object(`/repos/${slug}`, PistonEngineRepository)
+    }
+
+    run_job(language, version, files, main, args, stdin, compile_timeout, run_timeout){
+        return this.post(`/jobs`, {language, version, files, main, args, stdin, compile_timeout, run_timeout})
+    }
+}
+
+
+module.exports = {PistonEngine}

api-client/package.json

@@ -0,0 +1,11 @@
+{
+  "name": "piston-api-client",
+  "version": "1.0.0",
+  "description": "Wraps API of Piston Engine API",
+  "main": "index.cjs",
+  "author": "Thomas Hobson <thomas@hexf.me>",
+  "license": "MIT",
+  "dependencies": {
+    "node-fetch": "^2.6.1"
+  }
+}

api/Dockerfile

@@ -1,12 +1,9 @@
 FROM node:15.8.0-alpine3.13
-RUN apk add --no-cache gnupg tar bash coreutils shadow
+RUN apk add --no-cache gnupg tar bash coreutils shadow util-linux
+RUN userdel -r node
 RUN for i in $(seq 1000 1500); do \
         groupadd -g $i runner$i && \
-        useradd -M runner$i -g $i -u $i && \
-        echo "runner$i soft nproc 64" >> /etc/security/limits.conf && \
-        echo "runner$i hard nproc 64" >> /etc/security/limits.conf && \ 
-        echo "runner$i soft nofile 2048" >> /etc/security/limits.conf && \
-        echo "runner$i hard nofile 2048" >> /etc/security/limits.conf ;\
+        useradd -M runner$i -g $i -u $i ; \
     done
 
 ENV NODE_ENV=production

api/src/config.js

@@ -107,6 +107,18 @@ const options = [
         desc: 'Max size of each stdio buffer',
         default: 1024,
         validators: []
+    },
+    {
+        key: 'max_process_count',
+        desc: 'Max number of processes per job',
+        default: 64,
+        validators: []
+    },
+    {
+        key: 'max_open_files',
+        desc: 'Max number of open files per job',
+        default: 2048,
+        validators: []
     }
 ];
 

api/src/executor/job.js

@@ -68,80 +68,77 @@ class Job {
         logger.debug('Primed job');
     }
 
-    async execute(){
-        if(this.state != job_states.PRIMED) throw new Error('Job must be in primed state, current state: ' + this.state.toString());
-        logger.info(`Executing job uuid=${this.uuid} uid=${this.uid} gid=${this.gid} runtime=${this.runtime.toString()}`);
-        logger.debug('Compiling');
-        const compile = this.runtime.compiled && await new Promise((resolve, reject) => {
-            const proc_call = ['unshare', '-n', '-r', 'bash', path.join(this.runtime.pkgdir, 'compile'),this.main, ...this.files].slice(!config.enable_unshare * 3)
+    async safe_call(file, args, timeout){
+        return await new Promise((resolve, reject) => {
+            const unshare = config.enable_unshare ? ['unshare','-n','-r'] : [];
+            const prlimit = [
+                'prlimit',
+                '--nproc=' + config.max_process_count,
+                '--nofile=' + config.max_open_files
+            ];
+
+            const proc_call = [
+                ...prlimit,
+                ...unshare,
+                'bash',file, ...args
+            ];
             var stdout = '';
             var stderr = '';
             const proc = cp.spawn(proc_call[0], proc_call.splice(1) ,{
                 env: this.runtime.env_vars,
-                stdio: ['pipe', 'pipe', 'pipe'],
+                stdio: 'pipe',
                 cwd: this.dir,
                 uid: this.uid,
-                gid: this.gid
+                gid: this.gid,
+                detached: true //dont kill the main process when we kill the group
             });
 
-            const kill_timeout = setTimeout(_ => proc.kill('SIGKILL'), this.timeouts.compile);
+            
+
+            const kill_timeout = setTimeout(_ => proc.kill('SIGKILL'), timeout);
 
             proc.stderr.on('data', d=>{if(stderr.length>config.output_max_size) proc.kill('SIGKILL'); else stderr += d;});
             proc.stdout.on('data', d=>{if(stdout.length>config.output_max_size) proc.kill('SIGKILL'); else stdout += d;});
+            function exit_cleanup(){
+                clearTimeout(kill_timeout);
+                proc.stderr.destroy();
+                proc.stdout.destroy();
+                try{
+                    process.kill(-proc.pid, 'SIGKILL');
+                }catch{
+                    // Process will be dead alread, so nothing to kill.
+                }
+            }
 
             proc.on('exit', (code, signal)=>{
-                clearTimeout(kill_timeout);
-                proc.stderr.destroy()
-                proc.stdout.destroy()
+                exit_cleanup();
                 
                 resolve({stdout, stderr, code, signal});
             });
 
             proc.on('error', (err) => {
-                clearTimeout(kill_timeout);
-                proc.stderr.destroy()
-                proc.stdout.destroy()
+                exit_cleanup();
 
                 reject({error: err, stdout, stderr});
             });
         });
+    }
+
+    async execute(){
+        if(this.state != job_states.PRIMED) throw new Error('Job must be in primed state, current state: ' + this.state.toString());
+        logger.info(`Executing job uuid=${this.uuid} uid=${this.uid} gid=${this.gid} runtime=${this.runtime.toString()}`);
+        logger.debug('Compiling');
+        const compile = this.runtime.compiled && await this.safe_call(
+            path.join(this.runtime.pkgdir, 'compile'),
+            [this.main, ...this.files],
+            this.timeouts.compile);
 
         logger.debug('Running');
 
-        const run = await new Promise((resolve, reject) => {
-            const proc_call = ['unshare', '-n', '-r', 'bash', path.join(this.runtime.pkgdir, 'run'), this.main, ...this.args].slice(!config.enable_unshare * 3);
-            var stdout = '';            
-            var stderr = '';
-            const proc = cp.spawn(proc_call[0], proc_call.slice(1) ,{
-                env: this.runtime.env_vars,
-                stdio: ['pipe', 'pipe', 'pipe'],
-                cwd: this.dir,
-                uid: this.uid,
-                gid: this.gid
-            });
-            
-            const kill_timeout = setTimeout(_ => proc.kill('SIGKILL'), this.timeouts.run);
-
-            proc.stderr.on('data', d=>{if(stderr.length>config.output_max_size) proc.kill('SIGKILL'); else stderr += d;});
-            proc.stdout.on('data', d=>{if(stdout.length>config.output_max_size) proc.kill('SIGKILL'); else stdout += d;});
-
-            proc.stdin.write(this.stdin)
-            proc.stdin.end()
-
-            proc.on('exit', (code, signal)=>{
-                clearTimeout(kill_timeout);
-                proc.stderr.destroy()
-                proc.stdout.destroy()
-                resolve({stdout, stderr, code, signal});
-            });
-
-            proc.on('error', (err) => {
-                clearTimeout(kill_timeout);
-                proc.stderr.destroy()
-                proc.stdout.destroy()
-                reject({error: err, stdout, stderr});
-            });
-        });
+        const run = await this.safe_call(
+            path.join(this.runtime.pkgdir, 'run'),
+            [this.main, ...this.args],
+            this.timeouts.run);
 
         this.state = job_states.EXECUTED;
 

cli/.gitignore

@@ -0,0 +1 @@
+node_modules

cli/commands/execute.js

@@ -0,0 +1,90 @@
+const {PistonEngine} = require('piston-api-client');
+const fs = require('fs');
+const path = require('path');
+const chalk = require('chalk');
+
+exports.command = ['execute <language> <language-version> <file> [args..]']
+exports.aliases = ['run']
+exports.describe = 'Executes file with the specified runner'
+
+exports.builder = {
+    stdin: {
+        boolean: true,
+        desc: 'Read input from stdin and pass to executor',
+        alias: ['i']
+    },
+    run_timeout: {
+        alias: ['rt', 'r'],
+        number: true,
+        desc: 'Milliseconds before killing run process',
+        default: 3000
+    },
+    compile_timeout: {
+        alias: ['ct', 'c'],
+        number: true,
+        desc: 'Milliseconds before killing compile process',
+        default: 10000,
+    },
+    files: {
+        alias: ['f'],
+        array: true,
+        desc: 'Additional files to add',
+    }
+}
+
+exports.handler = async function(argv){
+    
+    const api = new PistonEngine(argv['piston-url']);
+
+    const files = [...(argv.files || []),argv.file]
+        .map(file_path => ({
+            name: path.basename(file_path),
+            content: fs.readFileSync(file_path).toString()
+        }));
+
+
+    const stdin = (argv.stdin && await new Promise((resolve, _)=>{
+        var data = "";
+        process.stdin.on('data', d=> data += d)
+        process.stdin.on('end', _ => resolve(data))
+    })) || "";
+
+
+    const response = await api.run_job(
+        argv.language,
+        argv['language-version'],
+        files,
+        argv.file,
+        argv.args,
+        stdin,
+        argv.ct,
+        argv.rt
+    )
+
+    function step(name, ctx){
+        console.log(chalk.bold(`== ${name} ==`))
+        if(ctx.stdout){
+            console.log(" ",chalk.bold(`STDOUT`))
+            console.log("   ",ctx.stdout.replace(/\n/g,'\n    '))
+        }
+        if(ctx.stderr){
+            console.log(chalk.bold(`STDERR`))
+            console.log("   ",ctx.stderr.replace(/\n/g,'\n    '))
+        }
+
+        if(ctx.code)
+            console.log(
+                chalk.bold(`Exit Code:`),
+                chalk.bold[ctx.code > 0 ? 'red' : 'green'](ctx.code)
+                )
+        if(ctx.signal)
+            console.log(
+                chalk.bold(`Signal:`),
+                chalk.bold.yellow(ctx.signal)
+                )
+    }
+    
+    if(response.compile) step('Compile', response.compile)
+    step('Run', response.run)
+    
+}

cli/index.js

@@ -0,0 +1,14 @@
+#!/usr/bin/node
+require('yargs')(process.argv.slice(2))
+    .option('piston-url', {
+        alias: ['u'],
+        default: 'http://127.0.0.1:6969',
+        desc: 'Piston API URL',
+        string: true
+    })
+    .scriptName("piston")
+    .commandDir('commands')
+    .demandCommand()
+    .help()
+    .wrap(72)
+    .argv

cli/package.json

@@ -0,0 +1,13 @@
+{
+  "name": "piston-cli",
+  "version": "1.0.0",
+  "description": "Piston Execution Engine CLI tools",
+  "main": "index.js",
+  "author": "Thomas Hobson <thomas@hexf.me>",
+  "license": "MIT",
+  "dependencies": {
+    "chalk": "^4.1.0",
+    "piston-api-client": "file:../api-client",
+    "yargs": "^16.2.0"
+  }
+}

cli/yarn.lock

@@ -0,0 +1,139 @@
+# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+
+
+ansi-regex@^5.0.0:
+  version "5.0.0"
+  resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.0.tgz#388539f55179bf39339c81af30a654d69f87cb75"
+  integrity sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==
+
+ansi-styles@^4.0.0, ansi-styles@^4.1.0:
+  version "4.3.0"
+  resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-4.3.0.tgz#edd803628ae71c04c85ae7a0906edad34b648937"
+  integrity sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==
+  dependencies:
+    color-convert "^2.0.1"
+
+chalk@^4.1.0:
+  version "4.1.0"
+  resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.0.tgz#4e14870a618d9e2edd97dd8345fd9d9dc315646a"
+  integrity sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==
+  dependencies:
+    ansi-styles "^4.1.0"
+    supports-color "^7.1.0"
+
+cliui@^7.0.2:
+  version "7.0.4"
+  resolved "https://registry.yarnpkg.com/cliui/-/cliui-7.0.4.tgz#a0265ee655476fc807aea9df3df8df7783808b4f"
+  integrity sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==
+  dependencies:
+    string-width "^4.2.0"
+    strip-ansi "^6.0.0"
+    wrap-ansi "^7.0.0"
+
+color-convert@^2.0.1:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-2.0.1.tgz#72d3a68d598c9bdb3af2ad1e84f21d896abd4de3"
+  integrity sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==
+  dependencies:
+    color-name "~1.1.4"
+
+color-name@~1.1.4:
+  version "1.1.4"
+  resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2"
+  integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==
+
+emoji-regex@^8.0.0:
+  version "8.0.0"
+  resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-8.0.0.tgz#e818fd69ce5ccfcb404594f842963bf53164cc37"
+  integrity sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==
+
+escalade@^3.1.1:
+  version "3.1.1"
+  resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.1.tgz#d8cfdc7000965c5a0174b4a82eaa5c0552742e40"
+  integrity sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==
+
+get-caller-file@^2.0.5:
+  version "2.0.5"
+  resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-2.0.5.tgz#4f94412a82db32f36e3b0b9741f8a97feb031f7e"
+  integrity sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==
+
+has-flag@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-4.0.0.tgz#944771fd9c81c81265c4d6941860da06bb59479b"
+  integrity sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==
+
+is-fullwidth-code-point@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz#f116f8064fe90b3f7844a38997c0b75051269f1d"
+  integrity sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==
+
+node-fetch@^2.6.1:
+  version "2.6.1"
+  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.1.tgz#045bd323631f76ed2e2b55573394416b639a0052"
+  integrity sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==
+
+"piston-api-client@file:../api-client":
+  version "1.0.0"
+  dependencies:
+    node-fetch "^2.6.1"
+
+require-directory@^2.1.1:
+  version "2.1.1"
+  resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42"
+  integrity sha1-jGStX9MNqxyXbiNE/+f3kqam30I=
+
+string-width@^4.1.0, string-width@^4.2.0:
+  version "4.2.0"
+  resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.0.tgz#952182c46cc7b2c313d1596e623992bd163b72b5"
+  integrity sha512-zUz5JD+tgqtuDjMhwIg5uFVV3dtqZ9yQJlZVfq4I01/K5Paj5UHj7VyrQOJvzawSVlKpObApbfD0Ed6yJc+1eg==
+  dependencies:
+    emoji-regex "^8.0.0"
+    is-fullwidth-code-point "^3.0.0"
+    strip-ansi "^6.0.0"
+
+strip-ansi@^6.0.0:
+  version "6.0.0"
+  resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.0.tgz#0b1571dd7669ccd4f3e06e14ef1eed26225ae532"
+  integrity sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==
+  dependencies:
+    ansi-regex "^5.0.0"
+
+supports-color@^7.1.0:
+  version "7.2.0"
+  resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-7.2.0.tgz#1b7dcdcb32b8138801b3e478ba6a51caa89648da"
+  integrity sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==
+  dependencies:
+    has-flag "^4.0.0"
+
+wrap-ansi@^7.0.0:
+  version "7.0.0"
+  resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"
+  integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
+  dependencies:
+    ansi-styles "^4.0.0"
+    string-width "^4.1.0"
+    strip-ansi "^6.0.0"
+
+y18n@^5.0.5:
+  version "5.0.5"
+  resolved "https://registry.yarnpkg.com/y18n/-/y18n-5.0.5.tgz#8769ec08d03b1ea2df2500acef561743bbb9ab18"
+  integrity sha512-hsRUr4FFrvhhRH12wOdfs38Gy7k2FFzB9qgN9v3aLykRq0dRcdcpz5C9FxdS2NuhOrI/628b/KSTJ3rwHysYSg==
+
+yargs-parser@^20.2.2:
+  version "20.2.6"
+  resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-20.2.6.tgz#69f920addf61aafc0b8b89002f5d66e28f2d8b20"
+  integrity sha512-AP1+fQIWSM/sMiET8fyayjx/J+JmTPt2Mr0FkrgqB4todtfa53sOsrSAcIrJRD5XS20bKUwaDIuMkWKCEiQLKA==
+
+yargs@^16.2.0:
+  version "16.2.0"
+  resolved "https://registry.yarnpkg.com/yargs/-/yargs-16.2.0.tgz#1c82bf0f6b6a66eafce7ef30e376f49a12477f66"
+  integrity sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==
+  dependencies:
+    cliui "^7.0.2"
+    escalade "^3.1.1"
+    get-caller-file "^2.0.5"
+    require-directory "^2.1.1"
+    string-width "^4.2.0"
+    y18n "^5.0.5"
+    yargs-parser "^20.2.2"

docker-compose.yaml

@@ -18,7 +18,7 @@ services:
     build: repo
     command: >
       bash -c '/repo/make.sh &&
-      true || curl http://piston_api:6969/repos -XPOST -d "slug=local&url=file:///repo/index.yaml";
+      curl http://piston_api:6969/repos -XPOST -d "slug=local&url=file:///repo/index.yaml";
       echo -e "\nAn error here is fine, it just means its already added it. Perhaps you restarted this container"
       '
     volumes: