Compare commits

..

No commits in common. "2505b89fcfb76e83ee9d8fabf931c38af411879b" and "72e1eb145784123b960653871d0520813d6be4fa" have entirely different histories.

12 changed files with 56 additions and 441 deletions

View File

@ -1 +0,0 @@
node_modules

View File

@ -1,110 +0,0 @@
const fetch = require('node-fetch')
class APIWrapper {
#base;
constructor(base_url){
this.#base = base_url.toString()
}
async query(endpoint, options={}){
const url = new URL(endpoint, this.#base).href;
return await fetch(url, options)
.then(res=>res.json())
.then(res=>{if(res.data)return res.data; throw new Error(res.message)});
}
get(endpoint){
return this.query(endpoint);
}
post(endpoint, body={}){
return this.query(endpoint, {
method: 'post',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify(body)
})
}
delete(endpoint, body={}){
return this.query(endpoint, {
method: 'delete',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify(body)
})
}
get_child_object(endpoint, class_type){
return this.get(endpoint).then(x => new class_type(this, x))
}
get url_base(){
return this.#base
}
}
class PistonEngineRepositoryPackage extends APIWrapper {
constructor(repo, {language, language_version, author, buildfile, size, dependencies, installed}){
super(new URL(`/packages/${language}/${language_version}`,repo.url_base))
this.language = language;
this.language_version = language_version;
this.author = author;
this,buildfile = buildfile;
this.size = size;
this.dependencies = dependencies;
this.installed = installed;
}
install(){
return this.post('/', {});
}
uninstall(){
return this.delete('/', {});
}
}
class PistonEngineRepository extends APIWrapper {
constructor(engine, {slug, url, packages}){
super(new URL(`/repos/${slug}`, engine.url_base,))
this.slug = slug;
this.url = url;
this.package_count = packages
}
list_packages(){
return this.get(`/packages`).then(x=>x.packages)
}
get_package(language, language_version){
return this.get_child_object(`/packages/${language}/${language_version}`, PistonEngineRepositoryPackage)
}
}
class PistonEngine extends APIWrapper {
constructor(base_url = 'http://127.0.0.1:6969'){
super(base_url);
}
list_repos(){
return this.get(`/repos`);
}
add_repo(slug, url){
return this.post(`/repos`, {slug, url})
}
get_repo(slug){
return this.get_child_object(`/repos/${slug}`, PistonEngineRepository)
}
run_job(language, version, files, main, args, stdin, compile_timeout, run_timeout){
return this.post(`/jobs`, {language, version, files, main, args, stdin, compile_timeout, run_timeout})
}
}
module.exports = {PistonEngine}

View File

@ -1,11 +0,0 @@
{
"name": "piston-api-client",
"version": "1.0.0",
"description": "Wraps API of Piston Engine API",
"main": "index.cjs",
"author": "Thomas Hobson <thomas@hexf.me>",
"license": "MIT",
"dependencies": {
"node-fetch": "^2.6.1"
}
}

View File

@ -1,9 +1,12 @@
FROM node:15.8.0-alpine3.13
RUN apk add --no-cache gnupg tar bash coreutils shadow util-linux
RUN userdel -r node
RUN apk add --no-cache gnupg tar bash coreutils shadow
RUN for i in $(seq 1000 1500); do \
groupadd -g $i runner$i && \
useradd -M runner$i -g $i -u $i ; \
useradd -M runner$i -g $i -u $i && \
echo "runner$i soft nproc 64" >> /etc/security/limits.conf && \
echo "runner$i hard nproc 64" >> /etc/security/limits.conf && \
echo "runner$i soft nofile 2048" >> /etc/security/limits.conf && \
echo "runner$i hard nofile 2048" >> /etc/security/limits.conf ;\
done
ENV NODE_ENV=production

View File

@ -107,18 +107,6 @@ const options = [
desc: 'Max size of each stdio buffer',
default: 1024,
validators: []
},
{
key: 'max_process_count',
desc: 'Max number of processes per job',
default: 64,
validators: []
},
{
key: 'max_open_files',
desc: 'Max number of open files per job',
default: 2048,
validators: []
}
];

View File

@ -68,77 +68,80 @@ class Job {
logger.debug('Primed job');
}
async safe_call(file, args, timeout){
return await new Promise((resolve, reject) => {
const unshare = config.enable_unshare ? ['unshare','-n','-r'] : [];
const prlimit = [
'prlimit',
'--nproc=' + config.max_process_count,
'--nofile=' + config.max_open_files
];
const proc_call = [
...prlimit,
...unshare,
'bash',file, ...args
];
async execute(){
if(this.state != job_states.PRIMED) throw new Error('Job must be in primed state, current state: ' + this.state.toString());
logger.info(`Executing job uuid=${this.uuid} uid=${this.uid} gid=${this.gid} runtime=${this.runtime.toString()}`);
logger.debug('Compiling');
const compile = this.runtime.compiled && await new Promise((resolve, reject) => {
const proc_call = ['unshare', '-n', '-r', 'bash', path.join(this.runtime.pkgdir, 'compile'),this.main, ...this.files].slice(!config.enable_unshare * 3)
var stdout = '';
var stderr = '';
const proc = cp.spawn(proc_call[0], proc_call.splice(1) ,{
env: this.runtime.env_vars,
stdio: 'pipe',
stdio: ['pipe', 'pipe', 'pipe'],
cwd: this.dir,
uid: this.uid,
gid: this.gid,
detached: true //dont kill the main process when we kill the group
gid: this.gid
});
const kill_timeout = setTimeout(_ => proc.kill('SIGKILL'), timeout);
const kill_timeout = setTimeout(_ => proc.kill('SIGKILL'), this.timeouts.compile);
proc.stderr.on('data', d=>{if(stderr.length>config.output_max_size) proc.kill('SIGKILL'); else stderr += d;});
proc.stdout.on('data', d=>{if(stdout.length>config.output_max_size) proc.kill('SIGKILL'); else stdout += d;});
function exit_cleanup(){
clearTimeout(kill_timeout);
proc.stderr.destroy();
proc.stdout.destroy();
try{
process.kill(-proc.pid, 'SIGKILL');
}catch{
// Process will be dead alread, so nothing to kill.
}
}
proc.on('exit', (code, signal)=>{
exit_cleanup();
clearTimeout(kill_timeout);
proc.stderr.destroy()
proc.stdout.destroy()
resolve({stdout, stderr, code, signal});
});
proc.on('error', (err) => {
exit_cleanup();
clearTimeout(kill_timeout);
proc.stderr.destroy()
proc.stdout.destroy()
reject({error: err, stdout, stderr});
});
});
}
async execute(){
if(this.state != job_states.PRIMED) throw new Error('Job must be in primed state, current state: ' + this.state.toString());
logger.info(`Executing job uuid=${this.uuid} uid=${this.uid} gid=${this.gid} runtime=${this.runtime.toString()}`);
logger.debug('Compiling');
const compile = this.runtime.compiled && await this.safe_call(
path.join(this.runtime.pkgdir, 'compile'),
[this.main, ...this.files],
this.timeouts.compile);
logger.debug('Running');
const run = await this.safe_call(
path.join(this.runtime.pkgdir, 'run'),
[this.main, ...this.args],
this.timeouts.run);
const run = await new Promise((resolve, reject) => {
const proc_call = ['unshare', '-n', '-r', 'bash', path.join(this.runtime.pkgdir, 'run'), this.main, ...this.args].slice(!config.enable_unshare * 3);
var stdout = '';
var stderr = '';
const proc = cp.spawn(proc_call[0], proc_call.slice(1) ,{
env: this.runtime.env_vars,
stdio: ['pipe', 'pipe', 'pipe'],
cwd: this.dir,
uid: this.uid,
gid: this.gid
});
const kill_timeout = setTimeout(_ => proc.kill('SIGKILL'), this.timeouts.run);
proc.stderr.on('data', d=>{if(stderr.length>config.output_max_size) proc.kill('SIGKILL'); else stderr += d;});
proc.stdout.on('data', d=>{if(stdout.length>config.output_max_size) proc.kill('SIGKILL'); else stdout += d;});
proc.stdin.write(this.stdin)
proc.stdin.end()
proc.on('exit', (code, signal)=>{
clearTimeout(kill_timeout);
proc.stderr.destroy()
proc.stdout.destroy()
resolve({stdout, stderr, code, signal});
});
proc.on('error', (err) => {
clearTimeout(kill_timeout);
proc.stderr.destroy()
proc.stdout.destroy()
reject({error: err, stdout, stderr});
});
});
this.state = job_states.EXECUTED;

1
cli/.gitignore vendored
View File

@ -1 +0,0 @@
node_modules

View File

@ -1,90 +0,0 @@
const {PistonEngine} = require('piston-api-client');
const fs = require('fs');
const path = require('path');
const chalk = require('chalk');
exports.command = ['execute <language> <language-version> <file> [args..]']
exports.aliases = ['run']
exports.describe = 'Executes file with the specified runner'
exports.builder = {
stdin: {
boolean: true,
desc: 'Read input from stdin and pass to executor',
alias: ['i']
},
run_timeout: {
alias: ['rt', 'r'],
number: true,
desc: 'Milliseconds before killing run process',
default: 3000
},
compile_timeout: {
alias: ['ct', 'c'],
number: true,
desc: 'Milliseconds before killing compile process',
default: 10000,
},
files: {
alias: ['f'],
array: true,
desc: 'Additional files to add',
}
}
exports.handler = async function(argv){
const api = new PistonEngine(argv['piston-url']);
const files = [...(argv.files || []),argv.file]
.map(file_path => ({
name: path.basename(file_path),
content: fs.readFileSync(file_path).toString()
}));
const stdin = (argv.stdin && await new Promise((resolve, _)=>{
var data = "";
process.stdin.on('data', d=> data += d)
process.stdin.on('end', _ => resolve(data))
})) || "";
const response = await api.run_job(
argv.language,
argv['language-version'],
files,
argv.file,
argv.args,
stdin,
argv.ct,
argv.rt
)
function step(name, ctx){
console.log(chalk.bold(`== ${name} ==`))
if(ctx.stdout){
console.log(" ",chalk.bold(`STDOUT`))
console.log(" ",ctx.stdout.replace(/\n/g,'\n '))
}
if(ctx.stderr){
console.log(chalk.bold(`STDERR`))
console.log(" ",ctx.stderr.replace(/\n/g,'\n '))
}
if(ctx.code)
console.log(
chalk.bold(`Exit Code:`),
chalk.bold[ctx.code > 0 ? 'red' : 'green'](ctx.code)
)
if(ctx.signal)
console.log(
chalk.bold(`Signal:`),
chalk.bold.yellow(ctx.signal)
)
}
if(response.compile) step('Compile', response.compile)
step('Run', response.run)
}

View File

@ -1,14 +0,0 @@
#!/usr/bin/node
require('yargs')(process.argv.slice(2))
.option('piston-url', {
alias: ['u'],
default: 'http://127.0.0.1:6969',
desc: 'Piston API URL',
string: true
})
.scriptName("piston")
.commandDir('commands')
.demandCommand()
.help()
.wrap(72)
.argv

View File

@ -1,13 +0,0 @@
{
"name": "piston-cli",
"version": "1.0.0",
"description": "Piston Execution Engine CLI tools",
"main": "index.js",
"author": "Thomas Hobson <thomas@hexf.me>",
"license": "MIT",
"dependencies": {
"chalk": "^4.1.0",
"piston-api-client": "file:../api-client",
"yargs": "^16.2.0"
}
}

View File

@ -1,139 +0,0 @@
# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
# yarn lockfile v1
ansi-regex@^5.0.0:
version "5.0.0"
resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.0.tgz#388539f55179bf39339c81af30a654d69f87cb75"
integrity sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==
ansi-styles@^4.0.0, ansi-styles@^4.1.0:
version "4.3.0"
resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-4.3.0.tgz#edd803628ae71c04c85ae7a0906edad34b648937"
integrity sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==
dependencies:
color-convert "^2.0.1"
chalk@^4.1.0:
version "4.1.0"
resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.0.tgz#4e14870a618d9e2edd97dd8345fd9d9dc315646a"
integrity sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==
dependencies:
ansi-styles "^4.1.0"
supports-color "^7.1.0"
cliui@^7.0.2:
version "7.0.4"
resolved "https://registry.yarnpkg.com/cliui/-/cliui-7.0.4.tgz#a0265ee655476fc807aea9df3df8df7783808b4f"
integrity sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==
dependencies:
string-width "^4.2.0"
strip-ansi "^6.0.0"
wrap-ansi "^7.0.0"
color-convert@^2.0.1:
version "2.0.1"
resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-2.0.1.tgz#72d3a68d598c9bdb3af2ad1e84f21d896abd4de3"
integrity sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==
dependencies:
color-name "~1.1.4"
color-name@~1.1.4:
version "1.1.4"
resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2"
integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==
emoji-regex@^8.0.0:
version "8.0.0"
resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-8.0.0.tgz#e818fd69ce5ccfcb404594f842963bf53164cc37"
integrity sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==
escalade@^3.1.1:
version "3.1.1"
resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.1.tgz#d8cfdc7000965c5a0174b4a82eaa5c0552742e40"
integrity sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==
get-caller-file@^2.0.5:
version "2.0.5"
resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-2.0.5.tgz#4f94412a82db32f36e3b0b9741f8a97feb031f7e"
integrity sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==
has-flag@^4.0.0:
version "4.0.0"
resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-4.0.0.tgz#944771fd9c81c81265c4d6941860da06bb59479b"
integrity sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==
is-fullwidth-code-point@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz#f116f8064fe90b3f7844a38997c0b75051269f1d"
integrity sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==
node-fetch@^2.6.1:
version "2.6.1"
resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.1.tgz#045bd323631f76ed2e2b55573394416b639a0052"
integrity sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==
"piston-api-client@file:../api-client":
version "1.0.0"
dependencies:
node-fetch "^2.6.1"
require-directory@^2.1.1:
version "2.1.1"
resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42"
integrity sha1-jGStX9MNqxyXbiNE/+f3kqam30I=
string-width@^4.1.0, string-width@^4.2.0:
version "4.2.0"
resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.0.tgz#952182c46cc7b2c313d1596e623992bd163b72b5"
integrity sha512-zUz5JD+tgqtuDjMhwIg5uFVV3dtqZ9yQJlZVfq4I01/K5Paj5UHj7VyrQOJvzawSVlKpObApbfD0Ed6yJc+1eg==
dependencies:
emoji-regex "^8.0.0"
is-fullwidth-code-point "^3.0.0"
strip-ansi "^6.0.0"
strip-ansi@^6.0.0:
version "6.0.0"
resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.0.tgz#0b1571dd7669ccd4f3e06e14ef1eed26225ae532"
integrity sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==
dependencies:
ansi-regex "^5.0.0"
supports-color@^7.1.0:
version "7.2.0"
resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-7.2.0.tgz#1b7dcdcb32b8138801b3e478ba6a51caa89648da"
integrity sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==
dependencies:
has-flag "^4.0.0"
wrap-ansi@^7.0.0:
version "7.0.0"
resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"
integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
dependencies:
ansi-styles "^4.0.0"
string-width "^4.1.0"
strip-ansi "^6.0.0"
y18n@^5.0.5:
version "5.0.5"
resolved "https://registry.yarnpkg.com/y18n/-/y18n-5.0.5.tgz#8769ec08d03b1ea2df2500acef561743bbb9ab18"
integrity sha512-hsRUr4FFrvhhRH12wOdfs38Gy7k2FFzB9qgN9v3aLykRq0dRcdcpz5C9FxdS2NuhOrI/628b/KSTJ3rwHysYSg==
yargs-parser@^20.2.2:
version "20.2.6"
resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-20.2.6.tgz#69f920addf61aafc0b8b89002f5d66e28f2d8b20"
integrity sha512-AP1+fQIWSM/sMiET8fyayjx/J+JmTPt2Mr0FkrgqB4todtfa53sOsrSAcIrJRD5XS20bKUwaDIuMkWKCEiQLKA==
yargs@^16.2.0:
version "16.2.0"
resolved "https://registry.yarnpkg.com/yargs/-/yargs-16.2.0.tgz#1c82bf0f6b6a66eafce7ef30e376f49a12477f66"
integrity sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==
dependencies:
cliui "^7.0.2"
escalade "^3.1.1"
get-caller-file "^2.0.5"
require-directory "^2.1.1"
string-width "^4.2.0"
y18n "^5.0.5"
yargs-parser "^20.2.2"

View File

@ -18,7 +18,7 @@ services:
build: repo
command: >
bash -c '/repo/make.sh &&
curl http://piston_api:6969/repos -XPOST -d "slug=local&url=file:///repo/index.yaml";
true || curl http://piston_api:6969/repos -XPOST -d "slug=local&url=file:///repo/index.yaml";
echo -e "\nAn error here is fine, it just means its already added it. Perhaps you restarted this container"
'
volumes: