From 4870441574bced4f83777c9bc12235d5953ef83c Mon Sep 17 00:00:00 2001 From: Brikaa Date: Fri, 1 Oct 2021 21:41:09 +0200 Subject: [PATCH 01/72] config.js: timeout, overrides --- api/src/api/v2.js | 6 ++-- api/src/config.js | 74 ++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 76 insertions(+), 4 deletions(-) diff --git a/api/src/api/v2.js b/api/src/api/v2.js index e3e0522..e0abacd 100644 --- a/api/src/api/v2.js +++ b/api/src/api/v2.js @@ -109,8 +109,8 @@ function get_job(body){ stdin: stdin || "", files, timeouts: { - run: run_timeout || 3000, - compile: compile_timeout || 10000, + run: run_timeout || config.run_timeout, + compile: compile_timeout || config.compile_timeout, }, memory_limits: { run: run_memory_limit || config.run_memory_limit, @@ -228,7 +228,7 @@ router.post('/execute', async (req, res) => { return res.status(200).send(result); }catch(error){ - return res.status(400).json(error); + return res.status(400).json(error.to_string()); } }); diff --git a/api/src/config.js b/api/src/config.js index bbd7ae9..162c9d6 100644 --- a/api/src/config.js +++ b/api/src/config.js @@ -2,6 +2,46 @@ const fss = require('fs'); const Logger = require('logplease'); const logger = Logger.create('config'); +function parse_overrides(overrides) { + try { + return JSON.parse(overrides); + } + catch (e) { + return null; + } +} + +function validate_overrides(overrides, options) { + for (let language in overrides) { + for (let key in overrides[language]) { + if ( + ![ + 'max_process_count', 'max_open_files', 'max_file_size', + 'compile_memory_limit', 'run_memory_limit', 'compile_timeout', + 'run_timeout', 'output_max_size' + ].includes(key) + ) { + logger.error(`Invalid overridden option: ${key}`); + return false; + } + let option = options.find((o) => o.key == key); + let parser = option.parser; + let raw = overrides[language][key]; + let value = parser(raw); + let validators = option.validators; + for (let validator of validators) { + let response = validator(value, raw); + if (response !== true) { + logger.error(`Failed to validate overridden option: ${key}`, response); + return false; + } + } + overrides[language][key] = value; + } + } + return overrides; +} + const options = [ { key: 'log_level', @@ -91,6 +131,22 @@ const options = [ parser: parse_int, validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`], }, + { + key: 'compile_timeout', + desc: + 'Max time allowed for compile stage in milliseconds', + default: 10000, // 10 seconds + parser: parse_int, + validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`], + }, + { + key: 'run_timeout', + desc: + 'Max time allowed for run stage in milliseconds', + default: 3000, // 3 seconds + parser: parse_int, + validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`], + }, { key: 'compile_memory_limit', desc: @@ -120,6 +176,16 @@ const options = [ default: 64, parser: parse_int, validators: [(x) => x > 0 || `${x} cannot be negative`] + }, + { + key: 'limit_overrides', + desc: 'Per-language exceptions in JSON format for each of:\ + max_process_count, max_open_files, max_file_size, compile_memory_limit,\ + run_memory_limit, compile_timeout, run_timeout, output_max_size', + default: {}, + parser: parse_overrides, + validators: [(x) => !!x || `Invalid JSON format for the overrides\n${x}`] + // More validation is done after the configs are loaded } ]; @@ -138,7 +204,7 @@ options.forEach(option => { const parsed_val = parser(env_val); - const value = env_val || option.default; + const value = parsed_val || option.default; option.validators.for_each(validator => { let response = null; @@ -158,10 +224,16 @@ options.forEach(option => { config[option.key] = value; }); +let overrides = validate_overrides(config.limit_overrides, options) +errored = errored || !overrides; + if (errored) { process.exit(1); } +config.limit_overrides = overrides; +console.log(config.limit_overrides); + logger.info('Configuration successfully loaded'); module.exports = config; From 46ecd482bb0acd7a6bbf3559a13e0f4e68567a91 Mon Sep 17 00:00:00 2001 From: milindmadhukar <68477234+milindmadhukar@users.noreply.github.com> Date: Sat, 2 Oct 2021 23:38:11 +0530 Subject: [PATCH 02/72] Added Go-Piston to the Readme --- readme.md | 1 + 1 file changed, 1 insertion(+) diff --git a/readme.md b/readme.md index 02a3a03..868a743 100644 --- a/readme.md +++ b/readme.md @@ -83,6 +83,7 @@ The following are approved and endorsed extensions/utilities to the core Piston - [Node Piston Client](https://github.com/dthree/node-piston), a Node.js wrapper for accessing the Piston API. - [Piston4J](https://github.com/the-codeboy/Piston4J), a Java wrapper for accessing the Piston API. - [Pyston](https://github.com/ffaanngg/pyston), a Python wrapper for accessing the Piston API. +- [Go-Piston](https://github.com/milindmadhukar/go-piston), a Golang wrapper for accessing the Piston API.
From 78d0ce156fa60326e86d1df92f0e29433a3a6791 Mon Sep 17 00:00:00 2001 From: "Vargas, Dan" Date: Fri, 1 Oct 2021 14:16:30 -0500 Subject: [PATCH 03/72] prevent building empty package on pr --- .github/workflows/package-pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/package-pr.yaml b/.github/workflows/package-pr.yaml index bb264a3..cb646d3 100644 --- a/.github/workflows/package-pr.yaml +++ b/.github/workflows/package-pr.yaml @@ -53,7 +53,7 @@ jobs: - name: Build Packages run: | - PACKAGES=$(jq '.[]' -r ${HOME}/files.json | awk -F/ '{ print $2 "-" $3 }' | sort -u) + PACKAGES=$(jq '.[]' -r ${HOME}/files.json | awk -F/ '$2 && $3{ print $2 "-" $3 }' | sort -u) echo "Packages: $PACKAGES" docker pull docker.pkg.github.com/engineer-man/piston/repo-builder:latest docker build -t repo-builder repo From 94af5639bf5130c91aaa260cdf0fb151b04f73ce Mon Sep 17 00:00:00 2001 From: Brikaa Date: Sat, 2 Oct 2021 14:08:36 +0200 Subject: [PATCH 04/72] Add per-language constraint overrides --- api/src/api/v2.js | 93 ++++++++++++++------------------ api/src/config.js | 22 ++++---- api/src/job.js | 37 ++++++------- api/src/runtime.js | 48 ++++++++++++++++- packages/dotnet/5.0.201/build.sh | 0 5 files changed, 112 insertions(+), 88 deletions(-) mode change 100644 => 100755 packages/dotnet/5.0.201/build.sh diff --git a/api/src/api/v2.js b/api/src/api/v2.js index e0abacd..a3571e1 100644 --- a/api/src/api/v2.js +++ b/api/src/api/v2.js @@ -3,7 +3,6 @@ const router = express.Router(); const events = require('events'); -const config = require('../config'); const runtime = require('../runtime'); const { Job } = require('../job'); const package = require('../package'); @@ -13,7 +12,7 @@ const SIGNALS = ["SIGABRT","SIGALRM","SIGBUS","SIGCHLD","SIGCLD","SIGCONT","SIGE // ref: https://man7.org/linux/man-pages/man7/signal.7.html function get_job(body){ - const { + let { language, version, args, @@ -31,19 +30,16 @@ function get_job(body){ message: 'language is required as a string', }); } - if (!version || typeof version !== 'string') { return reject({ message: 'version is required as a string', }); } - if (!files || !Array.isArray(files)) { return reject({ message: 'files is required as an array', }); } - for (const [i, file] of files.entries()) { if (typeof file.content !== 'string') { return reject({ @@ -52,73 +48,64 @@ function get_job(body){ } } - if (compile_memory_limit) { - if (typeof compile_memory_limit !== 'number') { - return reject({ - message: 'if specified, compile_memory_limit must be a number', - }); - } - - if ( - config.compile_memory_limit >= 0 && - (compile_memory_limit > config.compile_memory_limit || - compile_memory_limit < 0) - ) { - return reject({ - message: - 'compile_memory_limit cannot exceed the configured limit of ' + - config.compile_memory_limit, - }); - } - } - - if (run_memory_limit) { - if (typeof run_memory_limit !== 'number') { - return reject({ - message: 'if specified, run_memory_limit must be a number', - }); - } - - if ( - config.run_memory_limit >= 0 && - (run_memory_limit > config.run_memory_limit || run_memory_limit < 0) - ) { - return reject({ - message: - 'run_memory_limit cannot exceed the configured limit of ' + - config.run_memory_limit, - }); - } - } - const rt = runtime.get_latest_runtime_matching_language_version( language, version ); - if (rt === undefined) { return reject({ message: `${language}-${version} runtime is unknown`, }); } + for (let constraint of ['memory_limit', 'timeout']) { + for (let type of ['compile', 'run']) { + let constraint_name = `${type}_${constraint}`; + let constraint_value = body[constraint_name]; + let configured_limit = rt[`${constraint}s`][type]; + if (!constraint_value) { + continue; + } + if (typeof constraint_value !== 'number') { + return reject({ + message: `If specified, ${constraint_name} must be a number` + }); + } + if (configured_limit <= 0) { + continue; + } + if (constraint_value > configured_limit) { + return reject({ + message: `${constraint_name} cannot exceed the configured limit of ${configured_limit}` + }); + } + if (constraint_value < 0) { + return reject({ + message: `${constraint_name} must be non-negative` + }); + } + } + } + + compile_timeout = compile_timeout || rt.timeouts.compile; + run_timeout = run_timeout || rt.timeouts.run; + compile_memory_limit = compile_memory_limit || rt.memory_limits.compile; + run_timeout = run_timeout || rt.timeouts.run; resolve(new Job({ runtime: rt, - alias: language, args: args || [], stdin: stdin || "", files, timeouts: { - run: run_timeout || config.run_timeout, - compile: compile_timeout || config.compile_timeout, + run: run_timeout, + compile: compile_timeout, }, memory_limits: { - run: run_memory_limit || config.run_memory_limit, - compile: compile_memory_limit || config.compile_memory_limit, + run: run_memory_limit, + compile: compile_memory_limit, } })); - }) - + }); } router.use((req, res, next) => { @@ -228,7 +215,7 @@ router.post('/execute', async (req, res) => { return res.status(200).send(result); }catch(error){ - return res.status(400).json(error.to_string()); + return res.status(400).json(error); } }); diff --git a/api/src/config.js b/api/src/config.js index 162c9d6..c191644 100644 --- a/api/src/config.js +++ b/api/src/config.js @@ -24,7 +24,7 @@ function validate_overrides(overrides, options) { logger.error(`Invalid overridden option: ${key}`); return false; } - let option = options.find((o) => o.key == key); + let option = options.find((o) => o.key === key); let parser = option.parser; let raw = overrides[language][key]; let value = parser(raw); @@ -38,8 +38,10 @@ function validate_overrides(overrides, options) { } overrides[language][key] = value; } + // Modifies the reference + options[options.index_of(options.find((o) => o.key === 'limit_overrides'))] = overrides; } - return overrides; + return true; } const options = [ @@ -184,8 +186,10 @@ const options = [ run_memory_limit, compile_timeout, run_timeout, output_max_size', default: {}, parser: parse_overrides, - validators: [(x) => !!x || `Invalid JSON format for the overrides\n${x}`] - // More validation is done after the configs are loaded + validators: [ + (x) => !!x || `Invalid JSON format for the overrides\n${x}`, + (overrides, _, options) => validate_overrides(overrides, options) || `Failed to validate the overrides` + ] } ]; @@ -208,8 +212,8 @@ options.forEach(option => { option.validators.for_each(validator => { let response = null; - if (env_val) response = validator(parsed_val, env_val); - else response = validator(value, value); + if (env_val) response = validator(parsed_val, env_val, options); + else response = validator(value, value, options); if (response !== true) { errored = true; @@ -224,16 +228,10 @@ options.forEach(option => { config[option.key] = value; }); -let overrides = validate_overrides(config.limit_overrides, options) -errored = errored || !overrides; - if (errored) { process.exit(1); } -config.limit_overrides = overrides; -console.log(config.limit_overrides); - logger.info('Configuration successfully loaded'); module.exports = config; diff --git a/api/src/job.js b/api/src/job.js index ecc4ab3..d4d19d9 100644 --- a/api/src/job.js +++ b/api/src/job.js @@ -30,7 +30,7 @@ setInterval(()=>{ class Job { - constructor({ runtime, files, args, stdin, timeouts, memory_limits }) { + constructor({ runtime, files, args, stdin }) { this.uuid = uuidv4(); this.runtime = runtime; this.files = files.map((file, i) => ({ @@ -40,8 +40,6 @@ class Job { this.args = args; this.stdin = stdin; - this.timeouts = timeouts; - this.memory_limits = memory_limits; this.uid = config.runner_uid_min + uid; this.gid = config.runner_gid_min + gid; @@ -102,9 +100,9 @@ class Job { const prlimit = [ 'prlimit', - '--nproc=' + config.max_process_count, - '--nofile=' + config.max_open_files, - '--fsize=' + config.max_file_size, + '--nproc=' + this.runtime.max_process_count , + '--nofile=' + this.runtime.max_open_files , + '--fsize=' + this.runtime.max_file_size , ]; if (memory_limit >= 0) { @@ -142,8 +140,6 @@ class Job { proc.kill(signal) }) } - - const kill_timeout = set_timeout( async _ => { @@ -156,7 +152,7 @@ class Job { proc.stderr.on('data', async data => { if(eventBus !== null) { eventBus.emit("stderr", data); - } else if (stderr.length > config.output_max_size) { + } else if (stderr.length > this.runtime.output_max_size) { logger.info(`stderr length exceeded uuid=${this.uuid}`) process.kill(proc.pid, 'SIGKILL') } else { @@ -168,7 +164,7 @@ class Job { proc.stdout.on('data', async data => { if(eventBus !== null){ eventBus.emit("stdout", data); - } else if (stdout.length > config.output_max_size) { + } else if (stdout.length > this.runtime.output_max_size) { logger.info(`stdout length exceeded uuid=${this.uuid}`) process.kill(proc.pid, 'SIGKILL') } else { @@ -223,8 +219,8 @@ class Job { compile = await this.safe_call( path.join(this.runtime.pkgdir, 'compile'), this.files.map(x => x.name), - this.timeouts.compile, - this.memory_limits.compile + this.runtime.timeouts.compile, + this.runtime.memory_limits.compile ); } @@ -233,8 +229,8 @@ class Job { const run = await this.safe_call( path.join(this.runtime.pkgdir, 'run'), [this.files[0].name, ...this.args], - this.timeouts.run, - this.memory_limits.run + this.runtime.timeouts.run, + this.runtime.memory_limits.run ); this.state = job_states.EXECUTED; @@ -266,8 +262,8 @@ class Job { const {error, code, signal} = await this.safe_call( path.join(this.runtime.pkgdir, 'compile'), this.files.map(x => x.name), - this.timeouts.compile, - this.memory_limits.compile, + this.runtime.timeouts.compile, + this.runtime.memory_limits.compile, eventBus ) @@ -279,14 +275,14 @@ class Job { const {error, code, signal} = await this.safe_call( path.join(this.runtime.pkgdir, 'run'), [this.files[0].name, ...this.args], - this.timeouts.run, - this.memory_limits.run, + this.runtime.timeouts.run, + this.runtime.memory_limits.run, eventBus ); eventBus.emit("exit", "run", {error, code, signal}) - + this.state = job_states.EXECUTED; } @@ -308,8 +304,7 @@ class Job { const proc_lines = proc_status.to_string().split("\n") const uid_line = proc_lines.find(line=>line.starts_with("Uid:")) const [_, ruid, euid, suid, fuid] = uid_line.split(/\s+/); - - + if(ruid == this.uid || euid == this.uid) return parse_int(proc_id) diff --git a/api/src/runtime.js b/api/src/runtime.js index 191fc5d..60d3c23 100644 --- a/api/src/runtime.js +++ b/api/src/runtime.js @@ -8,12 +8,54 @@ const path = require('path'); const runtimes = []; class Runtime { - constructor({ language, version, aliases, pkgdir, runtime }) { + constructor({ + language, version, aliases, pkgdir, runtime, timeouts, memory_limits, max_process_count, + max_open_files, max_file_size, output_max_size + }) { this.language = language; this.version = version; this.aliases = aliases || []; this.pkgdir = pkgdir; this.runtime = runtime; + this.timeouts = timeouts; + this.memory_limits = memory_limits; + this.max_process_count = max_process_count; + this.max_open_files = max_open_files; + this.max_file_size = max_file_size; + this.output_max_size = output_max_size; + } + + static compute_single_limit(language_name, limit_name, language_limit_overrides) { + return ( + config.limit_overrides[language_name] && config.limit_overrides[language_name][limit_name] + || language_limit_overrides && language_limit_overrides[limit_name] + || config[limit_name] + ); + } + + static compute_all_limits(language_name, language_limit_overrides) { + return { + timeouts: { + compile: + this.compute_single_limit(language_name, 'compile_timeout', language_limit_overrides), + run: + this.compute_single_limit(language_name, 'run_timeout', language_limit_overrides) + }, + memory_limits: { + compile: + this.compute_single_limit(language_name, 'compile_memory_limit', language_limit_overrides), + run: + this.compute_single_limit(language_name, 'run_memory_limit', language_limit_overrides) + }, + max_process_count: + this.compute_single_limit(language_name, 'max_process_count', language_limit_overrides), + max_open_files: + this.compute_single_limit(language_name, 'max_open_files', language_limit_overrides), + max_file_size: + this.compute_single_limit(language_name, 'max_file_size', language_limit_overrides), + output_max_size: + this.compute_single_limit(language_name, 'output_max_size', language_limit_overrides), + } } static load_package(package_dir) { @@ -21,7 +63,7 @@ class Runtime { fss.read_file_sync(path.join(package_dir, 'pkg-info.json')) ); - let { language, version, build_platform, aliases, provides } = info; + let { language, version, build_platform, aliases, provides, limit_overrides } = info; version = semver.parse(version); if (build_platform !== globals.platform) { @@ -41,6 +83,7 @@ class Runtime { version, pkgdir: package_dir, runtime: language, + ...Runtime.compute_all_limits(lang.language, lang.limit_overrides) }) ); }); @@ -51,6 +94,7 @@ class Runtime { version, aliases, pkgdir: package_dir, + ...Runtime.compute_all_limits(language, limit_overrides) }) ); } diff --git a/packages/dotnet/5.0.201/build.sh b/packages/dotnet/5.0.201/build.sh old mode 100644 new mode 100755 From c24937aeb7ca619e74efd63fd7fa873b266e3377 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Thu, 30 Sep 2021 08:11:47 +1300 Subject: [PATCH 05/72] Add self to license --- license | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/license b/license index 4f45aea..fd203f8 100644 --- a/license +++ b/license @@ -1,4 +1,4 @@ -Copyright (c) 2018-2021 Brian Seymour, EMKC Contributors +Copyright (c) 2018-2021 Brian Seymour, Thomas Hobson, EMKC Contributors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal From 5d392effccd7da2584bfee4ef8cd6d565deff8a5 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Fri, 1 Oct 2021 20:28:54 +1300 Subject: [PATCH 06/72] api: maximum concurrent jobs and potential fix for gcc --- api/src/config.js | 7 +++++++ api/src/job.js | 38 +++++++++++++++++++++++++++++++------- docs/configuration.md | 9 +++++++++ 3 files changed, 47 insertions(+), 7 deletions(-) diff --git a/api/src/config.js b/api/src/config.js index 84270aa..bbd7ae9 100644 --- a/api/src/config.js +++ b/api/src/config.js @@ -114,6 +114,13 @@ const options = [ 'https://github.com/engineer-man/piston/releases/download/pkgs/index', validators: [], }, + { + key: 'max_concurrent_jobs', + desc: 'Maximum number of concurrent jobs to run at one time', + default: 64, + parser: parse_int, + validators: [(x) => x > 0 || `${x} cannot be negative`] + } ]; logger.info(`Loading Configuration from environment`); diff --git a/api/src/job.js b/api/src/job.js index 683cda6..712dcd8 100644 --- a/api/src/job.js +++ b/api/src/job.js @@ -16,6 +16,19 @@ const job_states = { let uid = 0; let gid = 0; +let remainingJobSpaces = config.max_concurrent_jobs; +let jobQueue = []; + + +setInterval(()=>{ + // Every 10ms try resolve a new job, if there is an available slot + if(jobQueue.length > 0 && remainingJobSpaces > 0){ + jobQueue.shift()() + } +}, 10) + + + class Job { constructor({ runtime, files, args, stdin, timeouts, memory_limits }) { this.uuid = uuidv4(); @@ -48,8 +61,15 @@ class Job { } async prime() { - logger.info(`Priming job uuid=${this.uuid}`); + if(remainingJobSpaces < 1){ + logger.info(`Awaiting job slot uuid=${this.uuid}`) + await new Promise((resolve)=>{ + jobQueue.push(resolve) + }) + } + logger.info(`Priming job uuid=${this.uuid}`); + remainingJobSpaces--; logger.debug('Writing files to job cache'); logger.debug(`Transfering ownership uid=${this.uid} gid=${this.gid}`); @@ -152,21 +172,23 @@ class Job { } }); - const exit_cleanup = () => { + const exit_cleanup = async () => { clear_timeout(kill_timeout); proc.stderr.destroy(); proc.stdout.destroy(); + + await this.cleanup_processes() }; - proc.on('exit', (code, signal) => { - exit_cleanup(); + proc.on('exit', async (code, signal) => { + await exit_cleanup(); resolve({stdout, stderr, code, signal, output }); }); - proc.on('error', err => { - exit_cleanup(); + proc.on('error', async err => { + await exit_cleanup(); reject({ error: err, stdout, stderr, output }); }); @@ -339,11 +361,13 @@ class Job { async cleanup() { logger.info(`Cleaning up job uuid=${this.uuid}`); - await this.cleanup_processes(); await this.cleanup_filesystem(); + + remainingJobSpaces++; } } + module.exports = { Job, }; diff --git a/docs/configuration.md b/docs/configuration.md index 1388e9d..16a5df0 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -145,3 +145,12 @@ default: https://github.com/engineer-man/piston/releases/download/pkgs/index ``` URL for repository index, where packages will be downloaded from. + +## Maximum Concurrent Jobs + +```yaml +key: PISTON_MAX_CONCURRENT_JOBS +default: 64 +``` + +Maximum number of jobs to run concurrently. From 36d72383a53ebf752fb0ef28a22db422f668f987 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sat, 2 Oct 2021 00:07:37 +1300 Subject: [PATCH 07/72] rework process janitor Old process janitor required starting a `ps` process. This was problematic, as `ps` requires another entry in the process table, which in some cases was impossible as it was exhausted. --- api/src/job.js | 74 ++++++++++++++++++++++++++++++++------------------ 1 file changed, 47 insertions(+), 27 deletions(-) diff --git a/api/src/job.js b/api/src/job.js index 712dcd8..ecc4ab3 100644 --- a/api/src/job.js +++ b/api/src/job.js @@ -146,26 +146,31 @@ class Job { const kill_timeout = set_timeout( - _ => proc.kill('SIGKILL'), + async _ => { + logger.info(`Timeout exceeded timeout=${timeout} uuid=${this.uuid}`) + process.kill(proc.pid, 'SIGKILL') + }, timeout ); - proc.stderr.on('data', data => { + proc.stderr.on('data', async data => { if(eventBus !== null) { eventBus.emit("stderr", data); } else if (stderr.length > config.output_max_size) { - proc.kill('SIGKILL'); + logger.info(`stderr length exceeded uuid=${this.uuid}`) + process.kill(proc.pid, 'SIGKILL') } else { stderr += data; output += data; } }); - proc.stdout.on('data', data => { + proc.stdout.on('data', async data => { if(eventBus !== null){ eventBus.emit("stdout", data); } else if (stdout.length > config.output_max_size) { - proc.kill('SIGKILL'); + logger.info(`stdout length exceeded uuid=${this.uuid}`) + process.kill(proc.pid, 'SIGKILL') } else { stdout += data; output += data; @@ -179,6 +184,7 @@ class Job { proc.stdout.destroy(); await this.cleanup_processes() + logger.debug(`Finished exit cleanup uuid=${this.uuid}`) }; proc.on('exit', async (code, signal) => { @@ -284,36 +290,47 @@ class Job { this.state = job_states.EXECUTED; } - async cleanup_processes() { + async cleanup_processes(dont_wait = []) { let processes = [1]; + logger.debug(`Cleaning up processes uuid=${this.uuid}`) while (processes.length > 0) { - processes = await new Promise((resolve, reject) => - cp.execFile('ps', ['awwxo', 'pid,ruid'], (err, stdout) => { - if (err === null) { - const lines = stdout.split('\n').slice(1); //Remove header with slice - const procs = lines.map(line => { - const [pid, ruid] = line - .trim() - .split(/\s+/) - .map(n => parseInt(n)); + processes = [] - return { pid, ruid }; - }); - resolve(procs); - } else { - reject(error); - } - }) - ); + const proc_ids = await fs.readdir("/proc"); + + + processes = await Promise.all(proc_ids.map(async (proc_id) => { + if(isNaN(proc_id)) return -1; + try{ + const proc_status = await fs.read_file(path.join("/proc",proc_id,"status")); + const proc_lines = proc_status.to_string().split("\n") + const uid_line = proc_lines.find(line=>line.starts_with("Uid:")) + const [_, ruid, euid, suid, fuid] = uid_line.split(/\s+/); + + + if(ruid == this.uid || euid == this.uid) + return parse_int(proc_id) + + }catch{ + return -1 + } + + return -1 + })) + + processes = processes.filter(p => p > 0) + + if(processes.length > 0) + logger.debug(`Got processes to kill: ${processes} uuid=${this.uuid}`) + - processes = processes.filter(proc => proc.ruid === this.uid); for (const proc of processes) { // First stop the processes, but keep their resources allocated so they cant re-fork try { - process.kill(proc.pid, 'SIGSTOP'); + process.kill(proc, 'SIGSTOP'); } catch { // Could already be dead } @@ -322,14 +339,17 @@ class Job { for (const proc of processes) { // Then clear them out of the process tree try { - process.kill(proc.pid, 'SIGKILL'); + process.kill(proc, 'SIGKILL'); } catch { // Could already be dead and just needs to be waited on } - wait_pid(proc.pid); + if(!dont_wait.includes(proc)) + wait_pid(proc); } } + + logger.debug(`Cleaned up processes uuid=${this.uuid}`) } async cleanup_filesystem() { From 241b56a5e9014a1f5d0c863293b0319e1a339bbb Mon Sep 17 00:00:00 2001 From: Brikaa Date: Sun, 26 Sep 2021 14:02:03 +0200 Subject: [PATCH 08/72] Add semantic versioning in CONTRIBUTING.MD --- packages/CONTRIBUTING.MD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/CONTRIBUTING.MD b/packages/CONTRIBUTING.MD index 813f71e..b1ed6d3 100644 --- a/packages/CONTRIBUTING.MD +++ b/packages/CONTRIBUTING.MD @@ -2,7 +2,7 @@ ## Naming Languages -Languages should be named after their interpreters, and the command line binaries you call. +Languages should be named after their interpreters, and the command line binaries you call. The language version should use semantic versioning. For example, the full name of the standard python interpreter is `CPython`, however we would name it `python`, after the main binary which it provides. In the example of NodeJS, we would call this `node`, after the main binary. From b36ce650dd7fc70bcfcfad336dc1efca56af54f8 Mon Sep 17 00:00:00 2001 From: Brikaa Date: Sun, 26 Sep 2021 14:09:25 +0200 Subject: [PATCH 09/72] Add ./piston logs --- piston | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/piston b/piston index 28e18db..a14e7f5 100755 --- a/piston +++ b/piston @@ -19,6 +19,7 @@ case $1 in echo "Commands:" echo " select Select the environment" echo " docker_compose Interact directly with the docker-compose for the selected environment" + echo " logs Show docker-compose logs" echo echo " start Starts piston" echo " stop Stops piston" @@ -37,18 +38,19 @@ case $1 in echo " clean-repo Remove all packages from local repo" echo " build-pkg Build a package" echo " rebuild Build and restart the docker container" - + else echo " Switch to developement environment for more info" echo " > piston select dev" - + fi ;; select) echo "$2" > .piston_env ;; docker_compose) shift; docker_compose "$@";; + logs) docker_compose logs -f ;; restart) docker_compose restart ;; start) docker_compose up -d ;; From e5732ef459dc04ade6e2a145bddd14682f068b1e Mon Sep 17 00:00:00 2001 From: Dan Vargas Date: Fri, 6 Aug 2021 13:30:48 -0600 Subject: [PATCH 10/72] pkg(forte-1.0.0): add forte --- packages/forte/1.0.0/build.sh | 15 +++++++++++++++ packages/forte/1.0.0/environment | 4 ++++ packages/forte/1.0.0/metadata.json | 5 +++++ packages/forte/1.0.0/run | 3 +++ packages/forte/1.0.0/test.forte | 2 ++ readme.md | 1 + 6 files changed, 30 insertions(+) create mode 100755 packages/forte/1.0.0/build.sh create mode 100644 packages/forte/1.0.0/environment create mode 100644 packages/forte/1.0.0/metadata.json create mode 100644 packages/forte/1.0.0/run create mode 100644 packages/forte/1.0.0/test.forte diff --git a/packages/forte/1.0.0/build.sh b/packages/forte/1.0.0/build.sh new file mode 100755 index 0000000..b3e39a8 --- /dev/null +++ b/packages/forte/1.0.0/build.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +# the forter interpreter requries ruby +source ../../ruby/3.0.1/build.sh + +mkdir -p build + +git clone -q "https://github.com/judofyr/forter" build/forter +cd build/forter + +mv bin/* ../../bin/ +mv lib/* ../../lib/ + +cd ../../ +rm -rf build diff --git a/packages/forte/1.0.0/environment b/packages/forte/1.0.0/environment new file mode 100644 index 0000000..e75919d --- /dev/null +++ b/packages/forte/1.0.0/environment @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +# interpreter location +export PATH=$PWD/bin:$PATH diff --git a/packages/forte/1.0.0/metadata.json b/packages/forte/1.0.0/metadata.json new file mode 100644 index 0000000..fd4ec12 --- /dev/null +++ b/packages/forte/1.0.0/metadata.json @@ -0,0 +1,5 @@ +{ + "language": "forte", + "version": "1.0.0", + "aliases": ["forter"] +} diff --git a/packages/forte/1.0.0/run b/packages/forte/1.0.0/run new file mode 100644 index 0000000..79ee95b --- /dev/null +++ b/packages/forte/1.0.0/run @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +forter "$@" diff --git a/packages/forte/1.0.0/test.forte b/packages/forte/1.0.0/test.forte new file mode 100644 index 0000000..7482949 --- /dev/null +++ b/packages/forte/1.0.0/test.forte @@ -0,0 +1,2 @@ +1 PRINT "OK" +2 END diff --git a/readme.md b/readme.md index 95fdb07..2d8cb5d 100644 --- a/readme.md +++ b/readme.md @@ -338,6 +338,7 @@ Content-Type: application/json `elixir`, `emacs`, `erlang`, +`forte`, `fortran`, `go`, `golfscript`, From 883d584c15f2566ffd221c1f013027c87fcebf79 Mon Sep 17 00:00:00 2001 From: Brikaa Date: Sun, 26 Sep 2021 13:25:13 +0200 Subject: [PATCH 11/72] pkg(iverilog-11.0.0): Added iverilog 11.0.0 --- api/src/api/v2.js | 4 ++-- packages/iverilog/11.0.0/build.sh | 17 +++++++++++++++++ packages/iverilog/11.0.0/compile | 4 ++++ packages/iverilog/11.0.0/environment | 2 ++ packages/iverilog/11.0.0/metadata.json | 5 +++++ packages/iverilog/11.0.0/run | 4 ++++ packages/iverilog/11.0.0/test.verilog | 7 +++++++ readme.md | 1 + repo/Dockerfile | 2 +- 9 files changed, 43 insertions(+), 3 deletions(-) create mode 100755 packages/iverilog/11.0.0/build.sh create mode 100644 packages/iverilog/11.0.0/compile create mode 100644 packages/iverilog/11.0.0/environment create mode 100644 packages/iverilog/11.0.0/metadata.json create mode 100644 packages/iverilog/11.0.0/run create mode 100644 packages/iverilog/11.0.0/test.verilog diff --git a/api/src/api/v2.js b/api/src/api/v2.js index 215453b..e3e0522 100644 --- a/api/src/api/v2.js +++ b/api/src/api/v2.js @@ -146,7 +146,7 @@ router.ws('/connect', async (ws, req) => { eventBus.on("exit", (stage, status) => ws.send(JSON.stringify({type: "exit", stage, ...status}))) ws.on("message", async (data) => { - + try{ const msg = JSON.parse(data); @@ -194,7 +194,7 @@ router.ws('/connect', async (ws, req) => { } break; } - + }catch(error){ ws.send(JSON.stringify({type: "error", message: error.message})) ws.close(4002, "Notified Error") diff --git a/packages/iverilog/11.0.0/build.sh b/packages/iverilog/11.0.0/build.sh new file mode 100755 index 0000000..befb2fa --- /dev/null +++ b/packages/iverilog/11.0.0/build.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +PREFIX=$(realpath $(dirname $0)) + +mkdir -p build/iverilog +cd build/iverilog +curl -L https://github.com/steveicarus/iverilog/archive/refs/tags/v11_0.tar.gz -o iverilog.tar.gz +tar xzf iverilog.tar.gz --strip-components=1 + +chmod +x ./autoconf.sh +./autoconf.sh +./configure --prefix="$PREFIX" +make -j$(nproc) +make install -j$(nproc) + +cd ../../ +rm -rf build diff --git a/packages/iverilog/11.0.0/compile b/packages/iverilog/11.0.0/compile new file mode 100644 index 0000000..56f4b4e --- /dev/null +++ b/packages/iverilog/11.0.0/compile @@ -0,0 +1,4 @@ +#!/bin/bash + +rename 's/$/\.v/' "$@" # Add .v extension +iverilog *.v diff --git a/packages/iverilog/11.0.0/environment b/packages/iverilog/11.0.0/environment new file mode 100644 index 0000000..b482830 --- /dev/null +++ b/packages/iverilog/11.0.0/environment @@ -0,0 +1,2 @@ +#!/bin/bash +export PATH=$PWD/bin:$PATH diff --git a/packages/iverilog/11.0.0/metadata.json b/packages/iverilog/11.0.0/metadata.json new file mode 100644 index 0000000..5a35bde --- /dev/null +++ b/packages/iverilog/11.0.0/metadata.json @@ -0,0 +1,5 @@ +{ + "language": "iverilog", + "version": "11.0.0", + "aliases": ["verilog", "vvp"] +} diff --git a/packages/iverilog/11.0.0/run b/packages/iverilog/11.0.0/run new file mode 100644 index 0000000..39e898c --- /dev/null +++ b/packages/iverilog/11.0.0/run @@ -0,0 +1,4 @@ +#!/bin/bash + +shift +vvp a.out "$@" diff --git a/packages/iverilog/11.0.0/test.verilog b/packages/iverilog/11.0.0/test.verilog new file mode 100644 index 0000000..88fcd7a --- /dev/null +++ b/packages/iverilog/11.0.0/test.verilog @@ -0,0 +1,7 @@ +module hello; + initial + begin + $display("OK"); + $finish ; + end +endmodule diff --git a/readme.md b/readme.md index 2d8cb5d..1240cbe 100644 --- a/readme.md +++ b/readme.md @@ -344,6 +344,7 @@ Content-Type: application/json `golfscript`, `groovy`, `haskell`, +`iverilog`, `java`, `javascript`, `jelly`, diff --git a/repo/Dockerfile b/repo/Dockerfile index 56ca59d..de28c11 100644 --- a/repo/Dockerfile +++ b/repo/Dockerfile @@ -9,7 +9,7 @@ RUN apt-get update && apt-get install -y unzip autoconf build-essential libssl-d libncursesw5-dev python3-pip libgmp-dev libmpfr-dev python2 libffi-dev gfortran\ libreadline-dev libblas-dev liblapack-dev libpcre3-dev libarpack2-dev libfftw3-dev \ libglpk-dev libqhull-dev libqrupdate-dev libsuitesparse-dev libsundials-dev \ - libbz2-dev liblzma-dev libpcre2-dev && \ + libbz2-dev liblzma-dev libpcre2-dev gperf bison flex g++ && \ ln -sf /bin/bash /bin/sh && \ rm -rf /var/lib/apt/lists/* && \ update-alternatives --install /usr/bin/python python /usr/bin/python3.7 2 From 4f5ef06adf5f75477766e858cef88a8eb1e603bc Mon Sep 17 00:00:00 2001 From: Dan Vargas Date: Fri, 17 Sep 2021 09:59:24 -0500 Subject: [PATCH 12/72] pkg(freebasic-1.8.0): Add Freebasic --- packages/freebasic/1.8.0/build.sh | 5 +++++ packages/freebasic/1.8.0/compile | 4 ++++ packages/freebasic/1.8.0/environment | 4 ++++ packages/freebasic/1.8.0/metadata.json | 5 +++++ packages/freebasic/1.8.0/run | 5 +++++ packages/freebasic/1.8.0/test.bas | 1 + readme.md | 1 + 7 files changed, 25 insertions(+) create mode 100755 packages/freebasic/1.8.0/build.sh create mode 100644 packages/freebasic/1.8.0/compile create mode 100644 packages/freebasic/1.8.0/environment create mode 100644 packages/freebasic/1.8.0/metadata.json create mode 100644 packages/freebasic/1.8.0/run create mode 100644 packages/freebasic/1.8.0/test.bas diff --git a/packages/freebasic/1.8.0/build.sh b/packages/freebasic/1.8.0/build.sh new file mode 100755 index 0000000..bd58488 --- /dev/null +++ b/packages/freebasic/1.8.0/build.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +curl -L "https://sourceforge.net/projects/fbc/files/FreeBASIC-1.08.0/Binaries-Linux/FreeBASIC-1.08.0-linux-x86_64.tar.gz/download" -o freebasic.tar.gz +tar xf freebasic.tar.gz --strip-components=1 +rm freebasic.tar.gz diff --git a/packages/freebasic/1.8.0/compile b/packages/freebasic/1.8.0/compile new file mode 100644 index 0000000..b836b3d --- /dev/null +++ b/packages/freebasic/1.8.0/compile @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +# Compile bas files +fbc -lang qb -b "$@" -x out diff --git a/packages/freebasic/1.8.0/environment b/packages/freebasic/1.8.0/environment new file mode 100644 index 0000000..144c737 --- /dev/null +++ b/packages/freebasic/1.8.0/environment @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +# Path to fbc compiler +export PATH=$PWD/bin:$PATH diff --git a/packages/freebasic/1.8.0/metadata.json b/packages/freebasic/1.8.0/metadata.json new file mode 100644 index 0000000..1dcf1ff --- /dev/null +++ b/packages/freebasic/1.8.0/metadata.json @@ -0,0 +1,5 @@ +{ + "language": "freebasic", + "version": "1.8.0", + "aliases": ["bas", "fbc", "basic", "qbasic", "quickbasic"] +} diff --git a/packages/freebasic/1.8.0/run b/packages/freebasic/1.8.0/run new file mode 100644 index 0000000..610d7e6 --- /dev/null +++ b/packages/freebasic/1.8.0/run @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +# Run output file from compile with arguments +shift +./out "$@" diff --git a/packages/freebasic/1.8.0/test.bas b/packages/freebasic/1.8.0/test.bas new file mode 100644 index 0000000..b13a0ef --- /dev/null +++ b/packages/freebasic/1.8.0/test.bas @@ -0,0 +1 @@ +PRINT "OK" \ No newline at end of file diff --git a/readme.md b/readme.md index 1240cbe..02a3a03 100644 --- a/readme.md +++ b/readme.md @@ -340,6 +340,7 @@ Content-Type: application/json `erlang`, `forte`, `fortran`, +`freebasic`, `go`, `golfscript`, `groovy`, From 1c48ca2ab576ca27fdc755a8188a12225e0172e0 Mon Sep 17 00:00:00 2001 From: Hydrazer Date: Sat, 2 Oct 2021 16:50:37 -0600 Subject: [PATCH 13/72] pkg(japt-2.0.0): add japt --- packages/japt/2.0.0/build.sh | 6 ++++++ packages/japt/2.0.0/environment | 5 +++++ packages/japt/2.0.0/metadata.json | 5 +++++ packages/japt/2.0.0/run | 4 ++++ packages/japt/2.0.0/test.japt | 1 + readme.md | 1 + 6 files changed, 22 insertions(+) create mode 100644 packages/japt/2.0.0/build.sh create mode 100644 packages/japt/2.0.0/environment create mode 100644 packages/japt/2.0.0/metadata.json create mode 100644 packages/japt/2.0.0/run create mode 100644 packages/japt/2.0.0/test.japt diff --git a/packages/japt/2.0.0/build.sh b/packages/japt/2.0.0/build.sh new file mode 100644 index 0000000..d31a5cc --- /dev/null +++ b/packages/japt/2.0.0/build.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +# japt install +source ../../node/16.3.0/build.sh + +git clone -q "https://github.com/Hydrazer/japt.git" japt \ No newline at end of file diff --git a/packages/japt/2.0.0/environment b/packages/japt/2.0.0/environment new file mode 100644 index 0000000..1fd5f14 --- /dev/null +++ b/packages/japt/2.0.0/environment @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +# node and japt path +export PATH=$PWD/bin:$PATH +export JAPT_PATH=$PWD/japt \ No newline at end of file diff --git a/packages/japt/2.0.0/metadata.json b/packages/japt/2.0.0/metadata.json new file mode 100644 index 0000000..7a3e5aa --- /dev/null +++ b/packages/japt/2.0.0/metadata.json @@ -0,0 +1,5 @@ +{ + "language": "japt", + "version": "2.0.0", + "aliases": ["japt"] +} \ No newline at end of file diff --git a/packages/japt/2.0.0/run b/packages/japt/2.0.0/run new file mode 100644 index 0000000..80649c4 --- /dev/null +++ b/packages/japt/2.0.0/run @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +# japt only takes filename and stdin +node "$JAPT_PATH"/node.js "$1" \ No newline at end of file diff --git a/packages/japt/2.0.0/test.japt b/packages/japt/2.0.0/test.japt new file mode 100644 index 0000000..d096585 --- /dev/null +++ b/packages/japt/2.0.0/test.japt @@ -0,0 +1 @@ +"OK \ No newline at end of file diff --git a/readme.md b/readme.md index 02a3a03..5b41e6a 100644 --- a/readme.md +++ b/readme.md @@ -346,6 +346,7 @@ Content-Type: application/json `groovy`, `haskell`, `iverilog`, +`japt`, `java`, `javascript`, `jelly`, From c0f203537c0e345d312eb4879c9dd2c7a9d09ff7 Mon Sep 17 00:00:00 2001 From: Brikaa Date: Fri, 1 Oct 2021 21:41:09 +0200 Subject: [PATCH 14/72] config.js: timeout, overrides --- api/src/api/v2.js | 6 ++-- api/src/config.js | 74 ++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 76 insertions(+), 4 deletions(-) diff --git a/api/src/api/v2.js b/api/src/api/v2.js index e3e0522..e0abacd 100644 --- a/api/src/api/v2.js +++ b/api/src/api/v2.js @@ -109,8 +109,8 @@ function get_job(body){ stdin: stdin || "", files, timeouts: { - run: run_timeout || 3000, - compile: compile_timeout || 10000, + run: run_timeout || config.run_timeout, + compile: compile_timeout || config.compile_timeout, }, memory_limits: { run: run_memory_limit || config.run_memory_limit, @@ -228,7 +228,7 @@ router.post('/execute', async (req, res) => { return res.status(200).send(result); }catch(error){ - return res.status(400).json(error); + return res.status(400).json(error.to_string()); } }); diff --git a/api/src/config.js b/api/src/config.js index bbd7ae9..162c9d6 100644 --- a/api/src/config.js +++ b/api/src/config.js @@ -2,6 +2,46 @@ const fss = require('fs'); const Logger = require('logplease'); const logger = Logger.create('config'); +function parse_overrides(overrides) { + try { + return JSON.parse(overrides); + } + catch (e) { + return null; + } +} + +function validate_overrides(overrides, options) { + for (let language in overrides) { + for (let key in overrides[language]) { + if ( + ![ + 'max_process_count', 'max_open_files', 'max_file_size', + 'compile_memory_limit', 'run_memory_limit', 'compile_timeout', + 'run_timeout', 'output_max_size' + ].includes(key) + ) { + logger.error(`Invalid overridden option: ${key}`); + return false; + } + let option = options.find((o) => o.key == key); + let parser = option.parser; + let raw = overrides[language][key]; + let value = parser(raw); + let validators = option.validators; + for (let validator of validators) { + let response = validator(value, raw); + if (response !== true) { + logger.error(`Failed to validate overridden option: ${key}`, response); + return false; + } + } + overrides[language][key] = value; + } + } + return overrides; +} + const options = [ { key: 'log_level', @@ -91,6 +131,22 @@ const options = [ parser: parse_int, validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`], }, + { + key: 'compile_timeout', + desc: + 'Max time allowed for compile stage in milliseconds', + default: 10000, // 10 seconds + parser: parse_int, + validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`], + }, + { + key: 'run_timeout', + desc: + 'Max time allowed for run stage in milliseconds', + default: 3000, // 3 seconds + parser: parse_int, + validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`], + }, { key: 'compile_memory_limit', desc: @@ -120,6 +176,16 @@ const options = [ default: 64, parser: parse_int, validators: [(x) => x > 0 || `${x} cannot be negative`] + }, + { + key: 'limit_overrides', + desc: 'Per-language exceptions in JSON format for each of:\ + max_process_count, max_open_files, max_file_size, compile_memory_limit,\ + run_memory_limit, compile_timeout, run_timeout, output_max_size', + default: {}, + parser: parse_overrides, + validators: [(x) => !!x || `Invalid JSON format for the overrides\n${x}`] + // More validation is done after the configs are loaded } ]; @@ -138,7 +204,7 @@ options.forEach(option => { const parsed_val = parser(env_val); - const value = env_val || option.default; + const value = parsed_val || option.default; option.validators.for_each(validator => { let response = null; @@ -158,10 +224,16 @@ options.forEach(option => { config[option.key] = value; }); +let overrides = validate_overrides(config.limit_overrides, options) +errored = errored || !overrides; + if (errored) { process.exit(1); } +config.limit_overrides = overrides; +console.log(config.limit_overrides); + logger.info('Configuration successfully loaded'); module.exports = config; From a5c38581002e34906acf78d9221cc8aa3f0a6c1f Mon Sep 17 00:00:00 2001 From: Brikaa Date: Sat, 2 Oct 2021 14:08:36 +0200 Subject: [PATCH 15/72] Add per-language constraint overrides --- api/src/api/v2.js | 93 ++++++++++++++------------------ api/src/config.js | 22 ++++---- api/src/job.js | 37 ++++++------- api/src/runtime.js | 48 ++++++++++++++++- packages/dotnet/5.0.201/build.sh | 0 5 files changed, 112 insertions(+), 88 deletions(-) mode change 100644 => 100755 packages/dotnet/5.0.201/build.sh diff --git a/api/src/api/v2.js b/api/src/api/v2.js index e0abacd..a3571e1 100644 --- a/api/src/api/v2.js +++ b/api/src/api/v2.js @@ -3,7 +3,6 @@ const router = express.Router(); const events = require('events'); -const config = require('../config'); const runtime = require('../runtime'); const { Job } = require('../job'); const package = require('../package'); @@ -13,7 +12,7 @@ const SIGNALS = ["SIGABRT","SIGALRM","SIGBUS","SIGCHLD","SIGCLD","SIGCONT","SIGE // ref: https://man7.org/linux/man-pages/man7/signal.7.html function get_job(body){ - const { + let { language, version, args, @@ -31,19 +30,16 @@ function get_job(body){ message: 'language is required as a string', }); } - if (!version || typeof version !== 'string') { return reject({ message: 'version is required as a string', }); } - if (!files || !Array.isArray(files)) { return reject({ message: 'files is required as an array', }); } - for (const [i, file] of files.entries()) { if (typeof file.content !== 'string') { return reject({ @@ -52,73 +48,64 @@ function get_job(body){ } } - if (compile_memory_limit) { - if (typeof compile_memory_limit !== 'number') { - return reject({ - message: 'if specified, compile_memory_limit must be a number', - }); - } - - if ( - config.compile_memory_limit >= 0 && - (compile_memory_limit > config.compile_memory_limit || - compile_memory_limit < 0) - ) { - return reject({ - message: - 'compile_memory_limit cannot exceed the configured limit of ' + - config.compile_memory_limit, - }); - } - } - - if (run_memory_limit) { - if (typeof run_memory_limit !== 'number') { - return reject({ - message: 'if specified, run_memory_limit must be a number', - }); - } - - if ( - config.run_memory_limit >= 0 && - (run_memory_limit > config.run_memory_limit || run_memory_limit < 0) - ) { - return reject({ - message: - 'run_memory_limit cannot exceed the configured limit of ' + - config.run_memory_limit, - }); - } - } - const rt = runtime.get_latest_runtime_matching_language_version( language, version ); - if (rt === undefined) { return reject({ message: `${language}-${version} runtime is unknown`, }); } + for (let constraint of ['memory_limit', 'timeout']) { + for (let type of ['compile', 'run']) { + let constraint_name = `${type}_${constraint}`; + let constraint_value = body[constraint_name]; + let configured_limit = rt[`${constraint}s`][type]; + if (!constraint_value) { + continue; + } + if (typeof constraint_value !== 'number') { + return reject({ + message: `If specified, ${constraint_name} must be a number` + }); + } + if (configured_limit <= 0) { + continue; + } + if (constraint_value > configured_limit) { + return reject({ + message: `${constraint_name} cannot exceed the configured limit of ${configured_limit}` + }); + } + if (constraint_value < 0) { + return reject({ + message: `${constraint_name} must be non-negative` + }); + } + } + } + + compile_timeout = compile_timeout || rt.timeouts.compile; + run_timeout = run_timeout || rt.timeouts.run; + compile_memory_limit = compile_memory_limit || rt.memory_limits.compile; + run_timeout = run_timeout || rt.timeouts.run; resolve(new Job({ runtime: rt, - alias: language, args: args || [], stdin: stdin || "", files, timeouts: { - run: run_timeout || config.run_timeout, - compile: compile_timeout || config.compile_timeout, + run: run_timeout, + compile: compile_timeout, }, memory_limits: { - run: run_memory_limit || config.run_memory_limit, - compile: compile_memory_limit || config.compile_memory_limit, + run: run_memory_limit, + compile: compile_memory_limit, } })); - }) - + }); } router.use((req, res, next) => { @@ -228,7 +215,7 @@ router.post('/execute', async (req, res) => { return res.status(200).send(result); }catch(error){ - return res.status(400).json(error.to_string()); + return res.status(400).json(error); } }); diff --git a/api/src/config.js b/api/src/config.js index 162c9d6..c191644 100644 --- a/api/src/config.js +++ b/api/src/config.js @@ -24,7 +24,7 @@ function validate_overrides(overrides, options) { logger.error(`Invalid overridden option: ${key}`); return false; } - let option = options.find((o) => o.key == key); + let option = options.find((o) => o.key === key); let parser = option.parser; let raw = overrides[language][key]; let value = parser(raw); @@ -38,8 +38,10 @@ function validate_overrides(overrides, options) { } overrides[language][key] = value; } + // Modifies the reference + options[options.index_of(options.find((o) => o.key === 'limit_overrides'))] = overrides; } - return overrides; + return true; } const options = [ @@ -184,8 +186,10 @@ const options = [ run_memory_limit, compile_timeout, run_timeout, output_max_size', default: {}, parser: parse_overrides, - validators: [(x) => !!x || `Invalid JSON format for the overrides\n${x}`] - // More validation is done after the configs are loaded + validators: [ + (x) => !!x || `Invalid JSON format for the overrides\n${x}`, + (overrides, _, options) => validate_overrides(overrides, options) || `Failed to validate the overrides` + ] } ]; @@ -208,8 +212,8 @@ options.forEach(option => { option.validators.for_each(validator => { let response = null; - if (env_val) response = validator(parsed_val, env_val); - else response = validator(value, value); + if (env_val) response = validator(parsed_val, env_val, options); + else response = validator(value, value, options); if (response !== true) { errored = true; @@ -224,16 +228,10 @@ options.forEach(option => { config[option.key] = value; }); -let overrides = validate_overrides(config.limit_overrides, options) -errored = errored || !overrides; - if (errored) { process.exit(1); } -config.limit_overrides = overrides; -console.log(config.limit_overrides); - logger.info('Configuration successfully loaded'); module.exports = config; diff --git a/api/src/job.js b/api/src/job.js index ecc4ab3..d4d19d9 100644 --- a/api/src/job.js +++ b/api/src/job.js @@ -30,7 +30,7 @@ setInterval(()=>{ class Job { - constructor({ runtime, files, args, stdin, timeouts, memory_limits }) { + constructor({ runtime, files, args, stdin }) { this.uuid = uuidv4(); this.runtime = runtime; this.files = files.map((file, i) => ({ @@ -40,8 +40,6 @@ class Job { this.args = args; this.stdin = stdin; - this.timeouts = timeouts; - this.memory_limits = memory_limits; this.uid = config.runner_uid_min + uid; this.gid = config.runner_gid_min + gid; @@ -102,9 +100,9 @@ class Job { const prlimit = [ 'prlimit', - '--nproc=' + config.max_process_count, - '--nofile=' + config.max_open_files, - '--fsize=' + config.max_file_size, + '--nproc=' + this.runtime.max_process_count , + '--nofile=' + this.runtime.max_open_files , + '--fsize=' + this.runtime.max_file_size , ]; if (memory_limit >= 0) { @@ -142,8 +140,6 @@ class Job { proc.kill(signal) }) } - - const kill_timeout = set_timeout( async _ => { @@ -156,7 +152,7 @@ class Job { proc.stderr.on('data', async data => { if(eventBus !== null) { eventBus.emit("stderr", data); - } else if (stderr.length > config.output_max_size) { + } else if (stderr.length > this.runtime.output_max_size) { logger.info(`stderr length exceeded uuid=${this.uuid}`) process.kill(proc.pid, 'SIGKILL') } else { @@ -168,7 +164,7 @@ class Job { proc.stdout.on('data', async data => { if(eventBus !== null){ eventBus.emit("stdout", data); - } else if (stdout.length > config.output_max_size) { + } else if (stdout.length > this.runtime.output_max_size) { logger.info(`stdout length exceeded uuid=${this.uuid}`) process.kill(proc.pid, 'SIGKILL') } else { @@ -223,8 +219,8 @@ class Job { compile = await this.safe_call( path.join(this.runtime.pkgdir, 'compile'), this.files.map(x => x.name), - this.timeouts.compile, - this.memory_limits.compile + this.runtime.timeouts.compile, + this.runtime.memory_limits.compile ); } @@ -233,8 +229,8 @@ class Job { const run = await this.safe_call( path.join(this.runtime.pkgdir, 'run'), [this.files[0].name, ...this.args], - this.timeouts.run, - this.memory_limits.run + this.runtime.timeouts.run, + this.runtime.memory_limits.run ); this.state = job_states.EXECUTED; @@ -266,8 +262,8 @@ class Job { const {error, code, signal} = await this.safe_call( path.join(this.runtime.pkgdir, 'compile'), this.files.map(x => x.name), - this.timeouts.compile, - this.memory_limits.compile, + this.runtime.timeouts.compile, + this.runtime.memory_limits.compile, eventBus ) @@ -279,14 +275,14 @@ class Job { const {error, code, signal} = await this.safe_call( path.join(this.runtime.pkgdir, 'run'), [this.files[0].name, ...this.args], - this.timeouts.run, - this.memory_limits.run, + this.runtime.timeouts.run, + this.runtime.memory_limits.run, eventBus ); eventBus.emit("exit", "run", {error, code, signal}) - + this.state = job_states.EXECUTED; } @@ -308,8 +304,7 @@ class Job { const proc_lines = proc_status.to_string().split("\n") const uid_line = proc_lines.find(line=>line.starts_with("Uid:")) const [_, ruid, euid, suid, fuid] = uid_line.split(/\s+/); - - + if(ruid == this.uid || euid == this.uid) return parse_int(proc_id) diff --git a/api/src/runtime.js b/api/src/runtime.js index 191fc5d..60d3c23 100644 --- a/api/src/runtime.js +++ b/api/src/runtime.js @@ -8,12 +8,54 @@ const path = require('path'); const runtimes = []; class Runtime { - constructor({ language, version, aliases, pkgdir, runtime }) { + constructor({ + language, version, aliases, pkgdir, runtime, timeouts, memory_limits, max_process_count, + max_open_files, max_file_size, output_max_size + }) { this.language = language; this.version = version; this.aliases = aliases || []; this.pkgdir = pkgdir; this.runtime = runtime; + this.timeouts = timeouts; + this.memory_limits = memory_limits; + this.max_process_count = max_process_count; + this.max_open_files = max_open_files; + this.max_file_size = max_file_size; + this.output_max_size = output_max_size; + } + + static compute_single_limit(language_name, limit_name, language_limit_overrides) { + return ( + config.limit_overrides[language_name] && config.limit_overrides[language_name][limit_name] + || language_limit_overrides && language_limit_overrides[limit_name] + || config[limit_name] + ); + } + + static compute_all_limits(language_name, language_limit_overrides) { + return { + timeouts: { + compile: + this.compute_single_limit(language_name, 'compile_timeout', language_limit_overrides), + run: + this.compute_single_limit(language_name, 'run_timeout', language_limit_overrides) + }, + memory_limits: { + compile: + this.compute_single_limit(language_name, 'compile_memory_limit', language_limit_overrides), + run: + this.compute_single_limit(language_name, 'run_memory_limit', language_limit_overrides) + }, + max_process_count: + this.compute_single_limit(language_name, 'max_process_count', language_limit_overrides), + max_open_files: + this.compute_single_limit(language_name, 'max_open_files', language_limit_overrides), + max_file_size: + this.compute_single_limit(language_name, 'max_file_size', language_limit_overrides), + output_max_size: + this.compute_single_limit(language_name, 'output_max_size', language_limit_overrides), + } } static load_package(package_dir) { @@ -21,7 +63,7 @@ class Runtime { fss.read_file_sync(path.join(package_dir, 'pkg-info.json')) ); - let { language, version, build_platform, aliases, provides } = info; + let { language, version, build_platform, aliases, provides, limit_overrides } = info; version = semver.parse(version); if (build_platform !== globals.platform) { @@ -41,6 +83,7 @@ class Runtime { version, pkgdir: package_dir, runtime: language, + ...Runtime.compute_all_limits(lang.language, lang.limit_overrides) }) ); }); @@ -51,6 +94,7 @@ class Runtime { version, aliases, pkgdir: package_dir, + ...Runtime.compute_all_limits(language, limit_overrides) }) ); } diff --git a/packages/dotnet/5.0.201/build.sh b/packages/dotnet/5.0.201/build.sh old mode 100644 new mode 100755 From adae6fde2f22fd07cb99f142eee277b526ea7969 Mon Sep 17 00:00:00 2001 From: Brikaa Date: Tue, 14 Sep 2021 13:59:19 +0200 Subject: [PATCH 16/72] pkg(dotnet-5.0.201): Added F#.net, F# interactive and VB.net --- api/src/job.js | 6 +-- packages/dotnet/5.0.201/build.sh | 6 ++- packages/dotnet/5.0.201/compile | 31 ++++++++++--- packages/dotnet/5.0.201/environment | 3 +- packages/dotnet/5.0.201/metadata.json | 63 ++++++++++++++++++++++++++- packages/dotnet/5.0.201/run | 22 +++++++++- packages/dotnet/5.0.201/test.fs | 6 +++ packages/dotnet/5.0.201/test.fsx | 1 + packages/dotnet/5.0.201/test.vb | 9 ++++ readme.md | 3 +- 10 files changed, 135 insertions(+), 15 deletions(-) create mode 100644 packages/dotnet/5.0.201/test.fs create mode 100644 packages/dotnet/5.0.201/test.fsx create mode 100644 packages/dotnet/5.0.201/test.vb diff --git a/api/src/job.js b/api/src/job.js index d4d19d9..552463a 100644 --- a/api/src/job.js +++ b/api/src/job.js @@ -100,9 +100,9 @@ class Job { const prlimit = [ 'prlimit', - '--nproc=' + this.runtime.max_process_count , - '--nofile=' + this.runtime.max_open_files , - '--fsize=' + this.runtime.max_file_size , + '--nproc=' + this.runtime.max_process_count, + '--nofile=' + this.runtime.max_open_files, + '--fsize=' + this.runtime.max_file_size, ]; if (memory_limit >= 0) { diff --git a/packages/dotnet/5.0.201/build.sh b/packages/dotnet/5.0.201/build.sh index c685668..6318b07 100755 --- a/packages/dotnet/5.0.201/build.sh +++ b/packages/dotnet/5.0.201/build.sh @@ -7,8 +7,10 @@ rm dotnet.tar.gz # Cache nuget packages export DOTNET_CLI_HOME=$PWD ./dotnet new console -o cache_application +./dotnet new console -lang F# -o fs_cache_application +./dotnet new console -lang VB -o vb_cache_application # This calls a restore on the global-packages index ($DOTNET_CLI_HOME/.nuget/packages) # If we want to allow more packages, we could add them to this cache_application -rm -rf cache_application -# Get rid of it, we don't actually need the application - just the restore \ No newline at end of file +rm -rf cache_application fs_cache_application vb_cache_application +# Get rid of it, we don't actually need the application - just the restore diff --git a/packages/dotnet/5.0.201/compile b/packages/dotnet/5.0.201/compile index 8bfcc27..1c34213 100644 --- a/packages/dotnet/5.0.201/compile +++ b/packages/dotnet/5.0.201/compile @@ -1,15 +1,36 @@ #!/usr/bin/env bash +[ "${PISTON_LANGUAGE}" == "fsi" ] && exit 0 + export DOTNET_CLI_HOME=$PWD export HOME=$PWD -rename 's/$/\.cs/' "$@" # Add .cs extension - dotnet build --help > /dev/null # Shut the thing up -dotnet new console -o . --no-restore -rm Program.cs +case "${PISTON_LANGUAGE}" in + basic.net) + rename 's/$/\.vb/' "$@" # Add .vb extension + dotnet new console -lang VB -o . --no-restore + rm Program.vb + ;; + fsharp.net) + first_file=$1 + shift + rename 's/$/\.fs/' "$@" # Add .fs extension + dotnet new console -lang F# -o . --no-restore + mv $first_file Program.fs # For some reason F#.net doesn't work unless the file name is Program.fs + ;; + csharp.net) + rename 's/$/\.cs/' "$@" # Add .cs extension + dotnet new console -o . --no-restore + rm Program.cs + ;; + *) + echo "How did you get here? (${PISTON_LANGUAGE})" + exit 1 + ;; +esac dotnet restore --source $DOTNET_ROOT/.nuget/packages -dotnet build --no-restore \ No newline at end of file +dotnet build --no-restore diff --git a/packages/dotnet/5.0.201/environment b/packages/dotnet/5.0.201/environment index 596d56e..468463d 100644 --- a/packages/dotnet/5.0.201/environment +++ b/packages/dotnet/5.0.201/environment @@ -2,4 +2,5 @@ # Put 'export' statements here for environment variables export DOTNET_ROOT=$PWD -export PATH=$DOTNET_ROOT:$PATH \ No newline at end of file +export PATH=$DOTNET_ROOT:$PATH +export FSI_PATH=$(find $(pwd) -name fsi.dll) diff --git a/packages/dotnet/5.0.201/metadata.json b/packages/dotnet/5.0.201/metadata.json index 619265d..7c73c58 100644 --- a/packages/dotnet/5.0.201/metadata.json +++ b/packages/dotnet/5.0.201/metadata.json @@ -1,5 +1,66 @@ { "language": "dotnet", "version": "5.0.201", - "aliases": ["cs", "csharp"] + "provides": [ + { + "language": "basic.net", + "aliases": [ + "basic", + "visual-basic", + "visual-basic.net", + "vb", + "vb.net", + "vb-dotnet", + "dotnet-vb", + "basic-dotnet", + "dotnet-basic" + ], + "limit_overrides": { "max_process_count": 128 } + }, + { + "language": "fsharp.net", + "aliases": [ + "fsharp", + "fs", + "f#", + "fs.net", + "f#.net", + "fsharp-dotnet", + "fs-dotnet", + "f#-dotnet", + "dotnet-fsharp", + "dotnet-fs", + "dotnet-fs" + ], + "limit_overrides": { "max_process_count": 128 } + }, + { + "language": "csharp.net", + "aliases": [ + "csharp", + "c#", + "cs", + "c#.net", + "cs.net", + "c#-dotnet", + "cs-dotnet", + "csharp-dotnet", + "dotnet-c#", + "dotnet-cs", + "dotnet-csharp" + ], + "limit_overrides": { "max_process_count": 128 } + }, + { + "language": "fsi", + "aliases": [ + "fsx", + "fsharp-interactive", + "f#-interactive", + "dotnet-fsi", + "fsi-dotnet", + "fsi.net" + ] + } + ] } diff --git a/packages/dotnet/5.0.201/run b/packages/dotnet/5.0.201/run index 774a08a..6b5c995 100644 --- a/packages/dotnet/5.0.201/run +++ b/packages/dotnet/5.0.201/run @@ -3,5 +3,23 @@ # Put instructions to run the runtime export DOTNET_CLI_HOME=$PWD -shift -dotnet bin/Debug/net5.0/$(basename $(realpath .)).dll "$@" \ No newline at end of file +case "${PISTON_LANGUAGE}" in + basic.net) + ;& + fsharp.net) + ;& + csharp.net) + shift + dotnet bin/Debug/net5.0/$(basename $(realpath .)).dll "$@" + ;; + fsi) + FILENAME=$1 + rename 's/$/\.fsx/' $FILENAME # Add .fsx extension + shift + dotnet $FSI_PATH $FILENAME.fsx "$@" + ;; + *) + echo "How did you get here? (${PISTON_LANGUAGE})" + exit 1 + ;; +esac diff --git a/packages/dotnet/5.0.201/test.fs b/packages/dotnet/5.0.201/test.fs new file mode 100644 index 0000000..006ac10 --- /dev/null +++ b/packages/dotnet/5.0.201/test.fs @@ -0,0 +1,6 @@ +open System + +[] +let main argv = + printfn "OK" + 0 diff --git a/packages/dotnet/5.0.201/test.fsx b/packages/dotnet/5.0.201/test.fsx new file mode 100644 index 0000000..33d166f --- /dev/null +++ b/packages/dotnet/5.0.201/test.fsx @@ -0,0 +1 @@ +printfn "OK" diff --git a/packages/dotnet/5.0.201/test.vb b/packages/dotnet/5.0.201/test.vb new file mode 100644 index 0000000..291042e --- /dev/null +++ b/packages/dotnet/5.0.201/test.vb @@ -0,0 +1,9 @@ +Imports System + +Module Module1 + + Sub Main() + Console.WriteLine("OK") + End Sub + +End Module diff --git a/readme.md b/readme.md index 5b41e6a..4ee97f1 100644 --- a/readme.md +++ b/readme.md @@ -333,7 +333,6 @@ Content-Type: application/json `d`, `dart`, `dash`, -`dotnet`, `dragon`, `elixir`, `emacs`, @@ -341,6 +340,8 @@ Content-Type: application/json `forte`, `fortran`, `freebasic`, +`fsharp`, +`fsi`, `go`, `golfscript`, `groovy`, From aa4b94a23720ce275730e6732f771b0c6f521dc8 Mon Sep 17 00:00:00 2001 From: Brikaa Date: Mon, 4 Oct 2021 17:38:48 +0200 Subject: [PATCH 17/72] Add csharp.net, fsharp.net basic.net to readme --- readme.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/readme.md b/readme.md index 4ee97f1..6a55116 100644 --- a/readme.md +++ b/readme.md @@ -330,6 +330,7 @@ Content-Type: application/json `cow`, `crystal`, `csharp`, +`csharp.net`, `d`, `dart`, `dash`, @@ -340,7 +341,7 @@ Content-Type: application/json `forte`, `fortran`, `freebasic`, -`fsharp`, +`fsharp.net`, `fsi`, `go`, `golfscript`, @@ -383,6 +384,7 @@ Content-Type: application/json `swift`, `typescript`, `basic`, +`basic.net`, `vlang`, `yeethon`, `zig`, From 363daa2a244e44e9f0bdbd7482d4786d2b003c1a Mon Sep 17 00:00:00 2001 From: Brikaa Date: Mon, 4 Oct 2021 21:35:13 +0200 Subject: [PATCH 18/72] Document limit_overrides, timeouts, provides and local testing --- docs/configuration.md | 31 ++++++++++++++++++++++++++-- packages/CONTRIBUTING.MD | 44 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 71 insertions(+), 4 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 16a5df0..4c6601a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -86,11 +86,11 @@ key: PISTON_MAX_PROCESS_COUNT default: 64 ``` -Maximum number of processess allowed to to have open for a job. +Maximum number of processes allowed to to have open for a job. Resists against exhausting the process table, causing a full system lockup. -## Output Max Side +## Output Max Size ```yaml key: PISTON_OUTPUT_MAX_SIZE @@ -123,6 +123,19 @@ Maximum size for a singular file written to disk. Resists against large file writes to exhaust disk space. +## Compile/Run timeouts +```yaml +key: + - PISTON_COMPILE_TIMEOUT +default: 10000 + +key: + - PISTON_RUN_TIMEOUT +default: 3000 +``` +The maximum time that is allowed to be taken by a stage in milliseconds. +Use -1 for unlimited time. + ## Compile/Run memory limits ```yaml @@ -154,3 +167,17 @@ default: 64 ``` Maximum number of jobs to run concurrently. + +## Limit overrides + +```yaml +key: PISTON_LIMIT_OVERRIDES +default: {} +``` + +Per-language overrides/exceptions for the each of `max_process_count`, `max_open_files`, `max_file_size`, +`compile_memory_limit`, `run_memory_limit`, `compile_timeout`, `run_timeout`, `output_max_size`. Defined as follows: +``` +PISTON_LIMIT_OVERRIDES={"c++":{"max_process_count":128}} +``` +This will give `c++` a max_process_count of 128 regardless of the configuration. diff --git a/packages/CONTRIBUTING.MD b/packages/CONTRIBUTING.MD index b1ed6d3..53deda9 100644 --- a/packages/CONTRIBUTING.MD +++ b/packages/CONTRIBUTING.MD @@ -40,7 +40,7 @@ public class Test } ``` -8. Create a `metadata.json` file which contains metadata about the language and interpreter. This simply contains the language name, as in the folder name, the version as in the folder name, aliases that can be used to call this package, and finally a dependencies map. +8. Create a `metadata.json` file which contains metadata about the language and interpreter. This simply contains the language name, as in the folder name, the version as in the folder name, aliases that can be used to call this package, limit overrides (if any) that can be used to override the default constraints and finally a dependencies map. The dependencies map contains the keys as language names, and the values as semver selectors for packages. ```json { @@ -50,9 +50,49 @@ The dependencies map contains the keys as language names, and the values as semv "aliases": ["deno-ts", "deno-js"] } ``` +If the interpreter/compiler provides multiple languages, then the provides property should be used: +```json +{ + "language": "dotnet", + "version": "5.0.201", + "provides": [ + { + "language": "basic.net", + "aliases": [ + "basic", + "visual-basic", + "visual-basic.net", + "vb", + "vb.net", + "vb-dotnet", + "dotnet-vb", + "basic-dotnet", + "dotnet-basic" + ], + "limit_overrides": { "max_process_count": 128 } + }, + { + "language": "fsi", + "aliases": [ + "fsx", + "fsharp-interactive", + "f#-interactive", + "dotnet-fsi", + "fsi-dotnet", + "fsi.net" + ] + } + ] +} +``` 9. Test your package builds with running `make [language]-[version].pkg.tar.gz`. -If it all goes to plan, you should have a file named `[language]-[version].pkg.tar.gz`, in this case your good to go! +If it all goes to plan, you should have a file named `[language]-[version].pkg.tar.gz`, in this case you're good to go, albeit it is preferable to test the package locally as follows +```shell +./piston build-pkg [package] [version] +./piston ppman install [package]=[version] +./piston run [package] -l [version] packages/[package]/[version]/test.* +``` 10. Commit your changes, using message format of `pkg([language]-[version]): Added [language] [version]` Any additional commits regarding this package should start with `pkg([language]-[version]): ` From 82d6dfd3e9a7240fe0f045b6955cda00e1f6cd31 Mon Sep 17 00:00:00 2001 From: Niki4tap Date: Wed, 6 Oct 2021 12:52:59 +0000 Subject: [PATCH 19/72] pkg(llvm_ir-12.0.1): Added llvm_ir 12.0.1 --- packages/llvm_ir/12.0.1/build.sh | 6 ++++++ packages/llvm_ir/12.0.1/compile | 4 ++++ packages/llvm_ir/12.0.1/environment | 2 ++ packages/llvm_ir/12.0.1/metadata.json | 5 +++++ packages/llvm_ir/12.0.1/run | 4 ++++ packages/llvm_ir/12.0.1/test.ll | 10 ++++++++++ 6 files changed, 31 insertions(+) create mode 100755 packages/llvm_ir/12.0.1/build.sh create mode 100755 packages/llvm_ir/12.0.1/compile create mode 100644 packages/llvm_ir/12.0.1/environment create mode 100644 packages/llvm_ir/12.0.1/metadata.json create mode 100644 packages/llvm_ir/12.0.1/run create mode 100644 packages/llvm_ir/12.0.1/test.ll diff --git a/packages/llvm_ir/12.0.1/build.sh b/packages/llvm_ir/12.0.1/build.sh new file mode 100755 index 0000000..7afd41a --- /dev/null +++ b/packages/llvm_ir/12.0.1/build.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +curl -L "https://github.com/llvm/llvm-project/releases/download/llvmorg-12.0.1/clang+llvm-12.0.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz" -o llvm-ir.tar.xz + +tar xf llvm-ir.tar.xz clang+llvm-12.0.1-x86_64-linux-gnu-ubuntu-/bin --strip-components=1 + +rm llvm-ir.tar.xz diff --git a/packages/llvm_ir/12.0.1/compile b/packages/llvm_ir/12.0.1/compile new file mode 100755 index 0000000..082fb5c --- /dev/null +++ b/packages/llvm_ir/12.0.1/compile @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +llc "$@" -o binary.s +clang binary.s -o binary diff --git a/packages/llvm_ir/12.0.1/environment b/packages/llvm_ir/12.0.1/environment new file mode 100644 index 0000000..85a25b6 --- /dev/null +++ b/packages/llvm_ir/12.0.1/environment @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +export PATH=$PWD/bin:$PATH diff --git a/packages/llvm_ir/12.0.1/metadata.json b/packages/llvm_ir/12.0.1/metadata.json new file mode 100644 index 0000000..8ef2dc3 --- /dev/null +++ b/packages/llvm_ir/12.0.1/metadata.json @@ -0,0 +1,5 @@ +{ + "language": "llvm_ir", + "version": "12.0.1", + "aliases": ["llvm", "llvm-ir"] +} diff --git a/packages/llvm_ir/12.0.1/run b/packages/llvm_ir/12.0.1/run new file mode 100644 index 0000000..94b3b0b --- /dev/null +++ b/packages/llvm_ir/12.0.1/run @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +shift +binary "$@" diff --git a/packages/llvm_ir/12.0.1/test.ll b/packages/llvm_ir/12.0.1/test.ll new file mode 100644 index 0000000..35786b2 --- /dev/null +++ b/packages/llvm_ir/12.0.1/test.ll @@ -0,0 +1,10 @@ +@.str = private unnamed_addr constant [3 x i8] c"OK\n" + +declare i32 @puts(i8* nocapture) nounwind + +define i32 @main() { + %cast210 = getelementptr [3 x i8],[3 x i8]* @.str, i64 0, i64 0 + + call i32 @puts(i8* %cast210) + ret i32 0 +} From 478ccefa58ec399263f1afd303c4c2523fe41a6f Mon Sep 17 00:00:00 2001 From: Niki4tap Date: Wed, 6 Oct 2021 13:00:58 +0000 Subject: [PATCH 20/72] pkg(llvm_ir-12.0.1): Fixed test.ll --- packages/llvm_ir/12.0.1/test.ll | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/llvm_ir/12.0.1/test.ll b/packages/llvm_ir/12.0.1/test.ll index 35786b2..be7d5d3 100644 --- a/packages/llvm_ir/12.0.1/test.ll +++ b/packages/llvm_ir/12.0.1/test.ll @@ -1,9 +1,9 @@ -@.str = private unnamed_addr constant [3 x i8] c"OK\n" +@.str = private unnamed_addr constant [2 x i8] c"OK" declare i32 @puts(i8* nocapture) nounwind define i32 @main() { - %cast210 = getelementptr [3 x i8],[3 x i8]* @.str, i64 0, i64 0 + %cast210 = getelementptr [2 x i8],[2 x i8]* @.str, i64 0, i64 0 call i32 @puts(i8* %cast210) ret i32 0 From fbee9e6c22d9b80c2edfe7cae97c7c1c788ffa09 Mon Sep 17 00:00:00 2001 From: Niki4tap Date: Wed, 6 Oct 2021 13:32:54 +0000 Subject: [PATCH 21/72] pkg(llvm_ir-12.0.1): Add the package to readme and `ll` alias --- packages/llvm_ir/12.0.1/metadata.json | 2 +- readme.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/llvm_ir/12.0.1/metadata.json b/packages/llvm_ir/12.0.1/metadata.json index 8ef2dc3..4c92048 100644 --- a/packages/llvm_ir/12.0.1/metadata.json +++ b/packages/llvm_ir/12.0.1/metadata.json @@ -1,5 +1,5 @@ { "language": "llvm_ir", "version": "12.0.1", - "aliases": ["llvm", "llvm-ir"] + "aliases": ["llvm", "llvm-ir", "ll"] } diff --git a/readme.md b/readme.md index 987b40c..c64815a 100644 --- a/readme.md +++ b/readme.md @@ -356,6 +356,7 @@ Content-Type: application/json `julia`, `kotlin`, `lisp`, +`llvm_ir`, `lolcode`, `lua`, `nasm`, From cfbb62f5bf950ab09b0e5f3a44ea5025a3cc7357 Mon Sep 17 00:00:00 2001 From: Brikaa Date: Fri, 8 Oct 2021 17:20:10 +0200 Subject: [PATCH 22/72] Remove stopped package builder container --- piston | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piston b/piston index a14e7f5..1653eae 100755 --- a/piston +++ b/piston @@ -73,7 +73,7 @@ case $1 in echo "Building $PKGSLUG" echo "Ensuring latest builder image" docker build repo -t piston-repo-builder - docker run -v "$(realpath $(dirname "$0")):/piston" piston-repo-builder --no-server $PKGSLUG + docker run --rm -v "$(realpath $(dirname "$0")):/piston" piston-repo-builder --no-server $PKGSLUG ;; *) cd cli From 7ff87cf0f20de32549f527fedcb93f70c861363f Mon Sep 17 00:00:00 2001 From: Hydrazer Date: Fri, 8 Oct 2021 17:14:08 -0600 Subject: [PATCH 23/72] pkg(husk-1.0.0): add husk --- packages/husk/1.0.0/build.sh | 14 ++++++++++++++ packages/husk/1.0.0/environment | 6 ++++++ packages/husk/1.0.0/metadata.json | 5 +++++ packages/husk/1.0.0/run | 10 ++++++++++ packages/husk/1.0.0/test.husk | 1 + readme.md | 1 + 6 files changed, 37 insertions(+) create mode 100644 packages/husk/1.0.0/build.sh create mode 100644 packages/husk/1.0.0/environment create mode 100644 packages/husk/1.0.0/metadata.json create mode 100644 packages/husk/1.0.0/run create mode 100644 packages/husk/1.0.0/test.husk diff --git a/packages/husk/1.0.0/build.sh b/packages/husk/1.0.0/build.sh new file mode 100644 index 0000000..e2ee19c --- /dev/null +++ b/packages/husk/1.0.0/build.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +cp ../../haskell/9.0.1/build.sh ./haskell-build.sh +sed -Ei 's/9\.0\.1/8\.10\.7/g' ./haskell-build.sh +source ./haskell-build.sh + +# compile Husk from source +git clone -q "https://github.com/barbuz/husk.git" +cd husk +../bin/ghc -O2 Husk + +# cleanup +cd .. +rm -f haskell-build.sh \ No newline at end of file diff --git a/packages/husk/1.0.0/environment b/packages/husk/1.0.0/environment new file mode 100644 index 0000000..8b8421d --- /dev/null +++ b/packages/husk/1.0.0/environment @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +# haskell and husk path +export PATH=$PWD/bin:$PATH +export HUSK_PATH=$PWD/husk +export LANG=en_US.UTF8 diff --git a/packages/husk/1.0.0/metadata.json b/packages/husk/1.0.0/metadata.json new file mode 100644 index 0000000..69c5b7e --- /dev/null +++ b/packages/husk/1.0.0/metadata.json @@ -0,0 +1,5 @@ +{ + "language": "husk", + "version": "1.0.0", + "aliases": [] +} diff --git a/packages/husk/1.0.0/run b/packages/husk/1.0.0/run new file mode 100644 index 0000000..1fa65a0 --- /dev/null +++ b/packages/husk/1.0.0/run @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +# Store the current path because we'll need it to run the program file +PROGRAM_PATH=$PWD + +# For now, Husk can only be run within the folder that has the imported modules +cd $HUSK_PATH + +# Run Husk from file in unicode format with the given args +./Husk -uf "${PROGRAM_PATH}/${@}" \ No newline at end of file diff --git a/packages/husk/1.0.0/test.husk b/packages/husk/1.0.0/test.husk new file mode 100644 index 0000000..d096585 --- /dev/null +++ b/packages/husk/1.0.0/test.husk @@ -0,0 +1 @@ +"OK \ No newline at end of file diff --git a/readme.md b/readme.md index c64815a..b215138 100644 --- a/readme.md +++ b/readme.md @@ -348,6 +348,7 @@ Content-Type: application/json `golfscript`, `groovy`, `haskell`, +`husk`, `iverilog`, `japt`, `java`, From f2c91acbe69ded912077e217239227516f97b7a0 Mon Sep 17 00:00:00 2001 From: Brikaa Date: Fri, 8 Oct 2021 15:16:57 +0200 Subject: [PATCH 24/72] Piston lint --- .github/ISSUE_TEMPLATE/language-request.md | 1 - .github/PULL_REQUEST_TEMPLATE/package.md | 19 +- .github/workflows/api-push.yaml | 67 +++--- .github/workflows/package-pr.yaml | 228 ++++++++++----------- .github/workflows/package-push.yaml | 139 +++++++------ .github/workflows/repo-push.yaml | 56 ++--- .gitignore | 3 +- .prettierignore | 12 ++ api/.prettierrc.yaml => .prettierrc.yaml | 0 .readthedocs.yaml | 8 +- api/.gitignore | 3 +- api/.prettierignore | 1 - api/package-lock.json | 28 +-- api/package.json | 8 +- api/src/api/v2.js | 211 ++++++++++++------- api/src/config.js | 49 +++-- api/src/index.js | 2 - api/src/job.js | 166 +++++++-------- api/src/package.js | 3 +- api/src/runtime.js | 101 ++++++--- cli/.gitignore | 1 - cli/commands/execute.js | 227 +++++++++++--------- cli/commands/ppman.js | 4 +- cli/commands/ppman_commands/install.js | 22 +- cli/commands/ppman_commands/list.js | 18 +- cli/commands/ppman_commands/spec.js | 171 +++++++++------- cli/commands/ppman_commands/uninstall.js | 25 ++- cli/index.js | 9 +- docker-compose.dev.yaml | 42 ++-- docs/api-v2.md | 80 ++++---- docs/configuration.md | 24 ++- mkdocs.yml | 18 +- package-lock.json | 34 ++- package.json | 6 +- packages/cjam/0.6.5/metadata.json | 6 +- packages/cobol/3.1.2/metadata.json | 6 +- packages/deno/1.7.5/metadata.json | 2 +- packages/deno/1.7.5/test.deno.ts | 2 +- packages/dragon/1.9.8/metadata.json | 6 +- packages/forte/1.0.0/metadata.json | 6 +- packages/gcc/10.2.0/metadata.json | 2 +- packages/golfscript/1.0.0/metadata.json | 6 +- packages/groovy/3.0.7/metadata.json | 6 +- packages/japt/2.0.0/metadata.json | 8 +- packages/llvm_ir/12.0.1/metadata.json | 6 +- packages/mono/6.12.0/metadata.json | 8 +- packages/node/15.10.0/test.js | 2 +- packages/node/16.3.0/test.js | 2 +- packages/ocaml/4.12.0/metadata.json | 6 +- packages/octave/6.2.0/metadata.json | 6 +- packages/pyth/1.0.0/metadata.json | 6 +- packages/raku/6.100.0/metadata.json | 2 +- packages/typescript/4.2.3/test.ts | 2 +- piston | 12 +- pre-commit | 15 ++ readme.md | 109 +++++----- tests/readme.md | 2 +- 57 files changed, 1121 insertions(+), 893 deletions(-) create mode 100644 .prettierignore rename api/.prettierrc.yaml => .prettierrc.yaml (100%) delete mode 100644 api/.prettierignore delete mode 100644 cli/.gitignore create mode 100755 pre-commit diff --git a/.github/ISSUE_TEMPLATE/language-request.md b/.github/ISSUE_TEMPLATE/language-request.md index 5ae2661..3f42d90 100644 --- a/.github/ISSUE_TEMPLATE/language-request.md +++ b/.github/ISSUE_TEMPLATE/language-request.md @@ -4,7 +4,6 @@ about: Template for requesting language support title: Add [insert language name here] labels: package assignees: '' - --- Provide links to different compilers/interpreters that could be used to implement this language, and discuss pros/cons of each. diff --git a/.github/PULL_REQUEST_TEMPLATE/package.md b/.github/PULL_REQUEST_TEMPLATE/package.md index 6cd3c98..da59fe0 100644 --- a/.github/PULL_REQUEST_TEMPLATE/package.md +++ b/.github/PULL_REQUEST_TEMPLATE/package.md @@ -1,10 +1,11 @@ Checklist: -* [ ] The package builds locally with `./piston build-pkg [package] [version]` -* [ ] The package installs with `./piston ppman install [package]=[version]` -* [ ] The package runs the test code with `./piston run [package] -l [version] packages/[package]/[version]/test.*` -* [ ] Package files are placed in the correct directory -* [ ] No old package versions are removed -* [ ] All source files are deleted in the `build.sh` script -* [ ] `metadata.json`'s `language` and `version` fields match the directory path -* [ ] Any extensions the language may use are set as aliases -* [ ] Any alternative names the language is referred to are set as aliases. + +- [ ] The package builds locally with `./piston build-pkg [package] [version]` +- [ ] The package installs with `./piston ppman install [package]=[version]` +- [ ] The package runs the test code with `./piston run [package] -l [version] packages/[package]/[version]/test.*` +- [ ] Package files are placed in the correct directory +- [ ] No old package versions are removed +- [ ] All source files are deleted in the `build.sh` script +- [ ] `metadata.json`'s `language` and `version` fields match the directory path +- [ ] Any extensions the language may use are set as aliases +- [ ] Any alternative names the language is referred to are set as aliases. diff --git a/.github/workflows/api-push.yaml b/.github/workflows/api-push.yaml index bcf0472..dec3bce 100644 --- a/.github/workflows/api-push.yaml +++ b/.github/workflows/api-push.yaml @@ -1,39 +1,38 @@ name: Publish API image on: - push: - branches: - - master - - v3 - paths: - - api/** + push: + branches: + - master + - v3 + paths: + - api/** - jobs: - push_to_registry: - runs-on: ubuntu-latest - name: Build and Push Docker image to Github Packages - steps: - - name: Check out repo - uses: actions/checkout@v2 - - name: Login to GitHub registry - uses: docker/login-action@v1 - with: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - registry: docker.pkg.github.com - - name: Login to ghcr.io - uses: docker/login-action@v1 - with: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - registry: ghcr.io + push_to_registry: + runs-on: ubuntu-latest + name: Build and Push Docker image to Github Packages + steps: + - name: Check out repo + uses: actions/checkout@v2 + - name: Login to GitHub registry + uses: docker/login-action@v1 + with: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + registry: docker.pkg.github.com + - name: Login to ghcr.io + uses: docker/login-action@v1 + with: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + registry: ghcr.io - - name: Build and push API - uses: docker/build-push-action@v2 - with: - context: api - push: true - pull: true - tags: | - docker.pkg.github.com/engineer-man/piston/api - ghcr.io/engineer-man/piston + - name: Build and push API + uses: docker/build-push-action@v2 + with: + context: api + push: true + pull: true + tags: | + docker.pkg.github.com/engineer-man/piston/api + ghcr.io/engineer-man/piston diff --git a/.github/workflows/package-pr.yaml b/.github/workflows/package-pr.yaml index cb646d3..78101b0 100644 --- a/.github/workflows/package-pr.yaml +++ b/.github/workflows/package-pr.yaml @@ -1,140 +1,140 @@ -name: "Package Pull Requests" +name: 'Package Pull Requests' on: - pull_request: - types: - - opened - - edited - - reopened - - synchronize - paths: - - "packages/**" + pull_request: + types: + - opened + - edited + - reopened + - synchronize + paths: + - 'packages/**' jobs: - check-pkg: - name: Validate README - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Get list of changed files - uses: lots0logs/gh-action-get-changed-files@2.1.4 - with: - token: ${{ secrets.GITHUB_TOKEN }} + check-pkg: + name: Validate README + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Get list of changed files + uses: lots0logs/gh-action-get-changed-files@2.1.4 + with: + token: ${{ secrets.GITHUB_TOKEN }} - - name: Ensure README was updated - run: | - MISSING_LINES=$(comm -23 <(jq 'if .provides then .provides[].language else .language end' -r $(find packages -name "metadata.json" ) | sed -e 's/^/`/g' -e 's/$/`,/g' | sort -u) <(awk '/# Supported Languages/{flag=1; next} /
/{flag=0} flag' readme.md | sort -u)) + - name: Ensure README was updated + run: | + MISSING_LINES=$(comm -23 <(jq 'if .provides then .provides[].language else .language end' -r $(find packages -name "metadata.json" ) | sed -e 's/^/`/g' -e 's/$/`,/g' | sort -u) <(awk '/# Supported Languages/{flag=1; next} /
/{flag=0} flag' readme.md | sort -u)) - [[ $(echo $MISSING_LINES | wc -c) = "1" ]] && exit 0 + [[ $(echo $MISSING_LINES | wc -c) = "1" ]] && exit 0 - echo "README has supported languages missing: " - comm -23 <(jq 'if .provides then .provides[].language else .language end' -r $(find packages -name "metadata.json" ) | sed -e 's/^/`/g' -e 's/$/`,/g' | sort -u) <(awk '/# Supported Languages/{flag=1; next} /
/{flag=0} flag' readme.md | sort -u) - exit 1 + echo "README has supported languages missing: " + comm -23 <(jq 'if .provides then .provides[].language else .language end' -r $(find packages -name "metadata.json" ) | sed -e 's/^/`/g' -e 's/$/`,/g' | sort -u) <(awk '/# Supported Languages/{flag=1; next} /
/{flag=0} flag' readme.md | sort -u) + exit 1 - build-pkg: - name: Check that package builds - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 + build-pkg: + name: Check that package builds + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 - - name: Login to GitHub registry - uses: docker/login-action@v1 - with: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - registry: docker.pkg.github.com + - name: Login to GitHub registry + uses: docker/login-action@v1 + with: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + registry: docker.pkg.github.com - - name: Get list of changed files - uses: lots0logs/gh-action-get-changed-files@2.1.4 - with: - token: ${{ secrets.GITHUB_TOKEN }} + - name: Get list of changed files + uses: lots0logs/gh-action-get-changed-files@2.1.4 + with: + token: ${{ secrets.GITHUB_TOKEN }} - - name: Build Packages - run: | - PACKAGES=$(jq '.[]' -r ${HOME}/files.json | awk -F/ '$2 && $3{ print $2 "-" $3 }' | sort -u) - echo "Packages: $PACKAGES" - docker pull docker.pkg.github.com/engineer-man/piston/repo-builder:latest - docker build -t repo-builder repo - docker run -v "${{ github.workspace }}:/piston" repo-builder --no-server $PACKAGES - ls -la packages + - name: Build Packages + run: | + PACKAGES=$(jq '.[]' -r ${HOME}/files.json | awk -F/ '$2 && $3{ print $2 "-" $3 }' | sort -u) + echo "Packages: $PACKAGES" + docker pull docker.pkg.github.com/engineer-man/piston/repo-builder:latest + docker build -t repo-builder repo + docker run -v "${{ github.workspace }}:/piston" repo-builder --no-server $PACKAGES + ls -la packages - - name: Upload package as artifact - uses: actions/upload-artifact@v2 - with: - name: packages - path: packages/*.pkg.tar.gz + - name: Upload package as artifact + uses: actions/upload-artifact@v2 + with: + name: packages + path: packages/*.pkg.tar.gz - test-pkg: - name: Test package - runs-on: ubuntu-latest - needs: build-pkg - steps: - - uses: actions/checkout@v2 + test-pkg: + name: Test package + runs-on: ubuntu-latest + needs: build-pkg + steps: + - uses: actions/checkout@v2 - - uses: actions/download-artifact@v2 - with: - name: packages + - uses: actions/download-artifact@v2 + with: + name: packages - - name: Relocate downloaded packages - run: mv *.pkg.tar.gz packages/ + - name: Relocate downloaded packages + run: mv *.pkg.tar.gz packages/ - - name: Login to GitHub registry - uses: docker/login-action@v1 - with: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - registry: docker.pkg.github.com + - name: Login to GitHub registry + uses: docker/login-action@v1 + with: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + registry: docker.pkg.github.com - - name: Run tests - run: | - ls -la - docker run -v $(pwd)'/repo:/piston/repo' -v $(pwd)'/packages:/piston/packages' -d --name repo docker.pkg.github.com/engineer-man/piston/repo-builder --no-build - docker pull docker.pkg.github.com/engineer-man/piston/api - docker build -t piston-api api - docker run --network container:repo -v $(pwd)'/data:/piston' -e PISTON_LOG_LEVEL=DEBUG -e 'PISTON_REPO_URL=http://localhost:8000/index' -d --name api piston-api - echo Waiting for API to start.. - docker run --network container:api appropriate/curl -s --retry 10 --retry-connrefused http://localhost:2000/api/v2/runtimes + - name: Run tests + run: | + ls -la + docker run -v $(pwd)'/repo:/piston/repo' -v $(pwd)'/packages:/piston/packages' -d --name repo docker.pkg.github.com/engineer-man/piston/repo-builder --no-build + docker pull docker.pkg.github.com/engineer-man/piston/api + docker build -t piston-api api + docker run --network container:repo -v $(pwd)'/data:/piston' -e PISTON_LOG_LEVEL=DEBUG -e 'PISTON_REPO_URL=http://localhost:8000/index' -d --name api piston-api + echo Waiting for API to start.. + docker run --network container:api appropriate/curl -s --retry 10 --retry-connrefused http://localhost:2000/api/v2/runtimes - echo Waiting for Index to start.. - docker run --network container:repo appropriate/curl -s --retry 999 --retry-max-time 0 --retry-connrefused http://localhost:8000/index + echo Waiting for Index to start.. + docker run --network container:repo appropriate/curl -s --retry 999 --retry-max-time 0 --retry-connrefused http://localhost:8000/index - echo Adjusting index - sed -i 's/repo/localhost/g' repo/index + echo Adjusting index + sed -i 's/repo/localhost/g' repo/index - echo Listing Packages - PACKAGES_JSON=$(docker run --network container:api appropriate/curl -s http://localhost:2000/api/v2/packages) - echo $PACKAGES_JSON + echo Listing Packages + PACKAGES_JSON=$(docker run --network container:api appropriate/curl -s http://localhost:2000/api/v2/packages) + echo $PACKAGES_JSON - echo Getting CLI ready - docker run -v "$PWD/cli:/app" --entrypoint /bin/bash node:15 -c 'cd /app; npm i' + echo Getting CLI ready + docker run -v "$PWD/cli:/app" --entrypoint /bin/bash node:15 -c 'cd /app; npm i' - for package in $(jq -r '.[] | "\(.language)-\(.language_version)"' <<< "$PACKAGES_JSON") - do - echo "Testing $package" - PKG_PATH=$(sed 's|-|/|' <<< $package) - PKG_NAME=$(awk -F- '{ print $1 }' <<< $package) - PKG_VERSION=$(awk -F- '{ print $2 }' <<< $package) + for package in $(jq -r '.[] | "\(.language)-\(.language_version)"' <<< "$PACKAGES_JSON") + do + echo "Testing $package" + PKG_PATH=$(sed 's|-|/|' <<< $package) + PKG_NAME=$(awk -F- '{ print $1 }' <<< $package) + PKG_VERSION=$(awk -F- '{ print $2 }' <<< $package) - echo "Installing..." - docker run --network container:api appropriate/curl -sXPOST http://localhost:2000/api/v2/packages -H "Content-Type: application/json" -d "{\"language\":\"$PKG_NAME\",\"version\":\"$PKG_VERSION\"}" + echo "Installing..." + docker run --network container:api appropriate/curl -sXPOST http://localhost:2000/api/v2/packages -H "Content-Type: application/json" -d "{\"language\":\"$PKG_NAME\",\"version\":\"$PKG_VERSION\"}" - TEST_SCRIPTS=packages/$PKG_PATH/test.* - echo "Tests: $TEST_SCRIPTS" + TEST_SCRIPTS=packages/$PKG_PATH/test.* + echo "Tests: $TEST_SCRIPTS" - for tscript in $TEST_SCRIPTS - do - TEST_RUNTIME=$(awk -F. '{print $2}' <<< $(basename $tscript)) - echo Running $tscript with runtime=$TEST_RUNTIME - docker run --network container:api -v "$PWD/cli:/app" -v "$PWD/$(dirname $tscript):/pkg" node:15 /app/index.js run $TEST_RUNTIME -l $PKG_VERSION /pkg/$(basename $tscript) > test_output - cat test_output - grep "OK" test_output - done - done + for tscript in $TEST_SCRIPTS + do + TEST_RUNTIME=$(awk -F. '{print $2}' <<< $(basename $tscript)) + echo Running $tscript with runtime=$TEST_RUNTIME + docker run --network container:api -v "$PWD/cli:/app" -v "$PWD/$(dirname $tscript):/pkg" node:15 /app/index.js run $TEST_RUNTIME -l $PKG_VERSION /pkg/$(basename $tscript) > test_output + cat test_output + grep "OK" test_output + done + done - - name: Dump logs - if: ${{ always() }} - run: | - docker logs api - docker logs repo + - name: Dump logs + if: ${{ always() }} + run: | + docker logs api + docker logs repo diff --git a/.github/workflows/package-push.yaml b/.github/workflows/package-push.yaml index bbb44af..ee49487 100644 --- a/.github/workflows/package-push.yaml +++ b/.github/workflows/package-push.yaml @@ -1,78 +1,77 @@ name: 'Package Pushed' on: - push: - branches: - - master - - v3 - paths: - - packages/** - + push: + branches: + - master + - v3 + paths: + - packages/** jobs: - build-pkg: - name: Build package - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Login to GitHub registry - uses: docker/login-action@v1 - with: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - registry: docker.pkg.github.com + build-pkg: + name: Build package + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 - - name: Get list of changed files - uses: lots0logs/gh-action-get-changed-files@2.1.4 - with: - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Build Packages - run: | - PACKAGES=$(jq '.[]' -r ${HOME}/files.json | awk -F/ '{ print $2 "-" $3 }' | sort -u) - echo "Packages: $PACKAGES" - docker pull docker.pkg.github.com/engineer-man/piston/repo-builder:latest - docker build -t repo-builder repo - docker run -v "${{ github.workspace }}:/piston" repo-builder --no-server $PACKAGES - ls -la packages + - name: Login to GitHub registry + uses: docker/login-action@v1 + with: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + registry: docker.pkg.github.com - - name: Upload Packages - uses: svenstaro/upload-release-action@v2 - with: - repo_token: ${{ secrets.GITHUB_TOKEN }} - file: packages/*.pkg.tar.gz - tag: pkgs - overwrite: true - file_glob: true - create-index: - name: Create Index - runs-on: ubuntu-latest - needs: build-pkg - steps: - - name: "Download all release assets" - run: curl -s https://api.github.com/repos/engineer-man/piston/releases/latest | jq '.assets[].browser_download_url' -r | xargs -L 1 curl -sLO - - name: "Generate index file" - run: | - echo "" > index - BASEURL=https://github.com/engineer-man/piston/releases/download/pkgs/ - for pkg in *.pkg.tar.gz - do - PKGFILE=$(basename $pkg) - PKGFILENAME=$(echo $PKGFILE | sed 's/\.pkg\.tar\.gz//g') + - name: Get list of changed files + uses: lots0logs/gh-action-get-changed-files@2.1.4 + with: + token: ${{ secrets.GITHUB_TOKEN }} - PKGNAME=$(echo $PKGFILENAME | grep -oP '^\K.+(?=-)') - PKGVERSION=$(echo $PKGFILENAME | grep -oP '^.+-\K.+') - PKGCHECKSUM=$(sha256sum $PKGFILE | awk '{print $1}') - echo "$PKGNAME,$PKGVERSION,$PKGCHECKSUM,$BASEURL$PKGFILE" >> index - echo "Adding package $PKGNAME-$PKGVERSION" - done - - name: Upload index - uses: svenstaro/upload-release-action@v2 - with: - repo_token: ${{ secrets.GITHUB_TOKEN }} - file: index - tag: pkgs - overwrite: true - file_glob: true + - name: Build Packages + run: | + PACKAGES=$(jq '.[]' -r ${HOME}/files.json | awk -F/ '{ print $2 "-" $3 }' | sort -u) + echo "Packages: $PACKAGES" + docker pull docker.pkg.github.com/engineer-man/piston/repo-builder:latest + docker build -t repo-builder repo + docker run -v "${{ github.workspace }}:/piston" repo-builder --no-server $PACKAGES + ls -la packages + + - name: Upload Packages + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: packages/*.pkg.tar.gz + tag: pkgs + overwrite: true + file_glob: true + create-index: + name: Create Index + runs-on: ubuntu-latest + needs: build-pkg + steps: + - name: 'Download all release assets' + run: curl -s https://api.github.com/repos/engineer-man/piston/releases/latest | jq '.assets[].browser_download_url' -r | xargs -L 1 curl -sLO + - name: 'Generate index file' + run: | + echo "" > index + BASEURL=https://github.com/engineer-man/piston/releases/download/pkgs/ + for pkg in *.pkg.tar.gz + do + PKGFILE=$(basename $pkg) + PKGFILENAME=$(echo $PKGFILE | sed 's/\.pkg\.tar\.gz//g') + + PKGNAME=$(echo $PKGFILENAME | grep -oP '^\K.+(?=-)') + PKGVERSION=$(echo $PKGFILENAME | grep -oP '^.+-\K.+') + PKGCHECKSUM=$(sha256sum $PKGFILE | awk '{print $1}') + echo "$PKGNAME,$PKGVERSION,$PKGCHECKSUM,$BASEURL$PKGFILE" >> index + echo "Adding package $PKGNAME-$PKGVERSION" + done + - name: Upload index + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: index + tag: pkgs + overwrite: true + file_glob: true diff --git a/.github/workflows/repo-push.yaml b/.github/workflows/repo-push.yaml index b5a603c..c887b01 100644 --- a/.github/workflows/repo-push.yaml +++ b/.github/workflows/repo-push.yaml @@ -1,31 +1,31 @@ name: Publish Repo image on: - push: - branches: - - master - - v3 - paths: - - repo/** - -jobs: - push_to_registry: - runs-on: ubuntu-latest - name: Build and Push Docker image to Github Packages - steps: - - name: Check out repo - uses: actions/checkout@v2 - - name: Login to GitHub registry - uses: docker/login-action@v1 - with: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - registry: docker.pkg.github.com + push: + branches: + - master + - v3 + paths: + - repo/** - - name: Build and push repo - uses: docker/build-push-action@v2 - with: - context: repo - pull: true - push: true - tags: | - docker.pkg.github.com/engineer-man/piston/repo-builder \ No newline at end of file +jobs: + push_to_registry: + runs-on: ubuntu-latest + name: Build and Push Docker image to Github Packages + steps: + - name: Check out repo + uses: actions/checkout@v2 + - name: Login to GitHub registry + uses: docker/login-action@v1 + with: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + registry: docker.pkg.github.com + + - name: Build and push repo + uses: docker/build-push-action@v2 + with: + context: repo + pull: true + push: true + tags: | + docker.pkg.github.com/engineer-man/piston/repo-builder diff --git a/.gitignore b/.gitignore index 222be8c..eb53d81 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ data/ -.piston_env \ No newline at end of file +.piston_env +node_modules diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 0000000..bb310ab --- /dev/null +++ b/.prettierignore @@ -0,0 +1,12 @@ +node_modules +data/ +api/_piston +repo/build +packages/*/*/* +packages/*.pkg.tar.gz +!packages/*/*/metadata.json +!packages/*/*/build.sh +!packages/*/*/environment +!packages/*/*/run +!packages/*/*/compile +!packages/*/*/test.* diff --git a/api/.prettierrc.yaml b/.prettierrc.yaml similarity index 100% rename from api/.prettierrc.yaml rename to .prettierrc.yaml diff --git a/.readthedocs.yaml b/.readthedocs.yaml index e651ad5..70f2b5e 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -1,8 +1,8 @@ version: 2 mkdocs: - configuration: mkdocs.yml + configuration: mkdocs.yml python: - version: 3.7 - install: - - requirements: docs/requirements.txt + version: 3.7 + install: + - requirements: docs/requirements.txt diff --git a/api/.gitignore b/api/.gitignore index adbd330..4b5a9b8 100644 --- a/api/.gitignore +++ b/api/.gitignore @@ -1,2 +1 @@ -node_modules -_piston \ No newline at end of file +_piston diff --git a/api/.prettierignore b/api/.prettierignore deleted file mode 100644 index 3c3629e..0000000 --- a/api/.prettierignore +++ /dev/null @@ -1 +0,0 @@ -node_modules diff --git a/api/package-lock.json b/api/package-lock.json index 83df240..2b34fc4 100644 --- a/api/package-lock.json +++ b/api/package-lock.json @@ -1,12 +1,12 @@ { "name": "piston-api", - "version": "3.0.0", + "version": "3.1.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "piston-api", - "version": "3.0.0", + "version": "3.1.0", "license": "MIT", "dependencies": { "body-parser": "^1.19.0", @@ -20,9 +20,6 @@ "semver": "^7.3.4", "uuid": "^8.3.2", "waitpid": "git+https://github.com/HexF/node-waitpid.git" - }, - "devDependencies": { - "prettier": "2.2.1" } }, "node_modules/accepts": { @@ -409,18 +406,6 @@ "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" }, - "node_modules/prettier": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.2.1.tgz", - "integrity": "sha512-PqyhM2yCjg/oKkFPtTGUojv7gnZAoG80ttl45O6x2Ug/rMJw4wcc9k6aaf2hibP7BGVCCM33gZoGjyvt9mm16Q==", - "dev": true, - "bin": { - "prettier": "bin-prettier.js" - }, - "engines": { - "node": ">=10.13.0" - } - }, "node_modules/proxy-addr": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.6.tgz", @@ -595,7 +580,8 @@ } }, "node_modules/waitpid": { - "resolved": "git+ssh://git@github.com/HexF/node-waitpid.git#a08d116a5d993a747624fe72ff890167be8c34aa" + "resolved": "git+ssh://git@github.com/HexF/node-waitpid.git#a08d116a5d993a747624fe72ff890167be8c34aa", + "hasInstallScript": true }, "node_modules/ws": { "version": "7.5.3", @@ -913,12 +899,6 @@ "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" }, - "prettier": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.2.1.tgz", - "integrity": "sha512-PqyhM2yCjg/oKkFPtTGUojv7gnZAoG80ttl45O6x2Ug/rMJw4wcc9k6aaf2hibP7BGVCCM33gZoGjyvt9mm16Q==", - "dev": true - }, "proxy-addr": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.6.tgz", diff --git a/api/package.json b/api/package.json index e8e5b5d..fb107a5 100644 --- a/api/package.json +++ b/api/package.json @@ -16,11 +16,5 @@ "uuid": "^8.3.2", "waitpid": "git+https://github.com/HexF/node-waitpid.git" }, - "license": "MIT", - "scripts": { - "lint": "prettier . --write" - }, - "devDependencies": { - "prettier": "2.2.1" - } + "license": "MIT" } diff --git a/api/src/api/v2.js b/api/src/api/v2.js index a3571e1..a8fa9f0 100644 --- a/api/src/api/v2.js +++ b/api/src/api/v2.js @@ -8,10 +8,49 @@ const { Job } = require('../job'); const package = require('../package'); const logger = require('logplease').create('api/v2'); -const SIGNALS = ["SIGABRT","SIGALRM","SIGBUS","SIGCHLD","SIGCLD","SIGCONT","SIGEMT","SIGFPE","SIGHUP","SIGILL","SIGINFO","SIGINT","SIGIO","SIGIOT","SIGKILL","SIGLOST","SIGPIPE","SIGPOLL","SIGPROF","SIGPWR","SIGQUIT","SIGSEGV","SIGSTKFLT","SIGSTOP","SIGTSTP","SIGSYS","SIGTERM","SIGTRAP","SIGTTIN","SIGTTOU","SIGUNUSED","SIGURG","SIGUSR1","SIGUSR2","SIGVTALRM","SIGXCPU","SIGXFSZ","SIGWINCH"] +const SIGNALS = [ + 'SIGABRT', + 'SIGALRM', + 'SIGBUS', + 'SIGCHLD', + 'SIGCLD', + 'SIGCONT', + 'SIGEMT', + 'SIGFPE', + 'SIGHUP', + 'SIGILL', + 'SIGINFO', + 'SIGINT', + 'SIGIO', + 'SIGIOT', + 'SIGKILL', + 'SIGLOST', + 'SIGPIPE', + 'SIGPOLL', + 'SIGPROF', + 'SIGPWR', + 'SIGQUIT', + 'SIGSEGV', + 'SIGSTKFLT', + 'SIGSTOP', + 'SIGTSTP', + 'SIGSYS', + 'SIGTERM', + 'SIGTRAP', + 'SIGTTIN', + 'SIGTTOU', + 'SIGUNUSED', + 'SIGURG', + 'SIGUSR1', + 'SIGUSR2', + 'SIGVTALRM', + 'SIGXCPU', + 'SIGXFSZ', + 'SIGWINCH', +]; // ref: https://man7.org/linux/man-pages/man7/signal.7.html -function get_job(body){ +function get_job(body) { let { language, version, @@ -21,7 +60,7 @@ function get_job(body){ compile_memory_limit, run_memory_limit, run_timeout, - compile_timeout + compile_timeout, } = body; return new Promise((resolve, reject) => { @@ -68,7 +107,7 @@ function get_job(body){ } if (typeof constraint_value !== 'number') { return reject({ - message: `If specified, ${constraint_name} must be a number` + message: `If specified, ${constraint_name} must be a number`, }); } if (configured_limit <= 0) { @@ -76,12 +115,12 @@ function get_job(body){ } if (constraint_value > configured_limit) { return reject({ - message: `${constraint_name} cannot exceed the configured limit of ${configured_limit}` + message: `${constraint_name} cannot exceed the configured limit of ${configured_limit}`, }); } if (constraint_value < 0) { return reject({ - message: `${constraint_name} must be non-negative` + message: `${constraint_name} must be non-negative`, }); } } @@ -91,20 +130,22 @@ function get_job(body){ run_timeout = run_timeout || rt.timeouts.run; compile_memory_limit = compile_memory_limit || rt.memory_limits.compile; run_timeout = run_timeout || rt.timeouts.run; - resolve(new Job({ - runtime: rt, - args: args || [], - stdin: stdin || "", - files, - timeouts: { - run: run_timeout, - compile: compile_timeout, - }, - memory_limits: { - run: run_memory_limit, - compile: compile_memory_limit, - } - })); + resolve( + new Job({ + runtime: rt, + args: args || [], + stdin: stdin || '', + files, + timeouts: { + run: run_timeout, + compile: compile_timeout, + }, + memory_limits: { + run: run_memory_limit, + compile: compile_memory_limit, + }, + }) + ); }); } @@ -123,88 +164,104 @@ router.use((req, res, next) => { }); router.ws('/connect', async (ws, req) => { - let job = null; let eventBus = new events.EventEmitter(); - eventBus.on("stdout", (data) => ws.send(JSON.stringify({type: "data", stream: "stdout", data: data.toString()}))) - eventBus.on("stderr", (data) => ws.send(JSON.stringify({type: "data", stream: "stderr", data: data.toString()}))) - eventBus.on("stage", (stage)=> ws.send(JSON.stringify({type: "stage", stage}))) - eventBus.on("exit", (stage, status) => ws.send(JSON.stringify({type: "exit", stage, ...status}))) + eventBus.on('stdout', data => + ws.send( + JSON.stringify({ + type: 'data', + stream: 'stdout', + data: data.toString(), + }) + ) + ); + eventBus.on('stderr', data => + ws.send( + JSON.stringify({ + type: 'data', + stream: 'stderr', + data: data.toString(), + }) + ) + ); + eventBus.on('stage', stage => + ws.send(JSON.stringify({ type: 'stage', stage })) + ); + eventBus.on('exit', (stage, status) => + ws.send(JSON.stringify({ type: 'exit', stage, ...status })) + ); - ws.on("message", async (data) => { - - try{ + ws.on('message', async data => { + try { const msg = JSON.parse(data); - switch(msg.type){ - case "init": - if(job === null){ + switch (msg.type) { + case 'init': + if (job === null) { job = await get_job(msg); await job.prime(); - ws.send(JSON.stringify({ - type: "runtime", - language: job.runtime.language, - version: job.runtime.version.raw - })) + ws.send( + JSON.stringify({ + type: 'runtime', + language: job.runtime.language, + version: job.runtime.version.raw, + }) + ); await job.execute_interactive(eventBus); - ws.close(4999, "Job Completed"); - - }else{ - ws.close(4000, "Already Initialized"); + ws.close(4999, 'Job Completed'); + } else { + ws.close(4000, 'Already Initialized'); } break; - case "data": - if(job !== null){ - if(msg.stream === "stdin"){ - eventBus.emit("stdin", msg.data) - }else{ - ws.close(4004, "Can only write to stdin") + case 'data': + if (job !== null) { + if (msg.stream === 'stdin') { + eventBus.emit('stdin', msg.data); + } else { + ws.close(4004, 'Can only write to stdin'); + } + } else { + ws.close(4003, 'Not yet initialized'); } - }else{ - ws.close(4003, "Not yet initialized") - } - break; - case "signal": - if(job !== null){ - if(SIGNALS.includes(msg.signal)){ - eventBus.emit("signal", msg.signal) - }else{ - ws.close(4005, "Invalid signal") + break; + case 'signal': + if (job !== null) { + if (SIGNALS.includes(msg.signal)) { + eventBus.emit('signal', msg.signal); + } else { + ws.close(4005, 'Invalid signal'); + } + } else { + ws.close(4003, 'Not yet initialized'); } - }else{ - ws.close(4003, "Not yet initialized") - } - break; + break; } - - }catch(error){ - ws.send(JSON.stringify({type: "error", message: error.message})) - ws.close(4002, "Notified Error") + } catch (error) { + ws.send(JSON.stringify({ type: 'error', message: error.message })); + ws.close(4002, 'Notified Error'); // ws.close message is limited to 123 characters, so we notify over WS then close. } - }) + }); - ws.on("close", async ()=>{ - if(job !== null){ - await job.cleanup() + ws.on('close', async () => { + if (job !== null) { + await job.cleanup(); } - }) + }); - setTimeout(()=>{ + setTimeout(() => { //Terminate the socket after 1 second, if not initialized. - if(job === null) - ws.close(4001, "Initialization Timeout"); - }, 1000) -}) + if (job === null) ws.close(4001, 'Initialization Timeout'); + }, 1000); +}); router.post('/execute', async (req, res) => { - - try{ + try { const job = await get_job(req.body); await job.prime(); @@ -214,7 +271,7 @@ router.post('/execute', async (req, res) => { await job.cleanup(); return res.status(200).send(result); - }catch(error){ + } catch (error) { return res.status(400).json(error); } }); diff --git a/api/src/config.js b/api/src/config.js index c191644..1a9eb3d 100644 --- a/api/src/config.js +++ b/api/src/config.js @@ -5,8 +5,7 @@ const logger = Logger.create('config'); function parse_overrides(overrides) { try { return JSON.parse(overrides); - } - catch (e) { + } catch (e) { return null; } } @@ -16,15 +15,20 @@ function validate_overrides(overrides, options) { for (let key in overrides[language]) { if ( ![ - 'max_process_count', 'max_open_files', 'max_file_size', - 'compile_memory_limit', 'run_memory_limit', 'compile_timeout', - 'run_timeout', 'output_max_size' + 'max_process_count', + 'max_open_files', + 'max_file_size', + 'compile_memory_limit', + 'run_memory_limit', + 'compile_timeout', + 'run_timeout', + 'output_max_size', ].includes(key) ) { logger.error(`Invalid overridden option: ${key}`); return false; } - let option = options.find((o) => o.key === key); + let option = options.find(o => o.key === key); let parser = option.parser; let raw = overrides[language][key]; let value = parser(raw); @@ -32,14 +36,19 @@ function validate_overrides(overrides, options) { for (let validator of validators) { let response = validator(value, raw); if (response !== true) { - logger.error(`Failed to validate overridden option: ${key}`, response); + logger.error( + `Failed to validate overridden option: ${key}`, + response + ); return false; } } overrides[language][key] = value; } // Modifies the reference - options[options.index_of(options.find((o) => o.key === 'limit_overrides'))] = overrides; + options[ + options.index_of(options.find(o => o.key === 'limit_overrides')) + ] = overrides; } return true; } @@ -135,32 +144,28 @@ const options = [ }, { key: 'compile_timeout', - desc: - 'Max time allowed for compile stage in milliseconds', + desc: 'Max time allowed for compile stage in milliseconds', default: 10000, // 10 seconds parser: parse_int, validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`], }, { key: 'run_timeout', - desc: - 'Max time allowed for run stage in milliseconds', + desc: 'Max time allowed for run stage in milliseconds', default: 3000, // 3 seconds parser: parse_int, validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`], }, { key: 'compile_memory_limit', - desc: - 'Max memory usage for compile stage in bytes (set to -1 for no limit)', + desc: 'Max memory usage for compile stage in bytes (set to -1 for no limit)', default: -1, // no limit parser: parse_int, validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`], }, { key: 'run_memory_limit', - desc: - 'Max memory usage for run stage in bytes (set to -1 for no limit)', + desc: 'Max memory usage for run stage in bytes (set to -1 for no limit)', default: -1, // no limit parser: parse_int, validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`], @@ -177,7 +182,7 @@ const options = [ desc: 'Maximum number of concurrent jobs to run at one time', default: 64, parser: parse_int, - validators: [(x) => x > 0 || `${x} cannot be negative`] + validators: [x => x > 0 || `${x} cannot be negative`], }, { key: 'limit_overrides', @@ -187,10 +192,12 @@ const options = [ default: {}, parser: parse_overrides, validators: [ - (x) => !!x || `Invalid JSON format for the overrides\n${x}`, - (overrides, _, options) => validate_overrides(overrides, options) || `Failed to validate the overrides` - ] - } + x => !!x || `Invalid JSON format for the overrides\n${x}`, + (overrides, _, options) => + validate_overrides(overrides, options) || + `Failed to validate the overrides`, + ], + }, ]; logger.info(`Loading Configuration from environment`); diff --git a/api/src/index.js b/api/src/index.js index afd4d15..025f5ae 100644 --- a/api/src/index.js +++ b/api/src/index.js @@ -15,8 +15,6 @@ const logger = Logger.create('index'); const app = express(); expressWs(app); - - (async () => { logger.info('Setting loglevel to', config.log_level); Logger.setLogLevel(config.log_level); diff --git a/api/src/job.js b/api/src/job.js index 552463a..c2e8bc8 100644 --- a/api/src/job.js +++ b/api/src/job.js @@ -19,15 +19,12 @@ let gid = 0; let remainingJobSpaces = config.max_concurrent_jobs; let jobQueue = []; - -setInterval(()=>{ +setInterval(() => { // Every 10ms try resolve a new job, if there is an available slot - if(jobQueue.length > 0 && remainingJobSpaces > 0){ - jobQueue.shift()() + if (jobQueue.length > 0 && remainingJobSpaces > 0) { + jobQueue.shift()(); } -}, 10) - - +}, 10); class Job { constructor({ runtime, files, args, stdin }) { @@ -59,11 +56,11 @@ class Job { } async prime() { - if(remainingJobSpaces < 1){ - logger.info(`Awaiting job slot uuid=${this.uuid}`) - await new Promise((resolve)=>{ - jobQueue.push(resolve) - }) + if (remainingJobSpaces < 1) { + logger.info(`Awaiting job slot uuid=${this.uuid}`); + await new Promise(resolve => { + jobQueue.push(resolve); + }); } logger.info(`Priming job uuid=${this.uuid}`); @@ -79,10 +76,15 @@ class Job { let file_path = path.join(this.dir, file.name); const rel = path.relative(this.dir, file_path); - if(rel.startsWith("..")) - throw Error(`File path "${file.name}" tries to escape parent directory: ${rel}`) + if (rel.startsWith('..')) + throw Error( + `File path "${file.name}" tries to escape parent directory: ${rel}` + ); - await fs.mkdir(path.dirname(file_path), {recursive: true, mode: 0o700}) + await fs.mkdir(path.dirname(file_path), { + recursive: true, + mode: 0o700, + }); await fs.chown(path.dirname(file_path), this.uid, this.gid); await fs.write_file(file_path, file.content); @@ -127,34 +129,33 @@ class Job { detached: true, //give this process its own process group }); - if(eventBus === null){ + if (eventBus === null) { proc.stdin.write(this.stdin); proc.stdin.end(); proc.stdin.destroy(); - }else{ - eventBus.on("stdin", (data) => { + } else { + eventBus.on('stdin', data => { proc.stdin.write(data); - }) + }); - eventBus.on("kill", (signal) => { - proc.kill(signal) - }) + eventBus.on('kill', signal => { + proc.kill(signal); + }); } - const kill_timeout = set_timeout( - async _ => { - logger.info(`Timeout exceeded timeout=${timeout} uuid=${this.uuid}`) - process.kill(proc.pid, 'SIGKILL') - }, - timeout - ); + const kill_timeout = set_timeout(async _ => { + logger.info( + `Timeout exceeded timeout=${timeout} uuid=${this.uuid}` + ); + process.kill(proc.pid, 'SIGKILL'); + }, timeout); proc.stderr.on('data', async data => { - if(eventBus !== null) { - eventBus.emit("stderr", data); + if (eventBus !== null) { + eventBus.emit('stderr', data); } else if (stderr.length > this.runtime.output_max_size) { - logger.info(`stderr length exceeded uuid=${this.uuid}`) - process.kill(proc.pid, 'SIGKILL') + logger.info(`stderr length exceeded uuid=${this.uuid}`); + process.kill(proc.pid, 'SIGKILL'); } else { stderr += data; output += data; @@ -162,11 +163,11 @@ class Job { }); proc.stdout.on('data', async data => { - if(eventBus !== null){ - eventBus.emit("stdout", data); + if (eventBus !== null) { + eventBus.emit('stdout', data); } else if (stdout.length > this.runtime.output_max_size) { - logger.info(`stdout length exceeded uuid=${this.uuid}`) - process.kill(proc.pid, 'SIGKILL') + logger.info(`stdout length exceeded uuid=${this.uuid}`); + process.kill(proc.pid, 'SIGKILL'); } else { stdout += data; output += data; @@ -179,14 +180,14 @@ class Job { proc.stderr.destroy(); proc.stdout.destroy(); - await this.cleanup_processes() - logger.debug(`Finished exit cleanup uuid=${this.uuid}`) + await this.cleanup_processes(); + logger.debug(`Finished exit cleanup uuid=${this.uuid}`); }; proc.on('exit', async (code, signal) => { await exit_cleanup(); - resolve({stdout, stderr, code, signal, output }); + resolve({ stdout, stderr, code, signal, output }); }); proc.on('error', async err => { @@ -243,7 +244,7 @@ class Job { }; } - async execute_interactive(eventBus){ + async execute_interactive(eventBus) { if (this.state !== job_states.PRIMED) { throw new Error( 'Job must be in primed state, current state: ' + @@ -252,27 +253,27 @@ class Job { } logger.info( - `Interactively executing job uuid=${this.uuid} uid=${this.uid} gid=${ - this.gid - } runtime=${this.runtime.toString()}` + `Interactively executing job uuid=${this.uuid} uid=${ + this.uid + } gid=${this.gid} runtime=${this.runtime.toString()}` ); - if(this.runtime.compiled){ - eventBus.emit("stage", "compile") - const {error, code, signal} = await this.safe_call( + if (this.runtime.compiled) { + eventBus.emit('stage', 'compile'); + const { error, code, signal } = await this.safe_call( path.join(this.runtime.pkgdir, 'compile'), this.files.map(x => x.name), this.runtime.timeouts.compile, this.runtime.memory_limits.compile, eventBus - ) + ); - eventBus.emit("exit", "compile", {error, code, signal}) + eventBus.emit('exit', 'compile', { error, code, signal }); } logger.debug('Running'); - eventBus.emit("stage", "run") - const {error, code, signal} = await this.safe_call( + eventBus.emit('stage', 'run'); + const { error, code, signal } = await this.safe_call( path.join(this.runtime.pkgdir, 'run'), [this.files[0].name, ...this.args], this.runtime.timeouts.run, @@ -280,47 +281,50 @@ class Job { eventBus ); - eventBus.emit("exit", "run", {error, code, signal}) - + eventBus.emit('exit', 'run', { error, code, signal }); this.state = job_states.EXECUTED; } async cleanup_processes(dont_wait = []) { let processes = [1]; - logger.debug(`Cleaning up processes uuid=${this.uuid}`) + logger.debug(`Cleaning up processes uuid=${this.uuid}`); while (processes.length > 0) { - processes = [] + processes = []; + const proc_ids = await fs.readdir('/proc'); - const proc_ids = await fs.readdir("/proc"); + processes = await Promise.all( + proc_ids.map(async proc_id => { + if (isNaN(proc_id)) return -1; + try { + const proc_status = await fs.read_file( + path.join('/proc', proc_id, 'status') + ); + const proc_lines = proc_status.to_string().split('\n'); + const uid_line = proc_lines.find(line => + line.starts_with('Uid:') + ); + const [_, ruid, euid, suid, fuid] = + uid_line.split(/\s+/); + if (ruid == this.uid || euid == this.uid) + return parse_int(proc_id); + } catch { + return -1; + } - processes = await Promise.all(proc_ids.map(async (proc_id) => { - if(isNaN(proc_id)) return -1; - try{ - const proc_status = await fs.read_file(path.join("/proc",proc_id,"status")); - const proc_lines = proc_status.to_string().split("\n") - const uid_line = proc_lines.find(line=>line.starts_with("Uid:")) - const [_, ruid, euid, suid, fuid] = uid_line.split(/\s+/); - - if(ruid == this.uid || euid == this.uid) - return parse_int(proc_id) - - }catch{ - return -1 - } - - return -1 - })) - - processes = processes.filter(p => p > 0) - - if(processes.length > 0) - logger.debug(`Got processes to kill: ${processes} uuid=${this.uuid}`) + return -1; + }) + ); + processes = processes.filter(p => p > 0); + if (processes.length > 0) + logger.debug( + `Got processes to kill: ${processes} uuid=${this.uuid}` + ); for (const proc of processes) { // First stop the processes, but keep their resources allocated so they cant re-fork @@ -339,12 +343,11 @@ class Job { // Could already be dead and just needs to be waited on } - if(!dont_wait.includes(proc)) - wait_pid(proc); + if (!dont_wait.includes(proc)) wait_pid(proc); } } - logger.debug(`Cleaned up processes uuid=${this.uuid}`) + logger.debug(`Cleaned up processes uuid=${this.uuid}`); } async cleanup_filesystem() { @@ -382,7 +385,6 @@ class Job { } } - module.exports = { Job, }; diff --git a/api/src/package.js b/api/src/package.js index 1300c8c..4e4630f 100644 --- a/api/src/package.js +++ b/api/src/package.js @@ -74,9 +74,8 @@ class Package { await new Promise((resolve, reject) => { read_stream.on('data', chunk => hash.update(chunk)); read_stream.on('end', () => resolve()); - read_stream.on('error', error => reject(error)) + read_stream.on('error', error => reject(error)); }); - const cs = hash.digest('hex'); diff --git a/api/src/runtime.js b/api/src/runtime.js index 60d3c23..6c6f10e 100644 --- a/api/src/runtime.js +++ b/api/src/runtime.js @@ -9,8 +9,17 @@ const runtimes = []; class Runtime { constructor({ - language, version, aliases, pkgdir, runtime, timeouts, memory_limits, max_process_count, - max_open_files, max_file_size, output_max_size + language, + version, + aliases, + pkgdir, + runtime, + timeouts, + memory_limits, + max_process_count, + max_open_files, + max_file_size, + output_max_size, }) { this.language = language; this.version = version; @@ -25,37 +34,67 @@ class Runtime { this.output_max_size = output_max_size; } - static compute_single_limit(language_name, limit_name, language_limit_overrides) { + static compute_single_limit( + language_name, + limit_name, + language_limit_overrides + ) { return ( - config.limit_overrides[language_name] && config.limit_overrides[language_name][limit_name] - || language_limit_overrides && language_limit_overrides[limit_name] - || config[limit_name] + (config.limit_overrides[language_name] && + config.limit_overrides[language_name][limit_name]) || + (language_limit_overrides && + language_limit_overrides[limit_name]) || + config[limit_name] ); } static compute_all_limits(language_name, language_limit_overrides) { return { timeouts: { - compile: - this.compute_single_limit(language_name, 'compile_timeout', language_limit_overrides), - run: - this.compute_single_limit(language_name, 'run_timeout', language_limit_overrides) + compile: this.compute_single_limit( + language_name, + 'compile_timeout', + language_limit_overrides + ), + run: this.compute_single_limit( + language_name, + 'run_timeout', + language_limit_overrides + ), }, memory_limits: { - compile: - this.compute_single_limit(language_name, 'compile_memory_limit', language_limit_overrides), - run: - this.compute_single_limit(language_name, 'run_memory_limit', language_limit_overrides) + compile: this.compute_single_limit( + language_name, + 'compile_memory_limit', + language_limit_overrides + ), + run: this.compute_single_limit( + language_name, + 'run_memory_limit', + language_limit_overrides + ), }, - max_process_count: - this.compute_single_limit(language_name, 'max_process_count', language_limit_overrides), - max_open_files: - this.compute_single_limit(language_name, 'max_open_files', language_limit_overrides), - max_file_size: - this.compute_single_limit(language_name, 'max_file_size', language_limit_overrides), - output_max_size: - this.compute_single_limit(language_name, 'output_max_size', language_limit_overrides), - } + max_process_count: this.compute_single_limit( + language_name, + 'max_process_count', + language_limit_overrides + ), + max_open_files: this.compute_single_limit( + language_name, + 'max_open_files', + language_limit_overrides + ), + max_file_size: this.compute_single_limit( + language_name, + 'max_file_size', + language_limit_overrides + ), + output_max_size: this.compute_single_limit( + language_name, + 'output_max_size', + language_limit_overrides + ), + }; } static load_package(package_dir) { @@ -63,7 +102,14 @@ class Runtime { fss.read_file_sync(path.join(package_dir, 'pkg-info.json')) ); - let { language, version, build_platform, aliases, provides, limit_overrides } = info; + let { + language, + version, + build_platform, + aliases, + provides, + limit_overrides, + } = info; version = semver.parse(version); if (build_platform !== globals.platform) { @@ -83,7 +129,10 @@ class Runtime { version, pkgdir: package_dir, runtime: language, - ...Runtime.compute_all_limits(lang.language, lang.limit_overrides) + ...Runtime.compute_all_limits( + lang.language, + lang.limit_overrides + ), }) ); }); @@ -94,7 +143,7 @@ class Runtime { version, aliases, pkgdir: package_dir, - ...Runtime.compute_all_limits(language, limit_overrides) + ...Runtime.compute_all_limits(language, limit_overrides), }) ); } diff --git a/cli/.gitignore b/cli/.gitignore deleted file mode 100644 index b512c09..0000000 --- a/cli/.gitignore +++ /dev/null @@ -1 +0,0 @@ -node_modules \ No newline at end of file diff --git a/cli/commands/execute.js b/cli/commands/execute.js index abb1f63..c69d3a0 100644 --- a/cli/commands/execute.js +++ b/cli/commands/execute.js @@ -3,8 +3,44 @@ const path = require('path'); const chalk = require('chalk'); const WebSocket = require('ws'); -const SIGNALS = ["SIGABRT","SIGALRM","SIGBUS","SIGCHLD","SIGCLD","SIGCONT","SIGEMT","SIGFPE","SIGHUP","SIGILL","SIGINFO","SIGINT","SIGIO","SIGIOT","SIGLOST","SIGPIPE","SIGPOLL","SIGPROF","SIGPWR","SIGQUIT","SIGSEGV","SIGSTKFLT","SIGTSTP","SIGSYS","SIGTERM","SIGTRAP","SIGTTIN","SIGTTOU","SIGUNUSED","SIGURG","SIGUSR1","SIGUSR2","SIGVTALRM","SIGXCPU","SIGXFSZ","SIGWINCH"] - +const SIGNALS = [ + 'SIGABRT', + 'SIGALRM', + 'SIGBUS', + 'SIGCHLD', + 'SIGCLD', + 'SIGCONT', + 'SIGEMT', + 'SIGFPE', + 'SIGHUP', + 'SIGILL', + 'SIGINFO', + 'SIGINT', + 'SIGIO', + 'SIGIOT', + 'SIGLOST', + 'SIGPIPE', + 'SIGPOLL', + 'SIGPROF', + 'SIGPWR', + 'SIGQUIT', + 'SIGSEGV', + 'SIGSTKFLT', + 'SIGTSTP', + 'SIGSYS', + 'SIGTERM', + 'SIGTRAP', + 'SIGTTIN', + 'SIGTTOU', + 'SIGUNUSED', + 'SIGURG', + 'SIGUSR1', + 'SIGUSR2', + 'SIGVTALRM', + 'SIGXCPU', + 'SIGXFSZ', + 'SIGWINCH', +]; exports.command = ['execute [args..]']; exports.aliases = ['run']; @@ -15,18 +51,18 @@ exports.builder = { string: true, desc: 'Set the version of the language to use', alias: ['l'], - default: '*' + default: '*', }, stdin: { boolean: true, desc: 'Read input from stdin and pass to executor', - alias: ['i'] + alias: ['i'], }, run_timeout: { alias: ['rt', 'r'], number: true, desc: 'Milliseconds before killing run process', - default: 3000 + default: 3000, }, compile_timeout: { alias: ['ct', 'c'], @@ -42,117 +78,126 @@ exports.builder = { interactive: { boolean: true, alias: ['t'], - desc: 'Run interactively using WebSocket transport' + desc: 'Run interactively using WebSocket transport', }, status: { boolean: true, alias: ['s'], - desc: 'Output additional status to stderr' - } + desc: 'Output additional status to stderr', + }, }; -async function handle_interactive(files, argv){ - const ws = new WebSocket(argv.pistonUrl.replace("http", "ws") + "/api/v2/connect") +async function handle_interactive(files, argv) { + const ws = new WebSocket( + argv.pistonUrl.replace('http', 'ws') + '/api/v2/connect' + ); - const log_message = (process.stderr.isTTY && argv.status) ? console.error : ()=>{}; + const log_message = + process.stderr.isTTY && argv.status ? console.error : () => {}; - process.on("exit", ()=>{ + process.on('exit', () => { ws.close(); process.stdin.end(); process.stdin.destroy(); - process.exit(); - }) + process.exit(); + }); - for(const signal of SIGNALS){ - process.on(signal, ()=>{ - ws.send(JSON.stringify({type: 'signal', signal})) - }) + for (const signal of SIGNALS) { + process.on(signal, () => { + ws.send(JSON.stringify({ type: 'signal', signal })); + }); } - - - ws.on('open', ()=>{ + ws.on('open', () => { const request = { - type: "init", + type: 'init', language: argv.language, version: argv['language_version'], files: files, args: argv.args, compile_timeout: argv.ct, - run_timeout: argv.rt - } + run_timeout: argv.rt, + }; - ws.send(JSON.stringify(request)) - log_message(chalk.white.bold("Connected")) + ws.send(JSON.stringify(request)); + log_message(chalk.white.bold('Connected')); process.stdin.resume(); - process.stdin.on("data", (data) => { - ws.send(JSON.stringify({ - type: "data", - stream: "stdin", - data: data.toString() - })) - }) - }) + process.stdin.on('data', data => { + ws.send( + JSON.stringify({ + type: 'data', + stream: 'stdin', + data: data.toString(), + }) + ); + }); + }); - ws.on("close", (code, reason)=>{ + ws.on('close', (code, reason) => { log_message( - chalk.white.bold("Disconnected: "), - chalk.white.bold("Reason: "), + chalk.white.bold('Disconnected: '), + chalk.white.bold('Reason: '), chalk.yellow(`"${reason}"`), - chalk.white.bold("Code: "), - chalk.yellow(`"${code}"`), - ) - process.stdin.pause() - }) + chalk.white.bold('Code: '), + chalk.yellow(`"${code}"`) + ); + process.stdin.pause(); + }); - ws.on('message', function(data){ + ws.on('message', function (data) { const msg = JSON.parse(data); - - switch(msg.type){ - case "runtime": - log_message(chalk.bold.white("Runtime:"), chalk.yellow(`${msg.language} ${msg.version}`)) + + switch (msg.type) { + case 'runtime': + log_message( + chalk.bold.white('Runtime:'), + chalk.yellow(`${msg.language} ${msg.version}`) + ); break; - case "stage": - log_message(chalk.bold.white("Stage:"), chalk.yellow(msg.stage)) + case 'stage': + log_message( + chalk.bold.white('Stage:'), + chalk.yellow(msg.stage) + ); break; - case "data": - if(msg.stream == "stdout") process.stdout.write(msg.data) - else if(msg.stream == "stderr") process.stderr.write(msg.data) - else log_message(chalk.bold.red(`(${msg.stream}) `), msg.data) + case 'data': + if (msg.stream == 'stdout') process.stdout.write(msg.data); + else if (msg.stream == 'stderr') process.stderr.write(msg.data); + else log_message(chalk.bold.red(`(${msg.stream}) `), msg.data); break; - case "exit": - if(msg.signal === null) + case 'exit': + if (msg.signal === null) log_message( - chalk.white.bold("Stage"), + chalk.white.bold('Stage'), chalk.yellow(msg.stage), - chalk.white.bold("exited with code"), + chalk.white.bold('exited with code'), chalk.yellow(msg.code) - ) + ); else log_message( - chalk.white.bold("Stage"), + chalk.white.bold('Stage'), chalk.yellow(msg.stage), - chalk.white.bold("exited with signal"), + chalk.white.bold('exited with signal'), chalk.yellow(msg.signal) - ) - break; + ); + break; default: - log_message(chalk.red.bold("Unknown message:"), msg) + log_message(chalk.red.bold('Unknown message:'), msg); } - }) - + }); } async function run_non_interactively(files, argv) { - - - const stdin = (argv.stdin && await new Promise((resolve, _) => { - let data = ''; - process.stdin.on('data', d => data += d); - process.stdin.on('end', _ => resolve(data)); - })) || ''; + const stdin = + (argv.stdin && + (await new Promise((resolve, _) => { + let data = ''; + process.stdin.on('data', d => (data += d)); + process.stdin.on('end', _ => resolve(data)); + }))) || + ''; const request = { language: argv.language, @@ -161,7 +206,7 @@ async function run_non_interactively(files, argv) { args: argv.args, stdin, compile_timeout: argv.ct, - run_timeout: argv.rt + run_timeout: argv.rt, }; let { data: response } = await argv.axios.post('/api/v2/execute', request); @@ -170,13 +215,13 @@ async function run_non_interactively(files, argv) { console.log(chalk.bold(`== ${name} ==`)); if (ctx.stdout) { - console.log(chalk.bold(`STDOUT`)) - console.log(ctx.stdout.replace(/\n/g,'\n ')) + console.log(chalk.bold(`STDOUT`)); + console.log(ctx.stdout.replace(/\n/g, '\n ')); } if (ctx.stderr) { - console.log(chalk.bold(`STDERR`)) - console.log(ctx.stderr.replace(/\n/g,'\n ')) + console.log(chalk.bold(`STDERR`)); + console.log(ctx.stderr.replace(/\n/g, '\n ')); } if (ctx.code) { @@ -187,12 +232,9 @@ async function run_non_interactively(files, argv) { } if (ctx.signal) { - console.log( - chalk.bold(`Signal:`), - chalk.bold.yellow(ctx.signal) - ); + console.log(chalk.bold(`Signal:`), chalk.bold.yellow(ctx.signal)); } - } + }; if (response.compile) { step('Compile', response.compile); @@ -201,17 +243,14 @@ async function run_non_interactively(files, argv) { step('Run', response.run); } -exports.handler = async (argv) => { - const files = [...(argv.files || []),argv.file] - .map(file_path => { - return { - name: path.basename(file_path), - content: fs.readFileSync(file_path).toString() - }; - }); +exports.handler = async argv => { + const files = [...(argv.files || []), argv.file].map(file_path => { + return { + name: path.basename(file_path), + content: fs.readFileSync(file_path).toString(), + }; + }); - if(argv.interactive) await handle_interactive(files, argv); + if (argv.interactive) await handle_interactive(files, argv); else await run_non_interactively(files, argv); -} - - +}; diff --git a/cli/commands/ppman.js b/cli/commands/ppman.js index 8d1cb34..ad2c879 100644 --- a/cli/commands/ppman.js +++ b/cli/commands/ppman.js @@ -2,6 +2,4 @@ exports.command = 'ppman'; exports.aliases = ['pkg']; exports.describe = 'Package Manager'; -exports.builder = yargs => yargs - .commandDir('ppman_commands') - .demandCommand(); +exports.builder = yargs => yargs.commandDir('ppman_commands').demandCommand(); diff --git a/cli/commands/ppman_commands/install.js b/cli/commands/ppman_commands/install.js index 8b2baf9..a47665d 100644 --- a/cli/commands/ppman_commands/install.js +++ b/cli/commands/ppman_commands/install.js @@ -4,30 +4,31 @@ exports.command = ['install ']; exports.aliases = ['i']; exports.describe = 'Installs the named package'; - - //Splits the package into it's language and version function split_package(package) { - [language, language_version] = package.split("=") + [language, language_version] = package.split('='); res = { language: language, - version: language_version || "*" + version: language_version || '*', }; - return res + return res; } const msg_format = { - color: p => `${p.language ? chalk.green.bold('✓') : chalk.red.bold('❌')} Installation ${p.language ? 'succeeded' : 'failed: ' + p.message}`, - monochrome: p => `Installation ${p.language ? 'succeeded' : 'failed: ' + p.message}`, - json: JSON.stringify + color: p => + `${ + p.language ? chalk.green.bold('✓') : chalk.red.bold('❌') + } Installation ${p.language ? 'succeeded' : 'failed: ' + p.message}`, + monochrome: p => + `Installation ${p.language ? 'succeeded' : 'failed: ' + p.message}`, + json: JSON.stringify, }; exports.handler = async ({ axios, packages }) => { const requests = packages.map(package => split_package(package)); for (request of requests) { try { - const install = await axios.post(`/api/v2/packages`, request); console.log(msg_format.color(install.data)); @@ -35,5 +36,4 @@ exports.handler = async ({ axios, packages }) => { console.error(response.data.message); } } - -} +}; diff --git a/cli/commands/ppman_commands/list.js b/cli/commands/ppman_commands/list.js index a45030c..0b09667 100644 --- a/cli/commands/ppman_commands/list.js +++ b/cli/commands/ppman_commands/list.js @@ -5,17 +5,21 @@ exports.aliases = ['l']; exports.describe = 'Lists all available packages'; const msg_format = { - color: p => `${chalk[p.installed ? 'green':'red']('•')} ${p.language} ${p.language_version}`, - monochrome: p => `${p.language} ${p.language_version} ${p.installed ? '(INSTALLED)': ''}`, - json: JSON.stringify + color: p => + `${chalk[p.installed ? 'green' : 'red']('•')} ${p.language} ${ + p.language_version + }`, + monochrome: p => + `${p.language} ${p.language_version} ${ + p.installed ? '(INSTALLED)' : '' + }`, + json: JSON.stringify, }; exports.handler = async ({ axios }) => { const packages = await axios.get('/api/v2/packages'); - const pkg_msg = packages.data - .map(msg_format.color) - .join('\n'); + const pkg_msg = packages.data.map(msg_format.color).join('\n'); console.log(pkg_msg); -} +}; diff --git a/cli/commands/ppman_commands/spec.js b/cli/commands/ppman_commands/spec.js index d558810..9c04857 100644 --- a/cli/commands/ppman_commands/spec.js +++ b/cli/commands/ppman_commands/spec.js @@ -1,49 +1,53 @@ const chalk = require('chalk'); const fs = require('fs/promises'); -const minimatch = require("minimatch"); +const minimatch = require('minimatch'); const semver = require('semver'); exports.command = ['spec ']; exports.aliases = ['s']; -exports.describe = 'Install the packages described in the spec file, uninstalling packages which aren\'t in the list' +exports.describe = + "Install the packages described in the spec file, uninstalling packages which aren't in the list"; -function does_match(package, rule){ +function does_match(package, rule) { const nameMatch = minimatch(package.language, rule.package_selector); - const versionMatch = semver.satisfies(package.language_version, rule.version_selector) + const versionMatch = semver.satisfies( + package.language_version, + rule.version_selector + ); return nameMatch && versionMatch; } -exports.handler = async ({axios, specfile}) => { +exports.handler = async ({ axios, specfile }) => { const spec_contents = await fs.readFile(specfile); - const spec_lines = spec_contents.toString().split("\n"); + const spec_lines = spec_contents.toString().split('\n'); const rules = []; - for(const line of spec_lines){ + for (const line of spec_lines) { const rule = { _raw: line.trim(), comment: false, package_selector: null, version_selector: null, - negate: false + negate: false, }; - if(line.starts_with("#")){ + if (line.starts_with('#')) { rule.comment = true; - }else { + } else { let l = line.trim(); - if(line.starts_with("!")){ + if (line.starts_with('!')) { rule.negate = true; l = line.slice(1).trim(); } - const [pkg, ver] = l.split(" ", 2); + const [pkg, ver] = l.split(' ', 2); rule.package_selector = pkg; rule.version_selector = ver; } - if(rule._raw.length != 0) rules.push(rule); + if (rule._raw.length != 0) rules.push(rule); } const packages_req = await axios.get('/api/v2/packages'); @@ -53,108 +57,127 @@ exports.handler = async ({axios, specfile}) => { let ensure_packages = []; - for(const rule of rules){ - if(rule.comment) continue; + for (const rule of rules) { + if (rule.comment) continue; const matches = []; - if(!rule.negate){ - for(const package of packages){ - if(does_match(package, rule)) - matches.push(package) + if (!rule.negate) { + for (const package of packages) { + if (does_match(package, rule)) matches.push(package); } - const latest_matches = matches.filter( - pkg => { - const versions = matches - .filter(x=>x.language == pkg.language) - .map(x=>x.language_version).sort(semver.rcompare); - return versions[0] == pkg.language_version - } - ); + const latest_matches = matches.filter(pkg => { + const versions = matches + .filter(x => x.language == pkg.language) + .map(x => x.language_version) + .sort(semver.rcompare); + return versions[0] == pkg.language_version; + }); - for(const match of latest_matches){ - if(!ensure_packages.find(pkg => pkg.language == match.language && pkg.language_version == match.language_version)) - ensure_packages.push(match) + for (const match of latest_matches) { + if ( + !ensure_packages.find( + pkg => + pkg.language == match.language && + pkg.language_version == match.language_version + ) + ) + ensure_packages.push(match); } - }else{ + } else { ensure_packages = ensure_packages.filter( pkg => !does_match(pkg, rule) - ) + ); } - - } const operations = []; - for(const package of ensure_packages){ - if(!package.installed) + for (const package of ensure_packages) { + if (!package.installed) operations.push({ - type: "install", + type: 'install', package: package.language, - version: package.language_version + version: package.language_version, }); } - for(const installed_package of installed){ - if(!ensure_packages.find( - pkg => pkg.language == installed_package.language && - pkg.language_version == installed_package.language_version - )) + for (const installed_package of installed) { + if ( + !ensure_packages.find( + pkg => + pkg.language == installed_package.language && + pkg.language_version == installed_package.language_version + ) + ) operations.push({ - type: "uninstall", + type: 'uninstall', package: installed_package.language, - version: installed_package.language_version - }) + version: installed_package.language_version, + }); } - console.log(chalk.bold.yellow("Actions")) - for(const op of operations){ - console.log((op.type == "install" ? chalk.green("Install") : chalk.red("Uninstall")) + ` ${op.package} ${op.version}`) + console.log(chalk.bold.yellow('Actions')); + for (const op of operations) { + console.log( + (op.type == 'install' + ? chalk.green('Install') + : chalk.red('Uninstall')) + ` ${op.package} ${op.version}` + ); } - if(operations.length == 0){ - console.log(chalk.gray("None")) + if (operations.length == 0) { + console.log(chalk.gray('None')); } - for(const op of operations){ - if(op.type == "install"){ - try{ + for (const op of operations) { + if (op.type == 'install') { + try { const install = await axios.post(`/api/v2/packages`, { language: op.package, - version: op.version + version: op.version, }); - if(!install.data.language) + if (!install.data.language) throw new Error(install.data.message); // Go to exception handler - console.log(chalk.bold.green("Installed"), op.package, op.version) - - }catch(e){ - console.log(chalk.bold.red("Failed to install") + ` ${op.package} ${op.version}:`, e.message) + console.log( + chalk.bold.green('Installed'), + op.package, + op.version + ); + } catch (e) { + console.log( + chalk.bold.red('Failed to install') + + ` ${op.package} ${op.version}:`, + e.message + ); } - } - else if(op.type == "uninstall"){ - try{ + } else if (op.type == 'uninstall') { + try { const install = await axios.delete(`/api/v2/packages`, { data: { language: op.package, - version: op.version - } + version: op.version, + }, }); - if(!install.data.language) + if (!install.data.language) throw new Error(install.data.message); // Go to exception handler - console.log(chalk.bold.green("Uninstalled"), op.package, op.version) - - }catch(e){ - console.log(chalk.bold.red("Failed to uninstall") + ` ${op.package} ${op.version}:`, e.message) + console.log( + chalk.bold.green('Uninstalled'), + op.package, + op.version + ); + } catch (e) { + console.log( + chalk.bold.red('Failed to uninstall') + + ` ${op.package} ${op.version}:`, + e.message + ); } } } - - - -} \ No newline at end of file +}; diff --git a/cli/commands/ppman_commands/uninstall.js b/cli/commands/ppman_commands/uninstall.js index f174fdb..21d2198 100644 --- a/cli/commands/ppman_commands/uninstall.js +++ b/cli/commands/ppman_commands/uninstall.js @@ -6,31 +6,36 @@ exports.describe = 'Uninstalls the named package'; //Splits the package into it's language and version function split_package(package) { - [language, language_version] = package.split("=") + [language, language_version] = package.split('='); res = { language: language, - version: language_version || "*" + version: language_version || '*', }; - return res + return res; } const msg_format = { - color: p => `${p.language ? chalk.green.bold('✓') : chalk.red.bold('❌')} Uninstallation ${p.language ? 'succeeded' : 'failed: ' + p.message}`, - monochrome: p => `Uninstallation ${p.language ? 'succeeded' : 'failed: ' + p.message}`, - json: JSON.stringify + color: p => + `${ + p.language ? chalk.green.bold('✓') : chalk.red.bold('❌') + } Uninstallation ${p.language ? 'succeeded' : 'failed: ' + p.message}`, + monochrome: p => + `Uninstallation ${p.language ? 'succeeded' : 'failed: ' + p.message}`, + json: JSON.stringify, }; exports.handler = async ({ axios, packages }) => { const requests = packages.map(package => split_package(package)); for (request of requests) { try { - - const uninstall = await axios.delete(`/api/v2/packages`, { data: request }); + const uninstall = await axios.delete(`/api/v2/packages`, { + data: request, + }); console.log(msg_format.color(uninstall.data)); } catch ({ response }) { - console.error(response.data.message) + console.error(response.data.message); } } -} \ No newline at end of file +}; diff --git a/cli/index.js b/cli/index.js index c0c25ee..340cdab 100755 --- a/cli/index.js +++ b/cli/index.js @@ -6,8 +6,8 @@ const axios_instance = argv => { argv.axios = axios.create({ baseURL: argv['piston-url'], headers: { - 'Content-Type': 'application/json' - } + 'Content-Type': 'application/json', + }, }); return argv; @@ -18,12 +18,11 @@ require('yargs')(process.argv.slice(2)) alias: ['u'], default: 'http://127.0.0.1:2000', desc: 'Piston API URL', - string: true + string: true, }) .middleware(axios_instance) .scriptName('piston') .commandDir('commands') .demandCommand() .help() - .wrap(72) - .argv; + .wrap(72).argv; diff --git a/docker-compose.dev.yaml b/docker-compose.dev.yaml index 5c3b61e..25d8fe6 100644 --- a/docker-compose.dev.yaml +++ b/docker-compose.dev.yaml @@ -1,24 +1,24 @@ -version: "3.2" +version: '3.2' services: - api: - build: api - container_name: piston_api - cap_add: - - CAP_SYS_ADMIN - restart: always - ports: - - 2000:2000 - volumes: - - ./data/piston:/piston - environment: - - PISTON_REPO_URL=http://repo:8000/index - tmpfs: - - /piston/jobs:exec + api: + build: api + container_name: piston_api + cap_add: + - CAP_SYS_ADMIN + restart: always + ports: + - 2000:2000 + volumes: + - ./data/piston:/piston + environment: + - PISTON_REPO_URL=http://repo:8000/index + tmpfs: + - /piston/jobs:exec - repo: # Local testing of packages - build: repo - container_name: piston_repo - command: ["--no-build"] # Don't build anything - volumes: - - .:/piston + repo: # Local testing of packages + build: repo + container_name: piston_repo + command: ['--no-build'] # Don't build anything + volumes: + - .:/piston diff --git a/docs/api-v2.md b/docs/api-v2.md index 111b514..d5cb486 100644 --- a/docs/api-v2.md +++ b/docs/api-v2.md @@ -17,10 +17,10 @@ Returns a list of available languages, including the version, runtime and aliase #### Response -- `[].language`: Name of the language -- `[].version`: Version of the runtime -- `[].aliases`: List of alternative names that can be used for the language -- `[].runtime` (_optional_): Name of the runtime used to run the langage, only provided if alternative runtimes exist for the language +- `[].language`: Name of the language +- `[].version`: Version of the runtime +- `[].aliases`: List of alternative names that can be used for the language +- `[].runtime` (_optional_): Name of the runtime used to run the langage, only provided if alternative runtimes exist for the language #### Example @@ -55,34 +55,34 @@ Runs the given code, using the given runtime and arguments, returning the result #### Request -- `language`: Name or alias of a language listed in [runtimes](#runtimes) -- `version`: SemVer version selector of a language listed in [runtimes](#runtimes) -- `files`: An array of files which should be uploaded into the job context -- `files[].name` (_optional_): Name of file to be written, if none a random name is picked -- `files[].content`: Content of file to be written -- `stdin` (_optional_): Text to pass into stdin of the program. Defaults to blank string. -- `args` (_optional_): Arguments to pass to the program. Defaults to none -- `run_timeout` (_optional_): The maximum allowed time in milliseconds for the compile stage to finish before bailing out. Must be a number, less than or equal to the configured maximum timeout. -- `compile_timeout` (_optional_): The maximum allowed time in milliseconds for the run stage to finish before bailing out. Must be a number, less than or equal to the configured maximum timeout. Defaults to maximum. -- `compile_memory_limit` (_optional_): The maximum amount of memory the compile stage is allowed to use in bytes. Must be a number, less than or equal to the configured maximum. Defaults to maximum, or `-1` (no limit) if none is configured. -- `run_memory_limit` (_optional_): The maximum amount of memory the run stage is allowed to use in bytes. Must be a number, less than or equal to the configured maximum. Defaults to maximum, or `-1` (no limit) if none is configured. +- `language`: Name or alias of a language listed in [runtimes](#runtimes) +- `version`: SemVer version selector of a language listed in [runtimes](#runtimes) +- `files`: An array of files which should be uploaded into the job context +- `files[].name` (_optional_): Name of file to be written, if none a random name is picked +- `files[].content`: Content of file to be written +- `stdin` (_optional_): Text to pass into stdin of the program. Defaults to blank string. +- `args` (_optional_): Arguments to pass to the program. Defaults to none +- `run_timeout` (_optional_): The maximum allowed time in milliseconds for the compile stage to finish before bailing out. Must be a number, less than or equal to the configured maximum timeout. +- `compile_timeout` (_optional_): The maximum allowed time in milliseconds for the run stage to finish before bailing out. Must be a number, less than or equal to the configured maximum timeout. Defaults to maximum. +- `compile_memory_limit` (_optional_): The maximum amount of memory the compile stage is allowed to use in bytes. Must be a number, less than or equal to the configured maximum. Defaults to maximum, or `-1` (no limit) if none is configured. +- `run_memory_limit` (_optional_): The maximum amount of memory the run stage is allowed to use in bytes. Must be a number, less than or equal to the configured maximum. Defaults to maximum, or `-1` (no limit) if none is configured. #### Response -- `language`: Name (not alias) of the runtime used -- `version`: Version of the used runtime -- `run`: Results from the run stage -- `run.stdout`: stdout from run stage process -- `run.stderr`: stderr from run stage process -- `run.output`: stdout and stderr combined in order of data from run stage process -- `run.code`: Exit code from run process, or null if signal is not null -- `run.signal`: Signal from run process, or null if code is not null -- `compile` (_optional_): Results from the compile stage, only provided if the runtime has a compile stage -- `compile.stdout`: stdout from compile stage process -- `compile.stderr`: stderr from compile stage process -- `compile.output`: stdout and stderr combined in order of data from compile stage process -- `compile.code`: Exit code from compile process, or null if signal is not null -- `compile.signal`: Signal from compile process, or null if code is not null +- `language`: Name (not alias) of the runtime used +- `version`: Version of the used runtime +- `run`: Results from the run stage +- `run.stdout`: stdout from run stage process +- `run.stderr`: stderr from run stage process +- `run.output`: stdout and stderr combined in order of data from run stage process +- `run.code`: Exit code from run process, or null if signal is not null +- `run.signal`: Signal from run process, or null if code is not null +- `compile` (_optional_): Results from the compile stage, only provided if the runtime has a compile stage +- `compile.stdout`: stdout from compile stage process +- `compile.stderr`: stderr from compile stage process +- `compile.output`: stdout and stderr combined in order of data from compile stage process +- `compile.code`: Exit code from compile process, or null if signal is not null +- `compile.signal`: Signal from compile process, or null if code is not null #### Example @@ -133,9 +133,9 @@ Returns a list of all possible packages, and whether their installation status. #### Response -- `[].language`: Name of the contained runtime -- `[].language_version`: Version of the contained runtime -- `[].installed`: Status on the package being installed +- `[].language`: Name of the contained runtime +- `[].language_version`: Version of the contained runtime +- `[].installed`: Status on the package being installed #### Example @@ -167,13 +167,13 @@ Install the given package. #### Request -- `language`: Name of package from [package list](#get-apiv2packages) -- `version`: SemVer version selector for package from [package list](#get-apiv2packages) +- `language`: Name of package from [package list](#get-apiv2packages) +- `version`: SemVer version selector for package from [package list](#get-apiv2packages) #### Response -- `language`: Name of package installed -- `version`: Version of package installed +- `language`: Name of package installed +- `version`: Version of package installed #### Example @@ -203,13 +203,13 @@ Uninstall the given package. #### Request -- `language`: Name of package from [package list](#get-apiv2packages) -- `version`: SemVer version selector for package from [package list](#get-apiv2packages) +- `language`: Name of package from [package list](#get-apiv2packages) +- `version`: SemVer version selector for package from [package list](#get-apiv2packages) #### Response -- `language`: Name of package uninstalled -- `version`: Version of package uninstalled +- `language`: Name of package uninstalled +- `version`: Version of package uninstalled #### Example diff --git a/docs/configuration.md b/docs/configuration.md index 4c6601a..1a6f5bd 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -50,15 +50,15 @@ Absolute path to piston related data, including packages and job contexts. ```yaml key: - - PISTON_RUNNER_UID_MIN - - PISTON_RUNNER_UID_MAX - - PISTON_RUNNER_GID_MIN - - PISTON_RUNNER_GID_MAX + - PISTON_RUNNER_UID_MIN + - PISTON_RUNNER_UID_MAX + - PISTON_RUNNER_GID_MIN + - PISTON_RUNNER_GID_MAX default: - - 1001 - - 1500 - - 1001 - - 1500 + - 1001 + - 1500 + - 1001 + - 1500 ``` UID and GID ranges to use when executing jobs. @@ -124,6 +124,7 @@ Maximum size for a singular file written to disk. Resists against large file writes to exhaust disk space. ## Compile/Run timeouts + ```yaml key: - PISTON_COMPILE_TIMEOUT @@ -133,6 +134,7 @@ key: - PISTON_RUN_TIMEOUT default: 3000 ``` + The maximum time that is allowed to be taken by a stage in milliseconds. Use -1 for unlimited time. @@ -140,8 +142,8 @@ Use -1 for unlimited time. ```yaml key: - - PISTON_COMPILE_MEMORY_LIMIT - - PISTON_RUN_MEMORY_LIMIT + - PISTON_COMPILE_MEMORY_LIMIT + - PISTON_RUN_MEMORY_LIMIT default: -1 ``` @@ -177,7 +179,9 @@ default: {} Per-language overrides/exceptions for the each of `max_process_count`, `max_open_files`, `max_file_size`, `compile_memory_limit`, `run_memory_limit`, `compile_timeout`, `run_timeout`, `output_max_size`. Defined as follows: + ``` PISTON_LIMIT_OVERRIDES={"c++":{"max_process_count":128}} ``` + This will give `c++` a max_process_count of 128 regardless of the configuration. diff --git a/mkdocs.yml b/mkdocs.yml index 148ba91..a6ef999 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,15 +1,15 @@ site_name: Piston nav: - - Home: index.md - - Configuration: configuration.md - - API: api-v2.md + - Home: index.md + - Configuration: configuration.md + - API: api-v2.md theme: - name: readthedocs - highlightjs: true - hljs_languages: - - yaml - - json + name: readthedocs + highlightjs: true + hljs_languages: + - yaml + - json markdown_extensions: - - admonition + - admonition diff --git a/package-lock.json b/package-lock.json index ac734d4..5c51a1d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,32 @@ { - "name": "piston", - "lockfileVersion": 2, - "requires": true, - "packages": {} + "name": "piston", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "devDependencies": { + "prettier": "2.4.1" + } + }, + "node_modules/prettier": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.4.1.tgz", + "integrity": "sha512-9fbDAXSBcc6Bs1mZrDYb3XKzDLm4EXXL9sC1LqKP5rZkT6KRr/rf9amVUcODVXgguK/isJz0d0hP72WeaKWsvA==", + "dev": true, + "bin": { + "prettier": "bin-prettier.js" + }, + "engines": { + "node": ">=10.13.0" + } + } + }, + "dependencies": { + "prettier": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.4.1.tgz", + "integrity": "sha512-9fbDAXSBcc6Bs1mZrDYb3XKzDLm4EXXL9sC1LqKP5rZkT6KRr/rf9amVUcODVXgguK/isJz0d0hP72WeaKWsvA==", + "dev": true + } + } } diff --git a/package.json b/package.json index 0967ef4..8f07606 100644 --- a/package.json +++ b/package.json @@ -1 +1,5 @@ -{} +{ + "devDependencies": { + "prettier": "2.4.1" + } +} diff --git a/packages/cjam/0.6.5/metadata.json b/packages/cjam/0.6.5/metadata.json index af510fd..bd25bde 100644 --- a/packages/cjam/0.6.5/metadata.json +++ b/packages/cjam/0.6.5/metadata.json @@ -1,5 +1,5 @@ { - "language": "cjam", - "version": "0.6.5", - "aliases": [] + "language": "cjam", + "version": "0.6.5", + "aliases": [] } diff --git a/packages/cobol/3.1.2/metadata.json b/packages/cobol/3.1.2/metadata.json index cf3e7e1..0a80d3c 100644 --- a/packages/cobol/3.1.2/metadata.json +++ b/packages/cobol/3.1.2/metadata.json @@ -1,5 +1,5 @@ { - "language": "cobol", - "version": "3.1.2", - "aliases": ["cob"] + "language": "cobol", + "version": "3.1.2", + "aliases": ["cob"] } diff --git a/packages/deno/1.7.5/metadata.json b/packages/deno/1.7.5/metadata.json index d30608b..217a7c6 100644 --- a/packages/deno/1.7.5/metadata.json +++ b/packages/deno/1.7.5/metadata.json @@ -4,7 +4,7 @@ "provides": [ { "language": "typescript", - "aliases": ["deno-ts","deno"] + "aliases": ["deno-ts", "deno"] }, { "language": "javascript", diff --git a/packages/deno/1.7.5/test.deno.ts b/packages/deno/1.7.5/test.deno.ts index 56ed4a0..e106678 100644 --- a/packages/deno/1.7.5/test.deno.ts +++ b/packages/deno/1.7.5/test.deno.ts @@ -1 +1 @@ -console.log("OK") \ No newline at end of file +console.log('OK'); diff --git a/packages/dragon/1.9.8/metadata.json b/packages/dragon/1.9.8/metadata.json index 86cfc4c..3fbc015 100644 --- a/packages/dragon/1.9.8/metadata.json +++ b/packages/dragon/1.9.8/metadata.json @@ -1,5 +1,5 @@ { - "language": "dragon", - "version": "1.9.8", - "aliases": [] + "language": "dragon", + "version": "1.9.8", + "aliases": [] } diff --git a/packages/forte/1.0.0/metadata.json b/packages/forte/1.0.0/metadata.json index fd4ec12..f7f4137 100644 --- a/packages/forte/1.0.0/metadata.json +++ b/packages/forte/1.0.0/metadata.json @@ -1,5 +1,5 @@ { - "language": "forte", - "version": "1.0.0", - "aliases": ["forter"] + "language": "forte", + "version": "1.0.0", + "aliases": ["forter"] } diff --git a/packages/gcc/10.2.0/metadata.json b/packages/gcc/10.2.0/metadata.json index f969bf5..367de7c 100644 --- a/packages/gcc/10.2.0/metadata.json +++ b/packages/gcc/10.2.0/metadata.json @@ -3,7 +3,7 @@ "version": "10.2.0", "provides": [ { - "language":"c", + "language": "c", "aliases": ["gcc"] }, { diff --git a/packages/golfscript/1.0.0/metadata.json b/packages/golfscript/1.0.0/metadata.json index 4ef3a62..cb4f356 100644 --- a/packages/golfscript/1.0.0/metadata.json +++ b/packages/golfscript/1.0.0/metadata.json @@ -1,5 +1,5 @@ { - "language": "golfscript", - "version": "1.0.0", - "aliases": ["golfscript"] + "language": "golfscript", + "version": "1.0.0", + "aliases": ["golfscript"] } diff --git a/packages/groovy/3.0.7/metadata.json b/packages/groovy/3.0.7/metadata.json index b790007..34ab93d 100644 --- a/packages/groovy/3.0.7/metadata.json +++ b/packages/groovy/3.0.7/metadata.json @@ -1,5 +1,5 @@ { - "language": "groovy", - "version": "3.0.7", - "aliases": ["groovy", "gvy"] + "language": "groovy", + "version": "3.0.7", + "aliases": ["groovy", "gvy"] } diff --git a/packages/japt/2.0.0/metadata.json b/packages/japt/2.0.0/metadata.json index 7a3e5aa..ef0ff8d 100644 --- a/packages/japt/2.0.0/metadata.json +++ b/packages/japt/2.0.0/metadata.json @@ -1,5 +1,5 @@ { - "language": "japt", - "version": "2.0.0", - "aliases": ["japt"] -} \ No newline at end of file + "language": "japt", + "version": "2.0.0", + "aliases": ["japt"] +} diff --git a/packages/llvm_ir/12.0.1/metadata.json b/packages/llvm_ir/12.0.1/metadata.json index 4c92048..50dfbbc 100644 --- a/packages/llvm_ir/12.0.1/metadata.json +++ b/packages/llvm_ir/12.0.1/metadata.json @@ -1,5 +1,5 @@ { - "language": "llvm_ir", - "version": "12.0.1", - "aliases": ["llvm", "llvm-ir", "ll"] + "language": "llvm_ir", + "version": "12.0.1", + "aliases": ["llvm", "llvm-ir", "ll"] } diff --git a/packages/mono/6.12.0/metadata.json b/packages/mono/6.12.0/metadata.json index 4d09ae7..3f483a4 100644 --- a/packages/mono/6.12.0/metadata.json +++ b/packages/mono/6.12.0/metadata.json @@ -8,7 +8,13 @@ }, { "language": "basic", - "aliases": ["vb", "mono-vb", "mono-basic", "visual-basic", "visual basic"] + "aliases": [ + "vb", + "mono-vb", + "mono-basic", + "visual-basic", + "visual basic" + ] } ] } diff --git a/packages/node/15.10.0/test.js b/packages/node/15.10.0/test.js index 56ed4a0..e106678 100644 --- a/packages/node/15.10.0/test.js +++ b/packages/node/15.10.0/test.js @@ -1 +1 @@ -console.log("OK") \ No newline at end of file +console.log('OK'); diff --git a/packages/node/16.3.0/test.js b/packages/node/16.3.0/test.js index 56ed4a0..e106678 100644 --- a/packages/node/16.3.0/test.js +++ b/packages/node/16.3.0/test.js @@ -1 +1 @@ -console.log("OK") \ No newline at end of file +console.log('OK'); diff --git a/packages/ocaml/4.12.0/metadata.json b/packages/ocaml/4.12.0/metadata.json index ddbfb89..6c2f733 100644 --- a/packages/ocaml/4.12.0/metadata.json +++ b/packages/ocaml/4.12.0/metadata.json @@ -1,5 +1,5 @@ { - "language": "ocaml", - "version": "4.12.0", - "aliases": ["ocaml", "ml"] + "language": "ocaml", + "version": "4.12.0", + "aliases": ["ocaml", "ml"] } diff --git a/packages/octave/6.2.0/metadata.json b/packages/octave/6.2.0/metadata.json index ab9dbb1..0b209ce 100644 --- a/packages/octave/6.2.0/metadata.json +++ b/packages/octave/6.2.0/metadata.json @@ -1,5 +1,5 @@ { - "language": "octave", - "version": "6.2.0", - "aliases": ["matlab", "m"] + "language": "octave", + "version": "6.2.0", + "aliases": ["matlab", "m"] } diff --git a/packages/pyth/1.0.0/metadata.json b/packages/pyth/1.0.0/metadata.json index bcddb7a..e9bbfe9 100644 --- a/packages/pyth/1.0.0/metadata.json +++ b/packages/pyth/1.0.0/metadata.json @@ -1,5 +1,5 @@ { - "language": "pyth", - "version": "1.0.0", - "aliases": ["pyth"] + "language": "pyth", + "version": "1.0.0", + "aliases": ["pyth"] } diff --git a/packages/raku/6.100.0/metadata.json b/packages/raku/6.100.0/metadata.json index 7cda1ed..e1fbad8 100644 --- a/packages/raku/6.100.0/metadata.json +++ b/packages/raku/6.100.0/metadata.json @@ -2,4 +2,4 @@ "language": "raku", "version": "6.100.0", "aliases": ["raku", "rakudo", "perl6", "p6", "pl6"] -} \ No newline at end of file +} diff --git a/packages/typescript/4.2.3/test.ts b/packages/typescript/4.2.3/test.ts index 56ed4a0..e106678 100644 --- a/packages/typescript/4.2.3/test.ts +++ b/packages/typescript/4.2.3/test.ts @@ -1 +1 @@ -console.log("OK") \ No newline at end of file +console.log('OK'); diff --git a/piston b/piston index 1653eae..90d47ed 100755 --- a/piston +++ b/piston @@ -38,6 +38,7 @@ case $1 in echo " clean-repo Remove all packages from local repo" echo " build-pkg Build a package" echo " rebuild Build and restart the docker container" + echo " lint Lint the codebase using prettier" else @@ -53,7 +54,11 @@ case $1 in logs) docker_compose logs -f ;; restart) docker_compose restart ;; - start) docker_compose up -d ;; + start) + rm -f .git/hooks/pre-commit + ln -s $(realpath $(dirname "$0"))/pre-commit .git/hooks/pre-commit + docker_compose up -d + ;; stop) docker_compose down ;; bash) docker_compose exec api /bin/bash ;; @@ -75,6 +80,11 @@ case $1 in docker build repo -t piston-repo-builder docker run --rm -v "$(realpath $(dirname "$0")):/piston" piston-repo-builder --no-server $PKGSLUG ;; + + lint) + npm install + npx prettier --ignore-unknown --write . + ;; *) cd cli npm i > /dev/null diff --git a/pre-commit b/pre-commit new file mode 100755 index 0000000..f592c88 --- /dev/null +++ b/pre-commit @@ -0,0 +1,15 @@ +#!/bin/sh + +echo "Linting staged files..." +npm install + +FILES=$(git diff --cached --name-only --diff-filter=ACMR | sed 's| |\\ |g') +[ -z "$FILES" ] && exit 0 + +# Prettify all selected files +echo "$FILES" | xargs npx prettier --ignore-unknown --write + +# Add back the modified/prettified files to staging +echo "$FILES" | xargs git add + +exit 0 diff --git a/readme.md b/readme.md index b215138..add2106 100644 --- a/readme.md +++ b/readme.md @@ -47,9 +47,10 @@ While we are accepting pull requests for Hacktoberfest, we will reject any low-q If we see PR abuse for Hacktoberfest, we will stop providing Hacktoberfest approval for pull requests. We are accepting PRs for: -* Packages - updating package versions, adding new packages -* Documentation updates -* CLI/API improvements - please discuss these with us in the Discord first + +- Packages - updating package versions, adding new packages +- Documentation updates +- CLI/API improvements - please discuss these with us in the Discord first Any queries or concerns, ping @HexF#0015 in the Discord. @@ -66,11 +67,11 @@ Any queries or concerns, ping @HexF#0015 in the Discord. It's used in numerous places including: -- [EMKC Challenges](https://emkc.org/challenges) -- [EMKC Weekly Contests](https://emkc.org/contests) -- [Engineer Man Discord Server](https://discord.gg/engineerman) -- Web IDEs -- 200+ direct integrations +- [EMKC Challenges](https://emkc.org/challenges) +- [EMKC Weekly Contests](https://emkc.org/contests) +- [Engineer Man Discord Server](https://discord.gg/engineerman) +- Web IDEs +- 200+ direct integrations
@@ -78,19 +79,19 @@ It's used in numerous places including: The following are approved and endorsed extensions/utilities to the core Piston offering. -- [I Run Code](https://github.com/engineer-man/piston-bot), a Discord bot used in 4100+ servers to handle arbitrary code evaluation in Discord. To get this bot in your own server, go here: https://emkc.org/run. -- [Piston CLI](https://github.com/Shivansh-007/piston-cli), a universal shell supporting code highlighting, files, and interpretation without the need to download a language. -- [Node Piston Client](https://github.com/dthree/node-piston), a Node.js wrapper for accessing the Piston API. -- [Piston4J](https://github.com/the-codeboy/Piston4J), a Java wrapper for accessing the Piston API. -- [Pyston](https://github.com/ffaanngg/pyston), a Python wrapper for accessing the Piston API. -- [Go-Piston](https://github.com/milindmadhukar/go-piston), a Golang wrapper for accessing the Piston API. +- [I Run Code](https://github.com/engineer-man/piston-bot), a Discord bot used in 4100+ servers to handle arbitrary code evaluation in Discord. To get this bot in your own server, go here: https://emkc.org/run. +- [Piston CLI](https://github.com/Shivansh-007/piston-cli), a universal shell supporting code highlighting, files, and interpretation without the need to download a language. +- [Node Piston Client](https://github.com/dthree/node-piston), a Node.js wrapper for accessing the Piston API. +- [Piston4J](https://github.com/the-codeboy/Piston4J), a Java wrapper for accessing the Piston API. +- [Pyston](https://github.com/ffaanngg/pyston), a Python wrapper for accessing the Piston API. +- [Go-Piston](https://github.com/milindmadhukar/go-piston), a Golang wrapper for accessing the Piston API.
# Public API -- Requires no installation and you can use it immediately. -- Reference the Runtimes/Execute sections below to learn about the request and response formats. +- Requires no installation and you can use it immediately. +- Reference the Runtimes/Execute sections below to learn about the request and response formats.
@@ -115,9 +116,9 @@ POST https://emkc.org/api/v2/piston/execute ### Host System Package Dependencies -- Docker -- Docker Compose -- Node JS (>= 13, preferably >= 15) +- Docker +- Docker Compose +- Node JS (>= 13, preferably >= 15) ### After system dependencies are installed, clone this repository: @@ -142,7 +143,7 @@ The API will now be online with no language runtimes installed. To install runti ### Host System Package Dependencies -- Docker +- Docker ### Installation @@ -160,7 +161,7 @@ docker run \ ### Host System Package Dependencies -- Same as [All In One](#All-In-One) +- Same as [All In One](#All-In-One) ### Installation @@ -250,34 +251,34 @@ Content-Type: application/json `POST /api/v2/execute` This endpoint requests execution of some arbitrary code. -- `language` (**required**) The language to use for execution, must be a string and must be installed. -- `version` (**required**) The version of the language to use for execution, must be a string containing a SemVer selector for the version or the specific version number to use. -- `files` (**required**) An array of files containing code or other data that should be used for execution. The first file in this array is considered the main file. -- `files[].name` (_optional_) The name of the file to upload, must be a string containing no path or left out. -- `files[].content` (**required**) The content of the files to upload, must be a string containing text to write. -- `stdin` (_optional_) The text to pass as stdin to the program. Must be a string or left out. Defaults to blank string. -- `args` (_optional_) The arguments to pass to the program. Must be an array or left out. Defaults to `[]`. -- `compile_timeout` (_optional_) The maximum time allowed for the compile stage to finish before bailing out in milliseconds. Must be a number or left out. Defaults to `10000` (10 seconds). -- `run_timeout` (_optional_) The maximum time allowed for the run stage to finish before bailing out in milliseconds. Must be a number or left out. Defaults to `3000` (3 seconds). -- `compile_memory_limit` (_optional_) The maximum amount of memory the compile stage is allowed to use in bytes. Must be a number or left out. Defaults to `-1` (no limit) -- `run_memory_limit` (_optional_) The maximum amount of memory the run stage is allowed to use in bytes. Must be a number or left out. Defaults to `-1` (no limit) +- `language` (**required**) The language to use for execution, must be a string and must be installed. +- `version` (**required**) The version of the language to use for execution, must be a string containing a SemVer selector for the version or the specific version number to use. +- `files` (**required**) An array of files containing code or other data that should be used for execution. The first file in this array is considered the main file. +- `files[].name` (_optional_) The name of the file to upload, must be a string containing no path or left out. +- `files[].content` (**required**) The content of the files to upload, must be a string containing text to write. +- `stdin` (_optional_) The text to pass as stdin to the program. Must be a string or left out. Defaults to blank string. +- `args` (_optional_) The arguments to pass to the program. Must be an array or left out. Defaults to `[]`. +- `compile_timeout` (_optional_) The maximum time allowed for the compile stage to finish before bailing out in milliseconds. Must be a number or left out. Defaults to `10000` (10 seconds). +- `run_timeout` (_optional_) The maximum time allowed for the run stage to finish before bailing out in milliseconds. Must be a number or left out. Defaults to `3000` (3 seconds). +- `compile_memory_limit` (_optional_) The maximum amount of memory the compile stage is allowed to use in bytes. Must be a number or left out. Defaults to `-1` (no limit) +- `run_memory_limit` (_optional_) The maximum amount of memory the run stage is allowed to use in bytes. Must be a number or left out. Defaults to `-1` (no limit) ```json { - "language": "js", - "version": "15.10.0", - "files": [ - { - "name": "my_cool_code.js", - "content": "console.log(process.argv)" - } - ], - "stdin": "", - "args": ["1", "2", "3"], - "compile_timeout": 10000, - "run_timeout": 3000, - "compile_memory_limit": -1, - "run_memory_limit": -1 + "language": "js", + "version": "15.10.0", + "files": [ + { + "name": "my_cool_code.js", + "content": "console.log(process.argv)" + } + ], + "stdin": "", + "args": ["1", "2", "3"], + "compile_timeout": 10000, + "run_timeout": 3000, + "compile_memory_limit": -1, + "run_memory_limit": -1 } ``` @@ -409,14 +410,14 @@ Docker provides a great deal of security out of the box in that it's separate fr Piston takes additional steps to make it resistant to various privilege escalation, denial-of-service, and resource saturation threats. These steps include: -- Disabling outgoing network interaction -- Capping max processes at 256 by default (resists `:(){ :|: &}:;`, `while True: os.fork()`, etc.) -- Capping max files at 2048 (resists various file based attacks) -- Cleaning up all temp space after each execution (resists out of drive space attacks) -- Running as a variety of unprivileged users -- Capping runtime execution at 3 seconds -- Capping stdout to 65536 characters (resists yes/no bombs and runaway output) -- SIGKILLing misbehaving code +- Disabling outgoing network interaction +- Capping max processes at 256 by default (resists `:(){ :|: &}:;`, `while True: os.fork()`, etc.) +- Capping max files at 2048 (resists various file based attacks) +- Cleaning up all temp space after each execution (resists out of drive space attacks) +- Running as a variety of unprivileged users +- Capping runtime execution at 3 seconds +- Capping stdout to 65536 characters (resists yes/no bombs and runaway output) +- SIGKILLing misbehaving code
diff --git a/tests/readme.md b/tests/readme.md index 01ae419..746d0b9 100644 --- a/tests/readme.md +++ b/tests/readme.md @@ -6,4 +6,4 @@ Write exploits in any language supported by piston. Hopefully when running any files in this directory, piston will resist the attack. -Leave a comment in the code describing how the exploit works. \ No newline at end of file +Leave a comment in the code describing how the exploit works. From f8eb7053edeef0b5c546ae65e59e9077daf872e7 Mon Sep 17 00:00:00 2001 From: Brikaa Date: Sat, 9 Oct 2021 18:16:41 +0200 Subject: [PATCH 25/72] Fix typescript --- packages/typescript/4.2.3/run | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/typescript/4.2.3/run b/packages/typescript/4.2.3/run index 1d26f3f..5a8c60e 100644 --- a/packages/typescript/4.2.3/run +++ b/packages/typescript/4.2.3/run @@ -2,7 +2,7 @@ # Put instructions to run the runtime -CODE=$(sed 's/ts$/js/' <<<"$1") +CODE=$1.js shift node $CODE "$@" From 6416b4d8cb34697642d67d8ebcb8867bff9cab0a Mon Sep 17 00:00:00 2001 From: Hydrazer Date: Sat, 9 Oct 2021 10:28:26 -0600 Subject: [PATCH 26/72] pkg(vyxal-2.4.1): add vyxal --- packages/vyxal/2.4.1/build.sh | 18 ++++++++++++++++++ packages/vyxal/2.4.1/environment | 8 ++++++++ packages/vyxal/2.4.1/metadata.json | 5 +++++ packages/vyxal/2.4.1/run | 3 +++ packages/vyxal/2.4.1/test.vyxal | 1 + readme.md | 1 + 6 files changed, 36 insertions(+) create mode 100644 packages/vyxal/2.4.1/build.sh create mode 100644 packages/vyxal/2.4.1/environment create mode 100644 packages/vyxal/2.4.1/metadata.json create mode 100644 packages/vyxal/2.4.1/run create mode 100644 packages/vyxal/2.4.1/test.vyxal diff --git a/packages/vyxal/2.4.1/build.sh b/packages/vyxal/2.4.1/build.sh new file mode 100644 index 0000000..e7ce729 --- /dev/null +++ b/packages/vyxal/2.4.1/build.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +# get Python source +source ../../python/3.9.4/build.sh + +# add regex and pwn modules +bin/pip3 install regex pwn + +# make vyxal directory +mkdir vyxal +cd vyxal + +# Vyxal install +curl -L "https://github.com/Vyxal/Vyxal/archive/refs/tags/v2.4.1.tar.gz" -o vyxal.tar.xz +tar xf vyxal.tar.xz --strip-components=1 +rm vyxal.tar.xz + +cd .. \ No newline at end of file diff --git a/packages/vyxal/2.4.1/environment b/packages/vyxal/2.4.1/environment new file mode 100644 index 0000000..f0008c8 --- /dev/null +++ b/packages/vyxal/2.4.1/environment @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +# Python and Vyxal path +export PATH=$PWD/bin:$PATH +export VYXAL_PATH=$PWD/vyxal + +# export term to fix curses warning +export TERM=xterm \ No newline at end of file diff --git a/packages/vyxal/2.4.1/metadata.json b/packages/vyxal/2.4.1/metadata.json new file mode 100644 index 0000000..e5427fb --- /dev/null +++ b/packages/vyxal/2.4.1/metadata.json @@ -0,0 +1,5 @@ +{ + "language": "vyxal", + "version": "2.4.1", + "aliases": [] +} \ No newline at end of file diff --git a/packages/vyxal/2.4.1/run b/packages/vyxal/2.4.1/run new file mode 100644 index 0000000..c9b08a6 --- /dev/null +++ b/packages/vyxal/2.4.1/run @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +python3 "$VYXAL_PATH"/Vyxal.py "$1" \ No newline at end of file diff --git a/packages/vyxal/2.4.1/test.vyxal b/packages/vyxal/2.4.1/test.vyxal new file mode 100644 index 0000000..6d0cb6e --- /dev/null +++ b/packages/vyxal/2.4.1/test.vyxal @@ -0,0 +1 @@ +`OK \ No newline at end of file diff --git a/readme.md b/readme.md index b215138..998d07e 100644 --- a/readme.md +++ b/readme.md @@ -389,6 +389,7 @@ Content-Type: application/json `basic`, `basic.net`, `vlang`, +`vyxal`, `yeethon`, `zig`, From 7fc0b9efb86d563e2648b0d07655334577438769 Mon Sep 17 00:00:00 2001 From: Brikaa Date: Sun, 10 Oct 2021 14:40:58 +0200 Subject: [PATCH 27/72] Revert Mono error separation to work with websockets --- packages/mono/6.12.0/compile | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/packages/mono/6.12.0/compile b/packages/mono/6.12.0/compile index e3ae230..5246bc2 100644 --- a/packages/mono/6.12.0/compile +++ b/packages/mono/6.12.0/compile @@ -1,20 +1,13 @@ #!/bin/bash -check_errors () { - grep -q 'error [A-Z]\+[0-9]\+:' check.txt && cat check.txt 1>&2 || cat check.txt - rm check.txt -} - case "${PISTON_LANGUAGE}" in csharp) rename 's/$/\.cs/' "$@" # Add .cs extension - csc -out:out *.cs > check.txt - check_errors + csc -out:out *.cs ;; basic) rename 's/$/\.vb/' "$@" # Add .vb extension - vbnc -out:out *.vb > check.txt - check_errors + vbnc -out:out *.vb ;; *) echo "How did you get here? (${PISTON_LANGUAGE})" From f973ecf2818ba56362d5e1fa618703a8adc2209e Mon Sep 17 00:00:00 2001 From: Brikaa Date: Sun, 10 Oct 2021 17:18:31 +0200 Subject: [PATCH 28/72] Add infinite timeout --- api/src/job.js | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/api/src/job.js b/api/src/job.js index c2e8bc8..3ad77c7 100644 --- a/api/src/job.js +++ b/api/src/job.js @@ -143,12 +143,15 @@ class Job { }); } - const kill_timeout = set_timeout(async _ => { - logger.info( - `Timeout exceeded timeout=${timeout} uuid=${this.uuid}` - ); - process.kill(proc.pid, 'SIGKILL'); - }, timeout); + const kill_timeout = + (timeout >= 0 && + set_timeout(async _ => { + logger.info( + `Timeout exceeded timeout=${timeout} uuid=${this.uuid}` + ); + process.kill(proc.pid, 'SIGKILL'); + }, timeout)) || + null; proc.stderr.on('data', async data => { if (eventBus !== null) { From 5968090f502cead8a6368bf8182f07186be03b35 Mon Sep 17 00:00:00 2001 From: Brikaa Date: Sun, 10 Oct 2021 16:56:16 +0200 Subject: [PATCH 29/72] Improve precommit --- piston | 15 ++++++++++++--- pre-commit | 2 +- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/piston b/piston index 90d47ed..068895b 100755 --- a/piston +++ b/piston @@ -11,6 +11,13 @@ docker_compose(){ fi } +init_precommit() { + if [ $PISTON_ENV == "dev" ]; then + rm -f .git/hooks/pre-commit + ln -s $(realpath $(dirname "$0"))/pre-commit .git/hooks/pre-commit + fi +} + case $1 in help) echo "=== Piston Management ===" @@ -55,14 +62,16 @@ case $1 in restart) docker_compose restart ;; start) - rm -f .git/hooks/pre-commit - ln -s $(realpath $(dirname "$0"))/pre-commit .git/hooks/pre-commit + init_precommit docker_compose up -d ;; stop) docker_compose down ;; bash) docker_compose exec api /bin/bash ;; - rebuild) docker_compose build && docker_compose up -d ;; + rebuild) + init_precommit + docker_compose build && docker_compose up -d + ;; update) git pull diff --git a/pre-commit b/pre-commit index f592c88..288a5e3 100755 --- a/pre-commit +++ b/pre-commit @@ -1,7 +1,7 @@ #!/bin/sh echo "Linting staged files..." -npm install +npm install > /dev/null || exit 1 FILES=$(git diff --cached --name-only --diff-filter=ACMR | sed 's| |\\ |g') [ -z "$FILES" ] && exit 0 From 2f114d6e5462a573d1594edcb56be425c5a9bfdc Mon Sep 17 00:00:00 2001 From: Brikaa Date: Sun, 10 Oct 2021 19:11:16 +0200 Subject: [PATCH 30/72] Fix important constraints bug --- api/src/api/v2.js | 2 +- api/src/job.js | 21 ++++++++++++--------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/api/src/api/v2.js b/api/src/api/v2.js index a8fa9f0..f9a3f66 100644 --- a/api/src/api/v2.js +++ b/api/src/api/v2.js @@ -129,7 +129,7 @@ function get_job(body) { compile_timeout = compile_timeout || rt.timeouts.compile; run_timeout = run_timeout || rt.timeouts.run; compile_memory_limit = compile_memory_limit || rt.memory_limits.compile; - run_timeout = run_timeout || rt.timeouts.run; + run_memory_limit = run_memory_limit || rt.memory_limits.run; resolve( new Job({ runtime: rt, diff --git a/api/src/job.js b/api/src/job.js index c2e8bc8..2a9aa99 100644 --- a/api/src/job.js +++ b/api/src/job.js @@ -27,7 +27,7 @@ setInterval(() => { }, 10); class Job { - constructor({ runtime, files, args, stdin }) { + constructor({ runtime, files, args, stdin, timeouts, memory_limits }) { this.uuid = uuidv4(); this.runtime = runtime; this.files = files.map((file, i) => ({ @@ -38,6 +38,9 @@ class Job { this.args = args; this.stdin = stdin; + this.timeouts = timeouts; + this.memory_limits = memory_limits; + this.uid = config.runner_uid_min + uid; this.gid = config.runner_gid_min + gid; @@ -220,8 +223,8 @@ class Job { compile = await this.safe_call( path.join(this.runtime.pkgdir, 'compile'), this.files.map(x => x.name), - this.runtime.timeouts.compile, - this.runtime.memory_limits.compile + this.timeouts.compile, + this.memory_limits.compile ); } @@ -230,8 +233,8 @@ class Job { const run = await this.safe_call( path.join(this.runtime.pkgdir, 'run'), [this.files[0].name, ...this.args], - this.runtime.timeouts.run, - this.runtime.memory_limits.run + this.timeouts.run, + this.memory_limits.run ); this.state = job_states.EXECUTED; @@ -263,8 +266,8 @@ class Job { const { error, code, signal } = await this.safe_call( path.join(this.runtime.pkgdir, 'compile'), this.files.map(x => x.name), - this.runtime.timeouts.compile, - this.runtime.memory_limits.compile, + this.timeouts.compile, + this.memory_limits.compile, eventBus ); @@ -276,8 +279,8 @@ class Job { const { error, code, signal } = await this.safe_call( path.join(this.runtime.pkgdir, 'run'), [this.files[0].name, ...this.args], - this.runtime.timeouts.run, - this.runtime.memory_limits.run, + this.timeouts.run, + this.memory_limits.run, eventBus ); From afd71cc82db4b5eda7dcc3d0a6a892754a078747 Mon Sep 17 00:00:00 2001 From: Brikaa Date: Sun, 10 Oct 2021 22:01:26 +0200 Subject: [PATCH 31/72] Fix env_val bug --- api/src/config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/config.js b/api/src/config.js index 1a9eb3d..8d0020f 100644 --- a/api/src/config.js +++ b/api/src/config.js @@ -215,7 +215,7 @@ options.forEach(option => { const parsed_val = parser(env_val); - const value = parsed_val || option.default; + const value = env_val === undefined ? option.default : parsed_val; option.validators.for_each(validator => { let response = null; From 80eefaa6fb3d0389d66ef9d62067ae145634d6fb Mon Sep 17 00:00:00 2001 From: Brikaa Date: Mon, 11 Oct 2021 13:26:43 +0200 Subject: [PATCH 32/72] Increase zig compile timeout --- packages/zig/0.8.0/metadata.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/zig/0.8.0/metadata.json b/packages/zig/0.8.0/metadata.json index 7af8ed6..8c02d33 100644 --- a/packages/zig/0.8.0/metadata.json +++ b/packages/zig/0.8.0/metadata.json @@ -1,5 +1,8 @@ { "language": "zig", "version": "0.8.0", - "aliases": ["zig"] + "aliases": ["zig"], + "limit_overrides": { + "compile_timeout": 15000 + } } From 198d8ff061f0c1c756a1311297e7b9f042477c23 Mon Sep 17 00:00:00 2001 From: Dan Vargas Date: Tue, 12 Oct 2021 11:59:49 -0500 Subject: [PATCH 33/72] improve piston shell script - fix portability & using piston within a symlink - only install cli npm modules on update or first use - allow building packages with custom builder - fix all shellchecks except SC2164 --- piston | 39 +++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/piston b/piston index 90d47ed..e6cc13e 100755 --- a/piston +++ b/piston @@ -1,7 +1,10 @@ #!/usr/bin/env bash -cd "$(dirname "$0")" -PISTON_ENV=$(cat .piston_env || echo dev) +EXECUTION_PATH="$PWD" +PISTON_PATH="$(dirname "$(realpath "$0")")" + +cd "$PISTON_PATH" +PISTON_ENV=$(cat .piston_env 2> /dev/null || echo dev) docker_compose(){ if [ -f "docker-compose.$PISTON_ENV.yaml" ]; then @@ -32,13 +35,14 @@ case $1 in echo echo "Development Commands:" - if [ $PISTON_ENV == dev ]; then + if [ "$PISTON_ENV" == dev ]; then - echo " clean-pkgs Clean any package build artifacts on disk" - echo " clean-repo Remove all packages from local repo" - echo " build-pkg Build a package" - echo " rebuild Build and restart the docker container" - echo " lint Lint the codebase using prettier" + echo " clean-pkgs Clean any package build artifacts on disk" + echo " clean-repo Remove all packages from local repo" + echo " list-pkgs Lists all packages that can be built" + echo " build-pkg [builder] Build a package [with desired builder image]" + echo " rebuild Build and restart the docker container" + echo " lint Lint the codebase using prettier" else @@ -56,7 +60,7 @@ case $1 in restart) docker_compose restart ;; start) rm -f .git/hooks/pre-commit - ln -s $(realpath $(dirname "$0"))/pre-commit .git/hooks/pre-commit + ln -s "$PISTON_PATH/pre-commit" "$PISTON_PATH/.git/hooks/pre-commit" docker_compose up -d ;; stop) docker_compose down ;; @@ -66,6 +70,7 @@ case $1 in update) git pull + cd cli && npm i > /dev/null && cd - docker_compose pull docker_compose up -d ;; @@ -73,12 +78,15 @@ case $1 in clean-pkgs) git clean -fqXd packages ;; clean-repo) git clean -fqXd repo ;; + list-pkgs) find packages -depth 2 | awk -F/ '$2 && $3{ print $2 "-" $3 }' | column ;; + build-pkg) PKGSLUG="$2-$3" + BUILDER="${4:-piston-repo-builder}" echo "Building $PKGSLUG" echo "Ensuring latest builder image" - docker build repo -t piston-repo-builder - docker run --rm -v "$(realpath $(dirname "$0")):/piston" piston-repo-builder --no-server $PKGSLUG + docker build repo -t "$BUILDER" + docker run --rm -v "$PWD:/piston" "$BUILDER" --no-server "$PKGSLUG" ;; lint) @@ -86,9 +94,8 @@ case $1 in npx prettier --ignore-unknown --write . ;; *) - cd cli - npm i > /dev/null - cd ../ - node cli/index.js "$@" - ;; + [ -d ./cli/node_modules ] || npm i > /dev/null + cd "$EXECUTION_PATH" + node "${PISTON_PATH}/cli/index.js" "$@" + ;; esac From 24a352699da305e353b885560504bf8b9d3717b3 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Thu, 14 Oct 2021 00:46:49 +1300 Subject: [PATCH 34/72] Support for uploading files in base64/hex format --- api/src/job.js | 18 +++++++++++------- docs/api-v2.md | 1 + readme.md | 1 + 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/api/src/job.js b/api/src/job.js index 8948be1..bf7870b 100644 --- a/api/src/job.js +++ b/api/src/job.js @@ -16,12 +16,12 @@ const job_states = { let uid = 0; let gid = 0; -let remainingJobSpaces = config.max_concurrent_jobs; +let remaining_job_spaces = config.max_concurrent_jobs; let jobQueue = []; setInterval(() => { // Every 10ms try resolve a new job, if there is an available slot - if (jobQueue.length > 0 && remainingJobSpaces > 0) { + if (jobQueue.length > 0 && remaining_job_spaces > 0) { jobQueue.shift()(); } }, 10); @@ -33,6 +33,9 @@ class Job { this.files = files.map((file, i) => ({ name: file.name || `file${i}.code`, content: file.content, + encoding: ['base64', 'hex', 'utf8'].includes(file.encoding) + ? file.encoding + : 'utf8', })); this.args = args; @@ -59,7 +62,7 @@ class Job { } async prime() { - if (remainingJobSpaces < 1) { + if (remaining_job_spaces < 1) { logger.info(`Awaiting job slot uuid=${this.uuid}`); await new Promise(resolve => { jobQueue.push(resolve); @@ -67,7 +70,7 @@ class Job { } logger.info(`Priming job uuid=${this.uuid}`); - remainingJobSpaces--; + remaining_job_spaces--; logger.debug('Writing files to job cache'); logger.debug(`Transfering ownership uid=${this.uid} gid=${this.gid}`); @@ -76,8 +79,9 @@ class Job { await fs.chown(this.dir, this.uid, this.gid); for (const file of this.files) { - let file_path = path.join(this.dir, file.name); + const file_path = path.join(this.dir, file.name); const rel = path.relative(this.dir, file_path); + const file_content = Buffer.from(file.content, file.encoding); if (rel.startsWith('..')) throw Error( @@ -90,7 +94,7 @@ class Job { }); await fs.chown(path.dirname(file_path), this.uid, this.gid); - await fs.write_file(file_path, file.content); + await fs.write_file(file_path, file_content); await fs.chown(file_path, this.uid, this.gid); } @@ -387,7 +391,7 @@ class Job { await this.cleanup_filesystem(); - remainingJobSpaces++; + remaining_job_spaces++; } } diff --git a/docs/api-v2.md b/docs/api-v2.md index d5cb486..b25e142 100644 --- a/docs/api-v2.md +++ b/docs/api-v2.md @@ -60,6 +60,7 @@ Runs the given code, using the given runtime and arguments, returning the result - `files`: An array of files which should be uploaded into the job context - `files[].name` (_optional_): Name of file to be written, if none a random name is picked - `files[].content`: Content of file to be written +- `files[].encoding` (_optional_): The encoding scheme used for the file content. One of `base64`, `hex` or `utf8`. Defaults to `utf8`. - `stdin` (_optional_): Text to pass into stdin of the program. Defaults to blank string. - `args` (_optional_): Arguments to pass to the program. Defaults to none - `run_timeout` (_optional_): The maximum allowed time in milliseconds for the compile stage to finish before bailing out. Must be a number, less than or equal to the configured maximum timeout. diff --git a/readme.md b/readme.md index a2f3afa..d2f2e75 100644 --- a/readme.md +++ b/readme.md @@ -256,6 +256,7 @@ This endpoint requests execution of some arbitrary code. - `files` (**required**) An array of files containing code or other data that should be used for execution. The first file in this array is considered the main file. - `files[].name` (_optional_) The name of the file to upload, must be a string containing no path or left out. - `files[].content` (**required**) The content of the files to upload, must be a string containing text to write. +- `files[].encoding` (_optional_) The encoding scheme used for the file content. One of `base64`, `hex` or `utf8`. Defaults to `utf8`. - `stdin` (_optional_) The text to pass as stdin to the program. Must be a string or left out. Defaults to blank string. - `args` (_optional_) The arguments to pass to the program. Must be an array or left out. Defaults to `[]`. - `compile_timeout` (_optional_) The maximum time allowed for the compile stage to finish before bailing out in milliseconds. Must be a number or left out. Defaults to `10000` (10 seconds). From 0faea205dbdc05429087dc7cab804bae667f5ff3 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Thu, 14 Oct 2021 01:36:29 +1300 Subject: [PATCH 35/72] Only compile/run files in utf8 encoding --- api/src/job.js | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/api/src/job.js b/api/src/job.js index bf7870b..7acff22 100644 --- a/api/src/job.js +++ b/api/src/job.js @@ -222,6 +222,8 @@ class Job { } runtime=${this.runtime.toString()}` ); + const code_files = this.files.filter(file => file.encoding == 'utf8'); + logger.debug('Compiling'); let compile; @@ -229,7 +231,7 @@ class Job { if (this.runtime.compiled) { compile = await this.safe_call( path.join(this.runtime.pkgdir, 'compile'), - this.files.map(x => x.name), + code_files.map(x => x.name), this.timeouts.compile, this.memory_limits.compile ); @@ -239,7 +241,7 @@ class Job { const run = await this.safe_call( path.join(this.runtime.pkgdir, 'run'), - [this.files[0].name, ...this.args], + [code_files[0].name, ...this.args], this.timeouts.run, this.memory_limits.run ); @@ -268,11 +270,13 @@ class Job { } gid=${this.gid} runtime=${this.runtime.toString()}` ); + const code_files = this.files.filter(file => file.encoding == 'utf8'); + if (this.runtime.compiled) { eventBus.emit('stage', 'compile'); const { error, code, signal } = await this.safe_call( path.join(this.runtime.pkgdir, 'compile'), - this.files.map(x => x.name), + code_files.map(x => x.name), this.timeouts.compile, this.memory_limits.compile, eventBus @@ -285,7 +289,7 @@ class Job { eventBus.emit('stage', 'run'); const { error, code, signal } = await this.safe_call( path.join(this.runtime.pkgdir, 'run'), - [this.files[0].name, ...this.args], + [code_files[0].name, ...this.args], this.timeouts.run, this.memory_limits.run, eventBus From fecfed48fdd121658d3a8104e934a7ae99cb36b2 Mon Sep 17 00:00:00 2001 From: Pyroseza <8463626+Pyroseza@users.noreply.github.com> Date: Wed, 13 Oct 2021 16:56:12 +0100 Subject: [PATCH 36/72] added python 3.10 --- packages/python/3.10.0/build.sh | 20 ++++++++++++++++++++ packages/python/3.10.0/compile | 3 +++ packages/python/3.10.0/environment | 1 + packages/python/3.10.0/metadata.json | 5 +++++ packages/python/3.10.0/run | 3 +++ packages/python/3.10.0/test.py | 7 +++++++ 6 files changed, 39 insertions(+) create mode 100755 packages/python/3.10.0/build.sh create mode 100644 packages/python/3.10.0/compile create mode 100644 packages/python/3.10.0/environment create mode 100644 packages/python/3.10.0/metadata.json create mode 100644 packages/python/3.10.0/run create mode 100644 packages/python/3.10.0/test.py diff --git a/packages/python/3.10.0/build.sh b/packages/python/3.10.0/build.sh new file mode 100755 index 0000000..38796a1 --- /dev/null +++ b/packages/python/3.10.0/build.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +PREFIX=$(realpath $(dirname $0)) + +mkdir -p build + +cd build + +curl "https://www.python.org/ftp/python/3.10.0/Python-3.10.0.tgz" -o python.tar.gz +tar xzf python.tar.gz --strip-components=1 +rm python.tar.gz + +./configure --prefix "$PREFIX" --with-ensurepip=install +make -j$(nproc) +make install -j$(nproc) + +cd .. + +rm -rf build + diff --git a/packages/python/3.10.0/compile b/packages/python/3.10.0/compile new file mode 100644 index 0000000..e37a74c --- /dev/null +++ b/packages/python/3.10.0/compile @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +# Put instructions to compile source code, remove this file if the language does not require this stage diff --git a/packages/python/3.10.0/environment b/packages/python/3.10.0/environment new file mode 100644 index 0000000..977a5e8 --- /dev/null +++ b/packages/python/3.10.0/environment @@ -0,0 +1 @@ +export PATH=$PWD/bin:$PATH diff --git a/packages/python/3.10.0/metadata.json b/packages/python/3.10.0/metadata.json new file mode 100644 index 0000000..64baa95 --- /dev/null +++ b/packages/python/3.10.0/metadata.json @@ -0,0 +1,5 @@ +{ + "language": "python", + "version": "3.10.0", + "aliases": ["py", "py3", "python3"] +} diff --git a/packages/python/3.10.0/run b/packages/python/3.10.0/run new file mode 100644 index 0000000..d141a0d --- /dev/null +++ b/packages/python/3.10.0/run @@ -0,0 +1,3 @@ +#!/bin/bash + +python3.10 "$@" diff --git a/packages/python/3.10.0/test.py b/packages/python/3.10.0/test.py new file mode 100644 index 0000000..1784bfa --- /dev/null +++ b/packages/python/3.10.0/test.py @@ -0,0 +1,7 @@ +working = True + +match working: + case True: + print("OK") + case False: + print() From 5641f671d10027a67455ab9150e822f3b122ef89 Mon Sep 17 00:00:00 2001 From: Pyroseza <8463626+Pyroseza@users.noreply.github.com> Date: Wed, 13 Oct 2021 20:33:29 +0100 Subject: [PATCH 37/72] remove unneeded compile file from python 3.10.0 package --- packages/python/3.10.0/compile | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 packages/python/3.10.0/compile diff --git a/packages/python/3.10.0/compile b/packages/python/3.10.0/compile deleted file mode 100644 index e37a74c..0000000 --- a/packages/python/3.10.0/compile +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash - -# Put instructions to compile source code, remove this file if the language does not require this stage From 0f440c082b51d5223f4a90188b53b46d6aecbe56 Mon Sep 17 00:00:00 2001 From: Dan Vargas Date: Wed, 13 Oct 2021 16:23:49 -0500 Subject: [PATCH 38/72] fix pkg build action for when pkg files are deleted --- .github/workflows/package-pr.yaml | 2 +- .github/workflows/package-push.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/package-pr.yaml b/.github/workflows/package-pr.yaml index 78101b0..8ac17a3 100644 --- a/.github/workflows/package-pr.yaml +++ b/.github/workflows/package-pr.yaml @@ -53,7 +53,7 @@ jobs: - name: Build Packages run: | - PACKAGES=$(jq '.[]' -r ${HOME}/files.json | awk -F/ '$2 && $3{ print $2 "-" $3 }' | sort -u) + PACKAGES=$(jq '.[]' -r ${HOME}/files*.json | awk -F/ '$1~/packages/ && $2 && $3{ print $2 "-" $3 }' | sort -u) echo "Packages: $PACKAGES" docker pull docker.pkg.github.com/engineer-man/piston/repo-builder:latest docker build -t repo-builder repo diff --git a/.github/workflows/package-push.yaml b/.github/workflows/package-push.yaml index ee49487..9de6051 100644 --- a/.github/workflows/package-push.yaml +++ b/.github/workflows/package-push.yaml @@ -30,7 +30,7 @@ jobs: - name: Build Packages run: | - PACKAGES=$(jq '.[]' -r ${HOME}/files.json | awk -F/ '{ print $2 "-" $3 }' | sort -u) + PACKAGES=$(jq '.[]' -r ${HOME}/files*.json | awk -F/ '$1~/packages/ && $2 && $3{ print $2 "-" $3 }' | sort -u) echo "Packages: $PACKAGES" docker pull docker.pkg.github.com/engineer-man/piston/repo-builder:latest docker build -t repo-builder repo From 8403e0f512e7925457295b07da219eb6acdcc618 Mon Sep 17 00:00:00 2001 From: Pyroseza <8463626+Pyroseza@users.noreply.github.com> Date: Wed, 13 Oct 2021 22:30:38 +0100 Subject: [PATCH 39/72] added extra alias for python 3.10 --- packages/python/3.10.0/metadata.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/python/3.10.0/metadata.json b/packages/python/3.10.0/metadata.json index 64baa95..b767f8d 100644 --- a/packages/python/3.10.0/metadata.json +++ b/packages/python/3.10.0/metadata.json @@ -1,5 +1,5 @@ { "language": "python", "version": "3.10.0", - "aliases": ["py", "py3", "python3"] + "aliases": ["py", "py3", "python3", "python3.10"] } From dfd6beb6ed0730d096e379d4e1a084e71009a049 Mon Sep 17 00:00:00 2001 From: Brikaa Date: Thu, 14 Oct 2021 12:44:51 +0200 Subject: [PATCH 40/72] Fix piston script after rebase --- piston | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/piston b/piston index c59002d..fbd256d 100755 --- a/piston +++ b/piston @@ -17,7 +17,7 @@ docker_compose(){ init_precommit() { if [ $PISTON_ENV == "dev" ]; then rm -f .git/hooks/pre-commit - ln -s $(realpath $(dirname "$0"))/pre-commit .git/hooks/pre-commit + ln -s "$PISTON_PATH/pre-commit" "$PISTON_PATH/.git/hooks/pre-commit" fi } @@ -67,8 +67,6 @@ case $1 in restart) docker_compose restart ;; start) init_precommit - rm -f .git/hooks/pre-commit - ln -s "$PISTON_PATH/pre-commit" "$PISTON_PATH/.git/hooks/pre-commit" docker_compose up -d ;; stop) docker_compose down ;; From 7e5844bcb1c42a7852ad36e8b82daabcfb0ac59e Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Fri, 15 Oct 2021 16:14:06 +1300 Subject: [PATCH 41/72] require arleast 1 utf-8 encoded file --- api/src/api/v2.js | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/api/src/api/v2.js b/api/src/api/v2.js index f9a3f66..d772c5a 100644 --- a/api/src/api/v2.js +++ b/api/src/api/v2.js @@ -87,6 +87,15 @@ function get_job(body) { } } + if ( + files.filter(file => !file.encoding || file.encoding === 'utf8') + .length === 0 + ) { + return reject({ + message: 'files must include at least one utf8 encoded file', + }); + } + const rt = runtime.get_latest_runtime_matching_language_version( language, version From 6ca9606f81d396e95dfd3a4a78520cf176d33682 Mon Sep 17 00:00:00 2001 From: Brikaa Date: Sat, 16 Oct 2021 11:01:17 +0200 Subject: [PATCH 42/72] Adhere to project let, const practices in my code --- api/src/api/v2.js | 10 +++++----- api/src/config.js | 18 +++++++++--------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/api/src/api/v2.js b/api/src/api/v2.js index d772c5a..c0526f4 100644 --- a/api/src/api/v2.js +++ b/api/src/api/v2.js @@ -106,11 +106,11 @@ function get_job(body) { }); } - for (let constraint of ['memory_limit', 'timeout']) { - for (let type of ['compile', 'run']) { - let constraint_name = `${type}_${constraint}`; - let constraint_value = body[constraint_name]; - let configured_limit = rt[`${constraint}s`][type]; + for (const constraint of ['memory_limit', 'timeout']) { + for (const type of ['compile', 'run']) { + const constraint_name = `${type}_${constraint}`; + const constraint_value = body[constraint_name]; + const configured_limit = rt[`${constraint}s`][type]; if (!constraint_value) { continue; } diff --git a/api/src/config.js b/api/src/config.js index 8d0020f..85aa691 100644 --- a/api/src/config.js +++ b/api/src/config.js @@ -11,8 +11,8 @@ function parse_overrides(overrides) { } function validate_overrides(overrides, options) { - for (let language in overrides) { - for (let key in overrides[language]) { + for (const language in overrides) { + for (const key in overrides[language]) { if ( ![ 'max_process_count', @@ -28,13 +28,13 @@ function validate_overrides(overrides, options) { logger.error(`Invalid overridden option: ${key}`); return false; } - let option = options.find(o => o.key === key); - let parser = option.parser; - let raw = overrides[language][key]; - let value = parser(raw); - let validators = option.validators; - for (let validator of validators) { - let response = validator(value, raw); + const option = options.find(o => o.key === key); + const parser = option.parser; + const raw = overrides[language][key]; + const value = parser(raw); + const validators = option.validators; + for (const validator of validators) { + const response = validator(value, raw); if (response !== true) { logger.error( `Failed to validate overridden option: ${key}`, From 64833a04085b33948256caa4d6a787a9f15d08db Mon Sep 17 00:00:00 2001 From: Brikaa Date: Fri, 15 Oct 2021 14:25:33 +0200 Subject: [PATCH 43/72] pkg(file-0.0.1): Run executable --- api/src/api/v2.js | 19 ++++++++++--------- api/src/job.js | 8 ++++++-- packages/file/0.0.1/build.sh | 3 +++ packages/file/0.0.1/environment | 3 +++ packages/file/0.0.1/metadata.json | 5 +++++ packages/file/0.0.1/run | 6 ++++++ packages/file/0.0.1/test.file | Bin 0 -> 16568 bytes readme.md | 1 + 8 files changed, 34 insertions(+), 11 deletions(-) create mode 100755 packages/file/0.0.1/build.sh create mode 100644 packages/file/0.0.1/environment create mode 100644 packages/file/0.0.1/metadata.json create mode 100644 packages/file/0.0.1/run create mode 100755 packages/file/0.0.1/test.file diff --git a/api/src/api/v2.js b/api/src/api/v2.js index c0526f4..d6cd3f2 100644 --- a/api/src/api/v2.js +++ b/api/src/api/v2.js @@ -87,15 +87,6 @@ function get_job(body) { } } - if ( - files.filter(file => !file.encoding || file.encoding === 'utf8') - .length === 0 - ) { - return reject({ - message: 'files must include at least one utf8 encoded file', - }); - } - const rt = runtime.get_latest_runtime_matching_language_version( language, version @@ -106,6 +97,16 @@ function get_job(body) { }); } + if ( + rt.language !== 'file' && + files.filter(file => !file.encoding || file.encoding === 'utf8') + .length === 0 + ) { + return reject({ + message: 'files must include at least one utf8 encoded file', + }); + } + for (const constraint of ['memory_limit', 'timeout']) { for (const type of ['compile', 'run']) { const constraint_name = `${type}_${constraint}`; diff --git a/api/src/job.js b/api/src/job.js index 7acff22..ca95e6f 100644 --- a/api/src/job.js +++ b/api/src/job.js @@ -222,7 +222,9 @@ class Job { } runtime=${this.runtime.toString()}` ); - const code_files = this.files.filter(file => file.encoding == 'utf8'); + const code_files = + (this.runtime.language === 'file' && this.files) || + this.files.filter(file => file.encoding == 'utf8'); logger.debug('Compiling'); @@ -270,7 +272,9 @@ class Job { } gid=${this.gid} runtime=${this.runtime.toString()}` ); - const code_files = this.files.filter(file => file.encoding == 'utf8'); + const code_files = + (this.runtime.language === 'file' && this.files) || + this.files.filter(file => file.encoding == 'utf8'); if (this.runtime.compiled) { eventBus.emit('stage', 'compile'); diff --git a/packages/file/0.0.1/build.sh b/packages/file/0.0.1/build.sh new file mode 100755 index 0000000..d7ab47e --- /dev/null +++ b/packages/file/0.0.1/build.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +# Nothing to do here diff --git a/packages/file/0.0.1/environment b/packages/file/0.0.1/environment new file mode 100644 index 0000000..d5a7803 --- /dev/null +++ b/packages/file/0.0.1/environment @@ -0,0 +1,3 @@ +#!/bin/bash + +export PATH=$PWD:$PATH diff --git a/packages/file/0.0.1/metadata.json b/packages/file/0.0.1/metadata.json new file mode 100644 index 0000000..ae34ae9 --- /dev/null +++ b/packages/file/0.0.1/metadata.json @@ -0,0 +1,5 @@ +{ + "language": "file", + "version": "0.0.1", + "aliases": ["executable", "elf", "binary"] +} diff --git a/packages/file/0.0.1/run b/packages/file/0.0.1/run new file mode 100644 index 0000000..77ed0a6 --- /dev/null +++ b/packages/file/0.0.1/run @@ -0,0 +1,6 @@ +#!/bin/bash + +filename="$1" +chmod +x "$filename" +shift +./"$filename" "$@" diff --git a/packages/file/0.0.1/test.file b/packages/file/0.0.1/test.file new file mode 100755 index 0000000000000000000000000000000000000000..8c7a67d92841fc71dd40ef76305328833c4aab6e GIT binary patch literal 16568 zcmeHOZ*Wvs6~DV75Ck?0ZAwa&td-6{F-rmgL!H>&Bpcq^04Yf zZKzVI(VCS;(GN<;en`hZ)v2R6`hkAXmO*NrsScfSYVDs99b1D_D@AcqEv&zD?>(Eh zkEGLf`pJ2B=Iy!Xch2wJ`_8-X<=wn{KNsum@%wy&OO5!LAZ;qBkcc+SwCf5)M6`=# z_-zua#X{%{B&O^UC16$MIe)dR(|8Re@fxWzhkU<+1xxNBN#e~{rWPp+OQFw$c(r8G zx?6r%_md^tEA54dj1rQ+75RN~Kw;_RQEn!hrCCgLxTJ3Hv_6045lp-(&70CZmO}Gb zay}_1bhPVuZrK7G6_#$^u;#gCx1tG_)CQIQm+jQMX|GT7>OwM%aLWM&3zpp8&EQd9 z-frQ`yi2z?-*p(%`DCdtr`Rx(9@^5jVIWHzMTF;O6PcFyHeZ9|J={_)V=c9!NRFX?9$37)5ks;d;V$ip?(vGe5g=7 zzT~7+eK8L5QTh0qtP@7%wDsFe!_lQ(O1u2`IG`1m!{Uy4@Ndk6Q$IsE+~s!w=88}K z4Y{46e|R3e2l!lZP659bhr9d;z;dxlL?X%(qz0*&=Kf4BYZmN8-Zo8P#s{{WDJyU7 zPZw+}Kd`-PB$u@Y5L zIFUE)d?IZZ;>lQS7qCrmPAAP=!OmNWOguSY#`-s#eK#e+yDVN8$7@UGiZ;5P97~u9 z+Zs!=vM-%YjHK^DKTQnIq&}rgBApe`3wA2GemxD)Eu=c zhn7p4IHI3pjL&HNs0)v1ye^`6?9ck?8x>AtpT|5Q&pB|qkGPz1;PUye4bu)>-ABrB z&VlPcCDJhCz`0Lk;PFPW-zPoQ&vETfF?cAF#v%fNe?fq&Mo{*N*KYMpVo_759`FuwSN?Ju7;#(!CNQZ`fG{CU9V%h!!V)Hez8 zA12HBvwtd=%SWV6FF5DVK2y=@#pe9k6Rd-6Np`o8eCQo6eX~n%cIvHvkC&dg!zlgP z82`(;zJcD!+Go+CF}ci?qT>XbD>tkFzn5i**Pa4l$vadvdQ#wFh`OLohCY%TJ>HH6M z)V3gGas8k%-f<5+g;9FhUTPff_#Tw$H!hXS(Y+3_%uUj&)vY+~=pipTh?!P3DuOU0S-gKzqZ8}BQ5K2BwdLV@x)z z10T^oMdx+6b_r>vUrm1o#v{M31p5V1AEygA{SbPw_vF5CebX1HL0oNpy`s1DPIlb3 zUWn)?;-zP!gYnXZ=s>je=AbdzJOH@AcilxAL(@OLh=xyGtg)L~U(osOEuHTzz1Cg& zYqWgjAC2*oK4a@k#lO+`x@~Xt_UPW|9Z~b-zS&0UBx&T4l;g585@nyfA1?!52D}V- z8SpaTWx&gTmjN#WUIx4jco}$?8SvrTjo5jU@Lf~$QTpae-&7jUmCF;5yCIK34nsZ# zc@%OK^1j#0Ch1WhSX=$j`M-#xp8Z!GA$X6d53Bd`%p z-}Xa@i3zF@-HYTWr8TIw;h6XbVnK>PuqV*?@%klq*Buf!uDap+O>3?tnE1Ejn1Y`q z5{N>$jpHcr76NpkwjRLoAlj1wVu9fO{;p*U_naX&g_% zeznuyr}&sEi&t@+hTS52PaybJe>~9mm6})}G`S!aSo=V2ccA6|g+`!#e9^W*BpYat z23n$lwVi=bXP~h&5bO-p$?s}y$j=PoIKNj>NUwMq@G{_Kz{`M_0WSky2D}V-8Spaj z|B(S+C&%mJcpaRcM#(jKbf&D+8aygXm8dMpTc-7+dhw*3s{rG5dDmz=&zV-Szg~w& z^BgKPC!%uUQaMNRm|m2|bCNR)2<0`XSbwMOyasNi?&uQbCK|PD(2|9wi7CDQjXoVy zF_fsxA9*d_N{#Q<0ONgn5g+%vPPfl%1iAgyn$Ld!Evxf`yM*v{RGx28$HO;KsdK+D zEtC3;PQ)8p*XhvF-#bxR_w5SVqAy@X%RauKF2A>{^7_hcU0pYX)(#F8vvx7m+7jLr zZfR~UO0{*J2Hq+rrG0^T9Uqme3ta~vEDz`l9@6G3Am{R@>*U95__XW1h)1~4>qMc- zvI)P0^L?XX5l6*`b-{g6h3H`+1|@!#;Pt2M|5;Qh4&RTAPe}iU%6-K6BdE;P|DQv$S#$PGi<1#MeG+dU?TQ$zlAC8}qaT+W4J>%oRgW_rt zj4+~PC;AftE?oGR;lG?#iieaR--izZzg9t2zpH?+Kno?EXZHCS#qnP)_ LPtooQ z|8l|4Q^rroIIPXe9`Z;0DV?_s+VuQ9{;T0n?YhtRdcx;e&mI9z`4N%XHmNp<_(AyZ zj#lH_rT?nS;_m^j|3p*~4oW;ILfU9)>m=|{m8VjEp#6_%fByUM*gWxnKTn)j=D}%+ z`ds}K{mH!D8qQ%cd}1hV+KK%FK*26zElE<$F7P+)jG3e*{snlZa_0V#+)!e~Oxd}7 z!Aum#L^78d9kFaH6>gc^!^FD#w3*1~69-HyYv&J$efdPjGE>D&<^U)T&4j02}@7WRE9`Ay!s?(`zKyI6;w6L?Mr$07e4n#Y9V~A>{5_ST+CU&B;sdUyX7A&IE z#s%K15OHogzy&De%we1ZZMJZ1aH#frcY7*S487ICG_kRRPag5)o(h)-bzbeEKwpLm z1+LG%HN!=g8!(cEqD`1BVrU(}fWbDKc zq@7oCm}}`QcE5~@a5iUK;r-cScr>3Iwet1>2Q*YnWB*e+rJ-nNyqWGZ={}q&42y8; zKo(6XY3G&YE-PQa(5zBS*z(p$f(W`cI%12k9PeRN!~1hksZy|#B8+V>P-VWt`J5bP zVQX05YQrf+V@<_S_ncyI9Ze-NX#`VrTwxK$&7Q$MJjZkFokH{;jYls2Mylf8$H?!| z+AMv!PVz%I=-rk1^X=CoUn=~*sx%?CpyK4a_wB7=M=km9BE$4v{nk%r*^97a>&F36*|1~ToQK2p~pXVUN4AegJ>zJqI0Bp|UZlC5GPX3)C<;~I{qocU{^D=B$ zrY!G4a^v@V-e>F17xP)Z4ob2x?H!f-78k+Uspj?g~K@|j~%CLyAF6r510?YNOMl^KR Date: Fri, 15 Oct 2021 16:51:11 +0200 Subject: [PATCH 44/72] Check for base64 in CLI --- cli/commands/execute.js | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/cli/commands/execute.js b/cli/commands/execute.js index c69d3a0..4929ccc 100644 --- a/cli/commands/execute.js +++ b/cli/commands/execute.js @@ -245,9 +245,18 @@ async function run_non_interactively(files, argv) { exports.handler = async argv => { const files = [...(argv.files || []), argv.file].map(file_path => { + const encoding = + (fs + .readFileSync(file_path) + .toString() + .split('') + .some(x => x.charCodeAt(0) >= 128) && + 'base64') || + 'utf8'; return { name: path.basename(file_path), - content: fs.readFileSync(file_path).toString(), + content: fs.readFileSync(file_path).toString(encoding), + encoding, }; }); From 19252467cbd91920eb897cd6439f186b72b0df68 Mon Sep 17 00:00:00 2001 From: Brikaa Date: Fri, 15 Oct 2021 16:57:11 +0200 Subject: [PATCH 45/72] Use some() instead of filter() in v2.js --- api/src/api/v2.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/api/src/api/v2.js b/api/src/api/v2.js index d6cd3f2..4f0c3e2 100644 --- a/api/src/api/v2.js +++ b/api/src/api/v2.js @@ -99,8 +99,7 @@ function get_job(body) { if ( rt.language !== 'file' && - files.filter(file => !file.encoding || file.encoding === 'utf8') - .length === 0 + !files.some(file => !file.encoding || file.encoding === 'utf8') ) { return reject({ message: 'files must include at least one utf8 encoded file', From 52fb900603070f12c6c5b0bd7ab197aaecb897fa Mon Sep 17 00:00:00 2001 From: Brikaa Date: Sat, 16 Oct 2021 11:55:22 +0200 Subject: [PATCH 46/72] Don't read the file twice --- cli/commands/execute.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cli/commands/execute.js b/cli/commands/execute.js index 4929ccc..0d906bc 100644 --- a/cli/commands/execute.js +++ b/cli/commands/execute.js @@ -245,9 +245,9 @@ async function run_non_interactively(files, argv) { exports.handler = async argv => { const files = [...(argv.files || []), argv.file].map(file_path => { + const buffer = fs.readFileSync(file_path); const encoding = - (fs - .readFileSync(file_path) + (buffer .toString() .split('') .some(x => x.charCodeAt(0) >= 128) && @@ -255,7 +255,7 @@ exports.handler = async argv => { 'utf8'; return { name: path.basename(file_path), - content: fs.readFileSync(file_path).toString(encoding), + content: buffer.toString(encoding), encoding, }; }); From cd2b471eed06c2ff966f1d990a313a869ac4f7a5 Mon Sep 17 00:00:00 2001 From: Cory <34887996+ccall48@users.noreply.github.com> Date: Sun, 17 Oct 2021 11:46:17 +1100 Subject: [PATCH 47/72] add missing libs back into py v3.10 --- packages/python/3.10.0/build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/python/3.10.0/build.sh b/packages/python/3.10.0/build.sh index 38796a1..00c839a 100755 --- a/packages/python/3.10.0/build.sh +++ b/packages/python/3.10.0/build.sh @@ -18,3 +18,4 @@ cd .. rm -rf build +bin/pip3 install numpy scipy pandas pycrypto whoosh bcrypt passlib sympy From c7efa5372ac7d275a5e140995b4dff510171c14c Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Thu, 11 Nov 2021 19:27:54 +1300 Subject: [PATCH 48/72] api(job): Switch process cleanup to sync The system used to use async. This would result in execution is handed off to other processes. In the case of a forkbomb was used, it could circumvent the janitor as it consumed more CPU time which would prevent the process janitor from reading the process information in. --- api/src/job.js | 86 +++++++++++++++++++++++++++++++++----------------- 1 file changed, 57 insertions(+), 29 deletions(-) diff --git a/api/src/job.js b/api/src/job.js index ca95e6f..e6fafca 100644 --- a/api/src/job.js +++ b/api/src/job.js @@ -5,6 +5,7 @@ const path = require('path'); const config = require('./config'); const globals = require('./globals'); const fs = require('fs/promises'); +const fss = require('fs'); const wait_pid = require('waitpid'); const job_states = { @@ -184,24 +185,24 @@ class Job { } }); - const exit_cleanup = async () => { + const exit_cleanup = () => { clear_timeout(kill_timeout); proc.stderr.destroy(); proc.stdout.destroy(); - await this.cleanup_processes(); + this.cleanup_processes(); logger.debug(`Finished exit cleanup uuid=${this.uuid}`); }; - proc.on('exit', async (code, signal) => { - await exit_cleanup(); + proc.on('exit', (code, signal) => { + exit_cleanup(); resolve({ stdout, stderr, code, signal, output }); }); - proc.on('error', async err => { - await exit_cleanup(); + proc.on('error', err => { + exit_cleanup(); reject({ error: err, stdout, stderr, output }); }); @@ -304,38 +305,46 @@ class Job { this.state = job_states.EXECUTED; } - async cleanup_processes(dont_wait = []) { + cleanup_processes(dont_wait = []) { let processes = [1]; + const to_wait = []; logger.debug(`Cleaning up processes uuid=${this.uuid}`); while (processes.length > 0) { processes = []; - const proc_ids = await fs.readdir('/proc'); + const proc_ids = fss.readdir_sync('/proc'); - processes = await Promise.all( - proc_ids.map(async proc_id => { - if (isNaN(proc_id)) return -1; - try { - const proc_status = await fs.read_file( - path.join('/proc', proc_id, 'status') - ); - const proc_lines = proc_status.to_string().split('\n'); - const uid_line = proc_lines.find(line => - line.starts_with('Uid:') - ); - const [_, ruid, euid, suid, fuid] = - uid_line.split(/\s+/); + processes = proc_ids.map(proc_id => { + if (isNaN(proc_id)) return -1; + try { + const proc_status = fss.read_file_sync( + path.join('/proc', proc_id, 'status') + ); + const proc_lines = proc_status.to_string().split('\n'); + const state_line = proc_lines.find(line => + line.starts_with('State:') + ); + const uid_line = proc_lines.find(line => + line.starts_with('Uid:') + ); + const [_, ruid, euid, suid, fuid] = uid_line.split(/\s+/); - if (ruid == this.uid || euid == this.uid) - return parse_int(proc_id); - } catch { + const [_1, state, user_friendly] = state_line.split(/\s+/); + + if (state == 'Z') + // Zombie process, just needs to be waited return -1; - } + // We should kill in all other state (Sleep, Stopped & Running) + if (ruid == this.uid || euid == this.uid) + return parse_int(proc_id); + } catch { return -1; - }) - ); + } + + return -1; + }); processes = processes.filter(p => p > 0); @@ -348,8 +357,12 @@ class Job { // First stop the processes, but keep their resources allocated so they cant re-fork try { process.kill(proc, 'SIGSTOP'); - } catch { + } catch (e) { // Could already be dead + logger.debug( + `Got error while SIGSTOPping process ${proc}:`, + e + ); } } @@ -359,12 +372,26 @@ class Job { process.kill(proc, 'SIGKILL'); } catch { // Could already be dead and just needs to be waited on + logger.debug( + `Got error while SIGKILLing process ${proc}:`, + e + ); } - if (!dont_wait.includes(proc)) wait_pid(proc); + to_wait.push(proc); } } + logger.debug( + `Finished kill-loop, calling wait_pid to end any zombie processes uuid=${this.uuid}` + ); + + for (const proc of to_wait) { + if (dont_wait.includes(proc)) continue; + + wait_pid(proc); + } + logger.debug(`Cleaned up processes uuid=${this.uuid}`); } @@ -397,6 +424,7 @@ class Job { async cleanup() { logger.info(`Cleaning up job uuid=${this.uuid}`); + this.cleanup_processes(); // Run process janitor, just incase there are any residual processes somehow await this.cleanup_filesystem(); remaining_job_spaces++; From c091c117c7c1a5165c137e3d367adc8eea2662f4 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Thu, 11 Nov 2021 21:34:30 +1300 Subject: [PATCH 49/72] api(job): Decrease safe_call CPU time By increasing the niceness value of child processes, the scheduler gives less CPU Time to them. This allows the node process to get more CPU time, and can work through the event queue faster. --- api/src/job.js | 75 ++++++++++++++++++++++++++------------------------ 1 file changed, 39 insertions(+), 36 deletions(-) diff --git a/api/src/job.js b/api/src/job.js index e6fafca..eef2ffd 100644 --- a/api/src/job.js +++ b/api/src/job.js @@ -1,4 +1,5 @@ -const logger = require('logplease').create('job'); +const logplease = require('logplease'); +const logger = logplease.create('job'); const { v4: uuidv4 } = require('uuid'); const cp = require('child_process'); const path = require('path'); @@ -30,6 +31,9 @@ setInterval(() => { class Job { constructor({ runtime, files, args, stdin, timeouts, memory_limits }) { this.uuid = uuidv4(); + + this.logger = logplease.create(`job/${this.uuid}`); + this.runtime = runtime; this.files = files.map((file, i) => ({ name: file.name || `file${i}.code`, @@ -54,6 +58,8 @@ class Job { uid %= config.runner_uid_max - config.runner_uid_min + 1; gid %= config.runner_gid_max - config.runner_gid_min + 1; + this.logger.debug(`Assigned uid=${this.uid} gid=${this.gid}`); + this.state = job_states.READY; this.dir = path.join( config.data_directory, @@ -64,17 +70,17 @@ class Job { async prime() { if (remaining_job_spaces < 1) { - logger.info(`Awaiting job slot uuid=${this.uuid}`); + this.logger.info(`Awaiting job slot`); await new Promise(resolve => { jobQueue.push(resolve); }); } - logger.info(`Priming job uuid=${this.uuid}`); + this.logger.info(`Priming job`); remaining_job_spaces--; - logger.debug('Writing files to job cache'); + this.logger.debug('Writing files to job cache'); - logger.debug(`Transfering ownership uid=${this.uid} gid=${this.gid}`); + this.logger.debug(`Transfering ownership`); await fs.mkdir(this.dir, { mode: 0o700 }); await fs.chown(this.dir, this.uid, this.gid); @@ -101,7 +107,7 @@ class Job { this.state = job_states.PRIMED; - logger.debug('Primed job'); + this.logger.debug('Primed job'); } async safe_call(file, args, timeout, memory_limit, eventBus = null) { @@ -119,7 +125,14 @@ class Job { prlimit.push('--as=' + memory_limit); } - const proc_call = [...prlimit, ...nonetwork, 'bash', file, ...args]; + const proc_call = [ + 'nice', + ...prlimit, + ...nonetwork, + 'bash', + file, + ...args, + ]; var stdout = ''; var stderr = ''; @@ -154,9 +167,7 @@ class Job { const kill_timeout = (timeout >= 0 && set_timeout(async _ => { - logger.info( - `Timeout exceeded timeout=${timeout} uuid=${this.uuid}` - ); + this.logger.info(`Timeout exceeded timeout=${timeout}`); process.kill(proc.pid, 'SIGKILL'); }, timeout)) || null; @@ -165,7 +176,7 @@ class Job { if (eventBus !== null) { eventBus.emit('stderr', data); } else if (stderr.length > this.runtime.output_max_size) { - logger.info(`stderr length exceeded uuid=${this.uuid}`); + this.logger.info(`stderr length exceeded`); process.kill(proc.pid, 'SIGKILL'); } else { stderr += data; @@ -177,7 +188,7 @@ class Job { if (eventBus !== null) { eventBus.emit('stdout', data); } else if (stdout.length > this.runtime.output_max_size) { - logger.info(`stdout length exceeded uuid=${this.uuid}`); + this.logger.info(`stdout length exceeded`); process.kill(proc.pid, 'SIGKILL'); } else { stdout += data; @@ -192,7 +203,7 @@ class Job { proc.stdout.destroy(); this.cleanup_processes(); - logger.debug(`Finished exit cleanup uuid=${this.uuid}`); + this.logger.debug(`Finished exit cleanup`); }; proc.on('exit', (code, signal) => { @@ -217,17 +228,13 @@ class Job { ); } - logger.info( - `Executing job uuid=${this.uuid} uid=${this.uid} gid=${ - this.gid - } runtime=${this.runtime.toString()}` - ); + this.logger.info(`Executing job runtime=${this.runtime.toString()}`); const code_files = (this.runtime.language === 'file' && this.files) || this.files.filter(file => file.encoding == 'utf8'); - logger.debug('Compiling'); + this.logger.debug('Compiling'); let compile; @@ -240,7 +247,7 @@ class Job { ); } - logger.debug('Running'); + this.logger.debug('Running'); const run = await this.safe_call( path.join(this.runtime.pkgdir, 'run'), @@ -267,10 +274,8 @@ class Job { ); } - logger.info( - `Interactively executing job uuid=${this.uuid} uid=${ - this.uid - } gid=${this.gid} runtime=${this.runtime.toString()}` + this.logger.info( + `Interactively executing job runtime=${this.runtime.toString()}` ); const code_files = @@ -290,7 +295,7 @@ class Job { eventBus.emit('exit', 'compile', { error, code, signal }); } - logger.debug('Running'); + this.logger.debug('Running'); eventBus.emit('stage', 'run'); const { error, code, signal } = await this.safe_call( path.join(this.runtime.pkgdir, 'run'), @@ -308,7 +313,7 @@ class Job { cleanup_processes(dont_wait = []) { let processes = [1]; const to_wait = []; - logger.debug(`Cleaning up processes uuid=${this.uuid}`); + this.logger.debug(`Cleaning up processes`); while (processes.length > 0) { processes = []; @@ -349,9 +354,7 @@ class Job { processes = processes.filter(p => p > 0); if (processes.length > 0) - logger.debug( - `Got processes to kill: ${processes} uuid=${this.uuid}` - ); + this.logger.debug(`Got processes to kill: ${processes}`); for (const proc of processes) { // First stop the processes, but keep their resources allocated so they cant re-fork @@ -359,7 +362,7 @@ class Job { process.kill(proc, 'SIGSTOP'); } catch (e) { // Could already be dead - logger.debug( + this.logger.debug( `Got error while SIGSTOPping process ${proc}:`, e ); @@ -372,7 +375,7 @@ class Job { process.kill(proc, 'SIGKILL'); } catch { // Could already be dead and just needs to be waited on - logger.debug( + this.logger.debug( `Got error while SIGKILLing process ${proc}:`, e ); @@ -382,8 +385,8 @@ class Job { } } - logger.debug( - `Finished kill-loop, calling wait_pid to end any zombie processes uuid=${this.uuid}` + this.logger.debug( + `Finished kill-loop, calling wait_pid to end any zombie processes` ); for (const proc of to_wait) { @@ -392,7 +395,7 @@ class Job { wait_pid(proc); } - logger.debug(`Cleaned up processes uuid=${this.uuid}`); + this.logger.debug(`Cleaned up processes`); } async cleanup_filesystem() { @@ -413,7 +416,7 @@ class Job { } } catch (e) { // File was somehow deleted in the time that we read the dir to when we checked the file - logger.warn(`Error removing file ${file_path}: ${e}`); + this.logger.warn(`Error removing file ${file_path}: ${e}`); } } } @@ -422,7 +425,7 @@ class Job { } async cleanup() { - logger.info(`Cleaning up job uuid=${this.uuid}`); + this.logger.info(`Cleaning up job`); this.cleanup_processes(); // Run process janitor, just incase there are any residual processes somehow await this.cleanup_filesystem(); From c2d9367b993e7c1658b9d2c2bfa42f7745504fee Mon Sep 17 00:00:00 2001 From: VlaDexa <52157081+VlaDexa@users.noreply.github.com> Date: Thu, 18 Nov 2021 17:36:10 +0300 Subject: [PATCH 50/72] pkg(rust-1.56.1): Added rust 1.56.1 --- packages/rust/1.56.1/build.sh | 5 +++++ packages/rust/1.56.1/compile | 6 ++++++ packages/rust/1.56.1/environment | 5 +++++ packages/rust/1.56.1/metadata.json | 7 +++++++ packages/rust/1.56.1/run | 4 ++++ packages/rust/1.56.1/test.rs | 3 +++ 6 files changed, 30 insertions(+) create mode 100644 packages/rust/1.56.1/build.sh create mode 100644 packages/rust/1.56.1/compile create mode 100644 packages/rust/1.56.1/environment create mode 100644 packages/rust/1.56.1/metadata.json create mode 100644 packages/rust/1.56.1/run create mode 100644 packages/rust/1.56.1/test.rs diff --git a/packages/rust/1.56.1/build.sh b/packages/rust/1.56.1/build.sh new file mode 100644 index 0000000..6be556f --- /dev/null +++ b/packages/rust/1.56.1/build.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +curl -OL "https://static.rust-lang.org/dist/rust-1.56.1-x86_64-unknown-linux-gnu.tar.gz" +tar xzvf rust-1.56.1-x86_64-unknown-linux-gnu.tar.gz +rm rust-1.56.1-x86_64-unknown-linux-gnu.tar.gz diff --git a/packages/rust/1.56.1/compile b/packages/rust/1.56.1/compile new file mode 100644 index 0000000..201318a --- /dev/null +++ b/packages/rust/1.56.1/compile @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +# https://stackoverflow.com/questions/38041331/rust-compiler-cant-find-crate-for-std +# Rust compiler needs to find the stdlib to link against +rustc -o binary -L ${RUST_INSTALL_LOC}/rustc/lib -L ${RUST_INSTALL_LOC}/rust-std-x86_64-unknown-linux-gnu/lib/rustlib/x86_64-unknown-linux-gnu/lib "$@" +chmod +x binary diff --git a/packages/rust/1.56.1/environment b/packages/rust/1.56.1/environment new file mode 100644 index 0000000..c28b1b7 --- /dev/null +++ b/packages/rust/1.56.1/environment @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +# Put 'export' statements here for environment variables +export PATH=$PWD/rust-1.56.1-x86_64-unknown-linux-gnu/rustc/bin/:$PATH +export RUST_INSTALL_LOC=$PWD/rust-1.56.1-x86_64-unknown-linux-gnu diff --git a/packages/rust/1.56.1/metadata.json b/packages/rust/1.56.1/metadata.json new file mode 100644 index 0000000..1f7319a --- /dev/null +++ b/packages/rust/1.56.1/metadata.json @@ -0,0 +1,7 @@ +{ + "language": "rust", + "version": "1.56.1", + "aliases": [ + "rs" + ] +} diff --git a/packages/rust/1.56.1/run b/packages/rust/1.56.1/run new file mode 100644 index 0000000..d377dd9 --- /dev/null +++ b/packages/rust/1.56.1/run @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +shift +./binary "$@" diff --git a/packages/rust/1.56.1/test.rs b/packages/rust/1.56.1/test.rs new file mode 100644 index 0000000..9561664 --- /dev/null +++ b/packages/rust/1.56.1/test.rs @@ -0,0 +1,3 @@ +fn main() { + println!("OK"); +} From aeddcc7661b7cd2d6e50731b75338679697670bf Mon Sep 17 00:00:00 2001 From: VlaDexa <52157081+VlaDexa@users.noreply.github.com> Date: Sun, 21 Nov 2021 21:39:40 +0300 Subject: [PATCH 51/72] pkg(deno-1.16.2): Added deno 1.16.2 --- packages/deno/1.16.2/build.sh | 5 +++++ packages/deno/1.16.2/environment | 3 +++ packages/deno/1.16.2/metadata.json | 20 ++++++++++++++++++++ packages/deno/1.16.2/run | 3 +++ packages/deno/1.16.2/test.ts | 1 + 5 files changed, 32 insertions(+) create mode 100644 packages/deno/1.16.2/build.sh create mode 100644 packages/deno/1.16.2/environment create mode 100644 packages/deno/1.16.2/metadata.json create mode 100644 packages/deno/1.16.2/run create mode 100644 packages/deno/1.16.2/test.ts diff --git a/packages/deno/1.16.2/build.sh b/packages/deno/1.16.2/build.sh new file mode 100644 index 0000000..ec5c1d0 --- /dev/null +++ b/packages/deno/1.16.2/build.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +curl -OL https://github.com/denoland/deno/releases/download/v1.16.2/deno-x86_64-unknown-linux-gnu.zip +unzip -o deno-x86_64-unknown-linux-gnu.zip +rm deno-x86_64-unknown-linux-gnu.zip diff --git a/packages/deno/1.16.2/environment b/packages/deno/1.16.2/environment new file mode 100644 index 0000000..d5a7803 --- /dev/null +++ b/packages/deno/1.16.2/environment @@ -0,0 +1,3 @@ +#!/bin/bash + +export PATH=$PWD:$PATH diff --git a/packages/deno/1.16.2/metadata.json b/packages/deno/1.16.2/metadata.json new file mode 100644 index 0000000..8043242 --- /dev/null +++ b/packages/deno/1.16.2/metadata.json @@ -0,0 +1,20 @@ +{ + "language": "deno", + "version": "1.16.2", + "dependencies": {}, + "provides": [ + { + "language": "typescript", + "aliases": [ + "deno", + "deno-ts" + ] + }, + { + "language": "javascript", + "aliases": [ + "deno-js" + ] + } + ] +} diff --git a/packages/deno/1.16.2/run b/packages/deno/1.16.2/run new file mode 100644 index 0000000..a863353 --- /dev/null +++ b/packages/deno/1.16.2/run @@ -0,0 +1,3 @@ +#!/bin/bash + +deno run $@ diff --git a/packages/deno/1.16.2/test.ts b/packages/deno/1.16.2/test.ts new file mode 100644 index 0000000..0b555b7 --- /dev/null +++ b/packages/deno/1.16.2/test.ts @@ -0,0 +1 @@ +console.log("OK") From ed14e125e35dda8ae35026a8596632648c143902 Mon Sep 17 00:00:00 2001 From: VlaDexa <52157081+VlaDexa@users.noreply.github.com> Date: Sun, 21 Nov 2021 23:18:39 +0300 Subject: [PATCH 52/72] pkg(deno-1.16.2): Fix the test file --- packages/deno/1.16.2/{test.ts => test.deno.ts} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/deno/1.16.2/{test.ts => test.deno.ts} (100%) diff --git a/packages/deno/1.16.2/test.ts b/packages/deno/1.16.2/test.deno.ts similarity index 100% rename from packages/deno/1.16.2/test.ts rename to packages/deno/1.16.2/test.deno.ts From 81c66946378c30a6e0d5bc664fd5b3cf068bc239 Mon Sep 17 00:00:00 2001 From: VlaDexa <52157081+VlaDexa@users.noreply.github.com> Date: Sun, 21 Nov 2021 23:27:59 +0300 Subject: [PATCH 53/72] pkg(deno-1.16.2): Set the cache directory --- packages/deno/1.16.2/run | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/deno/1.16.2/run b/packages/deno/1.16.2/run index a863353..288919f 100644 --- a/packages/deno/1.16.2/run +++ b/packages/deno/1.16.2/run @@ -1,3 +1,3 @@ #!/bin/bash -deno run $@ +DENO_DIR=$PWD deno run $@ From 142e7912fa03fda7823ebf4b24133881341a7c5d Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Fri, 26 Nov 2021 02:32:28 +1300 Subject: [PATCH 54/72] api(config): respect $PORT as used by heroku if no explicit bind address provided --- api/src/config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/config.js b/api/src/config.js index 85aa691..5324097 100644 --- a/api/src/config.js +++ b/api/src/config.js @@ -68,7 +68,7 @@ const options = [ { key: 'bind_address', desc: 'Address to bind REST API on', - default: '0.0.0.0:2000', + default: `0.0.0.0:${process.env["PORT"] || 2000}`, validators: [], }, { From 0949610b61dfb9e09f1ad239be5815227069fe62 Mon Sep 17 00:00:00 2001 From: Hydrazer Date: Thu, 25 Nov 2021 07:39:41 -0700 Subject: [PATCH 55/72] pkg(befunge93-0.2.0): added befunge93 --- package-lock.json | 2 +- packages/befunge93/0.2.0/build.sh | 15 +++++++++++++++ packages/befunge93/0.2.0/environment | 5 +++++ packages/befunge93/0.2.0/metadata.json | 5 +++++ packages/befunge93/0.2.0/run | 4 ++++ packages/befunge93/0.2.0/test.b93 | 1 + readme.md | 1 + 7 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 packages/befunge93/0.2.0/build.sh create mode 100644 packages/befunge93/0.2.0/environment create mode 100644 packages/befunge93/0.2.0/metadata.json create mode 100644 packages/befunge93/0.2.0/run create mode 100644 packages/befunge93/0.2.0/test.b93 diff --git a/package-lock.json b/package-lock.json index 5c51a1d..2966933 100644 --- a/package-lock.json +++ b/package-lock.json @@ -29,4 +29,4 @@ "dev": true } } -} +} \ No newline at end of file diff --git a/packages/befunge93/0.2.0/build.sh b/packages/befunge93/0.2.0/build.sh new file mode 100644 index 0000000..de6bdbf --- /dev/null +++ b/packages/befunge93/0.2.0/build.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +# source python 2.7 +source ../../python/2.7.18/build.sh + +# clone befunge repo +git clone -q 'https://github.com/programble/befungee' befunge93 + +# go inside befunge93 so we can checkout +cd befunge93 + +# checkout the version 0.2.0 +git checkout tag/v0.2.0 + +cd .. \ No newline at end of file diff --git a/packages/befunge93/0.2.0/environment b/packages/befunge93/0.2.0/environment new file mode 100644 index 0000000..43f2db1 --- /dev/null +++ b/packages/befunge93/0.2.0/environment @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +# Put 'export' statements here for environment variables +export PATH=$PWD/bin:$PATH +export BEFUNGE93_PATH=$PWD/befunge93 \ No newline at end of file diff --git a/packages/befunge93/0.2.0/metadata.json b/packages/befunge93/0.2.0/metadata.json new file mode 100644 index 0000000..16c2643 --- /dev/null +++ b/packages/befunge93/0.2.0/metadata.json @@ -0,0 +1,5 @@ +{ + "language": "befunge93", + "version": "0.2.0", + "aliases": ["b93"] +} diff --git a/packages/befunge93/0.2.0/run b/packages/befunge93/0.2.0/run new file mode 100644 index 0000000..3d95114 --- /dev/null +++ b/packages/befunge93/0.2.0/run @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +# run the befunge program with the file name +python2.7 "$BEFUNGE93_PATH"/befungee.py "$1" \ No newline at end of file diff --git a/packages/befunge93/0.2.0/test.b93 b/packages/befunge93/0.2.0/test.b93 new file mode 100644 index 0000000..4cb5175 --- /dev/null +++ b/packages/befunge93/0.2.0/test.b93 @@ -0,0 +1 @@ +64+"KO">:#,_@ \ No newline at end of file diff --git a/readme.md b/readme.md index 1679568..bc19ab4 100644 --- a/readme.md +++ b/readme.md @@ -323,6 +323,7 @@ Content-Type: application/json `awk`, `bash`, +`befunge93`, `brainfuck`, `c`, `c++`, From 0085bd721736640f53af34658b68802770e81893 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 25 Nov 2021 14:56:13 +0000 Subject: [PATCH 56/72] build(deps): bump mkdocs from 1.1.2 to 1.2.3 in /docs Bumps [mkdocs](https://github.com/mkdocs/mkdocs) from 1.1.2 to 1.2.3. - [Release notes](https://github.com/mkdocs/mkdocs/releases) - [Commits](https://github.com/mkdocs/mkdocs/compare/1.1.2...1.2.3) --- updated-dependencies: - dependency-name: mkdocs dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 53dbf05..33ce51e 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1 +1 @@ -mkdocs==1.1.2 \ No newline at end of file +mkdocs==1.2.3 \ No newline at end of file From 6cee1e8c349646c0eb6d4b95a43b88b9dc5c8d3d Mon Sep 17 00:00:00 2001 From: Hydrazer Date: Sat, 27 Nov 2021 03:59:56 -0700 Subject: [PATCH 57/72] pkg(retina-1.2.0): added retina 1.2.0 --- packages/retina/1.2.0/build.sh | 22 ++++++++++++++++++++++ packages/retina/1.2.0/environment | 4 ++++ packages/retina/1.2.0/metadata.json | 5 +++++ packages/retina/1.2.0/run | 15 +++++++++++++++ packages/retina/1.2.0/test.ret | 1 + readme.md | 1 + 6 files changed, 48 insertions(+) create mode 100644 packages/retina/1.2.0/build.sh create mode 100644 packages/retina/1.2.0/environment create mode 100644 packages/retina/1.2.0/metadata.json create mode 100644 packages/retina/1.2.0/run create mode 100644 packages/retina/1.2.0/test.ret diff --git a/packages/retina/1.2.0/build.sh b/packages/retina/1.2.0/build.sh new file mode 100644 index 0000000..9712458 --- /dev/null +++ b/packages/retina/1.2.0/build.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +# get dotnet 2.2.8 as a dependency for retina +curl "https://download.visualstudio.microsoft.com/download/pr/022d9abf-35f0-4fd5-8d1c-86056df76e89/477f1ebb70f314054129a9f51e9ec8ec/dotnet-sdk-2.2.207-linux-x64.tar.gz" -Lo dotnet.tar.gz +tar xzf dotnet.tar.gz --strip-components=1 +rm dotnet.tar.gz + +export DOTNET_CLI_HOME=$PWD +./dotnet new console -o cache_application + +rm -rf cache_application + +# curl retina version 1.2.0 +curl -L "https://github.com/m-ender/retina/releases/download/v1.2.0/retina-linux-x64.tar.gz" -o retina.tar.xz +tar xf retina.tar.xz --strip-components=1 +rm retina.tar.xz + +# move the libhostfxr.so file to the current directory so we don't have to set DOTNET_ROOT +mv host/fxr/2.2.8/libhostfxr.so libhostfxr.so + +# give execute permissions to retina +chmod +x Retina \ No newline at end of file diff --git a/packages/retina/1.2.0/environment b/packages/retina/1.2.0/environment new file mode 100644 index 0000000..d9b5a4d --- /dev/null +++ b/packages/retina/1.2.0/environment @@ -0,0 +1,4 @@ +#!/bin/bash + +export PATH=$PWD/bin:$PATH +export RETINA_PATH=$PWD \ No newline at end of file diff --git a/packages/retina/1.2.0/metadata.json b/packages/retina/1.2.0/metadata.json new file mode 100644 index 0000000..f632d6b --- /dev/null +++ b/packages/retina/1.2.0/metadata.json @@ -0,0 +1,5 @@ +{ + "language": "retina", + "version": "1.2.0", + "aliases": ["ret"] +} diff --git a/packages/retina/1.2.0/run b/packages/retina/1.2.0/run new file mode 100644 index 0000000..bb64b05 --- /dev/null +++ b/packages/retina/1.2.0/run @@ -0,0 +1,15 @@ +#!/bin/bash + +# retina doesn't take interactive stdin and doesn't take args either + +# save the file for later +file="$1" + +# remove file from $@ +shift + +# join arguments on newline +stdin=`printf "%s\n" "$@"` + +# pass stdin into Retina command with file as argument +echo "$stdin" | "$RETINA_PATH"/Retina "$file" \ No newline at end of file diff --git a/packages/retina/1.2.0/test.ret b/packages/retina/1.2.0/test.ret new file mode 100644 index 0000000..c4b21da --- /dev/null +++ b/packages/retina/1.2.0/test.ret @@ -0,0 +1 @@ +K`OK \ No newline at end of file diff --git a/readme.md b/readme.md index bc19ab4..d36c1a1 100644 --- a/readme.md +++ b/readme.md @@ -381,6 +381,7 @@ Content-Type: application/json `pyth`, `python`, `python2`, +`retina`, `raku`, `rockstar`, `rscript`, From d76aa7527efea92936da7be971653b81fce4dbe4 Mon Sep 17 00:00:00 2001 From: Jonxslays <51417989+Jonxslays@users.noreply.github.com> Date: Sat, 27 Nov 2021 12:26:28 -0700 Subject: [PATCH 58/72] Add piston_rs to readme --- readme.md | 1 + 1 file changed, 1 insertion(+) diff --git a/readme.md b/readme.md index bc19ab4..65b3a6e 100644 --- a/readme.md +++ b/readme.md @@ -85,6 +85,7 @@ The following are approved and endorsed extensions/utilities to the core Piston - [Piston4J](https://github.com/the-codeboy/Piston4J), a Java wrapper for accessing the Piston API. - [Pyston](https://github.com/ffaanngg/pyston), a Python wrapper for accessing the Piston API. - [Go-Piston](https://github.com/milindmadhukar/go-piston), a Golang wrapper for accessing the Piston API. +- [piston_rs](https://github.com/Jonxslays/piston_rs), a Rust wrapper for accessing the Piston API.
From 132a835b0ba77593f0f13b9f6ce581628175c174 Mon Sep 17 00:00:00 2001 From: Hydrazer Date: Sat, 27 Nov 2021 20:00:11 -0700 Subject: [PATCH 59/72] pkg(racket-8.3.0): added racket 8.3.0 --- packages/racket/8.3.0/build.sh | 10 ++++++++++ packages/racket/8.3.0/environment | 5 +++++ packages/racket/8.3.0/metadata.json | 5 +++++ packages/racket/8.3.0/run | 3 +++ packages/racket/8.3.0/test.rkt | 3 +++ readme.md | 3 ++- 6 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 packages/racket/8.3.0/build.sh create mode 100644 packages/racket/8.3.0/environment create mode 100644 packages/racket/8.3.0/metadata.json create mode 100644 packages/racket/8.3.0/run create mode 100644 packages/racket/8.3.0/test.rkt diff --git a/packages/racket/8.3.0/build.sh b/packages/racket/8.3.0/build.sh new file mode 100644 index 0000000..187aef4 --- /dev/null +++ b/packages/racket/8.3.0/build.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +# curl racket 8.3 linux installation shell file +curl -L 'https://download.racket-lang.org/installers/8.3/racket-8.3-x86_64-linux-cs.sh' -o racket.sh + +# provide settings "no" "4" and "" to racket.sh +echo "no +4 +" | sh racket.sh + diff --git a/packages/racket/8.3.0/environment b/packages/racket/8.3.0/environment new file mode 100644 index 0000000..1a30eef --- /dev/null +++ b/packages/racket/8.3.0/environment @@ -0,0 +1,5 @@ +#!/bin/bash + +# Path to racket binary +export PATH=$PWD/bin:$PATH +export RACKET_PATH=$PWD/racket \ No newline at end of file diff --git a/packages/racket/8.3.0/metadata.json b/packages/racket/8.3.0/metadata.json new file mode 100644 index 0000000..14b902c --- /dev/null +++ b/packages/racket/8.3.0/metadata.json @@ -0,0 +1,5 @@ +{ + "language": "racket", + "version": "8.3.0", + "aliases": ["rkt"] +} diff --git a/packages/racket/8.3.0/run b/packages/racket/8.3.0/run new file mode 100644 index 0000000..227bfaa --- /dev/null +++ b/packages/racket/8.3.0/run @@ -0,0 +1,3 @@ +#!/bin/bash + +"$RACKET_PATH"/bin/racket "$@" \ No newline at end of file diff --git a/packages/racket/8.3.0/test.rkt b/packages/racket/8.3.0/test.rkt new file mode 100644 index 0000000..7e0825c --- /dev/null +++ b/packages/racket/8.3.0/test.rkt @@ -0,0 +1,3 @@ +#lang racket + +(display "OK") \ No newline at end of file diff --git a/readme.md b/readme.md index 542c917..ce66d22 100644 --- a/readme.md +++ b/readme.md @@ -382,8 +382,9 @@ Content-Type: application/json `pyth`, `python`, `python2`, -`retina`, +`racket`, `raku`, +`retina`, `rockstar`, `rscript`, `ruby`, From 8d32385b4151207fe6212cc1c21e2af2c09c7e46 Mon Sep 17 00:00:00 2001 From: Hydrazer Date: Sun, 28 Nov 2021 07:03:57 -0700 Subject: [PATCH 60/72] pkg(brachylog-1.0.0): added brachylog --- package-lock.json | 2 +- packages/brachylog/1.0.0/build.sh | 20 ++++++++++++++++++++ packages/brachylog/1.0.0/environment | 5 +++++ packages/brachylog/1.0.0/metadata.json | 5 +++++ packages/brachylog/1.0.0/run | 19 +++++++++++++++++++ packages/brachylog/1.0.0/test.brachylog | 1 + readme.md | 1 + 7 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 packages/brachylog/1.0.0/build.sh create mode 100644 packages/brachylog/1.0.0/environment create mode 100644 packages/brachylog/1.0.0/metadata.json create mode 100644 packages/brachylog/1.0.0/run create mode 100644 packages/brachylog/1.0.0/test.brachylog diff --git a/package-lock.json b/package-lock.json index 2966933..5c51a1d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -29,4 +29,4 @@ "dev": true } } -} \ No newline at end of file +} diff --git a/packages/brachylog/1.0.0/build.sh b/packages/brachylog/1.0.0/build.sh new file mode 100644 index 0000000..8f35ef2 --- /dev/null +++ b/packages/brachylog/1.0.0/build.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +# build prolog 8.2.4 as dependency +source ../../prolog/8.2.4/build.sh + +# curl brachylog 1.0.0 +curl -L "https://github.com/JCumin/Brachylog/archive/refs/tags/v1.0-ascii.tar.gz" -o brachylog.tar.gz +tar xzf brachylog.tar.gz --strip-components=1 +rm brachylog.tar.gz + +# move swi prolog to working directory +cp bin/swipl swipl + +# give execution permission to swipl +chmod +x swipl + +# add some code the branchylog.pl so we don't have to escape backslashes while using the interactive mode +echo ' + +:-feature(argv, [Code, Stdin]), run_from_atom(Code, Stdin, _), halt.' >> prolog_parser/brachylog.pl \ No newline at end of file diff --git a/packages/brachylog/1.0.0/environment b/packages/brachylog/1.0.0/environment new file mode 100644 index 0000000..7012c44 --- /dev/null +++ b/packages/brachylog/1.0.0/environment @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +# Put 'export' statements here for environment variables +export PATH=$PWD/bin:$PATH +export BRACHYLOG_PATH=$PWD \ No newline at end of file diff --git a/packages/brachylog/1.0.0/metadata.json b/packages/brachylog/1.0.0/metadata.json new file mode 100644 index 0000000..d5df839 --- /dev/null +++ b/packages/brachylog/1.0.0/metadata.json @@ -0,0 +1,5 @@ +{ + "language": "brachylog", + "version": "1.0.0", + "aliases": [] +} diff --git a/packages/brachylog/1.0.0/run b/packages/brachylog/1.0.0/run new file mode 100644 index 0000000..16ce3cd --- /dev/null +++ b/packages/brachylog/1.0.0/run @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +# save the file for later +file="$1" + +# remove the file from $@ +shift + +# save stdin as $@ joined by newlines +stdin=`printf "%s\n" "$@"` + +# save code as the contents of $file +code=`cat "$file"` + +# go to the directory where brachylog.pl is so the imports work +cd "$BRACHYLOG_PATH"/prolog_parser + +# run swi prolog with code and stdin +swipl -f brachylog.pl "$code" "$stdin" \ No newline at end of file diff --git a/packages/brachylog/1.0.0/test.brachylog b/packages/brachylog/1.0.0/test.brachylog new file mode 100644 index 0000000..d90d668 --- /dev/null +++ b/packages/brachylog/1.0.0/test.brachylog @@ -0,0 +1 @@ +"OK"w \ No newline at end of file diff --git a/readme.md b/readme.md index ce66d22..135d7ac 100644 --- a/readme.md +++ b/readme.md @@ -325,6 +325,7 @@ Content-Type: application/json `awk`, `bash`, `befunge93`, +`brachylog`, `brainfuck`, `c`, `c++`, From eb37d0ab72baeff063948dbffc567a506e5b6b28 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Mon, 29 Nov 2021 14:00:10 +1300 Subject: [PATCH 61/72] ci: don --- .github/workflows/package-pr.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/package-pr.yaml b/.github/workflows/package-pr.yaml index 8ac17a3..d5bfe78 100644 --- a/.github/workflows/package-pr.yaml +++ b/.github/workflows/package-pr.yaml @@ -4,7 +4,6 @@ on: pull_request: types: - opened - - edited - reopened - synchronize paths: From 20e71f617bb05093ad035e663c5d129d6d7705ec Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sat, 25 Dec 2021 13:39:12 +1300 Subject: [PATCH 62/72] Remove hacktoberfest notice --- readme.md | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/readme.md b/readme.md index 135d7ac..0cc0fa4 100644 --- a/readme.md +++ b/readme.md @@ -41,21 +41,6 @@
-# Notes About Hacktoberfest - -While we are accepting pull requests for Hacktoberfest, we will reject any low-quality PRs. -If we see PR abuse for Hacktoberfest, we will stop providing Hacktoberfest approval for pull requests. - -We are accepting PRs for: - -- Packages - updating package versions, adding new packages -- Documentation updates -- CLI/API improvements - please discuss these with us in the Discord first - -Any queries or concerns, ping @HexF#0015 in the Discord. - -
- # About

From 8d6ae0473354380d8bcb4044881bfa4b543ce7b7 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Tue, 5 Oct 2021 14:57:40 +1300 Subject: [PATCH 63/72] create packages flake --- docker-compose.dev.yaml | 24 ------------------ docker-compose.yaml | 14 ----------- packages/flake.lock | 41 +++++++++++++++++++++++++++++++ packages/flake.nix | 47 ++++++++++++++++++++++++++++++++++++ packages/node-javascript.nix | 17 +++++++++++++ 5 files changed, 105 insertions(+), 38 deletions(-) delete mode 100644 docker-compose.yaml create mode 100644 packages/flake.lock create mode 100644 packages/flake.nix create mode 100644 packages/node-javascript.nix diff --git a/docker-compose.dev.yaml b/docker-compose.dev.yaml index 25d8fe6..e69de29 100644 --- a/docker-compose.dev.yaml +++ b/docker-compose.dev.yaml @@ -1,24 +0,0 @@ -version: '3.2' - -services: - api: - build: api - container_name: piston_api - cap_add: - - CAP_SYS_ADMIN - restart: always - ports: - - 2000:2000 - volumes: - - ./data/piston:/piston - environment: - - PISTON_REPO_URL=http://repo:8000/index - tmpfs: - - /piston/jobs:exec - - repo: # Local testing of packages - build: repo - container_name: piston_repo - command: ['--no-build'] # Don't build anything - volumes: - - .:/piston diff --git a/docker-compose.yaml b/docker-compose.yaml deleted file mode 100644 index 8888f53..0000000 --- a/docker-compose.yaml +++ /dev/null @@ -1,14 +0,0 @@ -version: '3.2' - -services: - api: - image: ghcr.io/engineer-man/piston - container_name: piston_api - restart: always - ports: - - 2000:2000 - volumes: - - ./data/piston:/piston - tmpfs: - - /piston/jobs:exec - - /tmp:exec diff --git a/packages/flake.lock b/packages/flake.lock new file mode 100644 index 0000000..14753b7 --- /dev/null +++ b/packages/flake.lock @@ -0,0 +1,41 @@ +{ + "nodes": { + "flake-utils": { + "locked": { + "lastModified": 1631561581, + "narHash": "sha256-3VQMV5zvxaVLvqqUrNz3iJelLw30mIVSfZmAaauM3dA=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "7e5bf3925f6fbdfaf50a2a7ca0be2879c4261d19", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1633356775, + "narHash": "sha256-UBhRo1qy8xpOGTrjf7r2SFcULkFM+Wn4kchxN1ToDxs=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "38501bec61c1e9447aa4ffc01ba07c40f4515327", + "type": "github" + }, + "original": { + "id": "nixpkgs", + "type": "indirect" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/packages/flake.nix b/packages/flake.nix new file mode 100644 index 0000000..17a60d3 --- /dev/null +++ b/packages/flake.nix @@ -0,0 +1,47 @@ +{ + description = "Piston packages repo"; + inputs.flake-utils.url = "github:numtide/flake-utils"; + + outputs = { self, nixpkgs, flake-utils }: + let + system = "x86_64-linux"; + pkgs = nixpkgs.legacyPackages.${system}; + args = { + inherit pkgs; + piston = { + mkRuntime = { + language, + version, + runtime, + run, + compile? null, + aliases? [] + }: let + packageName = "${runtime}-${language}"; + compileFile = if compile != null then + pkgs.writeShellScript "compile" compile + else null; + runFile = pkgs.writeShellScript "run" run; + metadataFile = pkgs.writeText "metadata.json" (builtins.toJSON { + inherit language version runtime aliases; + }); + in pkgs.runCommandNoCC packageName {} + ( + '' + mkdir -p $out/piston + ln -s ${runFile} $out/piston/run + ln -s ${metadataFile} $out/piston/metadata.json + '' + ( + if compileFile != null then + '' + ln -s ${compileFile} $out/piston/compile + '' else "") + ); + }; + }; + in { + piston = { + "node-javascript" = import ./node-javascript.nix args; + }; + }; +} diff --git a/packages/node-javascript.nix b/packages/node-javascript.nix new file mode 100644 index 0000000..911e3c6 --- /dev/null +++ b/packages/node-javascript.nix @@ -0,0 +1,17 @@ +{pkgs, piston}: +piston.mkRuntime { + language = "javascript"; + version = pkgs.nodejs.version; + runtime = "node"; + + aliases = [ + "node-js" + "node-javascript" + "js" + ]; + + run = '' + ${pkgs.nodejs}/bin/node "$@" + ''; + +} \ No newline at end of file From e2f37b74933a1815a45483a139da8252c2919c4d Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Tue, 5 Oct 2021 14:57:49 +1300 Subject: [PATCH 64/72] install API in container --- api/Dockerfile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/api/Dockerfile b/api/Dockerfile index ec0d2a8..6e0ff79 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -13,11 +13,16 @@ RUN apt-get update && \ libncurses6 libncurses5 libedit-dev libseccomp-dev rename procps python3 \ libreadline-dev libblas-dev liblapack-dev libpcre3-dev libarpack2-dev \ libfftw3-dev libglpk-dev libqhull-dev libqrupdate-dev libsuitesparse-dev \ - libsundials-dev libpcre2-dev && \ + libsundials-dev libpcre2-dev curl sudo && \ rm -rf /var/lib/apt/lists/* RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen +RUN mkdir -m 0755 /nix && chown node /nix +RUN runuser -l node -c 'curl -L https://nixos.org/nix/install | sh -s -- --no-daemon ' +RUN runuser -l node -c 'source ~/.profile; nix-env -iA nixpkgs.nixUnstable' +RUN runuser -l node -c 'mkdir -p /home/node/.config/nix/; echo "experimental-features = nix-command flakes" >> /home/node/.config/nix/nix.conf' + WORKDIR /piston_api COPY ["package.json", "package-lock.json", "./"] RUN npm install From e06b59d82ca08d31a7799795e72d93c5e1de077e Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Tue, 5 Oct 2021 15:18:08 +1300 Subject: [PATCH 65/72] dockerfile to manage mounting of /nix --- api/Dockerfile | 7 +++++-- api/entrypoint.sh | 11 +++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) create mode 100755 api/entrypoint.sh diff --git a/api/Dockerfile b/api/Dockerfile index 6e0ff79..ace28d0 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -18,10 +18,11 @@ RUN apt-get update && \ RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen -RUN mkdir -m 0755 /nix && chown node /nix +RUN mkdir -m 0755 /nix && chown node /nix && touch /nix/piston_detected RUN runuser -l node -c 'curl -L https://nixos.org/nix/install | sh -s -- --no-daemon ' RUN runuser -l node -c 'source ~/.profile; nix-env -iA nixpkgs.nixUnstable' RUN runuser -l node -c 'mkdir -p /home/node/.config/nix/; echo "experimental-features = nix-command flakes" >> /home/node/.config/nix/nix.conf' +RUN cp -r /nix /var/nix WORKDIR /piston_api COPY ["package.json", "package-lock.json", "./"] @@ -30,5 +31,7 @@ COPY ./src ./src RUN make -C ./src/nosocket/ all && make -C ./src/nosocket/ install -CMD [ "node", "src"] +COPY ./entrypoint.sh . + +CMD [ "./entrypoint.sh"] EXPOSE 2000/tcp diff --git a/api/entrypoint.sh b/api/entrypoint.sh new file mode 100755 index 0000000..17966e5 --- /dev/null +++ b/api/entrypoint.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +echo "Starting Piston API" + +echo "Checking presense of nix store" +if [[ ! -f "/nix/piston_detected" ]]; then + echo "Nix Store is not loaded, assuming /nix has been mounted - copying contents" + cp -r /var/nix /nix +fi + +echo "Launching Piston API" +node src From 564da5a7ebbd550e554d9a29f68e3a51d3c1ffb8 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sun, 30 Jan 2022 18:41:24 +1300 Subject: [PATCH 66/72] BREAKING: replace custom build scripts with nix General: - Switched to yarn to better work with nix-based tooling - Switched package system to use nix. This stops double dependencies and slow cloud compile times, while providing more compile/runtime support to the Nix project - Removed container builder in favor of internal container tooling - Package versions no-longer need to be SemVer compliant - Removed "piston package spec" files, replaced with nix-flake based runtimes - Exported nosocket and piston-api as packages within the nix-flake - Removed repo container - Switched docker building to nix-based container outputting - Removed docker compose as this is a single container - Removed package commands from CLI Packages: - Move bash, clojure, cobol, node, python2, python3 to new format - Remainder of packages still need to be moved v2 API: - Removed "version" specifier. To select specific versions, use the v3 api - Removed "/package" endpoints as this doesn't work with the new nix-based system v3 API: - Duplicate of v2 API, except instead of passing in a language name an ID is used intead. --- .gitignore | 1 + api/Dockerfile | 37 - api/default.nix | 28 + api/entrypoint.sh | 7 +- api/package-lock.json | 1048 ------------- api/package.json | 24 +- api/src/api/v2.js | 131 +- api/src/api/v3.js | 236 +++ api/src/config.js | 7 +- api/src/globals.js | 11 - api/src/job.js | 6 +- api/src/package.js | 226 --- api/src/{index.js => pistond.js} | 29 +- api/src/runtime.js | 161 +- api/yarn.lock | 1406 ++++++++++++++++++ builder/.gitignore | 1 - builder/Dockerfile | 2 - builder/build.sh | 61 - cli/commands/ppman.js | 5 - cli/commands/ppman_commands/install.js | 39 - cli/commands/ppman_commands/list.js | 25 - cli/commands/ppman_commands/spec.js | 183 --- cli/commands/ppman_commands/uninstall.js | 41 - dev.pps | 8 - docker-compose.dev.yaml | 0 flake.lock | 26 + flake.nix | 118 ++ {api/src/nosocket => nosocket}/Makefile | 0 nosocket/default.nix | 24 + {api/src/nosocket => nosocket}/nosocket.c | 0 package-lock.json | 32 - package.json | 5 - packages/.gitignore | 8 - packages/CONTRIBUTING.MD | 100 -- packages/Makefile | 25 - packages/README.MD | 7 - packages/bash/5.1.0/build.sh | 21 - packages/bash/5.1.0/environment | 4 - packages/bash/5.1.0/metadata.json | 5 - packages/bash/5.1.0/run | 4 - packages/bash/5.1.0/test.bash.sh | 1 - packages/clojure/1.10.3/build.sh | 43 - packages/clojure/1.10.3/environment | 6 - packages/clojure/1.10.3/metadata.json | 5 - packages/clojure/1.10.3/run | 4 - packages/clojure/1.10.3/test.clj | 5 - packages/cobol/3.1.2/build.sh | 20 - packages/cobol/3.1.2/compile | 4 - packages/cobol/3.1.2/environment | 5 - packages/cobol/3.1.2/metadata.json | 5 - packages/cobol/3.1.2/run | 5 - packages/cobol/3.1.2/test.cob | 8 - packages/flake.lock | 41 - packages/flake.nix | 47 - packages/init | 64 - packages/node-javascript.nix | 17 - packages/node/15.10.0/build.sh | 4 - packages/node/15.10.0/environment | 1 - packages/node/15.10.0/metadata.json | 10 - packages/node/15.10.0/run | 3 - packages/node/15.10.0/test.js | 1 - packages/node/16.3.0/build.sh | 4 - packages/node/16.3.0/environment | 1 - packages/node/16.3.0/metadata.json | 10 - packages/node/16.3.0/run | 3 - packages/node/16.3.0/test.js | 1 - packages/python/2.7.18/build.sh | 23 - packages/python/2.7.18/environment | 1 - packages/python/2.7.18/metadata.json | 5 - packages/python/2.7.18/run | 3 - packages/python/2.7.18/test.py | 1 - packages/python/3.10.0-alpha.7/build.sh | 22 - packages/python/3.10.0-alpha.7/environment | 1 - packages/python/3.10.0-alpha.7/metadata.json | 5 - packages/python/3.10.0-alpha.7/run | 3 - packages/python/3.10.0-alpha.7/test.py | 7 - packages/python/3.5.10/build.sh | 21 - packages/python/3.5.10/environment | 1 - packages/python/3.5.10/metadata.json | 5 - packages/python/3.5.10/run | 3 - packages/python/3.5.10/test.py | 1 - packages/python/3.9.1/build.sh | 21 - packages/python/3.9.1/environment | 1 - packages/python/3.9.1/metadata.json | 5 - packages/python/3.9.1/run | 3 - packages/python/3.9.1/test.py | 1 - packages/python/3.9.4/build.sh | 21 - packages/python/3.9.4/environment | 1 - packages/python/3.9.4/metadata.json | 5 - packages/python/3.9.4/run | 3 - packages/python/3.9.4/test.py | 18 - packages/test.sh | 21 - public.pps | 14 - readme.md | 51 +- repo/.dockerignore | 1 - repo/.gitignore | 2 - repo/Dockerfile | 20 - repo/entrypoint.sh | 59 - repo/mkindex.sh | 23 - result | 1 + runtimes/.scaffold.nix | 65 + runtimes/bash.nix | 28 + runtimes/clojure.nix | 29 + runtimes/cobol-gnu-cobol.nix | 39 + runtimes/default.nix | 8 + runtimes/node-javascript.nix | 28 + runtimes/python2.nix | 25 + runtimes/python3.nix | 26 + runtimes/scaffold.sh | 37 + shell.nix | 8 +- 110 files changed, 2293 insertions(+), 2793 deletions(-) delete mode 100644 api/Dockerfile create mode 100644 api/default.nix delete mode 100644 api/package-lock.json create mode 100644 api/src/api/v3.js delete mode 100644 api/src/package.js rename api/src/{index.js => pistond.js} (74%) mode change 100644 => 100755 create mode 100644 api/yarn.lock delete mode 100644 builder/.gitignore delete mode 100644 builder/Dockerfile delete mode 100755 builder/build.sh delete mode 100644 cli/commands/ppman.js delete mode 100644 cli/commands/ppman_commands/install.js delete mode 100644 cli/commands/ppman_commands/list.js delete mode 100644 cli/commands/ppman_commands/spec.js delete mode 100644 cli/commands/ppman_commands/uninstall.js delete mode 100644 dev.pps delete mode 100644 docker-compose.dev.yaml create mode 100644 flake.lock create mode 100644 flake.nix rename {api/src/nosocket => nosocket}/Makefile (100%) create mode 100644 nosocket/default.nix rename {api/src/nosocket => nosocket}/nosocket.c (100%) delete mode 100644 package-lock.json delete mode 100644 package.json delete mode 100644 packages/.gitignore delete mode 100644 packages/CONTRIBUTING.MD delete mode 100644 packages/Makefile delete mode 100644 packages/README.MD delete mode 100755 packages/bash/5.1.0/build.sh delete mode 100644 packages/bash/5.1.0/environment delete mode 100644 packages/bash/5.1.0/metadata.json delete mode 100644 packages/bash/5.1.0/run delete mode 100644 packages/bash/5.1.0/test.bash.sh delete mode 100755 packages/clojure/1.10.3/build.sh delete mode 100644 packages/clojure/1.10.3/environment delete mode 100644 packages/clojure/1.10.3/metadata.json delete mode 100644 packages/clojure/1.10.3/run delete mode 100644 packages/clojure/1.10.3/test.clj delete mode 100755 packages/cobol/3.1.2/build.sh delete mode 100755 packages/cobol/3.1.2/compile delete mode 100644 packages/cobol/3.1.2/environment delete mode 100644 packages/cobol/3.1.2/metadata.json delete mode 100755 packages/cobol/3.1.2/run delete mode 100644 packages/cobol/3.1.2/test.cob delete mode 100644 packages/flake.lock delete mode 100644 packages/flake.nix delete mode 100755 packages/init delete mode 100644 packages/node-javascript.nix delete mode 100755 packages/node/15.10.0/build.sh delete mode 100644 packages/node/15.10.0/environment delete mode 100644 packages/node/15.10.0/metadata.json delete mode 100644 packages/node/15.10.0/run delete mode 100644 packages/node/15.10.0/test.js delete mode 100755 packages/node/16.3.0/build.sh delete mode 100644 packages/node/16.3.0/environment delete mode 100644 packages/node/16.3.0/metadata.json delete mode 100644 packages/node/16.3.0/run delete mode 100644 packages/node/16.3.0/test.js delete mode 100755 packages/python/2.7.18/build.sh delete mode 100644 packages/python/2.7.18/environment delete mode 100644 packages/python/2.7.18/metadata.json delete mode 100644 packages/python/2.7.18/run delete mode 100644 packages/python/2.7.18/test.py delete mode 100755 packages/python/3.10.0-alpha.7/build.sh delete mode 100644 packages/python/3.10.0-alpha.7/environment delete mode 100644 packages/python/3.10.0-alpha.7/metadata.json delete mode 100644 packages/python/3.10.0-alpha.7/run delete mode 100644 packages/python/3.10.0-alpha.7/test.py delete mode 100755 packages/python/3.5.10/build.sh delete mode 100644 packages/python/3.5.10/environment delete mode 100644 packages/python/3.5.10/metadata.json delete mode 100644 packages/python/3.5.10/run delete mode 100644 packages/python/3.5.10/test.py delete mode 100755 packages/python/3.9.1/build.sh delete mode 100644 packages/python/3.9.1/environment delete mode 100644 packages/python/3.9.1/metadata.json delete mode 100644 packages/python/3.9.1/run delete mode 100644 packages/python/3.9.1/test.py delete mode 100755 packages/python/3.9.4/build.sh delete mode 100644 packages/python/3.9.4/environment delete mode 100644 packages/python/3.9.4/metadata.json delete mode 100644 packages/python/3.9.4/run delete mode 100644 packages/python/3.9.4/test.py delete mode 100755 packages/test.sh delete mode 100755 public.pps delete mode 100644 repo/.dockerignore delete mode 100644 repo/.gitignore delete mode 100644 repo/Dockerfile delete mode 100755 repo/entrypoint.sh delete mode 100755 repo/mkindex.sh create mode 120000 result create mode 100644 runtimes/.scaffold.nix create mode 100644 runtimes/bash.nix create mode 100644 runtimes/clojure.nix create mode 100644 runtimes/cobol-gnu-cobol.nix create mode 100644 runtimes/default.nix create mode 100644 runtimes/node-javascript.nix create mode 100644 runtimes/python2.nix create mode 100644 runtimes/python3.nix create mode 100755 runtimes/scaffold.sh diff --git a/.gitignore b/.gitignore index eb53d81..6d3d2cb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ data/ .piston_env node_modules +result diff --git a/api/Dockerfile b/api/Dockerfile deleted file mode 100644 index ace28d0..0000000 --- a/api/Dockerfile +++ /dev/null @@ -1,37 +0,0 @@ -FROM node:15.10.0-buster-slim - -ENV DEBIAN_FRONTEND=noninteractive - -RUN dpkg-reconfigure -p critical dash -RUN for i in $(seq 1001 1500); do \ - groupadd -g $i runner$i && \ - useradd -M runner$i -g $i -u $i ; \ - done -RUN apt-get update && \ - apt-get install -y libxml2 gnupg tar coreutils util-linux libc6-dev \ - binutils build-essential locales libpcre3-dev libevent-dev libgmp3-dev \ - libncurses6 libncurses5 libedit-dev libseccomp-dev rename procps python3 \ - libreadline-dev libblas-dev liblapack-dev libpcre3-dev libarpack2-dev \ - libfftw3-dev libglpk-dev libqhull-dev libqrupdate-dev libsuitesparse-dev \ - libsundials-dev libpcre2-dev curl sudo && \ - rm -rf /var/lib/apt/lists/* - -RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen - -RUN mkdir -m 0755 /nix && chown node /nix && touch /nix/piston_detected -RUN runuser -l node -c 'curl -L https://nixos.org/nix/install | sh -s -- --no-daemon ' -RUN runuser -l node -c 'source ~/.profile; nix-env -iA nixpkgs.nixUnstable' -RUN runuser -l node -c 'mkdir -p /home/node/.config/nix/; echo "experimental-features = nix-command flakes" >> /home/node/.config/nix/nix.conf' -RUN cp -r /nix /var/nix - -WORKDIR /piston_api -COPY ["package.json", "package-lock.json", "./"] -RUN npm install -COPY ./src ./src - -RUN make -C ./src/nosocket/ all && make -C ./src/nosocket/ install - -COPY ./entrypoint.sh . - -CMD [ "./entrypoint.sh"] -EXPOSE 2000/tcp diff --git a/api/default.nix b/api/default.nix new file mode 100644 index 0000000..cc6cb70 --- /dev/null +++ b/api/default.nix @@ -0,0 +1,28 @@ +{pkgs, ...}: +with pkgs; { + package = mkYarnPackage { + name = "piston"; + src = ./.; + + yarnPreBuild = '' + mkdir -p $HOME/.node-gyp/${nodejs.version} + echo 9 > $HOME/.node-gyp/${nodejs.version}/installVersion + ln -sfv ${nodejs}/include $HOME/.node-gyp/${nodejs.version} + export npm_config_nodedir=${nodejs} + ''; + + pkgConfig = { + waitpid = { + buildInputs = [ + gcc + gnumake + python3 + ]; + + postInstall = '' + yarn --offline run install + ''; + }; + }; + }; +} \ No newline at end of file diff --git a/api/entrypoint.sh b/api/entrypoint.sh index 17966e5..aedcedd 100755 --- a/api/entrypoint.sh +++ b/api/entrypoint.sh @@ -4,8 +4,11 @@ echo "Starting Piston API" echo "Checking presense of nix store" if [[ ! -f "/nix/piston_detected" ]]; then echo "Nix Store is not loaded, assuming /nix has been mounted - copying contents" - cp -r /var/nix /nix + cp -rp /var/nix/* /nix fi +echo "Adding nix to env" +. ~/.profile + echo "Launching Piston API" -node src +node src \ No newline at end of file diff --git a/api/package-lock.json b/api/package-lock.json deleted file mode 100644 index 2b34fc4..0000000 --- a/api/package-lock.json +++ /dev/null @@ -1,1048 +0,0 @@ -{ - "name": "piston-api", - "version": "3.1.0", - "lockfileVersion": 2, - "requires": true, - "packages": { - "": { - "name": "piston-api", - "version": "3.1.0", - "license": "MIT", - "dependencies": { - "body-parser": "^1.19.0", - "chownr": "^2.0.0", - "express": "^4.17.1", - "express-ws": "^5.0.2", - "is-docker": "^2.1.1", - "logplease": "^1.2.15", - "nocamel": "HexF/nocamel#patch-1", - "node-fetch": "^2.6.1", - "semver": "^7.3.4", - "uuid": "^8.3.2", - "waitpid": "git+https://github.com/HexF/node-waitpid.git" - } - }, - "node_modules/accepts": { - "version": "1.3.7", - "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.7.tgz", - "integrity": "sha512-Il80Qs2WjYlJIBNzNkK6KYqlVMTbZLXgHx2oT0pU/fjRHyEp+PEfEPY0R3WCwAGVOtauxh1hOxNgIf5bv7dQpA==", - "dependencies": { - "mime-types": "~2.1.24", - "negotiator": "0.6.2" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/array-flatten": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", - "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" - }, - "node_modules/body-parser": { - "version": "1.19.0", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.0.tgz", - "integrity": "sha512-dhEPs72UPbDnAQJ9ZKMNTP6ptJaionhP5cBb541nXPlW60Jepo9RV/a4fX4XWW9CuFNK22krhrj1+rgzifNCsw==", - "dependencies": { - "bytes": "3.1.0", - "content-type": "~1.0.4", - "debug": "2.6.9", - "depd": "~1.1.2", - "http-errors": "1.7.2", - "iconv-lite": "0.4.24", - "on-finished": "~2.3.0", - "qs": "6.7.0", - "raw-body": "2.4.0", - "type-is": "~1.6.17" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/bytes": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz", - "integrity": "sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg==", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/chownr": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz", - "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==", - "engines": { - "node": ">=10" - } - }, - "node_modules/content-disposition": { - "version": "0.5.3", - "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.3.tgz", - "integrity": "sha512-ExO0774ikEObIAEV9kDo50o+79VCUdEB6n6lzKgGwupcVeRlhrj3qGAfwq8G6uBJjkqLrhT0qEYFcWng8z1z0g==", - "dependencies": { - "safe-buffer": "5.1.2" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/content-type": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", - "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/cookie": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.0.tgz", - "integrity": "sha512-+Hp8fLp57wnUSt0tY0tHEXh4voZRDnoIrZPqlo3DPiI4y9lwg/jqx+1Om94/W6ZaPDOUbnjOt/99w66zk+l1Xg==", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/cookie-signature": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", - "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw=" - }, - "node_modules/debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "dependencies": { - "ms": "2.0.0" - } - }, - "node_modules/depd": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", - "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak=", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/destroy": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", - "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=" - }, - "node_modules/ee-first": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", - "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" - }, - "node_modules/encodeurl": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", - "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/escape-html": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", - "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=" - }, - "node_modules/etag": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", - "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/express": { - "version": "4.17.1", - "resolved": "https://registry.npmjs.org/express/-/express-4.17.1.tgz", - "integrity": "sha512-mHJ9O79RqluphRrcw2X/GTh3k9tVv8YcoyY4Kkh4WDMUYKRZUq0h1o0w2rrrxBqM7VoeUVqgb27xlEMXTnYt4g==", - "dependencies": { - "accepts": "~1.3.7", - "array-flatten": "1.1.1", - "body-parser": "1.19.0", - "content-disposition": "0.5.3", - "content-type": "~1.0.4", - "cookie": "0.4.0", - "cookie-signature": "1.0.6", - "debug": "2.6.9", - "depd": "~1.1.2", - "encodeurl": "~1.0.2", - "escape-html": "~1.0.3", - "etag": "~1.8.1", - "finalhandler": "~1.1.2", - "fresh": "0.5.2", - "merge-descriptors": "1.0.1", - "methods": "~1.1.2", - "on-finished": "~2.3.0", - "parseurl": "~1.3.3", - "path-to-regexp": "0.1.7", - "proxy-addr": "~2.0.5", - "qs": "6.7.0", - "range-parser": "~1.2.1", - "safe-buffer": "5.1.2", - "send": "0.17.1", - "serve-static": "1.14.1", - "setprototypeof": "1.1.1", - "statuses": "~1.5.0", - "type-is": "~1.6.18", - "utils-merge": "1.0.1", - "vary": "~1.1.2" - }, - "engines": { - "node": ">= 0.10.0" - } - }, - "node_modules/express-ws": { - "version": "5.0.2", - "resolved": "https://registry.npmjs.org/express-ws/-/express-ws-5.0.2.tgz", - "integrity": "sha512-0uvmuk61O9HXgLhGl3QhNSEtRsQevtmbL94/eILaliEADZBHZOQUAiHFrGPrgsjikohyrmSG5g+sCfASTt0lkQ==", - "dependencies": { - "ws": "^7.4.6" - }, - "engines": { - "node": ">=4.5.0" - }, - "peerDependencies": { - "express": "^4.0.0 || ^5.0.0-alpha.1" - } - }, - "node_modules/finalhandler": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.2.tgz", - "integrity": "sha512-aAWcW57uxVNrQZqFXjITpW3sIUQmHGG3qSb9mUah9MgMC4NeWhNOlNjXEYq3HjRAvL6arUviZGGJsBg6z0zsWA==", - "dependencies": { - "debug": "2.6.9", - "encodeurl": "~1.0.2", - "escape-html": "~1.0.3", - "on-finished": "~2.3.0", - "parseurl": "~1.3.3", - "statuses": "~1.5.0", - "unpipe": "~1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/forwarded": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz", - "integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ=", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/fresh": { - "version": "0.5.2", - "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", - "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/http-errors": { - "version": "1.7.2", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz", - "integrity": "sha512-uUQBt3H/cSIVfch6i1EuPNy/YsRSOUBXTVfZ+yR7Zjez3qjBz6i9+i4zjNaoqcoFVI4lQJ5plg63TvGfRSDCRg==", - "dependencies": { - "depd": "~1.1.2", - "inherits": "2.0.3", - "setprototypeof": "1.1.1", - "statuses": ">= 1.5.0 < 2", - "toidentifier": "1.0.0" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/iconv-lite": { - "version": "0.4.24", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", - "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", - "dependencies": { - "safer-buffer": ">= 2.1.2 < 3" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/inherits": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", - "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" - }, - "node_modules/ipaddr.js": { - "version": "1.9.1", - "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", - "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", - "engines": { - "node": ">= 0.10" - } - }, - "node_modules/is-docker": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-2.1.1.tgz", - "integrity": "sha512-ZOoqiXfEwtGknTiuDEy8pN2CfE3TxMHprvNer1mXiqwkOT77Rw3YVrUQ52EqAOU3QAWDQ+bQdx7HJzrv7LS2Hw==", - "bin": { - "is-docker": "cli.js" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/logplease": { - "version": "1.2.15", - "resolved": "https://registry.npmjs.org/logplease/-/logplease-1.2.15.tgz", - "integrity": "sha512-jLlHnlsPSJjpwUfcNyUxXCl33AYg2cHhIf9QhGL2T4iPT0XPB+xP1LRKFPgIg1M/sg9kAJvy94w9CzBNrfnstA==" - }, - "node_modules/lru-cache": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", - "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "dependencies": { - "yallist": "^4.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/media-typer": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", - "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/merge-descriptors": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", - "integrity": "sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E=" - }, - "node_modules/methods": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", - "integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4=", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/mime": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", - "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", - "bin": { - "mime": "cli.js" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/mime-db": { - "version": "1.46.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.46.0.tgz", - "integrity": "sha512-svXaP8UQRZ5K7or+ZmfNhg2xX3yKDMUzqadsSqi4NCH/KomcH75MAMYAGVlvXn4+b/xOPhS3I2uHKRUzvjY7BQ==", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/mime-types": { - "version": "2.1.29", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.29.tgz", - "integrity": "sha512-Y/jMt/S5sR9OaqteJtslsFZKWOIIqMACsJSiHghlCAyhf7jfVYjKBmLiX8OgpWeW+fjJ2b+Az69aPFPkUOY6xQ==", - "dependencies": { - "mime-db": "1.46.0" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" - }, - "node_modules/negotiator": { - "version": "0.6.2", - "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz", - "integrity": "sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw==", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/nocamel": { - "resolved": "git+ssh://git@github.com/HexF/nocamel.git#89a5bfbbd07c72c302d968b967d0f4fe54846544" - }, - "node_modules/node-fetch": { - "version": "2.6.1", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", - "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==", - "engines": { - "node": "4.x || >=6.0.0" - } - }, - "node_modules/on-finished": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", - "integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=", - "dependencies": { - "ee-first": "1.1.1" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/parseurl": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", - "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/path-to-regexp": { - "version": "0.1.7", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", - "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" - }, - "node_modules/proxy-addr": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.6.tgz", - "integrity": "sha512-dh/frvCBVmSsDYzw6n926jv974gddhkFPfiN8hPOi30Wax25QZyZEGveluCgliBnqmuM+UJmBErbAUFIoDbjOw==", - "dependencies": { - "forwarded": "~0.1.2", - "ipaddr.js": "1.9.1" - }, - "engines": { - "node": ">= 0.10" - } - }, - "node_modules/qs": { - "version": "6.7.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz", - "integrity": "sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ==", - "engines": { - "node": ">=0.6" - } - }, - "node_modules/range-parser": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", - "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/raw-body": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.0.tgz", - "integrity": "sha512-4Oz8DUIwdvoa5qMJelxipzi/iJIi40O5cGV1wNYp5hvZP8ZN0T+jiNkL0QepXs+EsQ9XJ8ipEDoiH70ySUJP3Q==", - "dependencies": { - "bytes": "3.1.0", - "http-errors": "1.7.2", - "iconv-lite": "0.4.24", - "unpipe": "1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" - }, - "node_modules/safer-buffer": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" - }, - "node_modules/semver": { - "version": "7.3.5", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz", - "integrity": "sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==", - "dependencies": { - "lru-cache": "^6.0.0" - }, - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/send": { - "version": "0.17.1", - "resolved": "https://registry.npmjs.org/send/-/send-0.17.1.tgz", - "integrity": "sha512-BsVKsiGcQMFwT8UxypobUKyv7irCNRHk1T0G680vk88yf6LBByGcZJOTJCrTP2xVN6yI+XjPJcNuE3V4fT9sAg==", - "dependencies": { - "debug": "2.6.9", - "depd": "~1.1.2", - "destroy": "~1.0.4", - "encodeurl": "~1.0.2", - "escape-html": "~1.0.3", - "etag": "~1.8.1", - "fresh": "0.5.2", - "http-errors": "~1.7.2", - "mime": "1.6.0", - "ms": "2.1.1", - "on-finished": "~2.3.0", - "range-parser": "~1.2.1", - "statuses": "~1.5.0" - }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/send/node_modules/ms": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", - "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==" - }, - "node_modules/serve-static": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.1.tgz", - "integrity": "sha512-JMrvUwE54emCYWlTI+hGrGv5I8dEwmco/00EvkzIIsR7MqrHonbD9pO2MOfFnpFntl7ecpZs+3mW+XbQZu9QCg==", - "dependencies": { - "encodeurl": "~1.0.2", - "escape-html": "~1.0.3", - "parseurl": "~1.3.3", - "send": "0.17.1" - }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/setprototypeof": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.1.tgz", - "integrity": "sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw==" - }, - "node_modules/statuses": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", - "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/toidentifier": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz", - "integrity": "sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw==", - "engines": { - "node": ">=0.6" - } - }, - "node_modules/type-is": { - "version": "1.6.18", - "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", - "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", - "dependencies": { - "media-typer": "0.3.0", - "mime-types": "~2.1.24" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/unpipe": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", - "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/utils-merge": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", - "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=", - "engines": { - "node": ">= 0.4.0" - } - }, - "node_modules/uuid": { - "version": "8.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", - "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", - "bin": { - "uuid": "dist/bin/uuid" - } - }, - "node_modules/vary": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", - "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/waitpid": { - "resolved": "git+ssh://git@github.com/HexF/node-waitpid.git#a08d116a5d993a747624fe72ff890167be8c34aa", - "hasInstallScript": true - }, - "node_modules/ws": { - "version": "7.5.3", - "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.3.tgz", - "integrity": "sha512-kQ/dHIzuLrS6Je9+uv81ueZomEwH0qVYstcAQ4/Z93K8zeko9gtAbttJWzoC5ukqXY1PpoouV3+VSOqEAFt5wg==", - "engines": { - "node": ">=8.3.0" - }, - "peerDependencies": { - "bufferutil": "^4.0.1", - "utf-8-validate": "^5.0.2" - }, - "peerDependenciesMeta": { - "bufferutil": { - "optional": true - }, - "utf-8-validate": { - "optional": true - } - } - }, - "node_modules/yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" - } - }, - "dependencies": { - "accepts": { - "version": "1.3.7", - "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.7.tgz", - "integrity": "sha512-Il80Qs2WjYlJIBNzNkK6KYqlVMTbZLXgHx2oT0pU/fjRHyEp+PEfEPY0R3WCwAGVOtauxh1hOxNgIf5bv7dQpA==", - "requires": { - "mime-types": "~2.1.24", - "negotiator": "0.6.2" - } - }, - "array-flatten": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", - "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" - }, - "body-parser": { - "version": "1.19.0", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.0.tgz", - "integrity": "sha512-dhEPs72UPbDnAQJ9ZKMNTP6ptJaionhP5cBb541nXPlW60Jepo9RV/a4fX4XWW9CuFNK22krhrj1+rgzifNCsw==", - "requires": { - "bytes": "3.1.0", - "content-type": "~1.0.4", - "debug": "2.6.9", - "depd": "~1.1.2", - "http-errors": "1.7.2", - "iconv-lite": "0.4.24", - "on-finished": "~2.3.0", - "qs": "6.7.0", - "raw-body": "2.4.0", - "type-is": "~1.6.17" - } - }, - "bytes": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz", - "integrity": "sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg==" - }, - "chownr": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz", - "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==" - }, - "content-disposition": { - "version": "0.5.3", - "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.3.tgz", - "integrity": "sha512-ExO0774ikEObIAEV9kDo50o+79VCUdEB6n6lzKgGwupcVeRlhrj3qGAfwq8G6uBJjkqLrhT0qEYFcWng8z1z0g==", - "requires": { - "safe-buffer": "5.1.2" - } - }, - "content-type": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", - "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==" - }, - "cookie": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.0.tgz", - "integrity": "sha512-+Hp8fLp57wnUSt0tY0tHEXh4voZRDnoIrZPqlo3DPiI4y9lwg/jqx+1Om94/W6ZaPDOUbnjOt/99w66zk+l1Xg==" - }, - "cookie-signature": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", - "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw=" - }, - "debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "requires": { - "ms": "2.0.0" - } - }, - "depd": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", - "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak=" - }, - "destroy": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", - "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=" - }, - "ee-first": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", - "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" - }, - "encodeurl": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", - "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=" - }, - "escape-html": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", - "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=" - }, - "etag": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", - "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=" - }, - "express": { - "version": "4.17.1", - "resolved": "https://registry.npmjs.org/express/-/express-4.17.1.tgz", - "integrity": "sha512-mHJ9O79RqluphRrcw2X/GTh3k9tVv8YcoyY4Kkh4WDMUYKRZUq0h1o0w2rrrxBqM7VoeUVqgb27xlEMXTnYt4g==", - "requires": { - "accepts": "~1.3.7", - "array-flatten": "1.1.1", - "body-parser": "1.19.0", - "content-disposition": "0.5.3", - "content-type": "~1.0.4", - "cookie": "0.4.0", - "cookie-signature": "1.0.6", - "debug": "2.6.9", - "depd": "~1.1.2", - "encodeurl": "~1.0.2", - "escape-html": "~1.0.3", - "etag": "~1.8.1", - "finalhandler": "~1.1.2", - "fresh": "0.5.2", - "merge-descriptors": "1.0.1", - "methods": "~1.1.2", - "on-finished": "~2.3.0", - "parseurl": "~1.3.3", - "path-to-regexp": "0.1.7", - "proxy-addr": "~2.0.5", - "qs": "6.7.0", - "range-parser": "~1.2.1", - "safe-buffer": "5.1.2", - "send": "0.17.1", - "serve-static": "1.14.1", - "setprototypeof": "1.1.1", - "statuses": "~1.5.0", - "type-is": "~1.6.18", - "utils-merge": "1.0.1", - "vary": "~1.1.2" - } - }, - "express-ws": { - "version": "5.0.2", - "resolved": "https://registry.npmjs.org/express-ws/-/express-ws-5.0.2.tgz", - "integrity": "sha512-0uvmuk61O9HXgLhGl3QhNSEtRsQevtmbL94/eILaliEADZBHZOQUAiHFrGPrgsjikohyrmSG5g+sCfASTt0lkQ==", - "requires": { - "ws": "^7.4.6" - } - }, - "finalhandler": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.2.tgz", - "integrity": "sha512-aAWcW57uxVNrQZqFXjITpW3sIUQmHGG3qSb9mUah9MgMC4NeWhNOlNjXEYq3HjRAvL6arUviZGGJsBg6z0zsWA==", - "requires": { - "debug": "2.6.9", - "encodeurl": "~1.0.2", - "escape-html": "~1.0.3", - "on-finished": "~2.3.0", - "parseurl": "~1.3.3", - "statuses": "~1.5.0", - "unpipe": "~1.0.0" - } - }, - "forwarded": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz", - "integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ=" - }, - "fresh": { - "version": "0.5.2", - "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", - "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=" - }, - "http-errors": { - "version": "1.7.2", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz", - "integrity": "sha512-uUQBt3H/cSIVfch6i1EuPNy/YsRSOUBXTVfZ+yR7Zjez3qjBz6i9+i4zjNaoqcoFVI4lQJ5plg63TvGfRSDCRg==", - "requires": { - "depd": "~1.1.2", - "inherits": "2.0.3", - "setprototypeof": "1.1.1", - "statuses": ">= 1.5.0 < 2", - "toidentifier": "1.0.0" - } - }, - "iconv-lite": { - "version": "0.4.24", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", - "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", - "requires": { - "safer-buffer": ">= 2.1.2 < 3" - } - }, - "inherits": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", - "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" - }, - "ipaddr.js": { - "version": "1.9.1", - "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", - "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==" - }, - "is-docker": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-2.1.1.tgz", - "integrity": "sha512-ZOoqiXfEwtGknTiuDEy8pN2CfE3TxMHprvNer1mXiqwkOT77Rw3YVrUQ52EqAOU3QAWDQ+bQdx7HJzrv7LS2Hw==" - }, - "logplease": { - "version": "1.2.15", - "resolved": "https://registry.npmjs.org/logplease/-/logplease-1.2.15.tgz", - "integrity": "sha512-jLlHnlsPSJjpwUfcNyUxXCl33AYg2cHhIf9QhGL2T4iPT0XPB+xP1LRKFPgIg1M/sg9kAJvy94w9CzBNrfnstA==" - }, - "lru-cache": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", - "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "requires": { - "yallist": "^4.0.0" - } - }, - "media-typer": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", - "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=" - }, - "merge-descriptors": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", - "integrity": "sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E=" - }, - "methods": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", - "integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4=" - }, - "mime": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", - "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==" - }, - "mime-db": { - "version": "1.46.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.46.0.tgz", - "integrity": "sha512-svXaP8UQRZ5K7or+ZmfNhg2xX3yKDMUzqadsSqi4NCH/KomcH75MAMYAGVlvXn4+b/xOPhS3I2uHKRUzvjY7BQ==" - }, - "mime-types": { - "version": "2.1.29", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.29.tgz", - "integrity": "sha512-Y/jMt/S5sR9OaqteJtslsFZKWOIIqMACsJSiHghlCAyhf7jfVYjKBmLiX8OgpWeW+fjJ2b+Az69aPFPkUOY6xQ==", - "requires": { - "mime-db": "1.46.0" - } - }, - "ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" - }, - "negotiator": { - "version": "0.6.2", - "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz", - "integrity": "sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw==" - }, - "nocamel": { - "version": "git+ssh://git@github.com/HexF/nocamel.git#89a5bfbbd07c72c302d968b967d0f4fe54846544", - "from": "nocamel@HexF/nocamel#patch-1" - }, - "node-fetch": { - "version": "2.6.1", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", - "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==" - }, - "on-finished": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", - "integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=", - "requires": { - "ee-first": "1.1.1" - } - }, - "parseurl": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", - "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==" - }, - "path-to-regexp": { - "version": "0.1.7", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", - "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" - }, - "proxy-addr": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.6.tgz", - "integrity": "sha512-dh/frvCBVmSsDYzw6n926jv974gddhkFPfiN8hPOi30Wax25QZyZEGveluCgliBnqmuM+UJmBErbAUFIoDbjOw==", - "requires": { - "forwarded": "~0.1.2", - "ipaddr.js": "1.9.1" - } - }, - "qs": { - "version": "6.7.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz", - "integrity": "sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ==" - }, - "range-parser": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", - "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==" - }, - "raw-body": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.0.tgz", - "integrity": "sha512-4Oz8DUIwdvoa5qMJelxipzi/iJIi40O5cGV1wNYp5hvZP8ZN0T+jiNkL0QepXs+EsQ9XJ8ipEDoiH70ySUJP3Q==", - "requires": { - "bytes": "3.1.0", - "http-errors": "1.7.2", - "iconv-lite": "0.4.24", - "unpipe": "1.0.0" - } - }, - "safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" - }, - "safer-buffer": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" - }, - "semver": { - "version": "7.3.5", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz", - "integrity": "sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==", - "requires": { - "lru-cache": "^6.0.0" - } - }, - "send": { - "version": "0.17.1", - "resolved": "https://registry.npmjs.org/send/-/send-0.17.1.tgz", - "integrity": "sha512-BsVKsiGcQMFwT8UxypobUKyv7irCNRHk1T0G680vk88yf6LBByGcZJOTJCrTP2xVN6yI+XjPJcNuE3V4fT9sAg==", - "requires": { - "debug": "2.6.9", - "depd": "~1.1.2", - "destroy": "~1.0.4", - "encodeurl": "~1.0.2", - "escape-html": "~1.0.3", - "etag": "~1.8.1", - "fresh": "0.5.2", - "http-errors": "~1.7.2", - "mime": "1.6.0", - "ms": "2.1.1", - "on-finished": "~2.3.0", - "range-parser": "~1.2.1", - "statuses": "~1.5.0" - }, - "dependencies": { - "ms": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", - "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==" - } - } - }, - "serve-static": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.1.tgz", - "integrity": "sha512-JMrvUwE54emCYWlTI+hGrGv5I8dEwmco/00EvkzIIsR7MqrHonbD9pO2MOfFnpFntl7ecpZs+3mW+XbQZu9QCg==", - "requires": { - "encodeurl": "~1.0.2", - "escape-html": "~1.0.3", - "parseurl": "~1.3.3", - "send": "0.17.1" - } - }, - "setprototypeof": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.1.tgz", - "integrity": "sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw==" - }, - "statuses": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", - "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=" - }, - "toidentifier": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz", - "integrity": "sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw==" - }, - "type-is": { - "version": "1.6.18", - "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", - "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", - "requires": { - "media-typer": "0.3.0", - "mime-types": "~2.1.24" - } - }, - "unpipe": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", - "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=" - }, - "utils-merge": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", - "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=" - }, - "uuid": { - "version": "8.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", - "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==" - }, - "vary": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", - "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=" - }, - "waitpid": { - "version": "git+ssh://git@github.com/HexF/node-waitpid.git#a08d116a5d993a747624fe72ff890167be8c34aa", - "from": "waitpid@git+https://github.com/HexF/node-waitpid.git" - }, - "ws": { - "version": "7.5.3", - "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.3.tgz", - "integrity": "sha512-kQ/dHIzuLrS6Je9+uv81ueZomEwH0qVYstcAQ4/Z93K8zeko9gtAbttJWzoC5ukqXY1PpoouV3+VSOqEAFt5wg==", - "requires": {} - }, - "yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" - } - } -} diff --git a/api/package.json b/api/package.json index fb107a5..49e55db 100644 --- a/api/package.json +++ b/api/package.json @@ -1,20 +1,28 @@ { "name": "piston-api", - "version": "3.1.0", + "version": "4.0.0", "description": "API for piston - a high performance code execution engine", - "main": "src/index.js", + "main": "src/pistond.js", "dependencies": { "body-parser": "^1.19.0", - "chownr": "^2.0.0", "express": "^4.17.1", "express-ws": "^5.0.2", - "is-docker": "^2.1.1", "logplease": "^1.2.15", - "nocamel": "HexF/nocamel#patch-1", "node-fetch": "^2.6.1", - "semver": "^7.3.4", "uuid": "^8.3.2", - "waitpid": "git+https://github.com/HexF/node-waitpid.git" + "nocamel": "git://github.com/HexF/nocamel.git#patch-1", + "waitpid": "git://github.com/HexF/node-waitpid.git" }, - "license": "MIT" + "license": "MIT", + "scripts": { + "lint": "prettier . --write", + "prepack": "yarn2nix > yarn.nix" + }, + "devDependencies": { + "node2nix": "^1.6.0", + "prettier": "2.2.1" + }, + "bin": { + "pistond": "./src/pistond.js" + } } diff --git a/api/src/api/v2.js b/api/src/api/v2.js index 4f0c3e2..86294b4 100644 --- a/api/src/api/v2.js +++ b/api/src/api/v2.js @@ -5,8 +5,7 @@ const events = require('events'); const runtime = require('../runtime'); const { Job } = require('../job'); -const package = require('../package'); -const logger = require('logplease').create('api/v2'); +const logger = require('logplease').create('api/v3'); const SIGNALS = [ 'SIGABRT', @@ -53,7 +52,6 @@ const SIGNALS = [ function get_job(body) { let { language, - version, args, stdin, files, @@ -69,11 +67,7 @@ function get_job(body) { message: 'language is required as a string', }); } - if (!version || typeof version !== 'string') { - return reject({ - message: 'version is required as a string', - }); - } + if (!files || !Array.isArray(files)) { return reject({ message: 'files is required as an array', @@ -87,10 +81,50 @@ function get_job(body) { } } - const rt = runtime.get_latest_runtime_matching_language_version( - language, - version - ); + if (compile_memory_limit) { + if (typeof compile_memory_limit !== 'number') { + return reject({ + message: 'if specified, compile_memory_limit must be a number', + }); + } + + if ( + config.compile_memory_limit >= 0 && + (compile_memory_limit > config.compile_memory_limit || + compile_memory_limit < 0) + ) { + return reject({ + message: + 'compile_memory_limit cannot exceed the configured limit of ' + + config.compile_memory_limit, + }); + } + } + + if (run_memory_limit) { + if (typeof run_memory_limit !== 'number') { + return reject({ + message: 'if specified, run_memory_limit must be a number', + }); + } + + if ( + config.run_memory_limit >= 0 && + (run_memory_limit > config.run_memory_limit || run_memory_limit < 0) + ) { + return reject({ + message: + 'run_memory_limit cannot exceed the configured limit of ' + + config.run_memory_limit, + }); + } + } + + const rt = runtime.find(rt => [ + ...rt.aliases, + rt.language + ].includes(rt.language)) + if (rt === undefined) { return reject({ message: `${language}-${version} runtime is unknown`, @@ -298,77 +332,4 @@ router.get('/runtimes', (req, res) => { return res.status(200).send(runtimes); }); -router.get('/packages', async (req, res) => { - logger.debug('Request to list packages'); - let packages = await package.get_package_list(); - - packages = packages.map(pkg => { - return { - language: pkg.language, - language_version: pkg.version.raw, - installed: pkg.installed, - }; - }); - - return res.status(200).send(packages); -}); - -router.post('/packages', async (req, res) => { - logger.debug('Request to install package'); - - const { language, version } = req.body; - - const pkg = await package.get_package(language, version); - - if (pkg == null) { - return res.status(404).send({ - message: `Requested package ${language}-${version} does not exist`, - }); - } - - try { - const response = await pkg.install(); - - return res.status(200).send(response); - } catch (e) { - logger.error( - `Error while installing package ${pkg.language}-${pkg.version}:`, - e.message - ); - - return res.status(500).send({ - message: e.message, - }); - } -}); - -router.delete('/packages', async (req, res) => { - logger.debug('Request to uninstall package'); - - const { language, version } = req.body; - - const pkg = await package.get_package(language, version); - - if (pkg == null) { - return res.status(404).send({ - message: `Requested package ${language}-${version} does not exist`, - }); - } - - try { - const response = await pkg.uninstall(); - - return res.status(200).send(response); - } catch (e) { - logger.error( - `Error while uninstalling package ${pkg.language}-${pkg.version}:`, - e.message - ); - - return res.status(500).send({ - message: e.message, - }); - } -}); - module.exports = router; diff --git a/api/src/api/v3.js b/api/src/api/v3.js new file mode 100644 index 0000000..3e8b530 --- /dev/null +++ b/api/src/api/v3.js @@ -0,0 +1,236 @@ +const express = require('express'); +const router = express.Router(); + +const events = require('events'); + +const config = require('../config'); +const runtime = require('../runtime'); +const { Job } = require('../job'); +const logger = require('logplease').create('api/v3'); + +const SIGNALS = ["SIGABRT","SIGALRM","SIGBUS","SIGCHLD","SIGCLD","SIGCONT","SIGEMT","SIGFPE","SIGHUP","SIGILL","SIGINFO","SIGINT","SIGIO","SIGIOT","SIGKILL","SIGLOST","SIGPIPE","SIGPOLL","SIGPROF","SIGPWR","SIGQUIT","SIGSEGV","SIGSTKFLT","SIGSTOP","SIGTSTP","SIGSYS","SIGTERM","SIGTRAP","SIGTTIN","SIGTTOU","SIGUNUSED","SIGURG","SIGUSR1","SIGUSR2","SIGVTALRM","SIGXCPU","SIGXFSZ","SIGWINCH"] +// ref: https://man7.org/linux/man-pages/man7/signal.7.html + +function get_job(body){ + const { + runtime_id, + args, + stdin, + files, + compile_memory_limit, + run_memory_limit, + run_timeout, + compile_timeout + } = body; + + return new Promise((resolve, reject) => { + if (typeof runtime_id !== 'number') { + return reject({ + message: 'runtime_id is required as a number' + }); + } + + if (!Array.isArray(files)) { + return reject({ + message: 'files is required as an array', + }); + } + + for (const [i, file] of files.entries()) { + if (typeof file.content !== 'string') { + return reject({ + message: `files[${i}].content is required as a string`, + }); + } + } + + if (compile_memory_limit) { + if (typeof compile_memory_limit !== 'number') { + return reject({ + message: 'if specified, compile_memory_limit must be a number', + }); + } + + if ( + config.compile_memory_limit >= 0 && + (compile_memory_limit > config.compile_memory_limit || + compile_memory_limit < 0) + ) { + return reject({ + message: + 'compile_memory_limit cannot exceed the configured limit of ' + + config.compile_memory_limit, + }); + } + } + + if (run_memory_limit) { + if (typeof run_memory_limit !== 'number') { + return reject({ + message: 'if specified, run_memory_limit must be a number', + }); + } + + if ( + config.run_memory_limit >= 0 && + (run_memory_limit > config.run_memory_limit || run_memory_limit < 0) + ) { + return reject({ + message: + 'run_memory_limit cannot exceed the configured limit of ' + + config.run_memory_limit, + }); + } + } + const rt = runtime[runtime_id]; + + + if (rt === undefined) { + return reject({ + message: `Runtime #${runtime_id} is unknown`, + }); + } + + resolve(new Job({ + runtime: rt, + args: args || [], + stdin: stdin || "", + files, + timeouts: { + run: run_timeout || 3000, + compile: compile_timeout || 10000, + }, + memory_limits: { + run: run_memory_limit || config.run_memory_limit, + compile: compile_memory_limit || config.compile_memory_limit, + } + })); + }) + +} + +router.use((req, res, next) => { + if (['GET', 'HEAD', 'OPTIONS'].includes(req.method)) { + return next(); + } + + if (!req.headers['content-type'].startsWith('application/json')) { + return res.status(415).send({ + message: 'requests must be of type application/json', + }); + } + + next(); +}); + +router.ws('/connect', async (ws, req) => { + + let job = null; + let eventBus = new events.EventEmitter(); + + eventBus.on("stdout", (data) => ws.send(JSON.stringify({type: "data", stream: "stdout", data: data.toString()}))) + eventBus.on("stderr", (data) => ws.send(JSON.stringify({type: "data", stream: "stderr", data: data.toString()}))) + eventBus.on("stage", (stage)=> ws.send(JSON.stringify({type: "stage", stage}))) + eventBus.on("exit", (stage, status) => ws.send(JSON.stringify({type: "exit", stage, ...status}))) + + ws.on("message", async (data) => { + + try{ + const msg = JSON.parse(data); + + switch(msg.type){ + case "init": + if(job === null){ + job = await get_job(msg); + + await job.prime(); + + ws.send(JSON.stringify({ + type: "runtime", + language: job.runtime.language, + version: job.runtime.version.raw + })) + + await job.execute_interactive(eventBus); + + ws.close(4999, "Job Completed"); + + }else{ + ws.close(4000, "Already Initialized"); + } + break; + case "data": + if(job !== null){ + if(msg.stream === "stdin"){ + eventBus.emit("stdin", msg.data) + }else{ + ws.close(4004, "Can only write to stdin") + } + }else{ + ws.close(4003, "Not yet initialized") + } + break; + case "signal": + if(job !== null){ + if(SIGNALS.includes(msg.signal)){ + eventBus.emit("signal", msg.signal) + }else{ + ws.close(4005, "Invalid signal") + } + }else{ + ws.close(4003, "Not yet initialized") + } + break; + } + + }catch(error){ + ws.send(JSON.stringify({type: "error", message: error.message})) + ws.close(4002, "Notified Error") + // ws.close message is limited to 123 characters, so we notify over WS then close. + } + }) + + ws.on("close", async ()=>{ + if(job !== null){ + await job.cleanup() + } + }) + + setTimeout(()=>{ + //Terminate the socket after 1 second, if not initialized. + if(job === null) + ws.close(4001, "Initialization Timeout"); + }, 1000) +}) + +router.post('/execute', async (req, res) => { + + try{ + const job = await get_job(req.body); + await job.prime(); + + const result = await job.execute(); + + await job.cleanup(); + + return res.status(200).send(result); + }catch(error){ + return res.status(400).json(error); + } +}); + +router.get('/runtimes', (req, res) => { + const runtimes = runtime.map(rt => { + return { + language: rt.language, + version: rt.version.raw, + aliases: rt.aliases, + runtime: rt.runtime, + id: rt.id + }; + }); + + return res.status(200).send(runtimes); +}); + +module.exports = router; diff --git a/api/src/config.js b/api/src/config.js index 5324097..8ddc3f5 100644 --- a/api/src/config.js +++ b/api/src/config.js @@ -171,10 +171,9 @@ const options = [ validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`], }, { - key: 'repo_url', - desc: 'URL of repo index', - default: - 'https://github.com/engineer-man/piston/releases/download/pkgs/index', + key: 'flake_path', + desc: 'Path to nix flake defining runtimes to install', + default: 'github:engineer-man/piston?directory=packages', validators: [], }, { diff --git a/api/src/globals.js b/api/src/globals.js index 933d2ca..f2a6459 100644 --- a/api/src/globals.js +++ b/api/src/globals.js @@ -1,20 +1,9 @@ // Globals are things the user shouldn't change in config, but is good to not use inline constants for -const is_docker = require('is-docker'); -const fs = require('fs'); -const platform = `${is_docker() ? 'docker' : 'baremetal'}-${fs - .read_file_sync('/etc/os-release') - .toString() - .split('\n') - .find(x => x.startsWith('ID')) - .replace('ID=', '')}`; module.exports = { data_directories: { - packages: 'packages', jobs: 'jobs', }, version: require('../package.json').version, - platform, - pkg_installed_file: '.ppman-installed', //Used as indication for if a package was installed clean_directories: ['/dev/shm', '/run/lock', '/tmp', '/var/tmp'], }; diff --git a/api/src/job.js b/api/src/job.js index eef2ffd..e05f017 100644 --- a/api/src/job.js +++ b/api/src/job.js @@ -240,8 +240,8 @@ class Job { if (this.runtime.compiled) { compile = await this.safe_call( - path.join(this.runtime.pkgdir, 'compile'), - code_files.map(x => x.name), + this.runtime.compile, + this.files.map(x => x.name), this.timeouts.compile, this.memory_limits.compile ); @@ -262,7 +262,7 @@ class Job { compile, run, language: this.runtime.language, - version: this.runtime.version.raw, + version: this.runtime.version, }; } diff --git a/api/src/package.js b/api/src/package.js deleted file mode 100644 index 4e4630f..0000000 --- a/api/src/package.js +++ /dev/null @@ -1,226 +0,0 @@ -const logger = require('logplease').create('package'); -const semver = require('semver'); -const config = require('./config'); -const globals = require('./globals'); -const fetch = require('node-fetch'); -const path = require('path'); -const fs = require('fs/promises'); -const fss = require('fs'); -const cp = require('child_process'); -const crypto = require('crypto'); -const runtime = require('./runtime'); -const chownr = require('chownr'); -const util = require('util'); - -class Package { - constructor({ language, version, download, checksum }) { - this.language = language; - this.version = semver.parse(version); - this.checksum = checksum; - this.download = download; - } - - get installed() { - return fss.exists_sync( - path.join(this.install_path, globals.pkg_installed_file) - ); - } - - get install_path() { - return path.join( - config.data_directory, - globals.data_directories.packages, - this.language, - this.version.raw - ); - } - - async install() { - if (this.installed) { - throw new Error('Already installed'); - } - - logger.info(`Installing ${this.language}-${this.version.raw}`); - - if (fss.exists_sync(this.install_path)) { - logger.warn( - `${this.language}-${this.version.raw} has residual files. Removing them.` - ); - await fs.rm(this.install_path, { recursive: true, force: true }); - } - - logger.debug(`Making directory ${this.install_path}`); - await fs.mkdir(this.install_path, { recursive: true }); - - logger.debug( - `Downloading package from ${this.download} in to ${this.install_path}` - ); - const pkgpath = path.join(this.install_path, 'pkg.tar.gz'); - const download = await fetch(this.download); - - const file_stream = fss.create_write_stream(pkgpath); - await new Promise((resolve, reject) => { - download.body.pipe(file_stream); - download.body.on('error', reject); - - file_stream.on('finish', resolve); - }); - - logger.debug('Validating checksums'); - logger.debug(`Assert sha256(pkg.tar.gz) == ${this.checksum}`); - const hash = crypto.create_hash('sha256'); - - const read_stream = fss.create_read_stream(pkgpath); - await new Promise((resolve, reject) => { - read_stream.on('data', chunk => hash.update(chunk)); - read_stream.on('end', () => resolve()); - read_stream.on('error', error => reject(error)); - }); - - const cs = hash.digest('hex'); - - if (cs !== this.checksum) { - throw new Error(`Checksum miss-match want: ${val} got: ${cs}`); - } - - logger.debug( - `Extracting package files from archive ${pkgpath} in to ${this.install_path}` - ); - - await new Promise((resolve, reject) => { - const proc = cp.exec( - `bash -c 'cd "${this.install_path}" && tar xzf ${pkgpath}'` - ); - - proc.once('exit', (code, _) => { - code === 0 ? resolve() : reject(); - }); - - proc.stdout.pipe(process.stdout); - proc.stderr.pipe(process.stderr); - - proc.once('error', reject); - }); - - logger.debug('Registering runtime'); - runtime.load_package(this.install_path); - - logger.debug('Caching environment'); - const get_env_command = `cd ${this.install_path}; source environment; env`; - - const envout = await new Promise((resolve, reject) => { - let stdout = ''; - - const proc = cp.spawn( - 'env', - ['-i', 'bash', '-c', `${get_env_command}`], - { - stdio: ['ignore', 'pipe', 'pipe'], - } - ); - - proc.once('exit', (code, _) => { - code === 0 ? resolve(stdout) : reject(); - }); - - proc.stdout.on('data', data => { - stdout += data; - }); - - proc.once('error', reject); - }); - - const filtered_env = envout - .split('\n') - .filter( - l => - !['PWD', 'OLDPWD', '_', 'SHLVL'].includes( - l.split('=', 2)[0] - ) - ) - .join('\n'); - - await fs.write_file(path.join(this.install_path, '.env'), filtered_env); - - logger.debug('Changing Ownership of package directory'); - await util.promisify(chownr)(this.install_path, 0, 0); - - logger.debug('Writing installed state to disk'); - await fs.write_file( - path.join(this.install_path, globals.pkg_installed_file), - Date.now().toString() - ); - - logger.info(`Installed ${this.language}-${this.version.raw}`); - - return { - language: this.language, - version: this.version.raw, - }; - } - - async uninstall() { - logger.info(`Uninstalling ${this.language}-${this.version.raw}`); - - logger.debug('Finding runtime'); - const found_runtime = runtime.get_runtime_by_name_and_version( - this.language, - this.version.raw - ); - - if (!found_runtime) { - logger.error( - `Uninstalling ${this.language}-${this.version.raw} failed: Not installed` - ); - throw new Error( - `${this.language}-${this.version.raw} is not installed` - ); - } - - logger.debug('Unregistering runtime'); - found_runtime.unregister(); - - logger.debug('Cleaning files from disk'); - await fs.rmdir(this.install_path, { recursive: true }); - - logger.info(`Uninstalled ${this.language}-${this.version.raw}`); - - return { - language: this.language, - version: this.version.raw, - }; - } - - static async get_package_list() { - const repo_content = await fetch(config.repo_url).then(x => x.text()); - - const entries = repo_content.split('\n').filter(x => x.length > 0); - - return entries.map(line => { - const [language, version, checksum, download] = line.split(',', 4); - - return new Package({ - language, - version, - checksum, - download, - }); - }); - } - - static async get_package(lang, version) { - const packages = await Package.get_package_list(); - - const candidates = packages.filter(pkg => { - return ( - pkg.language == lang && semver.satisfies(pkg.version, version) - ); - }); - - candidates.sort((a, b) => semver.rcompare(a.version, b.version)); - - return candidates[0] || null; - } -} - -module.exports = Package; diff --git a/api/src/index.js b/api/src/pistond.js old mode 100644 new mode 100755 similarity index 74% rename from api/src/index.js rename to api/src/pistond.js index 025f5ae..36e12e3 --- a/api/src/index.js +++ b/api/src/pistond.js @@ -5,6 +5,7 @@ const express = require('express'); const expressWs = require('express-ws'); const globals = require('./globals'); const config = require('./config'); +const cp = require('child_process'); const path = require('path'); const fs = require('fs/promises'); const fss = require('fs'); @@ -37,28 +38,11 @@ expressWs(app); }); logger.info('Loading packages'); - const pkgdir = path.join( - config.data_directory, - globals.data_directories.packages - ); - const pkglist = await fs.readdir(pkgdir); - - const languages = await Promise.all( - pkglist.map(lang => { - return fs.readdir(path.join(pkgdir, lang)).then(x => { - return x.map(y => path.join(pkgdir, lang, y)); - }); - }) - ); - - const installed_languages = languages - .flat() - .filter(pkg => - fss.exists_sync(path.join(pkg, globals.pkg_installed_file)) - ); - - installed_languages.for_each(pkg => runtime.load_package(pkg)); + const runtimes_data = cp.execSync(`nix eval --json ${config.flake_path}#pistonRuntimes --apply builtins.attrNames`).toString(); + const runtimes = JSON.parse(runtimes_data); + + runtimes.for_each(pkg => runtime.load_runtime(pkg)); logger.info('Starting API Server'); logger.debug('Constructing Express App'); @@ -76,8 +60,9 @@ expressWs(app); logger.debug('Registering Routes'); const api_v2 = require('./api/v2'); + const api_v3 = require('./api/v3'); app.use('/api/v2', api_v2); - app.use('/api/v2', api_v2); + app.use('/api/v3', api_v3); app.use((req, res, next) => { return res.status(404).send({ message: 'Not Found' }); diff --git a/api/src/runtime.js b/api/src/runtime.js index 6c6f10e..9ebd724 100644 --- a/api/src/runtime.js +++ b/api/src/runtime.js @@ -1,5 +1,5 @@ const logger = require('logplease').create('runtime'); -const semver = require('semver'); +const cp = require('child_process'); const config = require('./config'); const globals = require('./globals'); const fss = require('fs'); @@ -7,32 +7,44 @@ const path = require('path'); const runtimes = []; + class Runtime { constructor({ language, version, aliases, - pkgdir, runtime, + run, + compile, + packageSupport, + flake_key, timeouts, memory_limits, max_process_count, max_open_files, max_file_size, - output_max_size, + output_max_size }) { this.language = language; - this.version = version; - this.aliases = aliases || []; - this.pkgdir = pkgdir; this.runtime = runtime; + this.timeouts = timeouts; this.memory_limits = memory_limits; this.max_process_count = max_process_count; this.max_open_files = max_open_files; this.max_file_size = max_file_size; this.output_max_size = output_max_size; + + this.aliases = aliases; + this.version = version; + + this.run = run; + this.compile = compile; + + this.flake_key = flake_key; + this.package_support = packageSupport; } + static compute_single_limit( language_name, @@ -97,122 +109,61 @@ class Runtime { }; } - static load_package(package_dir) { - let info = JSON.parse( - fss.read_file_sync(path.join(package_dir, 'pkg-info.json')) - ); + ensure_built(){ + logger.info(`Ensuring ${this} is built`); - let { - language, - version, - build_platform, - aliases, - provides, - limit_overrides, - } = info; - version = semver.parse(version); + const flake_key = this.flake_key; - if (build_platform !== globals.platform) { - logger.warn( - `Package ${language}-${version} was built for platform ${build_platform}, ` + - `but our platform is ${globals.platform}` - ); + function _ensure_built(key){ + const command = `nix build ${config.flake_path}#pistonRuntimes.${flake_key}.metadata.${key} --no-link`; + cp.execSync(command, {stdio: "pipe"}) } - if (provides) { - // Multiple languages in 1 package - provides.forEach(lang => { - runtimes.push( - new Runtime({ - language: lang.language, - aliases: lang.aliases, - version, - pkgdir: package_dir, - runtime: language, - ...Runtime.compute_all_limits( - lang.language, - lang.limit_overrides - ), - }) - ); - }); - } else { - runtimes.push( - new Runtime({ - language, - version, - aliases, - pkgdir: package_dir, - ...Runtime.compute_all_limits(language, limit_overrides), - }) - ); - } + _ensure_built("run"); + if(this.compiled) _ensure_built("compile"); + + logger.debug(`Finished ensuring ${this} is installed`) + + } + + static load_runtime(flake_key){ + logger.info(`Loading ${flake_key}`) + const metadata_command = `nix eval --json ${config.flake_path}#pistonRuntimes.${flake_key}.metadata`; + const metadata = JSON.parse(cp.execSync(metadata_command)); + + const this_runtime = new Runtime({ + ...metadata, + ...Runtime.compute_all_limits( + metadata.language, + metadata.limit_overrides + ), + flake_key + }); + + this_runtime.ensure_built(); + + runtimes.push(this_runtime); + + + logger.debug(`Package ${flake_key} was loaded`); - logger.debug(`Package ${language}-${version} was loaded`); } get compiled() { - if (this._compiled === undefined) { - this._compiled = fss.exists_sync(path.join(this.pkgdir, 'compile')); - } - - return this._compiled; + return this.compile !== null; } - get env_vars() { - if (!this._env_vars) { - const env_file = path.join(this.pkgdir, '.env'); - const env_content = fss.read_file_sync(env_file).toString(); - - this._env_vars = {}; - - env_content - .trim() - .split('\n') - .map(line => line.split('=', 2)) - .forEach(([key, val]) => { - this._env_vars[key.trim()] = val.trim(); - }); - } - - return this._env_vars; + get id(){ + return runtimes.indexOf(this); } toString() { - return `${this.language}-${this.version.raw}`; + return `${this.language}-${this.version}`; } - unregister() { - const index = runtimes.indexOf(this); - runtimes.splice(index, 1); //Remove from runtimes list - } } module.exports = runtimes; module.exports.Runtime = Runtime; -module.exports.get_runtimes_matching_language_version = function (lang, ver) { - return runtimes.filter( - rt => - (rt.language == lang || rt.aliases.includes(lang)) && - semver.satisfies(rt.version, ver) - ); -}; -module.exports.get_latest_runtime_matching_language_version = function ( - lang, - ver -) { - return module.exports - .get_runtimes_matching_language_version(lang, ver) - .sort((a, b) => semver.rcompare(a.version, b.version))[0]; -}; +module.exports.load_runtime = Runtime.load_runtime; -module.exports.get_runtime_by_name_and_version = function (runtime, ver) { - return runtimes.find( - rt => - (rt.runtime == runtime || - (rt.runtime === undefined && rt.language == runtime)) && - semver.satisfies(rt.version, ver) - ); -}; - -module.exports.load_package = Runtime.load_package; diff --git a/api/yarn.lock b/api/yarn.lock new file mode 100644 index 0000000..eaef672 --- /dev/null +++ b/api/yarn.lock @@ -0,0 +1,1406 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +abbrev@1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8" + integrity sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q== + +accepts@~1.3.7: + version "1.3.7" + resolved "https://registry.yarnpkg.com/accepts/-/accepts-1.3.7.tgz#531bc726517a3b2b41f850021c6cc15eaab507cd" + integrity sha512-Il80Qs2WjYlJIBNzNkK6KYqlVMTbZLXgHx2oT0pU/fjRHyEp+PEfEPY0R3WCwAGVOtauxh1hOxNgIf5bv7dQpA== + dependencies: + mime-types "~2.1.24" + negotiator "0.6.2" + +ajv@^6.12.3: + version "6.12.6" + resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.12.6.tgz#baf5a62e802b07d977034586f8c3baf5adf26df4" + integrity sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g== + dependencies: + fast-deep-equal "^3.1.1" + fast-json-stable-stringify "^2.0.0" + json-schema-traverse "^0.4.1" + uri-js "^4.2.2" + +ansi-regex@^2.0.0: + version "2.1.1" + resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-2.1.1.tgz#c3b33ab5ee360d86e0e628f0468ae7ef27d654df" + integrity sha1-w7M6te42DYbg5ijwRorn7yfWVN8= + +ansi-regex@^5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.1.tgz#082cb2c89c9fe8659a311a53bd6a4dc5301db304" + integrity sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ== + +aproba@^1.0.3: + version "1.2.0" + resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.2.0.tgz#6802e6264efd18c790a1b0d517f0f2627bf2c94a" + integrity sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw== + +are-we-there-yet@~1.1.2: + version "1.1.7" + resolved "https://registry.yarnpkg.com/are-we-there-yet/-/are-we-there-yet-1.1.7.tgz#b15474a932adab4ff8a50d9adfa7e4e926f21146" + integrity sha512-nxwy40TuMiUGqMyRHgCSWZ9FM4VAoRP4xUYSTv5ImRog+h9yISPbVH7H8fASCIzYn9wlEv4zvFL7uKDMCFQm3g== + dependencies: + delegates "^1.0.0" + readable-stream "^2.0.6" + +array-flatten@1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/array-flatten/-/array-flatten-1.1.1.tgz#9a5f699051b1e7073328f2a008968b64ea2955d2" + integrity sha1-ml9pkFGx5wczKPKgCJaLZOopVdI= + +asn1@~0.2.3: + version "0.2.6" + resolved "https://registry.yarnpkg.com/asn1/-/asn1-0.2.6.tgz#0d3a7bb6e64e02a90c0303b31f292868ea09a08d" + integrity sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ== + dependencies: + safer-buffer "~2.1.0" + +assert-plus@1.0.0, assert-plus@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/assert-plus/-/assert-plus-1.0.0.tgz#f12e0f3c5d77b0b1cdd9146942e4e96c1e4dd525" + integrity sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU= + +asynckit@^0.4.0: + version "0.4.0" + resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" + integrity sha1-x57Zf380y48robyXkLzDZkdLS3k= + +aws-sign2@~0.7.0: + version "0.7.0" + resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8" + integrity sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg= + +aws4@^1.8.0: + version "1.11.0" + resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.11.0.tgz#d61f46d83b2519250e2784daf5b09479a8b41c59" + integrity sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA== + +balanced-match@^1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee" + integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw== + +base64-js@1.5.x: + version "1.5.1" + resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" + integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA== + +bcrypt-pbkdf@^1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz#a4301d389b6a43f9b67ff3ca11a3f6637e360e9e" + integrity sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4= + dependencies: + tweetnacl "^0.14.3" + +body-parser@1.19.1, body-parser@^1.19.0: + version "1.19.1" + resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.19.1.tgz#1499abbaa9274af3ecc9f6f10396c995943e31d4" + integrity sha512-8ljfQi5eBk8EJfECMrgqNGWPEY5jWP+1IzkzkGdFFEwFQZZyaZ21UqdaHktgiMlH0xLHqIFtE/u2OYE5dOtViA== + dependencies: + bytes "3.1.1" + content-type "~1.0.4" + debug "2.6.9" + depd "~1.1.2" + http-errors "1.8.1" + iconv-lite "0.4.24" + on-finished "~2.3.0" + qs "6.9.6" + raw-body "2.4.2" + type-is "~1.6.18" + +brace-expansion@^1.1.7: + version "1.1.11" + resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd" + integrity sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA== + dependencies: + balanced-match "^1.0.0" + concat-map "0.0.1" + +buffer-from@^1.0.0: + version "1.1.2" + resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.2.tgz#2b146a6fd72e80b4f55d255f35ed59a3a9a41bd5" + integrity sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ== + +builtins@^1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/builtins/-/builtins-1.0.3.tgz#cb94faeb61c8696451db36534e1422f94f0aee88" + integrity sha1-y5T662HIaWRR2zZTThQi+U8K7og= + +bytes@3.1.1: + version "3.1.1" + resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.1.tgz#3f018291cb4cbad9accb6e6970bca9c8889e879a" + integrity sha512-dWe4nWO/ruEOY7HkUJ5gFt1DCFV9zPRoJr8pV0/ASQermOZjtq8jMjOprC0Kd10GLN+l7xaUPvxzJFWtxGu8Fg== + +caseless@~0.12.0: + version "0.12.0" + resolved "https://registry.yarnpkg.com/caseless/-/caseless-0.12.0.tgz#1b681c21ff84033c826543090689420d187151dc" + integrity sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw= + +chownr@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/chownr/-/chownr-2.0.0.tgz#15bfbe53d2eab4cf70f18a8cd68ebe5b3cb1dece" + integrity sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ== + +code-point-at@^1.0.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/code-point-at/-/code-point-at-1.1.0.tgz#0d070b4d043a5bea33a2f1a40e2edb3d9a4ccf77" + integrity sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c= + +combined-stream@^1.0.6, combined-stream@~1.0.6: + version "1.0.8" + resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f" + integrity sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg== + dependencies: + delayed-stream "~1.0.0" + +concat-map@0.0.1: + version "0.0.1" + resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" + integrity sha1-2Klr13/Wjfd5OnMDajug1UBdR3s= + +concat-stream@^1.5.2: + version "1.6.2" + resolved "https://registry.yarnpkg.com/concat-stream/-/concat-stream-1.6.2.tgz#904bdf194cd3122fc675c77fc4ac3d4ff0fd1a34" + integrity sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw== + dependencies: + buffer-from "^1.0.0" + inherits "^2.0.3" + readable-stream "^2.2.2" + typedarray "^0.0.6" + +config-chain@~1.1.8: + version "1.1.13" + resolved "https://registry.yarnpkg.com/config-chain/-/config-chain-1.1.13.tgz#fad0795aa6a6cdaff9ed1b68e9dff94372c232f4" + integrity sha512-qj+f8APARXHrM0hraqXYb2/bOVSV4PvJQlNZ/DVj0QrmNM2q2euizkeuVckQ57J+W0mRH6Hvi+k50M4Jul2VRQ== + dependencies: + ini "^1.3.4" + proto-list "~1.2.1" + +console-control-strings@^1.0.0, console-control-strings@~1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/console-control-strings/-/console-control-strings-1.1.0.tgz#3d7cf4464db6446ea644bf4b39507f9851008e8e" + integrity sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4= + +content-disposition@0.5.4: + version "0.5.4" + resolved "https://registry.yarnpkg.com/content-disposition/-/content-disposition-0.5.4.tgz#8b82b4efac82512a02bb0b1dcec9d2c5e8eb5bfe" + integrity sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ== + dependencies: + safe-buffer "5.2.1" + +content-type@~1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/content-type/-/content-type-1.0.4.tgz#e138cc75e040c727b1966fe5e5f8c9aee256fe3b" + integrity sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA== + +cookie-signature@1.0.6: + version "1.0.6" + resolved "https://registry.yarnpkg.com/cookie-signature/-/cookie-signature-1.0.6.tgz#e303a882b342cc3ee8ca513a79999734dab3ae2c" + integrity sha1-4wOogrNCzD7oylE6eZmXNNqzriw= + +cookie@0.4.1: + version "0.4.1" + resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.4.1.tgz#afd713fe26ebd21ba95ceb61f9a8116e50a537d1" + integrity sha512-ZwrFkGJxUR3EIoXtO+yVE69Eb7KlixbaeAWfBQB9vVsNn/o+Yw69gBWSSDK825hQNdN+wF8zELf3dFNl/kxkUA== + +core-util-is@1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" + integrity sha1-tf1UIgqivFq1eqtxQMlAdUUDwac= + +core-util-is@~1.0.0: + version "1.0.3" + resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.3.tgz#a6042d3634c2b27e9328f837b965fac83808db85" + integrity sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ== + +dashdash@^1.12.0: + version "1.14.1" + resolved "https://registry.yarnpkg.com/dashdash/-/dashdash-1.14.1.tgz#853cfa0f7cbe2fed5de20326b8dd581035f6e2f0" + integrity sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA= + dependencies: + assert-plus "^1.0.0" + +debug@2.6.9: + version "2.6.9" + resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" + integrity sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA== + dependencies: + ms "2.0.0" + +delayed-stream@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" + integrity sha1-3zrhmayt+31ECqrgsp4icrJOxhk= + +delegates@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/delegates/-/delegates-1.0.0.tgz#84c6e159b81904fdca59a0ef44cd870d31250f9a" + integrity sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o= + +depd@~1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/depd/-/depd-1.1.2.tgz#9bcd52e14c097763e749b274c4346ed2e560b5a9" + integrity sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak= + +destroy@~1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/destroy/-/destroy-1.0.4.tgz#978857442c44749e4206613e37946205826abd80" + integrity sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA= + +ecc-jsbn@~0.1.1: + version "0.1.2" + resolved "https://registry.yarnpkg.com/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz#3a83a904e54353287874c564b7549386849a98c9" + integrity sha1-OoOpBOVDUyh4dMVkt1SThoSamMk= + dependencies: + jsbn "~0.1.0" + safer-buffer "^2.1.0" + +ee-first@1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/ee-first/-/ee-first-1.1.1.tgz#590c61156b0ae2f4f0255732a158b266bc56b21d" + integrity sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0= + +emoji-regex@^8.0.0: + version "8.0.0" + resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-8.0.0.tgz#e818fd69ce5ccfcb404594f842963bf53164cc37" + integrity sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A== + +encodeurl@~1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.2.tgz#ad3ff4c86ec2d029322f5a02c3a9a606c95b3f59" + integrity sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k= + +escape-html@~1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/escape-html/-/escape-html-1.0.3.tgz#0258eae4d3d0c0974de1c169188ef0051d1d1988" + integrity sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg= + +etag@~1.8.1: + version "1.8.1" + resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887" + integrity sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc= + +express-ws@^5.0.2: + version "5.0.2" + resolved "https://registry.yarnpkg.com/express-ws/-/express-ws-5.0.2.tgz#5b02d41b937d05199c6c266d7cc931c823bda8eb" + integrity sha512-0uvmuk61O9HXgLhGl3QhNSEtRsQevtmbL94/eILaliEADZBHZOQUAiHFrGPrgsjikohyrmSG5g+sCfASTt0lkQ== + dependencies: + ws "^7.4.6" + +express@^4.17.1: + version "4.17.2" + resolved "https://registry.yarnpkg.com/express/-/express-4.17.2.tgz#c18369f265297319beed4e5558753cc8c1364cb3" + integrity sha512-oxlxJxcQlYwqPWKVJJtvQiwHgosH/LrLSPA+H4UxpyvSS6jC5aH+5MoHFM+KABgTOt0APue4w66Ha8jCUo9QGg== + dependencies: + accepts "~1.3.7" + array-flatten "1.1.1" + body-parser "1.19.1" + content-disposition "0.5.4" + content-type "~1.0.4" + cookie "0.4.1" + cookie-signature "1.0.6" + debug "2.6.9" + depd "~1.1.2" + encodeurl "~1.0.2" + escape-html "~1.0.3" + etag "~1.8.1" + finalhandler "~1.1.2" + fresh "0.5.2" + merge-descriptors "1.0.1" + methods "~1.1.2" + on-finished "~2.3.0" + parseurl "~1.3.3" + path-to-regexp "0.1.7" + proxy-addr "~2.0.7" + qs "6.9.6" + range-parser "~1.2.1" + safe-buffer "5.2.1" + send "0.17.2" + serve-static "1.14.2" + setprototypeof "1.2.0" + statuses "~1.5.0" + type-is "~1.6.18" + utils-merge "1.0.1" + vary "~1.1.2" + +extend@~3.0.2: + version "3.0.2" + resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.2.tgz#f8b1136b4071fbd8eb140aff858b1019ec2915fa" + integrity sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g== + +extsprintf@1.3.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.3.0.tgz#96918440e3041a7a414f8c52e3c574eb3c3e1e05" + integrity sha1-lpGEQOMEGnpBT4xS48V06zw+HgU= + +extsprintf@^1.2.0: + version "1.4.1" + resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.4.1.tgz#8d172c064867f235c0c84a596806d279bf4bcc07" + integrity sha512-Wrk35e8ydCKDj/ArClo1VrPVmN8zph5V4AtHwIuHhvMXsKf73UT3BOD+azBIW+3wOJ4FhEH7zyaJCFvChjYvMA== + +fast-deep-equal@^3.1.1: + version "3.1.3" + resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz#3a7d56b559d6cbc3eb512325244e619a65c6c525" + integrity sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q== + +fast-json-stable-stringify@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz#874bf69c6f404c2b5d99c481341399fd55892633" + integrity sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw== + +finalhandler@~1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/finalhandler/-/finalhandler-1.1.2.tgz#b7e7d000ffd11938d0fdb053506f6ebabe9f587d" + integrity sha512-aAWcW57uxVNrQZqFXjITpW3sIUQmHGG3qSb9mUah9MgMC4NeWhNOlNjXEYq3HjRAvL6arUviZGGJsBg6z0zsWA== + dependencies: + debug "2.6.9" + encodeurl "~1.0.2" + escape-html "~1.0.3" + on-finished "~2.3.0" + parseurl "~1.3.3" + statuses "~1.5.0" + unpipe "~1.0.0" + +findit@2.0.x: + version "2.0.0" + resolved "https://registry.yarnpkg.com/findit/-/findit-2.0.0.tgz#6509f0126af4c178551cfa99394e032e13a4d56e" + integrity sha1-ZQnwEmr0wXhVHPqZOU4DLhOk1W4= + +foreachasync@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/foreachasync/-/foreachasync-3.0.0.tgz#5502987dc8714be3392097f32e0071c9dee07cf6" + integrity sha1-VQKYfchxS+M5IJfzLgBxyd7gfPY= + +forever-agent@~0.6.1: + version "0.6.1" + resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91" + integrity sha1-+8cfDEGt6zf5bFd60e1C2P2sypE= + +form-data@~2.3.2: + version "2.3.3" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.3.tgz#dcce52c05f644f298c6a7ab936bd724ceffbf3a6" + integrity sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ== + dependencies: + asynckit "^0.4.0" + combined-stream "^1.0.6" + mime-types "^2.1.12" + +forwarded@0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.2.0.tgz#2269936428aad4c15c7ebe9779a84bf0b2a81811" + integrity sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow== + +fresh@0.5.2: + version "0.5.2" + resolved "https://registry.yarnpkg.com/fresh/-/fresh-0.5.2.tgz#3d8cadd90d976569fa835ab1f8e4b23a105605a7" + integrity sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac= + +fs-extra@~0.6.1: + version "0.6.4" + resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-0.6.4.tgz#f46f0c75b7841f8d200b3348cd4d691d5a099d15" + integrity sha1-9G8MdbeEH40gCzNIzU1pHVoJnRU= + dependencies: + jsonfile "~1.0.1" + mkdirp "0.3.x" + ncp "~0.4.2" + rimraf "~2.2.0" + +fs-minipass@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/fs-minipass/-/fs-minipass-2.1.0.tgz#7f5036fdbf12c63c169190cbe4199c852271f9fb" + integrity sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg== + dependencies: + minipass "^3.0.0" + +fs.extra@1.3.x: + version "1.3.2" + resolved "https://registry.yarnpkg.com/fs.extra/-/fs.extra-1.3.2.tgz#dd023f93013bee24531f1b33514c37b20fd93349" + integrity sha1-3QI/kwE77iRTHxszUUw3sg/ZM0k= + dependencies: + fs-extra "~0.6.1" + mkdirp "~0.3.5" + walk "^2.3.9" + +fs.realpath@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" + integrity sha1-FQStJSMVjKpA20onh8sBQRmU6k8= + +function-bind@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.1.tgz#a56899d3ea3c9bab874bb9773b7c5ede92f4895d" + integrity sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A== + +gauge@~2.7.3: + version "2.7.4" + resolved "https://registry.yarnpkg.com/gauge/-/gauge-2.7.4.tgz#2c03405c7538c39d7eb37b317022e325fb018bf7" + integrity sha1-LANAXHU4w51+s3sxcCLjJfsBi/c= + dependencies: + aproba "^1.0.3" + console-control-strings "^1.0.0" + has-unicode "^2.0.0" + object-assign "^4.1.0" + signal-exit "^3.0.0" + string-width "^1.0.1" + strip-ansi "^3.0.1" + wide-align "^1.1.0" + +getpass@^0.1.1: + version "0.1.7" + resolved "https://registry.yarnpkg.com/getpass/-/getpass-0.1.7.tgz#5eff8e3e684d569ae4cb2b1282604e8ba62149fa" + integrity sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo= + dependencies: + assert-plus "^1.0.0" + +glob@^7.1.3: + version "7.2.0" + resolved "https://registry.yarnpkg.com/glob/-/glob-7.2.0.tgz#d15535af7732e02e948f4c41628bd910293f6023" + integrity sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q== + dependencies: + fs.realpath "^1.0.0" + inflight "^1.0.4" + inherits "2" + minimatch "^3.0.4" + once "^1.3.0" + path-is-absolute "^1.0.0" + +graceful-fs@^4.1.6: + version "4.2.9" + resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.9.tgz#041b05df45755e587a24942279b9d113146e1c96" + integrity sha512-NtNxqUcXgpW2iMrfqSfR73Glt39K+BLwWsPs94yR63v45T0Wbej7eRmL5cWfwEgqXnmjQp3zaJTshdRW/qC2ZQ== + +har-schema@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/har-schema/-/har-schema-2.0.0.tgz#a94c2224ebcac04782a0d9035521f24735b7ec92" + integrity sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI= + +har-validator@~5.1.3: + version "5.1.5" + resolved "https://registry.yarnpkg.com/har-validator/-/har-validator-5.1.5.tgz#1f0803b9f8cb20c0fa13822df1ecddb36bde1efd" + integrity sha512-nmT2T0lljbxdQZfspsno9hgrG3Uir6Ks5afism62poxqBM6sDnMEuPmzTq8XN0OEwqKLLdh1jQI3qyE66Nzb3w== + dependencies: + ajv "^6.12.3" + har-schema "^2.0.0" + +has-unicode@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/has-unicode/-/has-unicode-2.0.1.tgz#e0e6fe6a28cf51138855e086d1691e771de2a8b9" + integrity sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk= + +has@^1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/has/-/has-1.0.3.tgz#722d7cbfc1f6aa8241f16dd814e011e1f41e8796" + integrity sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw== + dependencies: + function-bind "^1.1.1" + +hosted-git-info@^2.1.4, hosted-git-info@^2.7.1: + version "2.8.9" + resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.8.9.tgz#dffc0bf9a21c02209090f2aa69429e1414daf3f9" + integrity sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw== + +http-errors@1.8.1: + version "1.8.1" + resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-1.8.1.tgz#7c3f28577cbc8a207388455dbd62295ed07bd68c" + integrity sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g== + dependencies: + depd "~1.1.2" + inherits "2.0.4" + setprototypeof "1.2.0" + statuses ">= 1.5.0 < 2" + toidentifier "1.0.1" + +http-signature@~1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.2.0.tgz#9aecd925114772f3d95b65a60abb8f7c18fbace1" + integrity sha1-muzZJRFHcvPZW2WmCruPfBj7rOE= + dependencies: + assert-plus "^1.0.0" + jsprim "^1.2.2" + sshpk "^1.7.0" + +iconv-lite@0.4.24: + version "0.4.24" + resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.24.tgz#2022b4b25fbddc21d2f524974a474aafe733908b" + integrity sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA== + dependencies: + safer-buffer ">= 2.1.2 < 3" + +inflight@^1.0.4: + version "1.0.6" + resolved "https://registry.yarnpkg.com/inflight/-/inflight-1.0.6.tgz#49bd6331d7d02d0c09bc910a1075ba8165b56df9" + integrity sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk= + dependencies: + once "^1.3.0" + wrappy "1" + +inherits@2, inherits@2.0.4, inherits@^2.0.3, inherits@~2.0.0, inherits@~2.0.3: + version "2.0.4" + resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" + integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== + +ini@^1.2.0, ini@^1.3.4: + version "1.3.8" + resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.8.tgz#a29da425b48806f34767a4efce397269af28432c" + integrity sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew== + +ipaddr.js@1.9.1: + version "1.9.1" + resolved "https://registry.yarnpkg.com/ipaddr.js/-/ipaddr.js-1.9.1.tgz#bff38543eeb8984825079ff3a2a8e6cbd46781b3" + integrity sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g== + +is-core-module@^2.8.1: + version "2.8.1" + resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.8.1.tgz#f59fdfca701d5879d0a6b100a40aa1560ce27211" + integrity sha512-SdNCUs284hr40hFTFP6l0IfZ/RSrMXF3qgoRHd3/79unUTvrFO/JoXwkGm+5J/Oe3E/b5GsnG330uUNgRpu1PA== + dependencies: + has "^1.0.3" + +is-fullwidth-code-point@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz#ef9e31386f031a7f0d643af82fde50c457ef00cb" + integrity sha1-754xOG8DGn8NZDr4L95QxFfvAMs= + dependencies: + number-is-nan "^1.0.0" + +is-fullwidth-code-point@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz#f116f8064fe90b3f7844a38997c0b75051269f1d" + integrity sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg== + +is-typedarray@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a" + integrity sha1-5HnICFjfDBsR3dppQPlgEfzaSpo= + +isarray@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" + integrity sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE= + +isstream@~0.1.2: + version "0.1.2" + resolved "https://registry.yarnpkg.com/isstream/-/isstream-0.1.2.tgz#47e63f7af55afa6f92e1500e690eb8b8529c099a" + integrity sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo= + +jsbn@~0.1.0: + version "0.1.1" + resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-0.1.1.tgz#a5e654c2e5a2deb5f201d96cefbca80c0ef2f513" + integrity sha1-peZUwuWi3rXyAdls77yoDA7y9RM= + +json-schema-traverse@^0.4.1: + version "0.4.1" + resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz#69f6a87d9513ab8bb8fe63bdb0979c448e684660" + integrity sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg== + +json-schema@0.4.0: + version "0.4.0" + resolved "https://registry.yarnpkg.com/json-schema/-/json-schema-0.4.0.tgz#f7de4cf6efab838ebaeb3236474cbba5a1930ab5" + integrity sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA== + +json-stringify-safe@~5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb" + integrity sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus= + +jsonfile@~1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-1.0.1.tgz#ea5efe40b83690b98667614a7392fc60e842c0dd" + integrity sha1-6l7+QLg2kLmGZ2FKc5L8YOhCwN0= + +jsprim@^1.2.2: + version "1.4.2" + resolved "https://registry.yarnpkg.com/jsprim/-/jsprim-1.4.2.tgz#712c65533a15c878ba59e9ed5f0e26d5b77c5feb" + integrity sha512-P2bSOMAc/ciLz6DzgjVlGJP9+BrJWu5UDGK70C2iweC5QBIeFf0ZXRvGjEj2uYgrY2MkAAhsSWHDWlFtEroZWw== + dependencies: + assert-plus "1.0.0" + extsprintf "1.3.0" + json-schema "0.4.0" + verror "1.10.0" + +logplease@^1.2.15: + version "1.2.15" + resolved "https://registry.yarnpkg.com/logplease/-/logplease-1.2.15.tgz#3da442e93751a5992cc19010a826b08d0293c48a" + integrity sha512-jLlHnlsPSJjpwUfcNyUxXCl33AYg2cHhIf9QhGL2T4iPT0XPB+xP1LRKFPgIg1M/sg9kAJvy94w9CzBNrfnstA== + +lru-cache@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-6.0.0.tgz#6d6fe6570ebd96aaf90fcad1dafa3b2566db3a94" + integrity sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA== + dependencies: + yallist "^4.0.0" + +media-typer@0.3.0: + version "0.3.0" + resolved "https://registry.yarnpkg.com/media-typer/-/media-typer-0.3.0.tgz#8710d7af0aa626f8fffa1ce00168545263255748" + integrity sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g= + +merge-descriptors@1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/merge-descriptors/-/merge-descriptors-1.0.1.tgz#b00aaa556dd8b44568150ec9d1b953f3f90cbb61" + integrity sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E= + +methods@~1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/methods/-/methods-1.1.2.tgz#5529a4d67654134edcc5266656835b0f851afcee" + integrity sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4= + +mime-db@1.51.0: + version "1.51.0" + resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.51.0.tgz#d9ff62451859b18342d960850dc3cfb77e63fb0c" + integrity sha512-5y8A56jg7XVQx2mbv1lu49NR4dokRnhZYTtL+KGfaa27uq4pSTXkwQkFJl4pkRMyNFz/EtYDSkiiEHx3F7UN6g== + +mime-types@^2.1.12, mime-types@~2.1.19, mime-types@~2.1.24: + version "2.1.34" + resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.34.tgz#5a712f9ec1503511a945803640fafe09d3793c24" + integrity sha512-6cP692WwGIs9XXdOO4++N+7qjqv0rqxxVvJ3VHPh/Sc9mVZcQP+ZGhkKiTvWMQRr2tbHkJP/Yn7Y0npb3ZBs4A== + dependencies: + mime-db "1.51.0" + +mime@1.6.0: + version "1.6.0" + resolved "https://registry.yarnpkg.com/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1" + integrity sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg== + +minimatch@^3.0.4: + version "3.0.4" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" + integrity sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA== + dependencies: + brace-expansion "^1.1.7" + +minimist@^1.2.5: + version "1.2.5" + resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.5.tgz#67d66014b66a6a8aaa0c083c5fd58df4e4e97602" + integrity sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw== + +minipass@^3.0.0: + version "3.1.6" + resolved "https://registry.yarnpkg.com/minipass/-/minipass-3.1.6.tgz#3b8150aa688a711a1521af5e8779c1d3bb4f45ee" + integrity sha512-rty5kpw9/z8SX9dmxblFA6edItUmwJgMeYDZRrwlIVN27i8gysGbznJwUggw2V/FVqFSDdWy040ZPS811DYAqQ== + dependencies: + yallist "^4.0.0" + +minizlib@^2.1.1: + version "2.1.2" + resolved "https://registry.yarnpkg.com/minizlib/-/minizlib-2.1.2.tgz#e90d3466ba209b932451508a11ce3d3632145931" + integrity sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg== + dependencies: + minipass "^3.0.0" + yallist "^4.0.0" + +mkdirp@0.3.x, mkdirp@~0.3.5: + version "0.3.5" + resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.3.5.tgz#de3e5f8961c88c787ee1368df849ac4413eca8d7" + integrity sha1-3j5fiWHIjHh+4TaN+EmsRBPsqNc= + +mkdirp@^0.5.0, mkdirp@^0.5.1: + version "0.5.5" + resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.5.tgz#d91cefd62d1436ca0f41620e251288d420099def" + integrity sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ== + dependencies: + minimist "^1.2.5" + +mkdirp@^1.0.3: + version "1.0.4" + resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e" + integrity sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw== + +ms@2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" + integrity sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g= + +ms@2.1.3: + version "2.1.3" + resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" + integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== + +ncp@~0.4.2: + version "0.4.2" + resolved "https://registry.yarnpkg.com/ncp/-/ncp-0.4.2.tgz#abcc6cbd3ec2ed2a729ff6e7c1fa8f01784a8574" + integrity sha1-q8xsvT7C7Spyn/bnwfqPAXhKhXQ= + +negotiator@0.6.2: + version "0.6.2" + resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.2.tgz#feacf7ccf525a77ae9634436a64883ffeca346fb" + integrity sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw== + +nijs@0.0.25: + version "0.0.25" + resolved "https://registry.yarnpkg.com/nijs/-/nijs-0.0.25.tgz#04b035cb530d46859d1018839a518c029133f676" + integrity sha1-BLA1y1MNRoWdEBiDmlGMApEz9nY= + dependencies: + optparse ">= 1.0.3" + slasp "0.0.4" + +"nocamel@git://github.com/HexF/nocamel.git#patch-1": + version "1.1.0" + resolved "git://github.com/HexF/nocamel.git#89a5bfbbd07c72c302d968b967d0f4fe54846544" + +node-fetch@^2.6.1: + version "2.6.7" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad" + integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ== + dependencies: + whatwg-url "^5.0.0" + +node2nix@^1.6.0: + version "1.10.0" + resolved "https://registry.yarnpkg.com/node2nix/-/node2nix-1.10.0.tgz#e206b9ca228e36878f09d9e8fd931097a2bb58ff" + integrity sha512-3hUEkGeUl9X2FQCeZ9K2/WiSdBjXVKMnGz7qQVGuWnyxwA8xqWi+dQueTd+YmOA7LmZAseQgok1CUcoTR7d2FQ== + dependencies: + base64-js "1.5.x" + findit "2.0.x" + fs.extra "1.3.x" + nijs "0.0.25" + npm-registry-client "8.6.x" + npmconf "2.1.x" + optparse "1.0.x" + semver "7.3.x" + slasp "0.0.4" + spdx-license-ids "3.0.x" + tar "6.1.x" + temp "0.9.x" + +nopt@~3.0.1: + version "3.0.6" + resolved "https://registry.yarnpkg.com/nopt/-/nopt-3.0.6.tgz#c6465dbf08abcd4db359317f79ac68a646b28ff9" + integrity sha1-xkZdvwirzU2zWTF/eaxopkayj/k= + dependencies: + abbrev "1" + +"normalize-package-data@~1.0.1 || ^2.0.0": + version "2.5.0" + resolved "https://registry.yarnpkg.com/normalize-package-data/-/normalize-package-data-2.5.0.tgz#e66db1838b200c1dfc233225d12cb36520e234a8" + integrity sha512-/5CMN3T0R4XTj4DcGaexo+roZSdSFW/0AOOTROrjxzCG1wrWXEsGbRKevjlIL+ZDE4sZlJr5ED4YW0yqmkK+eA== + dependencies: + hosted-git-info "^2.1.4" + resolve "^1.10.0" + semver "2 || 3 || 4 || 5" + validate-npm-package-license "^3.0.1" + +"npm-package-arg@^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0": + version "6.1.1" + resolved "https://registry.yarnpkg.com/npm-package-arg/-/npm-package-arg-6.1.1.tgz#02168cb0a49a2b75bf988a28698de7b529df5cb7" + integrity sha512-qBpssaL3IOZWi5vEKUKW0cO7kzLeT+EQO9W8RsLOZf76KF9E/K9+wH0C7t06HXPpaH8WH5xF1MExLuCwbTqRUg== + dependencies: + hosted-git-info "^2.7.1" + osenv "^0.1.5" + semver "^5.6.0" + validate-npm-package-name "^3.0.0" + +npm-registry-client@8.6.x: + version "8.6.0" + resolved "https://registry.yarnpkg.com/npm-registry-client/-/npm-registry-client-8.6.0.tgz#7f1529f91450732e89f8518e0f21459deea3e4c4" + integrity sha512-Qs6P6nnopig+Y8gbzpeN/dkt+n7IyVd8f45NTMotGk6Qo7GfBmzwYx6jRLoOOgKiMnaQfYxsuyQlD8Mc3guBhg== + dependencies: + concat-stream "^1.5.2" + graceful-fs "^4.1.6" + normalize-package-data "~1.0.1 || ^2.0.0" + npm-package-arg "^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0" + once "^1.3.3" + request "^2.74.0" + retry "^0.10.0" + safe-buffer "^5.1.1" + semver "2 >=2.2.1 || 3.x || 4 || 5" + slide "^1.1.3" + ssri "^5.2.4" + optionalDependencies: + npmlog "2 || ^3.1.0 || ^4.0.0" + +npmconf@2.1.x: + version "2.1.3" + resolved "https://registry.yarnpkg.com/npmconf/-/npmconf-2.1.3.tgz#1cbe5dd02e899d365fed7260b54055473f90a15c" + integrity sha512-iTK+HI68GceCoGOHAQiJ/ik1iDfI7S+cgyG8A+PP18IU3X83kRhQIRhAUNj4Bp2JMx6Zrt5kCiozYa9uGWTjhA== + dependencies: + config-chain "~1.1.8" + inherits "~2.0.0" + ini "^1.2.0" + mkdirp "^0.5.0" + nopt "~3.0.1" + once "~1.3.0" + osenv "^0.1.0" + safe-buffer "^5.1.1" + semver "2 || 3 || 4" + uid-number "0.0.5" + +"npmlog@2 || ^3.1.0 || ^4.0.0": + version "4.1.2" + resolved "https://registry.yarnpkg.com/npmlog/-/npmlog-4.1.2.tgz#08a7f2a8bf734604779a9efa4ad5cc717abb954b" + integrity sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg== + dependencies: + are-we-there-yet "~1.1.2" + console-control-strings "~1.1.0" + gauge "~2.7.3" + set-blocking "~2.0.0" + +number-is-nan@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/number-is-nan/-/number-is-nan-1.0.1.tgz#097b602b53422a522c1afb8790318336941a011d" + integrity sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0= + +oauth-sign@~0.9.0: + version "0.9.0" + resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.9.0.tgz#47a7b016baa68b5fa0ecf3dee08a85c679ac6455" + integrity sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ== + +object-assign@^4.1.0: + version "4.1.1" + resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" + integrity sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM= + +on-finished@~2.3.0: + version "2.3.0" + resolved "https://registry.yarnpkg.com/on-finished/-/on-finished-2.3.0.tgz#20f1336481b083cd75337992a16971aa2d906947" + integrity sha1-IPEzZIGwg811M3mSoWlxqi2QaUc= + dependencies: + ee-first "1.1.1" + +once@^1.3.0, once@^1.3.3: + version "1.4.0" + resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1" + integrity sha1-WDsap3WWHUsROsF9nFC6753Xa9E= + dependencies: + wrappy "1" + +once@~1.3.0: + version "1.3.3" + resolved "https://registry.yarnpkg.com/once/-/once-1.3.3.tgz#b2e261557ce4c314ec8304f3fa82663e4297ca20" + integrity sha1-suJhVXzkwxTsgwTz+oJmPkKXyiA= + dependencies: + wrappy "1" + +optparse@1.0.x, "optparse@>= 1.0.3": + version "1.0.5" + resolved "https://registry.yarnpkg.com/optparse/-/optparse-1.0.5.tgz#75e75a96506611eb1c65ba89018ff08a981e2c16" + integrity sha1-dedallBmEescZbqJAY/wipgeLBY= + +os-homedir@^1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/os-homedir/-/os-homedir-1.0.2.tgz#ffbc4988336e0e833de0c168c7ef152121aa7fb3" + integrity sha1-/7xJiDNuDoM94MFox+8VISGqf7M= + +os-tmpdir@^1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/os-tmpdir/-/os-tmpdir-1.0.2.tgz#bbe67406c79aa85c5cfec766fe5734555dfa1274" + integrity sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ= + +osenv@^0.1.0, osenv@^0.1.5: + version "0.1.5" + resolved "https://registry.yarnpkg.com/osenv/-/osenv-0.1.5.tgz#85cdfafaeb28e8677f416e287592b5f3f49ea410" + integrity sha512-0CWcCECdMVc2Rw3U5w9ZjqX6ga6ubk1xDVKxtBQPK7wis/0F2r9T6k4ydGYhecl7YUBxBVxhL5oisPsNxAPe2g== + dependencies: + os-homedir "^1.0.0" + os-tmpdir "^1.0.0" + +parseurl@~1.3.3: + version "1.3.3" + resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.3.tgz#9da19e7bee8d12dff0513ed5b76957793bc2e8d4" + integrity sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ== + +path-is-absolute@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f" + integrity sha1-F0uSaHNVNP+8es5r9TpanhtcX18= + +path-parse@^1.0.7: + version "1.0.7" + resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.7.tgz#fbc114b60ca42b30d9daf5858e4bd68bbedb6735" + integrity sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw== + +path-to-regexp@0.1.7: + version "0.1.7" + resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-0.1.7.tgz#df604178005f522f15eb4490e7247a1bfaa67f8c" + integrity sha1-32BBeABfUi8V60SQ5yR6G/qmf4w= + +performance-now@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b" + integrity sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns= + +prettier@2.2.1: + version "2.2.1" + resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.2.1.tgz#795a1a78dd52f073da0cd42b21f9c91381923ff5" + integrity sha512-PqyhM2yCjg/oKkFPtTGUojv7gnZAoG80ttl45O6x2Ug/rMJw4wcc9k6aaf2hibP7BGVCCM33gZoGjyvt9mm16Q== + +process-nextick-args@~2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2" + integrity sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag== + +proto-list@~1.2.1: + version "1.2.4" + resolved "https://registry.yarnpkg.com/proto-list/-/proto-list-1.2.4.tgz#212d5bfe1318306a420f6402b8e26ff39647a849" + integrity sha1-IS1b/hMYMGpCD2QCuOJv85ZHqEk= + +proxy-addr@~2.0.7: + version "2.0.7" + resolved "https://registry.yarnpkg.com/proxy-addr/-/proxy-addr-2.0.7.tgz#f19fe69ceab311eeb94b42e70e8c2070f9ba1025" + integrity sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg== + dependencies: + forwarded "0.2.0" + ipaddr.js "1.9.1" + +psl@^1.1.28: + version "1.8.0" + resolved "https://registry.yarnpkg.com/psl/-/psl-1.8.0.tgz#9326f8bcfb013adcc005fdff056acce020e51c24" + integrity sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ== + +punycode@^2.1.0, punycode@^2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec" + integrity sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A== + +qs@6.9.6: + version "6.9.6" + resolved "https://registry.yarnpkg.com/qs/-/qs-6.9.6.tgz#26ed3c8243a431b2924aca84cc90471f35d5a0ee" + integrity sha512-TIRk4aqYLNoJUbd+g2lEdz5kLWIuTMRagAXxl78Q0RiVjAOugHmeKNGdd3cwo/ktpf9aL9epCfFqWDEKysUlLQ== + +qs@~6.5.2: + version "6.5.3" + resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.3.tgz#3aeeffc91967ef6e35c0e488ef46fb296ab76aad" + integrity sha512-qxXIEh4pCGfHICj1mAJQ2/2XVZkjCDTcEgfoSQxc/fYivUZxTkk7L3bDBJSoNrEzXI17oUO5Dp07ktqE5KzczA== + +range-parser@~1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/range-parser/-/range-parser-1.2.1.tgz#3cf37023d199e1c24d1a55b84800c2f3e6468031" + integrity sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg== + +raw-body@2.4.2: + version "2.4.2" + resolved "https://registry.yarnpkg.com/raw-body/-/raw-body-2.4.2.tgz#baf3e9c21eebced59dd6533ac872b71f7b61cb32" + integrity sha512-RPMAFUJP19WIet/99ngh6Iv8fzAbqum4Li7AD6DtGaW2RpMB/11xDoalPiJMTbu6I3hkbMVkATvZrqb9EEqeeQ== + dependencies: + bytes "3.1.1" + http-errors "1.8.1" + iconv-lite "0.4.24" + unpipe "1.0.0" + +readable-stream@^2.0.6, readable-stream@^2.2.2: + version "2.3.7" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.7.tgz#1eca1cf711aef814c04f62252a36a62f6cb23b57" + integrity sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw== + dependencies: + core-util-is "~1.0.0" + inherits "~2.0.3" + isarray "~1.0.0" + process-nextick-args "~2.0.0" + safe-buffer "~5.1.1" + string_decoder "~1.1.1" + util-deprecate "~1.0.1" + +request@^2.74.0: + version "2.88.2" + resolved "https://registry.yarnpkg.com/request/-/request-2.88.2.tgz#d73c918731cb5a87da047e207234146f664d12b3" + integrity sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw== + dependencies: + aws-sign2 "~0.7.0" + aws4 "^1.8.0" + caseless "~0.12.0" + combined-stream "~1.0.6" + extend "~3.0.2" + forever-agent "~0.6.1" + form-data "~2.3.2" + har-validator "~5.1.3" + http-signature "~1.2.0" + is-typedarray "~1.0.0" + isstream "~0.1.2" + json-stringify-safe "~5.0.1" + mime-types "~2.1.19" + oauth-sign "~0.9.0" + performance-now "^2.1.0" + qs "~6.5.2" + safe-buffer "^5.1.2" + tough-cookie "~2.5.0" + tunnel-agent "^0.6.0" + uuid "^3.3.2" + +resolve@^1.10.0: + version "1.22.0" + resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.22.0.tgz#5e0b8c67c15df57a89bdbabe603a002f21731198" + integrity sha512-Hhtrw0nLeSrFQ7phPp4OOcVjLPIeMnRlr5mcnVuMe7M/7eBn98A3hmFRLoFo3DLZkivSYwhRUJTyPyWAk56WLw== + dependencies: + is-core-module "^2.8.1" + path-parse "^1.0.7" + supports-preserve-symlinks-flag "^1.0.0" + +retry@^0.10.0: + version "0.10.1" + resolved "https://registry.yarnpkg.com/retry/-/retry-0.10.1.tgz#e76388d217992c252750241d3d3956fed98d8ff4" + integrity sha1-52OI0heZLCUnUCQdPTlW/tmNj/Q= + +rimraf@~2.2.0: + version "2.2.8" + resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.2.8.tgz#e439be2aaee327321952730f99a8929e4fc50582" + integrity sha1-5Dm+Kq7jJzIZUnMPmaiSnk/FBYI= + +rimraf@~2.6.2: + version "2.6.3" + resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.6.3.tgz#b2d104fe0d8fb27cf9e0a1cda8262dd3833c6cab" + integrity sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA== + dependencies: + glob "^7.1.3" + +safe-buffer@5.2.1, safe-buffer@^5.0.1, safe-buffer@^5.1.1, safe-buffer@^5.1.2: + version "5.2.1" + resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6" + integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== + +safe-buffer@~5.1.0, safe-buffer@~5.1.1: + version "5.1.2" + resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" + integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g== + +"safer-buffer@>= 2.1.2 < 3", safer-buffer@^2.0.2, safer-buffer@^2.1.0, safer-buffer@~2.1.0: + version "2.1.2" + resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" + integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== + +"semver@2 >=2.2.1 || 3.x || 4 || 5", "semver@2 || 3 || 4 || 5", semver@^5.6.0: + version "5.7.1" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.1.tgz#a954f931aeba508d307bbf069eff0c01c96116f7" + integrity sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ== + +"semver@2 || 3 || 4": + version "4.3.6" + resolved "https://registry.yarnpkg.com/semver/-/semver-4.3.6.tgz#300bc6e0e86374f7ba61068b5b1ecd57fc6532da" + integrity sha1-MAvG4OhjdPe6YQaLWx7NV/xlMto= + +semver@7.3.x: + version "7.3.5" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.5.tgz#0b621c879348d8998e4b0e4be94b3f12e6018ef7" + integrity sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ== + dependencies: + lru-cache "^6.0.0" + +send@0.17.2: + version "0.17.2" + resolved "https://registry.yarnpkg.com/send/-/send-0.17.2.tgz#926622f76601c41808012c8bf1688fe3906f7820" + integrity sha512-UJYB6wFSJE3G00nEivR5rgWp8c2xXvJ3OPWPhmuteU0IKj8nKbG3DrjiOmLwpnHGYWAVwA69zmTm++YG0Hmwww== + dependencies: + debug "2.6.9" + depd "~1.1.2" + destroy "~1.0.4" + encodeurl "~1.0.2" + escape-html "~1.0.3" + etag "~1.8.1" + fresh "0.5.2" + http-errors "1.8.1" + mime "1.6.0" + ms "2.1.3" + on-finished "~2.3.0" + range-parser "~1.2.1" + statuses "~1.5.0" + +serve-static@1.14.2: + version "1.14.2" + resolved "https://registry.yarnpkg.com/serve-static/-/serve-static-1.14.2.tgz#722d6294b1d62626d41b43a013ece4598d292bfa" + integrity sha512-+TMNA9AFxUEGuC0z2mevogSnn9MXKb4fa7ngeRMJaaGv8vTwnIEkKi+QGvPt33HSnf8pRS+WGM0EbMtCJLKMBQ== + dependencies: + encodeurl "~1.0.2" + escape-html "~1.0.3" + parseurl "~1.3.3" + send "0.17.2" + +set-blocking@~2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/set-blocking/-/set-blocking-2.0.0.tgz#045f9782d011ae9a6803ddd382b24392b3d890f7" + integrity sha1-BF+XgtARrppoA93TgrJDkrPYkPc= + +setprototypeof@1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.2.0.tgz#66c9a24a73f9fc28cbe66b09fed3d33dcaf1b424" + integrity sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw== + +signal-exit@^3.0.0: + version "3.0.6" + resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.6.tgz#24e630c4b0f03fea446a2bd299e62b4a6ca8d0af" + integrity sha512-sDl4qMFpijcGw22U5w63KmD3cZJfBuFlVNbVMKje2keoKML7X2UzWbc4XrmEbDwg0NXJc3yv4/ox7b+JWb57kQ== + +slasp@0.0.4: + version "0.0.4" + resolved "https://registry.yarnpkg.com/slasp/-/slasp-0.0.4.tgz#9adc26ee729a0f95095851a5489f87a5258d57a9" + integrity sha1-mtwm7nKaD5UJWFGlSJ+HpSWNV6k= + +slide@^1.1.3: + version "1.1.6" + resolved "https://registry.yarnpkg.com/slide/-/slide-1.1.6.tgz#56eb027d65b4d2dce6cb2e2d32c4d4afc9e1d707" + integrity sha1-VusCfWW00tzmyy4tMsTUr8nh1wc= + +spdx-correct@^3.0.0: + version "3.1.1" + resolved "https://registry.yarnpkg.com/spdx-correct/-/spdx-correct-3.1.1.tgz#dece81ac9c1e6713e5f7d1b6f17d468fa53d89a9" + integrity sha512-cOYcUWwhCuHCXi49RhFRCyJEK3iPj1Ziz9DpViV3tbZOwXD49QzIN3MpOLJNxh2qwq2lJJZaKMVw9qNi4jTC0w== + dependencies: + spdx-expression-parse "^3.0.0" + spdx-license-ids "^3.0.0" + +spdx-exceptions@^2.1.0: + version "2.3.0" + resolved "https://registry.yarnpkg.com/spdx-exceptions/-/spdx-exceptions-2.3.0.tgz#3f28ce1a77a00372683eade4a433183527a2163d" + integrity sha512-/tTrYOC7PPI1nUAgx34hUpqXuyJG+DTHJTnIULG4rDygi4xu/tfgmq1e1cIRwRzwZgo4NLySi+ricLkZkw4i5A== + +spdx-expression-parse@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/spdx-expression-parse/-/spdx-expression-parse-3.0.1.tgz#cf70f50482eefdc98e3ce0a6833e4a53ceeba679" + integrity sha512-cbqHunsQWnJNE6KhVSMsMeH5H/L9EpymbzqTQ3uLwNCLZ1Q481oWaofqH7nO6V07xlXwY6PhQdQ2IedWx/ZK4Q== + dependencies: + spdx-exceptions "^2.1.0" + spdx-license-ids "^3.0.0" + +spdx-license-ids@3.0.x, spdx-license-ids@^3.0.0: + version "3.0.11" + resolved "https://registry.yarnpkg.com/spdx-license-ids/-/spdx-license-ids-3.0.11.tgz#50c0d8c40a14ec1bf449bae69a0ea4685a9d9f95" + integrity sha512-Ctl2BrFiM0X3MANYgj3CkygxhRmr9mi6xhejbdO960nF6EDJApTYpn0BQnDKlnNBULKiCN1n3w9EBkHK8ZWg+g== + +sshpk@^1.7.0: + version "1.17.0" + resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.17.0.tgz#578082d92d4fe612b13007496e543fa0fbcbe4c5" + integrity sha512-/9HIEs1ZXGhSPE8X6Ccm7Nam1z8KcoCqPdI7ecm1N33EzAetWahvQWVqLZtaZQ+IDKX4IyA2o0gBzqIMkAagHQ== + dependencies: + asn1 "~0.2.3" + assert-plus "^1.0.0" + bcrypt-pbkdf "^1.0.0" + dashdash "^1.12.0" + ecc-jsbn "~0.1.1" + getpass "^0.1.1" + jsbn "~0.1.0" + safer-buffer "^2.0.2" + tweetnacl "~0.14.0" + +ssri@^5.2.4: + version "5.3.0" + resolved "https://registry.yarnpkg.com/ssri/-/ssri-5.3.0.tgz#ba3872c9c6d33a0704a7d71ff045e5ec48999d06" + integrity sha512-XRSIPqLij52MtgoQavH/x/dU1qVKtWUAAZeOHsR9c2Ddi4XerFy3mc1alf+dLJKl9EUIm/Ht+EowFkTUOA6GAQ== + dependencies: + safe-buffer "^5.1.1" + +"statuses@>= 1.5.0 < 2", statuses@~1.5.0: + version "1.5.0" + resolved "https://registry.yarnpkg.com/statuses/-/statuses-1.5.0.tgz#161c7dac177659fd9811f43771fa99381478628c" + integrity sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow= + +string-width@^1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-1.0.2.tgz#118bdf5b8cdc51a2a7e70d211e07e2b0b9b107d3" + integrity sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M= + dependencies: + code-point-at "^1.0.0" + is-fullwidth-code-point "^1.0.0" + strip-ansi "^3.0.0" + +"string-width@^1.0.2 || 2 || 3 || 4": + version "4.2.3" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" + integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== + dependencies: + emoji-regex "^8.0.0" + is-fullwidth-code-point "^3.0.0" + strip-ansi "^6.0.1" + +string_decoder@~1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.1.1.tgz#9cf1611ba62685d7030ae9e4ba34149c3af03fc8" + integrity sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg== + dependencies: + safe-buffer "~5.1.0" + +strip-ansi@^3.0.0, strip-ansi@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-3.0.1.tgz#6a385fb8853d952d5ff05d0e8aaf94278dc63dcf" + integrity sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8= + dependencies: + ansi-regex "^2.0.0" + +strip-ansi@^6.0.1: + version "6.0.1" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" + integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== + dependencies: + ansi-regex "^5.0.1" + +supports-preserve-symlinks-flag@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz#6eda4bd344a3c94aea376d4cc31bc77311039e09" + integrity sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w== + +tar@6.1.x: + version "6.1.11" + resolved "https://registry.yarnpkg.com/tar/-/tar-6.1.11.tgz#6760a38f003afa1b2ffd0ffe9e9abbd0eab3d621" + integrity sha512-an/KZQzQUkZCkuoAA64hM92X0Urb6VpRhAFllDzz44U2mcD5scmT3zBc4VgVpkugF580+DQn8eAFSyoQt0tznA== + dependencies: + chownr "^2.0.0" + fs-minipass "^2.0.0" + minipass "^3.0.0" + minizlib "^2.1.1" + mkdirp "^1.0.3" + yallist "^4.0.0" + +temp@0.9.x: + version "0.9.4" + resolved "https://registry.yarnpkg.com/temp/-/temp-0.9.4.tgz#cd20a8580cb63635d0e4e9d4bd989d44286e7620" + integrity sha512-yYrrsWnrXMcdsnu/7YMYAofM1ktpL5By7vZhf15CrXijWWrEYZks5AXBudalfSWJLlnen/QUJUB5aoB0kqZUGA== + dependencies: + mkdirp "^0.5.1" + rimraf "~2.6.2" + +toidentifier@1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/toidentifier/-/toidentifier-1.0.1.tgz#3be34321a88a820ed1bd80dfaa33e479fbb8dd35" + integrity sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA== + +tough-cookie@~2.5.0: + version "2.5.0" + resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.5.0.tgz#cd9fb2a0aa1d5a12b473bd9fb96fa3dcff65ade2" + integrity sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g== + dependencies: + psl "^1.1.28" + punycode "^2.1.1" + +tr46@~0.0.3: + version "0.0.3" + resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a" + integrity sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o= + +tunnel-agent@^0.6.0: + version "0.6.0" + resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.6.0.tgz#27a5dea06b36b04a0a9966774b290868f0fc40fd" + integrity sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0= + dependencies: + safe-buffer "^5.0.1" + +tweetnacl@^0.14.3, tweetnacl@~0.14.0: + version "0.14.5" + resolved "https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64" + integrity sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q= + +type-is@~1.6.18: + version "1.6.18" + resolved "https://registry.yarnpkg.com/type-is/-/type-is-1.6.18.tgz#4e552cd05df09467dcbc4ef739de89f2cf37c131" + integrity sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g== + dependencies: + media-typer "0.3.0" + mime-types "~2.1.24" + +typedarray@^0.0.6: + version "0.0.6" + resolved "https://registry.yarnpkg.com/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777" + integrity sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c= + +uid-number@0.0.5: + version "0.0.5" + resolved "https://registry.yarnpkg.com/uid-number/-/uid-number-0.0.5.tgz#5a3db23ef5dbd55b81fce0ec9a2ac6fccdebb81e" + integrity sha1-Wj2yPvXb1VuB/ODsmirG/M3ruB4= + +unpipe@1.0.0, unpipe@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec" + integrity sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw= + +uri-js@^4.2.2: + version "4.4.1" + resolved "https://registry.yarnpkg.com/uri-js/-/uri-js-4.4.1.tgz#9b1a52595225859e55f669d928f88c6c57f2a77e" + integrity sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg== + dependencies: + punycode "^2.1.0" + +util-deprecate@~1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" + integrity sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8= + +utils-merge@1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713" + integrity sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM= + +uuid@^3.3.2: + version "3.4.0" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee" + integrity sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A== + +uuid@^8.3.2: + version "8.3.2" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2" + integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg== + +validate-npm-package-license@^3.0.1: + version "3.0.4" + resolved "https://registry.yarnpkg.com/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz#fc91f6b9c7ba15c857f4cb2c5defeec39d4f410a" + integrity sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew== + dependencies: + spdx-correct "^3.0.0" + spdx-expression-parse "^3.0.0" + +validate-npm-package-name@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/validate-npm-package-name/-/validate-npm-package-name-3.0.0.tgz#5fa912d81eb7d0c74afc140de7317f0ca7df437e" + integrity sha1-X6kS2B630MdK/BQN5zF/DKffQ34= + dependencies: + builtins "^1.0.3" + +vary@~1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/vary/-/vary-1.1.2.tgz#2299f02c6ded30d4a5961b0b9f74524a18f634fc" + integrity sha1-IpnwLG3tMNSllhsLn3RSShj2NPw= + +verror@1.10.0: + version "1.10.0" + resolved "https://registry.yarnpkg.com/verror/-/verror-1.10.0.tgz#3a105ca17053af55d6e270c1f8288682e18da400" + integrity sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA= + dependencies: + assert-plus "^1.0.0" + core-util-is "1.0.2" + extsprintf "^1.2.0" + +"waitpid@git://github.com/HexF/node-waitpid.git": + version "0.1.1" + resolved "git://github.com/HexF/node-waitpid.git#a08d116a5d993a747624fe72ff890167be8c34aa" + +walk@^2.3.9: + version "2.3.15" + resolved "https://registry.yarnpkg.com/walk/-/walk-2.3.15.tgz#1b4611e959d656426bc521e2da5db3acecae2424" + integrity sha512-4eRTBZljBfIISK1Vnt69Gvr2w/wc3U6Vtrw7qiN5iqYJPH7LElcYh/iU4XWhdCy2dZqv1ToMyYlybDylfG/5Vg== + dependencies: + foreachasync "^3.0.0" + +webidl-conversions@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871" + integrity sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE= + +whatwg-url@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d" + integrity sha1-lmRU6HZUYuN2RNNib2dCzotwll0= + dependencies: + tr46 "~0.0.3" + webidl-conversions "^3.0.0" + +wide-align@^1.1.0: + version "1.1.5" + resolved "https://registry.yarnpkg.com/wide-align/-/wide-align-1.1.5.tgz#df1d4c206854369ecf3c9a4898f1b23fbd9d15d3" + integrity sha512-eDMORYaPNZ4sQIuuYPDHdQvf4gyCF9rEEV/yPxGfwPkRodwEgiMUUXTx/dex+Me0wxx53S+NgUHaP7y3MGlDmg== + dependencies: + string-width "^1.0.2 || 2 || 3 || 4" + +wrappy@1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" + integrity sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8= + +ws@^7.4.6: + version "7.5.6" + resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.6.tgz#e59fc509fb15ddfb65487ee9765c5a51dec5fe7b" + integrity sha512-6GLgCqo2cy2A2rjCNFlxQS6ZljG/coZfZXclldI8FB/1G3CCI36Zd8xy2HrFVACi8tfk5XrgLQEk+P0Tnz9UcA== + +yallist@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72" + integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== diff --git a/builder/.gitignore b/builder/.gitignore deleted file mode 100644 index c795b05..0000000 --- a/builder/.gitignore +++ /dev/null @@ -1 +0,0 @@ -build \ No newline at end of file diff --git a/builder/Dockerfile b/builder/Dockerfile deleted file mode 100644 index a769f35..0000000 --- a/builder/Dockerfile +++ /dev/null @@ -1,2 +0,0 @@ -FROM ghcr.io/engineer-man/piston:latest -ADD . /piston/packages/ \ No newline at end of file diff --git a/builder/build.sh b/builder/build.sh deleted file mode 100755 index 8559eaf..0000000 --- a/builder/build.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env bash - -# Build a container using the spec file provided - -help_msg(){ - echo "Usage: $0 [specfile] [tag]" - echo - echo "$1" - - exit 1 -} - -cleanup(){ - echo "Exiting..." - docker stop builder_piston_instance && docker rm builder_piston_instance -} - -fetch_packages(){ - local port=$((5535 + $RANDOM % 60000)) - mkdir build - # Start a piston container - docker run \ - -v "$PWD/build":'/piston/packages' \ - --tmpfs /piston/jobs \ - -dit \ - -p $port:2000 \ - --name builder_piston_instance \ - ghcr.io/engineer-man/piston - - # Ensure the CLI is installed - cd ../cli - npm i - cd - - - # Evalulate the specfile - ../cli/index.js -u "http://127.0.0.1:$port" ppman spec $1 -} - -build_container(){ - docker build -t $1 -f "$(dirname $0)/Dockerfile" "$PWD/build" -} - - -SPEC_FILE=$1 -TAG=$2 - -[ -z "$SPEC_FILE" ] && help_msg "specfile is required" -[ -z "$TAG" ] && help_msg "tag is required" - -[ -f "$SPEC_FILE" ] || help_msg "specfile does not exist" - -which node || help_msg "nodejs is required" -which npm || help_msg "npm is required" - -trap cleanup EXIT - -fetch_packages $SPEC_FILE -build_container $TAG - -echo "Start your custom piston container with" -echo "$ docker run --tmpfs /piston/jobs -dit -p 2000:2000 $TAG" \ No newline at end of file diff --git a/cli/commands/ppman.js b/cli/commands/ppman.js deleted file mode 100644 index ad2c879..0000000 --- a/cli/commands/ppman.js +++ /dev/null @@ -1,5 +0,0 @@ -exports.command = 'ppman'; -exports.aliases = ['pkg']; -exports.describe = 'Package Manager'; - -exports.builder = yargs => yargs.commandDir('ppman_commands').demandCommand(); diff --git a/cli/commands/ppman_commands/install.js b/cli/commands/ppman_commands/install.js deleted file mode 100644 index a47665d..0000000 --- a/cli/commands/ppman_commands/install.js +++ /dev/null @@ -1,39 +0,0 @@ -const chalk = require('chalk'); - -exports.command = ['install ']; -exports.aliases = ['i']; -exports.describe = 'Installs the named package'; - -//Splits the package into it's language and version -function split_package(package) { - [language, language_version] = package.split('='); - - res = { - language: language, - version: language_version || '*', - }; - return res; -} - -const msg_format = { - color: p => - `${ - p.language ? chalk.green.bold('✓') : chalk.red.bold('❌') - } Installation ${p.language ? 'succeeded' : 'failed: ' + p.message}`, - monochrome: p => - `Installation ${p.language ? 'succeeded' : 'failed: ' + p.message}`, - json: JSON.stringify, -}; - -exports.handler = async ({ axios, packages }) => { - const requests = packages.map(package => split_package(package)); - for (request of requests) { - try { - const install = await axios.post(`/api/v2/packages`, request); - - console.log(msg_format.color(install.data)); - } catch ({ response }) { - console.error(response.data.message); - } - } -}; diff --git a/cli/commands/ppman_commands/list.js b/cli/commands/ppman_commands/list.js deleted file mode 100644 index 0b09667..0000000 --- a/cli/commands/ppman_commands/list.js +++ /dev/null @@ -1,25 +0,0 @@ -const chalk = require('chalk'); - -exports.command = ['list']; -exports.aliases = ['l']; -exports.describe = 'Lists all available packages'; - -const msg_format = { - color: p => - `${chalk[p.installed ? 'green' : 'red']('•')} ${p.language} ${ - p.language_version - }`, - monochrome: p => - `${p.language} ${p.language_version} ${ - p.installed ? '(INSTALLED)' : '' - }`, - json: JSON.stringify, -}; - -exports.handler = async ({ axios }) => { - const packages = await axios.get('/api/v2/packages'); - - const pkg_msg = packages.data.map(msg_format.color).join('\n'); - - console.log(pkg_msg); -}; diff --git a/cli/commands/ppman_commands/spec.js b/cli/commands/ppman_commands/spec.js deleted file mode 100644 index 9c04857..0000000 --- a/cli/commands/ppman_commands/spec.js +++ /dev/null @@ -1,183 +0,0 @@ -const chalk = require('chalk'); -const fs = require('fs/promises'); -const minimatch = require('minimatch'); -const semver = require('semver'); - -exports.command = ['spec ']; -exports.aliases = ['s']; -exports.describe = - "Install the packages described in the spec file, uninstalling packages which aren't in the list"; - -function does_match(package, rule) { - const nameMatch = minimatch(package.language, rule.package_selector); - const versionMatch = semver.satisfies( - package.language_version, - rule.version_selector - ); - - return nameMatch && versionMatch; -} - -exports.handler = async ({ axios, specfile }) => { - const spec_contents = await fs.readFile(specfile); - const spec_lines = spec_contents.toString().split('\n'); - - const rules = []; - - for (const line of spec_lines) { - const rule = { - _raw: line.trim(), - comment: false, - package_selector: null, - version_selector: null, - negate: false, - }; - - if (line.starts_with('#')) { - rule.comment = true; - } else { - let l = line.trim(); - if (line.starts_with('!')) { - rule.negate = true; - l = line.slice(1).trim(); - } - - const [pkg, ver] = l.split(' ', 2); - rule.package_selector = pkg; - rule.version_selector = ver; - } - - if (rule._raw.length != 0) rules.push(rule); - } - - const packages_req = await axios.get('/api/v2/packages'); - const packages = packages_req.data; - - const installed = packages.filter(pkg => pkg.installed); - - let ensure_packages = []; - - for (const rule of rules) { - if (rule.comment) continue; - - const matches = []; - - if (!rule.negate) { - for (const package of packages) { - if (does_match(package, rule)) matches.push(package); - } - - const latest_matches = matches.filter(pkg => { - const versions = matches - .filter(x => x.language == pkg.language) - .map(x => x.language_version) - .sort(semver.rcompare); - return versions[0] == pkg.language_version; - }); - - for (const match of latest_matches) { - if ( - !ensure_packages.find( - pkg => - pkg.language == match.language && - pkg.language_version == match.language_version - ) - ) - ensure_packages.push(match); - } - } else { - ensure_packages = ensure_packages.filter( - pkg => !does_match(pkg, rule) - ); - } - } - - const operations = []; - - for (const package of ensure_packages) { - if (!package.installed) - operations.push({ - type: 'install', - package: package.language, - version: package.language_version, - }); - } - - for (const installed_package of installed) { - if ( - !ensure_packages.find( - pkg => - pkg.language == installed_package.language && - pkg.language_version == installed_package.language_version - ) - ) - operations.push({ - type: 'uninstall', - package: installed_package.language, - version: installed_package.language_version, - }); - } - - console.log(chalk.bold.yellow('Actions')); - for (const op of operations) { - console.log( - (op.type == 'install' - ? chalk.green('Install') - : chalk.red('Uninstall')) + ` ${op.package} ${op.version}` - ); - } - - if (operations.length == 0) { - console.log(chalk.gray('None')); - } - - for (const op of operations) { - if (op.type == 'install') { - try { - const install = await axios.post(`/api/v2/packages`, { - language: op.package, - version: op.version, - }); - - if (!install.data.language) - throw new Error(install.data.message); // Go to exception handler - - console.log( - chalk.bold.green('Installed'), - op.package, - op.version - ); - } catch (e) { - console.log( - chalk.bold.red('Failed to install') + - ` ${op.package} ${op.version}:`, - e.message - ); - } - } else if (op.type == 'uninstall') { - try { - const install = await axios.delete(`/api/v2/packages`, { - data: { - language: op.package, - version: op.version, - }, - }); - - if (!install.data.language) - throw new Error(install.data.message); // Go to exception handler - - console.log( - chalk.bold.green('Uninstalled'), - op.package, - op.version - ); - } catch (e) { - console.log( - chalk.bold.red('Failed to uninstall') + - ` ${op.package} ${op.version}:`, - e.message - ); - } - } - } -}; diff --git a/cli/commands/ppman_commands/uninstall.js b/cli/commands/ppman_commands/uninstall.js deleted file mode 100644 index 21d2198..0000000 --- a/cli/commands/ppman_commands/uninstall.js +++ /dev/null @@ -1,41 +0,0 @@ -const chalk = require('chalk'); - -exports.command = ['uninstall ']; -exports.aliases = ['u']; -exports.describe = 'Uninstalls the named package'; - -//Splits the package into it's language and version -function split_package(package) { - [language, language_version] = package.split('='); - - res = { - language: language, - version: language_version || '*', - }; - return res; -} - -const msg_format = { - color: p => - `${ - p.language ? chalk.green.bold('✓') : chalk.red.bold('❌') - } Uninstallation ${p.language ? 'succeeded' : 'failed: ' + p.message}`, - monochrome: p => - `Uninstallation ${p.language ? 'succeeded' : 'failed: ' + p.message}`, - json: JSON.stringify, -}; - -exports.handler = async ({ axios, packages }) => { - const requests = packages.map(package => split_package(package)); - for (request of requests) { - try { - const uninstall = await axios.delete(`/api/v2/packages`, { - data: request, - }); - - console.log(msg_format.color(uninstall.data)); - } catch ({ response }) { - console.error(response.data.message); - } - } -}; diff --git a/dev.pps b/dev.pps deleted file mode 100644 index b2e5a8c..0000000 --- a/dev.pps +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env -S piston ppman spec - -# Development Piston Packages -# Defines packages to be installed by developers - -# All packages, latest version -# Don't use this when connected to public repo, in excess of 10GB -* * diff --git a/docker-compose.dev.yaml b/docker-compose.dev.yaml deleted file mode 100644 index e69de29..0000000 diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..17242b3 --- /dev/null +++ b/flake.lock @@ -0,0 +1,26 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1643456774, + "narHash": "sha256-abP2nVe3bsndDQgkGxoLdBqHRzisYJSO6cwdEi+AMVc=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "5cf5cad0da6244da30be1b6da2ff3d44b6f3ebe5", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..cce2f11 --- /dev/null +++ b/flake.nix @@ -0,0 +1,118 @@ +{ + description = "Piston packages repo"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs"; + + outputs = { self, nixpkgs }: + let + system = "x86_64-linux"; + pkgs = nixpkgs.legacyPackages.${system}; + args = { + inherit pkgs; + piston = { + mkRuntime = { + language, + version, + runtime? null, + run, + compile? null, + packages? null, + aliases? [], + tests + }: let + compileFile = if compile != null then + pkgs.writeShellScript "compile" compile + else null; + runFile = pkgs.writeShellScript "run" run; + metadata = { + inherit language version runtime aliases; + run = runFile; + compile = compileFile; + packageSupport = packages != null; + }; + in { + inherit packages metadata; + tests = if (builtins.length tests) > 0 then + tests + else abort "Language ${language} doesn't provide any tests"; + }; + mkTest = { + files, + args? [], + stdin? "", + packages? [], + main? null + }: { + inherit files args stdin packages; + main = if main == null then + ( + if (builtins.length (builtins.attrNames files)) == 1 then + (builtins.head (builtins.attrNames files)) + else abort "Could not determine the main file for test - specify it using the 'main' parameter" + ) + else main; + }; + }; + }; + allRuntimes = import ./runtimes args; + in { + piston = args.piston; + pistonRuntimes = { + "bash" = allRuntimes.bash; + }; + + legacyPackages."${system}" = { + piston = (import ./api { inherit pkgs; }).package; + nosocket = (import ./nosocket { inherit pkgs; }).package; + }; + + containerImage = pkgs.dockerTools.buildLayeredImageWithNixDb { + name = "piston"; + tag = "latest"; + + contents = with pkgs; [ + self.legacyPackages."${system}".piston + self.legacyPackages."${system}".nosocket + bash + nixFlakes + coreutils-full + cacert.out + git + gnutar + gzip + gnugrep + util-linux + ]; + + extraCommands = '' + mkdir -p piston/jobs etc/nix {,var/}tmp run/lock + echo -e "experimental-features = nix-command flakes" >> etc/nix/nix.conf + echo "nixbld:x:30000:nixbld1,nixbld10,nixbld11,nixbld12,nixbld13,nixbld14,nixbld15,nixbld16,nixbld17,nixbld18,nixbld19,nixbld2,nixbld20,nixbld21,nixbld22,nixbld23,nixbld24,nixbld25,nixbld26,nixbld27,nixbld28,nixbld29,nixbld3,nixbld30,nixbld31,nixbld32,nixbld4,nixbld5,nixbld6,nixbld7,nixbld8,nixbld9" >> etc/group + for i in $(seq 1 32) + do + echo "nixbld$i:x:$(( $i + 30000 )):30000:Nix build user $i:/var/empty:/run/current-system/sw/bin/nologin" >> etc/passwd + done + ''; + + config = { + Cmd = [ + "${self.legacyPackages."${system}".piston}/bin/pistond" + ]; + + Env = [ + "NIX_PAGER=cat" + "USER=nobody" + "SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt" + "GIT_SSL_CAINFO=/etc/ssl/certs/ca-bundle.crt" + "NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt" + + ]; + + ExposedPorts = { + "2000/tcp" = {}; + }; + }; + }; + + + }; +} diff --git a/api/src/nosocket/Makefile b/nosocket/Makefile similarity index 100% rename from api/src/nosocket/Makefile rename to nosocket/Makefile diff --git a/nosocket/default.nix b/nosocket/default.nix new file mode 100644 index 0000000..35efc18 --- /dev/null +++ b/nosocket/default.nix @@ -0,0 +1,24 @@ +{pkgs, ...}: +with pkgs; { + package = stdenv.mkDerivation { + name = "nosocket-1.0.0"; + + dontUnpack = true; + + + src = ./nosocket.c; + + buildInputs = [ + libseccomp + ]; + + buildPhase = '' + gcc $src -O2 -Wall -lseccomp -o nosocket + ''; + + installPhase = '' + mkdir -p $out/bin + cp nosocket $out/bin + ''; + }; +} \ No newline at end of file diff --git a/api/src/nosocket/nosocket.c b/nosocket/nosocket.c similarity index 100% rename from api/src/nosocket/nosocket.c rename to nosocket/nosocket.c diff --git a/package-lock.json b/package-lock.json deleted file mode 100644 index 5c51a1d..0000000 --- a/package-lock.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "name": "piston", - "lockfileVersion": 2, - "requires": true, - "packages": { - "": { - "devDependencies": { - "prettier": "2.4.1" - } - }, - "node_modules/prettier": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.4.1.tgz", - "integrity": "sha512-9fbDAXSBcc6Bs1mZrDYb3XKzDLm4EXXL9sC1LqKP5rZkT6KRr/rf9amVUcODVXgguK/isJz0d0hP72WeaKWsvA==", - "dev": true, - "bin": { - "prettier": "bin-prettier.js" - }, - "engines": { - "node": ">=10.13.0" - } - } - }, - "dependencies": { - "prettier": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.4.1.tgz", - "integrity": "sha512-9fbDAXSBcc6Bs1mZrDYb3XKzDLm4EXXL9sC1LqKP5rZkT6KRr/rf9amVUcODVXgguK/isJz0d0hP72WeaKWsvA==", - "dev": true - } - } -} diff --git a/package.json b/package.json deleted file mode 100644 index 8f07606..0000000 --- a/package.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "devDependencies": { - "prettier": "2.4.1" - } -} diff --git a/packages/.gitignore b/packages/.gitignore deleted file mode 100644 index d98ab0b..0000000 --- a/packages/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -*/*/* -*.pkg.tar.gz -!*/*/metadata.json -!*/*/build.sh -!*/*/environment -!*/*/run -!*/*/compile -!*/*/test.* \ No newline at end of file diff --git a/packages/CONTRIBUTING.MD b/packages/CONTRIBUTING.MD deleted file mode 100644 index 53deda9..0000000 --- a/packages/CONTRIBUTING.MD +++ /dev/null @@ -1,100 +0,0 @@ -# Contributing packages to the Piston Repository - -## Naming Languages - -Languages should be named after their interpreters, and the command line binaries you call. The language version should use semantic versioning. -For example, the full name of the standard python interpreter is `CPython`, however we would name it `python`, after the main binary which it provides. -In the example of NodeJS, we would call this `node`, after the main binary. - -## Creating new languages - -See [deno/1.7.5/](deno/1.7.5/) or any other directory for examples. - -1. Create a new branch on your fork of engineer-man/piston - -2. Create directories named `[language]/[version]`. See Naming Languages for how to determine the name for your language - -3. Create a file named `build.sh`, adding a shebang for bash `#!/bin/bash` on the first line. -In this file put any steps to compile the specified langauge. -This script should download sources, compile sources and output binaries. They should be dumped into the current working directory, removing any files which aren't required in the process. - -4. Create a file named `run`, containing bash script to run the interpreter. -The first argument given to this script (`$1`) is the name of the main file, with the remaining ones as program arguments. -STDIN is piped directly into the run file, and as such nothing special is required to deal with STDIN, except leaving it open. - -5. Create a file named `compile`, containing bash script to compile sources into binaries. This is only required if the language requires a compling stage. -The first argument is always the main file, followed the names of the other files as additional arguements. If the language does not require a compile stage, don't create a compile file. - -6. Create a file named `environment`, containing `export` statements which edit the environment variables accordingly. The `$PWD` variable should be used, and is set inside the package directory when running on the target system. - -7. Create a test script starting with test, with the file extension of the language. This script should simply output the phrase `OK`. For example, for mono we would create `test.cs` with the content: -```cs -using System; - -public class Test -{ - public static void Main(string[] args) - { - Console.WriteLine("OK"); - } -} -``` - -8. Create a `metadata.json` file which contains metadata about the language and interpreter. This simply contains the language name, as in the folder name, the version as in the folder name, aliases that can be used to call this package, limit overrides (if any) that can be used to override the default constraints and finally a dependencies map. -The dependencies map contains the keys as language names, and the values as semver selectors for packages. -```json -{ - "language": "deno", - "version": "1.7.5", - "dependencies": {}, - "aliases": ["deno-ts", "deno-js"] -} -``` -If the interpreter/compiler provides multiple languages, then the provides property should be used: -```json -{ - "language": "dotnet", - "version": "5.0.201", - "provides": [ - { - "language": "basic.net", - "aliases": [ - "basic", - "visual-basic", - "visual-basic.net", - "vb", - "vb.net", - "vb-dotnet", - "dotnet-vb", - "basic-dotnet", - "dotnet-basic" - ], - "limit_overrides": { "max_process_count": 128 } - }, - { - "language": "fsi", - "aliases": [ - "fsx", - "fsharp-interactive", - "f#-interactive", - "dotnet-fsi", - "fsi-dotnet", - "fsi.net" - ] - } - ] -} -``` - -9. Test your package builds with running `make [language]-[version].pkg.tar.gz`. -If it all goes to plan, you should have a file named `[language]-[version].pkg.tar.gz`, in this case you're good to go, albeit it is preferable to test the package locally as follows -```shell -./piston build-pkg [package] [version] -./piston ppman install [package]=[version] -./piston run [package] -l [version] packages/[package]/[version]/test.* -``` - -10. Commit your changes, using message format of `pkg([language]-[version]): Added [language] [version]` -Any additional commits regarding this package should start with `pkg([language]-[version]): ` - -11. Create a pull request (currently to v3 branch), referencing an Issue number (if there is one associated). diff --git a/packages/Makefile b/packages/Makefile deleted file mode 100644 index 22566f2..0000000 --- a/packages/Makefile +++ /dev/null @@ -1,25 +0,0 @@ -PACKAGES=$(subst /,-,$(shell find * -maxdepth 1 -mindepth 1 -type d)) -BUILD_PLATFORM=$(or ${PLATFORM},baremetal-$(shell grep -oP "^ID=\K.+" /etc/os-release)) - -help: - @echo "You probably don't want to build all package" - @echo "If you do run $`make build-all$`" - @echo - @echo "Run $`make [language]-[version].pkg.tar.gz$` to build a specific language" - -build build-all: $(addsuffix .pkg.tar.gz, ${PACKAGES}) - - -define PKG_RULE -$(1).pkg.tar.gz: $(subst -,/,$(1)) $(subst -,/,$(1))/pkg-info.json - cd $$< && chmod +x ./build.sh && ./build.sh - rm -f $$@ - - tar czf $$@ -C $$< . -endef - -$(foreach pkg,$(PACKAGES),$(eval $(call PKG_RULE,$(pkg)))) - -%/pkg-info.json: %/metadata.json - jq '.build_platform="${BUILD_PLATFORM}"' $< > $@ - diff --git a/packages/README.MD b/packages/README.MD deleted file mode 100644 index bcc3a93..0000000 --- a/packages/README.MD +++ /dev/null @@ -1,7 +0,0 @@ -# Piston Package Build Scripts - -## Building - -```bash -make build-[name]-[version] -``` diff --git a/packages/bash/5.1.0/build.sh b/packages/bash/5.1.0/build.sh deleted file mode 100755 index 657ae7a..0000000 --- a/packages/bash/5.1.0/build.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash - -# Put instructions to build your package in here -PREFIX=$(realpath $(dirname $0)) - -mkdir -p build - -cd build - -curl "https://ftp.gnu.org/gnu/bash/bash-5.1.tar.gz" -o bash.tar.gz - -tar xzf bash.tar.gz --strip-components=1 - -# === autoconf based === -./configure --prefix "$PREFIX" - -make -j$(nproc) -make install -j$(nproc) -cd ../ -rm -rf build - diff --git a/packages/bash/5.1.0/environment b/packages/bash/5.1.0/environment deleted file mode 100644 index 780b668..0000000 --- a/packages/bash/5.1.0/environment +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash - -# Put 'export' statements here for environment variables -export PATH=$PWD/bin:$PATH diff --git a/packages/bash/5.1.0/metadata.json b/packages/bash/5.1.0/metadata.json deleted file mode 100644 index 9ac1250..0000000 --- a/packages/bash/5.1.0/metadata.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "language": "bash", - "version": "5.1.0", - "aliases": ["sh"] -} diff --git a/packages/bash/5.1.0/run b/packages/bash/5.1.0/run deleted file mode 100644 index a8f3ffb..0000000 --- a/packages/bash/5.1.0/run +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash - -# Put instructions to run the runtime -bash "$@" diff --git a/packages/bash/5.1.0/test.bash.sh b/packages/bash/5.1.0/test.bash.sh deleted file mode 100644 index 727518f..0000000 --- a/packages/bash/5.1.0/test.bash.sh +++ /dev/null @@ -1 +0,0 @@ -echo "OK" \ No newline at end of file diff --git a/packages/clojure/1.10.3/build.sh b/packages/clojure/1.10.3/build.sh deleted file mode 100755 index 4633daf..0000000 --- a/packages/clojure/1.10.3/build.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash - -# Installation location -PREFIX=$(realpath $(dirname $0)) - -# Clojure depends on Java (build and runtime) -mkdir -p java -cd java -curl "https://download.java.net/java/GA/jdk15.0.2/0d1cfde4252546c6931946de8db48ee2/7/GPL/openjdk-15.0.2_linux-x64_bin.tar.gz" -o java.tar.gz -tar xzf java.tar.gz --strip-components=1 -rm java.tar.gz -cd .. - -# Clojure depends on Maven (build) -mkdir -p maven -cd maven -curl "https://apache.claz.org/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz" -o maven.tar.gz -tar xzf maven.tar.gz --strip-components=1 -rm maven.tar.gz -cd .. - -# Adding java and maven to the path for building -export PATH=$PWD/java/bin:$PWD/maven/bin:$PATH -export JAVA_HOME=$PWD/java - -# Clojure download -mkdir -p build -cd build -git clone -q "https://github.com/clojure/clojure.git" . -git checkout -b clojure-1.10.3 aaf73b12467df80f5db3e086550a33fee0e1b39e # commit for 1.10.3 release - -# Build using maven -mvn -Plocal -Dmaven.test.skip=true package - -# Get ridda that m2 bloat from Maven and remove Maven itself -cd ../ -rm -rf ~/.m2 -rm -rf maven/ - -# Move the jar for easier reference and cleanup -mkdir -p bin -mv build/clojure.jar bin -rm -rf build diff --git a/packages/clojure/1.10.3/environment b/packages/clojure/1.10.3/environment deleted file mode 100644 index e7ef3b1..0000000 --- a/packages/clojure/1.10.3/environment +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -# Clojure requires JAVA_HOME to be set and java binary to be in the path -export JAVA_HOME=$PWD/java -export CLOJURE_PATH=$PWD/bin -export PATH=$PWD/java/bin:$PATH diff --git a/packages/clojure/1.10.3/metadata.json b/packages/clojure/1.10.3/metadata.json deleted file mode 100644 index 847ecb6..0000000 --- a/packages/clojure/1.10.3/metadata.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "language": "clojure", - "version": "1.10.3", - "aliases": ["clojure", "clj"] -} diff --git a/packages/clojure/1.10.3/run b/packages/clojure/1.10.3/run deleted file mode 100644 index a8901af..0000000 --- a/packages/clojure/1.10.3/run +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash - -# Run clojure with Java referencing the clojure jar location -java -jar $CLOJURE_PATH/clojure.jar "$@" diff --git a/packages/clojure/1.10.3/test.clj b/packages/clojure/1.10.3/test.clj deleted file mode 100644 index 7f50202..0000000 --- a/packages/clojure/1.10.3/test.clj +++ /dev/null @@ -1,5 +0,0 @@ -(ns clojure.examples.main - (:gen-class)) -(defn main [] - (println "OK")) -(main) \ No newline at end of file diff --git a/packages/cobol/3.1.2/build.sh b/packages/cobol/3.1.2/build.sh deleted file mode 100755 index 1156fa8..0000000 --- a/packages/cobol/3.1.2/build.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash - -# Put instructions to build your package in here -PREFIX=$(realpath $(dirname $0)) - -mkdir -p build - -cd build - -curl -OL "https://downloads.sourceforge.net/project/gnucobol/gnucobol/3.1/gnucobol-3.1.2.tar.xz" - -tar xf gnucobol-3.1.2.tar.xz --strip-components=1 - -# === autoconf based === -./configure --prefix "$PREFIX" --without-db - -make -j$(nproc) -make install -j$(nproc) -cd ../ -rm -rf build diff --git a/packages/cobol/3.1.2/compile b/packages/cobol/3.1.2/compile deleted file mode 100755 index 051eb75..0000000 --- a/packages/cobol/3.1.2/compile +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash -cobc -o binary --free -x -L lib "$@" -chmod +x binary - diff --git a/packages/cobol/3.1.2/environment b/packages/cobol/3.1.2/environment deleted file mode 100644 index ca711d7..0000000 --- a/packages/cobol/3.1.2/environment +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash - -export PATH=$PWD/bin:$PATH -export LD_LIBRARY_PATH=$PWD/lib - diff --git a/packages/cobol/3.1.2/metadata.json b/packages/cobol/3.1.2/metadata.json deleted file mode 100644 index 0a80d3c..0000000 --- a/packages/cobol/3.1.2/metadata.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "language": "cobol", - "version": "3.1.2", - "aliases": ["cob"] -} diff --git a/packages/cobol/3.1.2/run b/packages/cobol/3.1.2/run deleted file mode 100755 index 9dcedfa..0000000 --- a/packages/cobol/3.1.2/run +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash - -shift -./binary "$@" - diff --git a/packages/cobol/3.1.2/test.cob b/packages/cobol/3.1.2/test.cob deleted file mode 100644 index 1a03e66..0000000 --- a/packages/cobol/3.1.2/test.cob +++ /dev/null @@ -1,8 +0,0 @@ -*> Test Program -identification division. -program-id. ok-test. - -procedure division. -display "OK" -goback. -end program ok-test. diff --git a/packages/flake.lock b/packages/flake.lock deleted file mode 100644 index 14753b7..0000000 --- a/packages/flake.lock +++ /dev/null @@ -1,41 +0,0 @@ -{ - "nodes": { - "flake-utils": { - "locked": { - "lastModified": 1631561581, - "narHash": "sha256-3VQMV5zvxaVLvqqUrNz3iJelLw30mIVSfZmAaauM3dA=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "7e5bf3925f6fbdfaf50a2a7ca0be2879c4261d19", - "type": "github" - }, - "original": { - "owner": "numtide", - "repo": "flake-utils", - "type": "github" - } - }, - "nixpkgs": { - "locked": { - "lastModified": 1633356775, - "narHash": "sha256-UBhRo1qy8xpOGTrjf7r2SFcULkFM+Wn4kchxN1ToDxs=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "38501bec61c1e9447aa4ffc01ba07c40f4515327", - "type": "github" - }, - "original": { - "id": "nixpkgs", - "type": "indirect" - } - }, - "root": { - "inputs": { - "flake-utils": "flake-utils", - "nixpkgs": "nixpkgs" - } - } - }, - "root": "root", - "version": 7 -} diff --git a/packages/flake.nix b/packages/flake.nix deleted file mode 100644 index 17a60d3..0000000 --- a/packages/flake.nix +++ /dev/null @@ -1,47 +0,0 @@ -{ - description = "Piston packages repo"; - inputs.flake-utils.url = "github:numtide/flake-utils"; - - outputs = { self, nixpkgs, flake-utils }: - let - system = "x86_64-linux"; - pkgs = nixpkgs.legacyPackages.${system}; - args = { - inherit pkgs; - piston = { - mkRuntime = { - language, - version, - runtime, - run, - compile? null, - aliases? [] - }: let - packageName = "${runtime}-${language}"; - compileFile = if compile != null then - pkgs.writeShellScript "compile" compile - else null; - runFile = pkgs.writeShellScript "run" run; - metadataFile = pkgs.writeText "metadata.json" (builtins.toJSON { - inherit language version runtime aliases; - }); - in pkgs.runCommandNoCC packageName {} - ( - '' - mkdir -p $out/piston - ln -s ${runFile} $out/piston/run - ln -s ${metadataFile} $out/piston/metadata.json - '' + ( - if compileFile != null then - '' - ln -s ${compileFile} $out/piston/compile - '' else "") - ); - }; - }; - in { - piston = { - "node-javascript" = import ./node-javascript.nix args; - }; - }; -} diff --git a/packages/init b/packages/init deleted file mode 100755 index 68a303b..0000000 --- a/packages/init +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env bash - -if [[ $# -lt 3 ]]; then - echo "Usage: $0 [name] [version] [source]" - echo "" - echo "Initializes an empty package" - exit 1 -fi - -NAME=$1 -VERSION=$2 -SOURCE=$3 - -DIR=$NAME/$VERSION - -mkdir -p $DIR - -build_instructions(){ - echo 'PREFIX=$(realpath $(dirname $0))' - echo - echo 'mkdir -p build' - echo - echo 'cd build' - echo - echo "curl \"$SOURCE\" -o $NAME.tar.gz" - echo - echo "tar xzf $NAME.tar.gz --strip-components=1" - echo - - echo "# === autoconf based ===" - echo './configure --prefix "$PREFIX"' - echo - echo 'make -j$(nproc)' - echo 'make install -j$(nproc)' - - echo 'cd ../' - echo 'rm -rf build' - -} - -cd $DIR - -for name in build.sh environment run compile; do - echo "#!/usr/bin/env bash" > "$name" - echo "" >> "$name" -done - -echo "# Put instructions to build your package in here" >> build.sh -echo "" -build_instructions >> build.sh - -echo "# Put 'export' statements here for environment variables" >> environment -echo "export PATH=\$PWD/bin:\$PATH" >> environment - -echo "# Put instructions to run the runtime" >> run -echo "$NAME-$VERSION \"\$@\"" >> run - -echo "# Put instructions to compile source code, remove this file if the language does not require this stage" >> compile - -jq '.language = "'$NAME'" | .version = "'$VERSION'" | .aliases = []' <<< "{}" > metadata.json - -cd - > /dev/null - -echo $DIR diff --git a/packages/node-javascript.nix b/packages/node-javascript.nix deleted file mode 100644 index 911e3c6..0000000 --- a/packages/node-javascript.nix +++ /dev/null @@ -1,17 +0,0 @@ -{pkgs, piston}: -piston.mkRuntime { - language = "javascript"; - version = pkgs.nodejs.version; - runtime = "node"; - - aliases = [ - "node-js" - "node-javascript" - "js" - ]; - - run = '' - ${pkgs.nodejs}/bin/node "$@" - ''; - -} \ No newline at end of file diff --git a/packages/node/15.10.0/build.sh b/packages/node/15.10.0/build.sh deleted file mode 100755 index 6a84798..0000000 --- a/packages/node/15.10.0/build.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -curl "https://nodejs.org/dist/v15.10.0/node-v15.10.0-linux-x64.tar.xz" -o node.tar.xz -tar xf node.tar.xz --strip-components=1 -rm node.tar.xz diff --git a/packages/node/15.10.0/environment b/packages/node/15.10.0/environment deleted file mode 100644 index bd0ff98..0000000 --- a/packages/node/15.10.0/environment +++ /dev/null @@ -1 +0,0 @@ -export PATH=$PWD/bin:$PATH \ No newline at end of file diff --git a/packages/node/15.10.0/metadata.json b/packages/node/15.10.0/metadata.json deleted file mode 100644 index 3675016..0000000 --- a/packages/node/15.10.0/metadata.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "language": "node", - "version": "15.10.0", - "provides": [ - { - "language": "javascript", - "aliases": ["node-javascript", "node-js", "javascript", "js"] - } - ] -} diff --git a/packages/node/15.10.0/run b/packages/node/15.10.0/run deleted file mode 100644 index 6d1fdee..0000000 --- a/packages/node/15.10.0/run +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -node "$@" \ No newline at end of file diff --git a/packages/node/15.10.0/test.js b/packages/node/15.10.0/test.js deleted file mode 100644 index e106678..0000000 --- a/packages/node/15.10.0/test.js +++ /dev/null @@ -1 +0,0 @@ -console.log('OK'); diff --git a/packages/node/16.3.0/build.sh b/packages/node/16.3.0/build.sh deleted file mode 100755 index bbc19db..0000000 --- a/packages/node/16.3.0/build.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -curl "https://nodejs.org/dist/v16.3.0/node-v16.3.0-linux-x64.tar.xz" -o node.tar.xz -tar xf node.tar.xz --strip-components=1 -rm node.tar.xz diff --git a/packages/node/16.3.0/environment b/packages/node/16.3.0/environment deleted file mode 100644 index 977a5e8..0000000 --- a/packages/node/16.3.0/environment +++ /dev/null @@ -1 +0,0 @@ -export PATH=$PWD/bin:$PATH diff --git a/packages/node/16.3.0/metadata.json b/packages/node/16.3.0/metadata.json deleted file mode 100644 index ccd146a..0000000 --- a/packages/node/16.3.0/metadata.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "language": "node", - "version": "16.3.0", - "provides": [ - { - "language": "javascript", - "aliases": ["node-javascript", "node-js", "javascript", "js"] - } - ] -} diff --git a/packages/node/16.3.0/run b/packages/node/16.3.0/run deleted file mode 100644 index 6d1fdee..0000000 --- a/packages/node/16.3.0/run +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -node "$@" \ No newline at end of file diff --git a/packages/node/16.3.0/test.js b/packages/node/16.3.0/test.js deleted file mode 100644 index e106678..0000000 --- a/packages/node/16.3.0/test.js +++ /dev/null @@ -1 +0,0 @@ -console.log('OK'); diff --git a/packages/python/2.7.18/build.sh b/packages/python/2.7.18/build.sh deleted file mode 100755 index 131ba96..0000000 --- a/packages/python/2.7.18/build.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -PREFIX=$(realpath $(dirname $0)) - -mkdir -p build - -cd build - -curl "https://www.python.org/ftp/python/2.7.18/Python-2.7.18.tgz" -o python.tar.gz -tar xzf python.tar.gz --strip-components=1 -rm python.tar.gz - -./configure --prefix "$PREFIX" --with-ensurepip=install -make -j$(nproc) -make install -j$(nproc) - -cd .. - -rm -rf build -bin/pip2 install -U pip==20.3.* -# Upgrade pip to latest supported version - -bin/pip2 install numpy scipy pycrypto whoosh bcrypt passlib diff --git a/packages/python/2.7.18/environment b/packages/python/2.7.18/environment deleted file mode 100644 index bd0ff98..0000000 --- a/packages/python/2.7.18/environment +++ /dev/null @@ -1 +0,0 @@ -export PATH=$PWD/bin:$PATH \ No newline at end of file diff --git a/packages/python/2.7.18/metadata.json b/packages/python/2.7.18/metadata.json deleted file mode 100644 index b825f8e..0000000 --- a/packages/python/2.7.18/metadata.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "language": "python2", - "version": "2.7.18", - "aliases": ["py2", "python2"] -} diff --git a/packages/python/2.7.18/run b/packages/python/2.7.18/run deleted file mode 100644 index 9858f52..0000000 --- a/packages/python/2.7.18/run +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -python2.7 "$@" \ No newline at end of file diff --git a/packages/python/2.7.18/test.py b/packages/python/2.7.18/test.py deleted file mode 100644 index d9708c5..0000000 --- a/packages/python/2.7.18/test.py +++ /dev/null @@ -1 +0,0 @@ -print "OK" \ No newline at end of file diff --git a/packages/python/3.10.0-alpha.7/build.sh b/packages/python/3.10.0-alpha.7/build.sh deleted file mode 100755 index 914353f..0000000 --- a/packages/python/3.10.0-alpha.7/build.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -PREFIX=$(realpath $(dirname $0)) - -mkdir -p build - -cd build - -curl "https://www.python.org/ftp/python/3.10.0/Python-3.10.0a7.tgz" -o python.tar.gz -tar xzf python.tar.gz --strip-components=1 -rm python.tar.gz - -./configure --prefix "$PREFIX" --with-ensurepip=install -make -j$(nproc) -make install -j$(nproc) - -cd .. - -rm -rf build - -# This is alpha version, hence most of the libraries are not compatible with python3.10.0 -# bin/pip3 install numpy scipy pandas pycrypto whoosh bcrypt passlib diff --git a/packages/python/3.10.0-alpha.7/environment b/packages/python/3.10.0-alpha.7/environment deleted file mode 100644 index bd0ff98..0000000 --- a/packages/python/3.10.0-alpha.7/environment +++ /dev/null @@ -1 +0,0 @@ -export PATH=$PWD/bin:$PATH \ No newline at end of file diff --git a/packages/python/3.10.0-alpha.7/metadata.json b/packages/python/3.10.0-alpha.7/metadata.json deleted file mode 100644 index 5232a19..0000000 --- a/packages/python/3.10.0-alpha.7/metadata.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "language": "python", - "version": "3.10.0-alpha.7", - "aliases": ["py", "py3", "python3"] -} diff --git a/packages/python/3.10.0-alpha.7/run b/packages/python/3.10.0-alpha.7/run deleted file mode 100644 index ff60df2..0000000 --- a/packages/python/3.10.0-alpha.7/run +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -python3.10 "$@" \ No newline at end of file diff --git a/packages/python/3.10.0-alpha.7/test.py b/packages/python/3.10.0-alpha.7/test.py deleted file mode 100644 index c3135e2..0000000 --- a/packages/python/3.10.0-alpha.7/test.py +++ /dev/null @@ -1,7 +0,0 @@ -working = True - -match working: - case True: - print("OK") - case False: - print() \ No newline at end of file diff --git a/packages/python/3.5.10/build.sh b/packages/python/3.5.10/build.sh deleted file mode 100755 index bced203..0000000 --- a/packages/python/3.5.10/build.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -PREFIX=$(realpath $(dirname $0)) - -mkdir -p build - -cd build - -curl "https://www.python.org/ftp/python/3.5.10/Python-3.5.10.tgz" -o python.tar.gz -tar xzf python.tar.gz --strip-components=1 -rm python.tar.gz - -./configure --prefix "$PREFIX" --with-ensurepip=install -make -j$(nproc) -make install -j$(nproc) - -cd .. - -rm -rf build - -bin/pip3 install numpy scipy pandas pycrypto whoosh bcrypt passlib diff --git a/packages/python/3.5.10/environment b/packages/python/3.5.10/environment deleted file mode 100644 index bd0ff98..0000000 --- a/packages/python/3.5.10/environment +++ /dev/null @@ -1 +0,0 @@ -export PATH=$PWD/bin:$PATH \ No newline at end of file diff --git a/packages/python/3.5.10/metadata.json b/packages/python/3.5.10/metadata.json deleted file mode 100644 index 5cc61c1..0000000 --- a/packages/python/3.5.10/metadata.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "language": "python", - "version": "3.5.10", - "aliases": ["py", "py3", "python3"] -} diff --git a/packages/python/3.5.10/run b/packages/python/3.5.10/run deleted file mode 100644 index aafa4ab..0000000 --- a/packages/python/3.5.10/run +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -python3.5 "$@" \ No newline at end of file diff --git a/packages/python/3.5.10/test.py b/packages/python/3.5.10/test.py deleted file mode 100644 index 3f2688f..0000000 --- a/packages/python/3.5.10/test.py +++ /dev/null @@ -1 +0,0 @@ -print("OK") \ No newline at end of file diff --git a/packages/python/3.9.1/build.sh b/packages/python/3.9.1/build.sh deleted file mode 100755 index fcae7c7..0000000 --- a/packages/python/3.9.1/build.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -PREFIX=$(realpath $(dirname $0)) - -mkdir -p build - -cd build - -curl "https://www.python.org/ftp/python/3.9.1/Python-3.9.1.tgz" -o python.tar.gz -tar xzf python.tar.gz --strip-components=1 -rm python.tar.gz - -./configure --prefix "$PREFIX" --with-ensurepip=install -make -j$(nproc) -make install -j$(nproc) - -cd .. - -rm -rf build - -bin/pip3 install numpy scipy pandas pycrypto whoosh bcrypt passlib diff --git a/packages/python/3.9.1/environment b/packages/python/3.9.1/environment deleted file mode 100644 index bd0ff98..0000000 --- a/packages/python/3.9.1/environment +++ /dev/null @@ -1 +0,0 @@ -export PATH=$PWD/bin:$PATH \ No newline at end of file diff --git a/packages/python/3.9.1/metadata.json b/packages/python/3.9.1/metadata.json deleted file mode 100644 index 2a0970f..0000000 --- a/packages/python/3.9.1/metadata.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "language": "python", - "version": "3.9.1", - "aliases": ["py", "py3", "python3"] -} diff --git a/packages/python/3.9.1/run b/packages/python/3.9.1/run deleted file mode 100644 index 450bb76..0000000 --- a/packages/python/3.9.1/run +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -python3.9 "$@" \ No newline at end of file diff --git a/packages/python/3.9.1/test.py b/packages/python/3.9.1/test.py deleted file mode 100644 index 3f2688f..0000000 --- a/packages/python/3.9.1/test.py +++ /dev/null @@ -1 +0,0 @@ -print("OK") \ No newline at end of file diff --git a/packages/python/3.9.4/build.sh b/packages/python/3.9.4/build.sh deleted file mode 100755 index 1966bd9..0000000 --- a/packages/python/3.9.4/build.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -PREFIX=$(realpath $(dirname $0)) - -mkdir -p build - -cd build - -curl "https://www.python.org/ftp/python/3.9.4/Python-3.9.4.tgz" -o python.tar.gz -tar xzf python.tar.gz --strip-components=1 -rm python.tar.gz - -./configure --prefix "$PREFIX" --with-ensurepip=install -make -j$(nproc) -make install -j$(nproc) - -cd .. - -rm -rf build - -bin/pip3 install numpy scipy pandas pycrypto whoosh bcrypt passlib sympy diff --git a/packages/python/3.9.4/environment b/packages/python/3.9.4/environment deleted file mode 100644 index bd0ff98..0000000 --- a/packages/python/3.9.4/environment +++ /dev/null @@ -1 +0,0 @@ -export PATH=$PWD/bin:$PATH \ No newline at end of file diff --git a/packages/python/3.9.4/metadata.json b/packages/python/3.9.4/metadata.json deleted file mode 100644 index 63f96dc..0000000 --- a/packages/python/3.9.4/metadata.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "language": "python", - "version": "3.9.4", - "aliases": ["py", "py3", "python3"] -} diff --git a/packages/python/3.9.4/run b/packages/python/3.9.4/run deleted file mode 100644 index 450bb76..0000000 --- a/packages/python/3.9.4/run +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -python3.9 "$@" \ No newline at end of file diff --git a/packages/python/3.9.4/test.py b/packages/python/3.9.4/test.py deleted file mode 100644 index f8bcd5c..0000000 --- a/packages/python/3.9.4/test.py +++ /dev/null @@ -1,18 +0,0 @@ -execute = (execute_with := lambda *a, **k: lambda f: f(*a, **k))() - - -@int -@execute -class n: __int__ = lambda _: 69 - - -@execute -class cout: __lshift__ = print - - -@execute_with(n) -def output(n): - return "OK" - - -cout << output diff --git a/packages/test.sh b/packages/test.sh deleted file mode 100755 index 615e2db..0000000 --- a/packages/test.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash - -AUTH_HEADER="Authorization: $API_KEY" - -for test_file in */*/test.* -do - IFS='/' read -ra test_parts <<< "$test_file" - IFS='.' read -ra file_parts <<< "$(basename $test_file)" - language=${file_parts[1]} - lang_ver=${test_parts[1]} - - test_src=$(python3 -c "import json; print(json.dumps(open('$test_file').read()))") - - json='{"language":"'$language'","version":"'$lang_ver'","files":[{"content":'$test_src'}]}' - - result=$(curl -s -XPOST -H "Content-Type: application/json" -d "$json" https://emkc.org/api/v2/piston/execute -H $AUTH_HEADER) - - echo "==$test_file: $language-$lang_ver==" - #jq '.' <<<"$result" - jq -r 'if (.run.stdout | contains("OK") ) then (.run.stdout) else (.compile.output + .run.output) end' <<<$result -done diff --git a/public.pps b/public.pps deleted file mode 100755 index f032032..0000000 --- a/public.pps +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env -S piston ppman spec - -# Public Piston Packages -# Defines packages to be installed on the public piston installation - -# All packages, latest version -* * - -# Except python -!python * - -# Install python 3.* and 2.* -python 3.* -python 2.* diff --git a/readme.md b/readme.md index 0cc0fa4..2bd0c1f 100644 --- a/readme.md +++ b/readme.md @@ -215,6 +215,7 @@ Content-Type: application/json [ { + "id": 1, "language": "bash", "version": "5.1.0", "aliases": [ @@ -222,6 +223,7 @@ Content-Type: application/json ] }, { + "id": 2, "language": "brainfuck", "version": "2.7.3", "aliases": [ @@ -237,35 +239,32 @@ Content-Type: application/json `POST /api/v2/execute` This endpoint requests execution of some arbitrary code. -- `language` (**required**) The language to use for execution, must be a string and must be installed. -- `version` (**required**) The version of the language to use for execution, must be a string containing a SemVer selector for the version or the specific version number to use. -- `files` (**required**) An array of files containing code or other data that should be used for execution. The first file in this array is considered the main file. -- `files[].name` (_optional_) The name of the file to upload, must be a string containing no path or left out. -- `files[].content` (**required**) The content of the files to upload, must be a string containing text to write. -- `files[].encoding` (_optional_) The encoding scheme used for the file content. One of `base64`, `hex` or `utf8`. Defaults to `utf8`. -- `stdin` (_optional_) The text to pass as stdin to the program. Must be a string or left out. Defaults to blank string. -- `args` (_optional_) The arguments to pass to the program. Must be an array or left out. Defaults to `[]`. -- `compile_timeout` (_optional_) The maximum time allowed for the compile stage to finish before bailing out in milliseconds. Must be a number or left out. Defaults to `10000` (10 seconds). -- `run_timeout` (_optional_) The maximum time allowed for the run stage to finish before bailing out in milliseconds. Must be a number or left out. Defaults to `3000` (3 seconds). -- `compile_memory_limit` (_optional_) The maximum amount of memory the compile stage is allowed to use in bytes. Must be a number or left out. Defaults to `-1` (no limit) -- `run_memory_limit` (_optional_) The maximum amount of memory the run stage is allowed to use in bytes. Must be a number or left out. Defaults to `-1` (no limit) +- `language` (**required**) The language to use for execution, must be a string and must be installed. +- `files` (**required**) An array of files containing code or other data that should be used for execution. The first file in this array is considered the main file. +- `files[].name` (_optional_) The name of the file to upload, must be a string containing no path or left out. +- `files[].content` (**required**) The content of the files to upload, must be a string containing text to write. +- `stdin` (_optional_) The text to pass as stdin to the program. Must be a string or left out. Defaults to blank string. +- `args` (_optional_) The arguments to pass to the program. Must be an array or left out. Defaults to `[]`. +- `compile_timeout` (_optional_) The maximum time allowed for the compile stage to finish before bailing out in milliseconds. Must be a number or left out. Defaults to `10000` (10 seconds). +- `run_timeout` (_optional_) The maximum time allowed for the run stage to finish before bailing out in milliseconds. Must be a number or left out. Defaults to `3000` (3 seconds). +- `compile_memory_limit` (_optional_) The maximum amount of memory the compile stage is allowed to use in bytes. Must be a number or left out. Defaults to `-1` (no limit) +- `run_memory_limit` (_optional_) The maximum amount of memory the run stage is allowed to use in bytes. Must be a number or left out. Defaults to `-1` (no limit) ```json { - "language": "js", - "version": "15.10.0", - "files": [ - { - "name": "my_cool_code.js", - "content": "console.log(process.argv)" - } - ], - "stdin": "", - "args": ["1", "2", "3"], - "compile_timeout": 10000, - "run_timeout": 3000, - "compile_memory_limit": -1, - "run_memory_limit": -1 + "language": "js", + "files": [ + { + "name": "my_cool_code.js", + "content": "console.log(process.argv)" + } + ], + "stdin": "", + "args": ["1", "2", "3"], + "compile_timeout": 10000, + "run_timeout": 3000, + "compile_memory_limit": -1, + "run_memory_limit": -1 } ``` diff --git a/repo/.dockerignore b/repo/.dockerignore deleted file mode 100644 index d1470fc..0000000 --- a/repo/.dockerignore +++ /dev/null @@ -1 +0,0 @@ -*.pkg.tar.gz \ No newline at end of file diff --git a/repo/.gitignore b/repo/.gitignore deleted file mode 100644 index c3607c0..0000000 --- a/repo/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -*.pkg.tar.gz -index \ No newline at end of file diff --git a/repo/Dockerfile b/repo/Dockerfile deleted file mode 100644 index de28c11..0000000 --- a/repo/Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -FROM debian:buster-slim -ENV DEBIAN_FRONTEND noninteractive -RUN apt-get update && apt-get install -y unzip autoconf build-essential libssl-dev \ - pkg-config zlib1g-dev libargon2-dev libsodium-dev libcurl4-openssl-dev \ - sqlite3 libsqlite3-dev libonig-dev libxml2 libxml2-dev bc curl git \ - linux-headers-amd64 perl xz-utils python3 python3-pip gnupg jq zlib1g-dev \ - cmake cmake-doc extra-cmake-modules build-essential gcc binutils bash coreutils \ - util-linux pciutils usbutils coreutils binutils findutils grep libncurses5-dev \ - libncursesw5-dev python3-pip libgmp-dev libmpfr-dev python2 libffi-dev gfortran\ - libreadline-dev libblas-dev liblapack-dev libpcre3-dev libarpack2-dev libfftw3-dev \ - libglpk-dev libqhull-dev libqrupdate-dev libsuitesparse-dev libsundials-dev \ - libbz2-dev liblzma-dev libpcre2-dev gperf bison flex g++ && \ - ln -sf /bin/bash /bin/sh && \ - rm -rf /var/lib/apt/lists/* && \ - update-alternatives --install /usr/bin/python python /usr/bin/python3.7 2 - -ADD entrypoint.sh mkindex.sh / - -ENTRYPOINT ["bash","/entrypoint.sh"] -CMD ["--no-build"] diff --git a/repo/entrypoint.sh b/repo/entrypoint.sh deleted file mode 100755 index 6c47e37..0000000 --- a/repo/entrypoint.sh +++ /dev/null @@ -1,59 +0,0 @@ -cd /piston/packages - -SERVER=1 -BUILD=1 -CI=0 - -echo "Running through arguments.." - -for pkg in "$@" -do - shift - if [[ "$pkg" = "--no-server" ]]; then - echo "Not starting index server after builds" - SERVER=0 - elif [[ "$pkg" = "--no-build" ]]; then - echo "Building no more package" - BUILD=0 - elif [[ "$pkg" = "--ci" ]]; then - echo "Running in CI mode, --no-build, --no-server" - BUILD=0 - SERVER=0 - CI=1 - else - if [[ $BUILD -eq 1 ]]; then - echo "Building package $pkg" - make -j16 $pkg.pkg.tar.gz PLATFORM=docker-debian - echo "Done with package $pkg" - elif [[ $CI -eq 1 ]]; then - echo "Commit SHA: $pkg" - - cd .. - echo "Changed files:" - git diff --name-only $pkg^1 $pkg - PACKAGES=$(git diff --name-only $pkg^1 $pkg | awk -F/ '{ print $2 "-" $3 }' | sort -u) - cd packages - - echo "Building packages: $PACKAGES" - for package in "$PACKAGES"; do - make -j16 $package.pkg.tar.gz PLATFORM=docker-debian - done - - else - echo "Building was disabled, skipping $pkg build=$BUILD ci=$CI" - fi - fi -done - -cd /piston/repo -echo "Creating index" -./mkindex.sh -echo "Index created" - -if [[ $SERVER -eq 1 ]]; then - echo "Starting index server.." - python3 -m http.server -else - echo "Skipping starting index server" -fi -exit 0 \ No newline at end of file diff --git a/repo/mkindex.sh b/repo/mkindex.sh deleted file mode 100755 index c93b444..0000000 --- a/repo/mkindex.sh +++ /dev/null @@ -1,23 +0,0 @@ -BASEURL=http://repo:8000/ - -i=0 - -echo "" > index - -for pkg in $(find ../packages -type f -name "*.pkg.tar.gz") -do - - cp $pkg . - - PKGFILE=$(basename $pkg) - PKGFILENAME=$(echo $PKGFILE | sed 's/\.pkg\.tar\.gz//g') - - PKGNAME=$(echo $PKGFILENAME | grep -oP '^\K.+(?=-)') - PKGVERSION=$(echo $PKGFILENAME | grep -oP '^.+-\K.+') - PKGCHECKSUM=$(sha256sum $PKGFILE | awk '{print $1}') - - echo "$PKGNAME,$PKGVERSION,$PKGCHECKSUM,$BASEURL$PKGFILE" >> index - echo "Adding package $PKGNAME-$PKGVERSION" - - ((i=i+1)) -done \ No newline at end of file diff --git a/result b/result new file mode 120000 index 0000000..d60c5ac --- /dev/null +++ b/result @@ -0,0 +1 @@ +/nix/store/1idnmsddirm1hpxw40r17wsg9p9dscb9-piston.tar.gz \ No newline at end of file diff --git a/runtimes/.scaffold.nix b/runtimes/.scaffold.nix new file mode 100644 index 0000000..51b68aa --- /dev/null +++ b/runtimes/.scaffold.nix @@ -0,0 +1,65 @@ +{pkgs, piston, ...}: +let + # Put your package here, preferibly from nixpkgs. + pkg = pkgs.%RUNTIME%; # this may be incorrect - it is guessed +in piston.mkRuntime { + # Name of the language implemented by this runtime + language = "%LANGUAGE%"; + + # The version of the language + # Usually this is specified on the package + version = pkg.version; + + # Name of the runtime + # This line should be kept if the runtime differs from the language + runtime = "%RUNTIME%"; + + aliases = [ + # Put extensions in here, and other common names + # Example: + # "js" + # "node-javascript" + ]; + + + + # This is the lines of a shell script to compile source code. + # Arguments passed to this script are all the provided source files + # The CWD of this script is a temp directory for a job + # + # If the language only supports JIT compiling, simply remove this line + # See ./python3.nix and ./node-javascript.nix for examples + # + # No shebang needs to be added to this - that is done automatically. + compile = '' + ${pkg}/bin/%RUNTIME% --compile "$@" + ''; + + # This is the lines of a shell script to evaluate a file at $1 + # The remaining arguments are the arguments to launch the application with + # The CWD of this script is a temp directory for a job + # + # If the compile stage is used, $1 still contains the name of the source file. + # It is up to your script to determine the filename of the emitted binary + # + # No shebang needs to be added to this - that is done automatically. + run = '' + ${pkg}/bin/%RUNTIME% "$@" + ''; + + # Specify a list of tests. + # These should output "OK" to STDOUT if everything looks good + tests = [ + (piston.mkTest { + files = { + "test.js" = '' + console.log("OK"); + ''; + }; + args = []; + stdin = ""; + packages = []; + main = "test.js"; + }) + ]; +} \ No newline at end of file diff --git a/runtimes/bash.nix b/runtimes/bash.nix new file mode 100644 index 0000000..9ed8cce --- /dev/null +++ b/runtimes/bash.nix @@ -0,0 +1,28 @@ +{pkgs, piston, ...}: +let + pkg = pkgs.bash; +in piston.mkRuntime { + language = "bash"; + + version = pkg.version; + + runtime = "bash"; + + aliases = [ + "sh" + ]; + + run = '' + ${pkg}/bin/bash "$@" + ''; + + tests = [ + (piston.mkTest { + files = { + "test.sh" = '' + echo OK + ''; + }; + }) + ]; +} \ No newline at end of file diff --git a/runtimes/clojure.nix b/runtimes/clojure.nix new file mode 100644 index 0000000..67c796c --- /dev/null +++ b/runtimes/clojure.nix @@ -0,0 +1,29 @@ +{pkgs, piston, ...}: +let + pkg = pkgs.clojure; +in piston.mkRuntime { + language = "clojure"; + version = pkg.version + ""; #Clojure has X.X.X.X versioning, we want X.X.X + + aliases = [ + "clj" + ]; + + run = '' + ${pkg}/bin/clojure "$@" + ''; + + tests = [ + (piston.mkTest { + files = { + "test.clj" = '' + (ns clojure.examples.main + (:gen-class)) + (defn main [] + (println "OK")) + (main) + ''; + }; + }) + ]; +} \ No newline at end of file diff --git a/runtimes/cobol-gnu-cobol.nix b/runtimes/cobol-gnu-cobol.nix new file mode 100644 index 0000000..af72f5d --- /dev/null +++ b/runtimes/cobol-gnu-cobol.nix @@ -0,0 +1,39 @@ +{pkgs, piston, ...}: +let + pkg = pkgs.gnu-cobol; +in piston.mkRuntime { + language = "cobol"; + version = pkg.version; + runtime = "gnu-cobol"; + + aliases = [ + "cob" + ]; + + compile = '' + ${pkg}/bin/cobc -o binary --free -x -L lib "$@" + chmod +x binary + ''; + + run = '' + shift + ./binary "$@" + ''; + + tests = [ + (piston.mkTest { + files = { + "test.cob" = '' + *> Test Program + identification division. + program-id. ok-test. + + procedure division. + display "OK" + goback. + end program ok-test. + ''; + }; + }) + ]; +} \ No newline at end of file diff --git a/runtimes/default.nix b/runtimes/default.nix new file mode 100644 index 0000000..2c3ddc4 --- /dev/null +++ b/runtimes/default.nix @@ -0,0 +1,8 @@ +args: { + "node-javascript" = import ./node-javascript.nix args; + "python2" = import ./python2.nix args; + "python3" = import ./python3.nix args; + "bash" = import ./bash.nix args; + "clojure" = import ./clojure.nix args; + "cobol-gnu-cobol" = import ./cobol-gnu-cobol.nix args; +} diff --git a/runtimes/node-javascript.nix b/runtimes/node-javascript.nix new file mode 100644 index 0000000..3bf2c78 --- /dev/null +++ b/runtimes/node-javascript.nix @@ -0,0 +1,28 @@ +{pkgs, piston, ...}: +let + pkg = pkgs.nodejs; +in piston.mkRuntime { + language = "javascript"; + version = pkg.version; + runtime = "node"; + + aliases = [ + "node-js" + "node-javascript" + "js" + ]; + + run = '' + ${pkg}/bin/node "$@" + ''; + + tests = [ + (piston.mkTest { + files = { + "test.js" = '' + console.log("OK"); + ''; + }; + }) + ]; +} \ No newline at end of file diff --git a/runtimes/python2.nix b/runtimes/python2.nix new file mode 100644 index 0000000..9b89a20 --- /dev/null +++ b/runtimes/python2.nix @@ -0,0 +1,25 @@ +{pkgs, piston, ...}: +let + pkg = pkgs.python2; +in piston.mkRuntime { + language = "python2"; + version = pkg.version; + + aliases = [ + "py2" + ]; + + run = '' + ${pkg}/bin/python2 "$@" + ''; + + tests = [ + (piston.mkTest { + files = { + "test.py" = '' + print "OK" + ''; + }; + }) + ]; +} \ No newline at end of file diff --git a/runtimes/python3.nix b/runtimes/python3.nix new file mode 100644 index 0000000..9dc2682 --- /dev/null +++ b/runtimes/python3.nix @@ -0,0 +1,26 @@ +{pkgs, piston, ...}: +let + pkg = pkgs.python3; +in piston.mkRuntime { + language = "python3"; + version = pkg.version; + + aliases = [ + "py3" + "py" + ]; + + run = '' + ${pkg}/bin/python3 "$@" + ''; + + tests = [ + (piston.mkTest { + files = { + "test.py" = '' + print("OK") + ''; + }; + }) + ]; +} \ No newline at end of file diff --git a/runtimes/scaffold.sh b/runtimes/scaffold.sh new file mode 100755 index 0000000..fb3aafe --- /dev/null +++ b/runtimes/scaffold.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env sh + +if [[ $# -eq 0 ]]; then + echo "Usage: $0 [runtime name]" + echo + echo "language name: The name of the language to add, e.g. javascript" + echo "runtime name: The name of the runtime to add, e.g. node" + echo " In most cases runtime won't be required, e.g. python3 is both a language and runtime" + exit 1; +fi + +LANGUAGE=$1 +RUNTIME=$1 +NAME=$1 + +if [[ $# -eq 2 ]]; then + RUNTIME=$2 + NAME=$LANGUAGE-$RUNTIME +fi + + +if grep "./$NAME.nix" default.nix > /dev/null; then + echo "ERROR: $NAME already exists" + echo "Remove it from default.nix and retry if this is intended" + exit 1 +else + sed -i '$d' default.nix + echo " \"$NAME\" = import ./$NAME.nix args;" >> default.nix + sed -e 's/%LANGUAGE%/'"$LANGUAGE"'/g' \ + -e 's/%RUNTIME%/'"$RUNTIME"'/g' \ + .scaffold.nix > $NAME.nix + git add $NAME.nix + echo "}" >> default.nix + + echo "Scaffolded $NAME" + echo "Edit $NAME.nix to get started" +fi \ No newline at end of file diff --git a/shell.nix b/shell.nix index 339bead..d8731fe 100644 --- a/shell.nix +++ b/shell.nix @@ -1,5 +1,11 @@ { pkgs ? import {} }: pkgs.mkShell { # nativeBuildInputs is usually what you want -- tools you need to run - nativeBuildInputs = with pkgs; [ nodejs-15_x jq mkdocs ]; + nativeBuildInputs = with pkgs; [ + nodejs-16_x + jq + mkdocs + yarn2nix + yarn + ]; } From e022e34a376a223c092e71f3163bb509457c043d Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sun, 30 Jan 2022 22:29:21 +1300 Subject: [PATCH 67/72] Add nix runtime testing and pre-installing runtimes --- api/default.nix | 47 +++++++++- api/entrypoint.sh | 14 --- api/package.json | 6 +- api/src/bin/install.js | 39 +++++++++ api/src/{ => bin}/pistond.js | 14 +-- api/src/bin/test.js | 112 ++++++++++++++++++++++++ api/src/config.js | 6 ++ api/src/runtime.js | 13 +-- flake.nix | 67 +++----------- piston | 163 ++++++++++++++++------------------- result | 1 - 11 files changed, 306 insertions(+), 176 deletions(-) delete mode 100755 api/entrypoint.sh create mode 100755 api/src/bin/install.js rename api/src/{ => bin}/pistond.js (85%) create mode 100755 api/src/bin/test.js delete mode 120000 result diff --git a/api/default.nix b/api/default.nix index cc6cb70..c194587 100644 --- a/api/default.nix +++ b/api/default.nix @@ -1,5 +1,5 @@ -{pkgs, ...}: -with pkgs; { +{pkgs, nosocket, ...}: +with pkgs; rec { package = mkYarnPackage { name = "piston"; src = ./.; @@ -25,4 +25,47 @@ with pkgs; { }; }; }; + container = pkgs.dockerTools.buildLayeredImageWithNixDb { + name = "piston"; + tag = "base-latest"; + + contents = with pkgs; [ + package + nosocket + bash + nixFlakes + coreutils-full + cacert.out + git + gnutar + gzip + gnugrep + util-linux + ]; + + extraCommands = '' + mkdir -p piston/{jobs,runtimes} etc/nix {,var/}tmp run/lock + echo -e "experimental-features = nix-command flakes" >> etc/nix/nix.conf + echo "nixbld:x:30000:nixbld1,nixbld10,nixbld11,nixbld12,nixbld13,nixbld14,nixbld15,nixbld16,nixbld17,nixbld18,nixbld19,nixbld2,nixbld20,nixbld21,nixbld22,nixbld23,nixbld24,nixbld25,nixbld26,nixbld27,nixbld28,nixbld29,nixbld3,nixbld30,nixbld31,nixbld32,nixbld4,nixbld5,nixbld6,nixbld7,nixbld8,nixbld9" >> etc/group + for i in $(seq 1 32) + do + echo "nixbld$i:x:$(( $i + 30000 )):30000:Nix build user $i:/var/empty:/run/current-system/sw/bin/nologin" >> etc/passwd + done + ''; + + config = { + Cmd = ["${package}/bin/pistond"]; + Env = [ + "NIX_PAGER=cat" + "USER=nobody" + "SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt" + "GIT_SSL_CAINFO=/etc/ssl/certs/ca-bundle.crt" + "NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt" + ]; + + ExposedPorts = { + "2000/tcp" = {}; + }; + }; + }; } \ No newline at end of file diff --git a/api/entrypoint.sh b/api/entrypoint.sh deleted file mode 100755 index aedcedd..0000000 --- a/api/entrypoint.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash -echo "Starting Piston API" - -echo "Checking presense of nix store" -if [[ ! -f "/nix/piston_detected" ]]; then - echo "Nix Store is not loaded, assuming /nix has been mounted - copying contents" - cp -rp /var/nix/* /nix -fi - -echo "Adding nix to env" -. ~/.profile - -echo "Launching Piston API" -node src \ No newline at end of file diff --git a/api/package.json b/api/package.json index 49e55db..0121f1b 100644 --- a/api/package.json +++ b/api/package.json @@ -2,7 +2,7 @@ "name": "piston-api", "version": "4.0.0", "description": "API for piston - a high performance code execution engine", - "main": "src/pistond.js", + "main": "src/bin/pistond.js", "dependencies": { "body-parser": "^1.19.0", "express": "^4.17.1", @@ -23,6 +23,8 @@ "prettier": "2.2.1" }, "bin": { - "pistond": "./src/pistond.js" + "pistond": "./src/bin/pistond.js", + "piston-install": "./src/bin/install.js", + "piston-test": "./src/bin/test.js" } } diff --git a/api/src/bin/install.js b/api/src/bin/install.js new file mode 100755 index 0000000..b9a2b72 --- /dev/null +++ b/api/src/bin/install.js @@ -0,0 +1,39 @@ +#!/usr/bin/env node +// Downloads all packages specified into the nix store +require('nocamel'); +const config = require('../config'); +const Logger = require('logplease'); +const logger = Logger.create('install'); +const cp = require('child_process'); + +logger.info('Setting loglevel to', config.log_level); +Logger.setLogLevel(config.log_level); + +const runtimes_path = `${config.flake_path}#pistonRuntimeSets.${config.runtime_set}`; +logger.info(`Installing runtimes in ${runtimes_path}`); + +logger.debug("Getting package listing"); +const runtimes = JSON.parse(cp.execSync(`nix eval --json ${runtimes_path} --apply builtins.attrNames`)); + +logger.info(`Got runtimes: ${runtimes}`); + +runtimes.forEach(runtime => { + logger.debug(`Loading metadata for ${runtime}`); + const runtime_metadata = JSON.parse(cp.execSync(`nix eval --json ${runtimes_path}.${runtime}.metadata`)); + const namever = `${runtime}-${runtime_metadata.version}`; + logger.info(`Installing ${namever}`); + + + function install(key){ + logger.debug(`Installing ${namever}-${key}`); + const command = `nix build ${runtimes_path}.${runtime}.metadata.${key} -o /piston/runtimes/${namever}-${key}`; + cp.execSync(command); + } + + install("run"); + if(runtime_metadata.compile) install("compile"); + + logger.info(`Installed ${namever}`); +}); + +logger.info("Done"); \ No newline at end of file diff --git a/api/src/pistond.js b/api/src/bin/pistond.js similarity index 85% rename from api/src/pistond.js rename to api/src/bin/pistond.js index 36e12e3..c56cc99 100755 --- a/api/src/pistond.js +++ b/api/src/bin/pistond.js @@ -3,16 +3,16 @@ require('nocamel'); const Logger = require('logplease'); const express = require('express'); const expressWs = require('express-ws'); -const globals = require('./globals'); -const config = require('./config'); +const globals = require('../globals'); +const config = require('../config'); const cp = require('child_process'); const path = require('path'); const fs = require('fs/promises'); const fss = require('fs'); const body_parser = require('body-parser'); -const runtime = require('./runtime'); +const runtime = require('../runtime'); -const logger = Logger.create('index'); +const logger = Logger.create('pistond'); const app = express(); expressWs(app); @@ -39,7 +39,7 @@ expressWs(app); logger.info('Loading packages'); - const runtimes_data = cp.execSync(`nix eval --json ${config.flake_path}#pistonRuntimes --apply builtins.attrNames`).toString(); + const runtimes_data = cp.execSync(`nix eval --json ${config.flake_path}#pistonRuntimeSets.${config.runtime_set} --apply builtins.attrNames`).toString(); const runtimes = JSON.parse(runtimes_data); runtimes.for_each(pkg => runtime.load_runtime(pkg)); @@ -59,8 +59,8 @@ expressWs(app); logger.debug('Registering Routes'); - const api_v2 = require('./api/v2'); - const api_v3 = require('./api/v3'); + const api_v2 = require('../api/v2'); + const api_v3 = require('../api/v3'); app.use('/api/v2', api_v2); app.use('/api/v3', api_v3); diff --git a/api/src/bin/test.js b/api/src/bin/test.js new file mode 100755 index 0000000..134ce2a --- /dev/null +++ b/api/src/bin/test.js @@ -0,0 +1,112 @@ +#!/usr/bin/env node +// Test the specified package +require('nocamel'); +const config = require('../config'); +const Logger = require('logplease'); +const logger = Logger.create('test'); +const cp = require('child_process'); +const runtime = require("../runtime"); +const { Job } = require('../job'); + +(async function(){ + logger.info('Setting loglevel to', config.log_level); + Logger.setLogLevel(config.log_level); + + + + let runtimes_to_test; + let failed = false; + + if(process.argv[2] === "--all"){ + // load all + runtimes_to_test = JSON.parse( + cp.execSync(`nix eval ${config.flake_path}#pistonRuntimes --json --apply builtins.attrNames`) + ); + }else{ + runtimes_to_test = [process.argv[2]]; + } + + + + for (const runtime_name of runtimes_to_test) { + + + const runtime_path = `${config.flake_path}#pistonRuntimes.${runtime_name}`; + logger.info(`Testing runtime ${runtime_path}`); + + logger.debug(`Loading runtime metadata`); + const metadata = JSON.parse(cp.execSync(`nix eval --json ${runtime_path}.metadata --json`)); + + logger.debug(`Loading runtime tests`); + const tests = JSON.parse(cp.execSync(`nix eval --json ${runtime_path}.tests --json`)); + + logger.debug(`Loading runtime`); + + const testable_runtime = new runtime.Runtime({ + ...metadata, + flake_path: runtime_path + }); + + testable_runtime.ensure_built(); + + + logger.info(`Running tests`); + + for (const test of tests) { + + const files = []; + + for (const file_name of Object.keys(test.files)) { + const file_content = test.files[file_name]; + const this_file = { + name: file_name, + content: file_content + }; + + if(file_name == test.main) + files.unshift(this_file); + else + files.push(this_file); + + } + + + const job = new Job({ + runtime: testable_runtime, + args: test.args || [], + stdin: test.stdin || "", + files, + timeouts: { + run: 3000, + compile: 10000 + }, + memory_limits: { + run: config.run_memory_limit, + compile: config.compile_memory_limit + } + }); + + await job.prime() + const result = await job.execute() + await job.cleanup() + + if(result.run.stdout.trim() !== "OK"){ + failed = true; + + logger.error("Test Failed:") + console.log(job, result) + }else{ + logger.info("Test Passed") + } + } + } + + if(failed) { + logger.error("One or more tests failed") + process.exit(1); + } + else { + logger.info("All tests passed") + process.exit(0); + } +})() \ No newline at end of file diff --git a/api/src/config.js b/api/src/config.js index 8ddc3f5..897a938 100644 --- a/api/src/config.js +++ b/api/src/config.js @@ -176,6 +176,12 @@ const options = [ default: 'github:engineer-man/piston?directory=packages', validators: [], }, + { + key: 'runtime_set', + desc: 'Key on the flake specified by flake_path to access runtimes from', + default: 'all', + validators: [] + }, { key: 'max_concurrent_jobs', desc: 'Maximum number of concurrent jobs to run at one time', diff --git a/api/src/runtime.js b/api/src/runtime.js index 9ebd724..e6f72fb 100644 --- a/api/src/runtime.js +++ b/api/src/runtime.js @@ -17,7 +17,7 @@ class Runtime { run, compile, packageSupport, - flake_key, + flake_path, timeouts, memory_limits, max_process_count, @@ -41,7 +41,7 @@ class Runtime { this.run = run; this.compile = compile; - this.flake_key = flake_key; + this.flake_path = flake_path; this.package_support = packageSupport; } @@ -112,10 +112,10 @@ class Runtime { ensure_built(){ logger.info(`Ensuring ${this} is built`); - const flake_key = this.flake_key; + const flake_path = this.flake_path; function _ensure_built(key){ - const command = `nix build ${config.flake_path}#pistonRuntimes.${flake_key}.metadata.${key} --no-link`; + const command = `nix build ${flake_path}.metadata.${key} --no-link`; cp.execSync(command, {stdio: "pipe"}) } @@ -128,7 +128,8 @@ class Runtime { static load_runtime(flake_key){ logger.info(`Loading ${flake_key}`) - const metadata_command = `nix eval --json ${config.flake_path}#pistonRuntimes.${flake_key}.metadata`; + const flake_path = `${config.flake_path}#pistonRuntimeSets.${config.runtime_set}.${flake_key}`; + const metadata_command = `nix eval --json ${flake_path}.metadata`; const metadata = JSON.parse(cp.execSync(metadata_command)); const this_runtime = new Runtime({ @@ -137,7 +138,7 @@ class Runtime { metadata.language, metadata.limit_overrides ), - flake_key + flake_path }); this_runtime.ensure_built(); diff --git a/flake.nix b/flake.nix index cce2f11..78b2ee1 100644 --- a/flake.nix +++ b/flake.nix @@ -6,6 +6,10 @@ let system = "x86_64-linux"; pkgs = nixpkgs.legacyPackages.${system}; + baseContainer = (import ./api { + inherit pkgs; + nosocket = self.legacyPackages."${system}".nosocket; + }).container; args = { inherit pkgs; piston = { @@ -53,66 +57,21 @@ }; }; }; - allRuntimes = import ./runtimes args; + runtimes = import ./runtimes args; + runtimeList = names: pkgs.lib.filterAttrs (n: v: n == "bash") runtimes; in { piston = args.piston; - pistonRuntimes = { - "bash" = allRuntimes.bash; + pistonRuntimes = runtimes; + pistonRuntimeSets = { + "all" = runtimes; + "bash-only" = runtimeList ["bash"]; }; - legacyPackages."${system}" = { - piston = (import ./api { inherit pkgs; }).package; + legacyPackages."${system}" = rec { nosocket = (import ./nosocket { inherit pkgs; }).package; + piston = (import ./api { inherit pkgs nosocket; }).package; }; - containerImage = pkgs.dockerTools.buildLayeredImageWithNixDb { - name = "piston"; - tag = "latest"; - - contents = with pkgs; [ - self.legacyPackages."${system}".piston - self.legacyPackages."${system}".nosocket - bash - nixFlakes - coreutils-full - cacert.out - git - gnutar - gzip - gnugrep - util-linux - ]; - - extraCommands = '' - mkdir -p piston/jobs etc/nix {,var/}tmp run/lock - echo -e "experimental-features = nix-command flakes" >> etc/nix/nix.conf - echo "nixbld:x:30000:nixbld1,nixbld10,nixbld11,nixbld12,nixbld13,nixbld14,nixbld15,nixbld16,nixbld17,nixbld18,nixbld19,nixbld2,nixbld20,nixbld21,nixbld22,nixbld23,nixbld24,nixbld25,nixbld26,nixbld27,nixbld28,nixbld29,nixbld3,nixbld30,nixbld31,nixbld32,nixbld4,nixbld5,nixbld6,nixbld7,nixbld8,nixbld9" >> etc/group - for i in $(seq 1 32) - do - echo "nixbld$i:x:$(( $i + 30000 )):30000:Nix build user $i:/var/empty:/run/current-system/sw/bin/nologin" >> etc/passwd - done - ''; - - config = { - Cmd = [ - "${self.legacyPackages."${system}".piston}/bin/pistond" - ]; - - Env = [ - "NIX_PAGER=cat" - "USER=nobody" - "SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt" - "GIT_SSL_CAINFO=/etc/ssl/certs/ca-bundle.crt" - "NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt" - - ]; - - ExposedPorts = { - "2000/tcp" = {}; - }; - }; - }; - - + container = baseContainer; }; } diff --git a/piston b/piston index fbd256d..67d2fa0 100755 --- a/piston +++ b/piston @@ -1,110 +1,93 @@ #!/usr/bin/env bash +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -EXECUTION_PATH="$PWD" -PISTON_PATH="$(dirname "$(realpath "$0")")" -cd "$PISTON_PATH" -PISTON_ENV=$(cat .piston_env 2> /dev/null || echo dev) +CONTAINER_NAME="piston_api" -docker_compose(){ - if [ -f "docker-compose.$PISTON_ENV.yaml" ]; then - docker-compose -f "docker-compose.$PISTON_ENV.yaml" "$@" - else - docker-compose "$@" - fi +IMAGE_TAG="base-latest" +IMAGE_NAME="ghcr.io/piston" +IMAGE_NAME_DEV="piston" + +SUBCOMMAND="$1" +shift + +cmd_build(){ + CONTAINER_PATH="$(nix build ".#container" --no-link --json | jq '.[0].outputs.out' -r)" + docker load -i $CONTAINER_PATH } -init_precommit() { - if [ $PISTON_ENV == "dev" ]; then - rm -f .git/hooks/pre-commit - ln -s "$PISTON_PATH/pre-commit" "$PISTON_PATH/.git/hooks/pre-commit" - fi -} +case "$SUBCOMMAND" in + logs) docker logs -f $CONTAINER_NAME;; -case $1 in - help) + restart) docker restart $CONTAINER_NAME ;; + start) + docker run \ + --rm \ + --name $CONTAINER_NAME \ + -it "$IMAGE_NAME:$IMAGE_TAG" + ;; + stop) docker stop $CONTAINER_NAME ;; + bash|shell) docker exec -it $CONTAINER_NAME bash ;; + + update) + git pull + docker pull "$IMAGE_NAME:$IMAGE_TAG" + ;; + + # dev commands + + build) cmd_build ;; + + start-dev) + cmd_build + docker run \ + --rm \ + -it \ + --name $CONTAINER_NAME \ + -e PISTON_FLAKE_PATH=/piston/packages \ + -v $PWD:/piston/packages \ + -it "$IMAGE_NAME_DEV:$IMAGE_TAG" + ;; + + test) + cmd_build + docker run \ + --rm \ + -it \ + -e PISTON_FLAKE_PATH=/piston/packages \ + -v $PWD:/piston/packages \ + --name piston_test_runner \ + -it "$IMAGE_NAME_DEV:$IMAGE_TAG" \ + piston-test $1 + + ;; + + *) echo "=== Piston Management ===" - echo "Current Environment: $PISTON_ENV" echo echo "Commands:" - echo " select Select the environment" - echo " docker_compose Interact directly with the docker-compose for the selected environment" - echo " logs Show docker-compose logs" + echo " logs Show piston logs" echo echo " start Starts piston" echo " stop Stops piston" echo " restart Restarts piston" - echo " bash Opens a bash shell for the piston_api container" + echo " shell Opens a bash shell for the api container" echo - echo " update Fetches and applies latest updates" + echo " update Fetches latest updates" echo - echo " Passthrough to piston cli tool" + echo " exec Execute the files on piston with language" echo echo "Development Commands:" - - if [ "$PISTON_ENV" == dev ]; then - - echo " clean-pkgs Clean any package build artifacts on disk" - echo " clean-repo Remove all packages from local repo" - echo " list-pkgs Lists all packages that can be built" - echo " build-pkg [builder] Build a package [with desired builder image]" - echo " rebuild Build and restart the docker container" - echo " lint Lint the codebase using prettier" - - else - - echo " Switch to developement environment for more info" - echo " > piston select dev" - - fi - ;; - - - select) echo "$2" > .piston_env ;; - docker_compose) shift; docker_compose "$@";; - logs) docker_compose logs -f ;; - - restart) docker_compose restart ;; - start) - init_precommit - docker_compose up -d - ;; - stop) docker_compose down ;; - bash) docker_compose exec api /bin/bash ;; - - rebuild) - init_precommit - docker_compose build && docker_compose up -d - ;; - - update) - git pull - cd cli && npm i > /dev/null && cd - - docker_compose pull - docker_compose up -d - ;; - - clean-pkgs) git clean -fqXd packages ;; - clean-repo) git clean -fqXd repo ;; - - list-pkgs) find packages -depth 2 | awk -F/ '$2 && $3{ print $2 "-" $3 }' | column ;; - - build-pkg) - PKGSLUG="$2-$3" - BUILDER="${4:-piston-repo-builder}" - echo "Building $PKGSLUG" - echo "Ensuring latest builder image" - docker build repo -t "$BUILDER" - docker run --rm -v "$PWD:/piston" "$BUILDER" --no-server "$PKGSLUG" - ;; - - lint) - npm install - npx prettier --ignore-unknown --write . - ;; - *) - [ -d ./cli/node_modules ] || npm i > /dev/null - cd "$EXECUTION_PATH" - node "${PISTON_PATH}/cli/index.js" "$@" + echo "Running some of these commands require a nix environment setup and on the path" + echo "See https://nixos.wiki/wiki/Nix_Installation_Guide#Stable_Nix" + echo + echo " start-dev Builds a container locally and starts piston" + echo " build Builds and loads the API container" + echo " scaffold [runtime] Initializes a new runtime" + echo " test Runs unit tests on the given runtime" + echo " Optionally set runtime to --all to test all" + echo " NOTE: This is only for the runtimes contained" + echo " within this repo" + echo ;; esac diff --git a/result b/result deleted file mode 120000 index d60c5ac..0000000 --- a/result +++ /dev/null @@ -1 +0,0 @@ -/nix/store/1idnmsddirm1hpxw40r17wsg9p9dscb9-piston.tar.gz \ No newline at end of file From 83e4a1a136f0d027cff88541a14bc9e4df7c6b82 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Mon, 31 Jan 2022 17:01:28 +1300 Subject: [PATCH 68/72] Fix issues after merging upstream Implements a simple container builder for runtime sets --- Dockerfile.withset | 12 +++++++ api/default.nix | 17 ++++++++++ api/src/bin/test.js | 81 ++++++++++++++++++++++----------------------- api/src/job.js | 6 +--- package-lock.json | 32 ++++++++++++++++++ package.json | 5 +++ piston | 37 ++++++++++++++++----- 7 files changed, 135 insertions(+), 55 deletions(-) create mode 100644 Dockerfile.withset create mode 100644 package-lock.json create mode 100644 package.json diff --git a/Dockerfile.withset b/Dockerfile.withset new file mode 100644 index 0000000..4fd9a0b --- /dev/null +++ b/Dockerfile.withset @@ -0,0 +1,12 @@ +# This "FROM" image is previously emitted by nix +FROM ghcr.io/engineer-man/piston:base-latest + +ENV PISTON_FLAKE_PATH=/piston/packages +COPY runtimes/ /piston/packages/runtimes +COPY flake.nix flake.lock /piston/packages/ + + +ARG RUNTIMESET=all +ENV PISTON_RUNTIME_SET=$RUNTIMESET + +RUN piston-install \ No newline at end of file diff --git a/api/default.nix b/api/default.nix index c194587..a4d4868 100644 --- a/api/default.nix +++ b/api/default.nix @@ -51,6 +51,8 @@ with pkgs; rec { do echo "nixbld$i:x:$(( $i + 30000 )):30000:Nix build user $i:/var/empty:/run/current-system/sw/bin/nologin" >> etc/passwd done + + chmod 1777 {,var/}tmp/ ''; config = { @@ -61,6 +63,21 @@ with pkgs; rec { "SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt" "GIT_SSL_CAINFO=/etc/ssl/certs/ca-bundle.crt" "NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt" + "PATH=${lib.concatStringsSep ":" [ + "/usr/local/sbin" + "/usr/local/bin" + "/usr/sbin" + "/usr/bin" + "/sbin" + "/bin" + "/root/.nix-profile/bin" + "/nix/var/nix/profiles/default/bin" + "/nix/var/nix/profiles/default/sbin" + ]}" + "MANPATH=${lib.concatStringsSep ":" [ + "/root/.nix-profile/share/man" + "/nix/var/nix/profiles/default/share/man" + ]}" ]; ExposedPorts = { diff --git a/api/src/bin/test.js b/api/src/bin/test.js index 134ce2a..64c467e 100755 --- a/api/src/bin/test.js +++ b/api/src/bin/test.js @@ -5,108 +5,105 @@ const config = require('../config'); const Logger = require('logplease'); const logger = Logger.create('test'); const cp = require('child_process'); -const runtime = require("../runtime"); +const runtime = require('../runtime'); const { Job } = require('../job'); -(async function(){ +(async function () { logger.info('Setting loglevel to', config.log_level); Logger.setLogLevel(config.log_level); - - let runtimes_to_test; let failed = false; - if(process.argv[2] === "--all"){ + if (process.argv[2] === '--all') { // load all runtimes_to_test = JSON.parse( - cp.execSync(`nix eval ${config.flake_path}#pistonRuntimes --json --apply builtins.attrNames`) + cp.execSync( + `nix eval ${config.flake_path}#pistonRuntimes --json --apply builtins.attrNames` + ) ); - }else{ + } else { runtimes_to_test = [process.argv[2]]; } - - for (const runtime_name of runtimes_to_test) { - - const runtime_path = `${config.flake_path}#pistonRuntimes.${runtime_name}`; logger.info(`Testing runtime ${runtime_path}`); logger.debug(`Loading runtime metadata`); - const metadata = JSON.parse(cp.execSync(`nix eval --json ${runtime_path}.metadata --json`)); + const metadata = JSON.parse( + cp.execSync(`nix eval --json ${runtime_path}.metadata --json`) + ); logger.debug(`Loading runtime tests`); - const tests = JSON.parse(cp.execSync(`nix eval --json ${runtime_path}.tests --json`)); + const tests = JSON.parse( + cp.execSync(`nix eval --json ${runtime_path}.tests --json`) + ); logger.debug(`Loading runtime`); const testable_runtime = new runtime.Runtime({ ...metadata, - flake_path: runtime_path + ...runtime.Runtime.compute_all_limits( + metadata.language, + metadata.limit_overrides + ), + flake_path: runtime_path, }); testable_runtime.ensure_built(); - logger.info(`Running tests`); for (const test of tests) { - const files = []; for (const file_name of Object.keys(test.files)) { const file_content = test.files[file_name]; const this_file = { name: file_name, - content: file_content + content: file_content, }; - if(file_name == test.main) - files.unshift(this_file); - else - files.push(this_file); - + if (file_name == test.main) files.unshift(this_file); + else files.push(this_file); } - const job = new Job({ runtime: testable_runtime, args: test.args || [], - stdin: test.stdin || "", + stdin: test.stdin || '', files, timeouts: { run: 3000, - compile: 10000 + compile: 10000, }, memory_limits: { run: config.run_memory_limit, - compile: config.compile_memory_limit - } + compile: config.compile_memory_limit, + }, }); - await job.prime() - const result = await job.execute() - await job.cleanup() - - if(result.run.stdout.trim() !== "OK"){ + await job.prime(); + const result = await job.execute(); + await job.cleanup(); + + if (result.run.stdout.trim() !== 'OK') { failed = true; - logger.error("Test Failed:") - console.log(job, result) - }else{ - logger.info("Test Passed") + logger.error('Test Failed:'); + console.log(job, result); + } else { + logger.info('Test Passed'); } } } - if(failed) { - logger.error("One or more tests failed") + if (failed) { + logger.error('One or more tests failed'); process.exit(1); - } - else { - logger.info("All tests passed") + } else { + logger.info('All tests passed'); process.exit(0); } -})() \ No newline at end of file +})(); diff --git a/api/src/job.js b/api/src/job.js index e05f017..c562693 100644 --- a/api/src/job.js +++ b/api/src/job.js @@ -139,10 +139,6 @@ class Job { var output = ''; const proc = cp.spawn(proc_call[0], proc_call.splice(1), { - env: { - ...this.runtime.env_vars, - PISTON_LANGUAGE: this.runtime.language, - }, stdio: 'pipe', cwd: this.dir, uid: this.uid, @@ -250,7 +246,7 @@ class Job { this.logger.debug('Running'); const run = await this.safe_call( - path.join(this.runtime.pkgdir, 'run'), + this.runtime.run, [code_files[0].name, ...this.args], this.timeouts.run, this.memory_limits.run diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..5c51a1d --- /dev/null +++ b/package-lock.json @@ -0,0 +1,32 @@ +{ + "name": "piston", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "devDependencies": { + "prettier": "2.4.1" + } + }, + "node_modules/prettier": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.4.1.tgz", + "integrity": "sha512-9fbDAXSBcc6Bs1mZrDYb3XKzDLm4EXXL9sC1LqKP5rZkT6KRr/rf9amVUcODVXgguK/isJz0d0hP72WeaKWsvA==", + "dev": true, + "bin": { + "prettier": "bin-prettier.js" + }, + "engines": { + "node": ">=10.13.0" + } + } + }, + "dependencies": { + "prettier": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.4.1.tgz", + "integrity": "sha512-9fbDAXSBcc6Bs1mZrDYb3XKzDLm4EXXL9sC1LqKP5rZkT6KRr/rf9amVUcODVXgguK/isJz0d0hP72WeaKWsvA==", + "dev": true + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..8f07606 --- /dev/null +++ b/package.json @@ -0,0 +1,5 @@ +{ + "devDependencies": { + "prettier": "2.4.1" + } +} diff --git a/piston b/piston index 67d2fa0..cacf23e 100755 --- a/piston +++ b/piston @@ -5,15 +5,16 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) CONTAINER_NAME="piston_api" IMAGE_TAG="base-latest" -IMAGE_NAME="ghcr.io/piston" +IMAGE_NAME="ghcr.io/engineer-man/piston" IMAGE_NAME_DEV="piston" SUBCOMMAND="$1" shift -cmd_build(){ +build_base(){ CONTAINER_PATH="$(nix build ".#container" --no-link --json | jq '.[0].outputs.out' -r)" docker load -i $CONTAINER_PATH + docker tag "$IMAGE_NAME_DEV:$IMAGE_TAG" "$IMAGE_NAME:$IMAGE_TAG" } case "$SUBCOMMAND" in @@ -22,9 +23,10 @@ case "$SUBCOMMAND" in restart) docker restart $CONTAINER_NAME ;; start) docker run \ + -p 2000:2000 \ --rm \ --name $CONTAINER_NAME \ - -it "$IMAGE_NAME:$IMAGE_TAG" + -d "$IMAGE_NAME:$IMAGE_TAG" ;; stop) docker stop $CONTAINER_NAME ;; bash|shell) docker exec -it $CONTAINER_NAME bash ;; @@ -35,22 +37,40 @@ case "$SUBCOMMAND" in ;; # dev commands + + scaffold) + pushd $SCRIPT_DIR/runtimes > /dev/null + ./scaffold.sh $1 $2 + popd > /dev/null + ;; - build) cmd_build ;; + build) + build_base + if [[ ! -z "$1" ]]; then + # $1 contains a variant to build + docker build \ + --build-arg RUNTIMESET=$1 \ + -f $SCRIPT_DIR/Dockerfile.withset \ + -t "$IMAGE_NAME_DEV:$1-latest" \ + . + fi + ;; start-dev) - cmd_build + build_base docker run \ --rm \ + -p 2000:2000 \ -it \ --name $CONTAINER_NAME \ + -e PISTON_LOG_LEVEL=DEBUG \ -e PISTON_FLAKE_PATH=/piston/packages \ -v $PWD:/piston/packages \ - -it "$IMAGE_NAME_DEV:$IMAGE_TAG" + -d "$IMAGE_NAME_DEV:$IMAGE_TAG" ;; test) - cmd_build + build_base docker run \ --rm \ -it \ @@ -82,7 +102,8 @@ case "$SUBCOMMAND" in echo "See https://nixos.wiki/wiki/Nix_Installation_Guide#Stable_Nix" echo echo " start-dev Builds a container locally and starts piston" - echo " build Builds and loads the API container" + echo " build [runtime-set] Builds and loads the API container optionally" + echo " including the runtime set within it" echo " scaffold [runtime] Initializes a new runtime" echo " test Runs unit tests on the given runtime" echo " Optionally set runtime to --all to test all" From 388b781acaeca4048a78f42c8f0589f2d3087640 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Mon, 31 Jan 2022 17:18:12 +1300 Subject: [PATCH 69/72] Update scaffold tooling to give test command --- runtimes/.scaffold.nix | 3 +++ runtimes/scaffold.sh | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/runtimes/.scaffold.nix b/runtimes/.scaffold.nix index 51b68aa..de7bc8f 100644 --- a/runtimes/.scaffold.nix +++ b/runtimes/.scaffold.nix @@ -49,6 +49,9 @@ in piston.mkRuntime { # Specify a list of tests. # These should output "OK" to STDOUT if everything looks good + # + # Run the following command to test the package: + # $ ./piston test %NAME% tests = [ (piston.mkTest { files = { diff --git a/runtimes/scaffold.sh b/runtimes/scaffold.sh index fb3aafe..21858bc 100755 --- a/runtimes/scaffold.sh +++ b/runtimes/scaffold.sh @@ -28,10 +28,13 @@ else echo " \"$NAME\" = import ./$NAME.nix args;" >> default.nix sed -e 's/%LANGUAGE%/'"$LANGUAGE"'/g' \ -e 's/%RUNTIME%/'"$RUNTIME"'/g' \ + -e 's/%NAME%/'"$NAME"'/g' \ .scaffold.nix > $NAME.nix - git add $NAME.nix echo "}" >> default.nix + git add $NAME.nix default.nix + echo "Scaffolded $NAME" echo "Edit $NAME.nix to get started" + echo "Once you are done, run ./piston test $NAME to test it" fi \ No newline at end of file From 8d6ecfaf37dda56c500cceb0ffbd2a9b77473acc Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Mon, 31 Jan 2022 17:46:48 +1300 Subject: [PATCH 70/72] fix scaffold script file naming --- runtimes/scaffold.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtimes/scaffold.sh b/runtimes/scaffold.sh index 21858bc..2a49ef2 100755 --- a/runtimes/scaffold.sh +++ b/runtimes/scaffold.sh @@ -15,7 +15,7 @@ NAME=$1 if [[ $# -eq 2 ]]; then RUNTIME=$2 - NAME=$LANGUAGE-$RUNTIME + NAME=$RUNTIME-$LANGUAGE fi From ddab59ccddb4e1a90e2d174baebcd0a4b2bb1197 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Mon, 31 Jan 2022 17:47:27 +1300 Subject: [PATCH 71/72] add limit overrides to runtime definitions --- api/src/bin/test.js | 2 +- api/src/runtime.js | 41 +++++++++++++++++------------------------ flake.nix | 3 ++- 3 files changed, 20 insertions(+), 26 deletions(-) diff --git a/api/src/bin/test.js b/api/src/bin/test.js index 64c467e..16de55d 100755 --- a/api/src/bin/test.js +++ b/api/src/bin/test.js @@ -46,7 +46,7 @@ const { Job } = require('../job'); ...metadata, ...runtime.Runtime.compute_all_limits( metadata.language, - metadata.limit_overrides + metadata.limitOverrides ), flake_path: runtime_path, }); diff --git a/api/src/runtime.js b/api/src/runtime.js index e6f72fb..b239426 100644 --- a/api/src/runtime.js +++ b/api/src/runtime.js @@ -7,7 +7,6 @@ const path = require('path'); const runtimes = []; - class Runtime { constructor({ language, @@ -23,11 +22,11 @@ class Runtime { max_process_count, max_open_files, max_file_size, - output_max_size + output_max_size, }) { this.language = language; this.runtime = runtime; - + this.timeouts = timeouts; this.memory_limits = memory_limits; this.max_process_count = max_process_count; @@ -36,15 +35,14 @@ class Runtime { this.output_max_size = output_max_size; this.aliases = aliases; - this.version = version; - + this.version = version; + this.run = run; this.compile = compile; this.flake_path = flake_path; this.package_support = packageSupport; } - static compute_single_limit( language_name, @@ -109,62 +107,57 @@ class Runtime { }; } - ensure_built(){ + ensure_built() { logger.info(`Ensuring ${this} is built`); const flake_path = this.flake_path; - function _ensure_built(key){ + function _ensure_built(key) { const command = `nix build ${flake_path}.metadata.${key} --no-link`; - cp.execSync(command, {stdio: "pipe"}) + cp.execSync(command, { stdio: 'pipe' }); } - _ensure_built("run"); - if(this.compiled) _ensure_built("compile"); - - logger.debug(`Finished ensuring ${this} is installed`) + _ensure_built('run'); + if (this.compiled) _ensure_built('compile'); + logger.debug(`Finished ensuring ${this} is installed`); } - static load_runtime(flake_key){ - logger.info(`Loading ${flake_key}`) + static load_runtime(flake_key) { + logger.info(`Loading ${flake_key}`); const flake_path = `${config.flake_path}#pistonRuntimeSets.${config.runtime_set}.${flake_key}`; const metadata_command = `nix eval --json ${flake_path}.metadata`; const metadata = JSON.parse(cp.execSync(metadata_command)); - + const this_runtime = new Runtime({ ...metadata, ...Runtime.compute_all_limits( metadata.language, - metadata.limit_overrides + metadata.limitOverrides ), - flake_path + flake_path, }); this_runtime.ensure_built(); runtimes.push(this_runtime); - - - logger.debug(`Package ${flake_key} was loaded`); + logger.debug(`Package ${flake_key} was loaded`); } get compiled() { return this.compile !== null; } - get id(){ + get id() { return runtimes.indexOf(this); } toString() { return `${this.language}-${this.version}`; } - } module.exports = runtimes; module.exports.Runtime = Runtime; module.exports.load_runtime = Runtime.load_runtime; - diff --git a/flake.nix b/flake.nix index 78b2ee1..fde9cee 100644 --- a/flake.nix +++ b/flake.nix @@ -21,6 +21,7 @@ compile? null, packages? null, aliases? [], + limitOverrides? {}, tests }: let compileFile = if compile != null then @@ -28,7 +29,7 @@ else null; runFile = pkgs.writeShellScript "run" run; metadata = { - inherit language version runtime aliases; + inherit language version runtime aliases limitOverrides; run = runFile; compile = compileFile; packageSupport = packages != null; From af5036d82c651f7de8c3eb2c615746c7c92116f5 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Mon, 31 Jan 2022 18:02:35 +1300 Subject: [PATCH 72/72] Convert some packages to nix-based Affected packages: - crystal - dart - dash - deno - elixir - erlang - gawk --- packages/crystal/0.36.1/build.sh | 7 ------ packages/crystal/0.36.1/compile | 5 ----- packages/crystal/0.36.1/environment | 1 - packages/crystal/0.36.1/metadata.json | 5 ----- packages/crystal/0.36.1/run | 4 ---- packages/crystal/0.36.1/test.cr | 1 - packages/dart/2.12.1/build.sh | 11 --------- packages/dart/2.12.1/environment | 4 ---- packages/dart/2.12.1/metadata.json | 5 ----- packages/dart/2.12.1/run | 4 ---- packages/dart/2.12.1/test.dart | 3 --- packages/dash/0.5.11/build.sh | 19 ---------------- packages/dash/0.5.11/environment | 4 ---- packages/dash/0.5.11/metadata.json | 5 ----- packages/dash/0.5.11/run | 4 ---- packages/dash/0.5.11/test.dash | 1 - packages/deno/1.16.2/build.sh | 5 ----- packages/deno/1.16.2/environment | 3 --- packages/deno/1.16.2/metadata.json | 20 ----------------- packages/deno/1.16.2/run | 3 --- packages/deno/1.16.2/test.deno.ts | 1 - packages/deno/1.7.5/build.sh | 5 ----- packages/deno/1.7.5/environment | 1 - packages/deno/1.7.5/metadata.json | 14 ------------ packages/deno/1.7.5/run | 2 -- packages/deno/1.7.5/test.deno.ts | 1 - packages/elixir/1.11.3/build.sh | 25 --------------------- packages/elixir/1.11.3/environment | 5 ----- packages/elixir/1.11.3/metadata.json | 5 ----- packages/elixir/1.11.3/run | 4 ---- packages/elixir/1.11.3/test.exs | 1 - packages/erlang/23.0.0/build.sh | 21 ----------------- packages/erlang/23.0.0/environment | 4 ---- packages/erlang/23.0.0/metadata.json | 5 ----- packages/erlang/23.0.0/run | 4 ---- packages/erlang/23.0.0/test.erl | 3 --- packages/file/0.0.1/build.sh | 3 --- packages/file/0.0.1/environment | 3 --- packages/file/0.0.1/metadata.json | 5 ----- packages/file/0.0.1/run | 6 ----- packages/file/0.0.1/test.file | Bin 16568 -> 0 bytes packages/gawk/5.1.0/build.sh | 21 ----------------- packages/gawk/5.1.0/environment | 4 ---- packages/gawk/5.1.0/metadata.json | 10 --------- packages/gawk/5.1.0/run | 4 ---- packages/gawk/5.1.0/test.awk | 1 - packages/python/3.10.0/build.sh | 21 ----------------- packages/python/3.10.0/environment | 1 - packages/python/3.10.0/metadata.json | 5 ----- packages/python/3.10.0/run | 3 --- packages/python/3.10.0/test.py | 7 ------ runtimes/crystal.nix | 31 ++++++++++++++++++++++++++ runtimes/dart.nix | 25 +++++++++++++++++++++ runtimes/dash.nix | 22 ++++++++++++++++++ runtimes/default.nix | 8 +++++++ runtimes/deno-javascript.nix | 27 ++++++++++++++++++++++ runtimes/deno-typescript.nix | 27 ++++++++++++++++++++++ runtimes/elixir.nix | 30 +++++++++++++++++++++++++ runtimes/erlang.nix | 28 +++++++++++++++++++++++ runtimes/gawk-awk.nix | 26 +++++++++++++++++++++ 60 files changed, 224 insertions(+), 309 deletions(-) delete mode 100755 packages/crystal/0.36.1/build.sh delete mode 100644 packages/crystal/0.36.1/compile delete mode 100644 packages/crystal/0.36.1/environment delete mode 100644 packages/crystal/0.36.1/metadata.json delete mode 100644 packages/crystal/0.36.1/run delete mode 100644 packages/crystal/0.36.1/test.cr delete mode 100755 packages/dart/2.12.1/build.sh delete mode 100644 packages/dart/2.12.1/environment delete mode 100644 packages/dart/2.12.1/metadata.json delete mode 100644 packages/dart/2.12.1/run delete mode 100644 packages/dart/2.12.1/test.dart delete mode 100755 packages/dash/0.5.11/build.sh delete mode 100644 packages/dash/0.5.11/environment delete mode 100644 packages/dash/0.5.11/metadata.json delete mode 100644 packages/dash/0.5.11/run delete mode 100644 packages/dash/0.5.11/test.dash delete mode 100644 packages/deno/1.16.2/build.sh delete mode 100644 packages/deno/1.16.2/environment delete mode 100644 packages/deno/1.16.2/metadata.json delete mode 100644 packages/deno/1.16.2/run delete mode 100644 packages/deno/1.16.2/test.deno.ts delete mode 100755 packages/deno/1.7.5/build.sh delete mode 100644 packages/deno/1.7.5/environment delete mode 100644 packages/deno/1.7.5/metadata.json delete mode 100644 packages/deno/1.7.5/run delete mode 100644 packages/deno/1.7.5/test.deno.ts delete mode 100755 packages/elixir/1.11.3/build.sh delete mode 100644 packages/elixir/1.11.3/environment delete mode 100644 packages/elixir/1.11.3/metadata.json delete mode 100644 packages/elixir/1.11.3/run delete mode 100644 packages/elixir/1.11.3/test.exs delete mode 100755 packages/erlang/23.0.0/build.sh delete mode 100644 packages/erlang/23.0.0/environment delete mode 100644 packages/erlang/23.0.0/metadata.json delete mode 100644 packages/erlang/23.0.0/run delete mode 100644 packages/erlang/23.0.0/test.erl delete mode 100755 packages/file/0.0.1/build.sh delete mode 100644 packages/file/0.0.1/environment delete mode 100644 packages/file/0.0.1/metadata.json delete mode 100644 packages/file/0.0.1/run delete mode 100755 packages/file/0.0.1/test.file delete mode 100644 packages/gawk/5.1.0/build.sh delete mode 100644 packages/gawk/5.1.0/environment delete mode 100644 packages/gawk/5.1.0/metadata.json delete mode 100644 packages/gawk/5.1.0/run delete mode 100644 packages/gawk/5.1.0/test.awk delete mode 100755 packages/python/3.10.0/build.sh delete mode 100644 packages/python/3.10.0/environment delete mode 100644 packages/python/3.10.0/metadata.json delete mode 100644 packages/python/3.10.0/run delete mode 100644 packages/python/3.10.0/test.py create mode 100644 runtimes/crystal.nix create mode 100644 runtimes/dart.nix create mode 100644 runtimes/dash.nix create mode 100644 runtimes/deno-javascript.nix create mode 100644 runtimes/deno-typescript.nix create mode 100644 runtimes/elixir.nix create mode 100644 runtimes/erlang.nix create mode 100644 runtimes/gawk-awk.nix diff --git a/packages/crystal/0.36.1/build.sh b/packages/crystal/0.36.1/build.sh deleted file mode 100755 index ba10f3f..0000000 --- a/packages/crystal/0.36.1/build.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -PREFIX=$(realpath $(dirname $0)) - -curl -L "https://github.com/crystal-lang/crystal/releases/download/0.36.1/crystal-0.36.1-1-linux-x86_64.tar.gz" -o crystal.tar.gz -tar xzf crystal.tar.gz --strip-components=1 -rm crystal.tar.gz diff --git a/packages/crystal/0.36.1/compile b/packages/crystal/0.36.1/compile deleted file mode 100644 index afbad6c..0000000 --- a/packages/crystal/0.36.1/compile +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash - -# Compile crystal files into out file -crystal build "$@" -o out --no-color && \ -chmod +x out diff --git a/packages/crystal/0.36.1/environment b/packages/crystal/0.36.1/environment deleted file mode 100644 index bd0ff98..0000000 --- a/packages/crystal/0.36.1/environment +++ /dev/null @@ -1 +0,0 @@ -export PATH=$PWD/bin:$PATH \ No newline at end of file diff --git a/packages/crystal/0.36.1/metadata.json b/packages/crystal/0.36.1/metadata.json deleted file mode 100644 index ee995eb..0000000 --- a/packages/crystal/0.36.1/metadata.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "language": "crystal", - "version": "0.36.1", - "aliases": ["crystal", "cr"] -} diff --git a/packages/crystal/0.36.1/run b/packages/crystal/0.36.1/run deleted file mode 100644 index 6955ba9..0000000 --- a/packages/crystal/0.36.1/run +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -shift # Filename is only used to compile -./out "$@" diff --git a/packages/crystal/0.36.1/test.cr b/packages/crystal/0.36.1/test.cr deleted file mode 100644 index 3fca0dd..0000000 --- a/packages/crystal/0.36.1/test.cr +++ /dev/null @@ -1 +0,0 @@ -puts("OK") \ No newline at end of file diff --git a/packages/dart/2.12.1/build.sh b/packages/dart/2.12.1/build.sh deleted file mode 100755 index d202d7f..0000000 --- a/packages/dart/2.12.1/build.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -curl -L "https://storage.googleapis.com/dart-archive/channels/stable/release/2.12.1/sdk/dartsdk-linux-x64-release.zip" -o dart.zip - -unzip dart.zip -rm dart.zip - -cp -r dart-sdk/* . -rm -rf dart-sdk - -chmod -R +rx bin diff --git a/packages/dart/2.12.1/environment b/packages/dart/2.12.1/environment deleted file mode 100644 index 780b668..0000000 --- a/packages/dart/2.12.1/environment +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash - -# Put 'export' statements here for environment variables -export PATH=$PWD/bin:$PATH diff --git a/packages/dart/2.12.1/metadata.json b/packages/dart/2.12.1/metadata.json deleted file mode 100644 index cec3d77..0000000 --- a/packages/dart/2.12.1/metadata.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "language": "dart", - "version": "2.12.1", - "aliases": [] -} diff --git a/packages/dart/2.12.1/run b/packages/dart/2.12.1/run deleted file mode 100644 index aae792a..0000000 --- a/packages/dart/2.12.1/run +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash - -# Put instructions to run the runtime -dart run "$@" diff --git a/packages/dart/2.12.1/test.dart b/packages/dart/2.12.1/test.dart deleted file mode 100644 index 27e87b2..0000000 --- a/packages/dart/2.12.1/test.dart +++ /dev/null @@ -1,3 +0,0 @@ -void main() { - print('OK'); -} \ No newline at end of file diff --git a/packages/dash/0.5.11/build.sh b/packages/dash/0.5.11/build.sh deleted file mode 100755 index 202d5aa..0000000 --- a/packages/dash/0.5.11/build.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash - -# Put instructions to build your package in here -PREFIX=$(realpath $(dirname $0)) - -mkdir -p build - -cd build - -curl "http://gondor.apana.org.au/~herbert/dash/files/dash-0.5.11.tar.gz" -o dash.tar.gz -tar xzf dash.tar.gz --strip-components=1 - -./configure --prefix "$PREFIX" && -make -j$(nproc) && -make install -j$(nproc) - -cd ../ - -rm -rf build diff --git a/packages/dash/0.5.11/environment b/packages/dash/0.5.11/environment deleted file mode 100644 index 780b668..0000000 --- a/packages/dash/0.5.11/environment +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash - -# Put 'export' statements here for environment variables -export PATH=$PWD/bin:$PATH diff --git a/packages/dash/0.5.11/metadata.json b/packages/dash/0.5.11/metadata.json deleted file mode 100644 index a2e5be5..0000000 --- a/packages/dash/0.5.11/metadata.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "language": "dash", - "version": "0.5.11", - "aliases": ["dash"] -} diff --git a/packages/dash/0.5.11/run b/packages/dash/0.5.11/run deleted file mode 100644 index 6a8a1da..0000000 --- a/packages/dash/0.5.11/run +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash - -# Put instructions to run the runtime -dash "$@" diff --git a/packages/dash/0.5.11/test.dash b/packages/dash/0.5.11/test.dash deleted file mode 100644 index 727518f..0000000 --- a/packages/dash/0.5.11/test.dash +++ /dev/null @@ -1 +0,0 @@ -echo "OK" \ No newline at end of file diff --git a/packages/deno/1.16.2/build.sh b/packages/deno/1.16.2/build.sh deleted file mode 100644 index ec5c1d0..0000000 --- a/packages/deno/1.16.2/build.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -curl -OL https://github.com/denoland/deno/releases/download/v1.16.2/deno-x86_64-unknown-linux-gnu.zip -unzip -o deno-x86_64-unknown-linux-gnu.zip -rm deno-x86_64-unknown-linux-gnu.zip diff --git a/packages/deno/1.16.2/environment b/packages/deno/1.16.2/environment deleted file mode 100644 index d5a7803..0000000 --- a/packages/deno/1.16.2/environment +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -export PATH=$PWD:$PATH diff --git a/packages/deno/1.16.2/metadata.json b/packages/deno/1.16.2/metadata.json deleted file mode 100644 index 8043242..0000000 --- a/packages/deno/1.16.2/metadata.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "language": "deno", - "version": "1.16.2", - "dependencies": {}, - "provides": [ - { - "language": "typescript", - "aliases": [ - "deno", - "deno-ts" - ] - }, - { - "language": "javascript", - "aliases": [ - "deno-js" - ] - } - ] -} diff --git a/packages/deno/1.16.2/run b/packages/deno/1.16.2/run deleted file mode 100644 index 288919f..0000000 --- a/packages/deno/1.16.2/run +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -DENO_DIR=$PWD deno run $@ diff --git a/packages/deno/1.16.2/test.deno.ts b/packages/deno/1.16.2/test.deno.ts deleted file mode 100644 index 0b555b7..0000000 --- a/packages/deno/1.16.2/test.deno.ts +++ /dev/null @@ -1 +0,0 @@ -console.log("OK") diff --git a/packages/deno/1.7.5/build.sh b/packages/deno/1.7.5/build.sh deleted file mode 100755 index 165d3b6..0000000 --- a/packages/deno/1.7.5/build.sh +++ /dev/null @@ -1,5 +0,0 @@ -curl -L https://github.com/denoland/deno/releases/download/v1.7.5/deno-x86_64-unknown-linux-gnu.zip --output deno.zip -unzip -o deno.zip -rm deno.zip - -chmod +x deno diff --git a/packages/deno/1.7.5/environment b/packages/deno/1.7.5/environment deleted file mode 100644 index 98fd770..0000000 --- a/packages/deno/1.7.5/environment +++ /dev/null @@ -1 +0,0 @@ -export PATH=$PWD:$PATH \ No newline at end of file diff --git a/packages/deno/1.7.5/metadata.json b/packages/deno/1.7.5/metadata.json deleted file mode 100644 index 217a7c6..0000000 --- a/packages/deno/1.7.5/metadata.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "language": "deno", - "version": "1.7.5", - "provides": [ - { - "language": "typescript", - "aliases": ["deno-ts", "deno"] - }, - { - "language": "javascript", - "aliases": ["deno-js"] - } - ] -} diff --git a/packages/deno/1.7.5/run b/packages/deno/1.7.5/run deleted file mode 100644 index d1b196f..0000000 --- a/packages/deno/1.7.5/run +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -DENO_DIR=$PWD deno run "$@" \ No newline at end of file diff --git a/packages/deno/1.7.5/test.deno.ts b/packages/deno/1.7.5/test.deno.ts deleted file mode 100644 index e106678..0000000 --- a/packages/deno/1.7.5/test.deno.ts +++ /dev/null @@ -1 +0,0 @@ -console.log('OK'); diff --git a/packages/elixir/1.11.3/build.sh b/packages/elixir/1.11.3/build.sh deleted file mode 100755 index 9328d8c..0000000 --- a/packages/elixir/1.11.3/build.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -source ../../erlang/23.0.0/build.sh - -export PATH=$PWD/bin:$PATH - -PREFIX=$(realpath $(dirname $0)) - -mkdir -p build - -cd build - -curl -L "https://github.com/elixir-lang/elixir/archive/v1.11.3.tar.gz" -o elixir.tar.gz -tar xzf elixir.tar.gz --strip-components=1 -rm elixir.tar.gz - -./configure --prefix "$PREFIX" -make -j$(nproc) - -cd .. - -cp -r build/bin . -cp -r build/lib . - -rm -rf build diff --git a/packages/elixir/1.11.3/environment b/packages/elixir/1.11.3/environment deleted file mode 100644 index ea24603..0000000 --- a/packages/elixir/1.11.3/environment +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash - -# Put 'export' statements here for environment variables -export LC_ALL=en_US.UTF-8 -export PATH=$PWD/bin:$PATH diff --git a/packages/elixir/1.11.3/metadata.json b/packages/elixir/1.11.3/metadata.json deleted file mode 100644 index 9c1a2fc..0000000 --- a/packages/elixir/1.11.3/metadata.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "language": "elixir", - "version": "1.11.3", - "aliases": ["elixir", "exs"] -} diff --git a/packages/elixir/1.11.3/run b/packages/elixir/1.11.3/run deleted file mode 100644 index 9b3ba9c..0000000 --- a/packages/elixir/1.11.3/run +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -# Put instructions to run the runtime -elixir "$@" \ No newline at end of file diff --git a/packages/elixir/1.11.3/test.exs b/packages/elixir/1.11.3/test.exs deleted file mode 100644 index bc837c6..0000000 --- a/packages/elixir/1.11.3/test.exs +++ /dev/null @@ -1 +0,0 @@ -IO.puts("OK") \ No newline at end of file diff --git a/packages/erlang/23.0.0/build.sh b/packages/erlang/23.0.0/build.sh deleted file mode 100755 index f282bc2..0000000 --- a/packages/erlang/23.0.0/build.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -PREFIX=$(realpath $(dirname $0)) - -mkdir -p build - -cd build - -curl "http://erlang.org/download/otp_src_23.0.tar.gz" -o erlang.tar.gz -tar xzf erlang.tar.gz --strip-components=1 -rm erlang.tar.gz - -export ERL_TOP=$(pwd) -./configure --prefix "$PREFIX" -make -j$(nproc) -make install -j$(nproc) - -cd .. - -rm -rf build - diff --git a/packages/erlang/23.0.0/environment b/packages/erlang/23.0.0/environment deleted file mode 100644 index 780b668..0000000 --- a/packages/erlang/23.0.0/environment +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash - -# Put 'export' statements here for environment variables -export PATH=$PWD/bin:$PATH diff --git a/packages/erlang/23.0.0/metadata.json b/packages/erlang/23.0.0/metadata.json deleted file mode 100644 index e82b4b3..0000000 --- a/packages/erlang/23.0.0/metadata.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "language": "erlang", - "version": "23.0.0", - "aliases": ["erlang", "erl", "escript"] -} diff --git a/packages/erlang/23.0.0/run b/packages/erlang/23.0.0/run deleted file mode 100644 index 135d9f4..0000000 --- a/packages/erlang/23.0.0/run +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -# Put instructions to run the runtime -escript "$@" \ No newline at end of file diff --git a/packages/erlang/23.0.0/test.erl b/packages/erlang/23.0.0/test.erl deleted file mode 100644 index d898d2c..0000000 --- a/packages/erlang/23.0.0/test.erl +++ /dev/null @@ -1,3 +0,0 @@ - -main(_) -> - io:format("OK~n"). \ No newline at end of file diff --git a/packages/file/0.0.1/build.sh b/packages/file/0.0.1/build.sh deleted file mode 100755 index d7ab47e..0000000 --- a/packages/file/0.0.1/build.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -# Nothing to do here diff --git a/packages/file/0.0.1/environment b/packages/file/0.0.1/environment deleted file mode 100644 index d5a7803..0000000 --- a/packages/file/0.0.1/environment +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -export PATH=$PWD:$PATH diff --git a/packages/file/0.0.1/metadata.json b/packages/file/0.0.1/metadata.json deleted file mode 100644 index ae34ae9..0000000 --- a/packages/file/0.0.1/metadata.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "language": "file", - "version": "0.0.1", - "aliases": ["executable", "elf", "binary"] -} diff --git a/packages/file/0.0.1/run b/packages/file/0.0.1/run deleted file mode 100644 index 77ed0a6..0000000 --- a/packages/file/0.0.1/run +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -filename="$1" -chmod +x "$filename" -shift -./"$filename" "$@" diff --git a/packages/file/0.0.1/test.file b/packages/file/0.0.1/test.file deleted file mode 100755 index 8c7a67d92841fc71dd40ef76305328833c4aab6e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16568 zcmeHOZ*Wvs6~DV75Ck?0ZAwa&td-6{F-rmgL!H>&Bpcq^04Yf zZKzVI(VCS;(GN<;en`hZ)v2R6`hkAXmO*NrsScfSYVDs99b1D_D@AcqEv&zD?>(Eh zkEGLf`pJ2B=Iy!Xch2wJ`_8-X<=wn{KNsum@%wy&OO5!LAZ;qBkcc+SwCf5)M6`=# z_-zua#X{%{B&O^UC16$MIe)dR(|8Re@fxWzhkU<+1xxNBN#e~{rWPp+OQFw$c(r8G zx?6r%_md^tEA54dj1rQ+75RN~Kw;_RQEn!hrCCgLxTJ3Hv_6045lp-(&70CZmO}Gb zay}_1bhPVuZrK7G6_#$^u;#gCx1tG_)CQIQm+jQMX|GT7>OwM%aLWM&3zpp8&EQd9 z-frQ`yi2z?-*p(%`DCdtr`Rx(9@^5jVIWHzMTF;O6PcFyHeZ9|J={_)V=c9!NRFX?9$37)5ks;d;V$ip?(vGe5g=7 zzT~7+eK8L5QTh0qtP@7%wDsFe!_lQ(O1u2`IG`1m!{Uy4@Ndk6Q$IsE+~s!w=88}K z4Y{46e|R3e2l!lZP659bhr9d;z;dxlL?X%(qz0*&=Kf4BYZmN8-Zo8P#s{{WDJyU7 zPZw+}Kd`-PB$u@Y5L zIFUE)d?IZZ;>lQS7qCrmPAAP=!OmNWOguSY#`-s#eK#e+yDVN8$7@UGiZ;5P97~u9 z+Zs!=vM-%YjHK^DKTQnIq&}rgBApe`3wA2GemxD)Eu=c zhn7p4IHI3pjL&HNs0)v1ye^`6?9ck?8x>AtpT|5Q&pB|qkGPz1;PUye4bu)>-ABrB z&VlPcCDJhCz`0Lk;PFPW-zPoQ&vETfF?cAF#v%fNe?fq&Mo{*N*KYMpVo_759`FuwSN?Ju7;#(!CNQZ`fG{CU9V%h!!V)Hez8 zA12HBvwtd=%SWV6FF5DVK2y=@#pe9k6Rd-6Np`o8eCQo6eX~n%cIvHvkC&dg!zlgP z82`(;zJcD!+Go+CF}ci?qT>XbD>tkFzn5i**Pa4l$vadvdQ#wFh`OLohCY%TJ>HH6M z)V3gGas8k%-f<5+g;9FhUTPff_#Tw$H!hXS(Y+3_%uUj&)vY+~=pipTh?!P3DuOU0S-gKzqZ8}BQ5K2BwdLV@x)z z10T^oMdx+6b_r>vUrm1o#v{M31p5V1AEygA{SbPw_vF5CebX1HL0oNpy`s1DPIlb3 zUWn)?;-zP!gYnXZ=s>je=AbdzJOH@AcilxAL(@OLh=xyGtg)L~U(osOEuHTzz1Cg& zYqWgjAC2*oK4a@k#lO+`x@~Xt_UPW|9Z~b-zS&0UBx&T4l;g585@nyfA1?!52D}V- z8SpaTWx&gTmjN#WUIx4jco}$?8SvrTjo5jU@Lf~$QTpae-&7jUmCF;5yCIK34nsZ# zc@%OK^1j#0Ch1WhSX=$j`M-#xp8Z!GA$X6d53Bd`%p z-}Xa@i3zF@-HYTWr8TIw;h6XbVnK>PuqV*?@%klq*Buf!uDap+O>3?tnE1Ejn1Y`q z5{N>$jpHcr76NpkwjRLoAlj1wVu9fO{;p*U_naX&g_% zeznuyr}&sEi&t@+hTS52PaybJe>~9mm6})}G`S!aSo=V2ccA6|g+`!#e9^W*BpYat z23n$lwVi=bXP~h&5bO-p$?s}y$j=PoIKNj>NUwMq@G{_Kz{`M_0WSky2D}V-8Spaj z|B(S+C&%mJcpaRcM#(jKbf&D+8aygXm8dMpTc-7+dhw*3s{rG5dDmz=&zV-Szg~w& z^BgKPC!%uUQaMNRm|m2|bCNR)2<0`XSbwMOyasNi?&uQbCK|PD(2|9wi7CDQjXoVy zF_fsxA9*d_N{#Q<0ONgn5g+%vPPfl%1iAgyn$Ld!Evxf`yM*v{RGx28$HO;KsdK+D zEtC3;PQ)8p*XhvF-#bxR_w5SVqAy@X%RauKF2A>{^7_hcU0pYX)(#F8vvx7m+7jLr zZfR~UO0{*J2Hq+rrG0^T9Uqme3ta~vEDz`l9@6G3Am{R@>*U95__XW1h)1~4>qMc- zvI)P0^L?XX5l6*`b-{g6h3H`+1|@!#;Pt2M|5;Qh4&RTAPe}iU%6-K6BdE;P|DQv$S#$PGi<1#MeG+dU?TQ$zlAC8}qaT+W4J>%oRgW_rt zj4+~PC;AftE?oGR;lG?#iieaR--izZzg9t2zpH?+Kno?EXZHCS#qnP)_ LPtooQ z|8l|4Q^rroIIPXe9`Z;0DV?_s+VuQ9{;T0n?YhtRdcx;e&mI9z`4N%XHmNp<_(AyZ zj#lH_rT?nS;_m^j|3p*~4oW;ILfU9)>m=|{m8VjEp#6_%fByUM*gWxnKTn)j=D}%+ z`ds}K{mH!D8qQ%cd}1hV+KK%FK*26zElE<$F7P+)jG3e*{snlZa_0V#+)!e~Oxd}7 z!Aum#L^78d9kFaH6>gc^!^FD#w3*1~69-HyYv&J$efdPjGE>D&<^U)T&4j02}@7WRE9`Ay!s?(`zKyI6;w6L?Mr$07e4n#Y9V~A>{5_ST+CU&B;sdUyX7A&IE z#s%K15OHogzy&De%we1ZZMJZ1aH#frcY7*S487ICG_kRRPag5)o(h)-bzbeEKwpLm z1+LG%HN!=g8!(cEqD`1BVrU(}fWbDKc zq@7oCm}}`QcE5~@a5iUK;r-cScr>3Iwet1>2Q*YnWB*e+rJ-nNyqWGZ={}q&42y8; zKo(6XY3G&YE-PQa(5zBS*z(p$f(W`cI%12k9PeRN!~1hksZy|#B8+V>P-VWt`J5bP zVQX05YQrf+V@<_S_ncyI9Ze-NX#`VrTwxK$&7Q$MJjZkFokH{;jYls2Mylf8$H?!| z+AMv!PVz%I=-rk1^X=CoUn=~*sx%?CpyK4a_wB7=M=km9BE$4v{nk%r*^97a>&F36*|1~ToQK2p~pXVUN4AegJ>zJqI0Bp|UZlC5GPX3)C<;~I{qocU{^D=B$ zrY!G4a^v@V-e>F17xP)Z4ob2x?H!f-78k+Uspj?g~K@|j~%CLyAF6r510?YNOMl^KR + io:format("OK~n"). + ''; + }; + }) + ]; +} \ No newline at end of file diff --git a/runtimes/gawk-awk.nix b/runtimes/gawk-awk.nix new file mode 100644 index 0000000..c5c9675 --- /dev/null +++ b/runtimes/gawk-awk.nix @@ -0,0 +1,26 @@ +{pkgs, piston, ...}: +let + pkg = pkgs.gawk; +in piston.mkRuntime { + language = "awk"; + version = pkg.version; + runtime = "gawk"; + + aliases = []; + + + run = '' + ${pkg}/bin/gawk -f "$@" + ''; + + tests = [ + (piston.mkTest { + files = { + "test.awk" = '' + {print "OK"} + ''; + }; + stdin = "\n"; # awk needs some line input + }) + ]; +} \ No newline at end of file