From 1fcb7604d77eb6b5f7bc7271529cd7d6d6d5d3b9 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sun, 13 Jun 2021 18:41:01 +1200 Subject: [PATCH 01/53] cli: add `ppman spec` command --- cli/commands/ppman_commands/spec.js | 161 ++++++++++++++++++++++++++++ cli/index.js | 2 +- cli/package-lock.json | 121 +++++++++++++++++++++ cli/package.json | 3 + cli/ppman_ops.js | 0 dev.pps | 8 ++ public.pps | 14 +++ 7 files changed, 308 insertions(+), 1 deletion(-) create mode 100644 cli/commands/ppman_commands/spec.js create mode 100644 cli/ppman_ops.js create mode 100644 dev.pps create mode 100755 public.pps diff --git a/cli/commands/ppman_commands/spec.js b/cli/commands/ppman_commands/spec.js new file mode 100644 index 0000000..8f54779 --- /dev/null +++ b/cli/commands/ppman_commands/spec.js @@ -0,0 +1,161 @@ +const chalk = require('chalk'); +const fs = require('fs/promises'); +const minimatch = require("minimatch"); +const semver = require('semver'); + +exports.command = ['spec ']; +exports.aliases = ['s']; +exports.describe = 'Install the packages described in the spec file, uninstalling packages which aren\'t in the list' + +function does_match(package, rule){ + const nameMatch = minimatch(package.language, rule.package_selector); + const versionMatch = semver.satisfies(package.language_version, rule.version_selector) + + return nameMatch && versionMatch; +} + +exports.handler = async ({axios, specfile}) => { + const spec_contents = await fs.readFile(specfile); + const spec_lines = spec_contents.toString().split("\n"); + + const rules = []; + + for(const line of spec_lines){ + const rule = { + _raw: line.trim(), + comment: false, + package_selector: null, + version_selector: null, + negate: false + }; + + if(line.starts_with("#")){ + rule.comment = true; + }else { + let l = line.trim(); + if(line.starts_with("!")){ + rule.negate = true; + l = line.slice(1).trim(); + } + + const [pkg, ver] = l.split(" ", 2); + rule.package_selector = pkg; + rule.version_selector = ver; + } + + if(rule._raw.length != 0) rules.push(rule); + } + + const packages_req = await axios.get('/api/v2/packages'); + const packages = packages_req.data; + + const installed = packages.filter(pkg => pkg.installed); + + const ensure_packages = []; + + for(const rule of rules){ + if(rule.comment) continue; + + const matches = []; + + if(!rule.negate){ + for(const package of packages){ + if(does_match(package, rule)) + matches.push(package) + } + + const latest_matches = matches.filter( + pkg => { + const versions = matches + .filter(x=>x.language == pkg.language) + .map(x=>x.language_version).sort(semver.rcompare); + return versions[0] == pkg.language_version + } + ); + + for(const match of latest_matches){ + if(!ensure_packages.find(pkg => pkg.language == match.language && pkg.language_version == match.language_version)) + ensure_packages.push(match) + } + }else{ + for(const package of ensure_packages){ + if(does_match(package, rule)) + ensure_packages.splice(ensure_packages.indexOf(package)) + } + } + + + } + + const operations = []; + + for(const package of ensure_packages){ + if(!package.installed) + operations.push({ + type: "install", + package: package.language, + version: package.language_version + }); + } + + for(const installed_package of installed){ + if(!ensure_packages.find( + pkg => pkg.language == installed_package.language && + pkg.language_version == installed_package.language_version + )) + operations.push({ + type: "uninstall", + package: installed_package.language, + version: installed_package.language_version + }) + } + + console.log(chalk.bold.yellow("Actions")) + for(const op of operations){ + console.log((op.type == "install" ? chalk.green("Install") : chalk.red("Uninstall")) + ` ${op.package} ${op.version}`) + } + + if(operations.length == 0){ + console.log(chalk.gray("None")) + } + + for(const op of operations){ + if(op.type == "install"){ + try{ + const install = await axios.post(`/api/v2/packages`, { + language: op.package, + version: op.version + }); + + if(!install.data.language) + throw new Error(install.data.message); // Go to exception handler + + console.log(chalk.bold.green("Installed"), op.package, op.version) + + }catch(e){ + console.log(chalk.bold.red("Failed to install") + ` ${op.package} ${op.version}:`, e.message) + } + } + else if(op.type == "uninstall"){ + try{ + const install = await axios.delete(`/api/v2/packages`, { + data: { + language: op.package, + version: op.version + } + }); + + if(!install.data.language) + throw new Error(install.data.message); // Go to exception handler + + console.log(chalk.bold.green("Uninstalled"), op.package, op.version) + + }catch(e){ + console.log(chalk.bold.red("Failed to uninstall") + ` ${op.package} ${op.version}:`, e.message) + } + } + } + + + +} \ No newline at end of file diff --git a/cli/index.js b/cli/index.js index d25ec7d..c0c25ee 100755 --- a/cli/index.js +++ b/cli/index.js @@ -1,5 +1,5 @@ #!/usr/bin/env node - +require('nocamel'); const axios = require('axios').default; const axios_instance = argv => { diff --git a/cli/package-lock.json b/cli/package-lock.json index 2d5532a..d564e5f 100644 --- a/cli/package-lock.json +++ b/cli/package-lock.json @@ -11,6 +11,9 @@ "dependencies": { "axios": "^0.21.1", "chalk": "^4.1.0", + "minimatch": "^3.0.4", + "nocamel": "^1.0.2", + "semver": "^7.3.5", "yargs": "^16.2.0" } }, @@ -41,6 +44,20 @@ "follow-redirects": "^1.10.0" } }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" + }, + "node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, "node_modules/chalk": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz", @@ -79,6 +96,11 @@ "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" + }, "node_modules/emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", @@ -124,6 +146,33 @@ "node": ">=8" } }, + "node_modules/lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/minimatch": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", + "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/nocamel": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/nocamel/-/nocamel-1.0.2.tgz", + "integrity": "sha512-CRkRSRLChj+H6e4lHS851QS6YGCoTETnSG/z+XGanxLSsTbBkvEeIWaIYMKzuBznFwWM0YcLGXsFyXg4xWYnWA==" + }, "node_modules/require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", @@ -132,6 +181,20 @@ "node": ">=0.10.0" } }, + "node_modules/semver": { + "version": "7.3.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz", + "integrity": "sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==", + "dependencies": { + "lru-cache": "^6.0.0" + }, + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/string-width": { "version": "4.2.2", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.2.tgz", @@ -188,6 +251,11 @@ "node": ">=10" } }, + "node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + }, "node_modules/yargs": { "version": "16.2.0", "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", @@ -236,6 +304,20 @@ "follow-redirects": "^1.10.0" } }, + "balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" + }, + "brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "requires": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, "chalk": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz", @@ -268,6 +350,11 @@ "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" }, + "concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" + }, "emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", @@ -298,11 +385,40 @@ "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==" }, + "lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "requires": { + "yallist": "^4.0.0" + } + }, + "minimatch": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", + "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "requires": { + "brace-expansion": "^1.1.7" + } + }, + "nocamel": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/nocamel/-/nocamel-1.0.2.tgz", + "integrity": "sha512-CRkRSRLChj+H6e4lHS851QS6YGCoTETnSG/z+XGanxLSsTbBkvEeIWaIYMKzuBznFwWM0YcLGXsFyXg4xWYnWA==" + }, "require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=" }, + "semver": { + "version": "7.3.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz", + "integrity": "sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==", + "requires": { + "lru-cache": "^6.0.0" + } + }, "string-width": { "version": "4.2.2", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.2.tgz", @@ -344,6 +460,11 @@ "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.5.tgz", "integrity": "sha512-hsRUr4FFrvhhRH12wOdfs38Gy7k2FFzB9qgN9v3aLykRq0dRcdcpz5C9FxdS2NuhOrI/628b/KSTJ3rwHysYSg==" }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + }, "yargs": { "version": "16.2.0", "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", diff --git a/cli/package.json b/cli/package.json index 6a998a4..6df989d 100644 --- a/cli/package.json +++ b/cli/package.json @@ -7,6 +7,9 @@ "dependencies": { "axios": "^0.21.1", "chalk": "^4.1.0", + "minimatch": "^3.0.4", + "nocamel": "^1.0.2", + "semver": "^7.3.5", "yargs": "^16.2.0" } } diff --git a/cli/ppman_ops.js b/cli/ppman_ops.js new file mode 100644 index 0000000..e69de29 diff --git a/dev.pps b/dev.pps new file mode 100644 index 0000000..b2e5a8c --- /dev/null +++ b/dev.pps @@ -0,0 +1,8 @@ +#!/usr/bin/env -S piston ppman spec + +# Development Piston Packages +# Defines packages to be installed by developers + +# All packages, latest version +# Don't use this when connected to public repo, in excess of 10GB +* * diff --git a/public.pps b/public.pps new file mode 100755 index 0000000..f032032 --- /dev/null +++ b/public.pps @@ -0,0 +1,14 @@ +#!/usr/bin/env -S piston ppman spec + +# Public Piston Packages +# Defines packages to be installed on the public piston installation + +# All packages, latest version +* * + +# Except python +!python * + +# Install python 3.* and 2.* +python 3.* +python 2.* From 6cfef7b7ce936077cd32b0e53cc8ce5b734670eb Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sun, 13 Jun 2021 18:45:39 +1200 Subject: [PATCH 02/53] cli: remove random file --- cli/ppman_ops.js | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 cli/ppman_ops.js diff --git a/cli/ppman_ops.js b/cli/ppman_ops.js deleted file mode 100644 index e69de29..0000000 From b24edf2ac1bfb9940fe4ce4dad12413864ff9e28 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sun, 13 Jun 2021 18:52:14 +1200 Subject: [PATCH 03/53] repo: dockerignore --- repo/.dockerignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 repo/.dockerignore diff --git a/repo/.dockerignore b/repo/.dockerignore new file mode 100644 index 0000000..d1470fc --- /dev/null +++ b/repo/.dockerignore @@ -0,0 +1 @@ +*.pkg.tar.gz \ No newline at end of file From 0598c5c9be2296581364cd8a75e1a07c32454ae7 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sun, 13 Jun 2021 18:54:59 +1200 Subject: [PATCH 04/53] Management tool additions * Command to build packages locally * Help message * Start/Stop/Restart commands * Select production/development environements * clean pkgs/repo commands --- .gitignore | 3 +- docker-compose.dev.yaml | 42 +++++++++++----------- piston | 77 +++++++++++++++++++++++++++++++++-------- 3 files changed, 86 insertions(+), 36 deletions(-) diff --git a/.gitignore b/.gitignore index adbb97d..222be8c 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ -data/ \ No newline at end of file +data/ +.piston_env \ No newline at end of file diff --git a/docker-compose.dev.yaml b/docker-compose.dev.yaml index 31772fe..5c3b61e 100644 --- a/docker-compose.dev.yaml +++ b/docker-compose.dev.yaml @@ -1,24 +1,24 @@ -version: '3.2' +version: "3.2" services: - api: - build: api - container_name: piston_api - cap_add: - - CAP_SYS_ADMIN - restart: always - ports: - - 2000:2000 - volumes: - - ./data/piston:/piston - environment: - - PISTON_REPO_URL=http://repo:8000/index - tmpfs: - - /piston/jobs:exec + api: + build: api + container_name: piston_api + cap_add: + - CAP_SYS_ADMIN + restart: always + ports: + - 2000:2000 + volumes: + - ./data/piston:/piston + environment: + - PISTON_REPO_URL=http://repo:8000/index + tmpfs: + - /piston/jobs:exec - repo: # Local testing of packages - build: repo - container_name: piston_repo - command: ['dart-2.12.1'] # Only build dart - volumes: - - .:/piston + repo: # Local testing of packages + build: repo + container_name: piston_repo + command: ["--no-build"] # Don't build anything + volumes: + - .:/piston diff --git a/piston b/piston index f58fd78..4d73571 100755 --- a/piston +++ b/piston @@ -1,23 +1,72 @@ #!/usr/bin/env bash +cd "$(dirname "$0")" + +PISTON_ENV=$(cat .piston_env || echo dev) + +docker_compose(){ + if [ -f "docker-compose.$PISTON_ENV.yaml" ]; then + docker-compose -f "docker-compose.$PISTON_ENV.yaml" "$@" + else + docker-compose "$@" + fi +} case $1 in - dev) - shift - docker-compose -f docker-compose.dev.yaml "$@" - ;; - prod) - shift - docker-compose -f docker-compose.yaml "$@" + help) + echo "=== Piston Management ===" + echo "Current Environment: $PISTON_ENV" + echo + echo "Commands:" + echo " select Select the environment" + echo " docker_compose Interact directly with the docker-compose for the selected environment" + echo + echo " start Starts piston" + echo " stop Stops piston" + echo " restart Restarts piston" + echo + echo " update Fetches and applies latest updates" + echo + echo " Passthrough to piston cli tool" + echo + echo "Development Commands:" + + if [ $PISTON_ENV == dev ]; then + + echo " clean-pkgs Clean any package build artifacts on disk" + echo " clean-repo Remove all packages from local repo" + echo " build-pkg Build a package" + + else + + echo " Switch to developement environment for more info" + echo " > piston switch dev" + + fi ;; + + + select) echo "$2" > .piston_env ;; + docker_compose) shift; docker_compose "$@";; + + restart) docker_compose restart ;; + start) docker_compose up -d ;; + stop) docker_compose down ;; + update) git pull - docker-compose pull api - docker-compose up -d api + docker_compose pull + docker_compose up -d ;; - clean-pkgs) - git clean -fqXd packages - ;; - *) - node cli/index.js "$@" + + clean-pkgs) git clean -fqXd packages ;; + clean-repo) git clean -fqXd repo ;; + + build-pkg) + PKGSLUG="$2-$3" + echo "Building $PKGSLUG" + echo "Ensuring latest builder image" + docker build repo -t piston-repo-builder + docker run -v "$(realpath $(dirname "$0")):/piston" piston-repo-builder --no-server $PKGSLUG ;; + *) node cli/index.js "$@" ;; esac \ No newline at end of file From 5a82e0308b6c92610a83780ed58164639ed70811 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sun, 13 Jun 2021 19:12:14 +1200 Subject: [PATCH 05/53] ensure cli always has packages installed --- piston | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/piston b/piston index 4d73571..2dc36fa 100755 --- a/piston +++ b/piston @@ -68,5 +68,10 @@ case $1 in docker build repo -t piston-repo-builder docker run -v "$(realpath $(dirname "$0")):/piston" piston-repo-builder --no-server $PKGSLUG ;; - *) node cli/index.js "$@" ;; + *) + cd cli + npm i > /dev/null + cd ../ + node cli/index.js "$@" + ;; esac \ No newline at end of file From 7d40abfb3fd182402d02f592bbe9c9c205ec235e Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sun, 13 Jun 2021 19:17:04 +1200 Subject: [PATCH 06/53] cli: fix issue where negate removes too many packages --- cli/commands/ppman_commands/spec.js | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/cli/commands/ppman_commands/spec.js b/cli/commands/ppman_commands/spec.js index 8f54779..d558810 100644 --- a/cli/commands/ppman_commands/spec.js +++ b/cli/commands/ppman_commands/spec.js @@ -51,7 +51,7 @@ exports.handler = async ({axios, specfile}) => { const installed = packages.filter(pkg => pkg.installed); - const ensure_packages = []; + let ensure_packages = []; for(const rule of rules){ if(rule.comment) continue; @@ -78,10 +78,9 @@ exports.handler = async ({axios, specfile}) => { ensure_packages.push(match) } }else{ - for(const package of ensure_packages){ - if(does_match(package, rule)) - ensure_packages.splice(ensure_packages.indexOf(package)) - } + ensure_packages = ensure_packages.filter( + pkg => !does_match(pkg, rule) + ) } From c0b218f47791117867feb4e5bb53eef6a4e67e9e Mon Sep 17 00:00:00 2001 From: g-w1 <58830309+g-w1@users.noreply.github.com> Date: Tue, 15 Jun 2021 21:25:12 -0400 Subject: [PATCH 07/53] update zig to 0.8.0 (#279) zig 0.8.0 got released, so this pr updates it --- packages/zig/0.7.1/build.sh | 2 +- packages/zig/0.8.0/build.sh | 10 ++++++++++ packages/zig/0.8.0/compile | 6 ++++++ packages/zig/0.8.0/environment | 4 ++++ packages/zig/0.8.0/metadata.json | 5 +++++ packages/zig/0.8.0/run | 4 ++++ packages/zig/0.8.0/test.zig | 6 ++++++ 7 files changed, 36 insertions(+), 1 deletion(-) create mode 100755 packages/zig/0.8.0/build.sh create mode 100644 packages/zig/0.8.0/compile create mode 100644 packages/zig/0.8.0/environment create mode 100644 packages/zig/0.8.0/metadata.json create mode 100644 packages/zig/0.8.0/run create mode 100644 packages/zig/0.8.0/test.zig diff --git a/packages/zig/0.7.1/build.sh b/packages/zig/0.7.1/build.sh index 9714794..284214f 100755 --- a/packages/zig/0.7.1/build.sh +++ b/packages/zig/0.7.1/build.sh @@ -7,4 +7,4 @@ curl -L "https://ziglang.org/download/0.7.1/zig-linux-x86_64-0.7.1.tar.xz" -o zi tar xf zig.tar.xz --strip-components=1 rm zig.tar.xz -cd ../ \ No newline at end of file +cd ../ diff --git a/packages/zig/0.8.0/build.sh b/packages/zig/0.8.0/build.sh new file mode 100755 index 0000000..b3bbaf5 --- /dev/null +++ b/packages/zig/0.8.0/build.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +mkdir -p bin +cd bin/ + +curl -L "https://ziglang.org/download/0.8.0/zig-linux-x86_64-0.8.0.tar.xz" -o zig.tar.xz +tar xf zig.tar.xz --strip-components=1 +rm zig.tar.xz + +cd ../ diff --git a/packages/zig/0.8.0/compile b/packages/zig/0.8.0/compile new file mode 100644 index 0000000..75ee6ba --- /dev/null +++ b/packages/zig/0.8.0/compile @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +# optimizing for small programs +rename 's/$/\.zig/' "$@" # Add .zig extension + +zig build-exe -O ReleaseSafe --color off --cache-dir . --global-cache-dir . --name out *.zig diff --git a/packages/zig/0.8.0/environment b/packages/zig/0.8.0/environment new file mode 100644 index 0000000..a85000c --- /dev/null +++ b/packages/zig/0.8.0/environment @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +# compiler path +export PATH=$PWD/bin:$PATH diff --git a/packages/zig/0.8.0/metadata.json b/packages/zig/0.8.0/metadata.json new file mode 100644 index 0000000..7af8ed6 --- /dev/null +++ b/packages/zig/0.8.0/metadata.json @@ -0,0 +1,5 @@ +{ + "language": "zig", + "version": "0.8.0", + "aliases": ["zig"] +} diff --git a/packages/zig/0.8.0/run b/packages/zig/0.8.0/run new file mode 100644 index 0000000..d96e06f --- /dev/null +++ b/packages/zig/0.8.0/run @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +shift # Filename is only used in compile step, so we can take it out here +./out "$@" diff --git a/packages/zig/0.8.0/test.zig b/packages/zig/0.8.0/test.zig new file mode 100644 index 0000000..548c703 --- /dev/null +++ b/packages/zig/0.8.0/test.zig @@ -0,0 +1,6 @@ +const std = @import("std"); + +pub fn main() !void { + const stdout = std.io.getStdOut().writer(); + try stdout.print("OK\n", .{}); +} \ No newline at end of file From ba9503d9f8ad9aa620fb0e12e25874a952058c95 Mon Sep 17 00:00:00 2001 From: Hydrazer <73801166+Hydrazer@users.noreply.github.com> Date: Fri, 18 Jun 2021 17:45:37 -0600 Subject: [PATCH 08/53] Adding Raku (#281) * Adding Raku probably doesn't work lol tried copying the other ones someone could probably implement it using the build instructions from this https://rakudo.org/downloads/rakudo/source * Update test.raku * edited raku implementation put it in version folder 6.d and removed the v from version in the json file * edited raku version changed the same stuff as last time but with different stuff * raku edit build.sh remove some unnecessary lines from the file * raku edit version editing version name again lol --- packages/raku/6.100.0/build.sh | 5 +++++ packages/raku/6.100.0/environment | 4 ++++ packages/raku/6.100.0/metadata.json | 5 +++++ packages/raku/6.100.0/run | 3 +++ packages/raku/6.100.0/test.raku | 1 + 5 files changed, 18 insertions(+) create mode 100644 packages/raku/6.100.0/build.sh create mode 100644 packages/raku/6.100.0/environment create mode 100644 packages/raku/6.100.0/metadata.json create mode 100644 packages/raku/6.100.0/run create mode 100644 packages/raku/6.100.0/test.raku diff --git a/packages/raku/6.100.0/build.sh b/packages/raku/6.100.0/build.sh new file mode 100644 index 0000000..20be2fc --- /dev/null +++ b/packages/raku/6.100.0/build.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +curl -L "https://rakudo.org/dl/rakudo/rakudo-moar-2021.05-01-linux-x86_64-gcc.tar.gz" -o raku.tar.xz +tar xf raku.tar.xz --strip-components=1 +rm raku.tar.xz \ No newline at end of file diff --git a/packages/raku/6.100.0/environment b/packages/raku/6.100.0/environment new file mode 100644 index 0000000..12ab624 --- /dev/null +++ b/packages/raku/6.100.0/environment @@ -0,0 +1,4 @@ +#!/bin/bash + +# Path to raku binary +export PATH=$PWD/bin:$PATH \ No newline at end of file diff --git a/packages/raku/6.100.0/metadata.json b/packages/raku/6.100.0/metadata.json new file mode 100644 index 0000000..7cda1ed --- /dev/null +++ b/packages/raku/6.100.0/metadata.json @@ -0,0 +1,5 @@ +{ + "language": "raku", + "version": "6.100.0", + "aliases": ["raku", "rakudo", "perl6", "p6", "pl6"] +} \ No newline at end of file diff --git a/packages/raku/6.100.0/run b/packages/raku/6.100.0/run new file mode 100644 index 0000000..6c2c077 --- /dev/null +++ b/packages/raku/6.100.0/run @@ -0,0 +1,3 @@ +#!/bin/bash + +raku "$@" \ No newline at end of file diff --git a/packages/raku/6.100.0/test.raku b/packages/raku/6.100.0/test.raku new file mode 100644 index 0000000..f863fac --- /dev/null +++ b/packages/raku/6.100.0/test.raku @@ -0,0 +1 @@ +say "OK" \ No newline at end of file From 95f9628abb8fe098d593463c37660fe8c7b5772a Mon Sep 17 00:00:00 2001 From: dc Date: Sat, 19 Jun 2021 01:46:51 -0700 Subject: [PATCH 09/53] Added cobol and raku to the languages list. (#282) --- readme.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/readme.md b/readme.md index 6165b24..9b864e3 100644 --- a/readme.md +++ b/readme.md @@ -276,6 +276,7 @@ Content-Type: application/json `brainfuck`, `cjam`, `clojure`, +`cobol`, `coffeescript`, `cow`, `crystal`, @@ -314,6 +315,7 @@ Content-Type: application/json `prolog`, `pure`, `python`, +`raku`, `rockstar`, `ruby`, `rust`, From 2c5ec5f7ec2434592166a4806c5605289047a8e0 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sat, 19 Jun 2021 21:24:28 +1200 Subject: [PATCH 10/53] Add container customizer (#280) --- builder/.gitignore | 1 + builder/Dockerfile | 2 ++ builder/build.sh | 61 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 64 insertions(+) create mode 100644 builder/.gitignore create mode 100644 builder/Dockerfile create mode 100755 builder/build.sh diff --git a/builder/.gitignore b/builder/.gitignore new file mode 100644 index 0000000..c795b05 --- /dev/null +++ b/builder/.gitignore @@ -0,0 +1 @@ +build \ No newline at end of file diff --git a/builder/Dockerfile b/builder/Dockerfile new file mode 100644 index 0000000..a769f35 --- /dev/null +++ b/builder/Dockerfile @@ -0,0 +1,2 @@ +FROM ghcr.io/engineer-man/piston:latest +ADD . /piston/packages/ \ No newline at end of file diff --git a/builder/build.sh b/builder/build.sh new file mode 100755 index 0000000..8559eaf --- /dev/null +++ b/builder/build.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash + +# Build a container using the spec file provided + +help_msg(){ + echo "Usage: $0 [specfile] [tag]" + echo + echo "$1" + + exit 1 +} + +cleanup(){ + echo "Exiting..." + docker stop builder_piston_instance && docker rm builder_piston_instance +} + +fetch_packages(){ + local port=$((5535 + $RANDOM % 60000)) + mkdir build + # Start a piston container + docker run \ + -v "$PWD/build":'/piston/packages' \ + --tmpfs /piston/jobs \ + -dit \ + -p $port:2000 \ + --name builder_piston_instance \ + ghcr.io/engineer-man/piston + + # Ensure the CLI is installed + cd ../cli + npm i + cd - + + # Evalulate the specfile + ../cli/index.js -u "http://127.0.0.1:$port" ppman spec $1 +} + +build_container(){ + docker build -t $1 -f "$(dirname $0)/Dockerfile" "$PWD/build" +} + + +SPEC_FILE=$1 +TAG=$2 + +[ -z "$SPEC_FILE" ] && help_msg "specfile is required" +[ -z "$TAG" ] && help_msg "tag is required" + +[ -f "$SPEC_FILE" ] || help_msg "specfile does not exist" + +which node || help_msg "nodejs is required" +which npm || help_msg "npm is required" + +trap cleanup EXIT + +fetch_packages $SPEC_FILE +build_container $TAG + +echo "Start your custom piston container with" +echo "$ docker run --tmpfs /piston/jobs -dit -p 2000:2000 $TAG" \ No newline at end of file From 6a368cf66f6e86d43fe32ebeccc1a6a9f0129dd0 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sat, 19 Jun 2021 21:58:19 +1200 Subject: [PATCH 11/53] ci: ensure README contains all packages in repo (#283) --- .github/workflows/package-pr.yaml | 41 ++++++++----- readme.md | 97 ++++++++++++++++++------------- 2 files changed, 84 insertions(+), 54 deletions(-) diff --git a/.github/workflows/package-pr.yaml b/.github/workflows/package-pr.yaml index 0cf4ccc..b3027ec 100644 --- a/.github/workflows/package-pr.yaml +++ b/.github/workflows/package-pr.yaml @@ -1,4 +1,4 @@ -name: 'Package Pull Requests' +name: "Package Pull Requests" on: pull_request: @@ -8,16 +8,37 @@ on: - reopened - synchronize paths: - - 'packages/**' + - "packages/**" jobs: + check-pkg: + name: Validate README + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Get list of changed files + uses: lots0logs/gh-action-get-changed-files@2.1.4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Ensure README was updated + run: | + MISSING_LINES=$(comm -23 <(jq 'if .provides then .provides[].language else .language end' -r $(find packages -name "metadata.json" ) | sed -e 's/^/`/g' -e 's/$/`,/g' | sort -u) <(awk '/# Supported Languages/{flag=1; next} /
/{flag=0} flag' readme.md | sort -u)) + + [[ $(echo $MISSING_LINES | wc -c) = "1" ]] && exit 0 + + echo "README has supported languages missing: " + comm -23 <(jq 'if .provides then .provides[].language else .language end' -r $(find packages -name "metadata.json" ) | sed -e 's/^/`/g' -e 's/$/`,/g' | sort -u) <(awk '/# Supported Languages/{flag=1; next} /
/{flag=0} flag' readme.md | sort -u) + exit 1 + build-pkg: name: Check that package builds runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v2 - + - name: Login to GitHub registry uses: docker/login-action@v1 with: @@ -29,7 +50,7 @@ jobs: uses: lots0logs/gh-action-get-changed-files@2.1.4 with: token: ${{ secrets.GITHUB_TOKEN }} - + - name: Build Packages run: | PACKAGES=$(jq '.[]' -r ${HOME}/files.json | awk -F/ '{ print $2 "-" $3 }' | sort -u) @@ -43,7 +64,6 @@ jobs: name: packages path: packages/*.pkg.tar.gz - test-pkg: name: Test package runs-on: ubuntu-latest @@ -54,7 +74,7 @@ jobs: - uses: actions/download-artifact@v2 with: name: packages - + - name: Relocate downloaded packages run: mv *.pkg.tar.gz packages/ @@ -109,17 +129,8 @@ jobs: done done - - name: Dump logs if: ${{ always() }} run: | docker logs api docker logs repo - - - - - - - - diff --git a/readme.md b/readme.md index 9b864e3..1a0f846 100644 --- a/readme.md +++ b/readme.md @@ -37,6 +37,7 @@ --- +
# About @@ -49,16 +50,19 @@
It's used in numerous places including: -* [EMKC Challenges](https://emkc.org/challenges) -* [EMKC Weekly Contests](https://emkc.org/contests) -* [Engineer Man Discord Server](https://discord.gg/engineerman) -* Web IDEs -* 200+ direct integrations + +- [EMKC Challenges](https://emkc.org/challenges) +- [EMKC Weekly Contests](https://emkc.org/contests) +- [Engineer Man Discord Server](https://discord.gg/engineerman) +- Web IDEs +- 200+ direct integrations
### Official Extensions + The following are approved and endorsed extensions/utilities to the core Piston offering. + - [I Run Code](https://github.com/engineer-man/piston-bot), a Discord bot used in 4100+ servers to handle arbitrary code evaluation in Discord. To get this bot in your own server, go here: https://emkc.org/run. - [Piston CLI](https://github.com/Shivansh-007/piston-cli), a universal shell supporting code highlighting, files, and interpretation without the need to download a language. @@ -72,13 +76,15 @@ The following are approved and endorsed extensions/utilities to the core Piston
When using the public Piston API, use the following two URLs: + ``` GET https://emkc.org/api/v2/piston/runtimes POST https://emkc.org/api/v2/piston/execute ``` + > Important Note: The Piston API is rate limited to 5 requests per second. If you have a need for more requests than that -and it's for a good cause, please reach out to me (EngineerMan#0001) on [Discord](https://discord.gg/engineerman) -so we can discuss potentially getting you an unlimited key. +> and it's for a good cause, please reach out to me (EngineerMan#0001) on [Discord](https://discord.gg/engineerman) +> so we can discuss potentially getting you an unlimited key.
@@ -109,8 +115,8 @@ docker-compose up -d api cd cli && npm i && cd - ``` -The API will now be online with no language runtimes installed. To install runtimes, [use the CLI](#cli). - +The API will now be online with no language runtimes installed. To install runtimes, [use the CLI](#cli). + ## Just Piston (no CLI) ### Host System Package Dependencies @@ -172,11 +178,13 @@ The container exposes an API on port 2000 by default. This is used by the CLI to carry out running jobs and package management. #### Runtimes Endpoint + `GET /api/v2/runtimes` This endpoint will return the supported languages along with the current version and aliases. To execute code for a particular language using the `/api/v2/execute` endpoint, either the name or one of the aliases must be provided, along with the version. Multiple versions of the same language may be present at the same time, and may be selected when running a job. + ```json HTTP/1.1 200 OK Content-Type: application/json @@ -201,47 +209,47 @@ Content-Type: application/json ``` #### Execute Endpoint + `POST /api/v2/execute` This endpoint requests execution of some arbitrary code. + - `language` (**required**) The language to use for execution, must be a string and must be installed. - `version` (**required**) The version of the language to use for execution, must be a string containing a SemVer selector for the version or the specific version number to use. - `files` (**required**) An array of files containing code or other data that should be used for execution. The first file in this array is considered the main file. -- `files[].name` (*optional*) The name of the file to upload, must be a string containing no path or left out. +- `files[].name` (_optional_) The name of the file to upload, must be a string containing no path or left out. - `files[].content` (**required**) The content of the files to upload, must be a string containing text to write. -- `stdin` (*optional*) The text to pass as stdin to the program. Must be a string or left out. Defaults to blank string. -- `args` (*optional*) The arguments to pass to the program. Must be an array or left out. Defaults to `[]`. -- `compile_timeout` (*optional*) The maximum time allowed for the compile stage to finish before bailing out in milliseconds. Must be a number or left out. Defaults to `10000` (10 seconds). -- `run_timeout` (*optional*) The maximum time allowed for the run stage to finish before bailing out in milliseconds. Must be a number or left out. Defaults to `3000` (3 seconds). -- `compile_memory_limit` (*optional*) The maximum amount of memory the compile stage is allowed to use in bytes. Must be a number or left out. Defaults to `-1` (no limit) -- `run_memory_limit` (*optional*) The maximum amount of memory the run stage is allowed to use in bytes. Must be a number or left out. Defaults to `-1` (no limit) +- `stdin` (_optional_) The text to pass as stdin to the program. Must be a string or left out. Defaults to blank string. +- `args` (_optional_) The arguments to pass to the program. Must be an array or left out. Defaults to `[]`. +- `compile_timeout` (_optional_) The maximum time allowed for the compile stage to finish before bailing out in milliseconds. Must be a number or left out. Defaults to `10000` (10 seconds). +- `run_timeout` (_optional_) The maximum time allowed for the run stage to finish before bailing out in milliseconds. Must be a number or left out. Defaults to `3000` (3 seconds). +- `compile_memory_limit` (_optional_) The maximum amount of memory the compile stage is allowed to use in bytes. Must be a number or left out. Defaults to `-1` (no limit) +- `run_memory_limit` (_optional_) The maximum amount of memory the run stage is allowed to use in bytes. Must be a number or left out. Defaults to `-1` (no limit) ```json { - "language": "js", - "version": "15.10.0", - "files": [ - { - "name": "my_cool_code.js", - "content": "console.log(process.argv)" - } - ], - "stdin": "", - "args": [ - "1", - "2", - "3" - ], - "compile_timeout": 10000, - "run_timeout": 3000, - "compile_memory_limit": -1, - "run_memory_limit": -1 + "language": "js", + "version": "15.10.0", + "files": [ + { + "name": "my_cool_code.js", + "content": "console.log(process.argv)" + } + ], + "stdin": "", + "args": ["1", "2", "3"], + "compile_timeout": 10000, + "run_timeout": 3000, + "compile_memory_limit": -1, + "run_memory_limit": -1 } ``` + A typical response upon successful execution will contain 1 or 2 keys `run` and `compile`. `compile` will only be present if the language requested requires a compile stage. Each of these keys has an identical structure, containing both a `stdout` and `stderr` key, which is a string containing the text outputted during the stage into each buffer. It also contains the `code` and `signal` which was returned from each process. + ```json HTTP/1.1 200 OK Content-Type: application/json @@ -260,6 +268,7 @@ Content-Type: application/json ``` If a problem exists with the request, a `400` status code is returned and the reason in the `message` key. + ```json HTTP/1.1 400 Bad Request Content-Type: application/json @@ -272,40 +281,45 @@ Content-Type: application/json
# Supported Languages + +`awk`, `bash`, `brainfuck`, +`c`, +`c++`, `cjam`, `clojure`, `cobol`, `coffeescript`, `cow`, `crystal`, +`csharp`, +`d`, `dart`, `dash`, -`deno`, `dotnet`, `dragon`, `elixir`, `emacs`, `erlang`, -`gawk`, -`gcc`, +`fortran`, `go`, `golfscript`, `groovy`, `haskell`, `java`, +`javascript`, `jelly`, `julia`, `kotlin`, `lisp`, `lolcode`, `lua`, -`mono`, `nasm`, +`nasm64`, `nim`, -`node`, `ocaml`, +`octave`, `osabie`, `paradoc`, `pascal`, @@ -314,7 +328,9 @@ Content-Type: application/json `ponylang`, `prolog`, `pure`, +`pyth`, `python`, +`python2`, `raku`, `rockstar`, `ruby`, @@ -338,9 +354,11 @@ The source file is either ran or compiled and ran (in the case of languages like
# Security + Docker provides a great deal of security out of the box in that it's separate from the system. Piston takes additional steps to make it resistant to various privilege escalation, denial-of-service, and resource saturation threats. These steps include: + - Disabling outgoing network interaction - Capping max processes at 256 by default (resists `:(){ :|: &}:;`, `while True: os.fork()`, etc.) - Capping max files at 2048 (resists various file based attacks) @@ -353,4 +371,5 @@ various privilege escalation, denial-of-service, and resource saturation threats
# License + Piston is licensed under the MIT license. From 64e95f63e071c1895420e5486a5ce87c76f727c7 Mon Sep 17 00:00:00 2001 From: Thomas Date: Sat, 19 Jun 2021 22:20:03 +1200 Subject: [PATCH 12/53] Update issue templates (#284) * Update issue templates * Create package.md --- .github/ISSUE_TEMPLATE/language-request.md | 10 ++++++++++ .github/PULL_REQUEST_TEMPLATE/package.md | 10 ++++++++++ 2 files changed, 20 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/language-request.md create mode 100644 .github/PULL_REQUEST_TEMPLATE/package.md diff --git a/.github/ISSUE_TEMPLATE/language-request.md b/.github/ISSUE_TEMPLATE/language-request.md new file mode 100644 index 0000000..5ae2661 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/language-request.md @@ -0,0 +1,10 @@ +--- +name: Language Request +about: Template for requesting language support +title: Add [insert language name here] +labels: package +assignees: '' + +--- + +Provide links to different compilers/interpreters that could be used to implement this language, and discuss pros/cons of each. diff --git a/.github/PULL_REQUEST_TEMPLATE/package.md b/.github/PULL_REQUEST_TEMPLATE/package.md new file mode 100644 index 0000000..6cd3c98 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE/package.md @@ -0,0 +1,10 @@ +Checklist: +* [ ] The package builds locally with `./piston build-pkg [package] [version]` +* [ ] The package installs with `./piston ppman install [package]=[version]` +* [ ] The package runs the test code with `./piston run [package] -l [version] packages/[package]/[version]/test.*` +* [ ] Package files are placed in the correct directory +* [ ] No old package versions are removed +* [ ] All source files are deleted in the `build.sh` script +* [ ] `metadata.json`'s `language` and `version` fields match the directory path +* [ ] Any extensions the language may use are set as aliases +* [ ] Any alternative names the language is referred to are set as aliases. From 929f3d3f2c2631a9abadf9d0b564a5134ffa0e40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luka=20Barbi=C4=87?= Date: Sat, 19 Jun 2021 13:02:36 +0200 Subject: [PATCH 13/53] add julia 1.6.1 package (#285) --- packages/julia/1.6.1/build.sh | 21 +++++++++++++++++++++ packages/julia/1.6.1/environment | 4 ++++ packages/julia/1.6.1/metadata.json | 5 +++++ packages/julia/1.6.1/run | 4 ++++ packages/julia/1.6.1/test.jl | 1 + 5 files changed, 35 insertions(+) create mode 100755 packages/julia/1.6.1/build.sh create mode 100644 packages/julia/1.6.1/environment create mode 100644 packages/julia/1.6.1/metadata.json create mode 100755 packages/julia/1.6.1/run create mode 100644 packages/julia/1.6.1/test.jl diff --git a/packages/julia/1.6.1/build.sh b/packages/julia/1.6.1/build.sh new file mode 100755 index 0000000..b2772ae --- /dev/null +++ b/packages/julia/1.6.1/build.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +# Install location +PREFIX=$(realpath $(dirname $0)) + +mkdir -p build +cd build + +# Download and extract Julia source +curl -L "https://github.com/JuliaLang/julia/releases/download/v1.6.1/julia-1.6.1.tar.gz" -o julia.tar.gz +tar xzf julia.tar.gz --strip-components=1 + +# Build +echo "JULIA_CPU_TARGET=generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base(1) +prefix=$PREFIX" > Make.user +make -j$(nproc) +make install -j$(nproc) + +# Cleanup +cd .. +rm -rf build diff --git a/packages/julia/1.6.1/environment b/packages/julia/1.6.1/environment new file mode 100644 index 0000000..e7d0c97 --- /dev/null +++ b/packages/julia/1.6.1/environment @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +# Add Julia binary to path +export PATH=$PWD/bin:$PATH diff --git a/packages/julia/1.6.1/metadata.json b/packages/julia/1.6.1/metadata.json new file mode 100644 index 0000000..be2fdea --- /dev/null +++ b/packages/julia/1.6.1/metadata.json @@ -0,0 +1,5 @@ +{ + "language": "julia", + "version": "1.6.1", + "aliases": ["jl"] +} diff --git a/packages/julia/1.6.1/run b/packages/julia/1.6.1/run new file mode 100755 index 0000000..6badf1c --- /dev/null +++ b/packages/julia/1.6.1/run @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +# Run without startup or history file +julia --startup-file=no --history-file=no "$@" diff --git a/packages/julia/1.6.1/test.jl b/packages/julia/1.6.1/test.jl new file mode 100644 index 0000000..dc98a27 --- /dev/null +++ b/packages/julia/1.6.1/test.jl @@ -0,0 +1 @@ +println("OK") From e859f83cefea9f94c7dd085855a418031062e758 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luka=20Barbi=C4=87?= Date: Sat, 19 Jun 2021 14:25:13 +0200 Subject: [PATCH 14/53] add node 16.3.0 package (#286) --- packages/node/16.3.0/build.sh | 4 ++++ packages/node/16.3.0/environment | 1 + packages/node/16.3.0/metadata.json | 10 ++++++++++ packages/node/16.3.0/run | 3 +++ packages/node/16.3.0/test.js | 1 + 5 files changed, 19 insertions(+) create mode 100755 packages/node/16.3.0/build.sh create mode 100644 packages/node/16.3.0/environment create mode 100644 packages/node/16.3.0/metadata.json create mode 100644 packages/node/16.3.0/run create mode 100644 packages/node/16.3.0/test.js diff --git a/packages/node/16.3.0/build.sh b/packages/node/16.3.0/build.sh new file mode 100755 index 0000000..bbc19db --- /dev/null +++ b/packages/node/16.3.0/build.sh @@ -0,0 +1,4 @@ +#!/bin/bash +curl "https://nodejs.org/dist/v16.3.0/node-v16.3.0-linux-x64.tar.xz" -o node.tar.xz +tar xf node.tar.xz --strip-components=1 +rm node.tar.xz diff --git a/packages/node/16.3.0/environment b/packages/node/16.3.0/environment new file mode 100644 index 0000000..977a5e8 --- /dev/null +++ b/packages/node/16.3.0/environment @@ -0,0 +1 @@ +export PATH=$PWD/bin:$PATH diff --git a/packages/node/16.3.0/metadata.json b/packages/node/16.3.0/metadata.json new file mode 100644 index 0000000..ccd146a --- /dev/null +++ b/packages/node/16.3.0/metadata.json @@ -0,0 +1,10 @@ +{ + "language": "node", + "version": "16.3.0", + "provides": [ + { + "language": "javascript", + "aliases": ["node-javascript", "node-js", "javascript", "js"] + } + ] +} diff --git a/packages/node/16.3.0/run b/packages/node/16.3.0/run new file mode 100644 index 0000000..6d1fdee --- /dev/null +++ b/packages/node/16.3.0/run @@ -0,0 +1,3 @@ +#!/bin/bash + +node "$@" \ No newline at end of file diff --git a/packages/node/16.3.0/test.js b/packages/node/16.3.0/test.js new file mode 100644 index 0000000..56ed4a0 --- /dev/null +++ b/packages/node/16.3.0/test.js @@ -0,0 +1 @@ +console.log("OK") \ No newline at end of file From 5bec67ff5fd0208ba0c2be01ac076a7b9a8619e4 Mon Sep 17 00:00:00 2001 From: dc Date: Wed, 23 Jun 2021 10:04:18 -0700 Subject: [PATCH 15/53] Proposal: Add Piston Node.js Client as an Official Extension I created a [Node.js client wrapper](https://github.com/dthree/node-piston) to access the Piston API easily from Node. This makes it very easy to access the public Piston API or any private API from Node, hope you like it! @realtux I don't mind moving this into the engineer-man account if you'd prefer that. --- readme.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/readme.md b/readme.md index 1a0f846..e6a96c8 100644 --- a/readme.md +++ b/readme.md @@ -65,7 +65,9 @@ The following are approved and endorsed extensions/utilities to the core Piston - [I Run Code](https://github.com/engineer-man/piston-bot), a Discord bot used in 4100+ servers to handle arbitrary code evaluation in Discord. To get this bot in your own server, go here: https://emkc.org/run. - [Piston CLI](https://github.com/Shivansh-007/piston-cli), a universal shell supporting code highlighting, files, and interpretation without the need to download a language. +- [Node Piston Client](https://github.com/dthree/node-piston), a Node.js wrapper for accessing the Piston API. +
# Public API From 322f98e4a053d9e336cb57fb9746f4762c198c6d Mon Sep 17 00:00:00 2001 From: the-codeboy <71213855+the-codeboy@users.noreply.github.com> Date: Wed, 7 Jul 2021 18:00:13 +0200 Subject: [PATCH 16/53] Add Piston4J as an Official Extension I made a [Java wrapper](https://github.com/the-codeboy/Piston4J) to access the Piston API and when I saw [this](https://github.com/engineer-man/piston/pull/287) pullrequest I thought I might also try adding it as an official exstension --- readme.md | 1 + 1 file changed, 1 insertion(+) diff --git a/readme.md b/readme.md index e6a96c8..21d8f3e 100644 --- a/readme.md +++ b/readme.md @@ -66,6 +66,7 @@ The following are approved and endorsed extensions/utilities to the core Piston - [I Run Code](https://github.com/engineer-man/piston-bot), a Discord bot used in 4100+ servers to handle arbitrary code evaluation in Discord. To get this bot in your own server, go here: https://emkc.org/run. - [Piston CLI](https://github.com/Shivansh-007/piston-cli), a universal shell supporting code highlighting, files, and interpretation without the need to download a language. - [Node Piston Client](https://github.com/dthree/node-piston), a Node.js wrapper for accessing the Piston API. +- [Piston4J](https://github.com/the-codeboy/Piston4J), a Java wrapper for accessing the Piston API.
From 8900bae7c0ebbf51f734bd276807ca5fc6d71fcf Mon Sep 17 00:00:00 2001 From: yrfriend Date: Thu, 8 Jul 2021 21:15:37 +0300 Subject: [PATCH 17/53] Fixed Go multiple file problem --- packages/go/1.16.2/run | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/go/1.16.2/run b/packages/go/1.16.2/run index da31af3..65c9a4f 100644 --- a/packages/go/1.16.2/run +++ b/packages/go/1.16.2/run @@ -1,6 +1,7 @@ #!/usr/bin/env bash mv $1 $1.go -filename=$1.go +#filename=$1.go +filename=*.go shift GOCACHE=$PWD go run $filename "$@" From c8beb86037955cd069d79ebb28f77bf1417be8eb Mon Sep 17 00:00:00 2001 From: Shiv <75499121+ffaanngg@users.noreply.github.com> Date: Mon, 12 Jul 2021 06:43:02 +0530 Subject: [PATCH 18/53] Added pyston --- readme.md | 1 + 1 file changed, 1 insertion(+) diff --git a/readme.md b/readme.md index 21d8f3e..c50f66f 100644 --- a/readme.md +++ b/readme.md @@ -67,6 +67,7 @@ The following are approved and endorsed extensions/utilities to the core Piston - [Piston CLI](https://github.com/Shivansh-007/piston-cli), a universal shell supporting code highlighting, files, and interpretation without the need to download a language. - [Node Piston Client](https://github.com/dthree/node-piston), a Node.js wrapper for accessing the Piston API. - [Piston4J](https://github.com/the-codeboy/Piston4J), a Java wrapper for accessing the Piston API. +- [Pyston](https://github.com/ffaanngg/pyston), an Python wrapper for accessing the Piston API.
From 2200283e50b0f8d70716d7e2427f1817115387e0 Mon Sep 17 00:00:00 2001 From: Shiv <75499121+ffaanngg@users.noreply.github.com> Date: Mon, 12 Jul 2021 06:43:20 +0530 Subject: [PATCH 19/53] Update readme.md --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index c50f66f..193d90b 100644 --- a/readme.md +++ b/readme.md @@ -67,7 +67,7 @@ The following are approved and endorsed extensions/utilities to the core Piston - [Piston CLI](https://github.com/Shivansh-007/piston-cli), a universal shell supporting code highlighting, files, and interpretation without the need to download a language. - [Node Piston Client](https://github.com/dthree/node-piston), a Node.js wrapper for accessing the Piston API. - [Piston4J](https://github.com/the-codeboy/Piston4J), a Java wrapper for accessing the Piston API. -- [Pyston](https://github.com/ffaanngg/pyston), an Python wrapper for accessing the Piston API. +- [Pyston](https://github.com/ffaanngg/pyston), a Python wrapper for accessing the Piston API.
From de89acb617470beffff6be2a3343302a51183ce7 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Fri, 16 Jul 2021 23:10:44 +1200 Subject: [PATCH 20/53] api: Implement Websocket transport for live data --- api/package-lock.json | 49 +++++++ api/package.json | 1 + api/src/api/v2.js | 299 +++++++++++++++++++++++++++--------------- api/src/index.js | 4 + api/src/job.js | 59 ++++++++- 5 files changed, 299 insertions(+), 113 deletions(-) diff --git a/api/package-lock.json b/api/package-lock.json index c46ae87..83df240 100644 --- a/api/package-lock.json +++ b/api/package-lock.json @@ -12,6 +12,7 @@ "body-parser": "^1.19.0", "chownr": "^2.0.0", "express": "^4.17.1", + "express-ws": "^5.0.2", "is-docker": "^2.1.1", "logplease": "^1.2.15", "nocamel": "HexF/nocamel#patch-1", @@ -196,6 +197,20 @@ "node": ">= 0.10.0" } }, + "node_modules/express-ws": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/express-ws/-/express-ws-5.0.2.tgz", + "integrity": "sha512-0uvmuk61O9HXgLhGl3QhNSEtRsQevtmbL94/eILaliEADZBHZOQUAiHFrGPrgsjikohyrmSG5g+sCfASTt0lkQ==", + "dependencies": { + "ws": "^7.4.6" + }, + "engines": { + "node": ">=4.5.0" + }, + "peerDependencies": { + "express": "^4.0.0 || ^5.0.0-alpha.1" + } + }, "node_modules/finalhandler": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.2.tgz", @@ -582,6 +597,26 @@ "node_modules/waitpid": { "resolved": "git+ssh://git@github.com/HexF/node-waitpid.git#a08d116a5d993a747624fe72ff890167be8c34aa" }, + "node_modules/ws": { + "version": "7.5.3", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.3.tgz", + "integrity": "sha512-kQ/dHIzuLrS6Je9+uv81ueZomEwH0qVYstcAQ4/Z93K8zeko9gtAbttJWzoC5ukqXY1PpoouV3+VSOqEAFt5wg==", + "engines": { + "node": ">=8.3.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": "^5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, "node_modules/yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", @@ -728,6 +763,14 @@ "vary": "~1.1.2" } }, + "express-ws": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/express-ws/-/express-ws-5.0.2.tgz", + "integrity": "sha512-0uvmuk61O9HXgLhGl3QhNSEtRsQevtmbL94/eILaliEADZBHZOQUAiHFrGPrgsjikohyrmSG5g+sCfASTt0lkQ==", + "requires": { + "ws": "^7.4.6" + } + }, "finalhandler": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.2.tgz", @@ -1010,6 +1053,12 @@ "version": "git+ssh://git@github.com/HexF/node-waitpid.git#a08d116a5d993a747624fe72ff890167be8c34aa", "from": "waitpid@git+https://github.com/HexF/node-waitpid.git" }, + "ws": { + "version": "7.5.3", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.3.tgz", + "integrity": "sha512-kQ/dHIzuLrS6Je9+uv81ueZomEwH0qVYstcAQ4/Z93K8zeko9gtAbttJWzoC5ukqXY1PpoouV3+VSOqEAFt5wg==", + "requires": {} + }, "yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", diff --git a/api/package.json b/api/package.json index ab34063..0c32198 100644 --- a/api/package.json +++ b/api/package.json @@ -7,6 +7,7 @@ "body-parser": "^1.19.0", "chownr": "^2.0.0", "express": "^4.17.1", + "express-ws": "^5.0.2", "is-docker": "^2.1.1", "logplease": "^1.2.15", "nocamel": "HexF/nocamel#patch-1", diff --git a/api/src/api/v2.js b/api/src/api/v2.js index 948dccf..487b10b 100644 --- a/api/src/api/v2.js +++ b/api/src/api/v2.js @@ -1,12 +1,124 @@ const express = require('express'); const router = express.Router(); +const events = require('events'); + const config = require('../config'); const runtime = require('../runtime'); const { Job } = require('../job'); const package = require('../package'); const logger = require('logplease').create('api/v2'); + +function get_job(body){ + const { + language, + version, + args, + stdin, + files, + compile_memory_limit, + run_memory_limit, + run_timeout, + compile_timeout + } = body; + + return new Promise((resolve, reject) => { + if (!language || typeof language !== 'string') { + return reject({ + message: 'language is required as a string', + }); + } + + if (!version || typeof version !== 'string') { + return reject({ + message: 'version is required as a string', + }); + } + + if (!files || !Array.isArray(files)) { + return reject({ + message: 'files is required as an array', + }); + } + + for (const [i, file] of files.entries()) { + if (typeof file.content !== 'string') { + return reject({ + message: `files[${i}].content is required as a string`, + }); + } + } + + if (compile_memory_limit) { + if (typeof compile_memory_limit !== 'number') { + return reject({ + message: 'if specified, compile_memory_limit must be a number', + }); + } + + if ( + config.compile_memory_limit >= 0 && + (compile_memory_limit > config.compile_memory_limit || + compile_memory_limit < 0) + ) { + return reject({ + message: + 'compile_memory_limit cannot exceed the configured limit of ' + + config.compile_memory_limit, + }); + } + } + + if (run_memory_limit) { + if (typeof run_memory_limit !== 'number') { + return reject({ + message: 'if specified, run_memory_limit must be a number', + }); + } + + if ( + config.run_memory_limit >= 0 && + (run_memory_limit > config.run_memory_limit || run_memory_limit < 0) + ) { + return reject({ + message: + 'run_memory_limit cannot exceed the configured limit of ' + + config.run_memory_limit, + }); + } + } + + const rt = runtime.get_latest_runtime_matching_language_version( + language, + version + ); + + if (rt === undefined) { + return reject({ + message: `${language}-${version} runtime is unknown`, + }); + } + + resolve(new Job({ + runtime: rt, + alias: language, + args: args || [], + stdin: '', + files, + timeouts: { + run: run_timeout || 3000, + compile: compile_timeout || 10000, + }, + memory_limits: { + run: run_memory_limit || config.run_memory_limit, + compile: compile_memory_limit || config.compile_memory_limit, + } + })); + }) + +} + router.use((req, res, next) => { if (['GET', 'HEAD', 'OPTIONS'].includes(req.method)) { return next(); @@ -21,118 +133,87 @@ router.use((req, res, next) => { next(); }); +router.ws('/connect', async (ws, req) => { + + let job = null; + let eventBus = new events.EventEmitter(); + + eventBus.on("stdout", (data) => ws.send(JSON.stringify({type: "data", stream: "stdout", data: data.toString()}))) + eventBus.on("stderr", (data) => ws.send(JSON.stringify({type: "data", stream: "stderr", data: data.toString()}))) + eventBus.on("stage", (stage)=> ws.send(JSON.stringify({type: "stage", stage}))) + eventBus.on("exit", (stage, status) => ws.send(JSON.stringify({type: "exit", stage, ...status}))) + + ws.on("message", async (data) => { + + try{ + const msg = JSON.parse(data); + + if(msg.type === "init"){ + if(job === null){ + const job = await get_job(msg); + + await job.prime(); + + ws.send(JSON.stringify({ + type: "runtime", + language: job.runtime.language, + version: job.runtime.version.raw + })) + + await job.execute_interactive(eventBus); + + ws.close(4999, "Job Completed"); + + }else{ + ws.close(4000, "Already Initialized"); + } + + }else if(msg.type === "data"){ + if(job !== null){ + if(msg.stream === "stdin"){ + eventBus.emit("stdin", msg.data) + }else{ + ws.close(4004, "Can only write to stdin") + } + }else{ + ws.close(4003, "Not yet initialized") + } + } + }catch(error){ + ws.send(JSON.stringify({type: "error", message: error.message})) + ws.close(4002, "Notified Error") + // ws.close message is limited to 123 characters, so we notify over WS then close. + } + }) + + ws.on("close", async ()=>{ + if(job !== null){ + await job.cleanup() + } + }) + + setTimeout(()=>{ + //Terminate the socket after 1 second, if not initialized. + //if(job === null) + // ws.close(4001, "Initialization Timeout"); + }, 1000) +}) + router.post('/execute', async (req, res) => { - const { - language, - version, - files, - stdin, - args, - run_timeout, - compile_timeout, - compile_memory_limit, - run_memory_limit, - } = req.body; - if (!language || typeof language !== 'string') { - return res.status(400).send({ - message: 'language is required as a string', - }); + try{ + const job = await get_job(req.body); + + await job.prime(); + + const result = await job.execute(); + + await job.cleanup(); + + return res.status(200).send(result); + }catch(error){ + return res.status(400).json(error); } - - if (!version || typeof version !== 'string') { - return res.status(400).send({ - message: 'version is required as a string', - }); - } - - if (!files || !Array.isArray(files)) { - return res.status(400).send({ - message: 'files is required as an array', - }); - } - - for (const [i, file] of files.entries()) { - if (typeof file.content !== 'string') { - return res.status(400).send({ - message: `files[${i}].content is required as a string`, - }); - } - } - - if (compile_memory_limit) { - if (typeof compile_memory_limit !== 'number') { - return res.status(400).send({ - message: 'if specified, compile_memory_limit must be a number', - }); - } - - if ( - config.compile_memory_limit >= 0 && - (compile_memory_limit > config.compile_memory_limit || - compile_memory_limit < 0) - ) { - return res.status(400).send({ - message: - 'compile_memory_limit cannot exceed the configured limit of ' + - config.compile_memory_limit, - }); - } - } - - if (run_memory_limit) { - if (typeof run_memory_limit !== 'number') { - return res.status(400).send({ - message: 'if specified, run_memory_limit must be a number', - }); - } - - if ( - config.run_memory_limit >= 0 && - (run_memory_limit > config.run_memory_limit || run_memory_limit < 0) - ) { - return res.status(400).send({ - message: - 'run_memory_limit cannot exceed the configured limit of ' + - config.run_memory_limit, - }); - } - } - - const rt = runtime.get_latest_runtime_matching_language_version( - language, - version - ); - - if (rt === undefined) { - return res.status(400).send({ - message: `${language}-${version} runtime is unknown`, - }); - } - - const job = new Job({ - runtime: rt, - alias: language, - files: files, - args: args || [], - stdin: stdin || '', - timeouts: { - run: run_timeout || 3000, - compile: compile_timeout || 10000, - }, - memory_limits: { - run: run_memory_limit || config.run_memory_limit, - compile: compile_memory_limit || config.compile_memory_limit, - }, - }); - - await job.prime(); - - const result = await job.execute(); - - await job.cleanup(); - - return res.status(200).send(result); }); router.get('/runtimes', (req, res) => { diff --git a/api/src/index.js b/api/src/index.js index ef16916..afd4d15 100644 --- a/api/src/index.js +++ b/api/src/index.js @@ -2,6 +2,7 @@ require('nocamel'); const Logger = require('logplease'); const express = require('express'); +const expressWs = require('express-ws'); const globals = require('./globals'); const config = require('./config'); const path = require('path'); @@ -12,6 +13,9 @@ const runtime = require('./runtime'); const logger = Logger.create('index'); const app = express(); +expressWs(app); + + (async () => { logger.info('Setting loglevel to', config.log_level); diff --git a/api/src/job.js b/api/src/job.js index d4b90ea..8001a76 100644 --- a/api/src/job.js +++ b/api/src/job.js @@ -69,7 +69,7 @@ class Job { logger.debug('Primed job'); } - async safe_call(file, args, timeout, memory_limit) { + async safe_call(file, args, timeout, memory_limit, eventBus = null) { return new Promise((resolve, reject) => { const nonetwork = config.disable_networking ? ['nosocket'] : []; @@ -102,9 +102,15 @@ class Job { detached: true, //give this process its own process group }); - proc.stdin.write(this.stdin); - proc.stdin.end(); - proc.stdin.destroy(); + if(eventBus === null){ + proc.stdin.write(this.stdin); + proc.stdin.end(); + proc.stdin.destroy(); + }else{ + eventBus.on("stdin", (data) => { + proc.stdin.write(data); + }) + } const kill_timeout = set_timeout( _ => proc.kill('SIGKILL'), @@ -115,6 +121,7 @@ class Job { if (stderr.length > config.output_max_size) { proc.kill('SIGKILL'); } else { + if(eventBus !== null) eventBus.emit("stderr", data); stderr += data; output += data; } @@ -124,6 +131,7 @@ class Job { if (stdout.length > config.output_max_size) { proc.kill('SIGKILL'); } else { + if(eventBus !== null) eventBus.emit("stdout", data); stdout += data; output += data; } @@ -196,6 +204,49 @@ class Job { }; } + async execute_interactive(eventBus){ + if (this.state !== job_states.PRIMED) { + throw new Error( + 'Job must be in primed state, current state: ' + + this.state.toString() + ); + } + + logger.info( + `Interactively executing job uuid=${this.uuid} uid=${this.uid} gid=${ + this.gid + } runtime=${this.runtime.toString()}` + ); + + if(this.runtime.compiled){ + eventBus.emit("stage", "compile") + const {error, code, signal} = await this.safe_call( + path.join(this.runtime.pkgdir, 'compile'), + this.files.map(x => x.name), + this.timeouts.compile, + this.memory_limits.compile, + eventBus + ) + + eventBus.emit("exit", "compile", {error, code, signal}) + } + + logger.debug('Running'); + eventBus.emit("stage", "run") + const {error, code, signal} = await this.safe_call( + path.join(this.runtime.pkgdir, 'run'), + [this.files[0].name, ...this.args], + this.timeouts.run, + this.memory_limits.run, + eventBus + ); + + eventBus.emit("exit", "run", {error, code, signal}) + + + this.state = job_states.EXECUTED; + } + async cleanup_processes() { let processes = [1]; From f58927d79a6a3d57b1773d8d717c6204a142bba7 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Fri, 16 Jul 2021 23:12:46 +1200 Subject: [PATCH 21/53] script: correct typo and add additional rebuild command --- piston | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/piston b/piston index 2dc36fa..7e3a469 100755 --- a/piston +++ b/piston @@ -35,11 +35,12 @@ case $1 in echo " clean-pkgs Clean any package build artifacts on disk" echo " clean-repo Remove all packages from local repo" echo " build-pkg Build a package" + echo " rebuild Build and restart the docker container" else echo " Switch to developement environment for more info" - echo " > piston switch dev" + echo " > piston select dev" fi ;; @@ -52,6 +53,8 @@ case $1 in start) docker_compose up -d ;; stop) docker_compose down ;; + rebuild) docker_compose build && docker_compose up -d ;; + update) git pull docker_compose pull From 3436648add766bea162e46b3f160402701f973f1 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sat, 17 Jul 2021 00:22:55 +1200 Subject: [PATCH 22/53] api: signaling process via ws --- api/src/api/v2.js | 48 +++++++++++++++++++++++++++++++---------------- api/src/job.js | 16 ++++++++++++---- 2 files changed, 44 insertions(+), 20 deletions(-) diff --git a/api/src/api/v2.js b/api/src/api/v2.js index 487b10b..76bab08 100644 --- a/api/src/api/v2.js +++ b/api/src/api/v2.js @@ -9,6 +9,8 @@ const { Job } = require('../job'); const package = require('../package'); const logger = require('logplease').create('api/v2'); +const SIGNALS = ["SIGABRT","SIGALRM","SIGBUS","SIGCHLD","SIGCLD","SIGCONT","SIGEMT","SIGFPE","SIGHUP","SIGILL","SIGINFO","SIGINT","SIGIO","SIGIOT","SIGKILL","SIGLOST","SIGPIPE","SIGPOLL","SIGPROF","SIGPWR","SIGQUIT","SIGSEGV","SIGSTKFLT","SIGSTOP","SIGTSTP","SIGSYS","SIGTERM","SIGTRAP","SIGTTIN","SIGTTOU","SIGUNUSED","SIGURG","SIGUSR1","SIGUSR2","SIGVTALRM","SIGXCPU","SIGXFSZ","SIGWINCH"] +// ref: https://man7.org/linux/man-pages/man7/signal.7.html function get_job(body){ const { @@ -148,27 +150,28 @@ router.ws('/connect', async (ws, req) => { try{ const msg = JSON.parse(data); - if(msg.type === "init"){ - if(job === null){ - const job = await get_job(msg); + switch(msg.type){ + case "init": + if(job === null){ + job = await get_job(msg); - await job.prime(); + await job.prime(); - ws.send(JSON.stringify({ - type: "runtime", - language: job.runtime.language, - version: job.runtime.version.raw - })) + ws.send(JSON.stringify({ + type: "runtime", + language: job.runtime.language, + version: job.runtime.version.raw + })) - await job.execute_interactive(eventBus); + await job.execute_interactive(eventBus); - ws.close(4999, "Job Completed"); + ws.close(4999, "Job Completed"); - }else{ - ws.close(4000, "Already Initialized"); - } - - }else if(msg.type === "data"){ + }else{ + ws.close(4000, "Already Initialized"); + } + break; + case "data": if(job !== null){ if(msg.stream === "stdin"){ eventBus.emit("stdin", msg.data) @@ -178,7 +181,20 @@ router.ws('/connect', async (ws, req) => { }else{ ws.close(4003, "Not yet initialized") } + break; + case "signal": + if(job !== null){ + if(SIGNALS.includes(msg.signal)){ + eventBus.emit("signal", msg.signal) + }else{ + ws.close(4005, "Invalid signal") + } + }else{ + ws.close(4003, "Not yet initialized") + } + break; } + }catch(error){ ws.send(JSON.stringify({type: "error", message: error.message})) ws.close(4002, "Notified Error") diff --git a/api/src/job.js b/api/src/job.js index 8001a76..d213bbc 100644 --- a/api/src/job.js +++ b/api/src/job.js @@ -110,7 +110,13 @@ class Job { eventBus.on("stdin", (data) => { proc.stdin.write(data); }) + + eventBus.on("kill", (signal) => { + proc.kill(signal) + }) } + + const kill_timeout = set_timeout( _ => proc.kill('SIGKILL'), @@ -118,20 +124,22 @@ class Job { ); proc.stderr.on('data', data => { - if (stderr.length > config.output_max_size) { + if(eventBus !== null) { + eventBus.emit("stderr", data); + } else if (stderr.length > config.output_max_size) { proc.kill('SIGKILL'); } else { - if(eventBus !== null) eventBus.emit("stderr", data); stderr += data; output += data; } }); proc.stdout.on('data', data => { - if (stdout.length > config.output_max_size) { + if(eventBus !== null){ + eventBus.emit("stdout", data); + } else if (stdout.length > config.output_max_size) { proc.kill('SIGKILL'); } else { - if(eventBus !== null) eventBus.emit("stdout", data); stdout += data; output += data; } From 4933577dae4833db342c88d76800a24d5ae237c5 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sat, 17 Jul 2021 00:23:45 +1200 Subject: [PATCH 23/53] cli: interactive run with -t flag --- cli/commands/execute.js | 134 +++++++++++++++++++++++++++++++++++++--- cli/package-lock.json | 27 ++++++++ cli/package.json | 1 + 3 files changed, 153 insertions(+), 9 deletions(-) diff --git a/cli/commands/execute.js b/cli/commands/execute.js index e273548..abb1f63 100644 --- a/cli/commands/execute.js +++ b/cli/commands/execute.js @@ -1,7 +1,10 @@ -//const fetch = require('node-fetch'); const fs = require('fs'); const path = require('path'); const chalk = require('chalk'); +const WebSocket = require('ws'); + +const SIGNALS = ["SIGABRT","SIGALRM","SIGBUS","SIGCHLD","SIGCLD","SIGCONT","SIGEMT","SIGFPE","SIGHUP","SIGILL","SIGINFO","SIGINT","SIGIO","SIGIOT","SIGLOST","SIGPIPE","SIGPOLL","SIGPROF","SIGPWR","SIGQUIT","SIGSEGV","SIGSTKFLT","SIGTSTP","SIGSYS","SIGTERM","SIGTRAP","SIGTTIN","SIGTTOU","SIGUNUSED","SIGURG","SIGUSR1","SIGUSR2","SIGVTALRM","SIGXCPU","SIGXFSZ","SIGWINCH"] + exports.command = ['execute [args..]']; exports.aliases = ['run']; @@ -35,17 +38,115 @@ exports.builder = { alias: ['f'], array: true, desc: 'Additional files to add', + }, + interactive: { + boolean: true, + alias: ['t'], + desc: 'Run interactively using WebSocket transport' + }, + status: { + boolean: true, + alias: ['s'], + desc: 'Output additional status to stderr' } }; -exports.handler = async (argv) => { - const files = [...(argv.files || []),argv.file] - .map(file_path => { - return { - name: path.basename(file_path), - content: fs.readFileSync(file_path).toString() - }; - }); +async function handle_interactive(files, argv){ + const ws = new WebSocket(argv.pistonUrl.replace("http", "ws") + "/api/v2/connect") + + const log_message = (process.stderr.isTTY && argv.status) ? console.error : ()=>{}; + + process.on("exit", ()=>{ + ws.close(); + process.stdin.end(); + process.stdin.destroy(); + process.exit(); + }) + + for(const signal of SIGNALS){ + process.on(signal, ()=>{ + ws.send(JSON.stringify({type: 'signal', signal})) + }) + } + + + + ws.on('open', ()=>{ + const request = { + type: "init", + language: argv.language, + version: argv['language_version'], + files: files, + args: argv.args, + compile_timeout: argv.ct, + run_timeout: argv.rt + } + + ws.send(JSON.stringify(request)) + log_message(chalk.white.bold("Connected")) + + process.stdin.resume(); + + process.stdin.on("data", (data) => { + ws.send(JSON.stringify({ + type: "data", + stream: "stdin", + data: data.toString() + })) + }) + }) + + ws.on("close", (code, reason)=>{ + log_message( + chalk.white.bold("Disconnected: "), + chalk.white.bold("Reason: "), + chalk.yellow(`"${reason}"`), + chalk.white.bold("Code: "), + chalk.yellow(`"${code}"`), + ) + process.stdin.pause() + }) + + ws.on('message', function(data){ + const msg = JSON.parse(data); + + switch(msg.type){ + case "runtime": + log_message(chalk.bold.white("Runtime:"), chalk.yellow(`${msg.language} ${msg.version}`)) + break; + case "stage": + log_message(chalk.bold.white("Stage:"), chalk.yellow(msg.stage)) + break; + case "data": + if(msg.stream == "stdout") process.stdout.write(msg.data) + else if(msg.stream == "stderr") process.stderr.write(msg.data) + else log_message(chalk.bold.red(`(${msg.stream}) `), msg.data) + break; + case "exit": + if(msg.signal === null) + log_message( + chalk.white.bold("Stage"), + chalk.yellow(msg.stage), + chalk.white.bold("exited with code"), + chalk.yellow(msg.code) + ) + else + log_message( + chalk.white.bold("Stage"), + chalk.yellow(msg.stage), + chalk.white.bold("exited with signal"), + chalk.yellow(msg.signal) + ) + break; + default: + log_message(chalk.red.bold("Unknown message:"), msg) + } + }) + +} + +async function run_non_interactively(files, argv) { + const stdin = (argv.stdin && await new Promise((resolve, _) => { let data = ''; @@ -99,3 +200,18 @@ exports.handler = async (argv) => { step('Run', response.run); } + +exports.handler = async (argv) => { + const files = [...(argv.files || []),argv.file] + .map(file_path => { + return { + name: path.basename(file_path), + content: fs.readFileSync(file_path).toString() + }; + }); + + if(argv.interactive) await handle_interactive(files, argv); + else await run_non_interactively(files, argv); +} + + diff --git a/cli/package-lock.json b/cli/package-lock.json index d564e5f..76a2cc6 100644 --- a/cli/package-lock.json +++ b/cli/package-lock.json @@ -14,6 +14,7 @@ "minimatch": "^3.0.4", "nocamel": "^1.0.2", "semver": "^7.3.5", + "ws": "^7.5.3", "yargs": "^16.2.0" } }, @@ -243,6 +244,26 @@ "node": ">=10" } }, + "node_modules/ws": { + "version": "7.5.3", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.3.tgz", + "integrity": "sha512-kQ/dHIzuLrS6Je9+uv81ueZomEwH0qVYstcAQ4/Z93K8zeko9gtAbttJWzoC5ukqXY1PpoouV3+VSOqEAFt5wg==", + "engines": { + "node": ">=8.3.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": "^5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, "node_modules/y18n": { "version": "5.0.5", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.5.tgz", @@ -455,6 +476,12 @@ "strip-ansi": "^6.0.0" } }, + "ws": { + "version": "7.5.3", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.3.tgz", + "integrity": "sha512-kQ/dHIzuLrS6Je9+uv81ueZomEwH0qVYstcAQ4/Z93K8zeko9gtAbttJWzoC5ukqXY1PpoouV3+VSOqEAFt5wg==", + "requires": {} + }, "y18n": { "version": "5.0.5", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.5.tgz", diff --git a/cli/package.json b/cli/package.json index 6df989d..9dcf061 100644 --- a/cli/package.json +++ b/cli/package.json @@ -10,6 +10,7 @@ "minimatch": "^3.0.4", "nocamel": "^1.0.2", "semver": "^7.3.5", + "ws": "^7.5.3", "yargs": "^16.2.0" } } From 5ace2bf0e42e986f5d8639629dbde06fec891867 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sat, 17 Jul 2021 00:25:19 +1200 Subject: [PATCH 24/53] api: websocket init timeout --- api/src/api/v2.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/src/api/v2.js b/api/src/api/v2.js index 76bab08..0c34d70 100644 --- a/api/src/api/v2.js +++ b/api/src/api/v2.js @@ -210,8 +210,8 @@ router.ws('/connect', async (ws, req) => { setTimeout(()=>{ //Terminate the socket after 1 second, if not initialized. - //if(job === null) - // ws.close(4001, "Initialization Timeout"); + if(job === null) + ws.close(4001, "Initialization Timeout"); }, 1000) }) From 230cb3abe1e5b57715bd60f6bd4760a7ca103bad Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sat, 17 Jul 2021 01:00:44 +1200 Subject: [PATCH 25/53] bump versions major --- api/package.json | 2 +- cli/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/package.json b/api/package.json index 0c32198..e8e5b5d 100644 --- a/api/package.json +++ b/api/package.json @@ -1,6 +1,6 @@ { "name": "piston-api", - "version": "3.0.0", + "version": "3.1.0", "description": "API for piston - a high performance code execution engine", "main": "src/index.js", "dependencies": { diff --git a/cli/package.json b/cli/package.json index 9dcf061..90e3e12 100644 --- a/cli/package.json +++ b/cli/package.json @@ -1,6 +1,6 @@ { "name": "piston-cli", - "version": "1.0.0", + "version": "1.1.0", "description": "Piston Execution Engine CLI tools", "main": "index.js", "license": "MIT", From 09e0a9497d5a388cdfa8bb3a4915a55ba2560392 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sat, 17 Jul 2021 14:44:21 +1200 Subject: [PATCH 26/53] basic api docs --- .envrc | 1 + .readthedocs.yaml | 8 ++ docs/api-v2.md | 234 ++++++++++++++++++++++++++++++++++++++++++ docs/index.md | 3 + docs/requirements.txt | 1 + mkdocs.yml | 4 + shell.nix | 2 +- 7 files changed, 252 insertions(+), 1 deletion(-) create mode 100644 .envrc create mode 100644 .readthedocs.yaml create mode 100644 docs/api-v2.md create mode 100644 docs/index.md create mode 100644 docs/requirements.txt create mode 100644 mkdocs.yml diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..17d6464 --- /dev/null +++ b/.envrc @@ -0,0 +1 @@ +use_nix \ No newline at end of file diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..e651ad5 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,8 @@ +version: 2 +mkdocs: + configuration: mkdocs.yml + +python: + version: 3.7 + install: + - requirements: docs/requirements.txt diff --git a/docs/api-v2.md b/docs/api-v2.md new file mode 100644 index 0000000..111b514 --- /dev/null +++ b/docs/api-v2.md @@ -0,0 +1,234 @@ +# API + +Piston exposes an API for managing packages and executing user-defined code. + +The API is broken in to 2 main sections - packages and jobs. + +The API is exposed from the container, by default on port 2000, at `/api/v2/`. + +All inputs are validated, and if an error occurs, a 4xx or 5xx status code is returned. +In this case, a JSON payload is sent back containing the error message as `message` + +## Runtimes + +### `GET /api/v2/runtimes` + +Returns a list of available languages, including the version, runtime and aliases. + +#### Response + +- `[].language`: Name of the language +- `[].version`: Version of the runtime +- `[].aliases`: List of alternative names that can be used for the language +- `[].runtime` (_optional_): Name of the runtime used to run the langage, only provided if alternative runtimes exist for the language + +#### Example + +``` +GET /api/v2/runtimes +``` + +```json +HTTP/1.1 200 OK +Content-Type: application/json + +[ + { + "language": "bash", + "version": "5.1.0", + "aliases": ["sh"] + }, + { + "language": "javascript", + "version": "15.10.0", + "aliases": ["node-javascript", "node-js", "javascript", "js"], + "runtime": "node" + } +] +``` + +## Execute + +### `POST /api/v2/execute` + +Runs the given code, using the given runtime and arguments, returning the result. + +#### Request + +- `language`: Name or alias of a language listed in [runtimes](#runtimes) +- `version`: SemVer version selector of a language listed in [runtimes](#runtimes) +- `files`: An array of files which should be uploaded into the job context +- `files[].name` (_optional_): Name of file to be written, if none a random name is picked +- `files[].content`: Content of file to be written +- `stdin` (_optional_): Text to pass into stdin of the program. Defaults to blank string. +- `args` (_optional_): Arguments to pass to the program. Defaults to none +- `run_timeout` (_optional_): The maximum allowed time in milliseconds for the compile stage to finish before bailing out. Must be a number, less than or equal to the configured maximum timeout. +- `compile_timeout` (_optional_): The maximum allowed time in milliseconds for the run stage to finish before bailing out. Must be a number, less than or equal to the configured maximum timeout. Defaults to maximum. +- `compile_memory_limit` (_optional_): The maximum amount of memory the compile stage is allowed to use in bytes. Must be a number, less than or equal to the configured maximum. Defaults to maximum, or `-1` (no limit) if none is configured. +- `run_memory_limit` (_optional_): The maximum amount of memory the run stage is allowed to use in bytes. Must be a number, less than or equal to the configured maximum. Defaults to maximum, or `-1` (no limit) if none is configured. + +#### Response + +- `language`: Name (not alias) of the runtime used +- `version`: Version of the used runtime +- `run`: Results from the run stage +- `run.stdout`: stdout from run stage process +- `run.stderr`: stderr from run stage process +- `run.output`: stdout and stderr combined in order of data from run stage process +- `run.code`: Exit code from run process, or null if signal is not null +- `run.signal`: Signal from run process, or null if code is not null +- `compile` (_optional_): Results from the compile stage, only provided if the runtime has a compile stage +- `compile.stdout`: stdout from compile stage process +- `compile.stderr`: stderr from compile stage process +- `compile.output`: stdout and stderr combined in order of data from compile stage process +- `compile.code`: Exit code from compile process, or null if signal is not null +- `compile.signal`: Signal from compile process, or null if code is not null + +#### Example + +```json +POST /api/v2/execute +Content-Type: application/json + +{ + "language": "js", + "version": "15.10.0", + "files": [ + { + "name": "my_cool_code.js", + "content": "console.log(process.argv)" + } + ], + "stdin": "", + "args": ["1", "2", "3"], + "compile_timeout": 10000, + "run_timeout": 3000, + "compile_memory_limit": -1, + "run_memory_limit": -1 +} +``` + +```json +HTTP/1.1 200 OK +Content-Type: application/json + +{ + "run": { + "stdout": "[\n '/piston/packages/node/15.10.0/bin/node',\n '/piston/jobs/e87afa0d-6c2a-40b8-a824-ffb9c5c6cb64/my_cool_code.js',\n '1',\n '2',\n '3'\n]\n", + "stderr": "", + "code": 0, + "signal": null, + "output": "[\n '/piston/packages/node/15.10.0/bin/node',\n '/piston/jobs/e87afa0d-6c2a-40b8-a824-ffb9c5c6cb64/my_cool_code.js',\n '1',\n '2',\n '3'\n]\n" + }, + "language": "javascript", + "version": "15.10.0" +} +``` + +## Packages + +### `GET /api/v2/packages` + +Returns a list of all possible packages, and whether their installation status. + +#### Response + +- `[].language`: Name of the contained runtime +- `[].language_version`: Version of the contained runtime +- `[].installed`: Status on the package being installed + +#### Example + +``` +GET /api/v2/packages +``` + +```json +HTTP/1.1 200 OK +Content-Type: application/json + +[ + { + "language": "node", + "language_version": "15.10.0", + "installed": true + }, + { + "language": "bash", + "language_version": "5.1.0", + "installed": true + } +] +``` + +### `POST /api/v2/packages` + +Install the given package. + +#### Request + +- `language`: Name of package from [package list](#get-apiv2packages) +- `version`: SemVer version selector for package from [package list](#get-apiv2packages) + +#### Response + +- `language`: Name of package installed +- `version`: Version of package installed + +#### Example + +```json +POST /api/v2/packages +Content-Type: application/json + +{ + "language": "bash", + "version": "5.x" +} +``` + +```json +HTTP/1.1 200 OK +Content-Type: application/json + +{ + "language": "bash", + "version": "5.1.0" +} +``` + +### `DELETE /api/v2/packages` + +Uninstall the given package. + +#### Request + +- `language`: Name of package from [package list](#get-apiv2packages) +- `version`: SemVer version selector for package from [package list](#get-apiv2packages) + +#### Response + +- `language`: Name of package uninstalled +- `version`: Version of package uninstalled + +#### Example + +```json +DELETE /api/v2/packages +Content-Type: application/json + +{ + "language": "bash", + "version": "5.x" +} +``` + +```json +HTTP/1.1 200 OK +Content-Type: application/json + +{ + "language": "bash", + "version": "5.1.0" +} +``` diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..a8e5891 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,3 @@ +# Piston + +These docs are a WIP diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..53dbf05 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1 @@ +mkdocs==1.1.2 \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..a9a86fe --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,4 @@ +site_name: Piston Documentation +nav: + - Home: index.md + - API: api-v2.md diff --git a/shell.nix b/shell.nix index 47b2d12..339bead 100644 --- a/shell.nix +++ b/shell.nix @@ -1,5 +1,5 @@ { pkgs ? import {} }: pkgs.mkShell { # nativeBuildInputs is usually what you want -- tools you need to run - nativeBuildInputs = with pkgs; [ nodejs-15_x jq ]; + nativeBuildInputs = with pkgs; [ nodejs-15_x jq mkdocs ]; } From 33ba39cbc7073e39b21fe3ef775f95fa944da002 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sat, 17 Jul 2021 14:46:40 +1200 Subject: [PATCH 27/53] docs: rtd theme --- mkdocs.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mkdocs.yml b/mkdocs.yml index a9a86fe..2030962 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,4 +1,7 @@ -site_name: Piston Documentation +site_name: Piston nav: - Home: index.md - API: api-v2.md + +theme: + name: readthedocs From acb590bf39786b24f90c92dca8959e6e070440bc Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sat, 17 Jul 2021 14:49:02 +1200 Subject: [PATCH 28/53] readme: add docs link --- readme.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/readme.md b/readme.md index 193d90b..01f3a2a 100644 --- a/readme.md +++ b/readme.md @@ -33,7 +33,8 @@ Supported Languages • Principles • Security • - License + License • + Documentation --- @@ -69,7 +70,6 @@ The following are approved and endorsed extensions/utilities to the core Piston - [Piston4J](https://github.com/the-codeboy/Piston4J), a Java wrapper for accessing the Piston API. - [Pyston](https://github.com/ffaanngg/pyston), a Python wrapper for accessing the Piston API. -
# Public API From 968390d5b69973ebdd56a35bb545fc864c53a586 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sat, 17 Jul 2021 15:35:43 +1200 Subject: [PATCH 29/53] docs: configuration options --- docs/configuration.md | 147 ++++++++++++++++++++++++++++++++++++++++++ mkdocs.yml | 8 +++ 2 files changed, 155 insertions(+) create mode 100644 docs/configuration.md diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 0000000..1388e9d --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,147 @@ +# Configuration + +Piston provides many different configuration options to tweak Piston to meet your needs. + +Configuration is specified through environment variables, prefixed with `PISTON_`. + +## Log Level + +```yaml +key: PISTON_LOG_LEVEL +default: INFO +``` + +Level of log output to provide. + +One of `DEBUG`, `INFO`, `WARN`, `ERROR` or `NONE` + +## Bind Address + +```yaml +key: PISTON_BIND_ADDRESS +default: 0.0.0.0:2000 +``` + +Port and IP address to bind the Piston API to. + + +!!! warning + Changing this value is not recommended. + + This changes the bind address inside the container, and thus serves no purpose when running in a container + +## Data Directory + +```yaml +key: PISTON_DATA_DIRECTORY +default: /piston +``` + +Absolute path to piston related data, including packages and job contexts. + + +!!! warning + Changing this value is not recommended. + + Some packages require absolute paths on disk at build time. + Due to this, some packages may break when changing this parameter. + +## Runner GID/UID range + +```yaml +key: + - PISTON_RUNNER_UID_MIN + - PISTON_RUNNER_UID_MAX + - PISTON_RUNNER_GID_MIN + - PISTON_RUNNER_GID_MAX +default: + - 1001 + - 1500 + - 1001 + - 1500 +``` + +UID and GID ranges to use when executing jobs. + + +!!! warning + Changing this value is not recommended. + + The piston container creates 500 users and groups by default, and reserves user/group 1000 for running the API. + Any processes run by these users will be killed when cleaning up a job. + +## Disable Networking + +```yaml +key: PISTON_DISABLE_NETWORKING +default: true +``` + +Disallows access to `socket` syscalls, effectively disabling networking for jobs run by piston. + +## Max Process Count + +```yaml +key: PISTON_MAX_PROCESS_COUNT +default: 64 +``` + +Maximum number of processess allowed to to have open for a job. + +Resists against exhausting the process table, causing a full system lockup. + +## Output Max Side + +```yaml +key: PISTON_OUTPUT_MAX_SIZE +default: 1024 +``` + +Maximum size of stdio buffers for each job. + +Resist against run-away output which could lead to memory exhaustion. + +## Max Open Files + +```yaml +key: PISTON_MAX_OPEN_FILES +default: 64 +``` + +Maximum number of open files at a given time by a job. + +Resists against writing many smaller files to exhaust inodes. + +## Max File Size + +```yaml +key: PISTON_MAX_FILE_SIZE +default: 10000000 #10MB +``` + +Maximum size for a singular file written to disk. + +Resists against large file writes to exhaust disk space. + +## Compile/Run memory limits + +```yaml +key: + - PISTON_COMPILE_MEMORY_LIMIT + - PISTON_RUN_MEMORY_LIMIT +default: -1 +``` + +Maximum memory allowed by a stage in bytes. +Use -1 for unlimited memory usage. + +Useful for running memory-limited contests. + +## Repository URL + +```yaml +key: PISTON_REPO_URL +default: https://github.com/engineer-man/piston/releases/download/pkgs/index +``` + +URL for repository index, where packages will be downloaded from. diff --git a/mkdocs.yml b/mkdocs.yml index 2030962..148ba91 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,7 +1,15 @@ site_name: Piston nav: - Home: index.md + - Configuration: configuration.md - API: api-v2.md theme: name: readthedocs + highlightjs: true + hljs_languages: + - yaml + - json + +markdown_extensions: + - admonition From 5cd84ae8167762bbfd19ea2cf348c0fa74081847 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Thu, 22 Jul 2021 14:58:41 +1200 Subject: [PATCH 30/53] api: allow paths when writing files (#302) --- api/src/job.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/api/src/job.js b/api/src/job.js index d4b90ea..bb78448 100644 --- a/api/src/job.js +++ b/api/src/job.js @@ -59,6 +59,13 @@ class Job { for (const file of this.files) { let file_path = path.join(this.dir, file.name); + const rel = path.relative(this.dir, file_path); + + if(rel.startsWith("..")) + throw Error(`File path "${file.name}" tries to escape parent directory: ${rel}`) + + await fs.mkdir(path.dirname(file_path), {recursive: true, mode: 0o700}) + await fs.chown(path.dirname(file_path), this.uid, this.gid); await fs.write_file(file_path, file.content); await fs.chown(file_path, this.uid, this.gid); @@ -139,7 +146,7 @@ class Job { proc.on('exit', (code, signal) => { exit_cleanup(); - resolve({ stdout, stderr, code, signal, output }); + resolve({stdout, stderr, code, signal, output }); }); proc.on('error', err => { From 2386684a050109e5f5236bb340b805ac0a4c243c Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Fri, 20 Aug 2021 01:05:13 +1200 Subject: [PATCH 31/53] api: fix file cleanup edge case Processes could still be spawned writing files after the app has cleaned the dir out, and is ready to clear it out. Dumb edge case, but oh well. --- api/src/job.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/api/src/job.js b/api/src/job.js index bb78448..07a7ee4 100644 --- a/api/src/job.js +++ b/api/src/job.js @@ -280,10 +280,8 @@ class Job { async cleanup() { logger.info(`Cleaning up job uuid=${this.uuid}`); - await Promise.all([ - this.cleanup_processes(), - this.cleanup_filesystem(), - ]); + await this.cleanup_processes(); + await this.cleanup_filesystem(); } } From c699688b36b7050af3a5001ffa7f786c8f33cc07 Mon Sep 17 00:00:00 2001 From: Hydrazer <73801166+Hydrazer@users.noreply.github.com> Date: Tue, 31 Aug 2021 09:08:13 -0600 Subject: [PATCH 32/53] fixed slurp for COW lang forked the original repo by BigZaphod and fixed the slurping issue when reading stdin for the Moo and oom commands --- packages/cow/1.0.0/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cow/1.0.0/build.sh b/packages/cow/1.0.0/build.sh index 3bf5938..4753cae 100755 --- a/packages/cow/1.0.0/build.sh +++ b/packages/cow/1.0.0/build.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # Grab the latest cow source from github -git clone -q https://github.com/BigZaphod/COW.git cow +git clone -q https://github.com/Hydrazer/COW.git cow # Generate the cow binary into bin mkdir -p bin From 30f2715c0197c32f2ac709990278d0cccfcd89df Mon Sep 17 00:00:00 2001 From: Brikaa Date: Tue, 7 Sep 2021 13:41:51 +0200 Subject: [PATCH 33/53] Add NodeJS version to docs --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index 01f3a2a..f092e16 100644 --- a/readme.md +++ b/readme.md @@ -100,7 +100,7 @@ POST https://emkc.org/api/v2/piston/execute - Docker - Docker Compose -- Node JS +- Node JS (>= 13, preferably >= 15) ### After system dependencies are installed, clone this repository: From dc80ed3a1d35d09cb73d2ba8a5e21e9815dc8eb1 Mon Sep 17 00:00:00 2001 From: Brikaa Date: Tue, 7 Sep 2021 13:50:10 +0200 Subject: [PATCH 34/53] Document testing packages locally --- readme.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/readme.md b/readme.md index f092e16..1e45681 100644 --- a/readme.md +++ b/readme.md @@ -139,6 +139,21 @@ docker run \ ghcr.io/engineer-man/piston ``` +## Piston for testing packages locally + +### Host System Package Dependencies + +- Same as [All In One](#All-In-One) + + +```sh +# Build the Docker containers +./piston start + +# For more help +./piston help +``` +
# Usage From 528eb0e2630d9fcaec8776bf43fb01ba4bb6a9f8 Mon Sep 17 00:00:00 2001 From: Brikaa Date: Tue, 7 Sep 2021 13:53:37 +0200 Subject: [PATCH 35/53] Installation --- readme.md | 1 + 1 file changed, 1 insertion(+) diff --git a/readme.md b/readme.md index 1e45681..3c9c8a8 100644 --- a/readme.md +++ b/readme.md @@ -145,6 +145,7 @@ docker run \ - Same as [All In One](#All-In-One) +### Installation ```sh # Build the Docker containers From 3c15de61447a89772ff1cd9978a4ec8e2fa156b5 Mon Sep 17 00:00:00 2001 From: Brikaa Date: Tue, 7 Sep 2021 19:21:52 +0200 Subject: [PATCH 36/53] pkg(pwsh-7.1.4): Added Powershell 7.1.4 --- packages/pwsh/7.1.4/build.sh | 6 ++++++ packages/pwsh/7.1.4/environment | 1 + packages/pwsh/7.1.4/metadata.json | 10 ++++++++++ packages/pwsh/7.1.4/run | 3 +++ packages/pwsh/7.1.4/test.ps1 | 1 + 5 files changed, 21 insertions(+) create mode 100755 packages/pwsh/7.1.4/build.sh create mode 100644 packages/pwsh/7.1.4/environment create mode 100644 packages/pwsh/7.1.4/metadata.json create mode 100644 packages/pwsh/7.1.4/run create mode 100644 packages/pwsh/7.1.4/test.ps1 diff --git a/packages/pwsh/7.1.4/build.sh b/packages/pwsh/7.1.4/build.sh new file mode 100755 index 0000000..3f4b070 --- /dev/null +++ b/packages/pwsh/7.1.4/build.sh @@ -0,0 +1,6 @@ +#!/bin/bash +curl -L https://github.com/PowerShell/PowerShell/releases/download/v7.1.4/powershell-7.1.4-linux-x64.tar.gz -o powershell.tar.gz +tar zxf powershell.tar.gz +rm powershell.tar.gz + +chmod +x pwsh diff --git a/packages/pwsh/7.1.4/environment b/packages/pwsh/7.1.4/environment new file mode 100644 index 0000000..42644cd --- /dev/null +++ b/packages/pwsh/7.1.4/environment @@ -0,0 +1 @@ +export PATH=$PWD:$PATH diff --git a/packages/pwsh/7.1.4/metadata.json b/packages/pwsh/7.1.4/metadata.json new file mode 100644 index 0000000..da90f76 --- /dev/null +++ b/packages/pwsh/7.1.4/metadata.json @@ -0,0 +1,10 @@ +{ + "language": "pwsh", + "version": "7.1.4", + "provides": [ + { + "language": "powershell", + "aliases": ["ps", "pwsh", "ps1"] + } + ] +} diff --git a/packages/pwsh/7.1.4/run b/packages/pwsh/7.1.4/run new file mode 100644 index 0000000..02a0be7 --- /dev/null +++ b/packages/pwsh/7.1.4/run @@ -0,0 +1,3 @@ +#!/bin/bash + +pwsh "$@" diff --git a/packages/pwsh/7.1.4/test.ps1 b/packages/pwsh/7.1.4/test.ps1 new file mode 100644 index 0000000..f0a4be3 --- /dev/null +++ b/packages/pwsh/7.1.4/test.ps1 @@ -0,0 +1 @@ +echo "OK" From 0a6e5140954c98fd654bbc469ba7f886f556b727 Mon Sep 17 00:00:00 2001 From: Brikaa Date: Tue, 7 Sep 2021 19:23:07 +0200 Subject: [PATCH 37/53] Add powershell to readme --- readme.md | 1 + 1 file changed, 1 insertion(+) diff --git a/readme.md b/readme.md index 3c9c8a8..2d1f3d6 100644 --- a/readme.md +++ b/readme.md @@ -346,6 +346,7 @@ Content-Type: application/json `perl`, `php`, `ponylang`, +`powershell`, `prolog`, `pure`, `pyth`, From 4577a02401cef45deb6b3c96e39b4cd7102dbff4 Mon Sep 17 00:00:00 2001 From: Brian Seymour Date: Wed, 8 Sep 2021 12:31:06 -0500 Subject: [PATCH 38/53] Update readme.md --- readme.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/readme.md b/readme.md index 3c9c8a8..f7dec40 100644 --- a/readme.md +++ b/readme.md @@ -88,7 +88,9 @@ POST https://emkc.org/api/v2/piston/execute > Important Note: The Piston API is rate limited to 5 requests per second. If you have a need for more requests than that > and it's for a good cause, please reach out to me (EngineerMan#0001) on [Discord](https://discord.gg/engineerman) -> so we can discuss potentially getting you an unlimited key. +> so we can discuss potentially getting you an unlimited key. What is and isn't a good cause is up to me, but, in general +> if your project is a) open source, b) helping people at not cost to them, and c) not likely to use tons of resources +> thereby impairing another's ability to enjoy Piston, you'll likely be granted a key.
From e197ca5b7a382dd77bd10a5c180b1fc4033029c6 Mon Sep 17 00:00:00 2001 From: Brian Seymour Date: Wed, 8 Sep 2021 12:32:04 -0500 Subject: [PATCH 39/53] Update readme.md --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index f7dec40..b9d8e12 100644 --- a/readme.md +++ b/readme.md @@ -89,7 +89,7 @@ POST https://emkc.org/api/v2/piston/execute > Important Note: The Piston API is rate limited to 5 requests per second. If you have a need for more requests than that > and it's for a good cause, please reach out to me (EngineerMan#0001) on [Discord](https://discord.gg/engineerman) > so we can discuss potentially getting you an unlimited key. What is and isn't a good cause is up to me, but, in general -> if your project is a) open source, b) helping people at not cost to them, and c) not likely to use tons of resources +> if your project is a) open source, b) helping people at no cost to them, and c) not likely to use tons of resources > thereby impairing another's ability to enjoy Piston, you'll likely be granted a key.
From 08b2fa094a12653af20846658b87d93d8fc533b6 Mon Sep 17 00:00:00 2001 From: Brikaa Date: Thu, 9 Sep 2021 20:20:01 +0200 Subject: [PATCH 40/53] pkg(mono-6.12.0): Added mono-basic 7.4 --- packages/mono/6.12.0/build.sh | 18 +++++++++++++++--- packages/mono/6.12.0/compile | 16 ++++++++++++++-- packages/mono/6.12.0/environment | 2 +- packages/mono/6.12.0/metadata.json | 4 ++++ packages/mono/6.12.0/test.vb | 9 +++++++++ 5 files changed, 43 insertions(+), 6 deletions(-) create mode 100644 packages/mono/6.12.0/test.vb diff --git a/packages/mono/6.12.0/build.sh b/packages/mono/6.12.0/build.sh index 2cecc07..7bb4b63 100755 --- a/packages/mono/6.12.0/build.sh +++ b/packages/mono/6.12.0/build.sh @@ -2,19 +2,31 @@ PREFIX=$(realpath $(dirname $0)) -mkdir -p build/mono +mkdir -p build/mono build/mono-basic cd build curl "https://download.mono-project.com/sources/mono/mono-6.12.0.122.tar.xz" -o mono.tar.xz +curl -L "https://github.com/mono/mono-basic/archive/refs/tags/4.7.tar.gz" -o mono-basic.tar.gz tar xf mono.tar.xz --strip-components=1 -C mono +tar xf mono-basic.tar.gz --strip-components=1 -C mono-basic +# Compiling Mono cd mono ./configure --prefix "$PREFIX" -make -j$(nproc) +make -j$(nproc) make install -j$(nproc) +export PATH="$PREFIX/bin:$PATH" # To be able to use mono commands + +# Compiling mono-basic +cd ../mono-basic +./configure --prefix="$PREFIX" + +make -j$(nproc) PLATFORM="linux" # Avoids conflict with the $PLATFORM variable we have +make install -j$(nproc) PLATFORM="linux" + +# Remove redundant files cd ../../ rm -rf build - diff --git a/packages/mono/6.12.0/compile b/packages/mono/6.12.0/compile index 8728714..5246bc2 100644 --- a/packages/mono/6.12.0/compile +++ b/packages/mono/6.12.0/compile @@ -1,4 +1,16 @@ #!/bin/bash -rename 's/$/\.cs/' "$@" # Add .cs extension -csc -out:out *.cs +case "${PISTON_LANGUAGE}" in + csharp) + rename 's/$/\.cs/' "$@" # Add .cs extension + csc -out:out *.cs + ;; + basic) + rename 's/$/\.vb/' "$@" # Add .vb extension + vbnc -out:out *.vb + ;; + *) + echo "How did you get here? (${PISTON_LANGUAGE})" + exit 1 + ;; +esac diff --git a/packages/mono/6.12.0/environment b/packages/mono/6.12.0/environment index bd0ff98..977a5e8 100644 --- a/packages/mono/6.12.0/environment +++ b/packages/mono/6.12.0/environment @@ -1 +1 @@ -export PATH=$PWD/bin:$PATH \ No newline at end of file +export PATH=$PWD/bin:$PATH diff --git a/packages/mono/6.12.0/metadata.json b/packages/mono/6.12.0/metadata.json index a053884..4d09ae7 100644 --- a/packages/mono/6.12.0/metadata.json +++ b/packages/mono/6.12.0/metadata.json @@ -5,6 +5,10 @@ { "language": "csharp", "aliases": ["mono", "mono-csharp", "mono-c#", "mono-cs", "c#", "cs"] + }, + { + "language": "basic", + "aliases": ["vb", "mono-vb", "mono-basic", "visual-basic", "visual basic"] } ] } diff --git a/packages/mono/6.12.0/test.vb b/packages/mono/6.12.0/test.vb new file mode 100644 index 0000000..291042e --- /dev/null +++ b/packages/mono/6.12.0/test.vb @@ -0,0 +1,9 @@ +Imports System + +Module Module1 + + Sub Main() + Console.WriteLine("OK") + End Sub + +End Module From dbf89dbb582e51d856e54fd00a81a45562b7fe68 Mon Sep 17 00:00:00 2001 From: Brikaa Date: Thu, 9 Sep 2021 20:21:41 +0200 Subject: [PATCH 41/53] Add basic to readme --- readme.md | 1 + 1 file changed, 1 insertion(+) diff --git a/readme.md b/readme.md index b9d8e12..4f75115 100644 --- a/readme.md +++ b/readme.md @@ -360,6 +360,7 @@ Content-Type: application/json `scala`, `swift`, `typescript`, +`basic`, `vlang`, `yeethon`, `zig`, From b3772c9502207f264357ca219b56d4d76ab376e3 Mon Sep 17 00:00:00 2001 From: Brikaa Date: Fri, 10 Sep 2021 15:33:36 +0200 Subject: [PATCH 42/53] pkg(mono-6.12.0): redirect mono error to stderr --- packages/mono/6.12.0/compile | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/mono/6.12.0/compile b/packages/mono/6.12.0/compile index 5246bc2..e3ae230 100644 --- a/packages/mono/6.12.0/compile +++ b/packages/mono/6.12.0/compile @@ -1,13 +1,20 @@ #!/bin/bash +check_errors () { + grep -q 'error [A-Z]\+[0-9]\+:' check.txt && cat check.txt 1>&2 || cat check.txt + rm check.txt +} + case "${PISTON_LANGUAGE}" in csharp) rename 's/$/\.cs/' "$@" # Add .cs extension - csc -out:out *.cs + csc -out:out *.cs > check.txt + check_errors ;; basic) rename 's/$/\.vb/' "$@" # Add .vb extension - vbnc -out:out *.vb + vbnc -out:out *.vb > check.txt + check_errors ;; *) echo "How did you get here? (${PISTON_LANGUAGE})" From 5004635c55d5de04c60cb1e1e313cba4c210195e Mon Sep 17 00:00:00 2001 From: Brikaa Date: Sat, 11 Sep 2021 14:20:36 +0200 Subject: [PATCH 43/53] Added bash shell option in piston script --- piston | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/piston b/piston index 2dc36fa..f979f44 100755 --- a/piston +++ b/piston @@ -23,24 +23,25 @@ case $1 in echo " start Starts piston" echo " stop Stops piston" echo " restart Restarts piston" + echo " bash Opens a bash shell for the piston_api container" echo echo " update Fetches and applies latest updates" echo echo " Passthrough to piston cli tool" - echo + echo echo "Development Commands:" - + if [ $PISTON_ENV == dev ]; then - + echo " clean-pkgs Clean any package build artifacts on disk" echo " clean-repo Remove all packages from local repo" echo " build-pkg Build a package" - + else - + echo " Switch to developement environment for more info" echo " > piston switch dev" - + fi ;; @@ -51,6 +52,7 @@ case $1 in restart) docker_compose restart ;; start) docker_compose up -d ;; stop) docker_compose down ;; + bash) docker_compose exec api /bin/bash ;; update) git pull @@ -74,4 +76,4 @@ case $1 in cd ../ node cli/index.js "$@" ;; -esac \ No newline at end of file +esac From 1250cf213adce6bb5d26615506d9823463c0a371 Mon Sep 17 00:00:00 2001 From: Brikaa Date: Sun, 12 Sep 2021 14:38:57 +0200 Subject: [PATCH 44/53] pkg(rscript-4.1.1): Added R 4.1.1 --- api/Dockerfile | 2 +- packages/rscript/4.1.1/build.sh | 16 ++++++++++++++++ packages/rscript/4.1.1/environment | 1 + packages/rscript/4.1.1/metadata.json | 5 +++++ packages/rscript/4.1.1/run | 2 ++ packages/rscript/4.1.1/test.r | 1 + repo/Dockerfile | 4 ++-- 7 files changed, 28 insertions(+), 3 deletions(-) create mode 100755 packages/rscript/4.1.1/build.sh create mode 100644 packages/rscript/4.1.1/environment create mode 100644 packages/rscript/4.1.1/metadata.json create mode 100644 packages/rscript/4.1.1/run create mode 100644 packages/rscript/4.1.1/test.r diff --git a/api/Dockerfile b/api/Dockerfile index 668c54a..ec0d2a8 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -13,7 +13,7 @@ RUN apt-get update && \ libncurses6 libncurses5 libedit-dev libseccomp-dev rename procps python3 \ libreadline-dev libblas-dev liblapack-dev libpcre3-dev libarpack2-dev \ libfftw3-dev libglpk-dev libqhull-dev libqrupdate-dev libsuitesparse-dev \ - libsundials-dev && \ + libsundials-dev libpcre2-dev && \ rm -rf /var/lib/apt/lists/* RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen diff --git a/packages/rscript/4.1.1/build.sh b/packages/rscript/4.1.1/build.sh new file mode 100755 index 0000000..9837c22 --- /dev/null +++ b/packages/rscript/4.1.1/build.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +PREFIX=$(realpath $(dirname $0)) + +mkdir build +cd build + +curl https://cloud.r-project.org/src/base/R-4/R-4.1.1.tar.gz -o R.tar.gz +tar xzf R.tar.gz --strip-components 1 + +./configure --prefix="$PREFIX" --with-x=no +make -j$(nproc) +make install -j$(nproc) + +cd ../ +rm -rf build diff --git a/packages/rscript/4.1.1/environment b/packages/rscript/4.1.1/environment new file mode 100644 index 0000000..977a5e8 --- /dev/null +++ b/packages/rscript/4.1.1/environment @@ -0,0 +1 @@ +export PATH=$PWD/bin:$PATH diff --git a/packages/rscript/4.1.1/metadata.json b/packages/rscript/4.1.1/metadata.json new file mode 100644 index 0000000..db16a76 --- /dev/null +++ b/packages/rscript/4.1.1/metadata.json @@ -0,0 +1,5 @@ +{ + "language": "rscript", + "version": "4.1.1", + "aliases": ["r"] +} diff --git a/packages/rscript/4.1.1/run b/packages/rscript/4.1.1/run new file mode 100644 index 0000000..d122eb8 --- /dev/null +++ b/packages/rscript/4.1.1/run @@ -0,0 +1,2 @@ +#/bin/bash +Rscript "$@" diff --git a/packages/rscript/4.1.1/test.r b/packages/rscript/4.1.1/test.r new file mode 100644 index 0000000..9273f27 --- /dev/null +++ b/packages/rscript/4.1.1/test.r @@ -0,0 +1 @@ +cat('OK') diff --git a/repo/Dockerfile b/repo/Dockerfile index 106fef4..56ca59d 100644 --- a/repo/Dockerfile +++ b/repo/Dockerfile @@ -8,7 +8,8 @@ RUN apt-get update && apt-get install -y unzip autoconf build-essential libssl-d util-linux pciutils usbutils coreutils binutils findutils grep libncurses5-dev \ libncursesw5-dev python3-pip libgmp-dev libmpfr-dev python2 libffi-dev gfortran\ libreadline-dev libblas-dev liblapack-dev libpcre3-dev libarpack2-dev libfftw3-dev \ - libglpk-dev libqhull-dev libqrupdate-dev libsuitesparse-dev libsundials-dev && \ + libglpk-dev libqhull-dev libqrupdate-dev libsuitesparse-dev libsundials-dev \ + libbz2-dev liblzma-dev libpcre2-dev && \ ln -sf /bin/bash /bin/sh && \ rm -rf /var/lib/apt/lists/* && \ update-alternatives --install /usr/bin/python python /usr/bin/python3.7 2 @@ -17,4 +18,3 @@ ADD entrypoint.sh mkindex.sh / ENTRYPOINT ["bash","/entrypoint.sh"] CMD ["--no-build"] - From 864e94739f29afa241b54b03055a3c15aa8d762e Mon Sep 17 00:00:00 2001 From: Brikaa Date: Sun, 12 Sep 2021 14:39:56 +0200 Subject: [PATCH 45/53] Add rscript to readme --- readme.md | 1 + 1 file changed, 1 insertion(+) diff --git a/readme.md b/readme.md index 919025c..f43f70a 100644 --- a/readme.md +++ b/readme.md @@ -356,6 +356,7 @@ Content-Type: application/json `python2`, `raku`, `rockstar`, +`rscript`, `ruby`, `rust`, `scala`, From 252987932c69cbb1104c94dbb2a253db481353d4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 12 Sep 2021 12:57:09 +0000 Subject: [PATCH 46/53] build(deps): bump axios from 0.21.1 to 0.21.2 in /cli Bumps [axios](https://github.com/axios/axios) from 0.21.1 to 0.21.2. - [Release notes](https://github.com/axios/axios/releases) - [Changelog](https://github.com/axios/axios/blob/master/CHANGELOG.md) - [Commits](https://github.com/axios/axios/compare/v0.21.1...v0.21.2) --- updated-dependencies: - dependency-name: axios dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- cli/package-lock.json | 41 ++++++++++++++++++++++++++--------------- cli/package.json | 2 +- 2 files changed, 27 insertions(+), 16 deletions(-) diff --git a/cli/package-lock.json b/cli/package-lock.json index d564e5f..ad65043 100644 --- a/cli/package-lock.json +++ b/cli/package-lock.json @@ -9,7 +9,7 @@ "version": "1.0.0", "license": "MIT", "dependencies": { - "axios": "^0.21.1", + "axios": "^0.21.2", "chalk": "^4.1.0", "minimatch": "^3.0.4", "nocamel": "^1.0.2", @@ -37,11 +37,11 @@ } }, "node_modules/axios": { - "version": "0.21.1", - "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.1.tgz", - "integrity": "sha512-dKQiRHxGD9PPRIUNIWvZhPTPpl1rf/OxTYKsqKUDjBwYylTvV7SjSHJb9ratfyzM6wCdLCOYLzs73qpg5c4iGA==", + "version": "0.21.2", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.2.tgz", + "integrity": "sha512-87otirqUw3e8CzHTMO+/9kh/FSgXt/eVDvipijwDtEuwbkySWZ9SBm6VEubmJ/kLKEoLQV/POhxXFb66bfekfg==", "dependencies": { - "follow-redirects": "^1.10.0" + "follow-redirects": "^1.14.0" } }, "node_modules/balanced-match": { @@ -115,11 +115,22 @@ } }, "node_modules/follow-redirects": { - "version": "1.13.3", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.13.3.tgz", - "integrity": "sha512-DUgl6+HDzB0iEptNQEXLx/KhTmDb8tZUHSeLqpnjpknR70H0nC2t9N73BK6fN4hOvJ84pKlIQVQ4k5FFlBedKA==", + "version": "1.14.3", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.3.tgz", + "integrity": "sha512-3MkHxknWMUtb23apkgz/83fDoe+y+qr0TdgacGIA7bew+QLBo3vdgEN2xEsuXNivpFy4CyDhBBZnNZOtalmenw==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], "engines": { "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } } }, "node_modules/get-caller-file": { @@ -297,11 +308,11 @@ } }, "axios": { - "version": "0.21.1", - "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.1.tgz", - "integrity": "sha512-dKQiRHxGD9PPRIUNIWvZhPTPpl1rf/OxTYKsqKUDjBwYylTvV7SjSHJb9ratfyzM6wCdLCOYLzs73qpg5c4iGA==", + "version": "0.21.2", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.2.tgz", + "integrity": "sha512-87otirqUw3e8CzHTMO+/9kh/FSgXt/eVDvipijwDtEuwbkySWZ9SBm6VEubmJ/kLKEoLQV/POhxXFb66bfekfg==", "requires": { - "follow-redirects": "^1.10.0" + "follow-redirects": "^1.14.0" } }, "balanced-match": { @@ -366,9 +377,9 @@ "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==" }, "follow-redirects": { - "version": "1.13.3", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.13.3.tgz", - "integrity": "sha512-DUgl6+HDzB0iEptNQEXLx/KhTmDb8tZUHSeLqpnjpknR70H0nC2t9N73BK6fN4hOvJ84pKlIQVQ4k5FFlBedKA==" + "version": "1.14.3", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.3.tgz", + "integrity": "sha512-3MkHxknWMUtb23apkgz/83fDoe+y+qr0TdgacGIA7bew+QLBo3vdgEN2xEsuXNivpFy4CyDhBBZnNZOtalmenw==" }, "get-caller-file": { "version": "2.0.5", diff --git a/cli/package.json b/cli/package.json index 6df989d..c244403 100644 --- a/cli/package.json +++ b/cli/package.json @@ -5,7 +5,7 @@ "main": "index.js", "license": "MIT", "dependencies": { - "axios": "^0.21.1", + "axios": "^0.21.2", "chalk": "^4.1.0", "minimatch": "^3.0.4", "nocamel": "^1.0.2", From 1566b4957412658b4a4ee979388da065484da4cb Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Mon, 13 Sep 2021 01:00:54 +1200 Subject: [PATCH 47/53] Build repo-builder if required --- .github/workflows/package-pr.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/package-pr.yaml b/.github/workflows/package-pr.yaml index b3027ec..a7f74c0 100644 --- a/.github/workflows/package-pr.yaml +++ b/.github/workflows/package-pr.yaml @@ -55,7 +55,9 @@ jobs: run: | PACKAGES=$(jq '.[]' -r ${HOME}/files.json | awk -F/ '{ print $2 "-" $3 }' | sort -u) echo "Packages: $PACKAGES" - docker run -v "${{ github.workspace }}:/piston" docker.pkg.github.com/engineer-man/piston/repo-builder:latest --no-server $PACKAGES + docker pull docker.pkg.github.com/engineer-man/piston/repo-builder:latest + docker build -t repo-builder repo + docker run -v "${{ github.workspace }}:/piston" repo-builder --no-server $PACKAGES ls -la packages - name: Upload package as artifact From d630b5ebe7bc6b50fd5ef2298f599d7d5d6a4cf7 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Mon, 13 Sep 2021 01:01:59 +1200 Subject: [PATCH 48/53] Build repo-builder if required --- .github/workflows/package-push.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/package-push.yaml b/.github/workflows/package-push.yaml index ad33f3e..bbb44af 100644 --- a/.github/workflows/package-push.yaml +++ b/.github/workflows/package-push.yaml @@ -33,7 +33,9 @@ jobs: run: | PACKAGES=$(jq '.[]' -r ${HOME}/files.json | awk -F/ '{ print $2 "-" $3 }' | sort -u) echo "Packages: $PACKAGES" - docker run -v "${{ github.workspace }}:/piston" docker.pkg.github.com/engineer-man/piston/repo-builder:latest --no-server $PACKAGES + docker pull docker.pkg.github.com/engineer-man/piston/repo-builder:latest + docker build -t repo-builder repo + docker run -v "${{ github.workspace }}:/piston" repo-builder --no-server $PACKAGES ls -la packages - name: Upload Packages @@ -73,4 +75,4 @@ jobs: file: index tag: pkgs overwrite: true - file_glob: true \ No newline at end of file + file_glob: true From dc20ec2bda9ccf35cb660822a2cbd553b8e05b90 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Mon, 13 Sep 2021 01:09:38 +1200 Subject: [PATCH 49/53] ci: Rebuild API container if required when testing --- .github/workflows/package-pr.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/package-pr.yaml b/.github/workflows/package-pr.yaml index a7f74c0..bb264a3 100644 --- a/.github/workflows/package-pr.yaml +++ b/.github/workflows/package-pr.yaml @@ -91,7 +91,9 @@ jobs: run: | ls -la docker run -v $(pwd)'/repo:/piston/repo' -v $(pwd)'/packages:/piston/packages' -d --name repo docker.pkg.github.com/engineer-man/piston/repo-builder --no-build - docker run --network container:repo -v $(pwd)'/data:/piston' -e PISTON_LOG_LEVEL=DEBUG -e 'PISTON_REPO_URL=http://localhost:8000/index' -d --name api docker.pkg.github.com/engineer-man/piston/api + docker pull docker.pkg.github.com/engineer-man/piston/api + docker build -t piston-api api + docker run --network container:repo -v $(pwd)'/data:/piston' -e PISTON_LOG_LEVEL=DEBUG -e 'PISTON_REPO_URL=http://localhost:8000/index' -d --name api piston-api echo Waiting for API to start.. docker run --network container:api appropriate/curl -s --retry 10 --retry-connrefused http://localhost:2000/api/v2/runtimes From 5e1a51c8133b9873a258822646a98571cc5d7b1b Mon Sep 17 00:00:00 2001 From: Brikaa Date: Wed, 15 Sep 2021 11:33:02 +0200 Subject: [PATCH 50/53] pkg(vlang-0.1.13): Fix command line arguments bug --- packages/vlang/0.1.13/run | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/vlang/0.1.13/run b/packages/vlang/0.1.13/run index 18b6c05..c999cca 100644 --- a/packages/vlang/0.1.13/run +++ b/packages/vlang/0.1.13/run @@ -6,8 +6,9 @@ export TMPDIR="$PWD" # Put instructions to run the runtime -rename 's/$/\.v/' "$@" # Add .v extension +filename=$1 + +rename 's/$/\.v/' $filename # Add .v extension -filename=$1.v shift -v run $filename "$@" \ No newline at end of file +v run $filename.v "$@" From 7cf99fb4f2c822eb01add42e0b7201cc37d6a636 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Wed, 22 Sep 2021 11:26:35 +1200 Subject: [PATCH 51/53] Update readme.md --- readme.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/readme.md b/readme.md index f43f70a..a154fe6 100644 --- a/readme.md +++ b/readme.md @@ -41,6 +41,20 @@
+# Notes About Hacktoberfest + +While we are accepting pull requests for Hacktoberfest, we will reject any low-quality PRs. +If we see PR abuse for Hacktoberfest, we will stop providing Hacktoberfest approval for pull requests. + +We are accepting PRs for: +* Packages - updating package versions, adding new packages +* Documentation updates +* CLI/API improvements - please discuss these with us in the Discord first + +Any queries or concerns, ping @HexF#0015 in the Discord. + +
+ # About

From 15e2e81d9681284e5c5043e86164644ec6cf653e Mon Sep 17 00:00:00 2001 From: Dan Vargas Date: Wed, 22 Sep 2021 16:31:19 -0500 Subject: [PATCH 52/53] fix stdin for normal execute --- api/src/api/v2.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/api/v2.js b/api/src/api/v2.js index 0c34d70..215453b 100644 --- a/api/src/api/v2.js +++ b/api/src/api/v2.js @@ -106,7 +106,7 @@ function get_job(body){ runtime: rt, alias: language, args: args || [], - stdin: '', + stdin: stdin || "", files, timeouts: { run: run_timeout || 3000, From 1835ab5cab354e79f05e052b4c58c3ecc2ba9617 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Thu, 30 Sep 2021 08:11:47 +1300 Subject: [PATCH 53/53] Add self to license --- license | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/license b/license index 4f45aea..fd203f8 100644 --- a/license +++ b/license @@ -1,4 +1,4 @@ -Copyright (c) 2018-2021 Brian Seymour, EMKC Contributors +Copyright (c) 2018-2021 Brian Seymour, Thomas Hobson, EMKC Contributors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal