From e022e34a376a223c092e71f3163bb509457c043d Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sun, 30 Jan 2022 22:29:21 +1300 Subject: [PATCH] Add nix runtime testing and pre-installing runtimes --- api/default.nix | 47 +++++++++- api/entrypoint.sh | 14 --- api/package.json | 6 +- api/src/bin/install.js | 39 +++++++++ api/src/{ => bin}/pistond.js | 14 +-- api/src/bin/test.js | 112 ++++++++++++++++++++++++ api/src/config.js | 6 ++ api/src/runtime.js | 13 +-- flake.nix | 67 +++----------- piston | 163 ++++++++++++++++------------------- result | 1 - 11 files changed, 306 insertions(+), 176 deletions(-) delete mode 100755 api/entrypoint.sh create mode 100755 api/src/bin/install.js rename api/src/{ => bin}/pistond.js (85%) create mode 100755 api/src/bin/test.js delete mode 120000 result diff --git a/api/default.nix b/api/default.nix index cc6cb70..c194587 100644 --- a/api/default.nix +++ b/api/default.nix @@ -1,5 +1,5 @@ -{pkgs, ...}: -with pkgs; { +{pkgs, nosocket, ...}: +with pkgs; rec { package = mkYarnPackage { name = "piston"; src = ./.; @@ -25,4 +25,47 @@ with pkgs; { }; }; }; + container = pkgs.dockerTools.buildLayeredImageWithNixDb { + name = "piston"; + tag = "base-latest"; + + contents = with pkgs; [ + package + nosocket + bash + nixFlakes + coreutils-full + cacert.out + git + gnutar + gzip + gnugrep + util-linux + ]; + + extraCommands = '' + mkdir -p piston/{jobs,runtimes} etc/nix {,var/}tmp run/lock + echo -e "experimental-features = nix-command flakes" >> etc/nix/nix.conf + echo "nixbld:x:30000:nixbld1,nixbld10,nixbld11,nixbld12,nixbld13,nixbld14,nixbld15,nixbld16,nixbld17,nixbld18,nixbld19,nixbld2,nixbld20,nixbld21,nixbld22,nixbld23,nixbld24,nixbld25,nixbld26,nixbld27,nixbld28,nixbld29,nixbld3,nixbld30,nixbld31,nixbld32,nixbld4,nixbld5,nixbld6,nixbld7,nixbld8,nixbld9" >> etc/group + for i in $(seq 1 32) + do + echo "nixbld$i:x:$(( $i + 30000 )):30000:Nix build user $i:/var/empty:/run/current-system/sw/bin/nologin" >> etc/passwd + done + ''; + + config = { + Cmd = ["${package}/bin/pistond"]; + Env = [ + "NIX_PAGER=cat" + "USER=nobody" + "SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt" + "GIT_SSL_CAINFO=/etc/ssl/certs/ca-bundle.crt" + "NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt" + ]; + + ExposedPorts = { + "2000/tcp" = {}; + }; + }; + }; } \ No newline at end of file diff --git a/api/entrypoint.sh b/api/entrypoint.sh deleted file mode 100755 index aedcedd..0000000 --- a/api/entrypoint.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash -echo "Starting Piston API" - -echo "Checking presense of nix store" -if [[ ! -f "/nix/piston_detected" ]]; then - echo "Nix Store is not loaded, assuming /nix has been mounted - copying contents" - cp -rp /var/nix/* /nix -fi - -echo "Adding nix to env" -. ~/.profile - -echo "Launching Piston API" -node src \ No newline at end of file diff --git a/api/package.json b/api/package.json index 49e55db..0121f1b 100644 --- a/api/package.json +++ b/api/package.json @@ -2,7 +2,7 @@ "name": "piston-api", "version": "4.0.0", "description": "API for piston - a high performance code execution engine", - "main": "src/pistond.js", + "main": "src/bin/pistond.js", "dependencies": { "body-parser": "^1.19.0", "express": "^4.17.1", @@ -23,6 +23,8 @@ "prettier": "2.2.1" }, "bin": { - "pistond": "./src/pistond.js" + "pistond": "./src/bin/pistond.js", + "piston-install": "./src/bin/install.js", + "piston-test": "./src/bin/test.js" } } diff --git a/api/src/bin/install.js b/api/src/bin/install.js new file mode 100755 index 0000000..b9a2b72 --- /dev/null +++ b/api/src/bin/install.js @@ -0,0 +1,39 @@ +#!/usr/bin/env node +// Downloads all packages specified into the nix store +require('nocamel'); +const config = require('../config'); +const Logger = require('logplease'); +const logger = Logger.create('install'); +const cp = require('child_process'); + +logger.info('Setting loglevel to', config.log_level); +Logger.setLogLevel(config.log_level); + +const runtimes_path = `${config.flake_path}#pistonRuntimeSets.${config.runtime_set}`; +logger.info(`Installing runtimes in ${runtimes_path}`); + +logger.debug("Getting package listing"); +const runtimes = JSON.parse(cp.execSync(`nix eval --json ${runtimes_path} --apply builtins.attrNames`)); + +logger.info(`Got runtimes: ${runtimes}`); + +runtimes.forEach(runtime => { + logger.debug(`Loading metadata for ${runtime}`); + const runtime_metadata = JSON.parse(cp.execSync(`nix eval --json ${runtimes_path}.${runtime}.metadata`)); + const namever = `${runtime}-${runtime_metadata.version}`; + logger.info(`Installing ${namever}`); + + + function install(key){ + logger.debug(`Installing ${namever}-${key}`); + const command = `nix build ${runtimes_path}.${runtime}.metadata.${key} -o /piston/runtimes/${namever}-${key}`; + cp.execSync(command); + } + + install("run"); + if(runtime_metadata.compile) install("compile"); + + logger.info(`Installed ${namever}`); +}); + +logger.info("Done"); \ No newline at end of file diff --git a/api/src/pistond.js b/api/src/bin/pistond.js similarity index 85% rename from api/src/pistond.js rename to api/src/bin/pistond.js index 36e12e3..c56cc99 100755 --- a/api/src/pistond.js +++ b/api/src/bin/pistond.js @@ -3,16 +3,16 @@ require('nocamel'); const Logger = require('logplease'); const express = require('express'); const expressWs = require('express-ws'); -const globals = require('./globals'); -const config = require('./config'); +const globals = require('../globals'); +const config = require('../config'); const cp = require('child_process'); const path = require('path'); const fs = require('fs/promises'); const fss = require('fs'); const body_parser = require('body-parser'); -const runtime = require('./runtime'); +const runtime = require('../runtime'); -const logger = Logger.create('index'); +const logger = Logger.create('pistond'); const app = express(); expressWs(app); @@ -39,7 +39,7 @@ expressWs(app); logger.info('Loading packages'); - const runtimes_data = cp.execSync(`nix eval --json ${config.flake_path}#pistonRuntimes --apply builtins.attrNames`).toString(); + const runtimes_data = cp.execSync(`nix eval --json ${config.flake_path}#pistonRuntimeSets.${config.runtime_set} --apply builtins.attrNames`).toString(); const runtimes = JSON.parse(runtimes_data); runtimes.for_each(pkg => runtime.load_runtime(pkg)); @@ -59,8 +59,8 @@ expressWs(app); logger.debug('Registering Routes'); - const api_v2 = require('./api/v2'); - const api_v3 = require('./api/v3'); + const api_v2 = require('../api/v2'); + const api_v3 = require('../api/v3'); app.use('/api/v2', api_v2); app.use('/api/v3', api_v3); diff --git a/api/src/bin/test.js b/api/src/bin/test.js new file mode 100755 index 0000000..134ce2a --- /dev/null +++ b/api/src/bin/test.js @@ -0,0 +1,112 @@ +#!/usr/bin/env node +// Test the specified package +require('nocamel'); +const config = require('../config'); +const Logger = require('logplease'); +const logger = Logger.create('test'); +const cp = require('child_process'); +const runtime = require("../runtime"); +const { Job } = require('../job'); + +(async function(){ + logger.info('Setting loglevel to', config.log_level); + Logger.setLogLevel(config.log_level); + + + + let runtimes_to_test; + let failed = false; + + if(process.argv[2] === "--all"){ + // load all + runtimes_to_test = JSON.parse( + cp.execSync(`nix eval ${config.flake_path}#pistonRuntimes --json --apply builtins.attrNames`) + ); + }else{ + runtimes_to_test = [process.argv[2]]; + } + + + + for (const runtime_name of runtimes_to_test) { + + + const runtime_path = `${config.flake_path}#pistonRuntimes.${runtime_name}`; + logger.info(`Testing runtime ${runtime_path}`); + + logger.debug(`Loading runtime metadata`); + const metadata = JSON.parse(cp.execSync(`nix eval --json ${runtime_path}.metadata --json`)); + + logger.debug(`Loading runtime tests`); + const tests = JSON.parse(cp.execSync(`nix eval --json ${runtime_path}.tests --json`)); + + logger.debug(`Loading runtime`); + + const testable_runtime = new runtime.Runtime({ + ...metadata, + flake_path: runtime_path + }); + + testable_runtime.ensure_built(); + + + logger.info(`Running tests`); + + for (const test of tests) { + + const files = []; + + for (const file_name of Object.keys(test.files)) { + const file_content = test.files[file_name]; + const this_file = { + name: file_name, + content: file_content + }; + + if(file_name == test.main) + files.unshift(this_file); + else + files.push(this_file); + + } + + + const job = new Job({ + runtime: testable_runtime, + args: test.args || [], + stdin: test.stdin || "", + files, + timeouts: { + run: 3000, + compile: 10000 + }, + memory_limits: { + run: config.run_memory_limit, + compile: config.compile_memory_limit + } + }); + + await job.prime() + const result = await job.execute() + await job.cleanup() + + if(result.run.stdout.trim() !== "OK"){ + failed = true; + + logger.error("Test Failed:") + console.log(job, result) + }else{ + logger.info("Test Passed") + } + } + } + + if(failed) { + logger.error("One or more tests failed") + process.exit(1); + } + else { + logger.info("All tests passed") + process.exit(0); + } +})() \ No newline at end of file diff --git a/api/src/config.js b/api/src/config.js index 8ddc3f5..897a938 100644 --- a/api/src/config.js +++ b/api/src/config.js @@ -176,6 +176,12 @@ const options = [ default: 'github:engineer-man/piston?directory=packages', validators: [], }, + { + key: 'runtime_set', + desc: 'Key on the flake specified by flake_path to access runtimes from', + default: 'all', + validators: [] + }, { key: 'max_concurrent_jobs', desc: 'Maximum number of concurrent jobs to run at one time', diff --git a/api/src/runtime.js b/api/src/runtime.js index 9ebd724..e6f72fb 100644 --- a/api/src/runtime.js +++ b/api/src/runtime.js @@ -17,7 +17,7 @@ class Runtime { run, compile, packageSupport, - flake_key, + flake_path, timeouts, memory_limits, max_process_count, @@ -41,7 +41,7 @@ class Runtime { this.run = run; this.compile = compile; - this.flake_key = flake_key; + this.flake_path = flake_path; this.package_support = packageSupport; } @@ -112,10 +112,10 @@ class Runtime { ensure_built(){ logger.info(`Ensuring ${this} is built`); - const flake_key = this.flake_key; + const flake_path = this.flake_path; function _ensure_built(key){ - const command = `nix build ${config.flake_path}#pistonRuntimes.${flake_key}.metadata.${key} --no-link`; + const command = `nix build ${flake_path}.metadata.${key} --no-link`; cp.execSync(command, {stdio: "pipe"}) } @@ -128,7 +128,8 @@ class Runtime { static load_runtime(flake_key){ logger.info(`Loading ${flake_key}`) - const metadata_command = `nix eval --json ${config.flake_path}#pistonRuntimes.${flake_key}.metadata`; + const flake_path = `${config.flake_path}#pistonRuntimeSets.${config.runtime_set}.${flake_key}`; + const metadata_command = `nix eval --json ${flake_path}.metadata`; const metadata = JSON.parse(cp.execSync(metadata_command)); const this_runtime = new Runtime({ @@ -137,7 +138,7 @@ class Runtime { metadata.language, metadata.limit_overrides ), - flake_key + flake_path }); this_runtime.ensure_built(); diff --git a/flake.nix b/flake.nix index cce2f11..78b2ee1 100644 --- a/flake.nix +++ b/flake.nix @@ -6,6 +6,10 @@ let system = "x86_64-linux"; pkgs = nixpkgs.legacyPackages.${system}; + baseContainer = (import ./api { + inherit pkgs; + nosocket = self.legacyPackages."${system}".nosocket; + }).container; args = { inherit pkgs; piston = { @@ -53,66 +57,21 @@ }; }; }; - allRuntimes = import ./runtimes args; + runtimes = import ./runtimes args; + runtimeList = names: pkgs.lib.filterAttrs (n: v: n == "bash") runtimes; in { piston = args.piston; - pistonRuntimes = { - "bash" = allRuntimes.bash; + pistonRuntimes = runtimes; + pistonRuntimeSets = { + "all" = runtimes; + "bash-only" = runtimeList ["bash"]; }; - legacyPackages."${system}" = { - piston = (import ./api { inherit pkgs; }).package; + legacyPackages."${system}" = rec { nosocket = (import ./nosocket { inherit pkgs; }).package; + piston = (import ./api { inherit pkgs nosocket; }).package; }; - containerImage = pkgs.dockerTools.buildLayeredImageWithNixDb { - name = "piston"; - tag = "latest"; - - contents = with pkgs; [ - self.legacyPackages."${system}".piston - self.legacyPackages."${system}".nosocket - bash - nixFlakes - coreutils-full - cacert.out - git - gnutar - gzip - gnugrep - util-linux - ]; - - extraCommands = '' - mkdir -p piston/jobs etc/nix {,var/}tmp run/lock - echo -e "experimental-features = nix-command flakes" >> etc/nix/nix.conf - echo "nixbld:x:30000:nixbld1,nixbld10,nixbld11,nixbld12,nixbld13,nixbld14,nixbld15,nixbld16,nixbld17,nixbld18,nixbld19,nixbld2,nixbld20,nixbld21,nixbld22,nixbld23,nixbld24,nixbld25,nixbld26,nixbld27,nixbld28,nixbld29,nixbld3,nixbld30,nixbld31,nixbld32,nixbld4,nixbld5,nixbld6,nixbld7,nixbld8,nixbld9" >> etc/group - for i in $(seq 1 32) - do - echo "nixbld$i:x:$(( $i + 30000 )):30000:Nix build user $i:/var/empty:/run/current-system/sw/bin/nologin" >> etc/passwd - done - ''; - - config = { - Cmd = [ - "${self.legacyPackages."${system}".piston}/bin/pistond" - ]; - - Env = [ - "NIX_PAGER=cat" - "USER=nobody" - "SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt" - "GIT_SSL_CAINFO=/etc/ssl/certs/ca-bundle.crt" - "NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt" - - ]; - - ExposedPorts = { - "2000/tcp" = {}; - }; - }; - }; - - + container = baseContainer; }; } diff --git a/piston b/piston index fbd256d..67d2fa0 100755 --- a/piston +++ b/piston @@ -1,110 +1,93 @@ #!/usr/bin/env bash +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -EXECUTION_PATH="$PWD" -PISTON_PATH="$(dirname "$(realpath "$0")")" -cd "$PISTON_PATH" -PISTON_ENV=$(cat .piston_env 2> /dev/null || echo dev) +CONTAINER_NAME="piston_api" -docker_compose(){ - if [ -f "docker-compose.$PISTON_ENV.yaml" ]; then - docker-compose -f "docker-compose.$PISTON_ENV.yaml" "$@" - else - docker-compose "$@" - fi +IMAGE_TAG="base-latest" +IMAGE_NAME="ghcr.io/piston" +IMAGE_NAME_DEV="piston" + +SUBCOMMAND="$1" +shift + +cmd_build(){ + CONTAINER_PATH="$(nix build ".#container" --no-link --json | jq '.[0].outputs.out' -r)" + docker load -i $CONTAINER_PATH } -init_precommit() { - if [ $PISTON_ENV == "dev" ]; then - rm -f .git/hooks/pre-commit - ln -s "$PISTON_PATH/pre-commit" "$PISTON_PATH/.git/hooks/pre-commit" - fi -} +case "$SUBCOMMAND" in + logs) docker logs -f $CONTAINER_NAME;; -case $1 in - help) + restart) docker restart $CONTAINER_NAME ;; + start) + docker run \ + --rm \ + --name $CONTAINER_NAME \ + -it "$IMAGE_NAME:$IMAGE_TAG" + ;; + stop) docker stop $CONTAINER_NAME ;; + bash|shell) docker exec -it $CONTAINER_NAME bash ;; + + update) + git pull + docker pull "$IMAGE_NAME:$IMAGE_TAG" + ;; + + # dev commands + + build) cmd_build ;; + + start-dev) + cmd_build + docker run \ + --rm \ + -it \ + --name $CONTAINER_NAME \ + -e PISTON_FLAKE_PATH=/piston/packages \ + -v $PWD:/piston/packages \ + -it "$IMAGE_NAME_DEV:$IMAGE_TAG" + ;; + + test) + cmd_build + docker run \ + --rm \ + -it \ + -e PISTON_FLAKE_PATH=/piston/packages \ + -v $PWD:/piston/packages \ + --name piston_test_runner \ + -it "$IMAGE_NAME_DEV:$IMAGE_TAG" \ + piston-test $1 + + ;; + + *) echo "=== Piston Management ===" - echo "Current Environment: $PISTON_ENV" echo echo "Commands:" - echo " select Select the environment" - echo " docker_compose Interact directly with the docker-compose for the selected environment" - echo " logs Show docker-compose logs" + echo " logs Show piston logs" echo echo " start Starts piston" echo " stop Stops piston" echo " restart Restarts piston" - echo " bash Opens a bash shell for the piston_api container" + echo " shell Opens a bash shell for the api container" echo - echo " update Fetches and applies latest updates" + echo " update Fetches latest updates" echo - echo " Passthrough to piston cli tool" + echo " exec Execute the files on piston with language" echo echo "Development Commands:" - - if [ "$PISTON_ENV" == dev ]; then - - echo " clean-pkgs Clean any package build artifacts on disk" - echo " clean-repo Remove all packages from local repo" - echo " list-pkgs Lists all packages that can be built" - echo " build-pkg [builder] Build a package [with desired builder image]" - echo " rebuild Build and restart the docker container" - echo " lint Lint the codebase using prettier" - - else - - echo " Switch to developement environment for more info" - echo " > piston select dev" - - fi - ;; - - - select) echo "$2" > .piston_env ;; - docker_compose) shift; docker_compose "$@";; - logs) docker_compose logs -f ;; - - restart) docker_compose restart ;; - start) - init_precommit - docker_compose up -d - ;; - stop) docker_compose down ;; - bash) docker_compose exec api /bin/bash ;; - - rebuild) - init_precommit - docker_compose build && docker_compose up -d - ;; - - update) - git pull - cd cli && npm i > /dev/null && cd - - docker_compose pull - docker_compose up -d - ;; - - clean-pkgs) git clean -fqXd packages ;; - clean-repo) git clean -fqXd repo ;; - - list-pkgs) find packages -depth 2 | awk -F/ '$2 && $3{ print $2 "-" $3 }' | column ;; - - build-pkg) - PKGSLUG="$2-$3" - BUILDER="${4:-piston-repo-builder}" - echo "Building $PKGSLUG" - echo "Ensuring latest builder image" - docker build repo -t "$BUILDER" - docker run --rm -v "$PWD:/piston" "$BUILDER" --no-server "$PKGSLUG" - ;; - - lint) - npm install - npx prettier --ignore-unknown --write . - ;; - *) - [ -d ./cli/node_modules ] || npm i > /dev/null - cd "$EXECUTION_PATH" - node "${PISTON_PATH}/cli/index.js" "$@" + echo "Running some of these commands require a nix environment setup and on the path" + echo "See https://nixos.wiki/wiki/Nix_Installation_Guide#Stable_Nix" + echo + echo " start-dev Builds a container locally and starts piston" + echo " build Builds and loads the API container" + echo " scaffold [runtime] Initializes a new runtime" + echo " test Runs unit tests on the given runtime" + echo " Optionally set runtime to --all to test all" + echo " NOTE: This is only for the runtimes contained" + echo " within this repo" + echo ;; esac diff --git a/result b/result deleted file mode 120000 index d60c5ac..0000000 --- a/result +++ /dev/null @@ -1 +0,0 @@ -/nix/store/1idnmsddirm1hpxw40r17wsg9p9dscb9-piston.tar.gz \ No newline at end of file