From 83e4a1a136f0d027cff88541a14bc9e4df7c6b82 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Mon, 31 Jan 2022 17:01:28 +1300 Subject: [PATCH] Fix issues after merging upstream Implements a simple container builder for runtime sets --- Dockerfile.withset | 12 +++++++ api/default.nix | 17 ++++++++++ api/src/bin/test.js | 81 ++++++++++++++++++++++----------------------- api/src/job.js | 6 +--- package-lock.json | 32 ++++++++++++++++++ package.json | 5 +++ piston | 37 ++++++++++++++++----- 7 files changed, 135 insertions(+), 55 deletions(-) create mode 100644 Dockerfile.withset create mode 100644 package-lock.json create mode 100644 package.json diff --git a/Dockerfile.withset b/Dockerfile.withset new file mode 100644 index 0000000..4fd9a0b --- /dev/null +++ b/Dockerfile.withset @@ -0,0 +1,12 @@ +# This "FROM" image is previously emitted by nix +FROM ghcr.io/engineer-man/piston:base-latest + +ENV PISTON_FLAKE_PATH=/piston/packages +COPY runtimes/ /piston/packages/runtimes +COPY flake.nix flake.lock /piston/packages/ + + +ARG RUNTIMESET=all +ENV PISTON_RUNTIME_SET=$RUNTIMESET + +RUN piston-install \ No newline at end of file diff --git a/api/default.nix b/api/default.nix index c194587..a4d4868 100644 --- a/api/default.nix +++ b/api/default.nix @@ -51,6 +51,8 @@ with pkgs; rec { do echo "nixbld$i:x:$(( $i + 30000 )):30000:Nix build user $i:/var/empty:/run/current-system/sw/bin/nologin" >> etc/passwd done + + chmod 1777 {,var/}tmp/ ''; config = { @@ -61,6 +63,21 @@ with pkgs; rec { "SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt" "GIT_SSL_CAINFO=/etc/ssl/certs/ca-bundle.crt" "NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt" + "PATH=${lib.concatStringsSep ":" [ + "/usr/local/sbin" + "/usr/local/bin" + "/usr/sbin" + "/usr/bin" + "/sbin" + "/bin" + "/root/.nix-profile/bin" + "/nix/var/nix/profiles/default/bin" + "/nix/var/nix/profiles/default/sbin" + ]}" + "MANPATH=${lib.concatStringsSep ":" [ + "/root/.nix-profile/share/man" + "/nix/var/nix/profiles/default/share/man" + ]}" ]; ExposedPorts = { diff --git a/api/src/bin/test.js b/api/src/bin/test.js index 134ce2a..64c467e 100755 --- a/api/src/bin/test.js +++ b/api/src/bin/test.js @@ -5,108 +5,105 @@ const config = require('../config'); const Logger = require('logplease'); const logger = Logger.create('test'); const cp = require('child_process'); -const runtime = require("../runtime"); +const runtime = require('../runtime'); const { Job } = require('../job'); -(async function(){ +(async function () { logger.info('Setting loglevel to', config.log_level); Logger.setLogLevel(config.log_level); - - let runtimes_to_test; let failed = false; - if(process.argv[2] === "--all"){ + if (process.argv[2] === '--all') { // load all runtimes_to_test = JSON.parse( - cp.execSync(`nix eval ${config.flake_path}#pistonRuntimes --json --apply builtins.attrNames`) + cp.execSync( + `nix eval ${config.flake_path}#pistonRuntimes --json --apply builtins.attrNames` + ) ); - }else{ + } else { runtimes_to_test = [process.argv[2]]; } - - for (const runtime_name of runtimes_to_test) { - - const runtime_path = `${config.flake_path}#pistonRuntimes.${runtime_name}`; logger.info(`Testing runtime ${runtime_path}`); logger.debug(`Loading runtime metadata`); - const metadata = JSON.parse(cp.execSync(`nix eval --json ${runtime_path}.metadata --json`)); + const metadata = JSON.parse( + cp.execSync(`nix eval --json ${runtime_path}.metadata --json`) + ); logger.debug(`Loading runtime tests`); - const tests = JSON.parse(cp.execSync(`nix eval --json ${runtime_path}.tests --json`)); + const tests = JSON.parse( + cp.execSync(`nix eval --json ${runtime_path}.tests --json`) + ); logger.debug(`Loading runtime`); const testable_runtime = new runtime.Runtime({ ...metadata, - flake_path: runtime_path + ...runtime.Runtime.compute_all_limits( + metadata.language, + metadata.limit_overrides + ), + flake_path: runtime_path, }); testable_runtime.ensure_built(); - logger.info(`Running tests`); for (const test of tests) { - const files = []; for (const file_name of Object.keys(test.files)) { const file_content = test.files[file_name]; const this_file = { name: file_name, - content: file_content + content: file_content, }; - if(file_name == test.main) - files.unshift(this_file); - else - files.push(this_file); - + if (file_name == test.main) files.unshift(this_file); + else files.push(this_file); } - const job = new Job({ runtime: testable_runtime, args: test.args || [], - stdin: test.stdin || "", + stdin: test.stdin || '', files, timeouts: { run: 3000, - compile: 10000 + compile: 10000, }, memory_limits: { run: config.run_memory_limit, - compile: config.compile_memory_limit - } + compile: config.compile_memory_limit, + }, }); - await job.prime() - const result = await job.execute() - await job.cleanup() - - if(result.run.stdout.trim() !== "OK"){ + await job.prime(); + const result = await job.execute(); + await job.cleanup(); + + if (result.run.stdout.trim() !== 'OK') { failed = true; - logger.error("Test Failed:") - console.log(job, result) - }else{ - logger.info("Test Passed") + logger.error('Test Failed:'); + console.log(job, result); + } else { + logger.info('Test Passed'); } } } - if(failed) { - logger.error("One or more tests failed") + if (failed) { + logger.error('One or more tests failed'); process.exit(1); - } - else { - logger.info("All tests passed") + } else { + logger.info('All tests passed'); process.exit(0); } -})() \ No newline at end of file +})(); diff --git a/api/src/job.js b/api/src/job.js index e05f017..c562693 100644 --- a/api/src/job.js +++ b/api/src/job.js @@ -139,10 +139,6 @@ class Job { var output = ''; const proc = cp.spawn(proc_call[0], proc_call.splice(1), { - env: { - ...this.runtime.env_vars, - PISTON_LANGUAGE: this.runtime.language, - }, stdio: 'pipe', cwd: this.dir, uid: this.uid, @@ -250,7 +246,7 @@ class Job { this.logger.debug('Running'); const run = await this.safe_call( - path.join(this.runtime.pkgdir, 'run'), + this.runtime.run, [code_files[0].name, ...this.args], this.timeouts.run, this.memory_limits.run diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..5c51a1d --- /dev/null +++ b/package-lock.json @@ -0,0 +1,32 @@ +{ + "name": "piston", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "devDependencies": { + "prettier": "2.4.1" + } + }, + "node_modules/prettier": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.4.1.tgz", + "integrity": "sha512-9fbDAXSBcc6Bs1mZrDYb3XKzDLm4EXXL9sC1LqKP5rZkT6KRr/rf9amVUcODVXgguK/isJz0d0hP72WeaKWsvA==", + "dev": true, + "bin": { + "prettier": "bin-prettier.js" + }, + "engines": { + "node": ">=10.13.0" + } + } + }, + "dependencies": { + "prettier": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.4.1.tgz", + "integrity": "sha512-9fbDAXSBcc6Bs1mZrDYb3XKzDLm4EXXL9sC1LqKP5rZkT6KRr/rf9amVUcODVXgguK/isJz0d0hP72WeaKWsvA==", + "dev": true + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..8f07606 --- /dev/null +++ b/package.json @@ -0,0 +1,5 @@ +{ + "devDependencies": { + "prettier": "2.4.1" + } +} diff --git a/piston b/piston index 67d2fa0..cacf23e 100755 --- a/piston +++ b/piston @@ -5,15 +5,16 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) CONTAINER_NAME="piston_api" IMAGE_TAG="base-latest" -IMAGE_NAME="ghcr.io/piston" +IMAGE_NAME="ghcr.io/engineer-man/piston" IMAGE_NAME_DEV="piston" SUBCOMMAND="$1" shift -cmd_build(){ +build_base(){ CONTAINER_PATH="$(nix build ".#container" --no-link --json | jq '.[0].outputs.out' -r)" docker load -i $CONTAINER_PATH + docker tag "$IMAGE_NAME_DEV:$IMAGE_TAG" "$IMAGE_NAME:$IMAGE_TAG" } case "$SUBCOMMAND" in @@ -22,9 +23,10 @@ case "$SUBCOMMAND" in restart) docker restart $CONTAINER_NAME ;; start) docker run \ + -p 2000:2000 \ --rm \ --name $CONTAINER_NAME \ - -it "$IMAGE_NAME:$IMAGE_TAG" + -d "$IMAGE_NAME:$IMAGE_TAG" ;; stop) docker stop $CONTAINER_NAME ;; bash|shell) docker exec -it $CONTAINER_NAME bash ;; @@ -35,22 +37,40 @@ case "$SUBCOMMAND" in ;; # dev commands + + scaffold) + pushd $SCRIPT_DIR/runtimes > /dev/null + ./scaffold.sh $1 $2 + popd > /dev/null + ;; - build) cmd_build ;; + build) + build_base + if [[ ! -z "$1" ]]; then + # $1 contains a variant to build + docker build \ + --build-arg RUNTIMESET=$1 \ + -f $SCRIPT_DIR/Dockerfile.withset \ + -t "$IMAGE_NAME_DEV:$1-latest" \ + . + fi + ;; start-dev) - cmd_build + build_base docker run \ --rm \ + -p 2000:2000 \ -it \ --name $CONTAINER_NAME \ + -e PISTON_LOG_LEVEL=DEBUG \ -e PISTON_FLAKE_PATH=/piston/packages \ -v $PWD:/piston/packages \ - -it "$IMAGE_NAME_DEV:$IMAGE_TAG" + -d "$IMAGE_NAME_DEV:$IMAGE_TAG" ;; test) - cmd_build + build_base docker run \ --rm \ -it \ @@ -82,7 +102,8 @@ case "$SUBCOMMAND" in echo "See https://nixos.wiki/wiki/Nix_Installation_Guide#Stable_Nix" echo echo " start-dev Builds a container locally and starts piston" - echo " build Builds and loads the API container" + echo " build [runtime-set] Builds and loads the API container optionally" + echo " including the runtime set within it" echo " scaffold [runtime] Initializes a new runtime" echo " test Runs unit tests on the given runtime" echo " Optionally set runtime to --all to test all"