Add nix runtime testing and pre-installing runtimes

api/default.nix

@@ -1,5 +1,5 @@
-{pkgs, ...}:
+{pkgs, nosocket, ...}:
-with pkgs; {
+with pkgs; rec {
   package = mkYarnPackage {
     name = "piston";
     src = ./.;
@@ -25,4 +25,47 @@ with pkgs; {
       };
     };
   };
+  container = pkgs.dockerTools.buildLayeredImageWithNixDb {
+    name = "piston";
+    tag = "base-latest";
+
+    contents = with pkgs; [
+      package
+      nosocket
+      bash
+      nixFlakes
+      coreutils-full
+      cacert.out
+      git
+      gnutar
+      gzip
+      gnugrep
+      util-linux
+    ];
+
+    extraCommands = ''
+      mkdir -p piston/{jobs,runtimes} etc/nix {,var/}tmp run/lock
+      echo -e "experimental-features = nix-command flakes" >> etc/nix/nix.conf
+      echo "nixbld:x:30000:nixbld1,nixbld10,nixbld11,nixbld12,nixbld13,nixbld14,nixbld15,nixbld16,nixbld17,nixbld18,nixbld19,nixbld2,nixbld20,nixbld21,nixbld22,nixbld23,nixbld24,nixbld25,nixbld26,nixbld27,nixbld28,nixbld29,nixbld3,nixbld30,nixbld31,nixbld32,nixbld4,nixbld5,nixbld6,nixbld7,nixbld8,nixbld9" >> etc/group
+      for i in $(seq 1 32)
+      do
+        echo "nixbld$i:x:$(( $i + 30000 )):30000:Nix build user $i:/var/empty:/run/current-system/sw/bin/nologin" >> etc/passwd
+      done
+    '';
+
+    config = {
+      Cmd = ["${package}/bin/pistond"];
+      Env = [
+        "NIX_PAGER=cat"
+        "USER=nobody"
+        "SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt"
+        "GIT_SSL_CAINFO=/etc/ssl/certs/ca-bundle.crt"
+        "NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt"
+      ];
+
+      ExposedPorts = {
+        "2000/tcp" = {};
+      };
+    };
+  };
 }

api/entrypoint.sh

@@ -1,14 +0,0 @@
-#!/usr/bin/env bash
-echo "Starting Piston API"
-
-echo "Checking presense of nix store"
-if [[ ! -f "/nix/piston_detected" ]]; then
-    echo "Nix Store is not loaded, assuming /nix has been mounted - copying contents"
-    cp -rp /var/nix/* /nix
-fi
-
-echo "Adding nix to env"
-. ~/.profile
-
-echo "Launching Piston API"
-node src

api/package.json

@@ -2,7 +2,7 @@
     "name": "piston-api",
     "version": "4.0.0",
     "description": "API for piston - a high performance code execution engine",
-    "main": "src/pistond.js",
+    "main": "src/bin/pistond.js",
     "dependencies": {
         "body-parser": "^1.19.0",
         "express": "^4.17.1",
@@ -23,6 +23,8 @@
         "prettier": "2.2.1"
     },
     "bin": {
-        "pistond": "./src/pistond.js"
+        "pistond": "./src/bin/pistond.js",
+        "piston-install": "./src/bin/install.js",
+        "piston-test": "./src/bin/test.js"
     }
 }

api/src/bin/install.js

@@ -0,0 +1,39 @@
+#!/usr/bin/env node
+// Downloads all packages specified into the nix store
+require('nocamel');
+const config = require('../config');
+const Logger = require('logplease');
+const logger = Logger.create('install');
+const cp = require('child_process');
+
+logger.info('Setting loglevel to', config.log_level);
+Logger.setLogLevel(config.log_level);
+
+const runtimes_path = `${config.flake_path}#pistonRuntimeSets.${config.runtime_set}`;
+logger.info(`Installing runtimes in ${runtimes_path}`);
+
+logger.debug("Getting package listing");
+const runtimes = JSON.parse(cp.execSync(`nix eval --json ${runtimes_path} --apply builtins.attrNames`));
+
+logger.info(`Got runtimes: ${runtimes}`);
+
+runtimes.forEach(runtime => {
+        logger.debug(`Loading metadata for ${runtime}`);
+        const runtime_metadata = JSON.parse(cp.execSync(`nix eval --json ${runtimes_path}.${runtime}.metadata`));
+        const namever = `${runtime}-${runtime_metadata.version}`;
+        logger.info(`Installing ${namever}`);
+        
+
+        function install(key){
+            logger.debug(`Installing ${namever}-${key}`);
+            const command = `nix build ${runtimes_path}.${runtime}.metadata.${key} -o /piston/runtimes/${namever}-${key}`;
+            cp.execSync(command);
+        }
+
+        install("run");
+        if(runtime_metadata.compile) install("compile");        
+
+        logger.info(`Installed ${namever}`);
+});
+
+logger.info("Done");

api/src/bin/pistond.js

@@ -3,16 +3,16 @@ require('nocamel');
 const Logger = require('logplease');
 const express = require('express');
 const expressWs = require('express-ws');
-const globals = require('./globals');
+const globals = require('../globals');
-const config = require('./config');
+const config = require('../config');
 const cp = require('child_process');
 const path = require('path');
 const fs = require('fs/promises');
 const fss = require('fs');
 const body_parser = require('body-parser');
-const runtime = require('./runtime');
+const runtime = require('../runtime');
 
-const logger = Logger.create('index');
+const logger = Logger.create('pistond');
 const app = express();
 expressWs(app);
 
@@ -39,7 +39,7 @@ expressWs(app);
 
     logger.info('Loading packages');
 
-    const runtimes_data = cp.execSync(`nix eval --json ${config.flake_path}#pistonRuntimes --apply builtins.attrNames`).toString();
+    const runtimes_data = cp.execSync(`nix eval --json ${config.flake_path}#pistonRuntimeSets.${config.runtime_set} --apply builtins.attrNames`).toString();
     const runtimes = JSON.parse(runtimes_data);
     
     runtimes.for_each(pkg => runtime.load_runtime(pkg));
@@ -59,8 +59,8 @@ expressWs(app);
 
     logger.debug('Registering Routes');
 
-    const api_v2 = require('./api/v2');
+    const api_v2 = require('../api/v2');
-    const api_v3 = require('./api/v3');
+    const api_v3 = require('../api/v3');
     app.use('/api/v2', api_v2);
     app.use('/api/v3', api_v3);
 

api/src/bin/test.js

@@ -0,0 +1,112 @@
+#!/usr/bin/env node
+// Test the specified package
+require('nocamel');
+const config = require('../config');
+const Logger = require('logplease');
+const logger = Logger.create('test');
+const cp = require('child_process');
+const runtime = require("../runtime");
+const { Job } = require('../job');
+
+(async function(){
+    logger.info('Setting loglevel to', config.log_level);
+    Logger.setLogLevel(config.log_level);
+
+    
+
+    let runtimes_to_test;
+    let failed = false;
+
+    if(process.argv[2] === "--all"){
+        // load all
+        runtimes_to_test = JSON.parse(
+            cp.execSync(`nix eval ${config.flake_path}#pistonRuntimes --json --apply builtins.attrNames`)
+        );
+    }else{
+        runtimes_to_test = [process.argv[2]];
+    }
+    
+
+
+    for (const runtime_name of runtimes_to_test) {
+        
+        
+        const runtime_path = `${config.flake_path}#pistonRuntimes.${runtime_name}`;
+        logger.info(`Testing runtime ${runtime_path}`);
+
+        logger.debug(`Loading runtime metadata`);
+        const metadata = JSON.parse(cp.execSync(`nix eval --json ${runtime_path}.metadata --json`));
+
+        logger.debug(`Loading runtime tests`);
+        const tests = JSON.parse(cp.execSync(`nix eval --json ${runtime_path}.tests --json`));
+
+        logger.debug(`Loading runtime`);
+
+        const testable_runtime = new runtime.Runtime({
+            ...metadata,
+            flake_path: runtime_path
+        });
+
+        testable_runtime.ensure_built();
+
+
+        logger.info(`Running tests`);
+
+        for (const test of tests) {
+
+            const files = [];
+
+            for (const file_name of Object.keys(test.files)) {
+                const file_content = test.files[file_name];
+                const this_file = {
+                    name: file_name,
+                    content: file_content
+                };
+
+                if(file_name == test.main)
+                    files.unshift(this_file);
+                else
+                    files.push(this_file);
+
+            }
+
+
+            const job = new Job({
+                runtime: testable_runtime,
+                args: test.args || [],
+                stdin: test.stdin || "",
+                files,
+                timeouts: {
+                    run: 3000,
+                    compile: 10000
+                },
+                memory_limits: {
+                    run: config.run_memory_limit,
+                    compile: config.compile_memory_limit
+                }
+            });
+
+            await job.prime()
+            const result = await job.execute()
+            await job.cleanup()
+            
+            if(result.run.stdout.trim() !== "OK"){
+                failed = true;
+
+                logger.error("Test Failed:")
+                console.log(job, result)
+            }else{
+                logger.info("Test Passed")
+            }
+        }
+    }
+
+    if(failed) {
+        logger.error("One or more tests failed")
+        process.exit(1);
+    }
+    else {
+        logger.info("All tests passed")
+        process.exit(0);
+    }
+})()

api/src/config.js

@@ -176,6 +176,12 @@ const options = [
         default: 'github:engineer-man/piston?directory=packages',
         validators: [],
     },
+    {
+        key: 'runtime_set',
+        desc: 'Key on the flake specified by flake_path to access runtimes from',
+        default: 'all',
+        validators: []
+    },
     {
         key: 'max_concurrent_jobs',
         desc: 'Maximum number of concurrent jobs to run at one time',

api/src/runtime.js

@@ -17,7 +17,7 @@ class Runtime {
         run,
         compile,
         packageSupport,
-        flake_key,
+        flake_path,
         timeouts,
         memory_limits,
         max_process_count,
@@ -41,7 +41,7 @@ class Runtime {
         this.run = run;
         this.compile = compile;
 
-        this.flake_key = flake_key;
+        this.flake_path = flake_path;
         this.package_support = packageSupport;
     }
     
@@ -112,10 +112,10 @@ class Runtime {
     ensure_built(){
         logger.info(`Ensuring ${this} is built`);
 
-        const flake_key = this.flake_key;
+        const flake_path = this.flake_path;
 
         function _ensure_built(key){
-            const command = `nix build ${config.flake_path}#pistonRuntimes.${flake_key}.metadata.${key} --no-link`;
+            const command = `nix build ${flake_path}.metadata.${key} --no-link`;
             cp.execSync(command, {stdio: "pipe"})
         }
 
@@ -128,7 +128,8 @@ class Runtime {
 
     static load_runtime(flake_key){
         logger.info(`Loading ${flake_key}`)
-        const metadata_command = `nix eval --json ${config.flake_path}#pistonRuntimes.${flake_key}.metadata`;
+        const flake_path = `${config.flake_path}#pistonRuntimeSets.${config.runtime_set}.${flake_key}`;
+        const metadata_command = `nix eval --json ${flake_path}.metadata`;
         const metadata = JSON.parse(cp.execSync(metadata_command));
         
         const this_runtime = new Runtime({
@@ -137,7 +138,7 @@ class Runtime {
                 metadata.language,
                 metadata.limit_overrides
             ),
-            flake_key
+            flake_path
         });
 
         this_runtime.ensure_built();

flake.nix

@@ -6,6 +6,10 @@
   let
     system = "x86_64-linux";
     pkgs = nixpkgs.legacyPackages.${system};
+    baseContainer = (import ./api {
+      inherit pkgs;
+      nosocket = self.legacyPackages."${system}".nosocket;
+    }).container;
     args = {
       inherit pkgs;
       piston = {
@@ -53,66 +57,21 @@
         };
       };
     };
-    allRuntimes = import ./runtimes args;
+    runtimes = import ./runtimes args;
+    runtimeList = names: pkgs.lib.filterAttrs (n: v: n == "bash") runtimes;
   in {
     piston = args.piston;
-    pistonRuntimes = {
+    pistonRuntimes = runtimes;
-      "bash" = allRuntimes.bash;
+    pistonRuntimeSets = {
+      "all" = runtimes;
+      "bash-only" = runtimeList ["bash"];
     };
 
-    legacyPackages."${system}" = {
+    legacyPackages."${system}" = rec {
-      piston = (import ./api { inherit pkgs; }).package;
       nosocket = (import ./nosocket { inherit pkgs; }).package;
+      piston = (import ./api { inherit pkgs nosocket; }).package;
     };
 
-    containerImage = pkgs.dockerTools.buildLayeredImageWithNixDb {
+    container = baseContainer;    
-      name = "piston";
-      tag = "latest";
-
-      contents = with pkgs; [
-        self.legacyPackages."${system}".piston
-        self.legacyPackages."${system}".nosocket
-        bash
-        nixFlakes
-        coreutils-full
-        cacert.out
-        git
-        gnutar
-        gzip
-        gnugrep
-        util-linux
-      ];
-
-      extraCommands = ''
-        mkdir -p piston/jobs etc/nix {,var/}tmp run/lock
-        echo -e "experimental-features = nix-command flakes" >> etc/nix/nix.conf
-        echo "nixbld:x:30000:nixbld1,nixbld10,nixbld11,nixbld12,nixbld13,nixbld14,nixbld15,nixbld16,nixbld17,nixbld18,nixbld19,nixbld2,nixbld20,nixbld21,nixbld22,nixbld23,nixbld24,nixbld25,nixbld26,nixbld27,nixbld28,nixbld29,nixbld3,nixbld30,nixbld31,nixbld32,nixbld4,nixbld5,nixbld6,nixbld7,nixbld8,nixbld9" >> etc/group
-        for i in $(seq 1 32)
-        do
-          echo "nixbld$i:x:$(( $i + 30000 )):30000:Nix build user $i:/var/empty:/run/current-system/sw/bin/nologin" >> etc/passwd
-        done
-      '';
-
-      config = {
-        Cmd = [
-          "${self.legacyPackages."${system}".piston}/bin/pistond"
-        ];
-
-        Env = [
-          "NIX_PAGER=cat"
-          "USER=nobody"
-          "SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt"
-          "GIT_SSL_CAINFO=/etc/ssl/certs/ca-bundle.crt"
-          "NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt"
-
-        ];
-
-        ExposedPorts = {
-          "2000/tcp" = {};
-        };
-      };
-    };
-
-    
   };
 }

piston

@@ -1,110 +1,93 @@
 #!/usr/bin/env bash
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
-EXECUTION_PATH="$PWD"
-PISTON_PATH="$(dirname "$(realpath "$0")")"
 
-cd "$PISTON_PATH"
+CONTAINER_NAME="piston_api"
-PISTON_ENV=$(cat .piston_env 2> /dev/null || echo dev)
 
-docker_compose(){
+IMAGE_TAG="base-latest"
-    if [ -f "docker-compose.$PISTON_ENV.yaml" ]; then
+IMAGE_NAME="ghcr.io/piston"
-        docker-compose -f "docker-compose.$PISTON_ENV.yaml" "$@"
+IMAGE_NAME_DEV="piston"
-    else
+
-        docker-compose "$@"
+SUBCOMMAND="$1"
-    fi
+shift
+
+cmd_build(){
+    CONTAINER_PATH="$(nix build ".#container" --no-link --json | jq '.[0].outputs.out' -r)"
+    docker load -i $CONTAINER_PATH
 }
 
-init_precommit() {
+case "$SUBCOMMAND" in
-    if [ $PISTON_ENV == "dev" ]; then
+    logs) docker logs -f $CONTAINER_NAME;;
-        rm -f .git/hooks/pre-commit
-        ln -s "$PISTON_PATH/pre-commit" "$PISTON_PATH/.git/hooks/pre-commit"
-    fi
-}
 
-case $1 in
+    restart) docker restart $CONTAINER_NAME ;;
-    help)
+    start) 
+        docker run \
+            --rm \
+            --name $CONTAINER_NAME \
+            -it "$IMAGE_NAME:$IMAGE_TAG"
+    ;;
+    stop) docker stop $CONTAINER_NAME ;;
+    bash|shell) docker exec -it $CONTAINER_NAME bash ;;
+
+    update)
+        git pull
+        docker pull "$IMAGE_NAME:$IMAGE_TAG"
+        ;;
+
+    # dev commands
+    
+    build) cmd_build ;;
+
+    start-dev) 
+        cmd_build
+        docker run \
+            --rm \
+            -it \
+            --name $CONTAINER_NAME \
+            -e PISTON_FLAKE_PATH=/piston/packages \
+            -v $PWD:/piston/packages \
+            -it "$IMAGE_NAME_DEV:$IMAGE_TAG"
+    ;;
+
+    test)
+        cmd_build
+        docker run \
+            --rm \
+            -it \
+            -e PISTON_FLAKE_PATH=/piston/packages \
+            -v $PWD:/piston/packages \
+            --name piston_test_runner \
+            -it "$IMAGE_NAME_DEV:$IMAGE_TAG" \
+            piston-test $1
+
+        ;;
+
+    *)
         echo "=== Piston Management ==="
-        echo "Current Environment: $PISTON_ENV"
         echo
         echo "Commands:"
-        echo " select <environment>           Select the environment"
+        echo " logs                           Show piston logs"
-        echo " docker_compose <args...>       Interact directly with the docker-compose for the selected environment"
-        echo " logs                           Show docker-compose logs"
         echo
         echo " start                          Starts piston"
         echo " stop                           Stops piston"
         echo " restart                        Restarts piston"
-        echo " bash                           Opens a bash shell for the piston_api container"
+        echo " shell                          Opens a bash shell for the api container"
         echo
-        echo " update                         Fetches and applies latest updates"
+        echo " update                         Fetches latest updates"
         echo
-        echo " <args..>                       Passthrough to piston cli tool"
+        echo " exec <language> <file>         Execute the files on piston with language"
         echo
         echo "Development Commands:"
-
+        echo "Running some of these commands require a nix environment setup and on the path"
-        if [ "$PISTON_ENV" == dev ]; then
+        echo "See https://nixos.wiki/wiki/Nix_Installation_Guide#Stable_Nix"
-
+        echo
-        echo " clean-pkgs                              Clean any package build artifacts on disk"
+        echo " start-dev                      Builds a container locally and starts piston"
-        echo " clean-repo                              Remove all packages from local repo"
+        echo " build                          Builds and loads the API container"
-        echo " list-pkgs                               Lists all packages that can be built"
+        echo " scaffold <language> [runtime]  Initializes a new runtime"
-        echo " build-pkg <package> <version> [builder] Build a package [with desired builder image]"
+        echo " test <runtime>                 Runs unit tests on the given runtime"
-        echo " rebuild                                 Build and restart the docker container"
+        echo "                                Optionally set runtime to --all to test all"
-        echo " lint                                    Lint the codebase using prettier"
+        echo "                                NOTE: This is only for the runtimes contained"
-
+        echo "                                      within this repo"
-        else
+        echo
-
-        echo " Switch to developement environment for more info"
-        echo " > piston select dev"
-
-        fi
-        ;;
-
-
-    select) echo "$2" > .piston_env ;;
-    docker_compose) shift; docker_compose "$@";;
-    logs) docker_compose logs -f ;;
-
-    restart) docker_compose restart ;;
-    start)
-        init_precommit
-        docker_compose up -d
-        ;;
-    stop) docker_compose down ;;
-    bash) docker_compose exec api /bin/bash ;;
-
-    rebuild)
-        init_precommit
-        docker_compose build && docker_compose up -d
-        ;;
-
-    update)
-        git pull
-        cd cli && npm i > /dev/null && cd -
-        docker_compose pull
-        docker_compose up -d
-        ;;
-
-    clean-pkgs) git clean -fqXd packages ;;
-    clean-repo) git clean -fqXd repo ;;
-
-    list-pkgs) find packages -depth 2 | awk -F/ '$2 && $3{ print $2 "-" $3 }' | column ;;
-
-    build-pkg)
-        PKGSLUG="$2-$3"
-        BUILDER="${4:-piston-repo-builder}"
-        echo "Building $PKGSLUG"
-        echo "Ensuring latest builder image"
-        docker build repo -t "$BUILDER"
-        docker run --rm -v "$PWD:/piston" "$BUILDER" --no-server "$PKGSLUG"
-        ;;
-
-    lint)
-        npm install
-        npx prettier --ignore-unknown --write .
-        ;;
-    *)
-        [ -d ./cli/node_modules ] || npm i > /dev/null
-        cd "$EXECUTION_PATH"
-        node "${PISTON_PATH}/cli/index.js" "$@"
         ;;
 esac

result

@@ -1 +0,0 @@
-/nix/store/1idnmsddirm1hpxw40r17wsg9p9dscb9-piston.tar.gz