Fix issues after merging upstream

Implements a simple container builder for runtime sets

Dockerfile.withset

@@ -0,0 +1,12 @@
+# This "FROM" image is previously emitted by nix
+FROM ghcr.io/engineer-man/piston:base-latest
+
+ENV PISTON_FLAKE_PATH=/piston/packages
+COPY runtimes/ /piston/packages/runtimes
+COPY flake.nix flake.lock /piston/packages/
+
+
+ARG RUNTIMESET=all
+ENV PISTON_RUNTIME_SET=$RUNTIMESET
+
+RUN piston-install

api/default.nix

@@ -51,6 +51,8 @@ with pkgs; rec {
       do
         echo "nixbld$i:x:$(( $i + 30000 )):30000:Nix build user $i:/var/empty:/run/current-system/sw/bin/nologin" >> etc/passwd
       done
+
+      chmod 1777 {,var/}tmp/
     '';
 
     config = {
@@ -61,6 +63,21 @@ with pkgs; rec {
         "SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt"
         "GIT_SSL_CAINFO=/etc/ssl/certs/ca-bundle.crt"
         "NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt"
+         "PATH=${lib.concatStringsSep ":" [
+           "/usr/local/sbin"
+           "/usr/local/bin"
+           "/usr/sbin"
+           "/usr/bin"
+           "/sbin"
+           "/bin"
+           "/root/.nix-profile/bin"
+           "/nix/var/nix/profiles/default/bin"
+           "/nix/var/nix/profiles/default/sbin"
+         ]}"
+         "MANPATH=${lib.concatStringsSep ":" [
+           "/root/.nix-profile/share/man"
+           "/nix/var/nix/profiles/default/share/man"
+         ]}"
       ];
 
       ExposedPorts = {

api/src/bin/test.js

@@ -5,108 +5,105 @@ const config = require('../config');
 const Logger = require('logplease');
 const logger = Logger.create('test');
 const cp = require('child_process');
-const runtime = require("../runtime");
+const runtime = require('../runtime');
 const { Job } = require('../job');
 
 (async function () {
     logger.info('Setting loglevel to', config.log_level);
     Logger.setLogLevel(config.log_level);
 
-    
-
     let runtimes_to_test;
     let failed = false;
 
-    if(process.argv[2] === "--all"){
+    if (process.argv[2] === '--all') {
         // load all
         runtimes_to_test = JSON.parse(
-            cp.execSync(`nix eval ${config.flake_path}#pistonRuntimes --json --apply builtins.attrNames`)
+            cp.execSync(
+                `nix eval ${config.flake_path}#pistonRuntimes --json --apply builtins.attrNames`
+            )
         );
     } else {
         runtimes_to_test = [process.argv[2]];
     }
 
-
-
     for (const runtime_name of runtimes_to_test) {
-        
-        
         const runtime_path = `${config.flake_path}#pistonRuntimes.${runtime_name}`;
         logger.info(`Testing runtime ${runtime_path}`);
 
         logger.debug(`Loading runtime metadata`);
-        const metadata = JSON.parse(cp.execSync(`nix eval --json ${runtime_path}.metadata --json`));
+        const metadata = JSON.parse(
+            cp.execSync(`nix eval --json ${runtime_path}.metadata --json`)
+        );
 
         logger.debug(`Loading runtime tests`);
-        const tests = JSON.parse(cp.execSync(`nix eval --json ${runtime_path}.tests --json`));
+        const tests = JSON.parse(
+            cp.execSync(`nix eval --json ${runtime_path}.tests --json`)
+        );
 
         logger.debug(`Loading runtime`);
 
         const testable_runtime = new runtime.Runtime({
             ...metadata,
-            flake_path: runtime_path
+            ...runtime.Runtime.compute_all_limits(
+                metadata.language,
+                metadata.limit_overrides
+            ),
+            flake_path: runtime_path,
         });
 
         testable_runtime.ensure_built();
 
-
         logger.info(`Running tests`);
 
         for (const test of tests) {
-
             const files = [];
 
             for (const file_name of Object.keys(test.files)) {
                 const file_content = test.files[file_name];
                 const this_file = {
                     name: file_name,
-                    content: file_content
+                    content: file_content,
                 };
 
-                if(file_name == test.main)
+                if (file_name == test.main) files.unshift(this_file);
-                    files.unshift(this_file);
+                else files.push(this_file);
-                else
-                    files.push(this_file);
-
             }
 
-
             const job = new Job({
                 runtime: testable_runtime,
                 args: test.args || [],
-                stdin: test.stdin || "",
+                stdin: test.stdin || '',
                 files,
                 timeouts: {
                     run: 3000,
-                    compile: 10000
+                    compile: 10000,
                 },
                 memory_limits: {
                     run: config.run_memory_limit,
-                    compile: config.compile_memory_limit
+                    compile: config.compile_memory_limit,
-                }
+                },
             });
 
-            await job.prime()
+            await job.prime();
-            const result = await job.execute()
+            const result = await job.execute();
-            await job.cleanup()
+            await job.cleanup();
 
-            if(result.run.stdout.trim() !== "OK"){
+            if (result.run.stdout.trim() !== 'OK') {
                 failed = true;
 
-                logger.error("Test Failed:")
+                logger.error('Test Failed:');
-                console.log(job, result)
+                console.log(job, result);
             } else {
-                logger.info("Test Passed")
+                logger.info('Test Passed');
             }
         }
     }
 
     if (failed) {
-        logger.error("One or more tests failed")
+        logger.error('One or more tests failed');
         process.exit(1);
-    }
+    } else {
-    else {
+        logger.info('All tests passed');
-        logger.info("All tests passed")
         process.exit(0);
     }
-})()
+})();

api/src/job.js

@@ -139,10 +139,6 @@ class Job {
             var output = '';
 
             const proc = cp.spawn(proc_call[0], proc_call.splice(1), {
-                env: {
-                    ...this.runtime.env_vars,
-                    PISTON_LANGUAGE: this.runtime.language,
-                },
                 stdio: 'pipe',
                 cwd: this.dir,
                 uid: this.uid,
@@ -250,7 +246,7 @@ class Job {
         this.logger.debug('Running');
 
         const run = await this.safe_call(
-            path.join(this.runtime.pkgdir, 'run'),
+            this.runtime.run,
             [code_files[0].name, ...this.args],
             this.timeouts.run,
             this.memory_limits.run

package-lock.json

@@ -0,0 +1,32 @@
+{
+    "name": "piston",
+    "lockfileVersion": 2,
+    "requires": true,
+    "packages": {
+        "": {
+            "devDependencies": {
+                "prettier": "2.4.1"
+            }
+        },
+        "node_modules/prettier": {
+            "version": "2.4.1",
+            "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.4.1.tgz",
+            "integrity": "sha512-9fbDAXSBcc6Bs1mZrDYb3XKzDLm4EXXL9sC1LqKP5rZkT6KRr/rf9amVUcODVXgguK/isJz0d0hP72WeaKWsvA==",
+            "dev": true,
+            "bin": {
+                "prettier": "bin-prettier.js"
+            },
+            "engines": {
+                "node": ">=10.13.0"
+            }
+        }
+    },
+    "dependencies": {
+        "prettier": {
+            "version": "2.4.1",
+            "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.4.1.tgz",
+            "integrity": "sha512-9fbDAXSBcc6Bs1mZrDYb3XKzDLm4EXXL9sC1LqKP5rZkT6KRr/rf9amVUcODVXgguK/isJz0d0hP72WeaKWsvA==",
+            "dev": true
+        }
+    }
+}

package.json

@@ -0,0 +1,5 @@
+{
+    "devDependencies": {
+        "prettier": "2.4.1"
+    }
+}

piston

@@ -5,15 +5,16 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 CONTAINER_NAME="piston_api"
 
 IMAGE_TAG="base-latest"
-IMAGE_NAME="ghcr.io/piston"
+IMAGE_NAME="ghcr.io/engineer-man/piston"
 IMAGE_NAME_DEV="piston"
 
 SUBCOMMAND="$1"
 shift
 
-cmd_build(){
+build_base(){
     CONTAINER_PATH="$(nix build ".#container" --no-link --json | jq '.[0].outputs.out' -r)"
     docker load -i $CONTAINER_PATH
+    docker tag "$IMAGE_NAME_DEV:$IMAGE_TAG" "$IMAGE_NAME:$IMAGE_TAG"
 }
 
 case "$SUBCOMMAND" in
@@ -22,9 +23,10 @@ case "$SUBCOMMAND" in
     restart) docker restart $CONTAINER_NAME ;;
     start) 
         docker run \
+            -p 2000:2000 \
             --rm \
             --name $CONTAINER_NAME \
-            -it "$IMAGE_NAME:$IMAGE_TAG"
+            -d "$IMAGE_NAME:$IMAGE_TAG"
     ;;
     stop) docker stop $CONTAINER_NAME ;;
     bash|shell) docker exec -it $CONTAINER_NAME bash ;;
@@ -36,21 +38,39 @@ case "$SUBCOMMAND" in
 
     # dev commands
 
-    build) cmd_build ;;
+    scaffold)
+        pushd $SCRIPT_DIR/runtimes > /dev/null
+        ./scaffold.sh $1 $2
+        popd > /dev/null
+        ;;
+    
+    build)
+        build_base
+        if [[ ! -z "$1" ]]; then
+            # $1 contains a variant to build
+            docker build \
+                --build-arg RUNTIMESET=$1 \
+                -f $SCRIPT_DIR/Dockerfile.withset \
+                -t "$IMAGE_NAME_DEV:$1-latest" \
+                .
+        fi
+        ;;
 
     start-dev) 
-        cmd_build
+        build_base
         docker run \
             --rm \
+            -p 2000:2000 \
             -it \
             --name $CONTAINER_NAME \
+            -e PISTON_LOG_LEVEL=DEBUG \
             -e PISTON_FLAKE_PATH=/piston/packages \
             -v $PWD:/piston/packages \
-            -it "$IMAGE_NAME_DEV:$IMAGE_TAG"
+            -d "$IMAGE_NAME_DEV:$IMAGE_TAG"
     ;;
 
     test)
-        cmd_build
+        build_base
         docker run \
             --rm \
             -it \
@@ -82,7 +102,8 @@ case "$SUBCOMMAND" in
         echo "See https://nixos.wiki/wiki/Nix_Installation_Guide#Stable_Nix"
         echo
         echo " start-dev                      Builds a container locally and starts piston"
-        echo " build                          Builds and loads the API container"
+        echo " build [runtime-set]            Builds and loads the API container optionally"
+        echo "                                including the runtime set within it"
         echo " scaffold <language> [runtime]  Initializes a new runtime"
         echo " test <runtime>                 Runs unit tests on the given runtime"
         echo "                                Optionally set runtime to --all to test all"