Add nix runtime testing and pre-installing runtimes

This commit is contained in:
Thomas Hobson 2022-01-30 22:29:21 +13:00
parent 564da5a7eb
commit e022e34a37
No known key found for this signature in database
GPG Key ID: 9F1FD9D87950DB6F
11 changed files with 306 additions and 176 deletions

View File

@ -1,5 +1,5 @@
{pkgs, ...}:
with pkgs; {
{pkgs, nosocket, ...}:
with pkgs; rec {
package = mkYarnPackage {
name = "piston";
src = ./.;
@ -25,4 +25,47 @@ with pkgs; {
};
};
};
container = pkgs.dockerTools.buildLayeredImageWithNixDb {
name = "piston";
tag = "base-latest";
contents = with pkgs; [
package
nosocket
bash
nixFlakes
coreutils-full
cacert.out
git
gnutar
gzip
gnugrep
util-linux
];
extraCommands = ''
mkdir -p piston/{jobs,runtimes} etc/nix {,var/}tmp run/lock
echo -e "experimental-features = nix-command flakes" >> etc/nix/nix.conf
echo "nixbld:x:30000:nixbld1,nixbld10,nixbld11,nixbld12,nixbld13,nixbld14,nixbld15,nixbld16,nixbld17,nixbld18,nixbld19,nixbld2,nixbld20,nixbld21,nixbld22,nixbld23,nixbld24,nixbld25,nixbld26,nixbld27,nixbld28,nixbld29,nixbld3,nixbld30,nixbld31,nixbld32,nixbld4,nixbld5,nixbld6,nixbld7,nixbld8,nixbld9" >> etc/group
for i in $(seq 1 32)
do
echo "nixbld$i:x:$(( $i + 30000 )):30000:Nix build user $i:/var/empty:/run/current-system/sw/bin/nologin" >> etc/passwd
done
'';
config = {
Cmd = ["${package}/bin/pistond"];
Env = [
"NIX_PAGER=cat"
"USER=nobody"
"SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt"
"GIT_SSL_CAINFO=/etc/ssl/certs/ca-bundle.crt"
"NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt"
];
ExposedPorts = {
"2000/tcp" = {};
};
};
};
}

View File

@ -1,14 +0,0 @@
#!/usr/bin/env bash
echo "Starting Piston API"
echo "Checking presense of nix store"
if [[ ! -f "/nix/piston_detected" ]]; then
echo "Nix Store is not loaded, assuming /nix has been mounted - copying contents"
cp -rp /var/nix/* /nix
fi
echo "Adding nix to env"
. ~/.profile
echo "Launching Piston API"
node src

View File

@ -2,7 +2,7 @@
"name": "piston-api",
"version": "4.0.0",
"description": "API for piston - a high performance code execution engine",
"main": "src/pistond.js",
"main": "src/bin/pistond.js",
"dependencies": {
"body-parser": "^1.19.0",
"express": "^4.17.1",
@ -23,6 +23,8 @@
"prettier": "2.2.1"
},
"bin": {
"pistond": "./src/pistond.js"
"pistond": "./src/bin/pistond.js",
"piston-install": "./src/bin/install.js",
"piston-test": "./src/bin/test.js"
}
}

39
api/src/bin/install.js Executable file
View File

@ -0,0 +1,39 @@
#!/usr/bin/env node
// Downloads all packages specified into the nix store
require('nocamel');
const config = require('../config');
const Logger = require('logplease');
const logger = Logger.create('install');
const cp = require('child_process');
logger.info('Setting loglevel to', config.log_level);
Logger.setLogLevel(config.log_level);
const runtimes_path = `${config.flake_path}#pistonRuntimeSets.${config.runtime_set}`;
logger.info(`Installing runtimes in ${runtimes_path}`);
logger.debug("Getting package listing");
const runtimes = JSON.parse(cp.execSync(`nix eval --json ${runtimes_path} --apply builtins.attrNames`));
logger.info(`Got runtimes: ${runtimes}`);
runtimes.forEach(runtime => {
logger.debug(`Loading metadata for ${runtime}`);
const runtime_metadata = JSON.parse(cp.execSync(`nix eval --json ${runtimes_path}.${runtime}.metadata`));
const namever = `${runtime}-${runtime_metadata.version}`;
logger.info(`Installing ${namever}`);
function install(key){
logger.debug(`Installing ${namever}-${key}`);
const command = `nix build ${runtimes_path}.${runtime}.metadata.${key} -o /piston/runtimes/${namever}-${key}`;
cp.execSync(command);
}
install("run");
if(runtime_metadata.compile) install("compile");
logger.info(`Installed ${namever}`);
});
logger.info("Done");

View File

@ -3,16 +3,16 @@ require('nocamel');
const Logger = require('logplease');
const express = require('express');
const expressWs = require('express-ws');
const globals = require('./globals');
const config = require('./config');
const globals = require('../globals');
const config = require('../config');
const cp = require('child_process');
const path = require('path');
const fs = require('fs/promises');
const fss = require('fs');
const body_parser = require('body-parser');
const runtime = require('./runtime');
const runtime = require('../runtime');
const logger = Logger.create('index');
const logger = Logger.create('pistond');
const app = express();
expressWs(app);
@ -39,7 +39,7 @@ expressWs(app);
logger.info('Loading packages');
const runtimes_data = cp.execSync(`nix eval --json ${config.flake_path}#pistonRuntimes --apply builtins.attrNames`).toString();
const runtimes_data = cp.execSync(`nix eval --json ${config.flake_path}#pistonRuntimeSets.${config.runtime_set} --apply builtins.attrNames`).toString();
const runtimes = JSON.parse(runtimes_data);
runtimes.for_each(pkg => runtime.load_runtime(pkg));
@ -59,8 +59,8 @@ expressWs(app);
logger.debug('Registering Routes');
const api_v2 = require('./api/v2');
const api_v3 = require('./api/v3');
const api_v2 = require('../api/v2');
const api_v3 = require('../api/v3');
app.use('/api/v2', api_v2);
app.use('/api/v3', api_v3);

112
api/src/bin/test.js Executable file
View File

@ -0,0 +1,112 @@
#!/usr/bin/env node
// Test the specified package
require('nocamel');
const config = require('../config');
const Logger = require('logplease');
const logger = Logger.create('test');
const cp = require('child_process');
const runtime = require("../runtime");
const { Job } = require('../job');
(async function(){
logger.info('Setting loglevel to', config.log_level);
Logger.setLogLevel(config.log_level);
let runtimes_to_test;
let failed = false;
if(process.argv[2] === "--all"){
// load all
runtimes_to_test = JSON.parse(
cp.execSync(`nix eval ${config.flake_path}#pistonRuntimes --json --apply builtins.attrNames`)
);
}else{
runtimes_to_test = [process.argv[2]];
}
for (const runtime_name of runtimes_to_test) {
const runtime_path = `${config.flake_path}#pistonRuntimes.${runtime_name}`;
logger.info(`Testing runtime ${runtime_path}`);
logger.debug(`Loading runtime metadata`);
const metadata = JSON.parse(cp.execSync(`nix eval --json ${runtime_path}.metadata --json`));
logger.debug(`Loading runtime tests`);
const tests = JSON.parse(cp.execSync(`nix eval --json ${runtime_path}.tests --json`));
logger.debug(`Loading runtime`);
const testable_runtime = new runtime.Runtime({
...metadata,
flake_path: runtime_path
});
testable_runtime.ensure_built();
logger.info(`Running tests`);
for (const test of tests) {
const files = [];
for (const file_name of Object.keys(test.files)) {
const file_content = test.files[file_name];
const this_file = {
name: file_name,
content: file_content
};
if(file_name == test.main)
files.unshift(this_file);
else
files.push(this_file);
}
const job = new Job({
runtime: testable_runtime,
args: test.args || [],
stdin: test.stdin || "",
files,
timeouts: {
run: 3000,
compile: 10000
},
memory_limits: {
run: config.run_memory_limit,
compile: config.compile_memory_limit
}
});
await job.prime()
const result = await job.execute()
await job.cleanup()
if(result.run.stdout.trim() !== "OK"){
failed = true;
logger.error("Test Failed:")
console.log(job, result)
}else{
logger.info("Test Passed")
}
}
}
if(failed) {
logger.error("One or more tests failed")
process.exit(1);
}
else {
logger.info("All tests passed")
process.exit(0);
}
})()

View File

@ -176,6 +176,12 @@ const options = [
default: 'github:engineer-man/piston?directory=packages',
validators: [],
},
{
key: 'runtime_set',
desc: 'Key on the flake specified by flake_path to access runtimes from',
default: 'all',
validators: []
},
{
key: 'max_concurrent_jobs',
desc: 'Maximum number of concurrent jobs to run at one time',

View File

@ -17,7 +17,7 @@ class Runtime {
run,
compile,
packageSupport,
flake_key,
flake_path,
timeouts,
memory_limits,
max_process_count,
@ -41,7 +41,7 @@ class Runtime {
this.run = run;
this.compile = compile;
this.flake_key = flake_key;
this.flake_path = flake_path;
this.package_support = packageSupport;
}
@ -112,10 +112,10 @@ class Runtime {
ensure_built(){
logger.info(`Ensuring ${this} is built`);
const flake_key = this.flake_key;
const flake_path = this.flake_path;
function _ensure_built(key){
const command = `nix build ${config.flake_path}#pistonRuntimes.${flake_key}.metadata.${key} --no-link`;
const command = `nix build ${flake_path}.metadata.${key} --no-link`;
cp.execSync(command, {stdio: "pipe"})
}
@ -128,7 +128,8 @@ class Runtime {
static load_runtime(flake_key){
logger.info(`Loading ${flake_key}`)
const metadata_command = `nix eval --json ${config.flake_path}#pistonRuntimes.${flake_key}.metadata`;
const flake_path = `${config.flake_path}#pistonRuntimeSets.${config.runtime_set}.${flake_key}`;
const metadata_command = `nix eval --json ${flake_path}.metadata`;
const metadata = JSON.parse(cp.execSync(metadata_command));
const this_runtime = new Runtime({
@ -137,7 +138,7 @@ class Runtime {
metadata.language,
metadata.limit_overrides
),
flake_key
flake_path
});
this_runtime.ensure_built();

View File

@ -6,6 +6,10 @@
let
system = "x86_64-linux";
pkgs = nixpkgs.legacyPackages.${system};
baseContainer = (import ./api {
inherit pkgs;
nosocket = self.legacyPackages."${system}".nosocket;
}).container;
args = {
inherit pkgs;
piston = {
@ -53,66 +57,21 @@
};
};
};
allRuntimes = import ./runtimes args;
runtimes = import ./runtimes args;
runtimeList = names: pkgs.lib.filterAttrs (n: v: n == "bash") runtimes;
in {
piston = args.piston;
pistonRuntimes = {
"bash" = allRuntimes.bash;
pistonRuntimes = runtimes;
pistonRuntimeSets = {
"all" = runtimes;
"bash-only" = runtimeList ["bash"];
};
legacyPackages."${system}" = {
piston = (import ./api { inherit pkgs; }).package;
legacyPackages."${system}" = rec {
nosocket = (import ./nosocket { inherit pkgs; }).package;
piston = (import ./api { inherit pkgs nosocket; }).package;
};
containerImage = pkgs.dockerTools.buildLayeredImageWithNixDb {
name = "piston";
tag = "latest";
contents = with pkgs; [
self.legacyPackages."${system}".piston
self.legacyPackages."${system}".nosocket
bash
nixFlakes
coreutils-full
cacert.out
git
gnutar
gzip
gnugrep
util-linux
];
extraCommands = ''
mkdir -p piston/jobs etc/nix {,var/}tmp run/lock
echo -e "experimental-features = nix-command flakes" >> etc/nix/nix.conf
echo "nixbld:x:30000:nixbld1,nixbld10,nixbld11,nixbld12,nixbld13,nixbld14,nixbld15,nixbld16,nixbld17,nixbld18,nixbld19,nixbld2,nixbld20,nixbld21,nixbld22,nixbld23,nixbld24,nixbld25,nixbld26,nixbld27,nixbld28,nixbld29,nixbld3,nixbld30,nixbld31,nixbld32,nixbld4,nixbld5,nixbld6,nixbld7,nixbld8,nixbld9" >> etc/group
for i in $(seq 1 32)
do
echo "nixbld$i:x:$(( $i + 30000 )):30000:Nix build user $i:/var/empty:/run/current-system/sw/bin/nologin" >> etc/passwd
done
'';
config = {
Cmd = [
"${self.legacyPackages."${system}".piston}/bin/pistond"
];
Env = [
"NIX_PAGER=cat"
"USER=nobody"
"SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt"
"GIT_SSL_CAINFO=/etc/ssl/certs/ca-bundle.crt"
"NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt"
];
ExposedPorts = {
"2000/tcp" = {};
};
};
};
container = baseContainer;
};
}

163
piston
View File

@ -1,110 +1,93 @@
#!/usr/bin/env bash
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
EXECUTION_PATH="$PWD"
PISTON_PATH="$(dirname "$(realpath "$0")")"
cd "$PISTON_PATH"
PISTON_ENV=$(cat .piston_env 2> /dev/null || echo dev)
CONTAINER_NAME="piston_api"
docker_compose(){
if [ -f "docker-compose.$PISTON_ENV.yaml" ]; then
docker-compose -f "docker-compose.$PISTON_ENV.yaml" "$@"
else
docker-compose "$@"
fi
IMAGE_TAG="base-latest"
IMAGE_NAME="ghcr.io/piston"
IMAGE_NAME_DEV="piston"
SUBCOMMAND="$1"
shift
cmd_build(){
CONTAINER_PATH="$(nix build ".#container" --no-link --json | jq '.[0].outputs.out' -r)"
docker load -i $CONTAINER_PATH
}
init_precommit() {
if [ $PISTON_ENV == "dev" ]; then
rm -f .git/hooks/pre-commit
ln -s "$PISTON_PATH/pre-commit" "$PISTON_PATH/.git/hooks/pre-commit"
fi
}
case "$SUBCOMMAND" in
logs) docker logs -f $CONTAINER_NAME;;
case $1 in
help)
restart) docker restart $CONTAINER_NAME ;;
start)
docker run \
--rm \
--name $CONTAINER_NAME \
-it "$IMAGE_NAME:$IMAGE_TAG"
;;
stop) docker stop $CONTAINER_NAME ;;
bash|shell) docker exec -it $CONTAINER_NAME bash ;;
update)
git pull
docker pull "$IMAGE_NAME:$IMAGE_TAG"
;;
# dev commands
build) cmd_build ;;
start-dev)
cmd_build
docker run \
--rm \
-it \
--name $CONTAINER_NAME \
-e PISTON_FLAKE_PATH=/piston/packages \
-v $PWD:/piston/packages \
-it "$IMAGE_NAME_DEV:$IMAGE_TAG"
;;
test)
cmd_build
docker run \
--rm \
-it \
-e PISTON_FLAKE_PATH=/piston/packages \
-v $PWD:/piston/packages \
--name piston_test_runner \
-it "$IMAGE_NAME_DEV:$IMAGE_TAG" \
piston-test $1
;;
*)
echo "=== Piston Management ==="
echo "Current Environment: $PISTON_ENV"
echo
echo "Commands:"
echo " select <environment> Select the environment"
echo " docker_compose <args...> Interact directly with the docker-compose for the selected environment"
echo " logs Show docker-compose logs"
echo " logs Show piston logs"
echo
echo " start Starts piston"
echo " stop Stops piston"
echo " restart Restarts piston"
echo " bash Opens a bash shell for the piston_api container"
echo " shell Opens a bash shell for the api container"
echo
echo " update Fetches and applies latest updates"
echo " update Fetches latest updates"
echo
echo " <args..> Passthrough to piston cli tool"
echo " exec <language> <file> Execute the files on piston with language"
echo
echo "Development Commands:"
if [ "$PISTON_ENV" == dev ]; then
echo " clean-pkgs Clean any package build artifacts on disk"
echo " clean-repo Remove all packages from local repo"
echo " list-pkgs Lists all packages that can be built"
echo " build-pkg <package> <version> [builder] Build a package [with desired builder image]"
echo " rebuild Build and restart the docker container"
echo " lint Lint the codebase using prettier"
else
echo " Switch to developement environment for more info"
echo " > piston select dev"
fi
;;
select) echo "$2" > .piston_env ;;
docker_compose) shift; docker_compose "$@";;
logs) docker_compose logs -f ;;
restart) docker_compose restart ;;
start)
init_precommit
docker_compose up -d
;;
stop) docker_compose down ;;
bash) docker_compose exec api /bin/bash ;;
rebuild)
init_precommit
docker_compose build && docker_compose up -d
;;
update)
git pull
cd cli && npm i > /dev/null && cd -
docker_compose pull
docker_compose up -d
;;
clean-pkgs) git clean -fqXd packages ;;
clean-repo) git clean -fqXd repo ;;
list-pkgs) find packages -depth 2 | awk -F/ '$2 && $3{ print $2 "-" $3 }' | column ;;
build-pkg)
PKGSLUG="$2-$3"
BUILDER="${4:-piston-repo-builder}"
echo "Building $PKGSLUG"
echo "Ensuring latest builder image"
docker build repo -t "$BUILDER"
docker run --rm -v "$PWD:/piston" "$BUILDER" --no-server "$PKGSLUG"
;;
lint)
npm install
npx prettier --ignore-unknown --write .
;;
*)
[ -d ./cli/node_modules ] || npm i > /dev/null
cd "$EXECUTION_PATH"
node "${PISTON_PATH}/cli/index.js" "$@"
echo "Running some of these commands require a nix environment setup and on the path"
echo "See https://nixos.wiki/wiki/Nix_Installation_Guide#Stable_Nix"
echo
echo " start-dev Builds a container locally and starts piston"
echo " build Builds and loads the API container"
echo " scaffold <language> [runtime] Initializes a new runtime"
echo " test <runtime> Runs unit tests on the given runtime"
echo " Optionally set runtime to --all to test all"
echo " NOTE: This is only for the runtimes contained"
echo " within this repo"
echo
;;
esac

1
result
View File

@ -1 +0,0 @@
/nix/store/1idnmsddirm1hpxw40r17wsg9p9dscb9-piston.tar.gz