Add nix runtime testing and pre-installing runtimes

This commit is contained in:
Thomas Hobson 2022-01-30 22:29:21 +13:00
parent 5bc793cd70
commit 0434877d03
No known key found for this signature in database
GPG Key ID: 9F1FD9D87950DB6F
11 changed files with 308 additions and 152 deletions

View File

@ -1,5 +1,5 @@
{pkgs, ...}: {pkgs, nosocket, ...}:
with pkgs; { with pkgs; rec {
package = mkYarnPackage { package = mkYarnPackage {
name = "piston"; name = "piston";
src = ./.; src = ./.;
@ -25,4 +25,47 @@ with pkgs; {
}; };
}; };
}; };
container = pkgs.dockerTools.buildLayeredImageWithNixDb {
name = "piston";
tag = "base-latest";
contents = with pkgs; [
package
nosocket
bash
nixFlakes
coreutils-full
cacert.out
git
gnutar
gzip
gnugrep
util-linux
];
extraCommands = ''
mkdir -p piston/{jobs,runtimes} etc/nix {,var/}tmp run/lock
echo -e "experimental-features = nix-command flakes" >> etc/nix/nix.conf
echo "nixbld:x:30000:nixbld1,nixbld10,nixbld11,nixbld12,nixbld13,nixbld14,nixbld15,nixbld16,nixbld17,nixbld18,nixbld19,nixbld2,nixbld20,nixbld21,nixbld22,nixbld23,nixbld24,nixbld25,nixbld26,nixbld27,nixbld28,nixbld29,nixbld3,nixbld30,nixbld31,nixbld32,nixbld4,nixbld5,nixbld6,nixbld7,nixbld8,nixbld9" >> etc/group
for i in $(seq 1 32)
do
echo "nixbld$i:x:$(( $i + 30000 )):30000:Nix build user $i:/var/empty:/run/current-system/sw/bin/nologin" >> etc/passwd
done
'';
config = {
Cmd = ["${package}/bin/pistond"];
Env = [
"NIX_PAGER=cat"
"USER=nobody"
"SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt"
"GIT_SSL_CAINFO=/etc/ssl/certs/ca-bundle.crt"
"NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt"
];
ExposedPorts = {
"2000/tcp" = {};
};
};
};
} }

View File

@ -1,14 +0,0 @@
#!/usr/bin/env bash
echo "Starting Piston API"
echo "Checking presense of nix store"
if [[ ! -f "/nix/piston_detected" ]]; then
echo "Nix Store is not loaded, assuming /nix has been mounted - copying contents"
cp -rp /var/nix/* /nix
fi
echo "Adding nix to env"
. ~/.profile
echo "Launching Piston API"
node src

View File

@ -2,7 +2,7 @@
"name": "piston-api", "name": "piston-api",
"version": "4.0.0", "version": "4.0.0",
"description": "API for piston - a high performance code execution engine", "description": "API for piston - a high performance code execution engine",
"main": "src/pistond.js", "main": "src/bin/pistond.js",
"dependencies": { "dependencies": {
"body-parser": "^1.19.0", "body-parser": "^1.19.0",
"express": "^4.17.1", "express": "^4.17.1",
@ -23,6 +23,8 @@
"prettier": "2.2.1" "prettier": "2.2.1"
}, },
"bin": { "bin": {
"pistond": "./src/pistond.js" "pistond": "./src/bin/pistond.js",
"piston-install": "./src/bin/install.js",
"piston-test": "./src/bin/test.js"
} }
} }

39
api/src/bin/install.js Executable file
View File

@ -0,0 +1,39 @@
#!/usr/bin/env node
// Downloads all packages specified into the nix store
require('nocamel');
const config = require('../config');
const Logger = require('logplease');
const logger = Logger.create('install');
const cp = require('child_process');
logger.info('Setting loglevel to', config.log_level);
Logger.setLogLevel(config.log_level);
const runtimes_path = `${config.flake_path}#pistonRuntimeSets.${config.runtime_set}`;
logger.info(`Installing runtimes in ${runtimes_path}`);
logger.debug("Getting package listing");
const runtimes = JSON.parse(cp.execSync(`nix eval --json ${runtimes_path} --apply builtins.attrNames`));
logger.info(`Got runtimes: ${runtimes}`);
runtimes.forEach(runtime => {
logger.debug(`Loading metadata for ${runtime}`);
const runtime_metadata = JSON.parse(cp.execSync(`nix eval --json ${runtimes_path}.${runtime}.metadata`));
const namever = `${runtime}-${runtime_metadata.version}`;
logger.info(`Installing ${namever}`);
function install(key){
logger.debug(`Installing ${namever}-${key}`);
const command = `nix build ${runtimes_path}.${runtime}.metadata.${key} -o /piston/runtimes/${namever}-${key}`;
cp.execSync(command);
}
install("run");
if(runtime_metadata.compile) install("compile");
logger.info(`Installed ${namever}`);
});
logger.info("Done");

View File

@ -3,16 +3,16 @@ require('nocamel');
const Logger = require('logplease'); const Logger = require('logplease');
const express = require('express'); const express = require('express');
const expressWs = require('express-ws'); const expressWs = require('express-ws');
const globals = require('./globals'); const globals = require('../globals');
const config = require('./config'); const config = require('../config');
const cp = require('child_process'); const cp = require('child_process');
const path = require('path'); const path = require('path');
const fs = require('fs/promises'); const fs = require('fs/promises');
const fss = require('fs'); const fss = require('fs');
const body_parser = require('body-parser'); const body_parser = require('body-parser');
const runtime = require('./runtime'); const runtime = require('../runtime');
const logger = Logger.create('index'); const logger = Logger.create('pistond');
const app = express(); const app = express();
expressWs(app); expressWs(app);
@ -41,7 +41,7 @@ expressWs(app);
logger.info('Loading packages'); logger.info('Loading packages');
const runtimes_data = cp.execSync(`nix eval --json ${config.flake_path}#pistonRuntimes --apply builtins.attrNames`).toString(); const runtimes_data = cp.execSync(`nix eval --json ${config.flake_path}#pistonRuntimeSets.${config.runtime_set} --apply builtins.attrNames`).toString();
const runtimes = JSON.parse(runtimes_data); const runtimes = JSON.parse(runtimes_data);
runtimes.for_each(pkg => runtime.load_runtime(pkg)); runtimes.for_each(pkg => runtime.load_runtime(pkg));
@ -61,8 +61,8 @@ expressWs(app);
logger.debug('Registering Routes'); logger.debug('Registering Routes');
const api_v2 = require('./api/v2'); const api_v2 = require('../api/v2');
const api_v3 = require('./api/v3'); const api_v3 = require('../api/v3');
app.use('/api/v2', api_v2); app.use('/api/v2', api_v2);
app.use('/api/v3', api_v3); app.use('/api/v3', api_v3);

112
api/src/bin/test.js Executable file
View File

@ -0,0 +1,112 @@
#!/usr/bin/env node
// Test the specified package
require('nocamel');
const config = require('../config');
const Logger = require('logplease');
const logger = Logger.create('test');
const cp = require('child_process');
const runtime = require("../runtime");
const { Job } = require('../job');
(async function(){
logger.info('Setting loglevel to', config.log_level);
Logger.setLogLevel(config.log_level);
let runtimes_to_test;
let failed = false;
if(process.argv[2] === "--all"){
// load all
runtimes_to_test = JSON.parse(
cp.execSync(`nix eval ${config.flake_path}#pistonRuntimes --json --apply builtins.attrNames`)
);
}else{
runtimes_to_test = [process.argv[2]];
}
for (const runtime_name of runtimes_to_test) {
const runtime_path = `${config.flake_path}#pistonRuntimes.${runtime_name}`;
logger.info(`Testing runtime ${runtime_path}`);
logger.debug(`Loading runtime metadata`);
const metadata = JSON.parse(cp.execSync(`nix eval --json ${runtime_path}.metadata --json`));
logger.debug(`Loading runtime tests`);
const tests = JSON.parse(cp.execSync(`nix eval --json ${runtime_path}.tests --json`));
logger.debug(`Loading runtime`);
const testable_runtime = new runtime.Runtime({
...metadata,
flake_path: runtime_path
});
testable_runtime.ensure_built();
logger.info(`Running tests`);
for (const test of tests) {
const files = [];
for (const file_name of Object.keys(test.files)) {
const file_content = test.files[file_name];
const this_file = {
name: file_name,
content: file_content
};
if(file_name == test.main)
files.unshift(this_file);
else
files.push(this_file);
}
const job = new Job({
runtime: testable_runtime,
args: test.args || [],
stdin: test.stdin || "",
files,
timeouts: {
run: 3000,
compile: 10000
},
memory_limits: {
run: config.run_memory_limit,
compile: config.compile_memory_limit
}
});
await job.prime()
const result = await job.execute()
await job.cleanup()
if(result.run.stdout.trim() !== "OK"){
failed = true;
logger.error("Test Failed:")
console.log(job, result)
}else{
logger.info("Test Passed")
}
}
}
if(failed) {
logger.error("One or more tests failed")
process.exit(1);
}
else {
logger.info("All tests passed")
process.exit(0);
}
})()

View File

@ -113,6 +113,12 @@ const options = [
default: 'github:engineer-man/piston?directory=packages', default: 'github:engineer-man/piston?directory=packages',
validators: [], validators: [],
}, },
{
key: 'runtime_set',
desc: 'Key on the flake specified by flake_path to access runtimes from',
default: 'all',
validators: []
},
{ {
key: 'max_concurrent_jobs', key: 'max_concurrent_jobs',
desc: 'Maximum number of concurrent jobs to run at one time', desc: 'Maximum number of concurrent jobs to run at one time',

View File

@ -9,7 +9,7 @@ const runtimes = [];
class Runtime { class Runtime {
constructor({ language, version, aliases, runtime, run, compile, packageSupport, flake_key }) { constructor({ language, version, aliases, runtime, run, compile, packageSupport, flake_path }) {
this.language = language; this.language = language;
this.runtime = runtime; this.runtime = runtime;
this.aliases = aliases; this.aliases = aliases;
@ -18,17 +18,17 @@ class Runtime {
this.run = run; this.run = run;
this.compile = compile; this.compile = compile;
this.flake_key = flake_key; this.flake_path = flake_path;
this.package_support = packageSupport; this.package_support = packageSupport;
} }
ensure_built(){ ensure_built(){
logger.info(`Ensuring ${this} is built`); logger.info(`Ensuring ${this} is built`);
const flake_key = this.flake_key; const flake_path = this.flake_path;
function _ensure_built(key){ function _ensure_built(key){
const command = `nix build ${config.flake_path}#pistonRuntimes.${flake_key}.metadata.${key} --no-link`; const command = `nix build ${flake_path}.metadata.${key} --no-link`;
cp.execSync(command, {stdio: "pipe"}) cp.execSync(command, {stdio: "pipe"})
} }
@ -41,12 +41,13 @@ class Runtime {
static load_runtime(flake_key){ static load_runtime(flake_key){
logger.info(`Loading ${flake_key}`) logger.info(`Loading ${flake_key}`)
const metadata_command = `nix eval --json ${config.flake_path}#pistonRuntimes.${flake_key}.metadata`; const flake_path = `${config.flake_path}#pistonRuntimeSets.${config.runtime_set}.${flake_key}`;
const metadata_command = `nix eval --json ${flake_path}.metadata`;
const metadata = JSON.parse(cp.execSync(metadata_command)); const metadata = JSON.parse(cp.execSync(metadata_command));
const this_runtime = new Runtime({ const this_runtime = new Runtime({
...metadata, ...metadata,
flake_key flake_path
}); });
this_runtime.ensure_built(); this_runtime.ensure_built();

View File

@ -6,6 +6,10 @@
let let
system = "x86_64-linux"; system = "x86_64-linux";
pkgs = nixpkgs.legacyPackages.${system}; pkgs = nixpkgs.legacyPackages.${system};
baseContainer = (import ./api {
inherit pkgs;
nosocket = self.legacyPackages."${system}".nosocket;
}).container;
args = { args = {
inherit pkgs; inherit pkgs;
piston = { piston = {
@ -53,66 +57,21 @@
}; };
}; };
}; };
allRuntimes = import ./runtimes args; runtimes = import ./runtimes args;
runtimeList = names: pkgs.lib.filterAttrs (n: v: n == "bash") runtimes;
in { in {
piston = args.piston; piston = args.piston;
pistonRuntimes = { pistonRuntimes = runtimes;
"bash" = allRuntimes.bash; pistonRuntimeSets = {
"all" = runtimes;
"bash-only" = runtimeList ["bash"];
}; };
legacyPackages."${system}" = { legacyPackages."${system}" = rec {
piston = (import ./api { inherit pkgs; }).package;
nosocket = (import ./nosocket { inherit pkgs; }).package; nosocket = (import ./nosocket { inherit pkgs; }).package;
piston = (import ./api { inherit pkgs nosocket; }).package;
}; };
containerImage = pkgs.dockerTools.buildLayeredImageWithNixDb { container = baseContainer;
name = "piston";
tag = "latest";
contents = with pkgs; [
self.legacyPackages."${system}".piston
self.legacyPackages."${system}".nosocket
bash
nixFlakes
coreutils-full
cacert.out
git
gnutar
gzip
gnugrep
util-linux
];
extraCommands = ''
mkdir -p piston/jobs etc/nix {,var/}tmp run/lock
echo -e "experimental-features = nix-command flakes" >> etc/nix/nix.conf
echo "nixbld:x:30000:nixbld1,nixbld10,nixbld11,nixbld12,nixbld13,nixbld14,nixbld15,nixbld16,nixbld17,nixbld18,nixbld19,nixbld2,nixbld20,nixbld21,nixbld22,nixbld23,nixbld24,nixbld25,nixbld26,nixbld27,nixbld28,nixbld29,nixbld3,nixbld30,nixbld31,nixbld32,nixbld4,nixbld5,nixbld6,nixbld7,nixbld8,nixbld9" >> etc/group
for i in $(seq 1 32)
do
echo "nixbld$i:x:$(( $i + 30000 )):30000:Nix build user $i:/var/empty:/run/current-system/sw/bin/nologin" >> etc/passwd
done
'';
config = {
Cmd = [
"${self.legacyPackages."${system}".piston}/bin/pistond"
];
Env = [
"NIX_PAGER=cat"
"USER=nobody"
"SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt"
"GIT_SSL_CAINFO=/etc/ssl/certs/ca-bundle.crt"
"NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt"
];
ExposedPorts = {
"2000/tcp" = {};
};
};
};
}; };
} }

141
piston
View File

@ -1,84 +1,93 @@
#!/usr/bin/env bash #!/usr/bin/env bash
cd "$(dirname "$0")" SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
PISTON_ENV=$(cat .piston_env || echo dev)
docker_compose(){ CONTAINER_NAME="piston_api"
if [ -f "docker-compose.$PISTON_ENV.yaml" ]; then
docker-compose -f "docker-compose.$PISTON_ENV.yaml" "$@" IMAGE_TAG="base-latest"
else IMAGE_NAME="ghcr.io/piston"
docker-compose "$@" IMAGE_NAME_DEV="piston"
fi
SUBCOMMAND="$1"
shift
cmd_build(){
CONTAINER_PATH="$(nix build ".#container" --no-link --json | jq '.[0].outputs.out' -r)"
docker load -i $CONTAINER_PATH
} }
case $1 in case "$SUBCOMMAND" in
help) logs) docker logs -f $CONTAINER_NAME;;
restart) docker restart $CONTAINER_NAME ;;
start)
docker run \
--rm \
--name $CONTAINER_NAME \
-it "$IMAGE_NAME:$IMAGE_TAG"
;;
stop) docker stop $CONTAINER_NAME ;;
bash|shell) docker exec -it $CONTAINER_NAME bash ;;
update)
git pull
docker pull "$IMAGE_NAME:$IMAGE_TAG"
;;
# dev commands
build) cmd_build ;;
start-dev)
cmd_build
docker run \
--rm \
-it \
--name $CONTAINER_NAME \
-e PISTON_FLAKE_PATH=/piston/packages \
-v $PWD:/piston/packages \
-it "$IMAGE_NAME_DEV:$IMAGE_TAG"
;;
test)
cmd_build
docker run \
--rm \
-it \
-e PISTON_FLAKE_PATH=/piston/packages \
-v $PWD:/piston/packages \
--name piston_test_runner \
-it "$IMAGE_NAME_DEV:$IMAGE_TAG" \
piston-test $1
;;
*)
echo "=== Piston Management ===" echo "=== Piston Management ==="
echo "Current Environment: $PISTON_ENV"
echo echo
echo "Commands:" echo "Commands:"
echo " select <environment> Select the environment" echo " logs Show piston logs"
echo " docker_compose <args...> Interact directly with the docker-compose for the selected environment"
echo " logs Show docker-compose logs"
echo echo
echo " start Starts piston" echo " start Starts piston"
echo " stop Stops piston" echo " stop Stops piston"
echo " restart Restarts piston" echo " restart Restarts piston"
echo " bash Opens a bash shell for the piston_api container" echo " shell Opens a bash shell for the api container"
echo echo
echo " update Fetches and applies latest updates" echo " update Fetches latest updates"
echo echo
echo " <args..> Passthrough to piston cli tool" echo " exec <language> <file> Execute the files on piston with language"
echo echo
echo "Development Commands:" echo "Development Commands:"
echo "Running some of these commands require a nix environment setup and on the path"
if [ $PISTON_ENV == dev ]; then echo "See https://nixos.wiki/wiki/Nix_Installation_Guide#Stable_Nix"
echo
echo " clean-pkgs Clean any package build artifacts on disk" echo " start-dev Builds a container locally and starts piston"
echo " clean-repo Remove all packages from local repo" echo " build Builds and loads the API container"
echo " build-pkg <package> <version> Build a package" echo " scaffold <language> [runtime] Initializes a new runtime"
echo " rebuild Build and restart the docker container" echo " test <runtime> Runs unit tests on the given runtime"
echo " Optionally set runtime to --all to test all"
else echo " NOTE: This is only for the runtimes contained"
echo " within this repo"
echo " Switch to developement environment for more info" echo
echo " > piston select dev"
fi
;;
select) echo "$2" > .piston_env ;;
docker_compose) shift; docker_compose "$@";;
logs) docker_compose logs -f ;;
restart) docker_compose restart ;;
start) docker_compose up -d ;;
stop) docker_compose down ;;
bash) docker_compose exec api /bin/bash ;;
rebuild) docker_compose build && docker_compose up -d ;;
update)
git pull
docker_compose pull
docker_compose up -d
;;
clean-pkgs) git clean -fqXd packages ;;
clean-repo) git clean -fqXd repo ;;
build-pkg)
PKGSLUG="$2-$3"
echo "Building $PKGSLUG"
echo "Ensuring latest builder image"
docker build repo -t piston-repo-builder
docker run -v "$(realpath $(dirname "$0")):/piston" piston-repo-builder --no-server $PKGSLUG
;;
*)
cd cli
npm i > /dev/null
cd ../
node cli/index.js "$@"
;; ;;
esac esac

1
result
View File

@ -1 +0,0 @@
/nix/store/1idnmsddirm1hpxw40r17wsg9p9dscb9-piston.tar.gz