Add nix runtime testing and pre-installing runtimes
This commit is contained in:
Thomas Hobson
parent
5bc793cd70
commit
0434877d03
|
|
@ -1,5 +1,5 @@
|
|||
{pkgs, ...}:
|
||||
with pkgs; {
|
||||
{pkgs, nosocket, ...}:
|
||||
with pkgs; rec {
|
||||
package = mkYarnPackage {
|
||||
name = "piston";
|
||||
src = ./.;
|
||||
|
|
@ -25,4 +25,47 @@ with pkgs; {
|
|||
};
|
||||
};
|
||||
};
|
||||
container = pkgs.dockerTools.buildLayeredImageWithNixDb {
|
||||
name = "piston";
|
||||
tag = "base-latest";
|
||||
|
||||
contents = with pkgs; [
|
||||
package
|
||||
nosocket
|
||||
bash
|
||||
nixFlakes
|
||||
coreutils-full
|
||||
cacert.out
|
||||
git
|
||||
gnutar
|
||||
gzip
|
||||
gnugrep
|
||||
util-linux
|
||||
];
|
||||
|
||||
extraCommands = ''
|
||||
mkdir -p piston/{jobs,runtimes} etc/nix {,var/}tmp run/lock
|
||||
echo -e "experimental-features = nix-command flakes" >> etc/nix/nix.conf
|
||||
echo "nixbld:x:30000:nixbld1,nixbld10,nixbld11,nixbld12,nixbld13,nixbld14,nixbld15,nixbld16,nixbld17,nixbld18,nixbld19,nixbld2,nixbld20,nixbld21,nixbld22,nixbld23,nixbld24,nixbld25,nixbld26,nixbld27,nixbld28,nixbld29,nixbld3,nixbld30,nixbld31,nixbld32,nixbld4,nixbld5,nixbld6,nixbld7,nixbld8,nixbld9" >> etc/group
|
||||
for i in $(seq 1 32)
|
||||
do
|
||||
echo "nixbld$i:x:$(( $i + 30000 )):30000:Nix build user $i:/var/empty:/run/current-system/sw/bin/nologin" >> etc/passwd
|
||||
done
|
||||
'';
|
||||
|
||||
config = {
|
||||
Cmd = ["${package}/bin/pistond"];
|
||||
Env = [
|
||||
"NIX_PAGER=cat"
|
||||
"USER=nobody"
|
||||
"SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt"
|
||||
"GIT_SSL_CAINFO=/etc/ssl/certs/ca-bundle.crt"
|
||||
"NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt"
|
||||
];
|
||||
|
||||
ExposedPorts = {
|
||||
"2000/tcp" = {};
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
@ -1,14 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
echo "Starting Piston API"
|
||||
|
||||
echo "Checking presense of nix store"
|
||||
if [[ ! -f "/nix/piston_detected" ]]; then
|
||||
echo "Nix Store is not loaded, assuming /nix has been mounted - copying contents"
|
||||
cp -rp /var/nix/* /nix
|
||||
fi
|
||||
|
||||
echo "Adding nix to env"
|
||||
. ~/.profile
|
||||
|
||||
echo "Launching Piston API"
|
||||
node src
|
||||
|
|
@ -2,7 +2,7 @@
|
|||
"name": "piston-api",
|
||||
"version": "4.0.0",
|
||||
"description": "API for piston - a high performance code execution engine",
|
||||
"main": "src/pistond.js",
|
||||
"main": "src/bin/pistond.js",
|
||||
"dependencies": {
|
||||
"body-parser": "^1.19.0",
|
||||
"express": "^4.17.1",
|
||||
|
|
@ -23,6 +23,8 @@
|
|||
"prettier": "2.2.1"
|
||||
},
|
||||
"bin": {
|
||||
"pistond": "./src/pistond.js"
|
||||
"pistond": "./src/bin/pistond.js",
|
||||
"piston-install": "./src/bin/install.js",
|
||||
"piston-test": "./src/bin/test.js"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,39 @@
|
|||
#!/usr/bin/env node
|
||||
// Downloads all packages specified into the nix store
|
||||
require('nocamel');
|
||||
const config = require('../config');
|
||||
const Logger = require('logplease');
|
||||
const logger = Logger.create('install');
|
||||
const cp = require('child_process');
|
||||
|
||||
logger.info('Setting loglevel to', config.log_level);
|
||||
Logger.setLogLevel(config.log_level);
|
||||
|
||||
const runtimes_path = `${config.flake_path}#pistonRuntimeSets.${config.runtime_set}`;
|
||||
logger.info(`Installing runtimes in ${runtimes_path}`);
|
||||
|
||||
logger.debug("Getting package listing");
|
||||
const runtimes = JSON.parse(cp.execSync(`nix eval --json ${runtimes_path} --apply builtins.attrNames`));
|
||||
|
||||
logger.info(`Got runtimes: ${runtimes}`);
|
||||
|
||||
runtimes.forEach(runtime => {
|
||||
logger.debug(`Loading metadata for ${runtime}`);
|
||||
const runtime_metadata = JSON.parse(cp.execSync(`nix eval --json ${runtimes_path}.${runtime}.metadata`));
|
||||
const namever = `${runtime}-${runtime_metadata.version}`;
|
||||
logger.info(`Installing ${namever}`);
|
||||
|
||||
|
||||
function install(key){
|
||||
logger.debug(`Installing ${namever}-${key}`);
|
||||
const command = `nix build ${runtimes_path}.${runtime}.metadata.${key} -o /piston/runtimes/${namever}-${key}`;
|
||||
cp.execSync(command);
|
||||
}
|
||||
|
||||
install("run");
|
||||
if(runtime_metadata.compile) install("compile");
|
||||
|
||||
logger.info(`Installed ${namever}`);
|
||||
});
|
||||
|
||||
logger.info("Done");
|
||||
|
|
@ -3,16 +3,16 @@ require('nocamel');
|
|||
const Logger = require('logplease');
|
||||
const express = require('express');
|
||||
const expressWs = require('express-ws');
|
||||
const globals = require('./globals');
|
||||
const config = require('./config');
|
||||
const globals = require('../globals');
|
||||
const config = require('../config');
|
||||
const cp = require('child_process');
|
||||
const path = require('path');
|
||||
const fs = require('fs/promises');
|
||||
const fss = require('fs');
|
||||
const body_parser = require('body-parser');
|
||||
const runtime = require('./runtime');
|
||||
const runtime = require('../runtime');
|
||||
|
||||
const logger = Logger.create('index');
|
||||
const logger = Logger.create('pistond');
|
||||
const app = express();
|
||||
expressWs(app);
|
||||
|
||||
|
|
@ -41,7 +41,7 @@ expressWs(app);
|
|||
|
||||
logger.info('Loading packages');
|
||||
|
||||
const runtimes_data = cp.execSync(`nix eval --json ${config.flake_path}#pistonRuntimes --apply builtins.attrNames`).toString();
|
||||
const runtimes_data = cp.execSync(`nix eval --json ${config.flake_path}#pistonRuntimeSets.${config.runtime_set} --apply builtins.attrNames`).toString();
|
||||
const runtimes = JSON.parse(runtimes_data);
|
||||
|
||||
runtimes.for_each(pkg => runtime.load_runtime(pkg));
|
||||
|
|
@ -61,8 +61,8 @@ expressWs(app);
|
|||
|
||||
logger.debug('Registering Routes');
|
||||
|
||||
const api_v2 = require('./api/v2');
|
||||
const api_v3 = require('./api/v3');
|
||||
const api_v2 = require('../api/v2');
|
||||
const api_v3 = require('../api/v3');
|
||||
app.use('/api/v2', api_v2);
|
||||
app.use('/api/v3', api_v3);
|
||||
|
||||
|
|
@ -0,0 +1,112 @@
|
|||
#!/usr/bin/env node
|
||||
// Test the specified package
|
||||
require('nocamel');
|
||||
const config = require('../config');
|
||||
const Logger = require('logplease');
|
||||
const logger = Logger.create('test');
|
||||
const cp = require('child_process');
|
||||
const runtime = require("../runtime");
|
||||
const { Job } = require('../job');
|
||||
|
||||
(async function(){
|
||||
logger.info('Setting loglevel to', config.log_level);
|
||||
Logger.setLogLevel(config.log_level);
|
||||
|
||||
|
||||
|
||||
let runtimes_to_test;
|
||||
let failed = false;
|
||||
|
||||
if(process.argv[2] === "--all"){
|
||||
// load all
|
||||
runtimes_to_test = JSON.parse(
|
||||
cp.execSync(`nix eval ${config.flake_path}#pistonRuntimes --json --apply builtins.attrNames`)
|
||||
);
|
||||
}else{
|
||||
runtimes_to_test = [process.argv[2]];
|
||||
}
|
||||
|
||||
|
||||
|
||||
for (const runtime_name of runtimes_to_test) {
|
||||
|
||||
|
||||
const runtime_path = `${config.flake_path}#pistonRuntimes.${runtime_name}`;
|
||||
logger.info(`Testing runtime ${runtime_path}`);
|
||||
|
||||
logger.debug(`Loading runtime metadata`);
|
||||
const metadata = JSON.parse(cp.execSync(`nix eval --json ${runtime_path}.metadata --json`));
|
||||
|
||||
logger.debug(`Loading runtime tests`);
|
||||
const tests = JSON.parse(cp.execSync(`nix eval --json ${runtime_path}.tests --json`));
|
||||
|
||||
logger.debug(`Loading runtime`);
|
||||
|
||||
const testable_runtime = new runtime.Runtime({
|
||||
...metadata,
|
||||
flake_path: runtime_path
|
||||
});
|
||||
|
||||
testable_runtime.ensure_built();
|
||||
|
||||
|
||||
logger.info(`Running tests`);
|
||||
|
||||
for (const test of tests) {
|
||||
|
||||
const files = [];
|
||||
|
||||
for (const file_name of Object.keys(test.files)) {
|
||||
const file_content = test.files[file_name];
|
||||
const this_file = {
|
||||
name: file_name,
|
||||
content: file_content
|
||||
};
|
||||
|
||||
if(file_name == test.main)
|
||||
files.unshift(this_file);
|
||||
else
|
||||
files.push(this_file);
|
||||
|
||||
}
|
||||
|
||||
|
||||
const job = new Job({
|
||||
runtime: testable_runtime,
|
||||
args: test.args || [],
|
||||
stdin: test.stdin || "",
|
||||
files,
|
||||
timeouts: {
|
||||
run: 3000,
|
||||
compile: 10000
|
||||
},
|
||||
memory_limits: {
|
||||
run: config.run_memory_limit,
|
||||
compile: config.compile_memory_limit
|
||||
}
|
||||
});
|
||||
|
||||
await job.prime()
|
||||
const result = await job.execute()
|
||||
await job.cleanup()
|
||||
|
||||
if(result.run.stdout.trim() !== "OK"){
|
||||
failed = true;
|
||||
|
||||
logger.error("Test Failed:")
|
||||
console.log(job, result)
|
||||
}else{
|
||||
logger.info("Test Passed")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(failed) {
|
||||
logger.error("One or more tests failed")
|
||||
process.exit(1);
|
||||
}
|
||||
else {
|
||||
logger.info("All tests passed")
|
||||
process.exit(0);
|
||||
}
|
||||
})()
|
||||
|
|
@ -113,6 +113,12 @@ const options = [
|
|||
default: 'github:engineer-man/piston?directory=packages',
|
||||
validators: [],
|
||||
},
|
||||
{
|
||||
key: 'runtime_set',
|
||||
desc: 'Key on the flake specified by flake_path to access runtimes from',
|
||||
default: 'all',
|
||||
validators: []
|
||||
},
|
||||
{
|
||||
key: 'max_concurrent_jobs',
|
||||
desc: 'Maximum number of concurrent jobs to run at one time',
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ const runtimes = [];
|
|||
|
||||
|
||||
class Runtime {
|
||||
constructor({ language, version, aliases, runtime, run, compile, packageSupport, flake_key }) {
|
||||
constructor({ language, version, aliases, runtime, run, compile, packageSupport, flake_path }) {
|
||||
this.language = language;
|
||||
this.runtime = runtime;
|
||||
this.aliases = aliases;
|
||||
|
|
@ -18,17 +18,17 @@ class Runtime {
|
|||
this.run = run;
|
||||
this.compile = compile;
|
||||
|
||||
this.flake_key = flake_key;
|
||||
this.flake_path = flake_path;
|
||||
this.package_support = packageSupport;
|
||||
}
|
||||
|
||||
ensure_built(){
|
||||
logger.info(`Ensuring ${this} is built`);
|
||||
|
||||
const flake_key = this.flake_key;
|
||||
const flake_path = this.flake_path;
|
||||
|
||||
function _ensure_built(key){
|
||||
const command = `nix build ${config.flake_path}#pistonRuntimes.${flake_key}.metadata.${key} --no-link`;
|
||||
const command = `nix build ${flake_path}.metadata.${key} --no-link`;
|
||||
cp.execSync(command, {stdio: "pipe"})
|
||||
}
|
||||
|
||||
|
|
@ -41,12 +41,13 @@ class Runtime {
|
|||
|
||||
static load_runtime(flake_key){
|
||||
logger.info(`Loading ${flake_key}`)
|
||||
const metadata_command = `nix eval --json ${config.flake_path}#pistonRuntimes.${flake_key}.metadata`;
|
||||
const flake_path = `${config.flake_path}#pistonRuntimeSets.${config.runtime_set}.${flake_key}`;
|
||||
const metadata_command = `nix eval --json ${flake_path}.metadata`;
|
||||
const metadata = JSON.parse(cp.execSync(metadata_command));
|
||||
|
||||
const this_runtime = new Runtime({
|
||||
...metadata,
|
||||
flake_key
|
||||
flake_path
|
||||
});
|
||||
|
||||
this_runtime.ensure_built();
|
||||
|
|
|
|||
67
flake.nix
67
flake.nix
|
|
@ -6,6 +6,10 @@
|
|||
let
|
||||
system = "x86_64-linux";
|
||||
pkgs = nixpkgs.legacyPackages.${system};
|
||||
baseContainer = (import ./api {
|
||||
inherit pkgs;
|
||||
nosocket = self.legacyPackages."${system}".nosocket;
|
||||
}).container;
|
||||
args = {
|
||||
inherit pkgs;
|
||||
piston = {
|
||||
|
|
@ -53,66 +57,21 @@
|
|||
};
|
||||
};
|
||||
};
|
||||
allRuntimes = import ./runtimes args;
|
||||
runtimes = import ./runtimes args;
|
||||
runtimeList = names: pkgs.lib.filterAttrs (n: v: n == "bash") runtimes;
|
||||
in {
|
||||
piston = args.piston;
|
||||
pistonRuntimes = {
|
||||
"bash" = allRuntimes.bash;
|
||||
pistonRuntimes = runtimes;
|
||||
pistonRuntimeSets = {
|
||||
"all" = runtimes;
|
||||
"bash-only" = runtimeList ["bash"];
|
||||
};
|
||||
|
||||
legacyPackages."${system}" = {
|
||||
piston = (import ./api { inherit pkgs; }).package;
|
||||
legacyPackages."${system}" = rec {
|
||||
nosocket = (import ./nosocket { inherit pkgs; }).package;
|
||||
piston = (import ./api { inherit pkgs nosocket; }).package;
|
||||
};
|
||||
|
||||
containerImage = pkgs.dockerTools.buildLayeredImageWithNixDb {
|
||||
name = "piston";
|
||||
tag = "latest";
|
||||
|
||||
contents = with pkgs; [
|
||||
self.legacyPackages."${system}".piston
|
||||
self.legacyPackages."${system}".nosocket
|
||||
bash
|
||||
nixFlakes
|
||||
coreutils-full
|
||||
cacert.out
|
||||
git
|
||||
gnutar
|
||||
gzip
|
||||
gnugrep
|
||||
util-linux
|
||||
];
|
||||
|
||||
extraCommands = ''
|
||||
mkdir -p piston/jobs etc/nix {,var/}tmp run/lock
|
||||
echo -e "experimental-features = nix-command flakes" >> etc/nix/nix.conf
|
||||
echo "nixbld:x:30000:nixbld1,nixbld10,nixbld11,nixbld12,nixbld13,nixbld14,nixbld15,nixbld16,nixbld17,nixbld18,nixbld19,nixbld2,nixbld20,nixbld21,nixbld22,nixbld23,nixbld24,nixbld25,nixbld26,nixbld27,nixbld28,nixbld29,nixbld3,nixbld30,nixbld31,nixbld32,nixbld4,nixbld5,nixbld6,nixbld7,nixbld8,nixbld9" >> etc/group
|
||||
for i in $(seq 1 32)
|
||||
do
|
||||
echo "nixbld$i:x:$(( $i + 30000 )):30000:Nix build user $i:/var/empty:/run/current-system/sw/bin/nologin" >> etc/passwd
|
||||
done
|
||||
'';
|
||||
|
||||
config = {
|
||||
Cmd = [
|
||||
"${self.legacyPackages."${system}".piston}/bin/pistond"
|
||||
];
|
||||
|
||||
Env = [
|
||||
"NIX_PAGER=cat"
|
||||
"USER=nobody"
|
||||
"SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt"
|
||||
"GIT_SSL_CAINFO=/etc/ssl/certs/ca-bundle.crt"
|
||||
"NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt"
|
||||
|
||||
];
|
||||
|
||||
ExposedPorts = {
|
||||
"2000/tcp" = {};
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
|
||||
container = baseContainer;
|
||||
};
|
||||
}
|
||||
|
|
|
|||
141
piston
141
piston
|
|
@ -1,84 +1,93 @@
|
|||
#!/usr/bin/env bash
|
||||
cd "$(dirname "$0")"
|
||||
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
|
||||
|
||||
PISTON_ENV=$(cat .piston_env || echo dev)
|
||||
|
||||
docker_compose(){
|
||||
if [ -f "docker-compose.$PISTON_ENV.yaml" ]; then
|
||||
docker-compose -f "docker-compose.$PISTON_ENV.yaml" "$@"
|
||||
else
|
||||
docker-compose "$@"
|
||||
fi
|
||||
CONTAINER_NAME="piston_api"
|
||||
|
||||
IMAGE_TAG="base-latest"
|
||||
IMAGE_NAME="ghcr.io/piston"
|
||||
IMAGE_NAME_DEV="piston"
|
||||
|
||||
SUBCOMMAND="$1"
|
||||
shift
|
||||
|
||||
cmd_build(){
|
||||
CONTAINER_PATH="$(nix build ".#container" --no-link --json | jq '.[0].outputs.out' -r)"
|
||||
docker load -i $CONTAINER_PATH
|
||||
}
|
||||
|
||||
case $1 in
|
||||
help)
|
||||
case "$SUBCOMMAND" in
|
||||
logs) docker logs -f $CONTAINER_NAME;;
|
||||
|
||||
restart) docker restart $CONTAINER_NAME ;;
|
||||
start)
|
||||
docker run \
|
||||
--rm \
|
||||
--name $CONTAINER_NAME \
|
||||
-it "$IMAGE_NAME:$IMAGE_TAG"
|
||||
;;
|
||||
stop) docker stop $CONTAINER_NAME ;;
|
||||
bash|shell) docker exec -it $CONTAINER_NAME bash ;;
|
||||
|
||||
update)
|
||||
git pull
|
||||
docker pull "$IMAGE_NAME:$IMAGE_TAG"
|
||||
;;
|
||||
|
||||
# dev commands
|
||||
|
||||
build) cmd_build ;;
|
||||
|
||||
start-dev)
|
||||
cmd_build
|
||||
docker run \
|
||||
--rm \
|
||||
-it \
|
||||
--name $CONTAINER_NAME \
|
||||
-e PISTON_FLAKE_PATH=/piston/packages \
|
||||
-v $PWD:/piston/packages \
|
||||
-it "$IMAGE_NAME_DEV:$IMAGE_TAG"
|
||||
;;
|
||||
|
||||
test)
|
||||
cmd_build
|
||||
docker run \
|
||||
--rm \
|
||||
-it \
|
||||
-e PISTON_FLAKE_PATH=/piston/packages \
|
||||
-v $PWD:/piston/packages \
|
||||
--name piston_test_runner \
|
||||
-it "$IMAGE_NAME_DEV:$IMAGE_TAG" \
|
||||
piston-test $1
|
||||
|
||||
;;
|
||||
|
||||
*)
|
||||
echo "=== Piston Management ==="
|
||||
echo "Current Environment: $PISTON_ENV"
|
||||
echo
|
||||
echo "Commands:"
|
||||
echo " select <environment> Select the environment"
|
||||
echo " docker_compose <args...> Interact directly with the docker-compose for the selected environment"
|
||||
echo " logs Show docker-compose logs"
|
||||
echo " logs Show piston logs"
|
||||
echo
|
||||
echo " start Starts piston"
|
||||
echo " stop Stops piston"
|
||||
echo " restart Restarts piston"
|
||||
echo " bash Opens a bash shell for the piston_api container"
|
||||
echo " shell Opens a bash shell for the api container"
|
||||
echo
|
||||
echo " update Fetches and applies latest updates"
|
||||
echo " update Fetches latest updates"
|
||||
echo
|
||||
echo " <args..> Passthrough to piston cli tool"
|
||||
echo " exec <language> <file> Execute the files on piston with language"
|
||||
echo
|
||||
echo "Development Commands:"
|
||||
|
||||
if [ $PISTON_ENV == dev ]; then
|
||||
|
||||
echo " clean-pkgs Clean any package build artifacts on disk"
|
||||
echo " clean-repo Remove all packages from local repo"
|
||||
echo " build-pkg <package> <version> Build a package"
|
||||
echo " rebuild Build and restart the docker container"
|
||||
|
||||
else
|
||||
|
||||
echo " Switch to developement environment for more info"
|
||||
echo " > piston select dev"
|
||||
|
||||
fi
|
||||
echo "Running some of these commands require a nix environment setup and on the path"
|
||||
echo "See https://nixos.wiki/wiki/Nix_Installation_Guide#Stable_Nix"
|
||||
echo
|
||||
echo " start-dev Builds a container locally and starts piston"
|
||||
echo " build Builds and loads the API container"
|
||||
echo " scaffold <language> [runtime] Initializes a new runtime"
|
||||
echo " test <runtime> Runs unit tests on the given runtime"
|
||||
echo " Optionally set runtime to --all to test all"
|
||||
echo " NOTE: This is only for the runtimes contained"
|
||||
echo " within this repo"
|
||||
echo
|
||||
;;
|
||||
|
||||
|
||||
select) echo "$2" > .piston_env ;;
|
||||
docker_compose) shift; docker_compose "$@";;
|
||||
logs) docker_compose logs -f ;;
|
||||
|
||||
restart) docker_compose restart ;;
|
||||
start) docker_compose up -d ;;
|
||||
stop) docker_compose down ;;
|
||||
bash) docker_compose exec api /bin/bash ;;
|
||||
|
||||
rebuild) docker_compose build && docker_compose up -d ;;
|
||||
|
||||
update)
|
||||
git pull
|
||||
docker_compose pull
|
||||
docker_compose up -d
|
||||
;;
|
||||
|
||||
clean-pkgs) git clean -fqXd packages ;;
|
||||
clean-repo) git clean -fqXd repo ;;
|
||||
|
||||
build-pkg)
|
||||
PKGSLUG="$2-$3"
|
||||
echo "Building $PKGSLUG"
|
||||
echo "Ensuring latest builder image"
|
||||
docker build repo -t piston-repo-builder
|
||||
docker run -v "$(realpath $(dirname "$0")):/piston" piston-repo-builder --no-server $PKGSLUG
|
||||
;;
|
||||
*)
|
||||
cd cli
|
||||
npm i > /dev/null
|
||||
cd ../
|
||||
node cli/index.js "$@"
|
||||
;;
|
||||
esac
|
||||
|
|
|
|||
Loading…
Reference in New Issue