Mascali
/
piston
mirror of https://github.com/engineer-man/piston.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Fix issues after merging upstream 

Implements a simple container builder for runtime sets
This commit is contained in:
Thomas Hobson 2022-01-31 17:01:28 +13:00
parent e022e34a37
commit 83e4a1a136
No known key found for this signature in database
GPG Key ID: 9F1FD9D87950DB6F
7 changed files with 135 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
Dockerfile.withset Normal file
View File

@ -0,0 +1,12 @@
# This "FROM" image is previously emitted by nix
FROM ghcr.io/engineer-man/piston:base-latest
ENV PISTON_FLAKE_PATH=/piston/packages
COPY runtimes/ /piston/packages/runtimes
COPY flake.nix flake.lock /piston/packages/
ARG RUNTIMESET=all
ENV PISTON_RUNTIME_SET=$RUNTIMESET
RUN piston-install

17
api/default.nix
View File

@ -51,6 +51,8 @@ with pkgs; rec {
do
echo "nixbld$i:x:$(( $i + 30000 )):30000:Nix build user $i:/var/empty:/run/current-system/sw/bin/nologin" >> etc/passwd
done
chmod 1777 {,var/}tmp/
'';
config = {
@ -61,6 +63,21 @@ with pkgs; rec {
"SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt"
"GIT_SSL_CAINFO=/etc/ssl/certs/ca-bundle.crt"
"NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt"
"PATH=${lib.concatStringsSep ":" [
"/usr/local/sbin"
"/usr/local/bin"
"/usr/sbin"
"/usr/bin"
"/sbin"
"/bin"
"/root/.nix-profile/bin"
"/nix/var/nix/profiles/default/bin"
"/nix/var/nix/profiles/default/sbin"
]}"
"MANPATH=${lib.concatStringsSep ":" [
"/root/.nix-profile/share/man"
"/nix/var/nix/profiles/default/share/man"
]}"
];
ExposedPorts = {

81
api/src/bin/test.js
View File

@ -5,108 +5,105 @@ const config = require('../config');
const Logger = require('logplease');
const logger = Logger.create('test');
const cp = require('child_process');
const runtime = require("../runtime");
const runtime = require('../runtime');
const { Job } = require('../job');
(async function(){
(async function () {
logger.info('Setting loglevel to', config.log_level);
Logger.setLogLevel(config.log_level);
let runtimes_to_test;
let failed = false;
if(process.argv[2] === "--all"){
if (process.argv[2] === '--all') {
// load all
runtimes_to_test = JSON.parse(
cp.execSync(`nix eval ${config.flake_path}#pistonRuntimes --json --apply builtins.attrNames`)
cp.execSync(
`nix eval ${config.flake_path}#pistonRuntimes --json --apply builtins.attrNames`
)
);
}else{
} else {
runtimes_to_test = [process.argv[2]];
}
for (const runtime_name of runtimes_to_test) {
const runtime_path = `${config.flake_path}#pistonRuntimes.${runtime_name}`;
logger.info(`Testing runtime ${runtime_path}`);
logger.debug(`Loading runtime metadata`);
const metadata = JSON.parse(cp.execSync(`nix eval --json ${runtime_path}.metadata --json`));
const metadata = JSON.parse(
cp.execSync(`nix eval --json ${runtime_path}.metadata --json`)
);
logger.debug(`Loading runtime tests`);
const tests = JSON.parse(cp.execSync(`nix eval --json ${runtime_path}.tests --json`));
const tests = JSON.parse(
cp.execSync(`nix eval --json ${runtime_path}.tests --json`)
);
logger.debug(`Loading runtime`);
const testable_runtime = new runtime.Runtime({
...metadata,
flake_path: runtime_path
...runtime.Runtime.compute_all_limits(
metadata.language,
metadata.limit_overrides
),
flake_path: runtime_path,
});
testable_runtime.ensure_built();
logger.info(`Running tests`);
for (const test of tests) {
const files = [];
for (const file_name of Object.keys(test.files)) {
const file_content = test.files[file_name];
const this_file = {
name: file_name,
content: file_content
content: file_content,
};
if(file_name == test.main)
files.unshift(this_file);
else
files.push(this_file);
if (file_name == test.main) files.unshift(this_file);
else files.push(this_file);
}
const job = new Job({
runtime: testable_runtime,
args: test.args || [],
stdin: test.stdin || "",
stdin: test.stdin || '',
files,
timeouts: {
run: 3000,
compile: 10000
compile: 10000,
},
memory_limits: {
run: config.run_memory_limit,
compile: config.compile_memory_limit
}
compile: config.compile_memory_limit,
},
});
await job.prime()
const result = await job.execute()
await job.cleanup()
if(result.run.stdout.trim() !== "OK"){
await job.prime();
const result = await job.execute();
await job.cleanup();
if (result.run.stdout.trim() !== 'OK') {
failed = true;
logger.error("Test Failed:")
console.log(job, result)
}else{
logger.info("Test Passed")
logger.error('Test Failed:');
console.log(job, result);
} else {
logger.info('Test Passed');
}
}
}
if(failed) {
logger.error("One or more tests failed")
if (failed) {
logger.error('One or more tests failed');
process.exit(1);
}
else {
logger.info("All tests passed")
} else {
logger.info('All tests passed');
process.exit(0);
}
})()
})();

6
api/src/job.js
View File

@ -139,10 +139,6 @@ class Job {
var output = '';
const proc = cp.spawn(proc_call[0], proc_call.splice(1), {
env: {
...this.runtime.env_vars,
PISTON_LANGUAGE: this.runtime.language,
},
stdio: 'pipe',
cwd: this.dir,
uid: this.uid,
@ -250,7 +246,7 @@ class Job {
this.logger.debug('Running');
const run = await this.safe_call(
path.join(this.runtime.pkgdir, 'run'),
this.runtime.run,
[code_files[0].name, ...this.args],
this.timeouts.run,
this.memory_limits.run

32
package-lock.json generated Normal file
View File

@ -0,0 +1,32 @@
{
"name": "piston",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"devDependencies": {
"prettier": "2.4.1"
}
},
"node_modules/prettier": {
"version": "2.4.1",
"resolved": "https://registry.npmjs.org/prettier/-/prettier-2.4.1.tgz",
"integrity": "sha512-9fbDAXSBcc6Bs1mZrDYb3XKzDLm4EXXL9sC1LqKP5rZkT6KRr/rf9amVUcODVXgguK/isJz0d0hP72WeaKWsvA==",
"dev": true,
"bin": {
"prettier": "bin-prettier.js"
},
"engines": {
"node": ">=10.13.0"
}
}
},
"dependencies": {
"prettier": {
"version": "2.4.1",
"resolved": "https://registry.npmjs.org/prettier/-/prettier-2.4.1.tgz",
"integrity": "sha512-9fbDAXSBcc6Bs1mZrDYb3XKzDLm4EXXL9sC1LqKP5rZkT6KRr/rf9amVUcODVXgguK/isJz0d0hP72WeaKWsvA==",
"dev": true
}
}
}

5
package.json Normal file
View File

@ -0,0 +1,5 @@
{
"devDependencies": {
"prettier": "2.4.1"
}
}

37
piston
View File

@ -5,15 +5,16 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
CONTAINER_NAME="piston_api"
IMAGE_TAG="base-latest"
IMAGE_NAME="ghcr.io/piston"
IMAGE_NAME="ghcr.io/engineer-man/piston"
IMAGE_NAME_DEV="piston"
SUBCOMMAND="$1"
shift
cmd_build(){
build_base(){
CONTAINER_PATH="$(nix build ".#container" --no-link --json | jq '.[0].outputs.out' -r)"
docker load -i $CONTAINER_PATH
docker tag "$IMAGE_NAME_DEV:$IMAGE_TAG" "$IMAGE_NAME:$IMAGE_TAG"
}
case "$SUBCOMMAND" in
@ -22,9 +23,10 @@ case "$SUBCOMMAND" in
restart) docker restart $CONTAINER_NAME ;;
start)
docker run \
-p 2000:2000 \
--rm \
--name $CONTAINER_NAME \
-it "$IMAGE_NAME:$IMAGE_TAG"
-d "$IMAGE_NAME:$IMAGE_TAG"
;;
stop) docker stop $CONTAINER_NAME ;;
bash|shell) docker exec -it $CONTAINER_NAME bash ;;
@ -35,22 +37,40 @@ case "$SUBCOMMAND" in
;;
# dev commands
scaffold)
pushd $SCRIPT_DIR/runtimes > /dev/null
./scaffold.sh $1 $2
popd > /dev/null
;;
build) cmd_build ;;
build)
build_base
if [[ ! -z "$1" ]]; then
# $1 contains a variant to build
docker build \
--build-arg RUNTIMESET=$1 \
-f $SCRIPT_DIR/Dockerfile.withset \
-t "$IMAGE_NAME_DEV:$1-latest" \
.
fi
;;
start-dev)
cmd_build
build_base
docker run \
--rm \
-p 2000:2000 \
-it \
--name $CONTAINER_NAME \
-e PISTON_LOG_LEVEL=DEBUG \
-e PISTON_FLAKE_PATH=/piston/packages \
-v $PWD:/piston/packages \
-it "$IMAGE_NAME_DEV:$IMAGE_TAG"
-d "$IMAGE_NAME_DEV:$IMAGE_TAG"
;;
test)
cmd_build
build_base
docker run \
--rm \
-it \
@ -82,7 +102,8 @@ case "$SUBCOMMAND" in
echo "See https://nixos.wiki/wiki/Nix_Installation_Guide#Stable_Nix"
echo
echo " start-dev Builds a container locally and starts piston"
echo " build Builds and loads the API container"
echo " build [runtime-set] Builds and loads the API container optionally"
echo " including the runtime set within it"
echo " scaffold <language> [runtime] Initializes a new runtime"
echo " test <runtime> Runs unit tests on the given runtime"
echo " Optionally set runtime to --all to test all"