Mascali
/
piston
mirror of https://github.com/engineer-man/piston.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #441 from Brikaa/nix-development 

Use nodemon and Docker volumes for development
This commit is contained in:
Thomas Hobson 2022-02-26 10:28:36 +13:00 committed by GitHub
parent 7596d6f46d 50978c4205
commit 16e4f08dfa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 801 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Dockerfile.withset
View File

@ -1,9 +1,9 @@
# This "FROM" image is previously emitted by nix
FROM ghcr.io/engineer-man/piston:base-latest
ENV PISTON_FLAKE_PATH=/piston/packages
COPY runtimes/ /piston/packages/runtimes
COPY flake.nix flake.lock /piston/packages/
ENV PISTON_FLAKE_PATH=/piston/src
COPY runtimes/ /piston/src/runtimes
COPY flake.nix flake.lock /piston/src/
ARG RUNTIMESET=all

74
api/default.nix
View File

@ -1,4 +1,4 @@
{pkgs, nosocket, ...}:
{pkgs, nosocket, isDev, ...}:
with pkgs; rec {
package = mkYarnPackage {
name = "piston";
@ -25,36 +25,54 @@ with pkgs; rec {
};
};
};
basePackages = with pkgs; [
package
nosocket
bash
nixFlakes
coreutils-full
cacert.out
git
gnutar
gzip
gnugrep
rename
util-linux
];
devPackages = with pkgs; [
nodejs-16_x
yarn
python3
gcc
gnumake
gnused
];
baseCommands = ''
mkdir -p piston/{jobs,runtimes} etc/nix {,var/}tmp run/lock
echo -e "experimental-features = nix-command flakes" >> etc/nix/nix.conf
echo "nixbld:x:30000:nixbld1,nixbld10,nixbld11,nixbld12,nixbld13,nixbld14,nixbld15,nixbld16,nixbld17,nixbld18,nixbld19,nixbld2,nixbld20,nixbld21,nixbld22,nixbld23,nixbld24,nixbld25,nixbld26,nixbld27,nixbld28,nixbld29,nixbld3,nixbld30,nixbld31,nixbld32,nixbld4,nixbld5,nixbld6,nixbld7,nixbld8,nixbld9" >> etc/group
for i in $(seq 1 32)
do
echo "nixbld$i:x:$(( $i + 30000 )):30000:Nix build user $i:/var/empty:/run/current-system/sw/bin/nologin" >> etc/passwd
done
chmod 1777 {,var/}tmp/
'';
devCommands = ''
mkdir -p usr/bin
ln -s /bin/env usr/bin/env
chmod -R 1777 usr
'';
container = pkgs.dockerTools.buildLayeredImageWithNixDb {
name = "piston";
name = if isDev then "piston" else "ghcr.io/engineer-man/piston";
tag = "base-latest";
contents = with pkgs; [
package
nosocket
bash
nixFlakes
coreutils-full
cacert.out
git
gnutar
gzip
gnugrep
rename
util-linux
];
contents = if isDev then basePackages ++ devPackages else basePackages;
extraCommands = ''
mkdir -p piston/{jobs,runtimes} etc/nix {,var/}tmp run/lock
echo -e "experimental-features = nix-command flakes" >> etc/nix/nix.conf
echo "nixbld:x:30000:nixbld1,nixbld10,nixbld11,nixbld12,nixbld13,nixbld14,nixbld15,nixbld16,nixbld17,nixbld18,nixbld19,nixbld2,nixbld20,nixbld21,nixbld22,nixbld23,nixbld24,nixbld25,nixbld26,nixbld27,nixbld28,nixbld29,nixbld3,nixbld30,nixbld31,nixbld32,nixbld4,nixbld5,nixbld6,nixbld7,nixbld8,nixbld9" >> etc/group
for i in $(seq 1 32)
do
echo "nixbld$i:x:$(( $i + 30000 )):30000:Nix build user $i:/var/empty:/run/current-system/sw/bin/nologin" >> etc/passwd
done
chmod 1777 {,var/}tmp/
'';
extraCommands = if isDev then baseCommands + devCommands else baseCommands;
config = {
Cmd = ["${package}/bin/pistond"];
@ -86,4 +104,4 @@ with pkgs; rec {
};
};
};
}
}

4
api/package.json
View File

@ -8,18 +8,20 @@
"express": "^4.17.1",
"express-ws": "^5.0.2",
"logplease": "^1.2.15",
"nocamel": "git://github.com/HexF/nocamel.git#patch-1",
"node-fetch": "^2.6.1",
"uuid": "^8.3.2",
"nocamel": "git://github.com/HexF/nocamel.git#patch-1",
"waitpid": "git://github.com/HexF/node-waitpid.git"
},
"license": "MIT",
"scripts": {
"dev": "nodemon .",
"lint": "prettier . --write",
"prepack": "yarn2nix > yarn.nix"
},
"devDependencies": {
"node2nix": "^1.6.0",
"nodemon": "^2.0.15",
"prettier": "2.2.1"
},
"bin": {

665
api/yarn.lock
View File

File diff suppressed because it is too large Load Diff

20
flake.nix
View File

@ -2,14 +2,11 @@
description = "Piston packages repo";
inputs.nixpkgs.url = "github:NixOS/nixpkgs";
outputs = { self, nixpkgs }:
outputs = { self, nixpkgs }:
let
system = "x86_64-linux";
pkgs = nixpkgs.legacyPackages.${system};
baseContainer = (import ./api {
inherit pkgs;
nosocket = self.legacyPackages."${system}".nosocket;
}).container;
nosocket = self.legacyPackages."${system}".nosocket;
args = {
inherit pkgs;
piston = {
@ -66,6 +63,7 @@
pistonRuntimeSets = {
"all" = runtimes;
"bash-only" = runtimeList ["bash"];
"none" = { };
};
legacyPackages."${system}" = rec {
@ -73,6 +71,16 @@
piston = (import ./api { inherit pkgs nosocket; }).package;
};
container = baseContainer;
containers = {
dev = (import ./api {
inherit pkgs nosocket;
isDev = true;
}).container;
prod = (import ./api {
inherit pkgs nosocket;
isDev = false;
}).container;
};
};
}

124
piston
View File

@ -3,6 +3,7 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
CONTAINER_NAME="piston_api"
DEV_VOLUME_NAME="piston_nix"
IMAGE_TAG="base-latest"
IMAGE_NAME="ghcr.io/engineer-man/piston"
@ -11,10 +12,11 @@ IMAGE_NAME_DEV="piston"
SUBCOMMAND="$1"
shift
build_base(){
CONTAINER_PATH="$(nix build ".#container" --no-link --json | jq '.[0].outputs.out' -r)" || exit 1
build_base() {
container_flake_key=$([[ $1 == "dev" ]] && echo "containers.dev" || echo "containers.prod")
CONTAINER_PATH="$(nix build ".#$container_flake_key" --no-link --json | jq '.[0].outputs.out' -r)"
echo "The image archive was created at: $CONTAINER_PATH"
docker load -i $CONTAINER_PATH || exit 1
docker tag "$IMAGE_NAME_DEV:$IMAGE_TAG" "$IMAGE_NAME:$IMAGE_TAG" || exit 1
}
case "$SUBCOMMAND" in
@ -36,28 +38,48 @@ case "$SUBCOMMAND" in
docker pull "$IMAGE_NAME:$IMAGE_TAG"
;;
# dev commands
scaffold)
pushd $SCRIPT_DIR/runtimes > /dev/null
./scaffold.sh $1 $2
popd > /dev/null
;;
build)
build_base
if [[ ! -z "$1" ]]; then
# $1 contains a variant to build
runtime_set=$1
docker build \
--build-arg RUNTIMESET=$1 \
-f $SCRIPT_DIR/Dockerfile.withset \
-t "$IMAGE_NAME_DEV:$1-latest" \
--build-arg RUNTIMESET=$runtime_set \
-f "$SCRIPT_DIR"/Dockerfile.withset \
-t "$IMAGE_NAME_DEV:$runtime_set-latest" \
.
fi
;;
# dev commands
scaffold)
pushd "$SCRIPT_DIR/runtimes" > /dev/null
./scaffold.sh $1 $2
popd > /dev/null
;;
build-dev)
echo "Removing the Nix volume if it exists"
docker volume rm -f $DEV_VOLUME_NAME || exit 1
echo "Building the base docker image"
build_base dev
echo "Installing the required node modules"
docker run \
--rm \
-p 2000:2000 \
-it \
--name $CONTAINER_NAME \
-e PISTON_LOG_LEVEL=DEBUG \
-e PISTON_FLAKE_PATH=/piston/src \
-e PISTON_RUNTIME_SET=none \
-v "$SCRIPT_DIR":/piston/src \
"$IMAGE_NAME_DEV:$IMAGE_TAG" \
bash -c "cd /piston/src/api && yarn install"
echo "Done building"
;;
start-dev)
build_base
runtime_set=all
if [[ ! -z "$1" ]]; then
runtime_set=$1
@ -68,52 +90,66 @@ case "$SUBCOMMAND" in
-it \
--name $CONTAINER_NAME \
-e PISTON_LOG_LEVEL=DEBUG \
-e PISTON_FLAKE_PATH=/piston/packages \
-e PISTON_FLAKE_PATH=/piston/src \
-e PISTON_RUNTIME_SET=$runtime_set \
-v $PWD:/piston/packages \
-d "$IMAGE_NAME_DEV:$IMAGE_TAG"
;;
-v "$SCRIPT_DIR":/piston/src \
-v $DEV_VOLUME_NAME:/nix \
-d "$IMAGE_NAME_DEV:$IMAGE_TAG" \
bash -c "cd /piston/src/api && yarn run dev"
;;
test)
build_base
docker run \
--rm \
-it \
-e PISTON_FLAKE_PATH=/piston/packages \
-v $PWD:/piston/packages \
-e PISTON_FLAKE_PATH=/piston/src \
-v "$SCRIPT_DIR":/piston/src \
--name piston_test_runner \
-it "$IMAGE_NAME_DEV:$IMAGE_TAG" \
"$IMAGE_NAME:$IMAGE_TAG" \
piston-test $1
;;
test-dev)
docker run \
--rm \
-it \
-e PISTON_FLAKE_PATH=/piston/src \
-v "$SCRIPT_DIR":/piston/src \
-v $DEV_VOLUME_NAME:/nix \
--name piston_test_runner \
"$IMAGE_NAME_DEV:$IMAGE_TAG" \
/piston/src/api/src/bin/test.js $1
;;
*)
echo "=== Piston Management ==="
echo
echo "Commands:"
echo " logs Show piston logs"
echo "======================"
echo "General Commands"
echo "======================"
echo " logs Show Piston logs"
echo " start Start Piston production container"
echo " stop Stop Piston"
echo " restart Restart Piston"
echo " shell Open a bash shell for the container"
echo " update Fetch latest updates"
echo " exec <language> <file> Execute the files on Piston with language"
echo
echo " start Starts piston"
echo " stop Stops piston"
echo " restart Restarts piston"
echo " shell Opens a bash shell for the api container"
echo
echo " update Fetches latest updates"
echo
echo " exec <language> <file> Execute the files on piston with language"
echo
echo "Development Commands:"
echo "======================"
echo "Development Commands"
echo "Running some of these commands require a nix environment setup and on the path"
echo "See https://nixos.wiki/wiki/Nix_Installation_Guide#Stable_Nix"
echo
echo " start-dev Builds a container locally and starts piston"
echo " build [runtime-set] Builds and loads the API container optionally"
echo " including the runtime set within it"
echo " scaffold <language> [runtime] Initializes a new runtime"
echo " test <runtime> Runs unit tests on the given runtime"
echo " Optionally set runtime to --all to test all"
echo " NOTE: This is only for the runtimes contained"
echo " within this repo"
echo "======================"
echo " build-dev Build the development image"
echo " build [runtime-set] Build the production image, and optionally"
echo " include a specified runtime set"
echo " start-dev Start Piston development container"
echo " scaffold <language> [runtime] Initialize a template for a new runtime"
echo " test <runtime> Run unit tests on the given runtime"
echo " Optionally set runtime to --all to test all"
echo " NOTE: This is only for the runtimes contained"
echo " within this repo"
echo " test-dev <runtime> test using the development container and volume"
echo
;;
esac