Merge pull request #441 from Brikaa/nix-development

Use nodemon and Docker volumes for development
This commit is contained in:
Thomas Hobson 2022-02-26 10:28:36 +13:00 committed by GitHub
commit 16e4f08dfa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 801 additions and 92 deletions

View File

@ -1,9 +1,9 @@
# This "FROM" image is previously emitted by nix # This "FROM" image is previously emitted by nix
FROM ghcr.io/engineer-man/piston:base-latest FROM ghcr.io/engineer-man/piston:base-latest
ENV PISTON_FLAKE_PATH=/piston/packages ENV PISTON_FLAKE_PATH=/piston/src
COPY runtimes/ /piston/packages/runtimes COPY runtimes/ /piston/src/runtimes
COPY flake.nix flake.lock /piston/packages/ COPY flake.nix flake.lock /piston/src/
ARG RUNTIMESET=all ARG RUNTIMESET=all

View File

@ -1,4 +1,4 @@
{pkgs, nosocket, ...}: {pkgs, nosocket, isDev, ...}:
with pkgs; rec { with pkgs; rec {
package = mkYarnPackage { package = mkYarnPackage {
name = "piston"; name = "piston";
@ -25,36 +25,54 @@ with pkgs; rec {
}; };
}; };
}; };
basePackages = with pkgs; [
package
nosocket
bash
nixFlakes
coreutils-full
cacert.out
git
gnutar
gzip
gnugrep
rename
util-linux
];
devPackages = with pkgs; [
nodejs-16_x
yarn
python3
gcc
gnumake
gnused
];
baseCommands = ''
mkdir -p piston/{jobs,runtimes} etc/nix {,var/}tmp run/lock
echo -e "experimental-features = nix-command flakes" >> etc/nix/nix.conf
echo "nixbld:x:30000:nixbld1,nixbld10,nixbld11,nixbld12,nixbld13,nixbld14,nixbld15,nixbld16,nixbld17,nixbld18,nixbld19,nixbld2,nixbld20,nixbld21,nixbld22,nixbld23,nixbld24,nixbld25,nixbld26,nixbld27,nixbld28,nixbld29,nixbld3,nixbld30,nixbld31,nixbld32,nixbld4,nixbld5,nixbld6,nixbld7,nixbld8,nixbld9" >> etc/group
for i in $(seq 1 32)
do
echo "nixbld$i:x:$(( $i + 30000 )):30000:Nix build user $i:/var/empty:/run/current-system/sw/bin/nologin" >> etc/passwd
done
chmod 1777 {,var/}tmp/
'';
devCommands = ''
mkdir -p usr/bin
ln -s /bin/env usr/bin/env
chmod -R 1777 usr
'';
container = pkgs.dockerTools.buildLayeredImageWithNixDb { container = pkgs.dockerTools.buildLayeredImageWithNixDb {
name = "piston"; name = if isDev then "piston" else "ghcr.io/engineer-man/piston";
tag = "base-latest"; tag = "base-latest";
contents = with pkgs; [ contents = if isDev then basePackages ++ devPackages else basePackages;
package
nosocket
bash
nixFlakes
coreutils-full
cacert.out
git
gnutar
gzip
gnugrep
rename
util-linux
];
extraCommands = '' extraCommands = if isDev then baseCommands + devCommands else baseCommands;
mkdir -p piston/{jobs,runtimes} etc/nix {,var/}tmp run/lock
echo -e "experimental-features = nix-command flakes" >> etc/nix/nix.conf
echo "nixbld:x:30000:nixbld1,nixbld10,nixbld11,nixbld12,nixbld13,nixbld14,nixbld15,nixbld16,nixbld17,nixbld18,nixbld19,nixbld2,nixbld20,nixbld21,nixbld22,nixbld23,nixbld24,nixbld25,nixbld26,nixbld27,nixbld28,nixbld29,nixbld3,nixbld30,nixbld31,nixbld32,nixbld4,nixbld5,nixbld6,nixbld7,nixbld8,nixbld9" >> etc/group
for i in $(seq 1 32)
do
echo "nixbld$i:x:$(( $i + 30000 )):30000:Nix build user $i:/var/empty:/run/current-system/sw/bin/nologin" >> etc/passwd
done
chmod 1777 {,var/}tmp/
'';
config = { config = {
Cmd = ["${package}/bin/pistond"]; Cmd = ["${package}/bin/pistond"];

View File

@ -8,18 +8,20 @@
"express": "^4.17.1", "express": "^4.17.1",
"express-ws": "^5.0.2", "express-ws": "^5.0.2",
"logplease": "^1.2.15", "logplease": "^1.2.15",
"nocamel": "git://github.com/HexF/nocamel.git#patch-1",
"node-fetch": "^2.6.1", "node-fetch": "^2.6.1",
"uuid": "^8.3.2", "uuid": "^8.3.2",
"nocamel": "git://github.com/HexF/nocamel.git#patch-1",
"waitpid": "git://github.com/HexF/node-waitpid.git" "waitpid": "git://github.com/HexF/node-waitpid.git"
}, },
"license": "MIT", "license": "MIT",
"scripts": { "scripts": {
"dev": "nodemon .",
"lint": "prettier . --write", "lint": "prettier . --write",
"prepack": "yarn2nix > yarn.nix" "prepack": "yarn2nix > yarn.nix"
}, },
"devDependencies": { "devDependencies": {
"node2nix": "^1.6.0", "node2nix": "^1.6.0",
"nodemon": "^2.0.15",
"prettier": "2.2.1" "prettier": "2.2.1"
}, },
"bin": { "bin": {

File diff suppressed because it is too large Load Diff

View File

@ -6,10 +6,7 @@
let let
system = "x86_64-linux"; system = "x86_64-linux";
pkgs = nixpkgs.legacyPackages.${system}; pkgs = nixpkgs.legacyPackages.${system};
baseContainer = (import ./api { nosocket = self.legacyPackages."${system}".nosocket;
inherit pkgs;
nosocket = self.legacyPackages."${system}".nosocket;
}).container;
args = { args = {
inherit pkgs; inherit pkgs;
piston = { piston = {
@ -66,6 +63,7 @@
pistonRuntimeSets = { pistonRuntimeSets = {
"all" = runtimes; "all" = runtimes;
"bash-only" = runtimeList ["bash"]; "bash-only" = runtimeList ["bash"];
"none" = { };
}; };
legacyPackages."${system}" = rec { legacyPackages."${system}" = rec {
@ -73,6 +71,16 @@
piston = (import ./api { inherit pkgs nosocket; }).package; piston = (import ./api { inherit pkgs nosocket; }).package;
}; };
container = baseContainer; containers = {
dev = (import ./api {
inherit pkgs nosocket;
isDev = true;
}).container;
prod = (import ./api {
inherit pkgs nosocket;
isDev = false;
}).container;
};
}; };
} }

124
piston
View File

@ -3,6 +3,7 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
CONTAINER_NAME="piston_api" CONTAINER_NAME="piston_api"
DEV_VOLUME_NAME="piston_nix"
IMAGE_TAG="base-latest" IMAGE_TAG="base-latest"
IMAGE_NAME="ghcr.io/engineer-man/piston" IMAGE_NAME="ghcr.io/engineer-man/piston"
@ -11,10 +12,11 @@ IMAGE_NAME_DEV="piston"
SUBCOMMAND="$1" SUBCOMMAND="$1"
shift shift
build_base(){ build_base() {
CONTAINER_PATH="$(nix build ".#container" --no-link --json | jq '.[0].outputs.out' -r)" || exit 1 container_flake_key=$([[ $1 == "dev" ]] && echo "containers.dev" || echo "containers.prod")
CONTAINER_PATH="$(nix build ".#$container_flake_key" --no-link --json | jq '.[0].outputs.out' -r)"
echo "The image archive was created at: $CONTAINER_PATH"
docker load -i $CONTAINER_PATH || exit 1 docker load -i $CONTAINER_PATH || exit 1
docker tag "$IMAGE_NAME_DEV:$IMAGE_TAG" "$IMAGE_NAME:$IMAGE_TAG" || exit 1
} }
case "$SUBCOMMAND" in case "$SUBCOMMAND" in
@ -36,28 +38,48 @@ case "$SUBCOMMAND" in
docker pull "$IMAGE_NAME:$IMAGE_TAG" docker pull "$IMAGE_NAME:$IMAGE_TAG"
;; ;;
# dev commands
scaffold)
pushd $SCRIPT_DIR/runtimes > /dev/null
./scaffold.sh $1 $2
popd > /dev/null
;;
build) build)
build_base build_base
if [[ ! -z "$1" ]]; then if [[ ! -z "$1" ]]; then
# $1 contains a variant to build # $1 contains a variant to build
runtime_set=$1
docker build \ docker build \
--build-arg RUNTIMESET=$1 \ --build-arg RUNTIMESET=$runtime_set \
-f $SCRIPT_DIR/Dockerfile.withset \ -f "$SCRIPT_DIR"/Dockerfile.withset \
-t "$IMAGE_NAME_DEV:$1-latest" \ -t "$IMAGE_NAME_DEV:$runtime_set-latest" \
. .
fi fi
;; ;;
# dev commands
scaffold)
pushd "$SCRIPT_DIR/runtimes" > /dev/null
./scaffold.sh $1 $2
popd > /dev/null
;;
build-dev)
echo "Removing the Nix volume if it exists"
docker volume rm -f $DEV_VOLUME_NAME || exit 1
echo "Building the base docker image"
build_base dev
echo "Installing the required node modules"
docker run \
--rm \
-p 2000:2000 \
-it \
--name $CONTAINER_NAME \
-e PISTON_LOG_LEVEL=DEBUG \
-e PISTON_FLAKE_PATH=/piston/src \
-e PISTON_RUNTIME_SET=none \
-v "$SCRIPT_DIR":/piston/src \
"$IMAGE_NAME_DEV:$IMAGE_TAG" \
bash -c "cd /piston/src/api && yarn install"
echo "Done building"
;;
start-dev) start-dev)
build_base
runtime_set=all runtime_set=all
if [[ ! -z "$1" ]]; then if [[ ! -z "$1" ]]; then
runtime_set=$1 runtime_set=$1
@ -68,52 +90,66 @@ case "$SUBCOMMAND" in
-it \ -it \
--name $CONTAINER_NAME \ --name $CONTAINER_NAME \
-e PISTON_LOG_LEVEL=DEBUG \ -e PISTON_LOG_LEVEL=DEBUG \
-e PISTON_FLAKE_PATH=/piston/packages \ -e PISTON_FLAKE_PATH=/piston/src \
-e PISTON_RUNTIME_SET=$runtime_set \ -e PISTON_RUNTIME_SET=$runtime_set \
-v $PWD:/piston/packages \ -v "$SCRIPT_DIR":/piston/src \
-d "$IMAGE_NAME_DEV:$IMAGE_TAG" -v $DEV_VOLUME_NAME:/nix \
;; -d "$IMAGE_NAME_DEV:$IMAGE_TAG" \
bash -c "cd /piston/src/api && yarn run dev"
;;
test) test)
build_base
docker run \ docker run \
--rm \ --rm \
-it \ -it \
-e PISTON_FLAKE_PATH=/piston/packages \ -e PISTON_FLAKE_PATH=/piston/src \
-v $PWD:/piston/packages \ -v "$SCRIPT_DIR":/piston/src \
--name piston_test_runner \ --name piston_test_runner \
-it "$IMAGE_NAME_DEV:$IMAGE_TAG" \ "$IMAGE_NAME:$IMAGE_TAG" \
piston-test $1 piston-test $1
;;
test-dev)
docker run \
--rm \
-it \
-e PISTON_FLAKE_PATH=/piston/src \
-v "$SCRIPT_DIR":/piston/src \
-v $DEV_VOLUME_NAME:/nix \
--name piston_test_runner \
"$IMAGE_NAME_DEV:$IMAGE_TAG" \
/piston/src/api/src/bin/test.js $1
;; ;;
*) *)
echo "=== Piston Management ===" echo "=== Piston Management ==="
echo echo
echo "Commands:" echo "======================"
echo " logs Show piston logs" echo "General Commands"
echo "======================"
echo " logs Show Piston logs"
echo " start Start Piston production container"
echo " stop Stop Piston"
echo " restart Restart Piston"
echo " shell Open a bash shell for the container"
echo " update Fetch latest updates"
echo " exec <language> <file> Execute the files on Piston with language"
echo echo
echo " start Starts piston" echo "======================"
echo " stop Stops piston" echo "Development Commands"
echo " restart Restarts piston"
echo " shell Opens a bash shell for the api container"
echo
echo " update Fetches latest updates"
echo
echo " exec <language> <file> Execute the files on piston with language"
echo
echo "Development Commands:"
echo "Running some of these commands require a nix environment setup and on the path" echo "Running some of these commands require a nix environment setup and on the path"
echo "See https://nixos.wiki/wiki/Nix_Installation_Guide#Stable_Nix" echo "See https://nixos.wiki/wiki/Nix_Installation_Guide#Stable_Nix"
echo echo "======================"
echo " start-dev Builds a container locally and starts piston" echo " build-dev Build the development image"
echo " build [runtime-set] Builds and loads the API container optionally" echo " build [runtime-set] Build the production image, and optionally"
echo " including the runtime set within it" echo " include a specified runtime set"
echo " scaffold <language> [runtime] Initializes a new runtime" echo " start-dev Start Piston development container"
echo " test <runtime> Runs unit tests on the given runtime" echo " scaffold <language> [runtime] Initialize a template for a new runtime"
echo " Optionally set runtime to --all to test all" echo " test <runtime> Run unit tests on the given runtime"
echo " NOTE: This is only for the runtimes contained" echo " Optionally set runtime to --all to test all"
echo " within this repo" echo " NOTE: This is only for the runtimes contained"
echo " within this repo"
echo " test-dev <runtime> test using the development container and volume"
echo echo
;; ;;
esac esac