diff --git a/.github/workflows/api-push.yaml b/.github/workflows/api-push.yaml deleted file mode 100644 index 7bafdc1..0000000 --- a/.github/workflows/api-push.yaml +++ /dev/null @@ -1,32 +0,0 @@ -name: Publish API image -on: - push: - branches: - - master - - v3 - paths: - - api/** - - -jobs: - push_to_registry: - runs-on: ubuntu-latest - name: Build and Push Docker image to Github Packages - steps: - - name: Check out repo - uses: actions/checkout@v2 - - name: Login to GitHub registry - uses: docker/login-action@v1 - with: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - registry: docker.pkg.github.com - - - name: Build and push API - uses: docker/build-push-action@v2 - with: - context: api - push: true - pull: true - tags: | - docker.pkg.github.com/engineer-man/piston/api \ No newline at end of file diff --git a/.github/workflows/package-pr.yaml b/.github/workflows/package-pr.yaml index cc5e46e..245e81a 100644 --- a/.github/workflows/package-pr.yaml +++ b/.github/workflows/package-pr.yaml @@ -11,78 +11,63 @@ on: - 'packages/**' jobs: - build-pkg: + check-build: name: Check that package builds runs-on: ubuntu-latest steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Login to GitHub registry - uses: docker/login-action@v1 - with: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - registry: docker.pkg.github.com - - - name: Get list of changed files - uses: lots0logs/gh-action-get-changed-files@2.1.4 + - name: Get PR Commits + id: 'get-pr-commits' + uses: tim-actions/get-pr-commits@master with: token: ${{ secrets.GITHUB_TOKEN }} - + - name: Check subsystem + uses: tim-actions/commit-message-checker-with-regex@v0.3.1 + with: + commits: ${{ steps.get-pr-commits.outputs.commits }} + pattern: '^[\s]*(pkg\([^:\s\n]+\))[\s]*:' + error: 'Your commit message must start with pkg([package])' + + - name: Get packages + id: 'get-packages' + shell: bash + run: | + COMMITS='${{ steps.get-pr-commits.outputs.commits }}' + PACKAGES=$(echo $COMMITS | jq .[].commit.message -r | grep -oP '^pkg\(\K[^:\h\n]+(?=\))' | sort -u) + echo "::set-output name=packages::$PACKAGES" + + - name: Checkout + uses: actions/checkout@v2 + + - name: Build docker containers + run: | + docker build -t piston_fs_repo repo + docker build -t piston_api api + - name: Build Packages run: | - PACKAGES=$(jq '.[]' -r ${HOME}/files.json | awk -F/ '{ print $2 "-" $3 }' | sort -u) - echo "Packages: $PACKAGES" - docker run -v "${{ github.workspace }}:/piston" docker.pkg.github.com/engineer-man/piston/repo-builder:latest --no-server $PACKAGES - ls -la packages - - - name: Upload package as artifact - uses: actions/upload-artifact@v2 - with: - name: packages - path: packages/*.pkg.tar.gz - - - test-pkg: - name: Test package - runs-on: ubuntu-latest - needs: build-pkg - steps: - - uses: actions/checkout@v2 - - - uses: actions/download-artifact@v2 - with: - name: packages - - - name: Write test config file - uses: DamianReeves/write-file-action@v1.0 - with: - path: data/config.yaml - contents: | - log_level: DEBUG - bind_address: 0.0.0.0:6969 - data_directory: /piston - runner_uid_min: 1100 - runner_uid_max: 1500 - runner_gid_min: 1100 - runner_gid_max: 1500 - enable_unshare: false - output_max_size: 1024 - max_process_count: 64 - max_open_files: 2048 - repo_url: http://piston_fs_repo:8000/index - - write-mode: overwrite + docker run -v $(pwd)'/repo:/repo' -v $(pwd)'/packages:/packages' piston_fs_repo ${{ steps.get-packages.outputs.packages }} - name: Run tests run: | - docker run -v $(pwd)'/repo:/piston/repo' -v $(pwd)'/packages:/piston/packages' --name piston_fs_repo docker.pkg.github.com/engineer-man/piston/repo-builder $PACKAGES & - docker run -v $(pwd)'/data:/piston' --name api docker.pkg.github.com/engineer-man/piston/api & + docker run -p 6969:6969 -v $(pwd)'/data:/piston' -v $(pwd)'/repo:/repo' --privileged --name api piston_api & echo Waiting for API to start.. - while [[ "$(curl -s -w '%{http_code}' http://api:6969/runtimes)" != "200" ]]; do sleep 5; done - + while [[ "$(curl -s -w '%{http_code}' http://127.0.0.1:6969/runtimes)" != "200" ]]; do sleep 5; done + echo Adding local repo + curl -s http://127.0.0.1:6969/repos -XPOST -d "slug=local&url=file:///repo/index.yaml" echo Testing packages + for pkg in "$(curl -s http://127.0.0.1:6969/repos/local/packages/ | jq '.data.packages[] | "\(.language)/\(.language_version)"' -r)" + do + PKG_SLUG=${pkg/\//-} + PKG_NAME=$(echo $pkg | cut -d'/' -f 1) + PKG_VERSION=$(echo $pkg | cut -d'/' -f 2) + echo Installing ${PKG_SLUG} + curl -sXPOST http://127.0.0.1:6969/repos/local/packages/${pkg} | jq '.language' -r || exit 1 + echo Testing ${PKG_SLUG} (using ${PKG_SLUG}.tf) + TEST_FILE=$(cat ${PKG_SLUG}.tf) + TEST_JSON=`jq -C '.language = "${PKG_NAME}" | .version = "${PKG_VERSION}" | .files=[] | .files[0]={} | .files[0].name="test" | .files[0].name.content="${TEST_FILE}" | .main = "test" | .args = [] | .stdin = "" | .compile_timeout = 10000 | .run_timeout = 3000' <<< '{}'` + curl -sXPOST http://127.0.0.1:6969/jobs -H 'Content-Type: application/json' -d "$TEST_JSON" > ${PKG_SLUG}.tr + jq '.run.stdout' ${PKG_SLUG}.tr | grep "OK" || exit 1 + done diff --git a/.github/workflows/repo-push.yaml b/.github/workflows/repo-push.yaml deleted file mode 100644 index b5a603c..0000000 --- a/.github/workflows/repo-push.yaml +++ /dev/null @@ -1,31 +0,0 @@ -name: Publish Repo image -on: - push: - branches: - - master - - v3 - paths: - - repo/** - -jobs: - push_to_registry: - runs-on: ubuntu-latest - name: Build and Push Docker image to Github Packages - steps: - - name: Check out repo - uses: actions/checkout@v2 - - name: Login to GitHub registry - uses: docker/login-action@v1 - with: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - registry: docker.pkg.github.com - - - name: Build and push repo - uses: docker/build-push-action@v2 - with: - context: repo - pull: true - push: true - tags: | - docker.pkg.github.com/engineer-man/piston/repo-builder \ No newline at end of file diff --git a/api/src/executor/job.js b/api/src/executor/job.js index 445c73a..ba739c4 100644 --- a/api/src/executor/job.js +++ b/api/src/executor/job.js @@ -81,8 +81,7 @@ class Job { const proc_call = [ ...prlimit, ...unshare, - 'bash',file, - ...args + 'bash',file, ...args ]; var stdout = ''; diff --git a/packages/bash/5.1.0/run b/packages/bash/5.1.0/run index a8f3ffb..be4ec50 100644 --- a/packages/bash/5.1.0/run +++ b/packages/bash/5.1.0/run @@ -1,4 +1,4 @@ #!/usr/bin/env bash # Put instructions to run the runtime -bash "$@" +bash $* diff --git a/packages/dart/2.12.1/run b/packages/dart/2.12.1/run index aae792a..dfdbd40 100644 --- a/packages/dart/2.12.1/run +++ b/packages/dart/2.12.1/run @@ -1,4 +1,4 @@ #!/usr/bin/env bash # Put instructions to run the runtime -dart run "$@" +dart run $* diff --git a/packages/deno/1.7.5/run b/packages/deno/1.7.5/run index d1b196f..9bcf3ef 100644 --- a/packages/deno/1.7.5/run +++ b/packages/deno/1.7.5/run @@ -1,2 +1,2 @@ #!/bin/bash -DENO_DIR=$PWD deno run "$@" \ No newline at end of file +DENO_DIR=$PWD deno run $* \ No newline at end of file diff --git a/packages/gawk/5.1.0/run b/packages/gawk/5.1.0/run index 5134ddf..567b400 100644 --- a/packages/gawk/5.1.0/run +++ b/packages/gawk/5.1.0/run @@ -1,4 +1,4 @@ #!/usr/bin/env bash # Put instructions to run the runtime -gawk-5.1.0 -f "$@" +gawk-5.1.0 -f $* diff --git a/packages/gcc/10.2.0/compile b/packages/gcc/10.2.0/compile index acc9ae4..e83346d 100644 --- a/packages/gcc/10.2.0/compile +++ b/packages/gcc/10.2.0/compile @@ -5,16 +5,16 @@ case "${PISTON_ALIAS}" in gcc | c) - gcc -std=c11 "$@" -lm + gcc -std=c11 $* -lm ;; g++ | c++ | cpp) - g++ -std=c++17 "$@" + g++ -std=c++17 $* ;; gccgo | go) - gccgo "$@" + gccgo $* ;; gdc | d) - gdc "$@" + gdc $* ;; *) echo "How did you get here? (${PISTON_ALIAS})" diff --git a/packages/gcc/10.2.0/run b/packages/gcc/10.2.0/run index 60ad16b..63e3443 100644 --- a/packages/gcc/10.2.0/run +++ b/packages/gcc/10.2.0/run @@ -3,4 +3,4 @@ # Put instructions to run the runtime shift # Discard main filename -./a.out "$@" +./a.out $* diff --git a/packages/java/15.0.2/run b/packages/java/15.0.2/run index 0837ba4..2215edd 100644 --- a/packages/java/15.0.2/run +++ b/packages/java/15.0.2/run @@ -1,4 +1,4 @@ #!/usr/bin/env bash # Put instructions to run the runtime -java "$@" +java $* diff --git a/packages/jelly/0.1.31/run b/packages/jelly/0.1.31/run index e07c1af..7f4dd38 100644 --- a/packages/jelly/0.1.31/run +++ b/packages/jelly/0.1.31/run @@ -1 +1 @@ -jelly fu "$@" \ No newline at end of file +jelly fu $* \ No newline at end of file diff --git a/packages/kotlin/1.4.31/compile b/packages/kotlin/1.4.31/compile index d042c39..8be38a7 100644 --- a/packages/kotlin/1.4.31/compile +++ b/packages/kotlin/1.4.31/compile @@ -1,4 +1,4 @@ #!/usr/bin/env bash # Put instructions to compile source code, remove this file if the language does not require this stage -kotlinc "$@" -include-runtime -d code.jar \ No newline at end of file +kotlinc $* -include-runtime -d code.jar \ No newline at end of file diff --git a/packages/mono/6.12.0/compile b/packages/mono/6.12.0/compile index 746c05e..3151cde 100644 --- a/packages/mono/6.12.0/compile +++ b/packages/mono/6.12.0/compile @@ -1,3 +1,3 @@ #!/bin/bash -csc "$@" \ No newline at end of file +csc $* \ No newline at end of file diff --git a/packages/mono/6.12.0/run b/packages/mono/6.12.0/run index 557ce76..9d9b61a 100644 --- a/packages/mono/6.12.0/run +++ b/packages/mono/6.12.0/run @@ -2,4 +2,4 @@ CODE=${1/cs/exe} shift -mono $CODE "$@" \ No newline at end of file +mono $CODE $* \ No newline at end of file diff --git a/packages/nasm/2.15.5/compile b/packages/nasm/2.15.5/compile index e481319..a8fe6da 100644 --- a/packages/nasm/2.15.5/compile +++ b/packages/nasm/2.15.5/compile @@ -5,11 +5,11 @@ case "${PISTON_ALIAS}" in nasm) - nasm -f elf32 -o binary.o "$@" + nasm -f elf32 -o binary.o $* ld -m elf_i386 binary.o -o binary ;; nasm64) - nasm -f elf64 -o binary.o "$@" + nasm -f elf64 -o binary.o $* ld -m elf_x86_64 binary.o -o binary ;; *) diff --git a/packages/nasm/2.15.5/run b/packages/nasm/2.15.5/run index f910a1c..53a6099 100644 --- a/packages/nasm/2.15.5/run +++ b/packages/nasm/2.15.5/run @@ -2,4 +2,4 @@ # Put instructions to run the runtime shift -./binary "$@" +./binary $* diff --git a/packages/node/15.10.0/run b/packages/node/15.10.0/run index 6d1fdee..004ca32 100644 --- a/packages/node/15.10.0/run +++ b/packages/node/15.10.0/run @@ -1,3 +1,3 @@ #!/bin/bash -node "$@" \ No newline at end of file +node $* \ No newline at end of file diff --git a/packages/php/8.0.2/run b/packages/php/8.0.2/run index 1261d95..2bce27a 100644 --- a/packages/php/8.0.2/run +++ b/packages/php/8.0.2/run @@ -1,3 +1,3 @@ #!/bin/bash -php "$@" \ No newline at end of file +php $* \ No newline at end of file diff --git a/packages/python/3.9.1/run b/packages/python/3.9.1/run index 450bb76..7b3205b 100644 --- a/packages/python/3.9.1/run +++ b/packages/python/3.9.1/run @@ -1,3 +1,3 @@ #!/bin/bash -python3.9 "$@" \ No newline at end of file +python3.9 $* \ No newline at end of file diff --git a/packages/typescript/4.2.3/compile b/packages/typescript/4.2.3/compile index fdf5f19..1258d16 100644 --- a/packages/typescript/4.2.3/compile +++ b/packages/typescript/4.2.3/compile @@ -2,4 +2,4 @@ # Put instructions to compile source code, remove this file if the language does not require this stage -tsc "$@" \ No newline at end of file +tsc $* \ No newline at end of file diff --git a/packages/typescript/4.2.3/run b/packages/typescript/4.2.3/run index 1d26f3f..dfd4249 100644 --- a/packages/typescript/4.2.3/run +++ b/packages/typescript/4.2.3/run @@ -5,4 +5,4 @@ CODE=$(sed 's/ts$/js/' <<<"$1") shift -node $CODE "$@" +node $CODE $* diff --git a/repo/Dockerfile b/repo/Dockerfile index 231289a..606b596 100644 --- a/repo/Dockerfile +++ b/repo/Dockerfile @@ -2,9 +2,10 @@ FROM debian:buster-slim RUN apt-get update && apt-get install -y unzip autoconf build-essential libssl-dev pkg-config zlib1g-dev libargon2-dev libsodium-dev libcurl4-openssl-dev sqlite3 libsqlite3-dev libonig-dev libxml2 libxml2-dev bc curl git linux-headers-amd64 perl xz-utils python3 python3-pip gnupg jq zlib1g-dev cmake cmake-doc extra-cmake-modules build-essential gcc binutils bash coreutils util-linux pciutils usbutils coreutils binutils findutils grep && \ ln -sf /bin/bash /bin/sh && \ + pip3 install 'yq==2.12.0' && \ rm -rf /var/lib/apt/lists/* -ADD entrypoint.sh mkindex.sh / +ADD *.sh / ENTRYPOINT ["bash","/entrypoint.sh"] -CMD ["--no-build"] +CMD ["all"] diff --git a/repo/README.MD b/repo/README.MD new file mode 100644 index 0000000..28f833f --- /dev/null +++ b/repo/README.MD @@ -0,0 +1,7 @@ +# Piston Filesystem Repo Builder + +This is just a simple POC for a repository tool to run locally. + +This only demonstrates building an unsigned python-3.9.1 package, however if it finds an `asc` file it will include it as the signature. + +Mount this whole directory into `/repo` in your API container if you wish to use it. \ No newline at end of file diff --git a/repo/entrypoint.sh b/repo/entrypoint.sh index d9fc973..54261dc 100755 --- a/repo/entrypoint.sh +++ b/repo/entrypoint.sh @@ -1,57 +1,11 @@ cd /piston/packages -SERVER=1 -BUILD=1 -CI=0 - -echo "Running through arguments.." - for pkg in "$@" do - shift - if [[ "$pkg" = "--no-server" ]]; then - echo "Not starting index server after builds" - SERVER=0 - elif [[ "$pkg" = "--no-build" ]]; then - echo "Building no more package" - BUILD=0 - elif [[ "$pkg" = "--ci" ]]; then - echo "Running in CI mode, --no-build, --no-server" - BUILD=0 - SERVER=0 - CI=1 - else - if [[ $BUILD -eq 1 ]]; then - echo "Building package $pkg" - make -j16 $pkg.pkg.tar.gz - echo "Done with package $pkg" - elif [[ $CI -eq 1 ]]; then - echo "Commit SHA: $pkg" - - echo "Changed files:" - git diff --name-only $pkg^1 $pkg - - PACKAGES=$(git diff --name-only $pkg^1 $pkg | awk -F/ '{ print $2 "-" $3 }' | sort -u) - echo "Building packages: $PACKAGES" - for package in "$PACKAGES"; do - make -j16 $package.pkg.tar.gz - done - - else - echo "Building was disabled, skipping $pkg build=$BUILD ci=$CI" - fi - fi + make -j16 $pkg.pkg.tar.gz done cd /piston/repo -echo "Creating index" ./mkindex.sh -echo "Index created" -if [[ $SERVER -eq 1 ]]; then - echo "Starting index server.." - python3 -m http.server -else - echo "Skipping starting index server" -fi -exit 0 \ No newline at end of file +python3 -m http.server \ No newline at end of file