diff --git a/.github/workflows/api-push.yaml b/.github/workflows/api-push.yaml new file mode 100644 index 0000000..7bafdc1 --- /dev/null +++ b/.github/workflows/api-push.yaml @@ -0,0 +1,32 @@ +name: Publish API image +on: + push: + branches: + - master + - v3 + paths: + - api/** + + +jobs: + push_to_registry: + runs-on: ubuntu-latest + name: Build and Push Docker image to Github Packages + steps: + - name: Check out repo + uses: actions/checkout@v2 + - name: Login to GitHub registry + uses: docker/login-action@v1 + with: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + registry: docker.pkg.github.com + + - name: Build and push API + uses: docker/build-push-action@v2 + with: + context: api + push: true + pull: true + tags: | + docker.pkg.github.com/engineer-man/piston/api \ No newline at end of file diff --git a/.github/workflows/package-pr.yaml b/.github/workflows/package-pr.yaml index 245e81a..cc5e46e 100644 --- a/.github/workflows/package-pr.yaml +++ b/.github/workflows/package-pr.yaml @@ -11,63 +11,78 @@ on: - 'packages/**' jobs: - check-build: + build-pkg: name: Check that package builds runs-on: ubuntu-latest steps: - - name: Get PR Commits - id: 'get-pr-commits' - uses: tim-actions/get-pr-commits@master - with: - token: ${{ secrets.GITHUB_TOKEN }} - - name: Check subsystem - uses: tim-actions/commit-message-checker-with-regex@v0.3.1 - with: - commits: ${{ steps.get-pr-commits.outputs.commits }} - pattern: '^[\s]*(pkg\([^:\s\n]+\))[\s]*:' - error: 'Your commit message must start with pkg([package])' - - - name: Get packages - id: 'get-packages' - shell: bash - run: | - COMMITS='${{ steps.get-pr-commits.outputs.commits }}' - PACKAGES=$(echo $COMMITS | jq .[].commit.message -r | grep -oP '^pkg\(\K[^:\h\n]+(?=\))' | sort -u) - echo "::set-output name=packages::$PACKAGES" - - name: Checkout uses: actions/checkout@v2 + + - name: Login to GitHub registry + uses: docker/login-action@v1 + with: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + registry: docker.pkg.github.com - - name: Build docker containers - run: | - docker build -t piston_fs_repo repo - docker build -t piston_api api - + - name: Get list of changed files + uses: lots0logs/gh-action-get-changed-files@2.1.4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + - name: Build Packages run: | - docker run -v $(pwd)'/repo:/repo' -v $(pwd)'/packages:/packages' piston_fs_repo ${{ steps.get-packages.outputs.packages }} + PACKAGES=$(jq '.[]' -r ${HOME}/files.json | awk -F/ '{ print $2 "-" $3 }' | sort -u) + echo "Packages: $PACKAGES" + docker run -v "${{ github.workspace }}:/piston" docker.pkg.github.com/engineer-man/piston/repo-builder:latest --no-server $PACKAGES + ls -la packages + + - name: Upload package as artifact + uses: actions/upload-artifact@v2 + with: + name: packages + path: packages/*.pkg.tar.gz + + + test-pkg: + name: Test package + runs-on: ubuntu-latest + needs: build-pkg + steps: + - uses: actions/checkout@v2 + + - uses: actions/download-artifact@v2 + with: + name: packages + + - name: Write test config file + uses: DamianReeves/write-file-action@v1.0 + with: + path: data/config.yaml + contents: | + log_level: DEBUG + bind_address: 0.0.0.0:6969 + data_directory: /piston + runner_uid_min: 1100 + runner_uid_max: 1500 + runner_gid_min: 1100 + runner_gid_max: 1500 + enable_unshare: false + output_max_size: 1024 + max_process_count: 64 + max_open_files: 2048 + repo_url: http://piston_fs_repo:8000/index + + write-mode: overwrite - name: Run tests run: | - docker run -p 6969:6969 -v $(pwd)'/data:/piston' -v $(pwd)'/repo:/repo' --privileged --name api piston_api & + docker run -v $(pwd)'/repo:/piston/repo' -v $(pwd)'/packages:/piston/packages' --name piston_fs_repo docker.pkg.github.com/engineer-man/piston/repo-builder $PACKAGES & + docker run -v $(pwd)'/data:/piston' --name api docker.pkg.github.com/engineer-man/piston/api & echo Waiting for API to start.. - while [[ "$(curl -s -w '%{http_code}' http://127.0.0.1:6969/runtimes)" != "200" ]]; do sleep 5; done - echo Adding local repo - curl -s http://127.0.0.1:6969/repos -XPOST -d "slug=local&url=file:///repo/index.yaml" + while [[ "$(curl -s -w '%{http_code}' http://api:6969/runtimes)" != "200" ]]; do sleep 5; done + echo Testing packages - for pkg in "$(curl -s http://127.0.0.1:6969/repos/local/packages/ | jq '.data.packages[] | "\(.language)/\(.language_version)"' -r)" - do - PKG_SLUG=${pkg/\//-} - PKG_NAME=$(echo $pkg | cut -d'/' -f 1) - PKG_VERSION=$(echo $pkg | cut -d'/' -f 2) - echo Installing ${PKG_SLUG} - curl -sXPOST http://127.0.0.1:6969/repos/local/packages/${pkg} | jq '.language' -r || exit 1 - echo Testing ${PKG_SLUG} (using ${PKG_SLUG}.tf) - TEST_FILE=$(cat ${PKG_SLUG}.tf) - TEST_JSON=`jq -C '.language = "${PKG_NAME}" | .version = "${PKG_VERSION}" | .files=[] | .files[0]={} | .files[0].name="test" | .files[0].name.content="${TEST_FILE}" | .main = "test" | .args = [] | .stdin = "" | .compile_timeout = 10000 | .run_timeout = 3000' <<< '{}'` - curl -sXPOST http://127.0.0.1:6969/jobs -H 'Content-Type: application/json' -d "$TEST_JSON" > ${PKG_SLUG}.tr - jq '.run.stdout' ${PKG_SLUG}.tr | grep "OK" || exit 1 - done diff --git a/.github/workflows/repo-push.yaml b/.github/workflows/repo-push.yaml new file mode 100644 index 0000000..b5a603c --- /dev/null +++ b/.github/workflows/repo-push.yaml @@ -0,0 +1,31 @@ +name: Publish Repo image +on: + push: + branches: + - master + - v3 + paths: + - repo/** + +jobs: + push_to_registry: + runs-on: ubuntu-latest + name: Build and Push Docker image to Github Packages + steps: + - name: Check out repo + uses: actions/checkout@v2 + - name: Login to GitHub registry + uses: docker/login-action@v1 + with: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + registry: docker.pkg.github.com + + - name: Build and push repo + uses: docker/build-push-action@v2 + with: + context: repo + pull: true + push: true + tags: | + docker.pkg.github.com/engineer-man/piston/repo-builder \ No newline at end of file diff --git a/api/src/executor/job.js b/api/src/executor/job.js index ba739c4..445c73a 100644 --- a/api/src/executor/job.js +++ b/api/src/executor/job.js @@ -81,7 +81,8 @@ class Job { const proc_call = [ ...prlimit, ...unshare, - 'bash',file, ...args + 'bash',file, + ...args ]; var stdout = ''; diff --git a/packages/bash/5.1.0/run b/packages/bash/5.1.0/run index be4ec50..a8f3ffb 100644 --- a/packages/bash/5.1.0/run +++ b/packages/bash/5.1.0/run @@ -1,4 +1,4 @@ #!/usr/bin/env bash # Put instructions to run the runtime -bash $* +bash "$@" diff --git a/packages/dart/2.12.1/run b/packages/dart/2.12.1/run index dfdbd40..aae792a 100644 --- a/packages/dart/2.12.1/run +++ b/packages/dart/2.12.1/run @@ -1,4 +1,4 @@ #!/usr/bin/env bash # Put instructions to run the runtime -dart run $* +dart run "$@" diff --git a/packages/deno/1.7.5/run b/packages/deno/1.7.5/run index 9bcf3ef..d1b196f 100644 --- a/packages/deno/1.7.5/run +++ b/packages/deno/1.7.5/run @@ -1,2 +1,2 @@ #!/bin/bash -DENO_DIR=$PWD deno run $* \ No newline at end of file +DENO_DIR=$PWD deno run "$@" \ No newline at end of file diff --git a/packages/gawk/5.1.0/run b/packages/gawk/5.1.0/run index 567b400..5134ddf 100644 --- a/packages/gawk/5.1.0/run +++ b/packages/gawk/5.1.0/run @@ -1,4 +1,4 @@ #!/usr/bin/env bash # Put instructions to run the runtime -gawk-5.1.0 -f $* +gawk-5.1.0 -f "$@" diff --git a/packages/gcc/10.2.0/compile b/packages/gcc/10.2.0/compile index e83346d..acc9ae4 100644 --- a/packages/gcc/10.2.0/compile +++ b/packages/gcc/10.2.0/compile @@ -5,16 +5,16 @@ case "${PISTON_ALIAS}" in gcc | c) - gcc -std=c11 $* -lm + gcc -std=c11 "$@" -lm ;; g++ | c++ | cpp) - g++ -std=c++17 $* + g++ -std=c++17 "$@" ;; gccgo | go) - gccgo $* + gccgo "$@" ;; gdc | d) - gdc $* + gdc "$@" ;; *) echo "How did you get here? (${PISTON_ALIAS})" diff --git a/packages/gcc/10.2.0/run b/packages/gcc/10.2.0/run index 63e3443..60ad16b 100644 --- a/packages/gcc/10.2.0/run +++ b/packages/gcc/10.2.0/run @@ -3,4 +3,4 @@ # Put instructions to run the runtime shift # Discard main filename -./a.out $* +./a.out "$@" diff --git a/packages/java/15.0.2/run b/packages/java/15.0.2/run index 2215edd..0837ba4 100644 --- a/packages/java/15.0.2/run +++ b/packages/java/15.0.2/run @@ -1,4 +1,4 @@ #!/usr/bin/env bash # Put instructions to run the runtime -java $* +java "$@" diff --git a/packages/jelly/0.1.31/run b/packages/jelly/0.1.31/run index 7f4dd38..e07c1af 100644 --- a/packages/jelly/0.1.31/run +++ b/packages/jelly/0.1.31/run @@ -1 +1 @@ -jelly fu $* \ No newline at end of file +jelly fu "$@" \ No newline at end of file diff --git a/packages/kotlin/1.4.31/compile b/packages/kotlin/1.4.31/compile index 8be38a7..d042c39 100644 --- a/packages/kotlin/1.4.31/compile +++ b/packages/kotlin/1.4.31/compile @@ -1,4 +1,4 @@ #!/usr/bin/env bash # Put instructions to compile source code, remove this file if the language does not require this stage -kotlinc $* -include-runtime -d code.jar \ No newline at end of file +kotlinc "$@" -include-runtime -d code.jar \ No newline at end of file diff --git a/packages/mono/6.12.0/compile b/packages/mono/6.12.0/compile index 3151cde..746c05e 100644 --- a/packages/mono/6.12.0/compile +++ b/packages/mono/6.12.0/compile @@ -1,3 +1,3 @@ #!/bin/bash -csc $* \ No newline at end of file +csc "$@" \ No newline at end of file diff --git a/packages/mono/6.12.0/run b/packages/mono/6.12.0/run index 9d9b61a..557ce76 100644 --- a/packages/mono/6.12.0/run +++ b/packages/mono/6.12.0/run @@ -2,4 +2,4 @@ CODE=${1/cs/exe} shift -mono $CODE $* \ No newline at end of file +mono $CODE "$@" \ No newline at end of file diff --git a/packages/nasm/2.15.5/compile b/packages/nasm/2.15.5/compile index a8fe6da..e481319 100644 --- a/packages/nasm/2.15.5/compile +++ b/packages/nasm/2.15.5/compile @@ -5,11 +5,11 @@ case "${PISTON_ALIAS}" in nasm) - nasm -f elf32 -o binary.o $* + nasm -f elf32 -o binary.o "$@" ld -m elf_i386 binary.o -o binary ;; nasm64) - nasm -f elf64 -o binary.o $* + nasm -f elf64 -o binary.o "$@" ld -m elf_x86_64 binary.o -o binary ;; *) diff --git a/packages/nasm/2.15.5/run b/packages/nasm/2.15.5/run index 53a6099..f910a1c 100644 --- a/packages/nasm/2.15.5/run +++ b/packages/nasm/2.15.5/run @@ -2,4 +2,4 @@ # Put instructions to run the runtime shift -./binary $* +./binary "$@" diff --git a/packages/node/15.10.0/run b/packages/node/15.10.0/run index 004ca32..6d1fdee 100644 --- a/packages/node/15.10.0/run +++ b/packages/node/15.10.0/run @@ -1,3 +1,3 @@ #!/bin/bash -node $* \ No newline at end of file +node "$@" \ No newline at end of file diff --git a/packages/php/8.0.2/run b/packages/php/8.0.2/run index 2bce27a..1261d95 100644 --- a/packages/php/8.0.2/run +++ b/packages/php/8.0.2/run @@ -1,3 +1,3 @@ #!/bin/bash -php $* \ No newline at end of file +php "$@" \ No newline at end of file diff --git a/packages/python/3.9.1/run b/packages/python/3.9.1/run index 7b3205b..450bb76 100644 --- a/packages/python/3.9.1/run +++ b/packages/python/3.9.1/run @@ -1,3 +1,3 @@ #!/bin/bash -python3.9 $* \ No newline at end of file +python3.9 "$@" \ No newline at end of file diff --git a/packages/typescript/4.2.3/compile b/packages/typescript/4.2.3/compile index 1258d16..fdf5f19 100644 --- a/packages/typescript/4.2.3/compile +++ b/packages/typescript/4.2.3/compile @@ -2,4 +2,4 @@ # Put instructions to compile source code, remove this file if the language does not require this stage -tsc $* \ No newline at end of file +tsc "$@" \ No newline at end of file diff --git a/packages/typescript/4.2.3/run b/packages/typescript/4.2.3/run index dfd4249..1d26f3f 100644 --- a/packages/typescript/4.2.3/run +++ b/packages/typescript/4.2.3/run @@ -5,4 +5,4 @@ CODE=$(sed 's/ts$/js/' <<<"$1") shift -node $CODE $* +node $CODE "$@" diff --git a/repo/Dockerfile b/repo/Dockerfile index 606b596..231289a 100644 --- a/repo/Dockerfile +++ b/repo/Dockerfile @@ -2,10 +2,9 @@ FROM debian:buster-slim RUN apt-get update && apt-get install -y unzip autoconf build-essential libssl-dev pkg-config zlib1g-dev libargon2-dev libsodium-dev libcurl4-openssl-dev sqlite3 libsqlite3-dev libonig-dev libxml2 libxml2-dev bc curl git linux-headers-amd64 perl xz-utils python3 python3-pip gnupg jq zlib1g-dev cmake cmake-doc extra-cmake-modules build-essential gcc binutils bash coreutils util-linux pciutils usbutils coreutils binutils findutils grep && \ ln -sf /bin/bash /bin/sh && \ - pip3 install 'yq==2.12.0' && \ rm -rf /var/lib/apt/lists/* -ADD *.sh / +ADD entrypoint.sh mkindex.sh / ENTRYPOINT ["bash","/entrypoint.sh"] -CMD ["all"] +CMD ["--no-build"] diff --git a/repo/README.MD b/repo/README.MD deleted file mode 100644 index 28f833f..0000000 --- a/repo/README.MD +++ /dev/null @@ -1,7 +0,0 @@ -# Piston Filesystem Repo Builder - -This is just a simple POC for a repository tool to run locally. - -This only demonstrates building an unsigned python-3.9.1 package, however if it finds an `asc` file it will include it as the signature. - -Mount this whole directory into `/repo` in your API container if you wish to use it. \ No newline at end of file diff --git a/repo/entrypoint.sh b/repo/entrypoint.sh index 54261dc..d9fc973 100755 --- a/repo/entrypoint.sh +++ b/repo/entrypoint.sh @@ -1,11 +1,57 @@ cd /piston/packages +SERVER=1 +BUILD=1 +CI=0 + +echo "Running through arguments.." + for pkg in "$@" do - make -j16 $pkg.pkg.tar.gz + shift + if [[ "$pkg" = "--no-server" ]]; then + echo "Not starting index server after builds" + SERVER=0 + elif [[ "$pkg" = "--no-build" ]]; then + echo "Building no more package" + BUILD=0 + elif [[ "$pkg" = "--ci" ]]; then + echo "Running in CI mode, --no-build, --no-server" + BUILD=0 + SERVER=0 + CI=1 + else + if [[ $BUILD -eq 1 ]]; then + echo "Building package $pkg" + make -j16 $pkg.pkg.tar.gz + echo "Done with package $pkg" + elif [[ $CI -eq 1 ]]; then + echo "Commit SHA: $pkg" + + echo "Changed files:" + git diff --name-only $pkg^1 $pkg + + PACKAGES=$(git diff --name-only $pkg^1 $pkg | awk -F/ '{ print $2 "-" $3 }' | sort -u) + echo "Building packages: $PACKAGES" + for package in "$PACKAGES"; do + make -j16 $package.pkg.tar.gz + done + + else + echo "Building was disabled, skipping $pkg build=$BUILD ci=$CI" + fi + fi done cd /piston/repo +echo "Creating index" ./mkindex.sh +echo "Index created" -python3 -m http.server \ No newline at end of file +if [[ $SERVER -eq 1 ]]; then + echo "Starting index server.." + python3 -m http.server +else + echo "Skipping starting index server" +fi +exit 0 \ No newline at end of file