ci(package): get list of packages from pr

.github/workflows/api-push.yaml

@@ -0,0 +1,32 @@
+name: Publish API image
+on:
+  push:
+    branches:
+      - master
+      - v3
+    paths:
+      - api/**
+
+  
+jobs:
+  push_to_registry:
+    runs-on: ubuntu-latest
+    name: Build and Push Docker image to Github Packages
+    steps:
+      - name: Check out repo
+        uses: actions/checkout@v2
+      - name: Login to GitHub registry
+        uses: docker/login-action@v1
+        with:
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+          registry: docker.pkg.github.com
+
+      - name: Build and push API
+        uses: docker/build-push-action@v2
+        with:
+          context: api
+          push: true
+          pull: true
+          tags: |
+            docker.pkg.github.com/engineer-man/piston/api

.github/workflows/package-pr.yaml

@@ -11,63 +11,78 @@ on:
       - 'packages/**'
 
 jobs:
-  check-build:
+  build-pkg:
     name: Check that package builds
     runs-on: ubuntu-latest
     steps:
-      - name: Get PR Commits
-        id: 'get-pr-commits'
-        uses: tim-actions/get-pr-commits@master
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-      - name: Check subsystem
-        uses: tim-actions/commit-message-checker-with-regex@v0.3.1
-        with:
-          commits: ${{ steps.get-pr-commits.outputs.commits }}
-          pattern: '^[\s]*(pkg\([^:\s\n]+\))[\s]*:'
-          error: 'Your commit message must start with pkg([package])'
-      
-      - name: Get packages
-        id: 'get-packages'
-        shell: bash
-        run: |
-          COMMITS='${{ steps.get-pr-commits.outputs.commits }}'
-          PACKAGES=$(echo $COMMITS | jq .[].commit.message -r | grep -oP '^pkg\(\K[^:\h\n]+(?=\))' | sort -u)
-          echo "::set-output name=packages::$PACKAGES"
-
       - name: Checkout
         uses: actions/checkout@v2
+      
+      - name: Login to GitHub registry
+        uses: docker/login-action@v1
+        with:
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+          registry: docker.pkg.github.com
 
-      - name: Build docker containers
+      - name: Get list of changed files
-        run: |
+        uses: lots0logs/gh-action-get-changed-files@2.1.4
-          docker build -t piston_fs_repo repo
+        with:
-          docker build -t piston_api api
+          token: ${{ secrets.GITHUB_TOKEN }}
-          
+                
       - name: Build Packages
         run: |
-          docker run -v $(pwd)'/repo:/repo' -v $(pwd)'/packages:/packages' piston_fs_repo ${{ steps.get-packages.outputs.packages }}
+          PACKAGES=$(jq '.[]' -r ${HOME}/files.json | awk -F/ '{ print $2 "-" $3 }' | sort -u)
+          echo "Packages: $PACKAGES"
+          docker run -v "${{ github.workspace }}:/piston" docker.pkg.github.com/engineer-man/piston/repo-builder:latest --no-server $PACKAGES
+          ls -la packages
+
+      - name: Upload package as artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: packages
+          path: packages/*.pkg.tar.gz
+
+
+  test-pkg:
+    name: Test package
+    runs-on: ubuntu-latest
+    needs: build-pkg
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/download-artifact@v2
+        with:
+          name: packages
+
+      - name: Write test config file
+        uses: DamianReeves/write-file-action@v1.0
+        with:
+          path: data/config.yaml
+          contents: |
+            log_level: DEBUG
+            bind_address: 0.0.0.0:6969
+            data_directory: /piston
+            runner_uid_min: 1100
+            runner_uid_max: 1500
+            runner_gid_min: 1100
+            runner_gid_max: 1500
+            enable_unshare: false
+            output_max_size: 1024
+            max_process_count: 64
+            max_open_files: 2048
+            repo_url: http://piston_fs_repo:8000/index
+
+          write-mode: overwrite
 
       - name: Run tests
         run: |
-          docker run -p 6969:6969 -v $(pwd)'/data:/piston' -v $(pwd)'/repo:/repo' --privileged --name api piston_api & 
+          docker run -v $(pwd)'/repo:/piston/repo' -v $(pwd)'/packages:/piston/packages' --name piston_fs_repo docker.pkg.github.com/engineer-man/piston/repo-builder $PACKAGES &
+          docker run -v $(pwd)'/data:/piston' --name api docker.pkg.github.com/engineer-man/piston/api & 
           echo Waiting for API to start..
-          while [[ "$(curl -s -w '%{http_code}' http://127.0.0.1:6969/runtimes)" != "200" ]]; do sleep 5; done
+          while [[ "$(curl -s -w '%{http_code}' http://api:6969/runtimes)" != "200" ]]; do sleep 5; done
-          echo Adding local repo
+
-          curl -s http://127.0.0.1:6969/repos -XPOST -d "slug=local&url=file:///repo/index.yaml"
           echo Testing packages
-          for pkg in "$(curl -s http://127.0.0.1:6969/repos/local/packages/ | jq '.data.packages[] | "\(.language)/\(.language_version)"' -r)"
-          do
-            PKG_SLUG=${pkg/\//-}
-            PKG_NAME=$(echo $pkg | cut -d'/' -f 1)
-            PKG_VERSION=$(echo $pkg | cut -d'/' -f 2)
-            echo Installing ${PKG_SLUG}
-            curl -sXPOST http://127.0.0.1:6969/repos/local/packages/${pkg} | jq '.language' -r || exit 1
-            echo Testing ${PKG_SLUG} (using ${PKG_SLUG}.tf)
-            TEST_FILE=$(cat ${PKG_SLUG}.tf)
-            TEST_JSON=`jq -C '.language = "${PKG_NAME}" | .version = "${PKG_VERSION}" | .files=[] | .files[0]={} | .files[0].name="test" | .files[0].name.content="${TEST_FILE}" | .main = "test" | .args = [] | .stdin = "" | .compile_timeout = 10000 | .run_timeout = 3000' <<< '{}'`
-            curl -sXPOST http://127.0.0.1:6969/jobs -H 'Content-Type: application/json' -d "$TEST_JSON" > ${PKG_SLUG}.tr
-            jq '.run.stdout' ${PKG_SLUG}.tr | grep "OK" || exit 1
-          done
 
 
 

.github/workflows/repo-push.yaml

@@ -0,0 +1,31 @@
+name: Publish Repo image
+on:
+  push:
+    branches:
+      - master
+      - v3
+    paths:
+      - repo/**
+  
+jobs:
+  push_to_registry:
+    runs-on: ubuntu-latest
+    name: Build and Push Docker image to Github Packages
+    steps:
+      - name: Check out repo
+        uses: actions/checkout@v2
+      - name: Login to GitHub registry
+        uses: docker/login-action@v1
+        with:
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+          registry: docker.pkg.github.com
+
+      - name: Build and push repo
+        uses: docker/build-push-action@v2
+        with:
+          context: repo
+          pull: true
+          push: true
+          tags: |
+            docker.pkg.github.com/engineer-man/piston/repo-builder

api/src/executor/job.js

@@ -81,7 +81,8 @@ class Job {
             const proc_call = [
                 ...prlimit,
                 ...unshare,
-                'bash',file, ...args
+                'bash',file,
+                ...args
             ];
 
             var stdout = '';

packages/bash/5.1.0/run

@@ -1,4 +1,4 @@
 #!/usr/bin/env bash
 
 # Put instructions to run the runtime
-bash $*
+bash "$@"

packages/dart/2.12.1/run

@@ -1,4 +1,4 @@
 #!/usr/bin/env bash
 
 # Put instructions to run the runtime
-dart run $*
+dart run "$@"

packages/deno/1.7.5/run

@@ -1,2 +1,2 @@
 #!/bin/bash
-DENO_DIR=$PWD deno run $*
+DENO_DIR=$PWD deno run "$@"

packages/gawk/5.1.0/run

@@ -1,4 +1,4 @@
 #!/usr/bin/env bash
 
 # Put instructions to run the runtime
-gawk-5.1.0 -f $*
+gawk-5.1.0 -f "$@"

packages/gcc/10.2.0/compile

@@ -5,16 +5,16 @@
 
 case "${PISTON_ALIAS}" in
     gcc | c)
-        gcc -std=c11 $* -lm
+        gcc -std=c11 "$@" -lm
         ;;
     g++ | c++ | cpp)
-        g++ -std=c++17 $*
+        g++ -std=c++17 "$@"
         ;;
     gccgo | go)
-        gccgo $*
+        gccgo "$@"
         ;;
     gdc | d)
-        gdc $*
+        gdc "$@"
         ;;
     *)
         echo "How did you get here? (${PISTON_ALIAS})"

packages/gcc/10.2.0/run

@@ -3,4 +3,4 @@
 # Put instructions to run the runtime
 
 shift # Discard main filename
-./a.out $*
+./a.out "$@"

packages/java/15.0.2/run

@@ -1,4 +1,4 @@
 #!/usr/bin/env bash
 
 # Put instructions to run the runtime
-java $*
+java "$@"

packages/jelly/0.1.31/run

@@ -1 +1 @@
-jelly fu $*
+jelly fu "$@"

packages/kotlin/1.4.31/compile

@@ -1,4 +1,4 @@
 #!/usr/bin/env bash
 
 # Put instructions to compile source code, remove this file if the language does not require this stage
-kotlinc $* -include-runtime -d code.jar
+kotlinc "$@" -include-runtime -d code.jar

packages/mono/6.12.0/compile

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-csc $*
+csc "$@"

packages/mono/6.12.0/run

@@ -2,4 +2,4 @@
 
 CODE=${1/cs/exe}
 shift
-mono $CODE $*
+mono $CODE "$@"

packages/nasm/2.15.5/compile

@@ -5,11 +5,11 @@
 
 case "${PISTON_ALIAS}" in
     nasm)
-        nasm -f elf32 -o binary.o $*
+        nasm -f elf32 -o binary.o "$@"
         ld -m elf_i386 binary.o -o binary
         ;;
     nasm64)
-        nasm -f elf64 -o binary.o $*
+        nasm -f elf64 -o binary.o "$@"
         ld -m elf_x86_64 binary.o -o binary
         ;;
     *)

packages/nasm/2.15.5/run

@@ -2,4 +2,4 @@
 
 # Put instructions to run the runtime
 shift
-./binary $*
+./binary "$@"

packages/node/15.10.0/run

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-node $*
+node "$@"

packages/php/8.0.2/run

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-php $*
+php "$@"

packages/python/3.9.1/run

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-python3.9 $*
+python3.9 "$@"

packages/typescript/4.2.3/compile

@@ -2,4 +2,4 @@
 
 # Put instructions to compile source code, remove this file if the language does not require this stage
 
-tsc $*
+tsc "$@"

packages/typescript/4.2.3/run

@@ -5,4 +5,4 @@
 CODE=$(sed 's/ts$/js/' <<<"$1")
 shift
 
-node $CODE $*
+node $CODE "$@"

repo/Dockerfile

@@ -2,10 +2,9 @@ FROM debian:buster-slim
 
 RUN apt-get update && apt-get install -y unzip autoconf build-essential libssl-dev pkg-config zlib1g-dev libargon2-dev libsodium-dev libcurl4-openssl-dev sqlite3 libsqlite3-dev libonig-dev libxml2 libxml2-dev bc curl git linux-headers-amd64 perl xz-utils python3 python3-pip gnupg jq zlib1g-dev cmake cmake-doc extra-cmake-modules build-essential gcc binutils bash coreutils util-linux pciutils usbutils coreutils binutils findutils grep && \
         ln -sf /bin/bash /bin/sh && \
-        pip3 install 'yq==2.12.0' && \
         rm -rf /var/lib/apt/lists/*
 
-ADD *.sh /
+ADD entrypoint.sh mkindex.sh /
 
 ENTRYPOINT ["bash","/entrypoint.sh"]
-CMD ["all"]
+CMD ["--no-build"]

repo/README.MD

@@ -1,7 +0,0 @@
-# Piston Filesystem Repo Builder
-
-This is just a simple POC for a repository tool to run locally.
-
-This only demonstrates building an unsigned python-3.9.1 package, however if it finds an `asc` file it will include it as the signature.
-
-Mount this whole directory into `/repo` in your API container if you wish to use it.

repo/entrypoint.sh

@@ -1,11 +1,57 @@
 cd /piston/packages
 
+SERVER=1
+BUILD=1
+CI=0
+
+echo "Running through arguments.."
+
 for pkg in "$@"
 do
-    make -j16 $pkg.pkg.tar.gz
+    shift
+    if [[ "$pkg" = "--no-server" ]]; then
+        echo "Not starting index server after builds"
+        SERVER=0
+    elif [[ "$pkg" = "--no-build" ]]; then
+        echo "Building no more package"
+        BUILD=0
+    elif [[ "$pkg" = "--ci" ]]; then
+        echo "Running in CI mode, --no-build, --no-server"
+        BUILD=0
+        SERVER=0
+        CI=1
+    else
+        if [[ $BUILD -eq 1 ]]; then
+            echo "Building package $pkg"
+            make -j16 $pkg.pkg.tar.gz
+            echo "Done with package $pkg"
+        elif [[ $CI -eq 1 ]]; then
+            echo "Commit SHA: $pkg"
+            
+            echo "Changed files:"
+            git diff --name-only $pkg^1 $pkg
+            
+            PACKAGES=$(git diff --name-only $pkg^1 $pkg | awk -F/ '{ print $2 "-" $3 }' | sort -u)
+            echo "Building packages: $PACKAGES"
+            for package in "$PACKAGES"; do
+                make -j16 $package.pkg.tar.gz
+            done
+
+        else
+            echo "Building was disabled, skipping $pkg build=$BUILD ci=$CI"
+        fi
+    fi
 done
 
 cd /piston/repo
+echo "Creating index"
 ./mkindex.sh
+echo "Index created"
 
-python3 -m http.server
+if [[ $SERVER -eq 1 ]]; then
+    echo "Starting index server.."
+    python3 -m http.server
+else
+    echo "Skipping starting index server"
+fi
+exit 0