.github/workflows/api-push.yaml

@@ -1,32 +0,0 @@
-name: Publish API image
-on:
-  push:
-    branches:
-      - master
-      - v3
-    paths:
-      - api/**
-
-  
-jobs:
-  push_to_registry:
-    runs-on: ubuntu-latest
-    name: Build and Push Docker image to Github Packages
-    steps:
-      - name: Check out repo
-        uses: actions/checkout@v2
-      - name: Login to GitHub registry
-        uses: docker/login-action@v1
-        with:
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-          registry: docker.pkg.github.com
-
-      - name: Build and push API
-        uses: docker/build-push-action@v2
-        with:
-          context: api
-          push: true
-          pull: true
-          tags: |
-            docker.pkg.github.com/engineer-man/piston/api

.github/workflows/package-pr.yaml

@@ -11,78 +11,63 @@ on:
       - 'packages/**'
 
 jobs:
-  build-pkg:
+  check-build:
     name: Check that package builds
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-      
-      - name: Login to GitHub registry
-        uses: docker/login-action@v1
-        with:
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-          registry: docker.pkg.github.com
-
-      - name: Get list of changed files
-        uses: lots0logs/gh-action-get-changed-files@2.1.4
+      - name: Get PR Commits
+        id: 'get-pr-commits'
+        uses: tim-actions/get-pr-commits@master
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
-                
+      - name: Check subsystem
+        uses: tim-actions/commit-message-checker-with-regex@v0.3.1
+        with:
+          commits: ${{ steps.get-pr-commits.outputs.commits }}
+          pattern: '^[\s]*(pkg\([^:\s\n]+\))[\s]*:'
+          error: 'Your commit message must start with pkg([package])'
+      
+      - name: Get packages
+        id: 'get-packages'
+        shell: bash
+        run: |
+          COMMITS='${{ steps.get-pr-commits.outputs.commits }}'
+          PACKAGES=$(echo $COMMITS | jq .[].commit.message -r | grep -oP '^pkg\(\K[^:\h\n]+(?=\))' | sort -u)
+          echo "::set-output name=packages::$PACKAGES"
+
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Build docker containers
+        run: |
+          docker build -t piston_fs_repo repo
+          docker build -t piston_api api
+          
       - name: Build Packages
         run: |
-          PACKAGES=$(jq '.[]' -r ${HOME}/files.json | awk -F/ '{ print $2 "-" $3 }' | sort -u)
-          echo "Packages: $PACKAGES"
-          docker run -v "${{ github.workspace }}:/piston" docker.pkg.github.com/engineer-man/piston/repo-builder:latest --no-server $PACKAGES
-          ls -la packages
-
-      - name: Upload package as artifact
-        uses: actions/upload-artifact@v2
-        with:
-          name: packages
-          path: packages/*.pkg.tar.gz
-
-
-  test-pkg:
-    name: Test package
-    runs-on: ubuntu-latest
-    needs: build-pkg
-    steps:
-      - uses: actions/checkout@v2
-
-      - uses: actions/download-artifact@v2
-        with:
-          name: packages
-
-      - name: Write test config file
-        uses: DamianReeves/write-file-action@v1.0
-        with:
-          path: data/config.yaml
-          contents: |
-            log_level: DEBUG
-            bind_address: 0.0.0.0:6969
-            data_directory: /piston
-            runner_uid_min: 1100
-            runner_uid_max: 1500
-            runner_gid_min: 1100
-            runner_gid_max: 1500
-            enable_unshare: false
-            output_max_size: 1024
-            max_process_count: 64
-            max_open_files: 2048
-            repo_url: http://piston_fs_repo:8000/index
-
-          write-mode: overwrite
+          docker run -v $(pwd)'/repo:/repo' -v $(pwd)'/packages:/packages' piston_fs_repo ${{ steps.get-packages.outputs.packages }}
 
       - name: Run tests
         run: |
-          docker run -v $(pwd)'/repo:/piston/repo' -v $(pwd)'/packages:/piston/packages' --name piston_fs_repo docker.pkg.github.com/engineer-man/piston/repo-builder $PACKAGES &
-          docker run -v $(pwd)'/data:/piston' --name api docker.pkg.github.com/engineer-man/piston/api & 
+          docker run -p 6969:6969 -v $(pwd)'/data:/piston' -v $(pwd)'/repo:/repo' --privileged --name api piston_api & 
           echo Waiting for API to start..
-          while [[ "$(curl -s -w '%{http_code}' http://api:6969/runtimes)" != "200" ]]; do sleep 5; done
-
+          while [[ "$(curl -s -w '%{http_code}' http://127.0.0.1:6969/runtimes)" != "200" ]]; do sleep 5; done
+          echo Adding local repo
+          curl -s http://127.0.0.1:6969/repos -XPOST -d "slug=local&url=file:///repo/index.yaml"
           echo Testing packages
+          for pkg in "$(curl -s http://127.0.0.1:6969/repos/local/packages/ | jq '.data.packages[] | "\(.language)/\(.language_version)"' -r)"
+          do
+            PKG_SLUG=${pkg/\//-}
+            PKG_NAME=$(echo $pkg | cut -d'/' -f 1)
+            PKG_VERSION=$(echo $pkg | cut -d'/' -f 2)
+            echo Installing ${PKG_SLUG}
+            curl -sXPOST http://127.0.0.1:6969/repos/local/packages/${pkg} | jq '.language' -r || exit 1
+            echo Testing ${PKG_SLUG} (using ${PKG_SLUG}.tf)
+            TEST_FILE=$(cat ${PKG_SLUG}.tf)
+            TEST_JSON=`jq -C '.language = "${PKG_NAME}" | .version = "${PKG_VERSION}" | .files=[] | .files[0]={} | .files[0].name="test" | .files[0].name.content="${TEST_FILE}" | .main = "test" | .args = [] | .stdin = "" | .compile_timeout = 10000 | .run_timeout = 3000' <<< '{}'`
+            curl -sXPOST http://127.0.0.1:6969/jobs -H 'Content-Type: application/json' -d "$TEST_JSON" > ${PKG_SLUG}.tr
+            jq '.run.stdout' ${PKG_SLUG}.tr | grep "OK" || exit 1
+          done
 
 
 

.github/workflows/repo-push.yaml

@@ -1,31 +0,0 @@
-name: Publish Repo image
-on:
-  push:
-    branches:
-      - master
-      - v3
-    paths:
-      - repo/**
-  
-jobs:
-  push_to_registry:
-    runs-on: ubuntu-latest
-    name: Build and Push Docker image to Github Packages
-    steps:
-      - name: Check out repo
-        uses: actions/checkout@v2
-      - name: Login to GitHub registry
-        uses: docker/login-action@v1
-        with:
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-          registry: docker.pkg.github.com
-
-      - name: Build and push repo
-        uses: docker/build-push-action@v2
-        with:
-          context: repo
-          pull: true
-          push: true
-          tags: |
-            docker.pkg.github.com/engineer-man/piston/repo-builder

api/src/executor/job.js

@@ -81,8 +81,7 @@ class Job {
             const proc_call = [
                 ...prlimit,
                 ...unshare,
-                'bash',file,
-                ...args
+                'bash',file, ...args
             ];
 
             var stdout = '';

packages/bash/5.1.0/run

@@ -1,4 +1,4 @@
 #!/usr/bin/env bash
 
 # Put instructions to run the runtime
-bash "$@"
+bash $*

packages/dart/2.12.1/run

@@ -1,4 +1,4 @@
 #!/usr/bin/env bash
 
 # Put instructions to run the runtime
-dart run "$@"
+dart run $*

packages/deno/1.7.5/run

@@ -1,2 +1,2 @@
 #!/bin/bash
-DENO_DIR=$PWD deno run "$@"
+DENO_DIR=$PWD deno run $*

packages/gawk/5.1.0/run

@@ -1,4 +1,4 @@
 #!/usr/bin/env bash
 
 # Put instructions to run the runtime
-gawk-5.1.0 -f "$@"
+gawk-5.1.0 -f $*

packages/gcc/10.2.0/compile

@@ -5,16 +5,16 @@
 
 case "${PISTON_ALIAS}" in
     gcc | c)
-        gcc -std=c11 "$@" -lm
+        gcc -std=c11 $* -lm
         ;;
     g++ | c++ | cpp)
-        g++ -std=c++17 "$@"
+        g++ -std=c++17 $*
         ;;
     gccgo | go)
-        gccgo "$@"
+        gccgo $*
         ;;
     gdc | d)
-        gdc "$@"
+        gdc $*
         ;;
     *)
         echo "How did you get here? (${PISTON_ALIAS})"

packages/gcc/10.2.0/run

@@ -3,4 +3,4 @@
 # Put instructions to run the runtime
 
 shift # Discard main filename
-./a.out "$@"
+./a.out $*

packages/java/15.0.2/run

@@ -1,4 +1,4 @@
 #!/usr/bin/env bash
 
 # Put instructions to run the runtime
-java "$@"
+java $*

packages/jelly/0.1.31/run

@@ -1 +1 @@
-jelly fu "$@"
+jelly fu $*

packages/kotlin/1.4.31/compile

@@ -1,4 +1,4 @@
 #!/usr/bin/env bash
 
 # Put instructions to compile source code, remove this file if the language does not require this stage
-kotlinc "$@" -include-runtime -d code.jar
+kotlinc $* -include-runtime -d code.jar

packages/mono/6.12.0/compile

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-csc "$@"
+csc $*

packages/mono/6.12.0/run

@@ -2,4 +2,4 @@
 
 CODE=${1/cs/exe}
 shift
-mono $CODE "$@"
+mono $CODE $*

packages/nasm/2.15.5/compile

@@ -5,11 +5,11 @@
 
 case "${PISTON_ALIAS}" in
     nasm)
-        nasm -f elf32 -o binary.o "$@"
+        nasm -f elf32 -o binary.o $*
         ld -m elf_i386 binary.o -o binary
         ;;
     nasm64)
-        nasm -f elf64 -o binary.o "$@"
+        nasm -f elf64 -o binary.o $*
         ld -m elf_x86_64 binary.o -o binary
         ;;
     *)

packages/nasm/2.15.5/run

@@ -2,4 +2,4 @@
 
 # Put instructions to run the runtime
 shift
-./binary "$@"
+./binary $*

packages/node/15.10.0/run

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-node "$@"
+node $*

packages/php/8.0.2/run

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-php "$@"
+php $*

packages/python/3.9.1/run

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-python3.9 "$@"
+python3.9 $*

packages/typescript/4.2.3/compile

@@ -2,4 +2,4 @@
 
 # Put instructions to compile source code, remove this file if the language does not require this stage
 
-tsc "$@"
+tsc $*

packages/typescript/4.2.3/run

@@ -5,4 +5,4 @@
 CODE=$(sed 's/ts$/js/' <<<"$1")
 shift
 
-node $CODE "$@"
+node $CODE $*

repo/Dockerfile

@@ -2,9 +2,10 @@ FROM debian:buster-slim
 
 RUN apt-get update && apt-get install -y unzip autoconf build-essential libssl-dev pkg-config zlib1g-dev libargon2-dev libsodium-dev libcurl4-openssl-dev sqlite3 libsqlite3-dev libonig-dev libxml2 libxml2-dev bc curl git linux-headers-amd64 perl xz-utils python3 python3-pip gnupg jq zlib1g-dev cmake cmake-doc extra-cmake-modules build-essential gcc binutils bash coreutils util-linux pciutils usbutils coreutils binutils findutils grep && \
         ln -sf /bin/bash /bin/sh && \
+        pip3 install 'yq==2.12.0' && \
         rm -rf /var/lib/apt/lists/*
 
-ADD entrypoint.sh mkindex.sh /
+ADD *.sh /
 
 ENTRYPOINT ["bash","/entrypoint.sh"]
-CMD ["--no-build"]
+CMD ["all"]

repo/README.MD

@@ -0,0 +1,7 @@
+# Piston Filesystem Repo Builder
+
+This is just a simple POC for a repository tool to run locally.
+
+This only demonstrates building an unsigned python-3.9.1 package, however if it finds an `asc` file it will include it as the signature.
+
+Mount this whole directory into `/repo` in your API container if you wish to use it.

repo/entrypoint.sh

@@ -1,57 +1,11 @@
 cd /piston/packages
 
-SERVER=1
-BUILD=1
-CI=0
-
-echo "Running through arguments.."
-
 for pkg in "$@"
 do
-    shift
-    if [[ "$pkg" = "--no-server" ]]; then
-        echo "Not starting index server after builds"
-        SERVER=0
-    elif [[ "$pkg" = "--no-build" ]]; then
-        echo "Building no more package"
-        BUILD=0
-    elif [[ "$pkg" = "--ci" ]]; then
-        echo "Running in CI mode, --no-build, --no-server"
-        BUILD=0
-        SERVER=0
-        CI=1
-    else
-        if [[ $BUILD -eq 1 ]]; then
-            echo "Building package $pkg"
-            make -j16 $pkg.pkg.tar.gz
-            echo "Done with package $pkg"
-        elif [[ $CI -eq 1 ]]; then
-            echo "Commit SHA: $pkg"
-            
-            echo "Changed files:"
-            git diff --name-only $pkg^1 $pkg
-            
-            PACKAGES=$(git diff --name-only $pkg^1 $pkg | awk -F/ '{ print $2 "-" $3 }' | sort -u)
-            echo "Building packages: $PACKAGES"
-            for package in "$PACKAGES"; do
-                make -j16 $package.pkg.tar.gz
-            done
-
-        else
-            echo "Building was disabled, skipping $pkg build=$BUILD ci=$CI"
-        fi
-    fi
+    make -j16 $pkg.pkg.tar.gz
 done
 
 cd /piston/repo
-echo "Creating index"
 ./mkindex.sh
-echo "Index created"
 
-if [[ $SERVER -eq 1 ]]; then
-    echo "Starting index server.."
-    python3 -m http.server
-else
-    echo "Skipping starting index server"
-fi
-exit 0
+python3 -m http.server