Merge remote-tracking branch 'upstream/v3'

.github/workflows/api-push.yaml

@@ -21,13 +21,12 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
           registry: docker.pkg.github.com
-      - name: Pull existing
-        run: docker pull docker.pkg.github.com/engineer-man/piston/api
 
       - name: Build and push API
         uses: docker/build-push-action@v2
         with:
           context: api
           push: true
+          pull: true
           tags: |
             docker.pkg.github.com/engineer-man/piston/api

.github/workflows/package-pr.yaml

@@ -25,13 +25,17 @@ jobs:
           password: ${{ secrets.GITHUB_TOKEN }}
           registry: docker.pkg.github.com
 
+      - name: Get list of changed files
+        uses: lots0logs/gh-action-get-changed-files@2.1.4
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+                
       - name: Build Packages
         run: |
-          SHALIST=$(curl "https://api.github.com/repos/$GITHUB_REPOSITORY/pulls/$(jq -r '.pull_request.number' $GITHUB_EVENT_PATH)"/commits | jq '.[].commit.tree.sha' -r)
+          PACKAGES=$(jq '.[]' -r ${HOME}/files.json | awk -F/ '{ print $2 "-" $3 }' | sort -u)
-          echo "Commits: $SHALIST"
+          echo "Packages: $PACKAGES"
-          git log
+          docker run -v "${{ github.workspace }}:/piston" docker.pkg.github.com/engineer-man/piston/repo-builder:latest --no-server $PACKAGES
-          docker run -v "${{ github.workspace }}:/piston" docker.pkg.github.com/engineer-man/piston/repo-builder:latest --ci $SHALIST
+          ls -la packages
-          ls -la packages repo
 
       - name: Upload package as artifact
         uses: actions/upload-artifact@v2
@@ -51,6 +55,9 @@ jobs:
         with:
           name: packages
       
+      - name: Relocate downloaded packages
+        run: mv *.pkg.tar.gz packages/
+
       - name: Write test config file
         uses: DamianReeves/write-file-action@v1.0
         with:
@@ -67,18 +74,67 @@ jobs:
             output_max_size: 1024
             max_process_count: 64
             max_open_files: 2048
-            repo_url: http://piston_fs_repo:8000/index
+            repo_url: http://localhost:8000/index
 
           write-mode: overwrite
 
+      - name: Login to GitHub registry
+        uses: docker/login-action@v1
+        with:
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+          registry: docker.pkg.github.com
+
       - name: Run tests
         run: |
-          docker run -v $(pwd)'/repo:/piston/repo' -v $(pwd)'/packages:/piston/packages' --name piston_fs_repo docker.pkg.github.com/engineer-man/piston/repo-builder $PACKAGES &
+          ls -la
-          docker run -v $(pwd)'/data:/piston' --name api docker.pkg.github.com/engineer-man/piston/api & 
+          docker run -v $(pwd)'/repo:/piston/repo' -v $(pwd)'/packages:/piston/packages' -d --name piston_fs_repo docker.pkg.github.com/engineer-man/piston/repo-builder --no-build
+          docker run --network container:piston_fs_repo -v $(pwd)'/data:/piston' -d --name api docker.pkg.github.com/engineer-man/piston/api
           echo Waiting for API to start..
-          while [[ "$(curl -s -w '%{http_code}' http://api:6969/runtimes)" != "200" ]]; do sleep 5; done
+          docker run --network container:api appropriate/curl -s --retry 10 --retry-connrefused http://localhost:6969/runtimes
+
+          echo Waiting for Index to start..
+          docker run --network container:piston_fs_repo appropriate/curl -s --retry 10 --retry-connrefused http://localhost:8000/index
+
+          echo Adjusting index
+          sed -i 's/piston_fs_repo/localhost/g' repo/index
+
+          echo Listing Packages
+          PACKAGES_JSON=$(docker run --network container:api appropriate/curl -s http://localhost:6969/packages)
+          echo $PACKAGES_JSON
+
+          echo Getting CLI ready
+          docker run  -v "$PWD/cli:/app" --entrypoint /bin/bash node:15 -c 'cd /app; yarn'
+
+          for package in $(jq -r '.[] | "\(.language)-\(.language_version)"' <<< "$PACKAGES_JSON")
+          do
+            echo "Testing $package"
+            PKG_PATH=$(sed 's|-|/|' <<< $package)
+            PKG_NAME=$(awk -F- '{ print $1 }' <<< $package)
+            PKG_VERSION=$(awk -F- '{ print $2 }' <<< $package)
+
+            echo "Installing..."
+            docker run --network container:api appropriate/curl -sXPOST http://localhost:6969/packages/$PKG_PATH
+
+            TEST_SCRIPTS=packages/$PKG_PATH/test.*
+            echo "Tests: $TEST_SCRIPTS"
+
+            for tscript in $TEST_SCRIPTS
+            do
+              TEST_RUNTIME=$(awk -F. '{print $2}' <<< $(basename $tscript))
+              echo Running $tscript with runtime=$TEST_RUNTIME
+              docker run --network container:api -v "$PWD/cli:/app" -v "$PWD/$(dirname $tscript):/pkg" node:15 /app/index.js run $TEST_RUNTIME $PKG_VERSION /pkg/$(basename $tscript) | grep OK
+            done
+          done
+
+
+      - name: Dump logs
+        if: ${{ always() }}
+        run: |
+          docker logs api
+          docker logs piston_fs_repo
+
 
-          echo Testing packages
 
 
 

.github/workflows/package-push.yaml

@@ -0,0 +1,76 @@
+name: 'Package Pushed'
+
+on:
+  push:
+    branches:
+      - master
+      - v3
+    paths:
+      - packages/**
+
+
+jobs:
+  build-pkg:
+    name: Build package
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      
+      - name: Login to GitHub registry
+        uses: docker/login-action@v1
+        with:
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+          registry: docker.pkg.github.com
+
+      - name: Get list of changed files
+        uses: lots0logs/gh-action-get-changed-files@2.1.4
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+                
+      - name: Build Packages
+        run: |
+          PACKAGES=$(jq '.[]' -r ${HOME}/files.json | awk -F/ '{ print $2 "-" $3 }' | sort -u)
+          echo "Packages: $PACKAGES"
+          docker run -v "${{ github.workspace }}:/piston" docker.pkg.github.com/engineer-man/piston/repo-builder:latest --no-server $PACKAGES
+          ls -la packages
+
+      - name: Upload Packages
+        uses: svenstaro/upload-release-action@v2
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          file: packages/*.pkg.tar.gz
+          tag: pkgs
+          overwrite: true
+          file_glob: true
+  create-index:
+    name: Create Index
+    runs-on: ubuntu-latest
+    needs: build-pkg
+    steps:
+      - name: "Download all release assets"
+        run: curl -s https://api.github.com/repos/engineer-man/piston/releases/latest | jq '.assets[].browser_download_url' -r | xargs -L 1 curl -sLO
+      - name: "Generate index file"
+        run: |
+          echo "" > index
+          BASEURL=https://github.com/engineer-man/piston/releases/download/pkgs/
+          for pkg in *.pkg.tar.gz
+          do
+            PKGFILE=$(basename $pkg)
+            PKGFILENAME=$(echo $PKGFILE | sed 's/\.pkg\.tar\.gz//g')
+
+            PKGNAME=$(echo $PKGFILENAME | grep -oP '^\K.+(?=-)')
+            PKGVERSION=$(echo $PKGFILENAME | grep -oP '^.+-\K.+')
+            PKGCHECKSUM=$(sha256sum $PKGFILE | awk '{print $1}')
+            echo "$PKGNAME,$PKGVERSION,$PKGCHECKSUM,$BASEURL$PKGFILE" >> index
+            echo "Adding package $PKGNAME-$PKGVERSION"
+          done
+      - name: Upload index
+        uses: svenstaro/upload-release-action@v2
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          file: index
+          tag: pkgs
+          overwrite: true
+          file_glob: true

.github/workflows/repo-push.yaml

@@ -20,13 +20,12 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
           registry: docker.pkg.github.com
-      - name: Pull existing
-        run: docker pull docker.pkg.github.com/engineer-man/piston/repo-builder
 
       - name: Build and push repo
         uses: docker/build-push-action@v2
         with:
           context: repo
+          pull: true
           push: true
           tags: |
             docker.pkg.github.com/engineer-man/piston/repo-builder

api/src/config.js

@@ -57,7 +57,7 @@ const options = [
     {
         key: 'runner_uid_min',
         desc: 'Minimum uid to use for runner',
-        default: 1000,
+        default: 1001,
         validators: []
     },
     {
@@ -69,7 +69,7 @@ const options = [
     {
         key: 'runner_gid_min',
         desc: 'Minimum gid to use for runner',
-        default: 1000,
+        default: 1001,
         validators: []
     },
     {
@@ -105,7 +105,7 @@ const options = [
     {
         key: 'repo_url',
         desc: 'URL of repo index',
-        default: 'https://github.com',
+        default: 'https://github.com/engineer-man/piston/releases/download/pkgs/index',
         validators: []
     }
 ];

cli/commands/execute.js

@@ -52,7 +52,7 @@ exports.handler = async function(argv){
         language: argv.language,
         version: argv['language-version'],
         files: files,
-        main: argv.file,
+        main: path.basename(argv.file),
         args: argv.args,
         stdin,
         compile_timeout: argv.ct,
@@ -85,6 +85,8 @@ exports.handler = async function(argv){
                 )
     }
 
+    
+    
     if(response.compile) step('Compile', response.compile)
     step('Run', response.run)
     

packages/Makefile

@@ -11,10 +11,11 @@ build build-all: $(addsuffix .pkg.tar.gz, ${PACKAGES})
 
 
 define PKG_RULE
-$(1).pkg.tar.gz: $(subst -,/,$(1))/ $(subst -,/,$(1))/pkg-info.json
+$(1).pkg.tar.gz: $(subst -,/,$(1)) $(subst -,/,$(1))/pkg-info.json
 	cd $$< && chmod +x ./build.sh && ./build.sh
 	rm -f $$@
-	tar czf $$@ $$< --transform='s|$$<||'
+
+	tar czf $$@ -C $$< .
 endef
 
 $(foreach pkg,$(PACKAGES),$(eval $(call PKG_RULE,$(pkg))))

packages/bash/5.1.0/build.sh

@@ -18,3 +18,4 @@ make -j$(nproc)
 make install -j$(nproc)
 cd ../
 rm -rf build
+

packages/gawk/5.1.0/build.sh

@@ -18,3 +18,4 @@ make -j$(nproc)
 make install -j$(nproc)
 cd ../
 rm -rf build
+

packages/gcc/10.2.0/build.sh

@@ -23,3 +23,4 @@ make -j$(nproc)
 make install -j$(nproc)
 cd ../
 rm -rf build obj
+

packages/go/1.16.2/build.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+curl -LO https://golang.org/dl/go1.16.2.linux-amd64.tar.gz
+tar -xzf go1.16.2.linux-amd64.tar.gz
+rm go1.16.2.linux-amd64.tar.gz
+

packages/go/1.16.2/environment

@@ -0,0 +1,2 @@
+export PATH=$PWD/go/bin:$PATH
+export GOPATH=$PWD/gopath

packages/go/1.16.2/metadata.json

@@ -0,0 +1,6 @@
+{
+    "language": "go",
+    "version": "1.16.2",
+    "author": "Victor Frazao <luiz_victor_frazao@hotmail.com>",
+    "aliases": ["go", "golang"]
+}

packages/go/1.16.2/run

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+GOCACHE=$PWD go run "$@"

packages/go/1.16.2/test.go

@@ -0,0 +1,7 @@
+package main
+
+import "fmt"
+
+func main() {
+	fmt.Println("OK")
+}

packages/java/15.0.2/build.sh

@@ -6,3 +6,4 @@ curl "https://download.java.net/java/GA/jdk15.0.2/0d1cfde4252546c6931946de8db48e
 
 tar xzf java.tar.gz --strip-components=1
 rm java.tar.gz
+

packages/julia/1.5.4/build.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+curl -OL https://julialang-s3.julialang.org/bin/linux/x64/1.5/julia-1.5.4-linux-x86_64.tar.gz
+tar zxvf julia-1.5.4-linux-x86_64.tar.gz
+rm julia-1.5.4-linux-x86_64.tar.gz
+

packages/julia/1.5.4/environment

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+export PATH=$PWD/julia-1.5.4/bin:$PATH

packages/julia/1.5.4/metadata.json

@@ -0,0 +1,6 @@
+{
+  "language": "julia",
+  "version": "1.5.4",
+  "aliases": ["jl"],
+  "author": "Victor Frazao <luiz_victor_frazao@hotmail.com>"
+}

packages/julia/1.5.4/run

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+# Put instructions to run the runtime
+julia --startup-file=no --history-file=no "$@"

packages/julia/1.5.4/test.jl

@@ -0,0 +1 @@
+println("OK")

packages/mono/6.12.0/build.sh

@@ -17,3 +17,4 @@ make install -j$(nproc)
 
 cd ../../
 rm -rf build
+

packages/nasm/2.15.5/build.sh

@@ -18,3 +18,4 @@ make -j$(nproc)
 make install -j$(nproc)
 cd ../
 rm -rf build
+

packages/php/8.0.2/environment

@@ -1 +1 @@
-export PATH=$PWD/php-8.0.2/bin:$PATH
+export PATH=$PWD/bin:$PATH

packages/python/3.9.1/build.sh

@@ -18,3 +18,4 @@ cd ..
 
 rm -rf build 
 
+

packages/ruby/2.5.1/build.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+PREFIX=$(realpath $(dirname $0))
+
+mkdir -p build
+
+cd build
+
+curl "https://cache.ruby-lang.org/pub/ruby/2.5/ruby-2.5.1.tar.gz" -o ruby.tar.gz
+tar xzf ruby.tar.gz --strip-components=1
+rm ruby.tar.gz
+
+./configure --prefix "$PREFIX"
+make -j$(nproc)
+make install -j$(nproc)
+
+cd ..
+
+rm -rf build 
+

packages/ruby/2.5.1/environment

@@ -0,0 +1 @@
+export PATH=$PWD/bin:$PATH

packages/ruby/2.5.1/metadata.json

@@ -0,0 +1,6 @@
+{
+    "language": "ruby",
+    "version": "2.5.1",
+    "author": "Dan Vargas <danvargas46@gmail.com>",
+    "aliases": ["ruby", "rb"]
+}

packages/ruby/2.5.1/run

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+ruby $*

packages/ruby/2.5.1/test.rb

@@ -0,0 +1 @@
+puts("OK")

repo/entrypoint.sh

@@ -2,8 +2,7 @@ cd /piston/packages
 
 SERVER=1
 BUILD=1
-
+CI=0
-ls -la /piston /piston/*
 
 echo "Running through arguments.."
 
@@ -20,22 +19,28 @@ do
         echo "Running in CI mode, --no-build, --no-server"
         BUILD=0
         SERVER=0
-        SHA=$1
+        CI=1
-        shift
-        echo "Commit Sha: $SHA"
-        PACKAGES=$(git diff-tree --no-commit-id --name-only -r $SHA | awk -F/ '{ print $2 "-" $3 }' | sort -u)
-        echo "Building packages: $PACKAGES"
-        for package in "$PACKAGES"; do
-            make -j16 $package.pkg.tar.gz
-        done
-
     else
         if [[ $BUILD -eq 1 ]]; then
             echo "Building package $pkg"
-            make -j16 $pkg.pkg.tar.gz
+            make -j16 $pkg.pkg.tar.gz PLATFORM=docker-debian
             echo "Done with package $pkg"
+        elif [[ $CI -eq 1 ]]; then
+            echo "Commit SHA: $pkg"
+            
+            cd ..
+            echo "Changed files:"
+            git diff --name-only $pkg^1 $pkg
+            PACKAGES=$(git diff --name-only $pkg^1 $pkg | awk -F/ '{ print $2 "-" $3 }' | sort -u)
+            cd packages
+
+            echo "Building packages: $PACKAGES"
+            for package in "$PACKAGES"; do
+                make -j16 $package.pkg.tar.gz PLATFORM=docker-debian
+            done
+
         else
-            echo "Building was disabled, skipping $pkg"
+            echo "Building was disabled, skipping $pkg build=$BUILD ci=$CI"
         fi
     fi
 done

repo/mkindex.sh

@@ -17,6 +17,7 @@ do
     PKGCHECKSUM=$(sha256sum $PKGFILE | awk '{print $1}')
 
     echo "$PKGNAME,$PKGVERSION,$PKGCHECKSUM,$BASEURL$PKGFILE" >> index
+    echo "Adding package $PKGNAME-$PKGVERSION"
     
     ((i=i+1))
 done