ci(package-pr): use absolute paths in docker

.github/workflows/package-pr.yaml

@@ -0,0 +1,75 @@
+name: 'Package Pull Requests'
+
+on:
+  pull_request:
+    types:
+      - opened
+      - edited
+      - reopened
+      - synchronize
+    paths:
+      - 'packages/**'
+
+jobs:
+  check-build:
+    name: Check that package builds
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get PR Commits
+        id: 'get-pr-commits'
+        uses: tim-actions/get-pr-commits@master
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Check subsystem
+        uses: tim-actions/commit-message-checker-with-regex@v0.3.1
+        with:
+          commits: ${{ steps.get-pr-commits.outputs.commits }}
+          pattern: '^[\s]*(pkg\([^:\s\n]+\))[\s]*:'
+          error: 'Your commit message must start with pkg([package])'
+      
+      - name: Get packages
+        id: 'get-packages'
+        shell: bash
+        run: |
+          COMMITS='${{ steps.get-pr-commits.outputs.commits }}'
+          PACKAGES=$(echo $COMMITS | jq .[].commit.message -r | grep -oP '^pkg\(\K[^:\h\n]+(?=\))' | sort -u)
+          echo "::set-output name=packages::$PACKAGES"
+
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Build docker containers
+        run: |
+          docker build -t piston_fs_repo repo
+          docker build -t piston_api api
+          
+      - name: Build Packages
+        run: |
+          docker run -v $(pwd)'/repo:/repo' -v $(pwd)'/packages:/packages' piston_fs_repo ${{ steps.get-packages.outputs.packages }}
+
+      - name: Run tests
+        run: |
+          docker run -dp 6969:6969 -v $(pwd)'/repo:/repo' --privileged --name api piston_api
+          echo Waiting for API to start..
+          bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:6969/runtimes)" != "200" ]]; do sleep 5; done'
+          echo Adding local repo
+          curl -s http://127.0.0.1:6969/repos -XPOST -d "slug=local&url=file:///repo/index.yaml"
+          echo Testing packages
+          for pkg in "$(curl -s http://127.0.0.1:6969/repos/local/packages/ | jq '.data.packages[] | "\(.language)/\(.language_version)"' -r)"
+          do
+            PKG_SLUG=${pkg/\//-}
+            PKG_NAME=$(echo $pkg | cut -d'/' -f 1)
+            PKG_VERSION=$(echo $pkg | cut -d'/' -f 2)
+            echo Installing ${PKG_SLUG}
+            curl -sXPOST http://127.0.0.1:6969/repos/local/packages/${pkg} | jq '.language' -r || exit 1
+            echo Testing ${PKG_SLUG} (using ${PKG_SLUG}.tf)
+            TEST_FILE=$(cat ${PKG_SLUG}.tf)
+            TEST_JSON=`jq -C '.language = "${PKG_NAME}" | .version = "${PKG_VERSION}" | .files=[] | .files[0]={} | .files[0].name="test" | .files[0].name.content="${TEST_FILE}" | .main = "test" | .args = [] | .stdin = "" | .compile_timeout = 10000 | .run_timeout = 3000' <<< '{}'`
+            curl -sXPOST http://127.0.0.1:6969/jobs -H 'Content-Type: application/json' -d "$TEST_JSON"
+          done
+
+
+
+
+
+

api/Dockerfile

@@ -1,6 +1,6 @@
 FROM node:15.8.0-buster-slim
 RUN dpkg-reconfigure -p critical dash
-RUN apt-get update && apt-get install -y gnupg tar coreutils util-linux \
+RUN apt-get update && apt-get install -y libxml2 gnupg tar coreutils util-linux \
   && rm -rf /var/lib/apt/lists/*
 
 RUN for i in $(seq 1001 1500); do \

docker-compose.yaml

@@ -16,11 +16,7 @@ services:
 
   piston_fs_repo: #Temporary solution until CI works
     build: repo
-    command: >
+    command: ['python'] # Only build python
-      bash -c '/repo/make.sh &&
-      curl http://piston_api:6969/repos -XPOST -d "slug=local&url=file:///repo/index.yaml";
-      echo -e "\nAn error here is fine, it just means its already added it. Perhaps you restarted this container"
-      '
     volumes:
       - ./repo:/repo
       - ./packages:/packages

packages/.gitignore

@@ -1,2 +1,3 @@
 build/
-*.pkg.tar.gz
+*.pkg.tar.gz
+*.tf

packages/CONTRIBUTING.MD

@@ -59,7 +59,21 @@ You should add in the path to any binaries. The current working directory for th
 If everything went well, you should now have a `[language]-[version].pkg.tar.gz` file sitting in the root.
 If not, read through your error logs, and if in doubt ask for help in #emkc-felix-piston in Discord.
 
-14. Commit your changes, using message format of `pkg([language]): Added [language] [version]`
+14. Create a source file for the app, which outputs `OK`, naming the file `[language-name].test`
+For example, `mono` would have a file named `csharp-mono.test` containing:
+```cs
+using System;
+
+public class Test
+{
+    public static void Main(string[] args)
+    {
+        Console.WriteLine("OK");
+    }
+}
+```
+
+15. Commit your changes, using message format of `pkg([language]): Added [language] [version]`
 Any additional commits regarding this package should start with `pkg([language]): `
 
-15. Create a pull request (currently to v3), referencing an Issue number (if there is one associated).
+16. Create a pull request (currently to v3), referencing an Issue number (if there is one associated).

packages/Makefile

@@ -5,6 +5,8 @@ PKG_FILES=$(filter-out common.mk,$(wildcard *.mk))
 PKG_SLUGS=$(foreach pkg, ${PKG_FILES}, $(addprefix $(shell make -f ${pkg} name VERSION=UNKNOWN)-, $(shell make -f ${pkg} versions VERSION=UNKNOWN)))
 
 # Functions
+CALL_MAKE=$(shell make -f $1 $2 VERSION=UNKNOWN)
+
 define pkg_info
 	$(eval PKG_SLUG=$(patsubst $1-%,%,$2))
 	$(eval PKG_VERSION=$(lastword $(subst -, ,${PKG_SLUG})))
@@ -12,18 +14,27 @@ define pkg_info
 	$(eval PKG_FILE=$(shell grep '^VERSIONS\s*=.*${PKG_VERSION}' $(shell grep "NAME\s*=\s*${PKG_NAME}" ${PKG_FILES} -l) -l))
 endef
 
+define per-lang
+build-$(call CALL_MAKE, $1, name):: $$(addprefix build-$(call CALL_MAKE, $1, name)-, $$(call CALL_MAKE, $1, versions))
+clean-$(call CALL_MAKE, $1, name):: $$(addprefix clean-$(call CALL_MAKE, $1, name)-, $$(call CALL_MAKE, $1, versions))
+endef
+
+
 # Targets
 
-build: $(foreach pkg, ${PKG_FILES}, $(addprefix build-$(shell make -f ${pkg} name VERSION=UNKNOWN)-, $(lastword $(shell make -f ${pkg} versions VERSION=UNKNOWN))))
+$(foreach e,${PKG_FILES}, $(eval $(call per-lang,${e})))
+
+build build-all: $(foreach pkg, ${PKG_FILES}, build-$(call CALL_MAKE, ${pkg}, name))
 
 $(addprefix build-, ${PKG_SLUGS}):
 	$(call pkg_info,build,$@)
 	$(MAKE) -f ${PKG_FILE} VERSION=${PKG_VERSION} build
 
-
+clean clean-all: $(foreach pkg, ${PKG_FILES}, clean-$(call CALL_MAKE, ${pkg}, name))
-
-clean: $(foreach pkg, ${PKG_FILES}, $(addprefix clean-$(shell make -f ${pkg} name VERSION=UNKNOWN)-, $(shell make -f ${pkg} versions VERSION=UNKNOWN)))
 	rm -rf build/
+
 $(addprefix clean-, ${PKG_SLUGS}):
 	$(call pkg_info,clean,$@)
 	$(MAKE) -f ${PKG_FILE} VERSION=${PKG_VERSION} clean
+
+

packages/common.mk

@@ -34,11 +34,12 @@ name:
 
 
 .NOTPARALLEL: build
-build: ${BUILD_DIR} ${PKG_FILE}
+build: ${BUILD_DIR} ${PKG_FILE} ${PKG_SLUG}.tf
 
 clean: 
 	rm -rf ${BUILD_DIR}
 	rm -f ${PKG_FILE}
+	rm -f ${PKG_SLUG}.tf
 
 # mkdir
 ${BUILD_DIR}:
@@ -61,7 +62,8 @@ $(patsubst %.json,%.jq,${INFO_FILE}):
 	echo '.build_platform="$(or ${PLATFORM}, baremetal-$(shell grep -oP "^ID=\K\w+" /etc/os-release ))"' >> $@
 	$(foreach dep, ${DEPENDENCIES}, echo '.dependencies.$(word 1,$(subst =, ,${dep}))="$(word 2,$(subst =, ,${dep}))"' >> $@)
 
-
+${PKG_SLUG}.tf:
+	cp ${NAME}.test $@
 
 # Helpers
 %/: %.tar.gz

packages/csharp-mono.test

@@ -0,0 +1,9 @@
+using System;
+
+public class Test
+{
+    public static void Main(string[] args)
+    {
+        Console.WriteLine("OK");
+    }
+}

packages/deno.test

@@ -0,0 +1 @@
+console.log("OK")

packages/python.mk

@@ -2,7 +2,7 @@ NAME=python
 AUTHOR=Thomas Hobson <thomas@hexf.me>
 DEPENDENCIES=
 COMPILED=false
-VERSIONS=2.7.1 3.5.1 3.9.1
+VERSIONS=3.5.1 3.9.1
 
 include common.mk
 
@@ -15,6 +15,7 @@ ${ENV_FILE}:
 
 ${BIN_DIR}: ${BUILD_DIR}Python-${VERSION}/
 	$(eval TMP_DIR=${PWD}/${BUILD_DIR}tmpout/)
+	
 	cd $< && ./configure --prefix ${PREFIX}
 	$(MAKE) -C $<
 	DESTDIR=${TMP_DIR} $(MAKE) -C $< altinstall

packages/python.test

@@ -0,0 +1 @@
+print("OK")

repo/Dockerfile

@@ -1,8 +1,11 @@
 FROM debian:buster-slim
 
-RUN apt-get update && apt-get install -y bc curl git linux-headers-amd64 perl xz-utils python3 python3-pip gnupg jq zlib1g-dev cmake cmake-doc extra-cmake-modules build-essential gcc binutils bash coreutils util-linux pciutils usbutils coreutils binutils findutils grep && \
+RUN apt-get update && apt-get install -y autoconf build-essential libssl-dev pkg-config zlib1g-dev libargon2-dev libsodium-dev libcurl4-openssl-dev sqlite3 libsqlite3-dev libonig-dev libxml2 libxml2-dev bc curl git linux-headers-amd64 perl xz-utils python3 python3-pip gnupg jq zlib1g-dev cmake cmake-doc extra-cmake-modules build-essential gcc binutils bash coreutils util-linux pciutils usbutils coreutils binutils findutils grep && \
         ln -sf /bin/bash /bin/sh && \
         pip3 install 'yq==2.12.0' && \
         rm -rf /var/lib/apt/lists/*
 
-CMD [ "bash", "/repo/make.sh" ]
+ADD *.sh /
+
+ENTRYPOINT ["bash","/entrypoint.sh"]
+CMD ["all"]

repo/entrypoint.sh

@@ -0,0 +1,10 @@
+cd /packages
+for pkg in "$*"
+do
+    make -j16 build-$pkg
+done
+
+cd /repo
+./mkindex.sh
+
+curl -s http://piston_api:6969/repos -XPOST -d "slug=local&url=file:///repo/index.yaml"

repo/make.sh

@@ -1,13 +0,0 @@
-#!/bin/bash -e
-
-cd /repo
-
-# Make packages
-pushd ../packages/
-make -j16
-popd
-
-
-# Make repo index
-
-./mkindex.sh