Merge branch 'v3' into v3-node

.github/workflows/package-pr.yaml

@@ -11,8 +11,8 @@ on:
       - 'packages/**'
 
 jobs:
-  packages-affected:
-    name: List affected packages
+  check-build:
+    name: Check that package builds
     runs-on: ubuntu-latest
     steps:
       - name: Get PR Commits
@@ -21,7 +21,6 @@ jobs:
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
       - name: Check subsystem
-        if: ${{ success() || failure() }}
         uses: tim-actions/commit-message-checker-with-regex@v0.3.1
         with:
           commits: ${{ steps.get-pr-commits.outputs.commits }}
@@ -33,17 +32,45 @@ jobs:
         shell: bash
         run: |
           COMMITS='${{ steps.get-pr-commits.outputs.commits }}'
-          PACKAGES=$(echo $COMMITS | jq .[].commit.message -r | grep -oP '^pkg\(\K[^:\h\n]+(?=\))' | sort -u | sed 's/^/* /')
-          PACKAGES=${PACKAGES//$'%'/'%25'}
-          PACKAGES=${PACKAGES//$'\n'/'%0A'}
-          PACKAGES=${PACKAGES//$'\r'/'%0D'}
+          PACKAGES=$(echo $COMMITS | jq .[].commit.message -r | grep -oP '^pkg\(\K[^:\h\n]+(?=\))' | sort -u)
           echo "::set-output name=packages::$PACKAGES"
 
-      - name: 'Comment PR'
-        uses: actions/github-script@0.3.0
-        if: github.event_name == 'pull_request'
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          script: |
-            const { issue: { number: issue_number }, repo: { owner, repo }  } = context;
-            github.issues.createComment({ issue_number, owner, repo, body: "Affected packages:\n${{ steps.get-packages.outputs.packages }}" });
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Build docker containers
+        run: |
+          docker build -t piston_fs_repo repo
+          docker build -t piston_api api
+          
+      - name: Build Packages
+        run: |
+          docker run -v $(pwd)'/repo:/repo' -v $(pwd)'/packages:/packages' piston_fs_repo ${{ steps.get-packages.outputs.packages }}
+
+      - name: Run tests
+        run: |
+          docker run -p 6969:6969 -v $(pwd)'/data:/piston' -v $(pwd)'/repo:/repo' --privileged --name api piston_api & 
+          echo Waiting for API to start..
+          while [[ "$(curl -s -w '%{http_code}' http://127.0.0.1:6969/runtimes)" != "200" ]]; do sleep 5; done
+          echo Adding local repo
+          curl -s http://127.0.0.1:6969/repos -XPOST -d "slug=local&url=file:///repo/index.yaml"
+          echo Testing packages
+          for pkg in "$(curl -s http://127.0.0.1:6969/repos/local/packages/ | jq '.data.packages[] | "\(.language)/\(.language_version)"' -r)"
+          do
+            PKG_SLUG=${pkg/\//-}
+            PKG_NAME=$(echo $pkg | cut -d'/' -f 1)
+            PKG_VERSION=$(echo $pkg | cut -d'/' -f 2)
+            echo Installing ${PKG_SLUG}
+            curl -sXPOST http://127.0.0.1:6969/repos/local/packages/${pkg} | jq '.language' -r || exit 1
+            echo Testing ${PKG_SLUG} (using ${PKG_SLUG}.tf)
+            TEST_FILE=$(cat ${PKG_SLUG}.tf)
+            TEST_JSON=`jq -C '.language = "${PKG_NAME}" | .version = "${PKG_VERSION}" | .files=[] | .files[0]={} | .files[0].name="test" | .files[0].name.content="${TEST_FILE}" | .main = "test" | .args = [] | .stdin = "" | .compile_timeout = 10000 | .run_timeout = 3000' <<< '{}'`
+            curl -sXPOST http://127.0.0.1:6969/jobs -H 'Content-Type: application/json' -d "$TEST_JSON" > ${PKG_SLUG}.tr
+            jq '.run.stdout' ${PKG_SLUG}.tr | grep "OK" || exit 1
+          done
+
+
+
+
+
+

README.MD

@@ -204,7 +204,7 @@ Content-Type: application/json
 
 # Supported Languages
 
-`python`,`node`
+`python`,`php`,`node`
 
 
 <br>

docker-compose.yaml

@@ -16,11 +16,7 @@ services:
 
   piston_fs_repo: #Temporary solution until CI works
     build: repo
-    command: >
-      bash -c '/repo/make.sh &&
-      curl http://piston_api:6969/repos -XPOST -d "slug=local&url=file:///repo/index.yaml";
-      echo -e "\nAn error here is fine, it just means its already added it. Perhaps you restarted this container"
-      '
+    command: ['python'] # Only build python
     volumes:
       - ./repo:/repo
       - ./packages:/packages

packages/.gitignore

@@ -1,2 +1,3 @@
 build/
 *.pkg.tar.gz
+*.tf

packages/Makefile

@@ -5,6 +5,8 @@ PKG_FILES=$(filter-out common.mk,$(wildcard *.mk))
 PKG_SLUGS=$(foreach pkg, ${PKG_FILES}, $(addprefix $(shell make -f ${pkg} name VERSION=UNKNOWN)-, $(shell make -f ${pkg} versions VERSION=UNKNOWN)))
 
 # Functions
+CALL_MAKE=$(shell make -f $1 $2 VERSION=UNKNOWN)
+
 define pkg_info
 	$(eval PKG_SLUG=$(patsubst $1-%,%,$2))
 	$(eval PKG_VERSION=$(lastword $(subst -, ,${PKG_SLUG})))
@@ -12,18 +14,27 @@ define pkg_info
 	$(eval PKG_FILE=$(shell grep '^VERSIONS\s*=.*${PKG_VERSION}' $(shell grep "NAME\s*=\s*${PKG_NAME}" ${PKG_FILES} -l) -l))
 endef
 
+define per-lang
+build-$(call CALL_MAKE, $1, name):: $$(addprefix build-$(call CALL_MAKE, $1, name)-, $$(call CALL_MAKE, $1, versions))
+clean-$(call CALL_MAKE, $1, name):: $$(addprefix clean-$(call CALL_MAKE, $1, name)-, $$(call CALL_MAKE, $1, versions))
+endef
+
+
 # Targets
 
-build: $(foreach pkg, ${PKG_FILES}, $(addprefix build-$(shell make -f ${pkg} name VERSION=UNKNOWN)-, $(lastword $(shell make -f ${pkg} versions VERSION=UNKNOWN))))
+$(foreach e,${PKG_FILES}, $(eval $(call per-lang,${e})))
+
+build build-all: $(foreach pkg, ${PKG_FILES}, build-$(call CALL_MAKE, ${pkg}, name))
 
 $(addprefix build-, ${PKG_SLUGS}):
 	$(call pkg_info,build,$@)
 	$(MAKE) -f ${PKG_FILE} VERSION=${PKG_VERSION} build
 
-
-
-clean: $(foreach pkg, ${PKG_FILES}, $(addprefix clean-$(shell make -f ${pkg} name VERSION=UNKNOWN)-, $(shell make -f ${pkg} versions VERSION=UNKNOWN)))
+clean clean-all: $(foreach pkg, ${PKG_FILES}, clean-$(call CALL_MAKE, ${pkg}, name))
 	rm -rf build/
+
 $(addprefix clean-, ${PKG_SLUGS}):
 	$(call pkg_info,clean,$@)
 	$(MAKE) -f ${PKG_FILE} VERSION=${PKG_VERSION} clean
+
+

packages/common.mk

@@ -34,11 +34,12 @@ name:
 
 
 .NOTPARALLEL: build
-build: ${BUILD_DIR} ${PKG_FILE}
+build: ${BUILD_DIR} ${PKG_FILE} ${PKG_SLUG}.tf
 
 clean: 
 	rm -rf ${BUILD_DIR}
 	rm -f ${PKG_FILE}
+	rm -f ${PKG_SLUG}.tf
 
 # mkdir
 ${BUILD_DIR}:
@@ -61,7 +62,8 @@ $(patsubst %.json,%.jq,${INFO_FILE}):
 	echo '.build_platform="$(or ${PLATFORM}, baremetal-$(shell grep -oP "^ID=\K\w+" /etc/os-release ))"' >> $@
 	$(foreach dep, ${DEPENDENCIES}, echo '.dependencies.$(word 1,$(subst =, ,${dep}))="$(word 2,$(subst =, ,${dep}))"' >> $@)
 
-
+${PKG_SLUG}.tf:
+	cp ${NAME}.test $@
 
 # Helpers
 %/: %.tar.gz

packages/php.mk

@@ -0,0 +1,24 @@
+NAME=php
+AUTHOR=Martin Kos <martinkos007@gmail.com>
+DEPENDENCIES=
+COMPILED=false
+VERSIONS=7.4.15 8.0.2
+
+include common.mk
+
+
+${RUN_FILE}:
+	echo 'php $$*' > $@
+
+${ENV_FILE}:
+	echo 'export PATH=$$PWD/bin:$$PATH' > $@
+
+${BIN_DIR}: ${BUILD_DIR}php-${VERSION}-sources/
+	$(eval TMP_DIR=${PWD}/${BUILD_DIR}tmpout)
+	cd $< && ./configure --prefix ${PREFIX}
+	$(MAKE) -C $<
+	INSTALL_ROOT=${TMP_DIR}/ $(MAKE) -C $< install
+	mv ${TMP_DIR}${PREFIX} ${BIN_DIR} && rm -r ${TMP_DIR}
+
+${BUILD_DIR}php-${VERSION}-sources.tar.gz:
+	curl "https://www.php.net/distributions/php-${VERSION}.tar.gz" -o $@

packages/php.test

@@ -0,0 +1,3 @@
+<?php
+echo "OK";
+?>

packages/python.mk

@@ -2,7 +2,7 @@ NAME=python
 AUTHOR=Thomas Hobson <thomas@hexf.me>
 DEPENDENCIES=
 COMPILED=false
-VERSIONS=2.7.1 3.5.1 3.9.1
+VERSIONS=3.5.1 3.9.1
 
 include common.mk
 
@@ -15,6 +15,7 @@ ${ENV_FILE}:
 
 ${BIN_DIR}: ${BUILD_DIR}Python-${VERSION}/
 	$(eval TMP_DIR=${PWD}/${BUILD_DIR}tmpout/)
+	
 	cd $< && ./configure --prefix ${PREFIX}
 	$(MAKE) -C $<
 	DESTDIR=${TMP_DIR} $(MAKE) -C $< altinstall

repo/Dockerfile

@@ -5,4 +5,7 @@ RUN apt-get update && apt-get install -y autoconf build-essential libssl-dev pkg
         pip3 install 'yq==2.12.0' && \
         rm -rf /var/lib/apt/lists/*
 
-CMD [ "bash", "/repo/make.sh" ]
+ADD *.sh /
+
+ENTRYPOINT ["bash","/entrypoint.sh"]
+CMD ["all"]

repo/entrypoint.sh

@@ -0,0 +1,10 @@
+cd /packages
+for pkg in "$*"
+do
+    make -j16 build-$pkg
+done
+
+cd /repo
+./mkindex.sh
+
+curl -s http://piston_api:6969/repos -XPOST -d "slug=local&url=file:///repo/index.yaml" || echo "WARNING: Could not add repository"

repo/make.sh

@@ -1,13 +0,0 @@
-#!/bin/bash -e
-
-cd /repo
-
-# Make packages
-pushd ../packages/
-make -j16
-popd
-
-
-# Make repo index
-
-./mkindex.sh