From 4626176222d2addf93c8e4f177ba8c5bfefa6e64 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sun, 14 Mar 2021 15:56:17 +1300 Subject: [PATCH 01/54] ci(package): use head commit --- .github/workflows/package-pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/package-pr.yaml b/.github/workflows/package-pr.yaml index b451a6b..94a2475 100644 --- a/.github/workflows/package-pr.yaml +++ b/.github/workflows/package-pr.yaml @@ -27,7 +27,7 @@ jobs: - name: Build Packages run: | - SHALIST=$(curl "https://api.github.com/repos/$GITHUB_REPOSITORY/pulls/$(jq -r '.pull_request.number' $GITHUB_EVENT_PATH)"/commits | jq '.[].commit.tree.sha' -r) + SHALIST=$(git rev-parse HEAD) echo "Commits: $SHALIST" git log docker run -v "${{ github.workspace }}:/piston" docker.pkg.github.com/engineer-man/piston/repo-builder:latest --ci $SHALIST From 84757560fc377a9aaa6978b448d2c3667ce1160a Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sun, 14 Mar 2021 16:03:40 +1300 Subject: [PATCH 02/54] repo: verbosity --- repo/entrypoint.sh | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/repo/entrypoint.sh b/repo/entrypoint.sh index cede653..429ead9 100755 --- a/repo/entrypoint.sh +++ b/repo/entrypoint.sh @@ -2,8 +2,7 @@ cd /piston/packages SERVER=1 BUILD=1 - -ls -la /piston /piston/* +CI=0 echo "Running through arguments.." @@ -20,22 +19,26 @@ do echo "Running in CI mode, --no-build, --no-server" BUILD=0 SERVER=0 - SHA=$1 - shift - echo "Commit Sha: $SHA" - PACKAGES=$(git diff-tree --no-commit-id --name-only -r $SHA | awk -F/ '{ print $2 "-" $3 }' | sort -u) - echo "Building packages: $PACKAGES" - for package in "$PACKAGES"; do - make -j16 $package.pkg.tar.gz - done - + CI=1 else if [[ $BUILD -eq 1 ]]; then echo "Building package $pkg" make -j16 $pkg.pkg.tar.gz echo "Done with package $pkg" + elif [[ $CI -eq 1 ]]; then + echo "Commit SHA: $pkg" + + echo "Changed files:" + git diff-tree --no-commit-id --name-only -r $pkg + + PACKAGES=$(git diff-tree --no-commit-id --name-only -r $pkg | awk -F/ '{ print $2 "-" $3 }' | sort -u) + echo "Building packages: $PACKAGES" + for package in "$PACKAGES"; do + make -j16 $package.pkg.tar.gz + done + else - echo "Building was disabled, skipping $pkg" + echo "Building was disabled, skipping $pkg build=$BUILD ci=$CI" fi fi done From 0ae6ee16325c7ba7c608ece54829539000ce49b3 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sun, 14 Mar 2021 16:05:11 +1300 Subject: [PATCH 03/54] ci(repo): pull using action --- .github/workflows/repo-push.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/repo-push.yaml b/.github/workflows/repo-push.yaml index 649b1cd..b5a603c 100644 --- a/.github/workflows/repo-push.yaml +++ b/.github/workflows/repo-push.yaml @@ -20,13 +20,12 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} registry: docker.pkg.github.com - - name: Pull existing - run: docker pull docker.pkg.github.com/engineer-man/piston/repo-builder - name: Build and push repo uses: docker/build-push-action@v2 with: context: repo + pull: true push: true tags: | docker.pkg.github.com/engineer-man/piston/repo-builder \ No newline at end of file From f1a74a0d686b62bffc1679fc3f337354b1e7d53c Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sun, 14 Mar 2021 16:06:42 +1300 Subject: [PATCH 04/54] ci(api): pull using action --- .github/workflows/api-push.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/api-push.yaml b/.github/workflows/api-push.yaml index ecf7a29..7bafdc1 100644 --- a/.github/workflows/api-push.yaml +++ b/.github/workflows/api-push.yaml @@ -21,13 +21,12 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} registry: docker.pkg.github.com - - name: Pull existing - run: docker pull docker.pkg.github.com/engineer-man/piston/api - name: Build and push API uses: docker/build-push-action@v2 with: context: api push: true + pull: true tags: | docker.pkg.github.com/engineer-man/piston/api \ No newline at end of file From 00dd38b83d59837e43524992d7949e298ca0cc2d Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sun, 14 Mar 2021 16:12:28 +1300 Subject: [PATCH 05/54] repo: get changed files --- repo/entrypoint.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/repo/entrypoint.sh b/repo/entrypoint.sh index 429ead9..d9fc973 100755 --- a/repo/entrypoint.sh +++ b/repo/entrypoint.sh @@ -29,9 +29,9 @@ do echo "Commit SHA: $pkg" echo "Changed files:" - git diff-tree --no-commit-id --name-only -r $pkg + git diff --name-only $pkg^1 $pkg - PACKAGES=$(git diff-tree --no-commit-id --name-only -r $pkg | awk -F/ '{ print $2 "-" $3 }' | sort -u) + PACKAGES=$(git diff --name-only $pkg^1 $pkg | awk -F/ '{ print $2 "-" $3 }' | sort -u) echo "Building packages: $PACKAGES" for package in "$PACKAGES"; do make -j16 $package.pkg.tar.gz From baf00ada3d357de90836b137900798381c9e4c51 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sun, 14 Mar 2021 16:21:01 +1300 Subject: [PATCH 06/54] ci(package): point to head --- .github/workflows/package-pr.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/package-pr.yaml b/.github/workflows/package-pr.yaml index 94a2475..5bf3834 100644 --- a/.github/workflows/package-pr.yaml +++ b/.github/workflows/package-pr.yaml @@ -27,10 +27,7 @@ jobs: - name: Build Packages run: | - SHALIST=$(git rev-parse HEAD) - echo "Commits: $SHALIST" - git log - docker run -v "${{ github.workspace }}:/piston" docker.pkg.github.com/engineer-man/piston/repo-builder:latest --ci $SHALIST + docker run -v "${{ github.workspace }}:/piston" docker.pkg.github.com/engineer-man/piston/repo-builder:latest --ci HEAD ls -la packages repo - name: Upload package as artifact From 7bca798c0bf65d079984d7e19d8950872bc99bad Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sun, 14 Mar 2021 16:38:48 +1300 Subject: [PATCH 07/54] ci(package): test --- .github/workflows/package-pr.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/package-pr.yaml b/.github/workflows/package-pr.yaml index 5bf3834..e6dcbcf 100644 --- a/.github/workflows/package-pr.yaml +++ b/.github/workflows/package-pr.yaml @@ -27,8 +27,9 @@ jobs: - name: Build Packages run: | - docker run -v "${{ github.workspace }}:/piston" docker.pkg.github.com/engineer-man/piston/repo-builder:latest --ci HEAD - ls -la packages repo + git diff --name-only HEAD^1 HEAD + echo docker run -v "${{ github.workspace }}:/piston" docker.pkg.github.com/engineer-man/piston/repo-builder:latest --no-server + ls -la packages - name: Upload package as artifact uses: actions/upload-artifact@v2 From 2c1ca22cdd05289ad8fce717058fad165a8fa4cb Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sun, 14 Mar 2021 16:41:52 +1300 Subject: [PATCH 08/54] ci(package): test --- .github/workflows/package-pr.yaml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/package-pr.yaml b/.github/workflows/package-pr.yaml index e6dcbcf..4d9454d 100644 --- a/.github/workflows/package-pr.yaml +++ b/.github/workflows/package-pr.yaml @@ -24,10 +24,15 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} registry: docker.pkg.github.com - + + - name: Get list of changed files + uses: lots0logs/gh-action-get-changed-files@2.1.4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + - name: Build Packages run: | - git diff --name-only HEAD^1 HEAD + cat ${HOME}/files.json echo docker run -v "${{ github.workspace }}:/piston" docker.pkg.github.com/engineer-man/piston/repo-builder:latest --no-server ls -la packages From abec6cf2444e354d4df90c5570162481c3f6cecd Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sun, 14 Mar 2021 16:43:50 +1300 Subject: [PATCH 09/54] ci(package): get list of packages from pr --- .github/workflows/package-pr.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/package-pr.yaml b/.github/workflows/package-pr.yaml index 4d9454d..cc5e46e 100644 --- a/.github/workflows/package-pr.yaml +++ b/.github/workflows/package-pr.yaml @@ -32,8 +32,9 @@ jobs: - name: Build Packages run: | - cat ${HOME}/files.json - echo docker run -v "${{ github.workspace }}:/piston" docker.pkg.github.com/engineer-man/piston/repo-builder:latest --no-server + PACKAGES=$(jq '.[]' -r ${HOME}/files.json | awk -F/ '{ print $2 "-" $3 }' | sort -u) + echo "Packages: $PACKAGES" + docker run -v "${{ github.workspace }}:/piston" docker.pkg.github.com/engineer-man/piston/repo-builder:latest --no-server $PACKAGES ls -la packages - name: Upload package as artifact From 81235d2c3900cd4ea0cc062c2b99149a9c12a579 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sun, 14 Mar 2021 17:38:56 +1300 Subject: [PATCH 10/54] ci(package): no build on test --- .github/workflows/package-pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/package-pr.yaml b/.github/workflows/package-pr.yaml index cc5e46e..3caebff 100644 --- a/.github/workflows/package-pr.yaml +++ b/.github/workflows/package-pr.yaml @@ -77,7 +77,7 @@ jobs: - name: Run tests run: | - docker run -v $(pwd)'/repo:/piston/repo' -v $(pwd)'/packages:/piston/packages' --name piston_fs_repo docker.pkg.github.com/engineer-man/piston/repo-builder $PACKAGES & + docker run -v $(pwd)'/repo:/piston/repo' -v $(pwd)'/packages:/piston/packages' --name piston_fs_repo docker.pkg.github.com/engineer-man/piston/repo-builder --no-build & docker run -v $(pwd)'/data:/piston' --name api docker.pkg.github.com/engineer-man/piston/api & echo Waiting for API to start.. while [[ "$(curl -s -w '%{http_code}' http://api:6969/runtimes)" != "200" ]]; do sleep 5; done From e1d9ae69c9d6b7294d60123e24034d715f3d9469 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sun, 14 Mar 2021 17:57:11 +1300 Subject: [PATCH 11/54] ci(packages): add testing --- .github/workflows/package-pr.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/package-pr.yaml b/.github/workflows/package-pr.yaml index 3caebff..5fbea65 100644 --- a/.github/workflows/package-pr.yaml +++ b/.github/workflows/package-pr.yaml @@ -77,12 +77,13 @@ jobs: - name: Run tests run: | - docker run -v $(pwd)'/repo:/piston/repo' -v $(pwd)'/packages:/piston/packages' --name piston_fs_repo docker.pkg.github.com/engineer-man/piston/repo-builder --no-build & - docker run -v $(pwd)'/data:/piston' --name api docker.pkg.github.com/engineer-man/piston/api & + docker run -v $(pwd)'/repo:/piston/repo' -v $(pwd)'/packages:/piston/packages' -d --name piston_fs_repo docker.pkg.github.com/engineer-man/piston/repo-builder --no-build + docker run -v $(pwd)'/data:/piston' --name api docker.pkg.github.com/engineer-man/piston/api -d echo Waiting for API to start.. - while [[ "$(curl -s -w '%{http_code}' http://api:6969/runtimes)" != "200" ]]; do sleep 5; done + docker run --network container:api appropriate/curl --retry 10 --retry-connrefused http://localhost:6969/runtimes - echo Testing packages + echo Listing Packages + docker run --network container:api appropriate/curl http://localhost:6969/packages From 8834462ac2c998391585a51020ef8b1bafcd54dd Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sun, 14 Mar 2021 17:59:21 +1300 Subject: [PATCH 12/54] ci(package): login to package registry --- .github/workflows/package-pr.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/package-pr.yaml b/.github/workflows/package-pr.yaml index 5fbea65..346dd31 100644 --- a/.github/workflows/package-pr.yaml +++ b/.github/workflows/package-pr.yaml @@ -75,6 +75,13 @@ jobs: write-mode: overwrite + - name: Login to GitHub registry + uses: docker/login-action@v1 + with: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + registry: docker.pkg.github.com + - name: Run tests run: | docker run -v $(pwd)'/repo:/piston/repo' -v $(pwd)'/packages:/piston/packages' -d --name piston_fs_repo docker.pkg.github.com/engineer-man/piston/repo-builder --no-build From 94ac5c42aab9b79a6427e2e56dd917e77a6f2131 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sun, 14 Mar 2021 18:01:45 +1300 Subject: [PATCH 13/54] ci(packge): move detached flag --- .github/workflows/package-pr.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/package-pr.yaml b/.github/workflows/package-pr.yaml index 346dd31..cf0a47a 100644 --- a/.github/workflows/package-pr.yaml +++ b/.github/workflows/package-pr.yaml @@ -81,11 +81,11 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} registry: docker.pkg.github.com - + - name: Run tests run: | docker run -v $(pwd)'/repo:/piston/repo' -v $(pwd)'/packages:/piston/packages' -d --name piston_fs_repo docker.pkg.github.com/engineer-man/piston/repo-builder --no-build - docker run -v $(pwd)'/data:/piston' --name api docker.pkg.github.com/engineer-man/piston/api -d + docker run -v $(pwd)'/data:/piston' -d --name api docker.pkg.github.com/engineer-man/piston/api echo Waiting for API to start.. docker run --network container:api appropriate/curl --retry 10 --retry-connrefused http://localhost:6969/runtimes From 72f0f614481a4b13f8da0cfa4adcc6fe66265f26 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sun, 14 Mar 2021 18:05:20 +1300 Subject: [PATCH 14/54] ci(package): wait for index to start --- .github/workflows/package-pr.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/package-pr.yaml b/.github/workflows/package-pr.yaml index cf0a47a..5661a1b 100644 --- a/.github/workflows/package-pr.yaml +++ b/.github/workflows/package-pr.yaml @@ -89,6 +89,9 @@ jobs: echo Waiting for API to start.. docker run --network container:api appropriate/curl --retry 10 --retry-connrefused http://localhost:6969/runtimes + echo Waiting for Index to start.. + docker run --network container:api appropriate/curl --retry 10 --retry-connrefused http://piston_fs_repo:8000/index + echo Listing Packages docker run --network container:api appropriate/curl http://localhost:6969/packages From aaadef6db914880af89aa8b5ca61619276d6bacd Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sun, 14 Mar 2021 18:08:37 +1300 Subject: [PATCH 15/54] ci(package): wait for index --- .github/workflows/package-pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/package-pr.yaml b/.github/workflows/package-pr.yaml index 5661a1b..2326894 100644 --- a/.github/workflows/package-pr.yaml +++ b/.github/workflows/package-pr.yaml @@ -90,7 +90,7 @@ jobs: docker run --network container:api appropriate/curl --retry 10 --retry-connrefused http://localhost:6969/runtimes echo Waiting for Index to start.. - docker run --network container:api appropriate/curl --retry 10 --retry-connrefused http://piston_fs_repo:8000/index + docker run --network container:piston_fs_repo appropriate/curl --retry 10 --retry-connrefused http://localhost:8000/index echo Listing Packages docker run --network container:api appropriate/curl http://localhost:6969/packages From c751653b3ace44e5e95c4ec24241a485d7664b94 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sun, 14 Mar 2021 18:11:39 +1300 Subject: [PATCH 16/54] ci(package): network index to api --- .github/workflows/package-pr.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/package-pr.yaml b/.github/workflows/package-pr.yaml index 2326894..66f9814 100644 --- a/.github/workflows/package-pr.yaml +++ b/.github/workflows/package-pr.yaml @@ -71,7 +71,7 @@ jobs: output_max_size: 1024 max_process_count: 64 max_open_files: 2048 - repo_url: http://piston_fs_repo:8000/index + repo_url: http://localhost:8000/index write-mode: overwrite @@ -85,7 +85,7 @@ jobs: - name: Run tests run: | docker run -v $(pwd)'/repo:/piston/repo' -v $(pwd)'/packages:/piston/packages' -d --name piston_fs_repo docker.pkg.github.com/engineer-man/piston/repo-builder --no-build - docker run -v $(pwd)'/data:/piston' -d --name api docker.pkg.github.com/engineer-man/piston/api + docker run --network container:piston_fs_repo -v $(pwd)'/data:/piston' -d --name api docker.pkg.github.com/engineer-man/piston/api echo Waiting for API to start.. docker run --network container:api appropriate/curl --retry 10 --retry-connrefused http://localhost:6969/runtimes From 6290fdfdbac4c695371fb8d3d7bb83507b6eb7b6 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sun, 14 Mar 2021 18:15:05 +1300 Subject: [PATCH 17/54] ci(package): dump logs --- .github/workflows/package-pr.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/package-pr.yaml b/.github/workflows/package-pr.yaml index 66f9814..efb2886 100644 --- a/.github/workflows/package-pr.yaml +++ b/.github/workflows/package-pr.yaml @@ -95,6 +95,9 @@ jobs: echo Listing Packages docker run --network container:api appropriate/curl http://localhost:6969/packages + docker logs api + docker logs piston_fs_repo + From ada44e41b53f2378f2b7e2a324d48c8f2eb05cf3 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sun, 14 Mar 2021 18:27:28 +1300 Subject: [PATCH 18/54] repo: logging verbosity --- repo/entrypoint.sh | 4 +++- repo/mkindex.sh | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/repo/entrypoint.sh b/repo/entrypoint.sh index d9fc973..a0f7e1a 100755 --- a/repo/entrypoint.sh +++ b/repo/entrypoint.sh @@ -28,10 +28,12 @@ do elif [[ $CI -eq 1 ]]; then echo "Commit SHA: $pkg" + cd .. echo "Changed files:" git diff --name-only $pkg^1 $pkg - PACKAGES=$(git diff --name-only $pkg^1 $pkg | awk -F/ '{ print $2 "-" $3 }' | sort -u) + cd packages + echo "Building packages: $PACKAGES" for package in "$PACKAGES"; do make -j16 $package.pkg.tar.gz diff --git a/repo/mkindex.sh b/repo/mkindex.sh index c3c72a0..082423e 100755 --- a/repo/mkindex.sh +++ b/repo/mkindex.sh @@ -17,6 +17,7 @@ do PKGCHECKSUM=$(sha256sum $PKGFILE | awk '{print $1}') echo "$PKGNAME,$PKGVERSION,$PKGCHECKSUM,$BASEURL$PKGFILE" >> index + echo "Adding package $PKGNAME-$PKGVERSION" ((i=i+1)) done \ No newline at end of file From 015365f0430a4b79351c5d7d8ca6ccd5ef2078fa Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sun, 14 Mar 2021 18:37:11 +1300 Subject: [PATCH 19/54] ci(package): put downloaded packages in correct folder --- .github/workflows/package-pr.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/package-pr.yaml b/.github/workflows/package-pr.yaml index efb2886..8b02e50 100644 --- a/.github/workflows/package-pr.yaml +++ b/.github/workflows/package-pr.yaml @@ -54,6 +54,9 @@ jobs: - uses: actions/download-artifact@v2 with: name: packages + + - name: Relocate downloaded packages + run: mv *.pkg.tar.gz packages/ - name: Write test config file uses: DamianReeves/write-file-action@v1.0 @@ -84,6 +87,7 @@ jobs: - name: Run tests run: | + ls -la docker run -v $(pwd)'/repo:/piston/repo' -v $(pwd)'/packages:/piston/packages' -d --name piston_fs_repo docker.pkg.github.com/engineer-man/piston/repo-builder --no-build docker run --network container:piston_fs_repo -v $(pwd)'/data:/piston' -d --name api docker.pkg.github.com/engineer-man/piston/api echo Waiting for API to start.. From 41d807dec4f497cc7be9f329b25c409073735f5f Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sun, 14 Mar 2021 19:07:54 +1300 Subject: [PATCH 20/54] ci(package): Add testing routine --- .github/workflows/package-pr.yaml | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/.github/workflows/package-pr.yaml b/.github/workflows/package-pr.yaml index 8b02e50..c1087a7 100644 --- a/.github/workflows/package-pr.yaml +++ b/.github/workflows/package-pr.yaml @@ -91,13 +91,35 @@ jobs: docker run -v $(pwd)'/repo:/piston/repo' -v $(pwd)'/packages:/piston/packages' -d --name piston_fs_repo docker.pkg.github.com/engineer-man/piston/repo-builder --no-build docker run --network container:piston_fs_repo -v $(pwd)'/data:/piston' -d --name api docker.pkg.github.com/engineer-man/piston/api echo Waiting for API to start.. - docker run --network container:api appropriate/curl --retry 10 --retry-connrefused http://localhost:6969/runtimes + docker run --network container:api appropriate/curl -s --retry 10 --retry-connrefused http://localhost:6969/runtimes echo Waiting for Index to start.. - docker run --network container:piston_fs_repo appropriate/curl --retry 10 --retry-connrefused http://localhost:8000/index + docker run --network container:piston_fs_repo appropriate/curl -s --retry 10 --retry-connrefused http://localhost:8000/index echo Listing Packages - docker run --network container:api appropriate/curl http://localhost:6969/packages + PACKAGES_JSON=$(docker run --network container:api appropriate/curl -s http://localhost:6969/packages) + echo $PACKAGES_JSON + + for package in $(jq -r '.[] | "\(.language)-\(.language_version)"' <<< "$PACKAGES_JSON") + do + echo "Testing $package" + PKG_PATH=$(sed 's|-|/|' <<< $package) + PKG_NAME=$(awk -F- '{ print $1 }' <<< $package) + PKG_VERSION=$(awk -F- '{ print $2 }' <<< $package) + + echo "Installing..." + docker run --network container:api appropriate/curl -sXPOST http://localhost:6969/packages/$PKG_PATH + + TEST_SCRIPTS=packages/$PKG_PATH/test.* + echo "Tests: $TEST_SCRIPTS" + + for tscript in $TEST_SCRIPTS + do + TEST_RUNTIME=$(awk -F. '{print $2}' <<< $(basename $tscript)) + echo Running $tscript with runtime=$TEST_RUNTIME + docker run --network container:api -v "$PWD/cli:/app" -v "$PWD/$(dirname $tscript):/pkg" node:15 /app/index.js run $TEST_RUNTIME $PKG_VERSION /pkg/$(basename $tscript) | grep OK + done + done docker logs api docker logs piston_fs_repo From ddbd4617f1f6b600421f99bb2bc9550c426bcefd Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sun, 14 Mar 2021 19:14:30 +1300 Subject: [PATCH 21/54] ci(package): run yarn before testing --- .github/workflows/package-pr.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/package-pr.yaml b/.github/workflows/package-pr.yaml index c1087a7..454ab54 100644 --- a/.github/workflows/package-pr.yaml +++ b/.github/workflows/package-pr.yaml @@ -100,6 +100,9 @@ jobs: PACKAGES_JSON=$(docker run --network container:api appropriate/curl -s http://localhost:6969/packages) echo $PACKAGES_JSON + echo Getting CLI ready + docker run -v "$PWD/cli:/app" --entrypoint yarn node:15 + for package in $(jq -r '.[] | "\(.language)-\(.language_version)"' <<< "$PACKAGES_JSON") do echo "Testing $package" From 935b6cd51769e91e67f91c8715e20aa80b64b5ee Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sun, 14 Mar 2021 19:21:41 +1300 Subject: [PATCH 22/54] ci(package): adjust index for localhost --- .github/workflows/package-pr.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/package-pr.yaml b/.github/workflows/package-pr.yaml index 454ab54..b6d18da 100644 --- a/.github/workflows/package-pr.yaml +++ b/.github/workflows/package-pr.yaml @@ -96,12 +96,15 @@ jobs: echo Waiting for Index to start.. docker run --network container:piston_fs_repo appropriate/curl -s --retry 10 --retry-connrefused http://localhost:8000/index + echo Adjusting index + sed -i 's/piston_fs_repo/localhost/g' repo/index + echo Listing Packages PACKAGES_JSON=$(docker run --network container:api appropriate/curl -s http://localhost:6969/packages) echo $PACKAGES_JSON echo Getting CLI ready - docker run -v "$PWD/cli:/app" --entrypoint yarn node:15 + docker run -v "$PWD/cli:/app" --entrypoint /bin/bash node:15 -c 'cd /app; yarn' for package in $(jq -r '.[] | "\(.language)-\(.language_version)"' <<< "$PACKAGES_JSON") do From 877473b1284001fa7fce9ccd7ffdde85608f155f Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sun, 14 Mar 2021 19:31:41 +1300 Subject: [PATCH 23/54] ci(package): Dump logs always --- .github/workflows/package-pr.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/package-pr.yaml b/.github/workflows/package-pr.yaml index b6d18da..30c5cfd 100644 --- a/.github/workflows/package-pr.yaml +++ b/.github/workflows/package-pr.yaml @@ -127,6 +127,10 @@ jobs: done done + + - name: Dump logs + if: ${{ always() }} + run: | docker logs api docker logs piston_fs_repo @@ -135,3 +139,5 @@ jobs: + + From 2ab508b4d22a10b6f0b851c180ed9e899355a964 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sun, 14 Mar 2021 19:36:12 +1300 Subject: [PATCH 24/54] cli: basename main file --- cli/commands/execute.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cli/commands/execute.js b/cli/commands/execute.js index 3d04ec6..b4baf35 100644 --- a/cli/commands/execute.js +++ b/cli/commands/execute.js @@ -52,7 +52,7 @@ exports.handler = async function(argv){ language: argv.language, version: argv['language-version'], files: files, - main: argv.file, + main: path.basename(argv.file), args: argv.args, stdin, compile_timeout: argv.ct, @@ -84,6 +84,8 @@ exports.handler = async function(argv){ chalk.bold.yellow(ctx.signal) ) } + + if(response.compile) step('Compile', response.compile) step('Run', response.run) From 5fbc0d7f1b50afb618d1c307cf256f9edff1acdc Mon Sep 17 00:00:00 2001 From: Dan Vargas Date: Sat, 13 Mar 2021 12:22:14 -0600 Subject: [PATCH 25/54] pkg(ruby-2.5.1): Added ruby 2.5.1 --- packages/ruby/2.5.1/build.sh | 19 +++++++++++++++++++ packages/ruby/2.5.1/environment | 1 + packages/ruby/2.5.1/metadata.json | 6 ++++++ packages/ruby/2.5.1/run | 3 +++ packages/ruby/2.5.1/test.rb | 1 + 5 files changed, 30 insertions(+) create mode 100755 packages/ruby/2.5.1/build.sh create mode 100644 packages/ruby/2.5.1/environment create mode 100644 packages/ruby/2.5.1/metadata.json create mode 100644 packages/ruby/2.5.1/run create mode 100644 packages/ruby/2.5.1/test.rb diff --git a/packages/ruby/2.5.1/build.sh b/packages/ruby/2.5.1/build.sh new file mode 100755 index 0000000..6f7def1 --- /dev/null +++ b/packages/ruby/2.5.1/build.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +PREFIX=$(realpath $(dirname $0)) + +mkdir -p build + +cd build + +curl "https://cache.ruby-lang.org/pub/ruby/2.5/ruby-2.5.1.tar.gz" -o ruby.tar.gz +tar xzf ruby.tar.gz --strip-components=1 +rm ruby.tar.gz + +./configure --prefix "$PREFIX" +make -j$(nproc) +make install -j$(nproc) + +cd .. + +rm -rf build diff --git a/packages/ruby/2.5.1/environment b/packages/ruby/2.5.1/environment new file mode 100644 index 0000000..bd0ff98 --- /dev/null +++ b/packages/ruby/2.5.1/environment @@ -0,0 +1 @@ +export PATH=$PWD/bin:$PATH \ No newline at end of file diff --git a/packages/ruby/2.5.1/metadata.json b/packages/ruby/2.5.1/metadata.json new file mode 100644 index 0000000..e22c78c --- /dev/null +++ b/packages/ruby/2.5.1/metadata.json @@ -0,0 +1,6 @@ +{ + "language": "ruby", + "version": "2.5.1", + "author": "Dan Vargas ", + "aliases": ["ruby", "rb"] +} \ No newline at end of file diff --git a/packages/ruby/2.5.1/run b/packages/ruby/2.5.1/run new file mode 100644 index 0000000..0512583 --- /dev/null +++ b/packages/ruby/2.5.1/run @@ -0,0 +1,3 @@ +#!/bin/bash + +ruby $* \ No newline at end of file diff --git a/packages/ruby/2.5.1/test.rb b/packages/ruby/2.5.1/test.rb new file mode 100644 index 0000000..3fca0dd --- /dev/null +++ b/packages/ruby/2.5.1/test.rb @@ -0,0 +1 @@ +puts("OK") \ No newline at end of file From a1aba52c3a152b3e69e10591e28a33edd9f7a65d Mon Sep 17 00:00:00 2001 From: Dan Vargas Date: Sat, 13 Mar 2021 12:49:09 -0600 Subject: [PATCH 26/54] pkg(php-8.0.2): Fix environment bin path --- packages/php/8.0.2/environment | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/php/8.0.2/environment b/packages/php/8.0.2/environment index 153022d..bd0ff98 100644 --- a/packages/php/8.0.2/environment +++ b/packages/php/8.0.2/environment @@ -1 +1 @@ -export PATH=$PWD/php-8.0.2/bin:$PATH \ No newline at end of file +export PATH=$PWD/bin:$PATH \ No newline at end of file From cbb71341e39134276c0c1f632381d9ea8cdc5513 Mon Sep 17 00:00:00 2001 From: Victor Frazao Date: Sat, 13 Mar 2021 18:40:32 -0500 Subject: [PATCH 27/54] pkg(go-1.16.2): Added go 1.16.2 --- packages/go/1.16.2/build.sh | 4 ++++ packages/go/1.16.2/environment | 2 ++ packages/go/1.16.2/metadata.json | 6 ++++++ packages/go/1.16.2/run | 3 +++ packages/go/1.16.2/test.go | 7 +++++++ 5 files changed, 22 insertions(+) create mode 100755 packages/go/1.16.2/build.sh create mode 100644 packages/go/1.16.2/environment create mode 100644 packages/go/1.16.2/metadata.json create mode 100644 packages/go/1.16.2/run create mode 100644 packages/go/1.16.2/test.go diff --git a/packages/go/1.16.2/build.sh b/packages/go/1.16.2/build.sh new file mode 100755 index 0000000..b2481f2 --- /dev/null +++ b/packages/go/1.16.2/build.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +wget https://golang.org/dl/go1.16.2.linux-amd64.tar.gz +tar -xzf go1.16.2.linux-amd64.tar.gz +rm go1.16.2.linux-amd64.tar.gz diff --git a/packages/go/1.16.2/environment b/packages/go/1.16.2/environment new file mode 100644 index 0000000..1788638 --- /dev/null +++ b/packages/go/1.16.2/environment @@ -0,0 +1,2 @@ +export PATH=$PWD/go/bin:$PATH +export GOPATH=$PWD/gopath \ No newline at end of file diff --git a/packages/go/1.16.2/metadata.json b/packages/go/1.16.2/metadata.json new file mode 100644 index 0000000..7f9e443 --- /dev/null +++ b/packages/go/1.16.2/metadata.json @@ -0,0 +1,6 @@ +{ + "language": "go", + "version": "1.16.2", + "author": "Victor Frazao ", + "aliases": ["go", "golang"] +} diff --git a/packages/go/1.16.2/run b/packages/go/1.16.2/run new file mode 100644 index 0000000..67fc46f --- /dev/null +++ b/packages/go/1.16.2/run @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +GOCACHE=$PWD go run $* diff --git a/packages/go/1.16.2/test.go b/packages/go/1.16.2/test.go new file mode 100644 index 0000000..c06f485 --- /dev/null +++ b/packages/go/1.16.2/test.go @@ -0,0 +1,7 @@ +package main + +import "fmt" + +func main() { + fmt.Println("OK") +} From 225960640919601853b66f88d33c3c6541264254 Mon Sep 17 00:00:00 2001 From: Thomas Date: Sun, 14 Mar 2021 17:47:20 +1300 Subject: [PATCH 28/54] Update build.sh --- packages/go/1.16.2/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/go/1.16.2/build.sh b/packages/go/1.16.2/build.sh index b2481f2..2021e1f 100755 --- a/packages/go/1.16.2/build.sh +++ b/packages/go/1.16.2/build.sh @@ -1,4 +1,4 @@ #!/usr/bin/env bash -wget https://golang.org/dl/go1.16.2.linux-amd64.tar.gz +curl https://golang.org/dl/go1.16.2.linux-amd64.tar.gz -o go1.16.2.linux-amd64.tar.gz tar -xzf go1.16.2.linux-amd64.tar.gz rm go1.16.2.linux-amd64.tar.gz From eac95d8c6bacfab94f686f123f743402acb43ba1 Mon Sep 17 00:00:00 2001 From: Thomas Date: Sun, 14 Mar 2021 17:48:05 +1300 Subject: [PATCH 29/54] pkg(go-1.16.2): use $@ --- packages/go/1.16.2/run | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/go/1.16.2/run b/packages/go/1.16.2/run index 67fc46f..09656af 100644 --- a/packages/go/1.16.2/run +++ b/packages/go/1.16.2/run @@ -1,3 +1,3 @@ #!/usr/bin/env bash -GOCACHE=$PWD go run $* +GOCACHE=$PWD go run "$@" From 4dade61599d59c941ff373198be050e07562bada Mon Sep 17 00:00:00 2001 From: Victor Frazao Date: Sun, 14 Mar 2021 10:44:26 -0400 Subject: [PATCH 30/54] pkg(go-1.16.2): Fix build.sh to follow redirects --- packages/go/1.16.2/build.sh | 2 +- packages/go/1.16.2/environment | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/go/1.16.2/build.sh b/packages/go/1.16.2/build.sh index 2021e1f..c45964c 100755 --- a/packages/go/1.16.2/build.sh +++ b/packages/go/1.16.2/build.sh @@ -1,4 +1,4 @@ #!/usr/bin/env bash -curl https://golang.org/dl/go1.16.2.linux-amd64.tar.gz -o go1.16.2.linux-amd64.tar.gz +curl -LO https://golang.org/dl/go1.16.2.linux-amd64.tar.gz tar -xzf go1.16.2.linux-amd64.tar.gz rm go1.16.2.linux-amd64.tar.gz diff --git a/packages/go/1.16.2/environment b/packages/go/1.16.2/environment index 1788638..88299c1 100644 --- a/packages/go/1.16.2/environment +++ b/packages/go/1.16.2/environment @@ -1,2 +1,2 @@ export PATH=$PWD/go/bin:$PATH -export GOPATH=$PWD/gopath \ No newline at end of file +export GOPATH=$PWD/gopath From 172cbd92a1b4f5f83ddb908a4ae9b734aba3c462 Mon Sep 17 00:00:00 2001 From: Victor Frazao Date: Sun, 14 Mar 2021 12:40:31 -0400 Subject: [PATCH 31/54] pkg(julia-1.5.4): Added julia 1.5.4 --- packages/julia/1.5.4/build.sh | 5 +++++ packages/julia/1.5.4/environment | 3 +++ packages/julia/1.5.4/metadata.json | 6 ++++++ packages/julia/1.5.4/run | 4 ++++ packages/julia/1.5.4/test.jl | 1 + 5 files changed, 19 insertions(+) create mode 100755 packages/julia/1.5.4/build.sh create mode 100644 packages/julia/1.5.4/environment create mode 100644 packages/julia/1.5.4/metadata.json create mode 100755 packages/julia/1.5.4/run create mode 100644 packages/julia/1.5.4/test.jl diff --git a/packages/julia/1.5.4/build.sh b/packages/julia/1.5.4/build.sh new file mode 100755 index 0000000..46beaeb --- /dev/null +++ b/packages/julia/1.5.4/build.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +curl -OL https://julialang-s3.julialang.org/bin/linux/x64/1.5/julia-1.5.4-linux-x86_64.tar.gz +tar zxvf julia-1.5.4-linux-x86_64.tar.gz +rm julia-1.5.4-linux-x86_64.tar.gz diff --git a/packages/julia/1.5.4/environment b/packages/julia/1.5.4/environment new file mode 100644 index 0000000..a20e0bf --- /dev/null +++ b/packages/julia/1.5.4/environment @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +export PATH=$PWD/julia-1.5.4/bin:$PATH diff --git a/packages/julia/1.5.4/metadata.json b/packages/julia/1.5.4/metadata.json new file mode 100644 index 0000000..414bc41 --- /dev/null +++ b/packages/julia/1.5.4/metadata.json @@ -0,0 +1,6 @@ +{ + "language": "julia", + "version": "1.5.4", + "aliases": [], + "author": "Victor Frazao " +} diff --git a/packages/julia/1.5.4/run b/packages/julia/1.5.4/run new file mode 100755 index 0000000..215025f --- /dev/null +++ b/packages/julia/1.5.4/run @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +# Put instructions to run the runtime +julia --startup-file=no --history-file=no "$@" diff --git a/packages/julia/1.5.4/test.jl b/packages/julia/1.5.4/test.jl new file mode 100644 index 0000000..dc98a27 --- /dev/null +++ b/packages/julia/1.5.4/test.jl @@ -0,0 +1 @@ +println("OK") From 4e26a48a6e153bda3d49543e929b0d59fd9ef33e Mon Sep 17 00:00:00 2001 From: Victor Frazao Date: Sun, 14 Mar 2021 12:54:57 -0400 Subject: [PATCH 32/54] pkg(julia-1.5.4): Add alias "jl" --- packages/julia/1.5.4/metadata.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/julia/1.5.4/metadata.json b/packages/julia/1.5.4/metadata.json index 414bc41..3dfba09 100644 --- a/packages/julia/1.5.4/metadata.json +++ b/packages/julia/1.5.4/metadata.json @@ -1,6 +1,6 @@ { "language": "julia", "version": "1.5.4", - "aliases": [], + "aliases": ["jl"], "author": "Victor Frazao " } From 11f415248d4b71d1ad67fcdf88f8d4f6c6e05c5b Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Mon, 15 Mar 2021 18:29:40 +1300 Subject: [PATCH 33/54] ci(package): automatically build and upload --- .github/workflows/package-push.yaml | 46 +++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 .github/workflows/package-push.yaml diff --git a/.github/workflows/package-push.yaml b/.github/workflows/package-push.yaml new file mode 100644 index 0000000..17cd7e0 --- /dev/null +++ b/.github/workflows/package-push.yaml @@ -0,0 +1,46 @@ +name: 'Package Pushed' + +on: + push: + branches: + - master + - v3 + paths: + - packages/** + + +jobs: + build-pkg: + name: Build package + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Login to GitHub registry + uses: docker/login-action@v1 + with: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + registry: docker.pkg.github.com + + - name: Get list of changed files + uses: lots0logs/gh-action-get-changed-files@2.1.4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Build Packages + run: | + PACKAGES=$(jq '.[]' -r ${HOME}/files.json | awk -F/ '{ print $2 "-" $3 }' | sort -u) + echo "Packages: $PACKAGES" + docker run -v "${{ github.workspace }}:/piston" docker.pkg.github.com/engineer-man/piston/repo-builder:latest --no-server $PACKAGES + ls -la packages + + - name: Upload Packages + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: packages/*.pkg.tar.gz + tag: pkgs + overwrite: true + file_glob: true From 7d7b83e9512d4737fa3c0b83e7466586bab77993 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Mon, 15 Mar 2021 18:30:34 +1300 Subject: [PATCH 34/54] pkg(bash-5.1.0): add correct file extension --- packages/bash/5.1.0/{test.bash => test.bash.sh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/bash/5.1.0/{test.bash => test.bash.sh} (100%) diff --git a/packages/bash/5.1.0/test.bash b/packages/bash/5.1.0/test.bash.sh similarity index 100% rename from packages/bash/5.1.0/test.bash rename to packages/bash/5.1.0/test.bash.sh From 7be0ca1626aa33da9fc62f7b0aa6269d274394ce Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Mon, 15 Mar 2021 18:48:01 +1300 Subject: [PATCH 35/54] ci(release): create index on release edit --- .github/workflows/package-index-release.yaml | 34 ++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .github/workflows/package-index-release.yaml diff --git a/.github/workflows/package-index-release.yaml b/.github/workflows/package-index-release.yaml new file mode 100644 index 0000000..ec9d456 --- /dev/null +++ b/.github/workflows/package-index-release.yaml @@ -0,0 +1,34 @@ +on: + release: + type: [edited] + +jobs: + create_index: + if: github.ref == 'pkgs' + runs-on: ubuntu-latest + steps: + - name: "Download all release assets" + run: curl -s https://api.github.com/repos/engineer-man/piston/releases/39776531 | jq '.assets[].browser_download_url' -r | xargs curl -sLO + - name: "Generate index file" + run: | + echo "" > index + BASEURL=https://github.com/engineer-man/piston/releases/download/pkgs/ + for pkg in *.pkg.tar.gz + do + PKGFILE=$(basename $pkg) + PKGFILENAME=$(echo $PKGFILE | sed 's/\.pkg\.tar\.gz//g') + + PKGNAME=$(echo $PKGFILENAME | grep -oP '^\K.+(?=-)') + PKGVERSION=$(echo $PKGFILENAME | grep -oP '^.+-\K.+') + PKGCHECKSUM=$(sha256sum $PKGFILE | awk '{print $1}') + echo "$PKGNAME,$PKGVERSION,$PKGCHECKSUM,$BASEURL$PKGFILE" >> index + echo "Adding package $PKGNAME-$PKGVERSION" + done + - name: Upload index + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: index + tag: pkgs + overwrite: true + file_glob: true \ No newline at end of file From 9589948ea79036ca606a8bfc17246c29b9308020 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Mon, 15 Mar 2021 18:49:09 +1300 Subject: [PATCH 36/54] pkg(deno-1.7.5): Give correct extension for CI testing --- packages/deno/1.7.5/{test.ts => test.deno.ts} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/deno/1.7.5/{test.ts => test.deno.ts} (100%) diff --git a/packages/deno/1.7.5/test.ts b/packages/deno/1.7.5/test.deno.ts similarity index 100% rename from packages/deno/1.7.5/test.ts rename to packages/deno/1.7.5/test.deno.ts From 49687d4a28b6569a4f7dc6f00f6123bd168b8ef3 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Mon, 15 Mar 2021 18:53:24 +1300 Subject: [PATCH 37/54] ci(release): name workflow --- .github/workflows/package-index-release.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/package-index-release.yaml b/.github/workflows/package-index-release.yaml index ec9d456..ff45f38 100644 --- a/.github/workflows/package-index-release.yaml +++ b/.github/workflows/package-index-release.yaml @@ -1,3 +1,5 @@ +name: "Package Index" + on: release: type: [edited] From 3c5336e5160cf762dae6c119d7887d0656a8ea55 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Mon, 15 Mar 2021 18:54:12 +1300 Subject: [PATCH 38/54] ci(release): use key types not type --- .github/workflows/package-index-release.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/package-index-release.yaml b/.github/workflows/package-index-release.yaml index ff45f38..db1920f 100644 --- a/.github/workflows/package-index-release.yaml +++ b/.github/workflows/package-index-release.yaml @@ -2,7 +2,8 @@ name: "Package Index" on: release: - type: [edited] + types: + - edited jobs: create_index: From 4c7ec9689e3107e78893c5728b345b1bc9b7824d Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Mon, 15 Mar 2021 18:56:32 +1300 Subject: [PATCH 39/54] ci(release): remove filters --- .github/workflows/package-index-release.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/package-index-release.yaml b/.github/workflows/package-index-release.yaml index db1920f..958a482 100644 --- a/.github/workflows/package-index-release.yaml +++ b/.github/workflows/package-index-release.yaml @@ -2,12 +2,9 @@ name: "Package Index" on: release: - types: - - edited jobs: create_index: - if: github.ref == 'pkgs' runs-on: ubuntu-latest steps: - name: "Download all release assets" From b9d4ca94dfcee5a8b6c080401ba512eeb1971d1e Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Mon, 15 Mar 2021 18:57:35 +1300 Subject: [PATCH 40/54] ci(release): add name to job --- .github/workflows/package-index-release.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/package-index-release.yaml b/.github/workflows/package-index-release.yaml index 958a482..07cd1a1 100644 --- a/.github/workflows/package-index-release.yaml +++ b/.github/workflows/package-index-release.yaml @@ -5,6 +5,7 @@ on: jobs: create_index: + name: Create Index runs-on: ubuntu-latest steps: - name: "Download all release assets" From e070c873220a8818d492d8b5110e7f84f2dbc7ac Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Mon, 15 Mar 2021 18:59:06 +1300 Subject: [PATCH 41/54] ci(release): manual dispatch --- .github/workflows/package-index-release.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/package-index-release.yaml b/.github/workflows/package-index-release.yaml index 07cd1a1..306b1af 100644 --- a/.github/workflows/package-index-release.yaml +++ b/.github/workflows/package-index-release.yaml @@ -2,6 +2,7 @@ name: "Package Index" on: release: + workflow_dispatch: jobs: create_index: From 53841e0d5092ffbbac670ce3f619dbdf12c28fd3 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Mon, 15 Mar 2021 19:03:02 +1300 Subject: [PATCH 42/54] ci(release): remove section from name --- .../{package-index-release.yaml => package-release.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{package-index-release.yaml => package-release.yaml} (100%) diff --git a/.github/workflows/package-index-release.yaml b/.github/workflows/package-release.yaml similarity index 100% rename from .github/workflows/package-index-release.yaml rename to .github/workflows/package-release.yaml From b02fe3eefb5a008420a88a85e7c885eb98e0e964 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Mon, 15 Mar 2021 19:14:02 +1300 Subject: [PATCH 43/54] ci(release): use latest release always --- .github/workflows/package-release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/package-release.yaml b/.github/workflows/package-release.yaml index 306b1af..778fba5 100644 --- a/.github/workflows/package-release.yaml +++ b/.github/workflows/package-release.yaml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest steps: - name: "Download all release assets" - run: curl -s https://api.github.com/repos/engineer-man/piston/releases/39776531 | jq '.assets[].browser_download_url' -r | xargs curl -sLO + run: curl -s https://api.github.com/repos/engineer-man/piston/releases/latest | jq '.assets[].browser_download_url' -r | xargs curl -sLO - name: "Generate index file" run: | echo "" > index From 9c2f562e019fe807660d42782e4a36c58c93855f Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Mon, 15 Mar 2021 19:16:38 +1300 Subject: [PATCH 44/54] repo: build with correct platform --- repo/entrypoint.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/repo/entrypoint.sh b/repo/entrypoint.sh index a0f7e1a..6c47e37 100755 --- a/repo/entrypoint.sh +++ b/repo/entrypoint.sh @@ -23,7 +23,7 @@ do else if [[ $BUILD -eq 1 ]]; then echo "Building package $pkg" - make -j16 $pkg.pkg.tar.gz + make -j16 $pkg.pkg.tar.gz PLATFORM=docker-debian echo "Done with package $pkg" elif [[ $CI -eq 1 ]]; then echo "Commit SHA: $pkg" @@ -36,7 +36,7 @@ do echo "Building packages: $PACKAGES" for package in "$PACKAGES"; do - make -j16 $package.pkg.tar.gz + make -j16 $package.pkg.tar.gz PLATFORM=docker-debian done else From 92a0b13ba7a4e8258884a941ecb5f96e82f87d3d Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Mon, 15 Mar 2021 19:18:01 +1300 Subject: [PATCH 45/54] ci(release): merge with push --- .github/workflows/package-push.yaml | 30 +++++++++++++++++++++ .github/workflows/package-release.yaml | 36 -------------------------- 2 files changed, 30 insertions(+), 36 deletions(-) delete mode 100644 .github/workflows/package-release.yaml diff --git a/.github/workflows/package-push.yaml b/.github/workflows/package-push.yaml index 17cd7e0..6cb3f56 100644 --- a/.github/workflows/package-push.yaml +++ b/.github/workflows/package-push.yaml @@ -44,3 +44,33 @@ jobs: tag: pkgs overwrite: true file_glob: true + create-index: + name: Create Index + runs-on: ubuntu-latest + needs: build-pkg + steps: + - name: "Download all release assets" + run: curl -s https://api.github.com/repos/engineer-man/piston/releases/latest | jq '.assets[].browser_download_url' -r | xargs curl -sLO + - name: "Generate index file" + run: | + echo "" > index + BASEURL=https://github.com/engineer-man/piston/releases/download/pkgs/ + for pkg in *.pkg.tar.gz + do + PKGFILE=$(basename $pkg) + PKGFILENAME=$(echo $PKGFILE | sed 's/\.pkg\.tar\.gz//g') + + PKGNAME=$(echo $PKGFILENAME | grep -oP '^\K.+(?=-)') + PKGVERSION=$(echo $PKGFILENAME | grep -oP '^.+-\K.+') + PKGCHECKSUM=$(sha256sum $PKGFILE | awk '{print $1}') + echo "$PKGNAME,$PKGVERSION,$PKGCHECKSUM,$BASEURL$PKGFILE" >> index + echo "Adding package $PKGNAME-$PKGVERSION" + done + - name: Upload index + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: index + tag: pkgs + overwrite: true + file_glob: true \ No newline at end of file diff --git a/.github/workflows/package-release.yaml b/.github/workflows/package-release.yaml deleted file mode 100644 index 778fba5..0000000 --- a/.github/workflows/package-release.yaml +++ /dev/null @@ -1,36 +0,0 @@ -name: "Package Index" - -on: - release: - workflow_dispatch: - -jobs: - create_index: - name: Create Index - runs-on: ubuntu-latest - steps: - - name: "Download all release assets" - run: curl -s https://api.github.com/repos/engineer-man/piston/releases/latest | jq '.assets[].browser_download_url' -r | xargs curl -sLO - - name: "Generate index file" - run: | - echo "" > index - BASEURL=https://github.com/engineer-man/piston/releases/download/pkgs/ - for pkg in *.pkg.tar.gz - do - PKGFILE=$(basename $pkg) - PKGFILENAME=$(echo $PKGFILE | sed 's/\.pkg\.tar\.gz//g') - - PKGNAME=$(echo $PKGFILENAME | grep -oP '^\K.+(?=-)') - PKGVERSION=$(echo $PKGFILENAME | grep -oP '^.+-\K.+') - PKGCHECKSUM=$(sha256sum $PKGFILE | awk '{print $1}') - echo "$PKGNAME,$PKGVERSION,$PKGCHECKSUM,$BASEURL$PKGFILE" >> index - echo "Adding package $PKGNAME-$PKGVERSION" - done - - name: Upload index - uses: svenstaro/upload-release-action@v2 - with: - repo_token: ${{ secrets.GITHUB_TOKEN }} - file: index - tag: pkgs - overwrite: true - file_glob: true \ No newline at end of file From fd8e25608ff94b88599ddef06da01b9723a081ad Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Mon, 15 Mar 2021 19:24:59 +1300 Subject: [PATCH 46/54] pkg: bump all binary download packages --- packages/dart/2.12.1/build.sh | 2 +- packages/deno/1.7.5/build.sh | 2 +- packages/go/1.16.2/build.sh | 1 + packages/java/15.0.2/build.sh | 1 + packages/jelly/0.1.31/build.sh | 2 +- packages/julia/1.5.4/build.sh | 1 + packages/kotlin/1.4.31/build.sh | 2 +- packages/node/15.10.0/build.sh | 2 +- packages/typescript/4.2.3/build.sh | 2 +- 9 files changed, 9 insertions(+), 6 deletions(-) diff --git a/packages/dart/2.12.1/build.sh b/packages/dart/2.12.1/build.sh index e01d7ce..d202d7f 100755 --- a/packages/dart/2.12.1/build.sh +++ b/packages/dart/2.12.1/build.sh @@ -8,4 +8,4 @@ rm dart.zip cp -r dart-sdk/* . rm -rf dart-sdk -chmod -R +rx bin \ No newline at end of file +chmod -R +rx bin diff --git a/packages/deno/1.7.5/build.sh b/packages/deno/1.7.5/build.sh index 0b0674c..165d3b6 100755 --- a/packages/deno/1.7.5/build.sh +++ b/packages/deno/1.7.5/build.sh @@ -2,4 +2,4 @@ curl -L https://github.com/denoland/deno/releases/download/v1.7.5/deno-x86_64-un unzip -o deno.zip rm deno.zip -chmod +x deno \ No newline at end of file +chmod +x deno diff --git a/packages/go/1.16.2/build.sh b/packages/go/1.16.2/build.sh index c45964c..5cf70f9 100755 --- a/packages/go/1.16.2/build.sh +++ b/packages/go/1.16.2/build.sh @@ -2,3 +2,4 @@ curl -LO https://golang.org/dl/go1.16.2.linux-amd64.tar.gz tar -xzf go1.16.2.linux-amd64.tar.gz rm go1.16.2.linux-amd64.tar.gz + diff --git a/packages/java/15.0.2/build.sh b/packages/java/15.0.2/build.sh index 3d56ee0..aef09a6 100755 --- a/packages/java/15.0.2/build.sh +++ b/packages/java/15.0.2/build.sh @@ -6,3 +6,4 @@ curl "https://download.java.net/java/GA/jdk15.0.2/0d1cfde4252546c6931946de8db48e tar xzf java.tar.gz --strip-components=1 rm java.tar.gz + diff --git a/packages/jelly/0.1.31/build.sh b/packages/jelly/0.1.31/build.sh index 8aec708..db57122 100755 --- a/packages/jelly/0.1.31/build.sh +++ b/packages/jelly/0.1.31/build.sh @@ -9,4 +9,4 @@ cd build/jelly ../../bin/python3.9 setup.py install --optimize=1 cd ../../ -rm -rf build \ No newline at end of file +rm -rf build diff --git a/packages/julia/1.5.4/build.sh b/packages/julia/1.5.4/build.sh index 46beaeb..d9e746c 100755 --- a/packages/julia/1.5.4/build.sh +++ b/packages/julia/1.5.4/build.sh @@ -3,3 +3,4 @@ curl -OL https://julialang-s3.julialang.org/bin/linux/x64/1.5/julia-1.5.4-linux-x86_64.tar.gz tar zxvf julia-1.5.4-linux-x86_64.tar.gz rm julia-1.5.4-linux-x86_64.tar.gz + diff --git a/packages/kotlin/1.4.31/build.sh b/packages/kotlin/1.4.31/build.sh index 4ea535d..e921dd5 100755 --- a/packages/kotlin/1.4.31/build.sh +++ b/packages/kotlin/1.4.31/build.sh @@ -7,4 +7,4 @@ unzip kotlin.zip rm kotlin.zip cp -r kotlinc/* . -rm -rf kotlinc \ No newline at end of file +rm -rf kotlinc diff --git a/packages/node/15.10.0/build.sh b/packages/node/15.10.0/build.sh index 25d419d..6a84798 100755 --- a/packages/node/15.10.0/build.sh +++ b/packages/node/15.10.0/build.sh @@ -1,4 +1,4 @@ #!/bin/bash curl "https://nodejs.org/dist/v15.10.0/node-v15.10.0-linux-x64.tar.xz" -o node.tar.xz tar xf node.tar.xz --strip-components=1 -rm node.tar.xz \ No newline at end of file +rm node.tar.xz diff --git a/packages/typescript/4.2.3/build.sh b/packages/typescript/4.2.3/build.sh index 83ab333..9182055 100755 --- a/packages/typescript/4.2.3/build.sh +++ b/packages/typescript/4.2.3/build.sh @@ -4,4 +4,4 @@ source ../../node/15.10.0/build.sh source ./environment -bin/npm install -g typescript@4.2.3 \ No newline at end of file +bin/npm install -g typescript@4.2.3 From 0493bcec312e1af4c4a7f615133227cd9ecc581d Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Mon, 15 Mar 2021 19:34:38 +1300 Subject: [PATCH 47/54] ci(package): run curl once per line --- .github/workflows/package-push.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/package-push.yaml b/.github/workflows/package-push.yaml index 6cb3f56..ad33f3e 100644 --- a/.github/workflows/package-push.yaml +++ b/.github/workflows/package-push.yaml @@ -50,7 +50,7 @@ jobs: needs: build-pkg steps: - name: "Download all release assets" - run: curl -s https://api.github.com/repos/engineer-man/piston/releases/latest | jq '.assets[].browser_download_url' -r | xargs curl -sLO + run: curl -s https://api.github.com/repos/engineer-man/piston/releases/latest | jq '.assets[].browser_download_url' -r | xargs -L 1 curl -sLO - name: "Generate index file" run: | echo "" > index From 92899bd880509d4ae709387db2aa6a3382b394b6 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Mon, 15 Mar 2021 19:34:51 +1300 Subject: [PATCH 48/54] pkg(bash-5.1.0): ci bump --- packages/bash/5.1.0/build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/bash/5.1.0/build.sh b/packages/bash/5.1.0/build.sh index 67ba7b8..657ae7a 100755 --- a/packages/bash/5.1.0/build.sh +++ b/packages/bash/5.1.0/build.sh @@ -18,3 +18,4 @@ make -j$(nproc) make install -j$(nproc) cd ../ rm -rf build + From 27598b236c3db209022e5f21a8f0c44437f04233 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Mon, 15 Mar 2021 19:38:57 +1300 Subject: [PATCH 49/54] pkg: bump remaining packages --- packages/gawk/5.1.0/build.sh | 1 + packages/gcc/10.2.0/build.sh | 1 + packages/mono/6.12.0/build.sh | 1 + packages/nasm/2.15.5/build.sh | 1 + packages/php/8.0.2/build.sh | 2 +- packages/python/3.9.1/build.sh | 1 + packages/ruby/2.5.1/build.sh | 1 + 7 files changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/gawk/5.1.0/build.sh b/packages/gawk/5.1.0/build.sh index 25a5566..81fb998 100644 --- a/packages/gawk/5.1.0/build.sh +++ b/packages/gawk/5.1.0/build.sh @@ -18,3 +18,4 @@ make -j$(nproc) make install -j$(nproc) cd ../ rm -rf build + diff --git a/packages/gcc/10.2.0/build.sh b/packages/gcc/10.2.0/build.sh index 942e4ac..0f4be80 100755 --- a/packages/gcc/10.2.0/build.sh +++ b/packages/gcc/10.2.0/build.sh @@ -23,3 +23,4 @@ make -j$(nproc) make install -j$(nproc) cd ../ rm -rf build obj + diff --git a/packages/mono/6.12.0/build.sh b/packages/mono/6.12.0/build.sh index 31bfaa9..2cecc07 100755 --- a/packages/mono/6.12.0/build.sh +++ b/packages/mono/6.12.0/build.sh @@ -17,3 +17,4 @@ make install -j$(nproc) cd ../../ rm -rf build + diff --git a/packages/nasm/2.15.5/build.sh b/packages/nasm/2.15.5/build.sh index 1e1f4c3..0f39f6f 100755 --- a/packages/nasm/2.15.5/build.sh +++ b/packages/nasm/2.15.5/build.sh @@ -18,3 +18,4 @@ make -j$(nproc) make install -j$(nproc) cd ../ rm -rf build + diff --git a/packages/php/8.0.2/build.sh b/packages/php/8.0.2/build.sh index 0a94615..b84a80d 100755 --- a/packages/php/8.0.2/build.sh +++ b/packages/php/8.0.2/build.sh @@ -17,4 +17,4 @@ make -j$(nproc) make install -j$(nproc) cd ../../ -rm -rf build \ No newline at end of file +rm -rf build diff --git a/packages/python/3.9.1/build.sh b/packages/python/3.9.1/build.sh index 875a191..ec2a2d5 100755 --- a/packages/python/3.9.1/build.sh +++ b/packages/python/3.9.1/build.sh @@ -18,3 +18,4 @@ cd .. rm -rf build + diff --git a/packages/ruby/2.5.1/build.sh b/packages/ruby/2.5.1/build.sh index 6f7def1..7403e87 100755 --- a/packages/ruby/2.5.1/build.sh +++ b/packages/ruby/2.5.1/build.sh @@ -17,3 +17,4 @@ make install -j$(nproc) cd .. rm -rf build + From 8b70b7d82ea6cdba1a296199738adb670c8513d7 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Mon, 15 Mar 2021 19:39:55 +1300 Subject: [PATCH 50/54] api: update default repo url --- api/src/config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/config.js b/api/src/config.js index 2ad3ffc..9af9b26 100644 --- a/api/src/config.js +++ b/api/src/config.js @@ -105,7 +105,7 @@ const options = [ { key: 'repo_url', desc: 'URL of repo index', - default: 'https://github.com', + default: 'https://github.com/engineer-man/piston/releases/download/pkgs/index', validators: [] } ]; From 8c6552d19e9ae1293f6f49a061aa32dac992f952 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Mon, 15 Mar 2021 22:08:37 +1300 Subject: [PATCH 51/54] api: start at user 1001 by default --- api/src/config.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/src/config.js b/api/src/config.js index 9af9b26..ade4a36 100644 --- a/api/src/config.js +++ b/api/src/config.js @@ -57,7 +57,7 @@ const options = [ { key: 'runner_uid_min', desc: 'Minimum uid to use for runner', - default: 1000, + default: 1001, validators: [] }, { @@ -69,7 +69,7 @@ const options = [ { key: 'runner_gid_min', desc: 'Minimum gid to use for runner', - default: 1000, + default: 1001, validators: [] }, { From c5938dfd40e4d11c47fc711f463c97aa3cd75e96 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Mon, 15 Mar 2021 22:40:51 +1300 Subject: [PATCH 52/54] pkg(deno-1.7.5): ci bump --- packages/deno/1.7.5/build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/deno/1.7.5/build.sh b/packages/deno/1.7.5/build.sh index 165d3b6..df146dc 100755 --- a/packages/deno/1.7.5/build.sh +++ b/packages/deno/1.7.5/build.sh @@ -3,3 +3,4 @@ unzip -o deno.zip rm deno.zip chmod +x deno + From 4a9ad97645a5eb9c3aa6790c482e30fe720b7578 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Mon, 15 Mar 2021 22:55:42 +1300 Subject: [PATCH 53/54] pkg(Makefile): update to strip dir properly --- packages/Makefile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/Makefile b/packages/Makefile index c0cf6fe..22566f2 100644 --- a/packages/Makefile +++ b/packages/Makefile @@ -11,10 +11,11 @@ build build-all: $(addsuffix .pkg.tar.gz, ${PACKAGES}) define PKG_RULE -$(1).pkg.tar.gz: $(subst -,/,$(1))/ $(subst -,/,$(1))/pkg-info.json +$(1).pkg.tar.gz: $(subst -,/,$(1)) $(subst -,/,$(1))/pkg-info.json cd $$< && chmod +x ./build.sh && ./build.sh rm -f $$@ - tar czf $$@ $$< --transform='s|$$<||' + + tar czf $$@ -C $$< . endef $(foreach pkg,$(PACKAGES),$(eval $(call PKG_RULE,$(pkg)))) From 90bc5848a12f12e65a7be432dee927fcf0f43772 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Mon, 15 Mar 2021 22:55:59 +1300 Subject: [PATCH 54/54] pkg(deno-1.7.5): bump ci --- packages/deno/1.7.5/build.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/deno/1.7.5/build.sh b/packages/deno/1.7.5/build.sh index df146dc..165d3b6 100755 --- a/packages/deno/1.7.5/build.sh +++ b/packages/deno/1.7.5/build.sh @@ -3,4 +3,3 @@ unzip -o deno.zip rm deno.zip chmod +x deno -