pkg(deno): Added deno version 1.7.5

Closes issue #119

api/Dockerfile

@@ -1,7 +1,9 @@
-FROM node:15.8.0-alpine3.13
+FROM node:15.8.0-buster-slim
-RUN apk add --no-cache gnupg tar bash coreutils shadow util-linux
+RUN dpkg-reconfigure -p critical dash
-RUN userdel -r node
+RUN apt-get update && apt-get install -y gnupg tar coreutils util-linux \
-RUN for i in $(seq 1000 1500); do \
+  && rm -rf /var/lib/apt/lists/*
+
+RUN for i in $(seq 1001 1500); do \
         groupadd -g $i runner$i && \
         useradd -M runner$i -g $i -u $i ; \
     done

api/src/executor/job.js

@@ -131,10 +131,12 @@ class Job {
         if(this.state != job_states.PRIMED) throw new Error('Job must be in primed state, current state: ' + this.state.toString());
         logger.info(`Executing job uuid=${this.uuid} uid=${this.uid} gid=${this.gid} runtime=${this.runtime.toString()}`);
         logger.debug('Compiling');
-        const compile = this.runtime.compiled && await this.safe_call(
+        var compile = undefined;
-            path.join(this.runtime.pkgdir, 'compile'),
+        if(this.runtime.compiled)
-            [this.main, ...this.files],
+            compile = await this.safe_call(
-            this.timeouts.compile);
+                path.join(this.runtime.pkgdir, 'compile'),
+                this.files.map(x=>x.name),
+                this.timeouts.compile);
 
         logger.debug('Running');
 

api/src/executor/routes.js

@@ -10,7 +10,8 @@ module.exports = {
         body('language')
             .isString(), // eslint-disable-line snakecasejs/snakecasejs
         body('version')
-            .isSemVer(), // eslint-disable-line snakecasejs/snakecasejs
+            .isString(), // eslint-disable-line snakecasejs/snakecasejs
+                         // isSemVer requires it to be a version, not a selector
         body('files')
             .isArray(), // eslint-disable-line snakecasejs/snakecasejs
         body('files.*.name')
@@ -20,7 +21,9 @@ module.exports = {
             .contains('/'),
         body('files.*.content')
             .isString(), // eslint-disable-line snakecasejs/snakecasejs
-        body('*_timeout')
+        body('compile_timeout')
+            .isNumeric(), // eslint-disable-line snakecasejs/snakecasejs
+        body('run_timeout')
             .isNumeric(), // eslint-disable-line snakecasejs/snakecasejs
         body('stdin')
             .isString(), // eslint-disable-line snakecasejs/snakecasejs

api/src/index.js

@@ -93,11 +93,17 @@ const app = express();
     app.get   ('/repos/:repo_slug/packages', ppman_routes.repo_packages_validators, validate, ppman_routes.repo_packages);
     app.get   ('/repos/:repo_slug/packages/:language/:version', ppman_routes.package_info_validators, validate, ppman_routes.package_info);
     app.post  ('/repos/:repo_slug/packages/:language/:version', ppman_routes.package_info_validators, validate, ppman_routes.package_install);
-    app.delete('/repos/:repo_slug/packages/:language/:version', ppman_routes.package_info_validators, validate, ppman_routes.package_uninstall); //TODO
+    app.delete('/repos/:repo_slug/packages/:language/:version', ppman_routes.package_info_validators, validate, ppman_routes.package_uninstall);
 
     const executor_routes = require('./executor/routes');
     app.post  ('/jobs', executor_routes.run_job_validators, validate, executor_routes.run_job);
 
+    app.get   ('/runtimes', (_, res) => res.json_success({runtimes: runtime.map(rt=>({
+            language: rt.language,
+            version: rt.version.raw,
+            author: rt.author
+        }))}))
+
     logger.debug('Calling app.listen');
     const [address,port] = config.bind_address.split(':');
 

api/src/ppman/routes.js

@@ -52,7 +52,7 @@ module.exports = {
         body('url')
             .notEmpty() // eslint-disable-line snakecasejs/snakecasejs
             .bail()
-            .isURL({require_protocol: true}) // eslint-disable-line snakecasejs/snakecasejs
+            .isURL({require_host: false, require_protocol: true, protocols: ['http','https','file']}) // eslint-disable-line snakecasejs/snakecasejs
 
     ],
     async repo_add(req, res){

api/src/runtime.js

@@ -48,7 +48,7 @@ class Runtime {
         return res;
     }
 
-    get compile(){
+    get compiled(){
         if(this.#compiled === undefined) this.#compiled = fss.exists_sync(path.join(this.pkgdir, 'compile'));
         return this.#compiled;
     }

cli/commands/ppman_commands/install.js

@@ -7,8 +7,8 @@ exports.describe = 'Installs the named package'
 
 
 const msg_format = {
-    'color': p => `${p.success ? chalk.green.bold('✓') : chalk.red.bold('❌')} Installation ${p.success ? "succeeded" : "failed: " + p.message}`,
+    'color': p => `${p.language ? chalk.green.bold('✓') : chalk.red.bold('❌')} Installation ${p.language ? "succeeded" : "failed: " + p.message}`,
-    'monochrome': p => `Installation ${p.success ? "succeeded" : "failed: " + p.message}`,
+    'monochrome': p => `Installation ${p.language ? "succeeded" : "failed: " + p.message}`,
     'json': JSON.stringify
 
 }

packages/Makefile

@@ -2,20 +2,19 @@
 
 # Variables
 PKG_FILES=$(filter-out common.mk,$(wildcard *.mk))
-PKG_SLUGS=$(foreach pkg, ${PKG_FILES}, $(addprefix $(shell make -f ${pkg} name)-, $(shell make -f ${pkg} versions)))
+PKG_SLUGS=$(foreach pkg, ${PKG_FILES}, $(addprefix $(shell make -f ${pkg} name VERSION=UNKNOWN)-, $(shell make -f ${pkg} versions VERSION=UNKNOWN)))
 
 # Functions
 define pkg_info
 	$(eval PKG_SLUG=$(patsubst $1-%,%,$2))
-	$(eval PKG_PARTS=$(subst -, ,${PKG_SLUG}))
+	$(eval PKG_VERSION=$(lastword $(subst -, ,${PKG_SLUG})))
-	$(eval PKG_NAME=$(word 1,${PKG_PARTS}))
+	$(eval PKG_NAME=$(patsubst %-${PKG_VERSION},%,${PKG_SLUG}))
-	$(eval PKG_VERSION=$(word 2,${PKG_PARTS}))
 	$(eval PKG_FILE=$(shell grep '^VERSIONS\s*=.*${PKG_VERSION}' $(shell grep "NAME\s*=\s*${PKG_NAME}" ${PKG_FILES} -l) -l))
 endef
 
 # Targets
 
-build: $(foreach pkg, ${PKG_FILES}, $(addprefix build-$(shell make -f ${pkg} name)-, $(lastword $(shell make -f ${pkg} versions))))
+build: $(foreach pkg, ${PKG_FILES}, $(addprefix build-$(shell make -f ${pkg} name VERSION=UNKNOWN)-, $(lastword $(shell make -f ${pkg} versions VERSION=UNKNOWN))))
 
 $(addprefix build-, ${PKG_SLUGS}):
 	$(call pkg_info,build,$@)
@@ -23,7 +22,7 @@ $(addprefix build-, ${PKG_SLUGS}):
 
 
 
-clean: $(foreach pkg, ${PKG_FILES}, $(addprefix clean-$(shell make -f ${pkg} name)-, $(shell make -f ${pkg} versions)))
+clean: $(foreach pkg, ${PKG_FILES}, $(addprefix clean-$(shell make -f ${pkg} name VERSION=UNKNOWN)-, $(shell make -f ${pkg} versions VERSION=UNKNOWN)))
 	rm -rf build/
 $(addprefix clean-, ${PKG_SLUGS}):
 	$(call pkg_info,clean,$@)

packages/common.mk

@@ -6,7 +6,8 @@ BIN_DIR=${BUILD_DIR}${PKG_SLUG}/
 RUN_FILE=${BUILD_DIR}run
 COMPILE_FILE=${BUILD_DIR}compile
 ENV_FILE=${BIN_DIR}environment
-INFO_FILE=${BUILD_DIR}pkg-info.jq
+INFO_FILE=${BUILD_DIR}pkg-info.json
+PREFIX=/piston/packages/${NAME}/${VERSION}/${PKG_SLUG}
 
 PKG_FILE=${PKG_SLUG}.pkg.tar.gz
 
@@ -31,7 +32,10 @@ versions:
 name:
 	@echo ${NAME}
 
+
+.NOTPARALLEL: build
 build: ${BUILD_DIR} ${PKG_FILE}
+
 clean: 
 	rm -rf ${BUILD_DIR}
 	rm -f ${PKG_FILE}
@@ -49,7 +53,7 @@ endif
 ${PKG_FILE}: ${PKG_TARGETS}
 	tar -czC ${BUILD_DIR} -f $@ ${patsubst ${BUILD_DIR}%,%,$?}
 	
-${INFO_FILE}:
+$(patsubst %.json,%.jq,${INFO_FILE}):
 	echo '.language="${NAME}"' > $@
 	echo '.version="${VERSION}"' >> $@
 	echo '.author="${AUTHOR}"' >> $@
@@ -60,11 +64,14 @@ ${INFO_FILE}:
 
 
 # Helpers
-
-%/: %.tgz
-	cd ${BUILD_DIR} && tar xzf $(patsubst ${BUILD_DIR}%,%,$<)
 %/: %.tar.gz
-	cd ${BUILD_DIR} && tar xzf $(patsubst ${BUILD_DIR}%,%,$<)
+	mkdir -p $@
-
+	tar xzf $< --strip-components=1 -C $@
+%/: %.tar.xz
+	mkdir -p $@
+	tar xf $< --strip-components=1 -C $@
+%/: %.zip
+	mkdir -p $@
+	unzip $< -d $@
 %.json: %.jq
 	jq '$(shell tr '\n' '|' < $<).' <<< "{}" > $@

packages/csharp-mono.mk

@@ -0,0 +1,34 @@
+NAME=csharp-mono
+AUTHOR=Thomas Hobson <thomas@hexf.me>
+DEPENDENCIES=
+COMPILED=true
+VERSIONS=6.12.0
+
+include common.mk
+
+
+VERSION_6.12.0_FULL=6.12.0.122
+
+VERSION_FULL=${VERSION_${VERSION}_FULL}
+
+
+${RUN_FILE}:
+	echo 'CODE=$${1/cs/exe}' > $@
+	echo 'shift' >> $@
+	echo 'mono $$CODE $$*' >> $@
+
+${COMPILE_FILE}:
+	echo 'csc $$*' > $@
+
+${ENV_FILE}:
+	echo 'export PATH=$$PWD/bin:$$PATH' > $@
+
+${BIN_DIR}: ${BUILD_DIR}mono-${VERSION_FULL}/
+	$(eval TMP_DIR=${PWD}/${BUILD_DIR}tmpout/)
+	cd $< && ./configure --prefix ${PREFIX}
+	$(MAKE) -j64 -C $<
+	DESTDIR=${TMP_DIR} $(MAKE) -C $< install
+	mv ${TMP_DIR}${PREFIX} ${BIN_DIR} && rm -r ${TMP_DIR}
+
+${BUILD_DIR}mono-${VERSION_FULL}.tar.xz: 
+	curl "https://download.mono-project.com/sources/mono/mono-${VERSION_FULL}.tar.xz" -o $@

packages/deno.mk

@@ -0,0 +1,22 @@
+NAME=deno
+AUTHOR=Thomas Hobson <thomas@hexf.me>
+DEPENDENCIES=
+COMPILED=false
+VERSIONS=1.7.5
+
+include common.mk
+
+
+${RUN_FILE}:
+	echo 'deno run $$*' > $@
+
+${ENV_FILE}:
+	echo 'export PATH=$$PWD:$$PATH' > $@
+
+${BIN_DIR}: ${BUILD_DIR}deno-x86_64-unknown-linux-gnu/
+	mkdir -p $@
+	mv $</deno $@
+	chmod +x $@/deno
+	
+${BUILD_DIR}deno-x86_64-unknown-linux-gnu.zip: 
+	curl -L "https://github.com/denoland/deno/releases/download/v${VERSION}/deno-x86_64-unknown-linux-gnu.zip" -o $@

packages/python.mk

@@ -14,9 +14,12 @@ ${ENV_FILE}:
 	echo 'export PATH=$$PWD/bin:$$PATH' > $@
 
 ${BIN_DIR}: ${BUILD_DIR}Python-${VERSION}/
-	cd $< && ./configure --prefix /
+	$(eval TMP_DIR=${PWD}/${BUILD_DIR}tmpout/)
-	$(MAKE) -j64 -C $<
+	cd $< && ./configure --prefix ${PREFIX}
-	DESTDIR=../${PKG_SLUG} $(MAKE) -j64 -C $< altinstall || true
+	$(MAKE) -C $<
+	DESTDIR=${TMP_DIR} $(MAKE) -C $< altinstall
+	mv ${TMP_DIR}${PREFIX} ${BIN_DIR} && rm -rf ${TMP_DIR}
 
-${BUILD_DIR}Python-${VERSION}.tgz: 
+
+${BUILD_DIR}Python-${VERSION}.tar.gz: 
 	curl "https://www.python.org/ftp/python/${VERSION}/Python-${VERSION}.tgz" -o $@

repo/Dockerfile

@@ -1,7 +1,8 @@
-FROM alpine:3.13
+FROM debian:buster-slim
 
-RUN apk add --no-cache python3 py3-pip gnupg jq zlib zlib-dev cmake cmake-doc extra-cmake-modules extra-cmake-modules-doc build-base gcc abuild binutils binutils-doc gcc-doc yq bash coreutils util-linux pciutils usbutils coreutils binutils findutils grep && \
+RUN apt-get update && apt-get install -y bc curl git linux-headers-amd64 perl xz-utils python3 python3-pip gnupg jq zlib1g-dev cmake cmake-doc extra-cmake-modules build-essential gcc binutils bash coreutils util-linux pciutils usbutils coreutils binutils findutils grep && \
         ln -sf /bin/bash /bin/sh && \
-        pip3 install 'yq==2.12.0'
+        pip3 install 'yq==2.12.0' && \
+        rm -rf /var/lib/apt/lists/*
 
 CMD [ "bash", "/repo/make.sh" ]

repo/make.sh

@@ -3,8 +3,8 @@
 cd /repo
 
 # Make packages
-pushd ../packages/python
+pushd ../packages/
-make build VERSIONS=3.9.1
+make -j16
 popd
 
 

repo/mkindex.sh

@@ -3,7 +3,7 @@ echo "baseurl: file://$PWD" >> index.yaml
 echo "keys: []" >> index.yaml
 echo "packages: []" >> index.yaml
 
-yq -yi '.keys[0] = "0x107DA02C7AE97B084746564B9F1FD9D87950DB6F"' index.yaml
+#yq -yi '.keys[0] = "0x107DA02C7AE97B084746564B9F1FD9D87950DB6F"' index.yaml
 
 i=-1
 
@@ -15,12 +15,12 @@ do
     PKGFILENAME=$(echo $PKGFILE | sed 's/\.pkg\.tar\.gz//g')
     PKGNAME=$(echo $PKGFILENAME | grep -oP '^\K.+(?=-)')
     PKGVERSION=$(echo $PKGFILENAME | grep -oP '^.+-\K.+')
-    BUILDFILE=https://github.com/engineer-man/piston/tree/v3/packages/python/
+    BUILDFILE=https://github.com/engineer-man/piston/tree/v3/packages/
     SIZE=$(tar tzvf $PKGFILE | sed 's/ \+/ /g' | cut -f3 -d' ' | sed '2,$s/^/+ /' | paste -sd' ' | bc)
 
     tar xzf $PKGFILE pkg-info.json
 
-    yq -yi ".packages[$i] = {} | .packages[$i].signature = \"$(cat ${pkg}.asc)\" | .packages[$i].buildfile = \"$BUILDFILE\" | .packages[$i].size = $SIZE | .packages[$i].download = \"$PKGFILE\" | .packages[$i].dependencies = $(jq .dependencies -r pkg-info.json) | .packages[$i].author = $(jq .author pkg-info.json) | .packages[$i].language =\"$PKGNAME\" | .packages[$i].version = \"$PKGVERSION\" | .packages[$i].checksums = {} | .packages[$i].checksums.sha256 = \"$(sha256sum $PKGFILE | awk '{print $1}')\"" index.yaml
+    yq -yi ".packages[$i] = {} | .packages[$i].signature = \"\" | .packages[$i].buildfile = \"$BUILDFILE\" | .packages[$i].size = $SIZE | .packages[$i].download = \"$PKGFILE\" | .packages[$i].dependencies = $(jq .dependencies -r pkg-info.json) | .packages[$i].author = $(jq .author pkg-info.json) | .packages[$i].language =\"$PKGNAME\" | .packages[$i].version = \"$PKGVERSION\" | .packages[$i].checksums = {} | .packages[$i].checksums.sha256 = \"$(sha256sum $PKGFILE | awk '{print $1}')\"" index.yaml
     
     rm pkg-info.json
 done