mirror of
https://github.com/engineer-man/piston.git
synced 2025-04-22 04:56:30 +02:00
Compare commits
No commits in common. "e6a1fd06a97efe4995c64f633fbc785d8104cfc2" and "ff69a28a68b00082937108473cc780afbc14d1e6" have entirely different histories.
e6a1fd06a9
...
ff69a28a68
15 changed files with 40 additions and 119 deletions
|
|
@ -1,9 +1,7 @@
|
|||
FROM node:15.8.0-buster-slim
|
||||
RUN dpkg-reconfigure -p critical dash
|
||||
RUN apt-get update && apt-get install -y gnupg tar coreutils util-linux \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN for i in $(seq 1001 1500); do \
|
||||
FROM node:15.8.0-alpine3.13
|
||||
RUN apk add --no-cache gnupg tar bash coreutils shadow util-linux
|
||||
RUN userdel -r node
|
||||
RUN for i in $(seq 1000 1500); do \
|
||||
groupadd -g $i runner$i && \
|
||||
useradd -M runner$i -g $i -u $i ; \
|
||||
done
|
||||
|
|
|
|||
|
|
@ -131,12 +131,10 @@ class Job {
|
|||
if(this.state != job_states.PRIMED) throw new Error('Job must be in primed state, current state: ' + this.state.toString());
|
||||
logger.info(`Executing job uuid=${this.uuid} uid=${this.uid} gid=${this.gid} runtime=${this.runtime.toString()}`);
|
||||
logger.debug('Compiling');
|
||||
var compile = undefined;
|
||||
if(this.runtime.compiled)
|
||||
compile = await this.safe_call(
|
||||
path.join(this.runtime.pkgdir, 'compile'),
|
||||
this.files.map(x=>x.name),
|
||||
this.timeouts.compile);
|
||||
const compile = this.runtime.compiled && await this.safe_call(
|
||||
path.join(this.runtime.pkgdir, 'compile'),
|
||||
[this.main, ...this.files],
|
||||
this.timeouts.compile);
|
||||
|
||||
logger.debug('Running');
|
||||
|
||||
|
|
|
|||
|
|
@ -10,8 +10,7 @@ module.exports = {
|
|||
body('language')
|
||||
.isString(), // eslint-disable-line snakecasejs/snakecasejs
|
||||
body('version')
|
||||
.isString(), // eslint-disable-line snakecasejs/snakecasejs
|
||||
// isSemVer requires it to be a version, not a selector
|
||||
.isSemVer(), // eslint-disable-line snakecasejs/snakecasejs
|
||||
body('files')
|
||||
.isArray(), // eslint-disable-line snakecasejs/snakecasejs
|
||||
body('files.*.name')
|
||||
|
|
@ -21,9 +20,7 @@ module.exports = {
|
|||
.contains('/'),
|
||||
body('files.*.content')
|
||||
.isString(), // eslint-disable-line snakecasejs/snakecasejs
|
||||
body('compile_timeout')
|
||||
.isNumeric(), // eslint-disable-line snakecasejs/snakecasejs
|
||||
body('run_timeout')
|
||||
body('*_timeout')
|
||||
.isNumeric(), // eslint-disable-line snakecasejs/snakecasejs
|
||||
body('stdin')
|
||||
.isString(), // eslint-disable-line snakecasejs/snakecasejs
|
||||
|
|
|
|||
|
|
@ -93,17 +93,11 @@ const app = express();
|
|||
app.get ('/repos/:repo_slug/packages', ppman_routes.repo_packages_validators, validate, ppman_routes.repo_packages);
|
||||
app.get ('/repos/:repo_slug/packages/:language/:version', ppman_routes.package_info_validators, validate, ppman_routes.package_info);
|
||||
app.post ('/repos/:repo_slug/packages/:language/:version', ppman_routes.package_info_validators, validate, ppman_routes.package_install);
|
||||
app.delete('/repos/:repo_slug/packages/:language/:version', ppman_routes.package_info_validators, validate, ppman_routes.package_uninstall);
|
||||
app.delete('/repos/:repo_slug/packages/:language/:version', ppman_routes.package_info_validators, validate, ppman_routes.package_uninstall); //TODO
|
||||
|
||||
const executor_routes = require('./executor/routes');
|
||||
app.post ('/jobs', executor_routes.run_job_validators, validate, executor_routes.run_job);
|
||||
|
||||
app.get ('/runtimes', (_, res) => res.json_success({runtimes: runtime.map(rt=>({
|
||||
language: rt.language,
|
||||
version: rt.version.raw,
|
||||
author: rt.author
|
||||
}))}))
|
||||
|
||||
logger.debug('Calling app.listen');
|
||||
const [address,port] = config.bind_address.split(':');
|
||||
|
||||
|
|
|
|||
|
|
@ -52,7 +52,7 @@ module.exports = {
|
|||
body('url')
|
||||
.notEmpty() // eslint-disable-line snakecasejs/snakecasejs
|
||||
.bail()
|
||||
.isURL({require_host: false, require_protocol: true, protocols: ['http','https','file']}) // eslint-disable-line snakecasejs/snakecasejs
|
||||
.isURL({require_protocol: true}) // eslint-disable-line snakecasejs/snakecasejs
|
||||
|
||||
],
|
||||
async repo_add(req, res){
|
||||
|
|
|
|||
|
|
@ -48,7 +48,7 @@ class Runtime {
|
|||
return res;
|
||||
}
|
||||
|
||||
get compiled(){
|
||||
get compile(){
|
||||
if(this.#compiled === undefined) this.#compiled = fss.exists_sync(path.join(this.pkgdir, 'compile'));
|
||||
return this.#compiled;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,8 +7,8 @@ exports.describe = 'Installs the named package'
|
|||
|
||||
|
||||
const msg_format = {
|
||||
'color': p => `${p.language ? chalk.green.bold('✓') : chalk.red.bold('❌')} Installation ${p.language ? "succeeded" : "failed: " + p.message}`,
|
||||
'monochrome': p => `Installation ${p.language ? "succeeded" : "failed: " + p.message}`,
|
||||
'color': p => `${p.success ? chalk.green.bold('✓') : chalk.red.bold('❌')} Installation ${p.success ? "succeeded" : "failed: " + p.message}`,
|
||||
'monochrome': p => `Installation ${p.success ? "succeeded" : "failed: " + p.message}`,
|
||||
'json': JSON.stringify
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,19 +2,20 @@
|
|||
|
||||
# Variables
|
||||
PKG_FILES=$(filter-out common.mk,$(wildcard *.mk))
|
||||
PKG_SLUGS=$(foreach pkg, ${PKG_FILES}, $(addprefix $(shell make -f ${pkg} name VERSION=UNKNOWN)-, $(shell make -f ${pkg} versions VERSION=UNKNOWN)))
|
||||
PKG_SLUGS=$(foreach pkg, ${PKG_FILES}, $(addprefix $(shell make -f ${pkg} name)-, $(shell make -f ${pkg} versions)))
|
||||
|
||||
# Functions
|
||||
define pkg_info
|
||||
$(eval PKG_SLUG=$(patsubst $1-%,%,$2))
|
||||
$(eval PKG_VERSION=$(lastword $(subst -, ,${PKG_SLUG})))
|
||||
$(eval PKG_NAME=$(patsubst %-${PKG_VERSION},%,${PKG_SLUG}))
|
||||
$(eval PKG_PARTS=$(subst -, ,${PKG_SLUG}))
|
||||
$(eval PKG_NAME=$(word 1,${PKG_PARTS}))
|
||||
$(eval PKG_VERSION=$(word 2,${PKG_PARTS}))
|
||||
$(eval PKG_FILE=$(shell grep '^VERSIONS\s*=.*${PKG_VERSION}' $(shell grep "NAME\s*=\s*${PKG_NAME}" ${PKG_FILES} -l) -l))
|
||||
endef
|
||||
|
||||
# Targets
|
||||
|
||||
build: $(foreach pkg, ${PKG_FILES}, $(addprefix build-$(shell make -f ${pkg} name VERSION=UNKNOWN)-, $(lastword $(shell make -f ${pkg} versions VERSION=UNKNOWN))))
|
||||
build: $(foreach pkg, ${PKG_FILES}, $(addprefix build-$(shell make -f ${pkg} name)-, $(lastword $(shell make -f ${pkg} versions))))
|
||||
|
||||
$(addprefix build-, ${PKG_SLUGS}):
|
||||
$(call pkg_info,build,$@)
|
||||
|
|
@ -22,7 +23,7 @@ $(addprefix build-, ${PKG_SLUGS}):
|
|||
|
||||
|
||||
|
||||
clean: $(foreach pkg, ${PKG_FILES}, $(addprefix clean-$(shell make -f ${pkg} name VERSION=UNKNOWN)-, $(shell make -f ${pkg} versions VERSION=UNKNOWN)))
|
||||
clean: $(foreach pkg, ${PKG_FILES}, $(addprefix clean-$(shell make -f ${pkg} name)-, $(shell make -f ${pkg} versions)))
|
||||
rm -rf build/
|
||||
$(addprefix clean-, ${PKG_SLUGS}):
|
||||
$(call pkg_info,clean,$@)
|
||||
|
|
|
|||
|
|
@ -6,8 +6,7 @@ BIN_DIR=${BUILD_DIR}${PKG_SLUG}/
|
|||
RUN_FILE=${BUILD_DIR}run
|
||||
COMPILE_FILE=${BUILD_DIR}compile
|
||||
ENV_FILE=${BIN_DIR}environment
|
||||
INFO_FILE=${BUILD_DIR}pkg-info.json
|
||||
PREFIX=/piston/packages/${NAME}/${VERSION}/${PKG_SLUG}
|
||||
INFO_FILE=${BUILD_DIR}pkg-info.jq
|
||||
|
||||
PKG_FILE=${PKG_SLUG}.pkg.tar.gz
|
||||
|
||||
|
|
@ -32,10 +31,7 @@ versions:
|
|||
name:
|
||||
@echo ${NAME}
|
||||
|
||||
|
||||
.NOTPARALLEL: build
|
||||
build: ${BUILD_DIR} ${PKG_FILE}
|
||||
|
||||
clean:
|
||||
rm -rf ${BUILD_DIR}
|
||||
rm -f ${PKG_FILE}
|
||||
|
|
@ -53,7 +49,7 @@ endif
|
|||
${PKG_FILE}: ${PKG_TARGETS}
|
||||
tar -czC ${BUILD_DIR} -f $@ ${patsubst ${BUILD_DIR}%,%,$?}
|
||||
|
||||
$(patsubst %.json,%.jq,${INFO_FILE}):
|
||||
${INFO_FILE}:
|
||||
echo '.language="${NAME}"' > $@
|
||||
echo '.version="${VERSION}"' >> $@
|
||||
echo '.author="${AUTHOR}"' >> $@
|
||||
|
|
@ -64,14 +60,11 @@ $(patsubst %.json,%.jq,${INFO_FILE}):
|
|||
|
||||
|
||||
# Helpers
|
||||
|
||||
%/: %.tgz
|
||||
cd ${BUILD_DIR} && tar xzf $(patsubst ${BUILD_DIR}%,%,$<)
|
||||
%/: %.tar.gz
|
||||
mkdir -p $@
|
||||
tar xzf $< --strip-components=1 -C $@
|
||||
%/: %.tar.xz
|
||||
mkdir -p $@
|
||||
tar xf $< --strip-components=1 -C $@
|
||||
%/: %.zip
|
||||
mkdir -p $@
|
||||
unzip $< -d $@
|
||||
cd ${BUILD_DIR} && tar xzf $(patsubst ${BUILD_DIR}%,%,$<)
|
||||
|
||||
%.json: %.jq
|
||||
jq '$(shell tr '\n' '|' < $<).' <<< "{}" > $@
|
||||
|
|
@ -1,34 +0,0 @@
|
|||
NAME=csharp-mono
|
||||
AUTHOR=Thomas Hobson <thomas@hexf.me>
|
||||
DEPENDENCIES=
|
||||
COMPILED=true
|
||||
VERSIONS=6.12.0
|
||||
|
||||
include common.mk
|
||||
|
||||
|
||||
VERSION_6.12.0_FULL=6.12.0.122
|
||||
|
||||
VERSION_FULL=${VERSION_${VERSION}_FULL}
|
||||
|
||||
|
||||
${RUN_FILE}:
|
||||
echo 'CODE=$${1/cs/exe}' > $@
|
||||
echo 'shift' >> $@
|
||||
echo 'mono $$CODE $$*' >> $@
|
||||
|
||||
${COMPILE_FILE}:
|
||||
echo 'csc $$*' > $@
|
||||
|
||||
${ENV_FILE}:
|
||||
echo 'export PATH=$$PWD/bin:$$PATH' > $@
|
||||
|
||||
${BIN_DIR}: ${BUILD_DIR}mono-${VERSION_FULL}/
|
||||
$(eval TMP_DIR=${PWD}/${BUILD_DIR}tmpout/)
|
||||
cd $< && ./configure --prefix ${PREFIX}
|
||||
$(MAKE) -j64 -C $<
|
||||
DESTDIR=${TMP_DIR} $(MAKE) -C $< install
|
||||
mv ${TMP_DIR}${PREFIX} ${BIN_DIR} && rm -r ${TMP_DIR}
|
||||
|
||||
${BUILD_DIR}mono-${VERSION_FULL}.tar.xz:
|
||||
curl "https://download.mono-project.com/sources/mono/mono-${VERSION_FULL}.tar.xz" -o $@
|
||||
|
|
@ -1,22 +0,0 @@
|
|||
NAME=deno
|
||||
AUTHOR=Thomas Hobson <thomas@hexf.me>
|
||||
DEPENDENCIES=
|
||||
COMPILED=false
|
||||
VERSIONS=1.7.5
|
||||
|
||||
include common.mk
|
||||
|
||||
|
||||
${RUN_FILE}:
|
||||
echo 'deno run $$*' > $@
|
||||
|
||||
${ENV_FILE}:
|
||||
echo 'export PATH=$$PWD:$$PATH' > $@
|
||||
|
||||
${BIN_DIR}: ${BUILD_DIR}deno-x86_64-unknown-linux-gnu/
|
||||
mkdir -p $@
|
||||
mv $</deno $@
|
||||
chmod +x $@/deno
|
||||
|
||||
${BUILD_DIR}deno-x86_64-unknown-linux-gnu.zip:
|
||||
curl -L "https://github.com/denoland/deno/releases/download/v${VERSION}/deno-x86_64-unknown-linux-gnu.zip" -o $@
|
||||
|
|
@ -14,12 +14,9 @@ ${ENV_FILE}:
|
|||
echo 'export PATH=$$PWD/bin:$$PATH' > $@
|
||||
|
||||
${BIN_DIR}: ${BUILD_DIR}Python-${VERSION}/
|
||||
$(eval TMP_DIR=${PWD}/${BUILD_DIR}tmpout/)
|
||||
cd $< && ./configure --prefix ${PREFIX}
|
||||
$(MAKE) -C $<
|
||||
DESTDIR=${TMP_DIR} $(MAKE) -C $< altinstall
|
||||
mv ${TMP_DIR}${PREFIX} ${BIN_DIR} && rm -rf ${TMP_DIR}
|
||||
cd $< && ./configure --prefix /
|
||||
$(MAKE) -j64 -C $<
|
||||
DESTDIR=../${PKG_SLUG} $(MAKE) -j64 -C $< altinstall || true
|
||||
|
||||
|
||||
${BUILD_DIR}Python-${VERSION}.tar.gz:
|
||||
${BUILD_DIR}Python-${VERSION}.tgz:
|
||||
curl "https://www.python.org/ftp/python/${VERSION}/Python-${VERSION}.tgz" -o $@
|
||||
|
|
|
|||
|
|
@ -1,8 +1,7 @@
|
|||
FROM debian:buster-slim
|
||||
FROM alpine:3.13
|
||||
|
||||
RUN apt-get update && apt-get install -y bc curl git linux-headers-amd64 perl xz-utils python3 python3-pip gnupg jq zlib1g-dev cmake cmake-doc extra-cmake-modules build-essential gcc binutils bash coreutils util-linux pciutils usbutils coreutils binutils findutils grep && \
|
||||
RUN apk add --no-cache python3 py3-pip gnupg jq zlib zlib-dev cmake cmake-doc extra-cmake-modules extra-cmake-modules-doc build-base gcc abuild binutils binutils-doc gcc-doc yq bash coreutils util-linux pciutils usbutils coreutils binutils findutils grep && \
|
||||
ln -sf /bin/bash /bin/sh && \
|
||||
pip3 install 'yq==2.12.0' && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
pip3 install 'yq==2.12.0'
|
||||
|
||||
CMD [ "bash", "/repo/make.sh" ]
|
||||
|
|
@ -3,8 +3,8 @@
|
|||
cd /repo
|
||||
|
||||
# Make packages
|
||||
pushd ../packages/
|
||||
make -j16
|
||||
pushd ../packages/python
|
||||
make build VERSIONS=3.9.1
|
||||
popd
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ echo "baseurl: file://$PWD" >> index.yaml
|
|||
echo "keys: []" >> index.yaml
|
||||
echo "packages: []" >> index.yaml
|
||||
|
||||
#yq -yi '.keys[0] = "0x107DA02C7AE97B084746564B9F1FD9D87950DB6F"' index.yaml
|
||||
yq -yi '.keys[0] = "0x107DA02C7AE97B084746564B9F1FD9D87950DB6F"' index.yaml
|
||||
|
||||
i=-1
|
||||
|
||||
|
|
@ -15,12 +15,12 @@ do
|
|||
PKGFILENAME=$(echo $PKGFILE | sed 's/\.pkg\.tar\.gz//g')
|
||||
PKGNAME=$(echo $PKGFILENAME | grep -oP '^\K.+(?=-)')
|
||||
PKGVERSION=$(echo $PKGFILENAME | grep -oP '^.+-\K.+')
|
||||
BUILDFILE=https://github.com/engineer-man/piston/tree/v3/packages/
|
||||
BUILDFILE=https://github.com/engineer-man/piston/tree/v3/packages/python/
|
||||
SIZE=$(tar tzvf $PKGFILE | sed 's/ \+/ /g' | cut -f3 -d' ' | sed '2,$s/^/+ /' | paste -sd' ' | bc)
|
||||
|
||||
tar xzf $PKGFILE pkg-info.json
|
||||
|
||||
yq -yi ".packages[$i] = {} | .packages[$i].signature = \"\" | .packages[$i].buildfile = \"$BUILDFILE\" | .packages[$i].size = $SIZE | .packages[$i].download = \"$PKGFILE\" | .packages[$i].dependencies = $(jq .dependencies -r pkg-info.json) | .packages[$i].author = $(jq .author pkg-info.json) | .packages[$i].language =\"$PKGNAME\" | .packages[$i].version = \"$PKGVERSION\" | .packages[$i].checksums = {} | .packages[$i].checksums.sha256 = \"$(sha256sum $PKGFILE | awk '{print $1}')\"" index.yaml
|
||||
yq -yi ".packages[$i] = {} | .packages[$i].signature = \"$(cat ${pkg}.asc)\" | .packages[$i].buildfile = \"$BUILDFILE\" | .packages[$i].size = $SIZE | .packages[$i].download = \"$PKGFILE\" | .packages[$i].dependencies = $(jq .dependencies -r pkg-info.json) | .packages[$i].author = $(jq .author pkg-info.json) | .packages[$i].language =\"$PKGNAME\" | .packages[$i].version = \"$PKGVERSION\" | .packages[$i].checksums = {} | .packages[$i].checksums.sha256 = \"$(sha256sum $PKGFILE | awk '{print $1}')\"" index.yaml
|
||||
|
||||
rm pkg-info.json
|
||||
done
|
||||
Loading…
Add table
Add a link
Reference in a new issue