Compare commits

...

27 Commits

Author SHA1 Message Date
Thomas Hobson 39adb13ba1
pkg(dart-2.12.1): Added dart 2.12.1 2021-03-14 01:24:59 +13:00
Thomas Hobson 05dc79d9f8
pkg(nasm-2.15.5): Add nasm 2.15.5 2021-03-14 01:04:46 +13:00
Thomas Hobson a5ba0f5ee6
pkg(kotlin-1.4.31): Added kotlin 1.4.31 2021-03-14 00:35:15 +13:00
Thomas Hobson 56adb6123c
pkg(java-15.0.2): Add java 15.0.2 2021-03-14 00:14:06 +13:00
Thomas Hobson 4db97b005b
deploy: allow exec on jobs dir 2021-03-13 23:29:33 +13:00
Thomas Hobson 9affe1e892
pkg(typescript-4.2.3): Add typescript 4.2.3 2021-03-13 23:28:55 +13:00
Thomas Hobson f852b60b69
pkg(gcc-10.2.0): Add c, c++ and D via gcc 10.2.0 2021-03-13 23:12:50 +13:00
Thomas Hobson 3bec1c3499
cli: fix breaks caused by refactor 2021-03-13 23:09:30 +13:00
Thomas Hobson 4621f34e47
api: fixes from refactor 2021-03-13 23:09:13 +13:00
Thomas Hobson 3ae32aad3f
api: fix errors from refactor + PISTON_ALIAS envvar 2021-03-13 20:57:21 +13:00
Thomas Hobson c6610671fa
pkg(bash-5.1.0): actually run the binary 2021-03-13 20:44:35 +13:00
Thomas Hobson 5819b75315
pkg(bash-5.1.0): Add bash 5.1.0 2021-03-13 20:43:04 +13:00
Thomas Hobson 92aa283c8f
pkg(gawk-5.1.0): Add gawk 2021-03-13 20:37:11 +13:00
Thomas Hobson c3617c55a8
Merge branch 'v3' of https://github.com/engineer-man/piston into v3 2021-03-13 19:54:04 +13:00
Thomas Hobson 535ce6ce97
dev(shell.nix): add jq to shell 2021-03-13 19:53:57 +13:00
Thomas Hobson 09779c6bf2
pkg(gitignore): update for new structure 2021-03-13 19:53:46 +13:00
Thomas Hobson 8745c0e2a3
pkg(jelly-0.1.31): Add jelly 0.1.31 2021-03-13 19:53:33 +13:00
Thomas Hobson eb38af782d
pkg(deno-1.7.5): relocate files 2021-03-13 19:53:14 +13:00
Thomas Hobson 509013f4a0
pkg(mono-6.12.0): relocate files 2021-03-13 19:53:00 +13:00
Thomas Hobson 7b4e072154
pkg(php-8.0.2): relocate files 2021-03-13 19:52:42 +13:00
Thomas Hobson 60ec517dcd
pkg(node-15.10.0): relocate files 2021-03-13 19:52:32 +13:00
Thomas Hobson a65ecf7386
repo: mount files in build container differently 2021-03-13 19:51:38 +13:00
Thomas Hobson 0dd099a38d
pkg(python-3.9.1): adjust for new layout 2021-03-13 19:51:11 +13:00
Thomas Hobson 69c854cb99
pkg: remove everything for relocation 2021-03-13 19:50:22 +13:00
Thomas Hobson 98782b0a17
pkg: relocate package files 2021-03-13 19:50:05 +13:00
Thomas Hobson 5e400cb782
cli: fix execute 2021-03-13 19:49:05 +13:00
Brian Seymour b3d18600cd api refactoring 2021-03-12 23:01:04 -06:00
100 changed files with 883 additions and 387 deletions

View File

@ -1,40 +0,0 @@
{
"env": {
"commonjs": true,
"es2021": true,
"node": true
},
"plugins": [
"snakecasejs"
],
"extends": "eslint:recommended",
"parser": "babel-eslint",
"parserOptions": {
"ecmaVersion": 12
},
"settings":
{
"snakecasejs/filter": ["ClassDeclaration", "NewExpression"],
"snakecasejs/whitelist": []
},
"rules": {
"indent": [
"error",
4
],
"linebreak-style": [
"error",
"unix"
],
"quotes": [
"error",
"single"
],
"semi": [
"error",
"always"
],
"no-unused-vars": ["error", { "argsIgnorePattern": "^_"}],
"snakecasejs/snakecasejs": "warn"
}
}

View File

@ -1,12 +1,12 @@
FROM node:15.8.0-buster-slim FROM node:15.8.0-buster-slim
RUN dpkg-reconfigure -p critical dash RUN dpkg-reconfigure -p critical dash
RUN apt-get update && apt-get install -y libxml2 gnupg tar coreutils util-linux \
&& rm -rf /var/lib/apt/lists/*
RUN for i in $(seq 1001 1500); do \ RUN for i in $(seq 1001 1500); do \
groupadd -g $i runner$i && \ groupadd -g $i runner$i && \
useradd -M runner$i -g $i -u $i ; \ useradd -M runner$i -g $i -u $i ; \
done done
RUN apt-get update && \
apt-get install -y libxml2 gnupg tar coreutils util-linux libc6-dev binutils && \
rm -rf /var/lib/apt/lists/*
ENV NODE_ENV=production ENV NODE_ENV=production
WORKDIR /piston_api WORKDIR /piston_api
@ -15,4 +15,4 @@ RUN yarn
COPY ./src ./src COPY ./src ./src
CMD [ "node", "src", "-m", "-c", "/piston/config.yaml"] CMD [ "node", "src", "-m", "-c", "/piston/config.yaml"]
EXPOSE 6969/tcp EXPOSE 6969/tcp

View File

@ -1,25 +1,20 @@
{ {
"name": "piston-api", "name": "piston-api",
"version": "3.0.0", "version": "3.0.0",
"description": "API for piston - a high performance code execution engine", "description": "API for piston - a high performance code execution engine",
"main": "src/index.js", "main": "src/index.js",
"dependencies": { "dependencies": {
"body-parser": "^1.19.0", "body-parser": "^1.19.0",
"express": "^4.17.1", "express": "^4.17.1",
"express-validator": "^6.10.0", "express-validator": "^6.10.0",
"is-docker": "^2.1.1", "is-docker": "^2.1.1",
"js-yaml": "^4.0.0", "js-yaml": "^4.0.0",
"logplease": "^1.2.15", "logplease": "^1.2.15",
"nocamel": "HexF/nocamel#patch-1", "nocamel": "HexF/nocamel#patch-1",
"node-fetch": "^2.6.1", "node-fetch": "^2.6.1",
"semver": "^7.3.4", "semver": "^7.3.4",
"uuid": "^8.3.2", "uuid": "^8.3.2",
"yargs": "^16.2.0" "yargs": "^16.2.0"
}, },
"devDependencies": { "license": "MIT"
"babel-eslint": "^10.1.0",
"eslint": "^7.20.0",
"eslint-plugin-snakecasejs": "^2.2.0"
},
"license": "MIT"
} }

View File

@ -1,25 +1,25 @@
const fss = require('fs'); const fss = require('fs');
const yargs = require('yargs'); const yargs = require('yargs');
const hide_bin = require('yargs/helpers').hideBin; //eslint-disable-line snakecasejs/snakecasejs const hide_bin = require('yargs/helpers').hideBin;
const Logger = require('logplease'); const Logger = require('logplease');
const logger = Logger.create('config'); const logger = Logger.create('config');
const yaml = require('js-yaml'); const yaml = require('js-yaml');
const header = `# const header = `#
# ____ _ _ # ____ _ _
# | _ \\(_)___| |_ ___ _ __ # | _ \\(_)___| |_ ___ _ __
# | |_) | / __| __/ _ \\| '_ \\ # | |_) | / __| __/ _ \\| '_ \\
# | __/| \\__ \\ || (_) | | | | # | __/| \\__ \\ || (_) | | | |
# |_| |_|___/\\__\\___/|_| |_| # |_| |_|___/\\__\\___/|_| |_|
# #
# A High performance code execution engine # A High performance code execution engine
# github.com/engineer-man/piston # github.com/engineer-man/piston
# #
`; `;
const argv = yargs(hide_bin(process.argv)) const argv = yargs(hide_bin(process.argv))
.usage('Usage: $0 -c [config]') .usage('Usage: $0 -c [config]')
.demandOption('c') //eslint-disable-line snakecasejs/snakecasejs .demandOption('c')
.option('config', { .option('config', {
alias: 'c', alias: 'c',
describe: 'config file to load from', describe: 'config file to load from',
@ -29,18 +29,18 @@ const argv = yargs(hide_bin(process.argv))
alias: 'm', alias: 'm',
type: 'boolean', type: 'boolean',
describe: 'create config file and populate defaults if it does not already exist' describe: 'create config file and populate defaults if it does not already exist'
}).argv; })
.argv;
const options = [ const options = [
{ {
key: 'log_level', key: 'log_level',
desc: 'Level of data to log', desc: 'Level of data to log',
default: 'INFO', default: 'INFO',
/* eslint-disable snakecasejs/snakecasejs */
options: Object.values(Logger.LogLevels), options: Object.values(Logger.LogLevels),
validators: [x=>Object.values(Logger.LogLevels).includes(x) || `Log level ${x} does not exist`] validators: [
/* eslint-enable snakecasejs/snakecasejs */ x => Object.values(Logger.LogLevels).includes(x) || `Log level ${x} does not exist`
]
}, },
{ {
key: 'bind_address', key: 'bind_address',
@ -110,68 +110,71 @@ const options = [
} }
]; ];
function make_default_config(){ const make_default_config = () => {
let content = header.split('\n'); let content = header.split('\n');
options.forEach(option => { options.forEach(option => {
content = content.concat(option.desc.split('\n').map(x=>`# ${x}`)); content = content.concat(option.desc.split('\n').map(x=>`# ${x}`));
if(option.options) if (option.options) {
content.push('# Options: ' + option.options.join(', ')); content.push('# Options: ' + option.options.join(', '));
}
content.push(`${option.key}: ${option.default}`); content.push(`${option.key}: ${option.default}`);
content.push(''); // New line between content.push(''); // New line between
}); });
return content.join('\n'); return content.join('\n');
} };
logger.info(`Loading Configuration from ${argv.config}`); logger.info(`Loading Configuration from ${argv.config}`);
if(argv['make-config']) if (argv['make-config']) {
logger.debug('Make configuration flag is set'); logger.debug('Make configuration flag is set');
}
if(!!argv['make-config'] && !fss.exists_sync(argv.config)){ if (!!argv['make-config'] && !fss.exists_sync(argv.config)) {
logger.info('Writing default configuration...'); logger.info('Writing default configuration...');
try { try {
fss.write_file_sync(argv.config, make_default_config()); fss.write_file_sync(argv.config, make_default_config());
} catch (err) { } catch (e) {
logger.error('Error writing default configuration:', err.message); logger.error('Error writing default configuration:', e.message);
process.exit(1); process.exit(1);
} }
} }
var config = {}; let config = {};
logger.debug('Reading config file'); logger.debug('Reading config file');
try{ try {
const cfg_content = fss.read_file_sync(argv.config); const cfg_content = fss.read_file_sync(argv.config);
config = yaml.load(cfg_content); config = yaml.load(cfg_content);
}catch(err){ } catch(err) {
logger.error('Error reading configuration file:', err.message); logger.error('Error reading configuration file:', err.message);
process.exit(1); process.exit(1);
} }
logger.debug('Validating config entries'); logger.debug('Validating config entries');
var errored=false; let errored = false;
options.forEach(option => { options.for_each(option => {
logger.debug('Checking option', option.key); logger.debug('Checking option', option.key);
var cfg_val = config[option.key]; let cfg_val = config[option.key];
if(cfg_val == undefined){ if (cfg_val === undefined) {
errored = true; errored = true;
logger.error(`Config key ${option.key} does not exist on currently loaded configuration`); logger.error(`Config key ${option.key} does not exist on currently loaded configuration`);
return; return;
} }
option.validators.forEach(validator => { option.validators.for_each(validator => {
var response = validator(cfg_val); let response = validator(cfg_val);
if(response !== true){
if (!response) {
errored = true; errored = true;
logger.error(`Config option ${option.key} failed validation:`, response); logger.error(`Config option ${option.key} failed validation:`, response);
return; return;
@ -179,9 +182,10 @@ options.forEach(option => {
}); });
}); });
if(errored) process.exit(1); if (errored) {
process.exit(1);
}
logger.info('Configuration successfully loaded'); logger.info('Configuration successfully loaded');
module.exports = config; module.exports = config;

View File

@ -1,5 +1,5 @@
const logger = require('logplease').create('executor/job'); const logger = require('logplease').create('executor/job');
const { v4: uuidv4 } = require('uuid'); const {v4: uuidv4} = require('uuid');
const cp = require('child_process'); const cp = require('child_process');
const path = require('path'); const path = require('path');
const config = require('../config'); const config = require('../config');
@ -12,11 +12,12 @@ const job_states = {
EXECUTED: Symbol('Executed and ready for cleanup') EXECUTED: Symbol('Executed and ready for cleanup')
}; };
var uid=0; let uid = 0;
var gid=0; let gid = 0;
class Job { class Job {
constructor({runtime, files, args, stdin, timeouts, main}){
constructor({ runtime, files, args, stdin, timeouts, main, alias }) {
this.uuid = uuidv4(); this.uuid = uuidv4();
this.runtime = runtime; this.runtime = runtime;
this.files = files; this.files = files;
@ -24,9 +25,13 @@ class Job {
this.stdin = stdin; this.stdin = stdin;
this.timeouts = timeouts; this.timeouts = timeouts;
this.main = main; this.main = main;
this.alias = alias;
if(!this.files.map(f=>f.name).includes(this.main)) let file_list = this.files.map(f => f.name);
if (!file_list.includes(this.main)) {
throw new Error(`Main file "${this.main}" will not be written to disk`); throw new Error(`Main file "${this.main}" will not be written to disk`);
}
this.uid = config.runner_uid_min + uid; this.uid = config.runner_uid_min + uid;
this.gid = config.runner_gid_min + gid; this.gid = config.runner_gid_min + gid;
@ -41,34 +46,30 @@ class Job {
this.dir = path.join(config.data_directory, globals.data_directories.jobs, this.uuid); this.dir = path.join(config.data_directory, globals.data_directories.jobs, this.uuid);
} }
async prime(){ async prime() {
logger.info(`Priming job uuid=${this.uuid}`); logger.info(`Priming job uuid=${this.uuid}`);
logger.debug('Writing files to job cache'); logger.debug('Writing files to job cache');
await fs.mkdir(this.dir, {mode:0o700});
const files = this.files.map(({name: file_name, content}) => {
return fs.write_file(path.join(this.dir, file_name), content);
});
await Promise.all(files);
logger.debug(`Transfering ownership uid=${this.uid} gid=${this.gid}`); logger.debug(`Transfering ownership uid=${this.uid} gid=${this.gid}`);
await fs.chown(this.dir, this.uid, this.gid);
const chowns = this.files.map(({name:file_name}) => {
return fs.chown(path.join(this.dir, file_name), this.uid, this.gid);
});
await Promise.all(chowns); await fs.mkdir(this.dir, { mode:0o700 });
await fs.chown(this.dir, this.uid, this.gid);
for (const file of this.files) {
let file_path = path.join(this.dir, file.name);
await fs.write_file(file_path, file.content);
await fs.chown(file_path, this.uid, this.gid);
}
this.state = job_states.PRIMED; this.state = job_states.PRIMED;
logger.debug('Primed job'); logger.debug('Primed job');
} }
async safe_call(file, args, timeout){ async safe_call(file, args, timeout) {
return await new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
const unshare = config.enable_unshare ? ['unshare','-n','-r'] : []; const unshare = config.enable_unshare ? ['unshare','-n','-r'] : [];
const prlimit = [ const prlimit = [
@ -87,70 +88,91 @@ class Job {
var stderr = ''; var stderr = '';
const proc = cp.spawn(proc_call[0], proc_call.splice(1) ,{ const proc = cp.spawn(proc_call[0], proc_call.splice(1) ,{
env: this.runtime.env_vars, env: {
...this.runtime.env_vars,
PISTON_ALIAS: this.alias
},
stdio: 'pipe', stdio: 'pipe',
cwd: this.dir, cwd: this.dir,
uid: this.uid, uid: this.uid,
gid: this.gid, gid: this.gid,
detached: true //give this process its own process group detached: true //give this process its own process group
}); });
proc.stdin.write(this.stdin); proc.stdin.write(this.stdin);
proc.stdin.end(); proc.stdin.end();
const kill_timeout = setTimeout(_ => proc.kill('SIGKILL'), timeout); const kill_timeout = set_timeout(_ => proc.kill('SIGKILL'), timeout);
proc.stderr.on('data', d=>{if(stderr.length>config.output_max_size) proc.kill('SIGKILL'); else stderr += d;}); proc.stderr.on('data', data => {
proc.stdout.on('data', d=>{if(stdout.length>config.output_max_size) proc.kill('SIGKILL'); else stdout += d;}); if (stderr.length > config.output_max_size) {
proc.kill('SIGKILL');
} else {
stderr += data;
}
});
proc.stdout.on('data', data => {
if (stdout.length > config.output_max_size) {
proc.kill('SIGKILL');
} else {
stdout += data;
}
});
const exit_cleanup = () => {
clear_timeout(kill_timeout);
function exit_cleanup(){
clearTimeout(kill_timeout);
proc.stderr.destroy(); proc.stderr.destroy();
proc.stdout.destroy(); proc.stdout.destroy();
try{
try {
process.kill(-proc.pid, 'SIGKILL'); process.kill(-proc.pid, 'SIGKILL');
}catch{ } catch {
// Process will be dead already, so nothing to kill. // Process will be dead already, so nothing to kill.
} }
} };
proc.on('exit', (code, signal)=>{ proc.on('exit', (code, signal)=>{
exit_cleanup(); exit_cleanup();
resolve({stdout, stderr, code, signal}); resolve({ stdout, stderr, code, signal });
}); });
proc.on('error', (err) => { proc.on('error', (err) => {
exit_cleanup(); exit_cleanup();
reject({error: err, stdout, stderr}); reject({ error: err, stdout, stderr });
}); });
}); });
} }
async execute(){ async execute() {
if(this.state != job_states.PRIMED) if (this.state !== job_states.PRIMED) {
throw new Error('Job must be in primed state, current state: ' + this.state.toString()); throw new Error('Job must be in primed state, current state: ' + this.state.toString());
}
logger.info(`Executing job uuid=${this.uuid} uid=${this.uid} gid=${this.gid} runtime=${this.runtime.toString()}`); logger.info(`Executing job uuid=${this.uuid} uid=${this.uid} gid=${this.gid} runtime=${this.runtime.toString()}`);
logger.debug('Compiling'); logger.debug('Compiling');
var compile = undefined; let compile;
if(this.runtime.compiled)
if (this.runtime.compiled) {
compile = await this.safe_call( compile = await this.safe_call(
path.join(this.runtime.pkgdir, 'compile'), path.join(this.runtime.pkgdir, 'compile'),
this.files.map(x=>x.name), this.files.map(x => x.name),
this.timeouts.compile); this.timeouts.compile
);
}
logger.debug('Running'); logger.debug('Running');
const run = await this.safe_call( const run = await this.safe_call(
path.join(this.runtime.pkgdir, 'run'), path.join(this.runtime.pkgdir, 'run'),
[this.main, ...this.args], [this.main, ...this.args],
this.timeouts.run); this.timeouts.run
);
this.state = job_states.EXECUTED; this.state = job_states.EXECUTED;
@ -158,13 +180,15 @@ class Job {
compile, compile,
run run
}; };
} }
async cleanup(){ async cleanup() {
logger.info(`Cleaning up job uuid=${this.uuid}`); logger.info(`Cleaning up job uuid=${this.uuid}`);
await fs.rm(this.dir, {recursive: true, force: true}); await fs.rm(this.dir, { recursive: true, force: true });
} }
} }
module.exports = {Job}; module.exports = {
Job
};

View File

@ -6,41 +6,49 @@ const { Job } = require('./job');
const { body } = require('express-validator'); const { body } = require('express-validator');
module.exports = { module.exports = {
run_job_validators: [ run_job_validators: [
body('language') body('language')
.isString(), // eslint-disable-line snakecasejs/snakecasejs .isString(),
body('version') body('version')
.isString(), // eslint-disable-line snakecasejs/snakecasejs .isString(),
// isSemVer requires it to be a version, not a selector // isSemVer requires it to be a version, not a selector
body('files') body('files')
.isArray(), // eslint-disable-line snakecasejs/snakecasejs .isArray(),
body('files.*.name') body('files.*.name')
.isString() // eslint-disable-line snakecasejs/snakecasejs .isString()
.bail() .bail()
.not() .not()
.contains('/'), .contains('/'),
body('files.*.content') body('files.*.content')
.isString(), // eslint-disable-line snakecasejs/snakecasejs .isString(),
body('compile_timeout') body('compile_timeout')
.isNumeric(), // eslint-disable-line snakecasejs/snakecasejs .isNumeric(),
body('run_timeout') body('run_timeout')
.isNumeric(), // eslint-disable-line snakecasejs/snakecasejs .isNumeric(),
body('stdin') body('stdin')
.isString(), // eslint-disable-line snakecasejs/snakecasejs .isString(),
body('args') body('args')
.isArray(), .isArray(),
body('args.*') body('args.*')
.isString() // eslint-disable-line snakecasejs/snakecasejs .isString()
], ],
async run_job(req, res){
// POST /jobs
// POST /jobs
async run_job(req, res) {
const runtime = get_latest_runtime_matching_language_version(req.body.language, req.body.version); const runtime = get_latest_runtime_matching_language_version(req.body.language, req.body.version);
if(runtime == undefined) return res.json_error(`${req.body.language}-${req.body.version} runtime is unknown`, 400);
if (runtime === undefined) {
return res
.status(400)
.send({
message: `${req.body.language}-${req.body.version} runtime is unknown`
});
}
const job = new Job({ const job = new Job({
runtime, runtime,
alias: req.body.language,
files: req.body.files, files: req.body.files,
args: req.body.args, args: req.body.args,
stdin: req.body.stdin, stdin: req.body.stdin,
@ -54,8 +62,12 @@ module.exports = {
await job.prime(); await job.prime();
const result = await job.execute(); const result = await job.execute();
res.json_success(result);
await job.cleanup(); await job.cleanup();
return res
.status(200)
.send(result);
} }
};
};

View File

@ -5,7 +5,7 @@ const platform = `${is_docker() ? 'docker' : 'baremetal'}-${
fss.read_file_sync('/etc/os-release') fss.read_file_sync('/etc/os-release')
.toString() .toString()
.split('\n') .split('\n')
.find(x=>x.startsWith('ID')) .find(x => x.startsWith('ID'))
.replace('ID=','') .replace('ID=','')
}`; }`;
@ -17,4 +17,4 @@ module.exports = {
version: require('../package.json').version, version: require('../package.json').version,
platform, platform,
pkg_installed_file: '.ppman-installed' //Used as indication for if a package was installed pkg_installed_file: '.ppman-installed' //Used as indication for if a package was installed
}; };

View File

@ -9,117 +9,103 @@ const fs = require('fs/promises');
const fss = require('fs'); const fss = require('fs');
const body_parser = require('body-parser'); const body_parser = require('body-parser');
const runtime = require('./runtime'); const runtime = require('./runtime');
const {validationResult} = require('express-validator'); //eslint-disable-line snakecasejs/snakecasejs const { validationResult } = require('express-validator');
const logger = Logger.create('index'); const logger = Logger.create('index');
const app = express(); const app = express();
(async () => { (async () => {
logger.info('Setting loglevel to',config.log_level); logger.info('Setting loglevel to',config.log_level);
Logger.setLogLevel(config.log_level); //eslint-disable-line snakecasejs/snakecasejs Logger.setLogLevel(config.log_level);
logger.debug('Ensuring data directories exist'); logger.debug('Ensuring data directories exist');
Object.values(globals.data_directories).forEach(dir => {
var data_path = path.join(config.data_directory, dir); Object.values(globals.data_directories).for_each(dir => {
let data_path = path.join(config.data_directory, dir);
logger.debug(`Ensuring ${data_path} exists`); logger.debug(`Ensuring ${data_path} exists`);
if(!fss.exists_sync(data_path)){
if (!fss.exists_sync(data_path)) {
logger.info(`${data_path} does not exist.. Creating..`); logger.info(`${data_path} does not exist.. Creating..`);
try{
try {
fss.mkdir_sync(data_path); fss.mkdir_sync(data_path);
}catch(err){ } catch(e) {
logger.error(`Failed to create ${data_path}: `, err.message); logger.error(`Failed to create ${data_path}: `, e.message);
} }
} }
}); });
logger.info('Loading packages'); logger.info('Loading packages');
const pkgdir = path.join(config.data_directory,globals.data_directories.packages); const pkgdir = path.join(config.data_directory,globals.data_directories.packages);
const pkglist = await fs.readdir(pkgdir); const pkglist = await fs.readdir(pkgdir);
const languages = await Promise.all( const languages = await Promise.all(
pkglist.map(lang=> pkglist.map(lang=>
fs.readdir(path.join(pkgdir,lang)) fs.readdir(path.join(pkgdir,lang))
.then(x=>x.map(y=>path.join(pkgdir, lang, y))) .then(x=>x.map(y=>path.join(pkgdir, lang, y)))
)); ));
const installed_languages = languages.flat()
.filter(pkg=>fss.exists_sync(path.join(pkg, globals.pkg_installed_file))); const installed_languages = languages
.flat()
.filter(pkg => fss.exists_sync(path.join(pkg, globals.pkg_installed_file)));
installed_languages.forEach(pkg => new runtime.Runtime(pkg)); installed_languages.forEach(pkg => new runtime.Runtime(pkg));
logger.info('Starting API Server'); logger.info('Starting API Server');
logger.debug('Constructing Express App'); logger.debug('Constructing Express App');
logger.debug('Registering custom message wrappers');
express.response.json_error = function(message, code) {
this.status(code);
return this.json({success: false, message, code});
};
express.response.json_success = function(obj) {
return this.json({success: true, data: obj});
};
logger.debug('Registering middleware'); logger.debug('Registering middleware');
app.use(body_parser.urlencoded({extended: true})); app.use(body_parser.urlencoded({ extended: true }));
app.use(body_parser.json()); app.use(body_parser.json());
const validate = (req, res, next) => {
const errors = validationResult(req);
if (!errors.isEmpty()) {
return res
.status(422)
.send({
message: errors.array()
});
}
function validate(req, res, next) {
const errors = validationResult(req); //eslint-disable-line snakecasejs/snakecasejs
if (!errors.isEmpty()) //eslint-disable-line snakecasejs/snakecasejs
return res.json_error(errors.array(), 422);
next(); next();
} };
logger.debug('Registering Routes'); logger.debug('Registering Routes');
const ppman_routes = require('./ppman/routes'); const ppman_routes = require('./ppman/routes');
const executor_routes = require('./executor/routes'); const executor_routes = require('./executor/routes');
app.get('/packages', ppman_routes.package_list);
app.get('/packages', app.post('/packages/:language/:version', ppman_routes.package_install);
ppman_routes.package_list app.delete('/packages/:language/:version', ppman_routes.package_uninstall);
);
app.post('/packages/:language/:version',
ppman_routes.package_install
);
app.delete('/packages/:language/:version',
ppman_routes.package_uninstall
);
app.post('/jobs', app.post('/jobs',
executor_routes.run_job_validators, executor_routes.run_job_validators,
validate, validate,
executor_routes.run_job); executor_routes.run_job
);
app.get('/runtimes', (req, res) => {
const runtimes = runtime
.map(rt => {
return {
language: rt.language,
version: rt.version.raw,
author: rt.author,
aliases: rt.aliases
};
});
function list_runtimes(_, res){ return res
const runtimes = runtime.map(rt => ( .status(200)
{ .send(runtimes);
language: rt.language,
version: rt.version.raw,
author: rt.author,
aliases: rt.aliases
}
));
return res.json_success({
runtimes
});
}
app.get('/runtimes', list_runtimes);
logger.debug('Calling app.listen');
const [address,port] = config.bind_address.split(':');
app.listen(port, address, ()=>{
logger.info('API server started on', config.bind_address);
}); });
})(); logger.debug('Calling app.listen');
const [ address, port ] = config.bind_address.split(':');
app.listen(port, address, () => {
logger.info('API server started on', config.bind_address);
});
})();

View File

@ -11,66 +11,73 @@ const crypto = require('crypto');
const runtime = require('../runtime'); const runtime = require('../runtime');
class Package { class Package {
constructor({language, version, download, checksum}){
constructor({ language, version, download, checksum }){
this.language = language; this.language = language;
this.version = semver.parse(version); this.version = semver.parse(version);
this.checksum = checksum; this.checksum = checksum;
this.download = download; this.download = download;
} }
get installed(){ get installed() {
return fss.exists_sync(path.join(this.install_path, globals.pkg_installed_file)); return fss.exists_sync(path.join(this.install_path, globals.pkg_installed_file));
} }
get download_url(){ get install_path() {
return this.download; return path.join(
} config.data_directory,
get install_path(){
return path.join(config.data_directory,
globals.data_directories.packages, globals.data_directories.packages,
this.language, this.language,
this.version.raw); this.version.raw
);
} }
async install(){ async install() {
if(this.installed) throw new Error('Already installed'); if (this.installed) {
throw new Error('Already installed');
}
logger.info(`Installing ${this.language}-${this.version.raw}`); logger.info(`Installing ${this.language}-${this.version.raw}`);
if(fss.exists_sync(this.install_path)){ if (fss.exists_sync(this.install_path)) {
logger.warn(`${this.language}-${this.version.raw} has residual files. Removing them.`); logger.warn(`${this.language}-${this.version.raw} has residual files. Removing them.`);
await fs.rm(this.install_path, {recursive: true, force: true}); await fs.rm(this.install_path, { recursive: true, force: true });
} }
logger.debug(`Making directory ${this.install_path}`); logger.debug(`Making directory ${this.install_path}`);
await fs.mkdir(this.install_path, {recursive: true}); await fs.mkdir(this.install_path, {recursive: true});
logger.debug(`Downloading package from ${this.download} in to ${this.install_path}`);
const pkgpath = path.join(this.install_path, 'pkg.tar.gz');
const download = await fetch(this.download);
logger.debug(`Downloading package from ${this.download_url} in to ${this.install_path}`);
const pkgpath = path.join(this.install_path, "pkg.tar.gz");
const download = await fetch(this.download_url);
const file_stream = fss.create_write_stream(pkgpath); const file_stream = fss.create_write_stream(pkgpath);
await new Promise((resolve, reject) => { await new Promise((resolve, reject) => {
download.body.pipe(file_stream) download.body.pipe(file_stream);
download.body.on("error", reject) download.body.on('error', reject);
file_stream.on("finish", resolve)
file_stream.on('finish', resolve);
}); });
logger.debug('Validating checksums'); logger.debug('Validating checksums');
logger.debug(`Assert sha256(pkg.tar.gz) == ${this.checksum}`) logger.debug(`Assert sha256(pkg.tar.gz) == ${this.checksum}`);
const cs = crypto.create_hash("sha256") const cs = crypto.create_hash("sha256")
.update(fss.readFileSync(pkgpath)) .update(fss.readFileSync(pkgpath))
.digest('hex'); .digest('hex');
if(cs != this.checksum) throw new Error(`Checksum miss-match want: ${val} got: ${cs}`);
if (cs !== this.checksum) {
throw new Error(`Checksum miss-match want: ${val} got: ${cs}`);
}
logger.debug(`Extracting package files from archive ${pkgpath} in to ${this.install_path}`); logger.debug(`Extracting package files from archive ${pkgpath} in to ${this.install_path}`);
await new Promise((resolve, reject)=>{ await new Promise((resolve, reject) => {
const proc = cp.exec(`bash -c 'cd "${this.install_path}" && tar xzf ${pkgpath}'`); const proc = cp.exec(`bash -c 'cd "${this.install_path}" && tar xzf ${pkgpath}'`);
proc.once('exit', (code,_)=>{
if(code == 0) resolve(); proc.once('exit', (code, _) => {
else reject(new Error('Failed to extract package')); code === 0 ? resolve() : reject();
}); });
proc.stdout.pipe(process.stdout); proc.stdout.pipe(process.stdout);
proc.stderr.pipe(process.stderr); proc.stderr.pipe(process.stderr);
@ -80,28 +87,35 @@ class Package {
logger.debug('Registering runtime'); logger.debug('Registering runtime');
new runtime.Runtime(this.install_path); new runtime.Runtime(this.install_path);
logger.debug('Caching environment'); logger.debug('Caching environment');
const get_env_command = `cd ${this.install_path}; source environment; env`; const get_env_command = `cd ${this.install_path}; source environment; env`;
const envout = await new Promise((resolve, reject)=>{ const envout = await new Promise((resolve, reject) => {
var stdout = ''; let stdout = '';
const proc = cp.spawn('env',['-i','bash','-c',`${get_env_command}`], {
stdio: ['ignore', 'pipe', 'pipe']}); const proc = cp
proc.once('exit', (code,_)=>{ .spawn(
if(code == 0) resolve(stdout); 'env',
else reject(new Error('Failed to cache environment')); ['-i','bash','-c',`${get_env_command}`],
{
stdio: ['ignore', 'pipe', 'pipe']
}
);
proc.once('exit', (code, _) => {
code === 0 ? resolve(stdout) : reject();
}); });
proc.stdout.on('data', (data)=>{ proc.stdout.on('data', data => {
stdout += data; stdout += data;
}); });
proc.once('error', reject); proc.once('error', reject);
}); });
const filtered_env = envout.split('\n') const filtered_env = envout
.filter(l=>!['PWD','OLDPWD','_', 'SHLVL'].includes(l.split('=',2)[0])) .split('\n')
.filter(l => !['PWD','OLDPWD','_', 'SHLVL'].includes(l.split('=',2)[0]))
.join('\n'); .join('\n');
await fs.write_file(path.join(this.install_path, '.env'), filtered_env); await fs.write_file(path.join(this.install_path, '.env'), filtered_env);
@ -116,7 +130,9 @@ class Package {
version: this.version.raw version: this.version.raw
}; };
} }
} }
module.exports = {
module.exports = {Package}; Package
};

View File

@ -4,66 +4,98 @@ const fetch = require('node-fetch');
const config = require('../config'); const config = require('../config');
const { Package } = require('./package'); const { Package } = require('./package');
const get_package_list = async () => {
const repo_content = await fetch(config.repo_url).then(x => x.text());
async function get_package_list(){ const entries = repo_content
const repo_content = await fetch(config.repo_url).then(x=>x.text()); .split('\n')
.filter(x => x.length > 0);
const entries = repo_content.split('\n').filter(x=>x.length > 0);
return entries.map(line => { return entries.map(line => {
const [language, version, checksum, download] = line.split(',',4); const [ language, version, checksum, download ] = line.split(',', 4);
return new Package({language, version, checksum, download});
})
}
return new Package({
language,
version,
checksum,
download
});
});
};
async function get_package(lang, version){ const get_package = async (lang, version) => {
const packages = await get_package_list(); const packages = await get_package_list();
const candidates = packages.filter(
pkg => pkg.language == lang && semver.satisfies(pkg.version, version)
);
return candidates.sort((a,b)=>semver.rcompare(a.version,b.version))[0] || null;
}
module.exports = { const candidates = packages
.filter(pkg => {
async package_list(req, res){ return pkg.language == lang && semver.satisfies(pkg.version, version)
// GET /packages
logger.debug('Request to list packages');
const packages = await get_package_list();
res.json_success({
packages: packages.map(pkg=>({
language: pkg.language,
language_version: pkg.version.raw,
installed: pkg.installed
}))
}); });
}, candidates.sort((a, b) => semver.rcompare(a.version, b.version));
async package_install(req,res){
// POST /packages/:language/:version
return candidates[0] || null;
};
module.exports = {
// GET /packages
async package_list(req, res) {
logger.debug('Request to list packages');
let packages = await get_package_list();
packages = packages
.map(pkg => {
return {
language: pkg.language,
language_version: pkg.version.raw,
installed: pkg.installed
};
});
return res
.status(200)
.send(packages);
},
// POST /packages/:language/:version
async package_install(req, res) {
logger.debug('Request to install package'); logger.debug('Request to install package');
const pkg = await get_package(req.params.language, req.params.version); const pkg = await get_package(req.params.language, req.params.version);
if(pkg == null) return res.json_error(`Requested package ${req.params.language}-${req.params.version} does not exist`, 404);
try{ if (pkg == null) {
const response = await pkg.install(); return res
return res.json_success(response); .status(404)
}catch(err){ .send({
logger.error(`Error while installing package ${pkg.language}-${pkg.version}:`, err.message); message: `Requested package ${req.params.language}-${req.params.version} does not exist`
res.json_error(err.message,500); });
} }
try {
const response = await pkg.install();
return res
.status(200)
.send(response);
} catch(e) {
logger.error(`Error while installing package ${pkg.language}-${pkg.version}:`, e.message);
return res
.status(500)
.send({
message: e.message
});
}
}, },
async package_uninstall(req,res){
// DELETE /packages/:language/:version
//res.json(req.body); //TODO // DELETE /packages/:language/:version
res.json_error('not implemented', 500); async package_uninstall(req, res) {
return res
.status(500)
.send({
message: 'Not implemented'
});
} }
};
};

View File

@ -8,53 +8,60 @@ const path = require('path');
const runtimes = []; const runtimes = [];
class Runtime { class Runtime {
#env_vars
#compiled
constructor(package_dir){ constructor(package_dir){
const {language, version, author, build_platform, aliases} = JSON.parse( let info = JSON.parse(
fss.read_file_sync(path.join(package_dir, 'pkg-info.json')) fss.read_file_sync(path.join(package_dir, 'pkg-info.json'))
); );
const { language, version, author, build_platform, aliases } = info;
this.pkgdir = package_dir; this.pkgdir = package_dir;
this.language = language; this.language = language;
this.version = semver.parse(version); this.version = semver.parse(version);
this.author = author; this.author = author;
this.aliases = aliases; this.aliases = aliases;
if(build_platform != globals.platform){ if (build_platform !== globals.platform) {
logger.warn(`Package ${language}-${version} was built for platform ${build_platform}, but our platform is ${globals.platform}`); logger.warn(
`Package ${language}-${version} was built for platform ${build_platform}, ` +
`but our platform is ${globals.platform}`
);
} }
logger.debug(`Package ${language}-${version} was loaded`); logger.debug(`Package ${language}-${version} was loaded`);
runtimes.push(this); runtimes.push(this);
} }
get env_file_path(){ get compiled() {
return path.join(this.pkgdir, 'environment'); if (this._compiled === undefined) {
this._compiled = fss.exists_sync(path.join(this.pkgdir, 'compile'));
}
return this._compiled;
} }
get compiled(){ get env_vars() {
if(this.#compiled === undefined) this.#compiled = fss.exists_sync(path.join(this.pkgdir, 'compile')); if (!this._env_vars) {
return this.#compiled;
}
get env_vars(){
if(!this.#env_vars){
const env_file = path.join(this.pkgdir, '.env'); const env_file = path.join(this.pkgdir, '.env');
const env_content = fss.read_file_sync(env_file).toString(); const env_content = fss.read_file_sync(env_file).toString();
this.#env_vars = {};
this._env_vars = {};
env_content env_content
.trim() .trim()
.split('\n') .split('\n')
.map(line => line.split('=',2)) .map(line => line.split('=',2))
.forEach(([key,val]) => { .forEach(([key,val]) => {
this.#env_vars[key.trim()] = val.trim(); this._env_vars[key.trim()] = val.trim();
}); });
} }
return this.#env_vars;
return this._env_vars;
} }
toString(){ toString() {
return `${this.language}-${this.version.raw}`; return `${this.language}-${this.version.raw}`;
} }
} }
@ -68,4 +75,3 @@ module.exports.get_latest_runtime_matching_language_version = function(lang, ver
return module.exports.get_runtimes_matching_language_version(lang, ver) return module.exports.get_runtimes_matching_language_version(lang, ver)
.sort((a,b) => semver.rcompare(a.version, b.version))[0]; .sort((a,b) => semver.rcompare(a.version, b.version))[0];
}; };

View File

@ -18,7 +18,7 @@ exports.handler = async function({axios}){
const packages = await axios.get('/packages'); const packages = await axios.get('/packages');
const pkg_msg = packages.data.data.packages const pkg_msg = packages.data
.map(msg_format.color) .map(msg_format.color)
.join('\n'); .join('\n');

View File

@ -11,11 +11,11 @@ services:
- ./data/piston:/piston - ./data/piston:/piston
- ./repo:/repo - ./repo:/repo
tmpfs: tmpfs:
- /piston/jobs - /piston/jobs:exec
piston_fs_repo: #Temporary solution until CI works piston_fs_repo: #Temporary solution until CI works
build: repo build: repo
command: ['deno-1.7.5'] # Only build deno command: ['typescript-4.2.3'] # Only build typescript
volumes: volumes:
- ./repo:/repo - ./repo:/piston/repo
- ./packages:/packages - ./packages:/piston/packages

14
packages/.gitignore vendored
View File

@ -1,8 +1,8 @@
*/* */*/*
*.pkg.tar.gz *.pkg.tar.gz
!*/metadata.json !*/*/metadata.json
!*/build.sh !*/*/build.sh
!*/environment !*/*/environment
!*/run !*/*/run
!*/compile !*/*/compile
!*/test.* !*/*/test.*

View File

@ -8,11 +8,11 @@ In the example of NodeJS, we would call this `node`, after the main binary.
## Creating new languages ## Creating new languages
See [deno-1.7.5/](deno-1.7.5/) or any other directory for examples. See [deno/1.7.5/](deno/1.7.5/) or any other directory for examples.
1. Create a new branch on your fork of engineer-man/piston 1. Create a new branch on your fork of engineer-man/piston
2. Create a directory named `[language]-[version]`. See Naming Languages for how to determine the name for your language 2. Create directories named `[language]/[version]`. See Naming Languages for how to determine the name for your language
3. Create a file named `build.sh`, adding a shebang for bash `#!/bin/bash` on the first line. 3. Create a file named `build.sh`, adding a shebang for bash `#!/bin/bash` on the first line.
In this file put any steps to compile the specified langauge. In this file put any steps to compile the specified langauge.

View File

@ -1,4 +1,4 @@
PACKAGES=$(shell find * -maxdepth 0 -type d) PACKAGES=$(subst /,-,$(shell find * -maxdepth 1 -mindepth 1 -type d))
BUILD_PLATFORM=$(or ${PLATFORM},baremetal-$(shell grep -oP "^ID=\K.+" /etc/os-release)) BUILD_PLATFORM=$(or ${PLATFORM},baremetal-$(shell grep -oP "^ID=\K.+" /etc/os-release))
help: help:
@ -9,14 +9,16 @@ help:
build build-all: $(addsuffix .pkg.tar.gz, ${PACKAGES}) build build-all: $(addsuffix .pkg.tar.gz, ${PACKAGES})
%.pkg.tar.gz: %/ %/pkg-info.json
cd $< && chmod +x ./build.sh && ./build.sh define PKG_RULE
rm -f $@ $(1).pkg.tar.gz: $(subst -,/,$(1))/ $(subst -,/,$(1))/pkg-info.json
tar czf $@ $* --transform='s|$*||' cd $$< && chmod +x ./build.sh && ./build.sh
rm -f $$@
tar czf $$@ $$< --transform='s|$$<||'
endef
$(foreach pkg,$(PACKAGES),$(eval $(call PKG_RULE,$(pkg))))
%/pkg-info.json: %/metadata.json %/pkg-info.json: %/metadata.json
jq '.build_platform="${BUILD_PLATFORM}"' $< > $@ jq '.build_platform="${BUILD_PLATFORM}"' $< > $@

20
packages/bash/5.1.0/build.sh vendored Executable file
View File

@ -0,0 +1,20 @@
#!/usr/bin/env bash
# Put instructions to build your package in here
PREFIX=$(realpath $(dirname $0))
mkdir -p build
cd build
curl "https://ftp.gnu.org/gnu/bash/bash-5.1.tar.gz" -o bash.tar.gz
tar xzf bash.tar.gz --strip-components=1
# === autoconf based ===
./configure --prefix "$PREFIX"
make -j$(nproc)
make install -j$(nproc)
cd ../
rm -rf build

4
packages/bash/5.1.0/environment vendored Normal file
View File

@ -0,0 +1,4 @@
#!/usr/bin/env bash
# Put 'export' statements here for environment variables
export PATH=$PWD/bin:$PATH

6
packages/bash/5.1.0/metadata.json vendored Normal file
View File

@ -0,0 +1,6 @@
{
"language": "bash",
"version": "5.1.0",
"aliases": ["sh"],
"author": "Thomas Hobson <git@hexf.me>"
}

4
packages/bash/5.1.0/run vendored Normal file
View File

@ -0,0 +1,4 @@
#!/usr/bin/env bash
# Put instructions to run the runtime
bash $*

1
packages/bash/5.1.0/test.bash vendored Normal file
View File

@ -0,0 +1 @@
echo "OK"

11
packages/dart/2.12.1/build.sh vendored Executable file
View File

@ -0,0 +1,11 @@
#!/usr/bin/env bash
curl -L "https://storage.googleapis.com/dart-archive/channels/stable/release/2.12.1/sdk/dartsdk-linux-x64-release.zip" -o dart.zip
unzip dart.zip
rm dart.zip
cp -r dart-sdk/* .
rm -rf dart-sdk
chmod -R +rx bin

4
packages/dart/2.12.1/environment vendored Normal file
View File

@ -0,0 +1,4 @@
#!/usr/bin/env bash
# Put 'export' statements here for environment variables
export PATH=$PWD/bin:$PATH

6
packages/dart/2.12.1/metadata.json vendored Normal file
View File

@ -0,0 +1,6 @@
{
"language": "dart",
"version": "2.12.1",
"aliases": [],
"author": "Thomas Hobson <git@hexf.me>"
}

4
packages/dart/2.12.1/run vendored Normal file
View File

@ -0,0 +1,4 @@
#!/usr/bin/env bash
# Put instructions to run the runtime
dart run $*

3
packages/dart/2.12.1/test.dart vendored Normal file
View File

@ -0,0 +1,3 @@
void main() {
print('OK');
}

20
packages/gawk/5.1.0/build.sh vendored Normal file
View File

@ -0,0 +1,20 @@
#!/usr/bin/env bash
# Put instructions to build your package in here
PREFIX=$(realpath $(dirname $0))
mkdir -p build
cd build
curl "https://ftp.gnu.org/gnu/gawk/gawk-5.1.0.tar.gz" -o gawk.tar.gz
tar xzf gawk.tar.gz --strip-components=1
# === autoconf based ===
./configure --prefix "$PREFIX"
make -j$(nproc)
make install -j$(nproc)
cd ../
rm -rf build

3
packages/gawk/5.1.0/compile vendored Normal file
View File

@ -0,0 +1,3 @@
#!/usr/bin/env bash
# Put instructions to compile source code, remove this file if the language does not require this stage

4
packages/gawk/5.1.0/environment vendored Normal file
View File

@ -0,0 +1,4 @@
#!/usr/bin/env bash
# Put 'export' statements here for environment variables
export PATH=$PWD/bin:$PATH

6
packages/gawk/5.1.0/metadata.json vendored Normal file
View File

@ -0,0 +1,6 @@
{
"language": "gawk",
"version": "5.1.0",
"aliases": ["awk"],
"author": "Thomas Hobson <git@hexf.me>"
}

4
packages/gawk/5.1.0/run vendored Normal file
View File

@ -0,0 +1,4 @@
#!/usr/bin/env bash
# Put instructions to run the runtime
gawk-5.1.0 -f $*

1
packages/gawk/5.1.0/test.awk vendored Normal file
View File

@ -0,0 +1 @@
{print "OK"}

25
packages/gcc/10.2.0/build.sh vendored Executable file
View File

@ -0,0 +1,25 @@
#!/usr/bin/env bash
# Put instructions to build your package in here
[[ -d "bin" ]] && exit 0
PREFIX=$(realpath $(dirname $0))
mkdir -p build obj
cd build
curl "https://ftp.gnu.org/gnu/gcc/gcc-10.2.0/gcc-10.2.0.tar.gz" -o gcc.tar.gz
tar xzf gcc.tar.gz --strip-components=1
./contrib/download_prerequisites
cd ../obj
# === autoconf based ===
../build/configure --prefix "$PREFIX" --enable-languages=c,c++,d --disable-multilib --disable-bootstrap
make -j$(nproc)
make install -j$(nproc)
cd ../
rm -rf build obj

25
packages/gcc/10.2.0/compile vendored Normal file
View File

@ -0,0 +1,25 @@
#!/usr/bin/env bash
# Put instructions to compile source code, remove this file if the language does not require this stage
case "${PISTON_ALIAS}" in
gcc | c)
gcc -std=c11 $* -lm
;;
g++ | c++ | cpp)
g++ -std=c++17 $*
;;
gccgo | go)
gccgo $*
;;
gdc | d)
gdc $*
;;
*)
echo "How did you get here? (${PISTON_ALIAS})"
exit 1
;;
esac
chmod +x a.out

4
packages/gcc/10.2.0/environment vendored Normal file
View File

@ -0,0 +1,4 @@
#!/usr/bin/env bash
# Put 'export' statements here for environment variables
export PATH=$PWD/bin:$PATH

6
packages/gcc/10.2.0/metadata.json vendored Normal file
View File

@ -0,0 +1,6 @@
{
"language": "gcc",
"version": "10.2.0",
"aliases": ["c","g++","c++","cpp","gdc","d"],
"author": "Thomas Hobson <git@hexf.me>"
}

6
packages/gcc/10.2.0/run vendored Normal file
View File

@ -0,0 +1,6 @@
#!/usr/bin/env bash
# Put instructions to run the runtime
shift # Discard main filename
./a.out $*

6
packages/gcc/10.2.0/test.c vendored Normal file
View File

@ -0,0 +1,6 @@
#include <stdio.h>
int main(void) {
printf("OK");
return 0;
}

6
packages/gcc/10.2.0/test.cpp vendored Normal file
View File

@ -0,0 +1,6 @@
#include <iostream>
int main(void) {
printf("OK");
return 0;
}

5
packages/gcc/10.2.0/test.d vendored Normal file
View File

@ -0,0 +1,5 @@
import std.stdio;
void main() {
writeln("OK");
}

65
packages/init Executable file
View File

@ -0,0 +1,65 @@
#!/usr/bin/env bash
if [[ $# -lt 3 ]]; then
echo "Usage: $0 [name] [version] [source]"
echo ""
echo "Initializes an empty package"
exit 1
fi
NAME=$1
VERSION=$2
AUTHOR="$(git config user.name) <$(git config user.email)>"
SOURCE=$3
DIR=$NAME/$VERSION
mkdir -p $DIR
build_instructions(){
echo 'PREFIX=$(realpath $(dirname $0))'
echo
echo 'mkdir -p build'
echo
echo 'cd build'
echo
echo "curl \"$SOURCE\" -o $NAME.tar.gz"
echo
echo "tar xzf $NAME.tar.gz --strip-components=1"
echo
echo "# === autoconf based ==="
echo './configure --prefix "$PREFIX"'
echo
echo 'make -j$(nproc)'
echo 'make install -j$(nproc)'
echo 'cd ../'
echo 'rm -rf build'
}
cd $DIR
for name in build.sh environment run compile; do
echo "#!/usr/bin/env bash" > "$name"
echo "" >> "$name"
done
echo "# Put instructions to build your package in here" >> build.sh
echo ""
build_instructions >> build.sh
echo "# Put 'export' statements here for environment variables" >> environment
echo "export PATH=\$PWD/bin:\$PATH" >> environment
echo "# Put instructions to run the runtime" >> run
echo "$NAME-$VERSION \$*" >> run
echo "# Put instructions to compile source code, remove this file if the language does not require this stage" >> compile
jq '.language = "'$NAME'" | .version = "'$VERSION'" | .aliases = [] | .author = "'"$AUTHOR"'"' <<< "{}" > metadata.json
cd - > /dev/null
echo $DIR

8
packages/java/15.0.2/build.sh vendored Executable file
View File

@ -0,0 +1,8 @@
#!/usr/bin/env bash
# Put instructions to build your package in here
curl "https://download.java.net/java/GA/jdk15.0.2/0d1cfde4252546c6931946de8db48ee2/7/GPL/openjdk-15.0.2_linux-x64_bin.tar.gz" -o java.tar.gz
tar xzf java.tar.gz --strip-components=1
rm java.tar.gz

4
packages/java/15.0.2/environment vendored Normal file
View File

@ -0,0 +1,4 @@
#!/usr/bin/env bash
# Put 'export' statements here for environment variables
export PATH=$PWD/bin:$PATH

6
packages/java/15.0.2/metadata.json vendored Normal file
View File

@ -0,0 +1,6 @@
{
"language": "java",
"version": "15.0.2",
"aliases": [],
"author": "Thomas Hobson <git@hexf.me>"
}

4
packages/java/15.0.2/run vendored Normal file
View File

@ -0,0 +1,4 @@
#!/usr/bin/env bash
# Put instructions to run the runtime
java $*

5
packages/java/15.0.2/test.java vendored Normal file
View File

@ -0,0 +1,5 @@
public class HelloWorld {
public static void main(String[] args) {
System.out.println("OK");
}
}

12
packages/jelly/0.1.31/build.sh vendored Executable file
View File

@ -0,0 +1,12 @@
#!/bin/bash
source ../../python/3.9.1/build.sh
mkdir -p build
git clone -q https://github.com/DennisMitchell/jellylanguage.git build/jelly
cd build/jelly
../../bin/python3.9 setup.py install --optimize=1
cd ../../
rm -rf build

6
packages/jelly/0.1.31/metadata.json vendored Normal file
View File

@ -0,0 +1,6 @@
{
"language": "jelly",
"version": "0.1.31",
"author": "Thomas Hobson <thomas@hexf.me>",
"aliases": []
}

1
packages/jelly/0.1.31/run vendored Normal file
View File

@ -0,0 +1 @@
jelly fu $*

1
packages/jelly/0.1.31/test.jelly vendored Normal file
View File

@ -0,0 +1 @@
“OK”

10
packages/kotlin/1.4.31/build.sh vendored Executable file
View File

@ -0,0 +1,10 @@
#!/usr/bin/env bash
source ../../java/15.0.2/build.sh
curl -L "https://github.com/JetBrains/kotlin/releases/download/v1.4.31/kotlin-compiler-1.4.31.zip" -o kotlin.zip
unzip kotlin.zip
rm kotlin.zip
cp -r kotlinc/* .
rm -rf kotlinc

4
packages/kotlin/1.4.31/compile vendored Normal file
View File

@ -0,0 +1,4 @@
#!/usr/bin/env bash
# Put instructions to compile source code, remove this file if the language does not require this stage
kotlinc $* -include-runtime -d code.jar

4
packages/kotlin/1.4.31/environment vendored Normal file
View File

@ -0,0 +1,4 @@
#!/usr/bin/env bash
# Put 'export' statements here for environment variables
export PATH=$PWD/bin:$PATH

6
packages/kotlin/1.4.31/metadata.json vendored Normal file
View File

@ -0,0 +1,6 @@
{
"language": "kotlin",
"version": "1.4.31",
"aliases": ["kt"],
"author": "Thomas Hobson <git@hexf.me>"
}

4
packages/kotlin/1.4.31/run vendored Normal file
View File

@ -0,0 +1,4 @@
#!/usr/bin/env bash
# Put instructions to run the runtime
java -jar code.jar

3
packages/kotlin/1.4.31/test.kt vendored Normal file
View File

@ -0,0 +1,3 @@
fun main() {
println("OK")
}

View File

@ -1,6 +1,8 @@
#!/bin/bash #!/bin/bash
mkdir -p build/tmp build/mono PREFIX=$(realpath $(dirname $0))
mkdir -p build/mono
cd build cd build
curl "https://download.mono-project.com/sources/mono/mono-6.12.0.122.tar.xz" -o mono.tar.xz curl "https://download.mono-project.com/sources/mono/mono-6.12.0.122.tar.xz" -o mono.tar.xz
@ -8,12 +10,10 @@ tar xf mono.tar.xz --strip-components=1 -C mono
cd mono cd mono
./configure --prefix /piston/packages/mono/6.12.0/mono-6.12.0 ./configure --prefix "$PREFIX"
make -j$(nproc) make -j$(nproc)
DESTDIR=build/tmp make install -j$(nproc) make install -j$(nproc)
mv build/tmp/piston/packages/mono/6.12.0/mono-6.12.0 ../../mono-6.12.0
cd ../../ cd ../../
rm -rf build rm -rf build

20
packages/nasm/2.15.5/build.sh vendored Executable file
View File

@ -0,0 +1,20 @@
#!/usr/bin/env bash
# Put instructions to build your package in here
PREFIX=$(realpath $(dirname $0))
mkdir -p build
cd build
curl -L "https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/nasm-2.15.05.tar.gz" -o nasm.tar.gz
tar xzf nasm.tar.gz --strip-components=1
# === autoconf based ===
./configure --prefix "$PREFIX"
make -j$(nproc)
make install -j$(nproc)
cd ../
rm -rf build

21
packages/nasm/2.15.5/compile vendored Normal file
View File

@ -0,0 +1,21 @@
#!/usr/bin/env bash
# Put instructions to compile source code, remove this file if the language does not require this stage
case "${PISTON_ALIAS}" in
nasm)
nasm -f elf32 -o binary.o $*
ld -m elf_i386 binary.o -o binary
;;
nasm64)
nasm -f elf64 -o binary.o $*
ld -m elf_x86_64 binary.o -o binary
;;
*)
echo "How did you get here? (${PISTON_ALIAS})"
exit 1
;;
esac
chmod +x ./binary

4
packages/nasm/2.15.5/environment vendored Normal file
View File

@ -0,0 +1,4 @@
#!/usr/bin/env bash
# Put 'export' statements here for environment variables
export PATH=$PWD/bin:$PATH

6
packages/nasm/2.15.5/metadata.json vendored Normal file
View File

@ -0,0 +1,6 @@
{
"language": "nasm",
"version": "2.15.5",
"aliases": ["nasm64"],
"author": "Thomas Hobson <git@hexf.me>"
}

5
packages/nasm/2.15.5/run vendored Normal file
View File

@ -0,0 +1,5 @@
#!/usr/bin/env bash
# Put instructions to run the runtime
shift
./binary $*

16
packages/nasm/2.15.5/test.nasm.asm vendored Normal file
View File

@ -0,0 +1,16 @@
SECTION .DATA
good: db 'OK',10
txtlen: equ $-good
SECTION .TEXT
GLOBAL _start
_start:
mov eax,4
mov ebx,1
mov ecx,good
mov edx,txtlen
int 80h
mov eax,1
mov ebx,0
int 80h

18
packages/nasm/2.15.5/test.nasm64.asm vendored Normal file
View File

@ -0,0 +1,18 @@
SECTION .data
good: db "OK", 0x0
txtlen: equ $ - good
SECTION .text
GLOBAL _start
_start:
;sys_write
mov rax, 1
mov rdi, 1
mov rsi, good
mov rdx, txtlen
syscall
;sys_exit
mov rax, 60
mov rdi, 0
syscall

1
packages/node/15.10.0/environment vendored Normal file
View File

@ -0,0 +1 @@
export PATH=$PWD/bin:$PATH

View File

@ -1,20 +0,0 @@
#!/bin/bash
mkdir -p build/tmp build/php
cd build
curl "https://www.php.net/distributions/php-8.0.2.tar.gz" -o php.tar.gz
tar xzf php.tar.gz --strip-components=1 -C php
cd php
./configure --prefix /piston/packages/php/8.0.2/php-8.0.2
make -j$(nproc)
INSTALL_ROOT=build/tmp make install -j$(nproc)
mv build/tmp/piston/packages/php/8.0.2/php-8.0.2 ../../php-8.0.2

20
packages/php/8.0.2/build.sh vendored Executable file
View File

@ -0,0 +1,20 @@
#!/bin/bash
PREFIX=$(realpath $(dirname $0))
mkdir -p build/php
cd build
curl "https://www.php.net/distributions/php-8.0.2.tar.gz" -o php.tar.gz
tar xzf php.tar.gz --strip-components=1 -C php
cd php
./configure --prefix "$PREFIX"
make -j$(nproc)
make install -j$(nproc)
cd ../../
rm -rf build

View File

@ -1 +0,0 @@
export PATH=$PWD:$PATH

View File

@ -1,10 +1,20 @@
#!/bin/bash #!/bin/bash
PREFIX=$(realpath $(dirname $0))
mkdir -p build
cd build
curl "https://www.python.org/ftp/python/3.9.1/Python-3.9.1.tgz" -o python.tar.gz curl "https://www.python.org/ftp/python/3.9.1/Python-3.9.1.tgz" -o python.tar.gz
tar xzf python.tar.gz --strip-components=1 tar xzf python.tar.gz --strip-components=1
rm python.tar.gz rm python.tar.gz
./configure --prefix /piston/packages/python/3.9.1/python-3.9.1 ./configure --prefix "$PREFIX" --with-ensurepip=install
make -j$(nproc) make -j$(nproc)
ln -s python python3.9 make install -j$(nproc)
cd ..
rm -rf build

1
packages/python/3.9.1/environment vendored Normal file
View File

@ -0,0 +1 @@
export PATH=$PWD/bin:$PATH

7
packages/typescript/4.2.3/build.sh vendored Executable file
View File

@ -0,0 +1,7 @@
#!/usr/bin/env bash
source ../../node/15.10.0/build.sh
source ./environment
bin/npm install -g typescript@4.2.3

5
packages/typescript/4.2.3/compile vendored Normal file
View File

@ -0,0 +1,5 @@
#!/usr/bin/env bash
# Put instructions to compile source code, remove this file if the language does not require this stage
tsc $*

4
packages/typescript/4.2.3/environment vendored Normal file
View File

@ -0,0 +1,4 @@
#!/usr/bin/env bash
# Put 'export' statements here for environment variables
export PATH=$PWD/bin:$PATH

View File

@ -0,0 +1,6 @@
{
"language": "typescript",
"version": "4.2.3",
"aliases": ["ts","node-ts","tsc"],
"author": "Thomas Hobson <git@hexf.me>"
}

8
packages/typescript/4.2.3/run vendored Normal file
View File

@ -0,0 +1,8 @@
#!/usr/bin/env bash
# Put instructions to run the runtime
CODE=$(sed 's/ts$/js/' <<<"$1")
shift
node $CODE $*

1
packages/typescript/4.2.3/test.ts vendored Normal file
View File

@ -0,0 +1 @@
console.log("OK")

View File

@ -1,11 +1,11 @@
cd /packages cd /piston/packages
for pkg in "$@" for pkg in "$@"
do do
make -j16 $pkg.pkg.tar.gz make -j16 $pkg.pkg.tar.gz
done done
cd /repo cd /piston/repo
./mkindex.sh ./mkindex.sh
python3 -m http.server python3 -m http.server

View File

@ -1,5 +1,5 @@
{ pkgs ? import <nixpkgs> {} }: { pkgs ? import <nixpkgs> {} }:
pkgs.mkShell { pkgs.mkShell {
# nativeBuildInputs is usually what you want -- tools you need to run # nativeBuildInputs is usually what you want -- tools you need to run
nativeBuildInputs = [ pkgs.nodejs-15_x pkgs.yarn ]; nativeBuildInputs = [ pkgs.nodejs-15_x pkgs.yarn pkgs.jq ];
} }