Merge branch 'v3' of https://github.com/engineer-man/piston into v3
This commit is contained in:
commit
c3617c55a8
|
@ -1,40 +0,0 @@
|
||||||
{
|
|
||||||
"env": {
|
|
||||||
"commonjs": true,
|
|
||||||
"es2021": true,
|
|
||||||
"node": true
|
|
||||||
},
|
|
||||||
"plugins": [
|
|
||||||
"snakecasejs"
|
|
||||||
],
|
|
||||||
"extends": "eslint:recommended",
|
|
||||||
"parser": "babel-eslint",
|
|
||||||
"parserOptions": {
|
|
||||||
"ecmaVersion": 12
|
|
||||||
},
|
|
||||||
"settings":
|
|
||||||
{
|
|
||||||
"snakecasejs/filter": ["ClassDeclaration", "NewExpression"],
|
|
||||||
"snakecasejs/whitelist": []
|
|
||||||
},
|
|
||||||
"rules": {
|
|
||||||
"indent": [
|
|
||||||
"error",
|
|
||||||
4
|
|
||||||
],
|
|
||||||
"linebreak-style": [
|
|
||||||
"error",
|
|
||||||
"unix"
|
|
||||||
],
|
|
||||||
"quotes": [
|
|
||||||
"error",
|
|
||||||
"single"
|
|
||||||
],
|
|
||||||
"semi": [
|
|
||||||
"error",
|
|
||||||
"always"
|
|
||||||
],
|
|
||||||
"no-unused-vars": ["error", { "argsIgnorePattern": "^_"}],
|
|
||||||
"snakecasejs/snakecasejs": "warn"
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,7 +1,8 @@
|
||||||
FROM node:15.8.0-buster-slim
|
FROM node:15.8.0-buster-slim
|
||||||
RUN dpkg-reconfigure -p critical dash
|
RUN dpkg-reconfigure -p critical dash
|
||||||
RUN apt-get update && apt-get install -y libxml2 gnupg tar coreutils util-linux \
|
RUN apt-get update && \
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
apt-get install -y libxml2 gnupg tar coreutils util-linux && \
|
||||||
|
rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
RUN for i in $(seq 1001 1500); do \
|
RUN for i in $(seq 1001 1500); do \
|
||||||
groupadd -g $i runner$i && \
|
groupadd -g $i runner$i && \
|
||||||
|
|
|
@ -16,10 +16,5 @@
|
||||||
"uuid": "^8.3.2",
|
"uuid": "^8.3.2",
|
||||||
"yargs": "^16.2.0"
|
"yargs": "^16.2.0"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
|
||||||
"babel-eslint": "^10.1.0",
|
|
||||||
"eslint": "^7.20.0",
|
|
||||||
"eslint-plugin-snakecasejs": "^2.2.0"
|
|
||||||
},
|
|
||||||
"license": "MIT"
|
"license": "MIT"
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
const fss = require('fs');
|
const fss = require('fs');
|
||||||
const yargs = require('yargs');
|
const yargs = require('yargs');
|
||||||
const hide_bin = require('yargs/helpers').hideBin; //eslint-disable-line snakecasejs/snakecasejs
|
const hide_bin = require('yargs/helpers').hideBin;
|
||||||
const Logger = require('logplease');
|
const Logger = require('logplease');
|
||||||
const logger = Logger.create('config');
|
const logger = Logger.create('config');
|
||||||
const yaml = require('js-yaml');
|
const yaml = require('js-yaml');
|
||||||
|
@ -19,7 +19,7 @@ const header = `#
|
||||||
`;
|
`;
|
||||||
const argv = yargs(hide_bin(process.argv))
|
const argv = yargs(hide_bin(process.argv))
|
||||||
.usage('Usage: $0 -c [config]')
|
.usage('Usage: $0 -c [config]')
|
||||||
.demandOption('c') //eslint-disable-line snakecasejs/snakecasejs
|
.demandOption('c')
|
||||||
.option('config', {
|
.option('config', {
|
||||||
alias: 'c',
|
alias: 'c',
|
||||||
describe: 'config file to load from',
|
describe: 'config file to load from',
|
||||||
|
@ -29,18 +29,18 @@ const argv = yargs(hide_bin(process.argv))
|
||||||
alias: 'm',
|
alias: 'm',
|
||||||
type: 'boolean',
|
type: 'boolean',
|
||||||
describe: 'create config file and populate defaults if it does not already exist'
|
describe: 'create config file and populate defaults if it does not already exist'
|
||||||
}).argv;
|
})
|
||||||
|
.argv;
|
||||||
|
|
||||||
const options = [
|
const options = [
|
||||||
{
|
{
|
||||||
key: 'log_level',
|
key: 'log_level',
|
||||||
desc: 'Level of data to log',
|
desc: 'Level of data to log',
|
||||||
default: 'INFO',
|
default: 'INFO',
|
||||||
/* eslint-disable snakecasejs/snakecasejs */
|
|
||||||
options: Object.values(Logger.LogLevels),
|
options: Object.values(Logger.LogLevels),
|
||||||
validators: [x=>Object.values(Logger.LogLevels).includes(x) || `Log level ${x} does not exist`]
|
validators: [
|
||||||
/* eslint-enable snakecasejs/snakecasejs */
|
x => Object.values(Logger.LogLevels).includes(x) || `Log level ${x} does not exist`
|
||||||
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
key: 'bind_address',
|
key: 'bind_address',
|
||||||
|
@ -110,14 +110,15 @@ const options = [
|
||||||
}
|
}
|
||||||
];
|
];
|
||||||
|
|
||||||
function make_default_config(){
|
const make_default_config = () => {
|
||||||
let content = header.split('\n');
|
let content = header.split('\n');
|
||||||
|
|
||||||
options.forEach(option => {
|
options.forEach(option => {
|
||||||
content = content.concat(option.desc.split('\n').map(x=>`# ${x}`));
|
content = content.concat(option.desc.split('\n').map(x=>`# ${x}`));
|
||||||
|
|
||||||
if(option.options)
|
if (option.options) {
|
||||||
content.push('# Options: ' + option.options.join(', '));
|
content.push('# Options: ' + option.options.join(', '));
|
||||||
|
}
|
||||||
|
|
||||||
content.push(`${option.key}: ${option.default}`);
|
content.push(`${option.key}: ${option.default}`);
|
||||||
|
|
||||||
|
@ -125,24 +126,25 @@ function make_default_config(){
|
||||||
});
|
});
|
||||||
|
|
||||||
return content.join('\n');
|
return content.join('\n');
|
||||||
}
|
};
|
||||||
|
|
||||||
logger.info(`Loading Configuration from ${argv.config}`);
|
logger.info(`Loading Configuration from ${argv.config}`);
|
||||||
|
|
||||||
if(argv['make-config'])
|
if (argv['make-config']) {
|
||||||
logger.debug('Make configuration flag is set');
|
logger.debug('Make configuration flag is set');
|
||||||
|
}
|
||||||
|
|
||||||
if (!!argv['make-config'] && !fss.exists_sync(argv.config)) {
|
if (!!argv['make-config'] && !fss.exists_sync(argv.config)) {
|
||||||
logger.info('Writing default configuration...');
|
logger.info('Writing default configuration...');
|
||||||
try {
|
try {
|
||||||
fss.write_file_sync(argv.config, make_default_config());
|
fss.write_file_sync(argv.config, make_default_config());
|
||||||
} catch (err) {
|
} catch (e) {
|
||||||
logger.error('Error writing default configuration:', err.message);
|
logger.error('Error writing default configuration:', e.message);
|
||||||
process.exit(1);
|
process.exit(1);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
var config = {};
|
let config = {};
|
||||||
|
|
||||||
logger.debug('Reading config file');
|
logger.debug('Reading config file');
|
||||||
|
|
||||||
|
@ -156,22 +158,23 @@ try{
|
||||||
|
|
||||||
logger.debug('Validating config entries');
|
logger.debug('Validating config entries');
|
||||||
|
|
||||||
var errored=false;
|
let errored = false;
|
||||||
|
|
||||||
options.forEach(option => {
|
options.for_each(option => {
|
||||||
logger.debug('Checking option', option.key);
|
logger.debug('Checking option', option.key);
|
||||||
|
|
||||||
var cfg_val = config[option.key];
|
let cfg_val = config[option.key];
|
||||||
|
|
||||||
if(cfg_val == undefined){
|
if (cfg_val === undefined) {
|
||||||
errored = true;
|
errored = true;
|
||||||
logger.error(`Config key ${option.key} does not exist on currently loaded configuration`);
|
logger.error(`Config key ${option.key} does not exist on currently loaded configuration`);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
option.validators.forEach(validator => {
|
option.validators.for_each(validator => {
|
||||||
var response = validator(cfg_val);
|
let response = validator(cfg_val);
|
||||||
if(response !== true){
|
|
||||||
|
if (!response) {
|
||||||
errored = true;
|
errored = true;
|
||||||
logger.error(`Config option ${option.key} failed validation:`, response);
|
logger.error(`Config option ${option.key} failed validation:`, response);
|
||||||
return;
|
return;
|
||||||
|
@ -179,9 +182,10 @@ options.forEach(option => {
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
if(errored) process.exit(1);
|
if (errored) {
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
logger.info('Configuration successfully loaded');
|
logger.info('Configuration successfully loaded');
|
||||||
|
|
||||||
module.exports = config;
|
module.exports = config;
|
||||||
|
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
const logger = require('logplease').create('executor/job');
|
const logger = require('logplease').create('executor/job');
|
||||||
const { v4: uuidv4 } = require('uuid');
|
const uuidv4 = require('uuid/v4');
|
||||||
const cp = require('child_process');
|
const cp = require('child_process');
|
||||||
const path = require('path');
|
const path = require('path');
|
||||||
const config = require('../config');
|
const config = require('../config');
|
||||||
|
@ -12,10 +12,11 @@ const job_states = {
|
||||||
EXECUTED: Symbol('Executed and ready for cleanup')
|
EXECUTED: Symbol('Executed and ready for cleanup')
|
||||||
};
|
};
|
||||||
|
|
||||||
var uid=0;
|
let uid = 0;
|
||||||
var gid=0;
|
let gid = 0;
|
||||||
|
|
||||||
class Job {
|
class Job {
|
||||||
|
|
||||||
constructor({ runtime, files, args, stdin, timeouts, main }) {
|
constructor({ runtime, files, args, stdin, timeouts, main }) {
|
||||||
this.uuid = uuidv4();
|
this.uuid = uuidv4();
|
||||||
this.runtime = runtime;
|
this.runtime = runtime;
|
||||||
|
@ -25,8 +26,11 @@ class Job {
|
||||||
this.timeouts = timeouts;
|
this.timeouts = timeouts;
|
||||||
this.main = main;
|
this.main = main;
|
||||||
|
|
||||||
if(!this.files.map(f=>f.name).includes(this.main))
|
let file_list = this.files.map(f => f.name);
|
||||||
|
|
||||||
|
if (!file_list.includes(this.main)) {
|
||||||
throw new Error(`Main file "${this.main}" will not be written to disk`);
|
throw new Error(`Main file "${this.main}" will not be written to disk`);
|
||||||
|
}
|
||||||
|
|
||||||
this.uid = config.runner_uid_min + uid;
|
this.uid = config.runner_uid_min + uid;
|
||||||
this.gid = config.runner_gid_min + gid;
|
this.gid = config.runner_gid_min + gid;
|
||||||
|
@ -46,29 +50,25 @@ class Job {
|
||||||
|
|
||||||
logger.debug('Writing files to job cache');
|
logger.debug('Writing files to job cache');
|
||||||
|
|
||||||
await fs.mkdir(this.dir, {mode:0o700});
|
|
||||||
|
|
||||||
const files = this.files.map(({name: file_name, content}) => {
|
|
||||||
return fs.write_file(path.join(this.dir, file_name), content);
|
|
||||||
});
|
|
||||||
|
|
||||||
await Promise.all(files);
|
|
||||||
|
|
||||||
logger.debug(`Transfering ownership uid=${this.uid} gid=${this.gid}`);
|
logger.debug(`Transfering ownership uid=${this.uid} gid=${this.gid}`);
|
||||||
|
|
||||||
|
await fs.mkdir(this.dir, { mode:0o700 });
|
||||||
await fs.chown(this.dir, this.uid, this.gid);
|
await fs.chown(this.dir, this.uid, this.gid);
|
||||||
|
|
||||||
const chowns = this.files.map(({name:file_name}) => {
|
for (const file of this.files) {
|
||||||
return fs.chown(path.join(this.dir, file_name), this.uid, this.gid);
|
let file_path = path.join(this.dir, file.name);
|
||||||
});
|
|
||||||
|
|
||||||
await Promise.all(chowns);
|
await fs.write_file(file_path, file.content);
|
||||||
|
await fs.chown(file_path, this.uid, this.gid);
|
||||||
|
}
|
||||||
|
|
||||||
this.state = job_states.PRIMED;
|
this.state = job_states.PRIMED;
|
||||||
|
|
||||||
logger.debug('Primed job');
|
logger.debug('Primed job');
|
||||||
}
|
}
|
||||||
|
|
||||||
async safe_call(file, args, timeout) {
|
async safe_call(file, args, timeout) {
|
||||||
return await new Promise((resolve, reject) => {
|
return new Promise((resolve, reject) => {
|
||||||
const unshare = config.enable_unshare ? ['unshare','-n','-r'] : [];
|
const unshare = config.enable_unshare ? ['unshare','-n','-r'] : [];
|
||||||
|
|
||||||
const prlimit = [
|
const prlimit = [
|
||||||
|
@ -98,22 +98,36 @@ class Job {
|
||||||
proc.stdin.write(this.stdin);
|
proc.stdin.write(this.stdin);
|
||||||
proc.stdin.end();
|
proc.stdin.end();
|
||||||
|
|
||||||
|
const kill_timeout = set_timeout(_ => proc.kill('SIGKILL'), timeout);
|
||||||
|
|
||||||
const kill_timeout = setTimeout(_ => proc.kill('SIGKILL'), timeout);
|
proc.stderr.on('data', data => {
|
||||||
|
if (stderr.length > config.output_max_size) {
|
||||||
|
proc.kill('SIGKILL');
|
||||||
|
} else {
|
||||||
|
stderr += data;
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
proc.stderr.on('data', d=>{if(stderr.length>config.output_max_size) proc.kill('SIGKILL'); else stderr += d;});
|
proc.stdout.on('data', data => {
|
||||||
proc.stdout.on('data', d=>{if(stdout.length>config.output_max_size) proc.kill('SIGKILL'); else stdout += d;});
|
if (stdout.length > config.output_max_size) {
|
||||||
|
proc.kill('SIGKILL');
|
||||||
|
} else {
|
||||||
|
stdout += data;
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
const exit_cleanup = () => {
|
||||||
|
clear_timeout(kill_timeout);
|
||||||
|
|
||||||
function exit_cleanup(){
|
|
||||||
clearTimeout(kill_timeout);
|
|
||||||
proc.stderr.destroy();
|
proc.stderr.destroy();
|
||||||
proc.stdout.destroy();
|
proc.stdout.destroy();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
process.kill(-proc.pid, 'SIGKILL');
|
process.kill(-proc.pid, 'SIGKILL');
|
||||||
} catch {
|
} catch {
|
||||||
// Process will be dead already, so nothing to kill.
|
// Process will be dead already, so nothing to kill.
|
||||||
}
|
}
|
||||||
}
|
};
|
||||||
|
|
||||||
proc.on('exit', (code, signal)=>{
|
proc.on('exit', (code, signal)=>{
|
||||||
exit_cleanup();
|
exit_cleanup();
|
||||||
|
@ -130,27 +144,31 @@ class Job {
|
||||||
}
|
}
|
||||||
|
|
||||||
async execute() {
|
async execute() {
|
||||||
if(this.state != job_states.PRIMED)
|
if (this.state !== job_states.PRIMED) {
|
||||||
throw new Error('Job must be in primed state, current state: ' + this.state.toString());
|
throw new Error('Job must be in primed state, current state: ' + this.state.toString());
|
||||||
|
}
|
||||||
|
|
||||||
logger.info(`Executing job uuid=${this.uuid} uid=${this.uid} gid=${this.gid} runtime=${this.runtime.toString()}`);
|
logger.info(`Executing job uuid=${this.uuid} uid=${this.uid} gid=${this.gid} runtime=${this.runtime.toString()}`);
|
||||||
|
|
||||||
logger.debug('Compiling');
|
logger.debug('Compiling');
|
||||||
|
|
||||||
var compile = undefined;
|
let compile;
|
||||||
if(this.runtime.compiled)
|
|
||||||
|
if (this.runtime.compiled) {
|
||||||
compile = await this.safe_call(
|
compile = await this.safe_call(
|
||||||
path.join(this.runtime.pkgdir, 'compile'),
|
path.join(this.runtime.pkgdir, 'compile'),
|
||||||
this.files.map(x => x.name),
|
this.files.map(x => x.name),
|
||||||
this.timeouts.compile);
|
this.timeouts.compile
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
logger.debug('Running');
|
logger.debug('Running');
|
||||||
|
|
||||||
const run = await this.safe_call(
|
const run = await this.safe_call(
|
||||||
path.join(this.runtime.pkgdir, 'run'),
|
path.join(this.runtime.pkgdir, 'run'),
|
||||||
[this.main, ...this.args],
|
[this.main, ...this.args],
|
||||||
this.timeouts.run);
|
this.timeouts.run
|
||||||
|
);
|
||||||
|
|
||||||
this.state = job_states.EXECUTED;
|
this.state = job_states.EXECUTED;
|
||||||
|
|
||||||
|
@ -158,13 +176,15 @@ class Job {
|
||||||
compile,
|
compile,
|
||||||
run
|
run
|
||||||
};
|
};
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async cleanup() {
|
async cleanup() {
|
||||||
logger.info(`Cleaning up job uuid=${this.uuid}`);
|
logger.info(`Cleaning up job uuid=${this.uuid}`);
|
||||||
await fs.rm(this.dir, { recursive: true, force: true });
|
await fs.rm(this.dir, { recursive: true, force: true });
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = {Job};
|
module.exports = {
|
||||||
|
Job
|
||||||
|
};
|
||||||
|
|
|
@ -6,38 +6,45 @@ const { Job } = require('./job');
|
||||||
const { body } = require('express-validator');
|
const { body } = require('express-validator');
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
|
|
||||||
run_job_validators: [
|
run_job_validators: [
|
||||||
body('language')
|
body('language')
|
||||||
.isString(), // eslint-disable-line snakecasejs/snakecasejs
|
.isString(),
|
||||||
body('version')
|
body('version')
|
||||||
.isString(), // eslint-disable-line snakecasejs/snakecasejs
|
.isString(),
|
||||||
// isSemVer requires it to be a version, not a selector
|
// isSemVer requires it to be a version, not a selector
|
||||||
body('files')
|
body('files')
|
||||||
.isArray(), // eslint-disable-line snakecasejs/snakecasejs
|
.isArray(),
|
||||||
body('files.*.name')
|
body('files.*.name')
|
||||||
.isString() // eslint-disable-line snakecasejs/snakecasejs
|
.isString()
|
||||||
.bail()
|
.bail()
|
||||||
.not()
|
.not()
|
||||||
.contains('/'),
|
.contains('/'),
|
||||||
body('files.*.content')
|
body('files.*.content')
|
||||||
.isString(), // eslint-disable-line snakecasejs/snakecasejs
|
.isString(),
|
||||||
body('compile_timeout')
|
body('compile_timeout')
|
||||||
.isNumeric(), // eslint-disable-line snakecasejs/snakecasejs
|
.isNumeric(),
|
||||||
body('run_timeout')
|
body('run_timeout')
|
||||||
.isNumeric(), // eslint-disable-line snakecasejs/snakecasejs
|
.isNumeric(),
|
||||||
body('stdin')
|
body('stdin')
|
||||||
.isString(), // eslint-disable-line snakecasejs/snakecasejs
|
.isString(),
|
||||||
body('args')
|
body('args')
|
||||||
.isArray(),
|
.isArray(),
|
||||||
body('args.*')
|
body('args.*')
|
||||||
.isString() // eslint-disable-line snakecasejs/snakecasejs
|
.isString()
|
||||||
],
|
],
|
||||||
async run_job(req, res){
|
|
||||||
// POST /jobs
|
// POST /jobs
|
||||||
|
async run_job(req, res) {
|
||||||
|
|
||||||
const runtime = get_latest_runtime_matching_language_version(req.body.language, req.body.version);
|
const runtime = get_latest_runtime_matching_language_version(req.body.language, req.body.version);
|
||||||
if(runtime == undefined) return res.json_error(`${req.body.language}-${req.body.version} runtime is unknown`, 400);
|
|
||||||
|
if (runtime === undefined) {
|
||||||
|
return res
|
||||||
|
.status(400)
|
||||||
|
.send({
|
||||||
|
message: `${req.body.language}-${req.body.version} runtime is unknown`
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
const job = new Job({
|
const job = new Job({
|
||||||
runtime,
|
runtime,
|
||||||
|
@ -54,8 +61,12 @@ module.exports = {
|
||||||
await job.prime();
|
await job.prime();
|
||||||
|
|
||||||
const result = await job.execute();
|
const result = await job.execute();
|
||||||
res.json_success(result);
|
|
||||||
|
|
||||||
await job.cleanup();
|
await job.cleanup();
|
||||||
|
|
||||||
|
return res
|
||||||
|
.status(200)
|
||||||
|
.send(result);
|
||||||
}
|
}
|
||||||
|
|
||||||
};
|
};
|
|
@ -9,111 +9,98 @@ const fs = require('fs/promises');
|
||||||
const fss = require('fs');
|
const fss = require('fs');
|
||||||
const body_parser = require('body-parser');
|
const body_parser = require('body-parser');
|
||||||
const runtime = require('./runtime');
|
const runtime = require('./runtime');
|
||||||
const {validationResult} = require('express-validator'); //eslint-disable-line snakecasejs/snakecasejs
|
const { validationResult } = require('express-validator');
|
||||||
|
|
||||||
const logger = Logger.create('index');
|
const logger = Logger.create('index');
|
||||||
const app = express();
|
const app = express();
|
||||||
|
|
||||||
(async () => {
|
(async () => {
|
||||||
logger.info('Setting loglevel to',config.log_level);
|
logger.info('Setting loglevel to',config.log_level);
|
||||||
Logger.setLogLevel(config.log_level); //eslint-disable-line snakecasejs/snakecasejs
|
Logger.setLogLevel(config.log_level);
|
||||||
|
|
||||||
logger.debug('Ensuring data directories exist');
|
logger.debug('Ensuring data directories exist');
|
||||||
Object.values(globals.data_directories).forEach(dir => {
|
|
||||||
var data_path = path.join(config.data_directory, dir);
|
Object.values(globals.data_directories).for_each(dir => {
|
||||||
|
let data_path = path.join(config.data_directory, dir);
|
||||||
|
|
||||||
logger.debug(`Ensuring ${data_path} exists`);
|
logger.debug(`Ensuring ${data_path} exists`);
|
||||||
|
|
||||||
if (!fss.exists_sync(data_path)) {
|
if (!fss.exists_sync(data_path)) {
|
||||||
logger.info(`${data_path} does not exist.. Creating..`);
|
logger.info(`${data_path} does not exist.. Creating..`);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
fss.mkdir_sync(data_path);
|
fss.mkdir_sync(data_path);
|
||||||
}catch(err){
|
} catch(e) {
|
||||||
logger.error(`Failed to create ${data_path}: `, err.message);
|
logger.error(`Failed to create ${data_path}: `, e.message);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
});
|
});
|
||||||
|
|
||||||
logger.info('Loading packages');
|
logger.info('Loading packages');
|
||||||
const pkgdir = path.join(config.data_directory,globals.data_directories.packages);
|
const pkgdir = path.join(config.data_directory,globals.data_directories.packages);
|
||||||
|
|
||||||
const pkglist = await fs.readdir(pkgdir);
|
const pkglist = await fs.readdir(pkgdir);
|
||||||
|
|
||||||
const languages = await Promise.all(
|
const languages = await Promise.all(
|
||||||
pkglist.map(lang=>
|
pkglist.map(lang=>
|
||||||
fs.readdir(path.join(pkgdir,lang))
|
fs.readdir(path.join(pkgdir,lang))
|
||||||
.then(x=>x.map(y=>path.join(pkgdir, lang, y)))
|
.then(x=>x.map(y=>path.join(pkgdir, lang, y)))
|
||||||
));
|
));
|
||||||
const installed_languages = languages.flat()
|
|
||||||
|
const installed_languages = languages
|
||||||
|
.flat()
|
||||||
.filter(pkg => fss.exists_sync(path.join(pkg, globals.pkg_installed_file)));
|
.filter(pkg => fss.exists_sync(path.join(pkg, globals.pkg_installed_file)));
|
||||||
|
|
||||||
installed_languages.forEach(pkg => new runtime.Runtime(pkg));
|
installed_languages.forEach(pkg => new runtime.Runtime(pkg));
|
||||||
|
|
||||||
logger.info('Starting API Server');
|
logger.info('Starting API Server');
|
||||||
|
|
||||||
logger.debug('Constructing Express App');
|
logger.debug('Constructing Express App');
|
||||||
|
|
||||||
logger.debug('Registering custom message wrappers');
|
|
||||||
|
|
||||||
express.response.json_error = function(message, code) {
|
|
||||||
this.status(code);
|
|
||||||
return this.json({success: false, message, code});
|
|
||||||
};
|
|
||||||
|
|
||||||
express.response.json_success = function(obj) {
|
|
||||||
return this.json({success: true, data: obj});
|
|
||||||
};
|
|
||||||
|
|
||||||
logger.debug('Registering middleware');
|
logger.debug('Registering middleware');
|
||||||
|
|
||||||
app.use(body_parser.urlencoded({ extended: true }));
|
app.use(body_parser.urlencoded({ extended: true }));
|
||||||
app.use(body_parser.json());
|
app.use(body_parser.json());
|
||||||
|
|
||||||
|
const validate = (req, res, next) => {
|
||||||
|
const errors = validationResult(req);
|
||||||
|
|
||||||
function validate(req, res, next) {
|
if (!errors.isEmpty()) {
|
||||||
const errors = validationResult(req); //eslint-disable-line snakecasejs/snakecasejs
|
return res
|
||||||
if (!errors.isEmpty()) //eslint-disable-line snakecasejs/snakecasejs
|
.status(422)
|
||||||
return res.json_error(errors.array(), 422);
|
.send({
|
||||||
next();
|
message: errors.array()
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
next();
|
||||||
|
};
|
||||||
|
|
||||||
logger.debug('Registering Routes');
|
logger.debug('Registering Routes');
|
||||||
|
|
||||||
const ppman_routes = require('./ppman/routes');
|
const ppman_routes = require('./ppman/routes');
|
||||||
const executor_routes = require('./executor/routes');
|
const executor_routes = require('./executor/routes');
|
||||||
|
|
||||||
|
app.get('/packages', ppman_routes.package_list);
|
||||||
app.get('/packages',
|
app.post('/packages/:language/:version', ppman_routes.package_install);
|
||||||
ppman_routes.package_list
|
app.delete('/packages/:language/:version', ppman_routes.package_uninstall);
|
||||||
);
|
|
||||||
|
|
||||||
app.post('/packages/:language/:version',
|
|
||||||
ppman_routes.package_install
|
|
||||||
);
|
|
||||||
|
|
||||||
app.delete('/packages/:language/:version',
|
|
||||||
ppman_routes.package_uninstall
|
|
||||||
);
|
|
||||||
|
|
||||||
app.post('/jobs',
|
app.post('/jobs',
|
||||||
executor_routes.run_job_validators,
|
executor_routes.run_job_validators,
|
||||||
validate,
|
validate,
|
||||||
executor_routes.run_job);
|
executor_routes.run_job
|
||||||
|
);
|
||||||
function list_runtimes(_, res){
|
app.get('/runtimes', (req, res) => {
|
||||||
const runtimes = runtime.map(rt => (
|
const runtimes = runtime
|
||||||
{
|
.map(rt => {
|
||||||
|
return {
|
||||||
language: rt.language,
|
language: rt.language,
|
||||||
version: rt.version.raw,
|
version: rt.version.raw,
|
||||||
author: rt.author,
|
author: rt.author,
|
||||||
aliases: rt.aliases
|
aliases: rt.aliases
|
||||||
}
|
};
|
||||||
));
|
|
||||||
|
|
||||||
return res.json_success({
|
|
||||||
runtimes
|
|
||||||
});
|
});
|
||||||
}
|
|
||||||
|
|
||||||
app.get('/runtimes', list_runtimes);
|
return res
|
||||||
|
.status(200)
|
||||||
|
.send(runtimes);
|
||||||
|
});
|
||||||
|
|
||||||
logger.debug('Calling app.listen');
|
logger.debug('Calling app.listen');
|
||||||
const [ address, port ] = config.bind_address.split(':');
|
const [ address, port ] = config.bind_address.split(':');
|
||||||
|
@ -121,5 +108,4 @@ const app = express();
|
||||||
app.listen(port, address, () => {
|
app.listen(port, address, () => {
|
||||||
logger.info('API server started on', config.bind_address);
|
logger.info('API server started on', config.bind_address);
|
||||||
});
|
});
|
||||||
|
|
||||||
})();
|
})();
|
|
@ -11,6 +11,7 @@ const crypto = require('crypto');
|
||||||
const runtime = require('../runtime');
|
const runtime = require('../runtime');
|
||||||
|
|
||||||
class Package {
|
class Package {
|
||||||
|
|
||||||
constructor({ language, version, download, checksum }){
|
constructor({ language, version, download, checksum }){
|
||||||
this.language = language;
|
this.language = language;
|
||||||
this.version = semver.parse(version);
|
this.version = semver.parse(version);
|
||||||
|
@ -22,19 +23,20 @@ class Package {
|
||||||
return fss.exists_sync(path.join(this.install_path, globals.pkg_installed_file));
|
return fss.exists_sync(path.join(this.install_path, globals.pkg_installed_file));
|
||||||
}
|
}
|
||||||
|
|
||||||
get download_url(){
|
|
||||||
return this.download;
|
|
||||||
}
|
|
||||||
|
|
||||||
get install_path() {
|
get install_path() {
|
||||||
return path.join(config.data_directory,
|
return path.join(
|
||||||
|
config.data_directory,
|
||||||
globals.data_directories.packages,
|
globals.data_directories.packages,
|
||||||
this.language,
|
this.language,
|
||||||
this.version.raw);
|
this.version.raw
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
async install() {
|
async install() {
|
||||||
if(this.installed) throw new Error('Already installed');
|
if (this.installed) {
|
||||||
|
throw new Error('Already installed');
|
||||||
|
}
|
||||||
|
|
||||||
logger.info(`Installing ${this.language}-${this.version.raw}`);
|
logger.info(`Installing ${this.language}-${this.version.raw}`);
|
||||||
|
|
||||||
if (fss.exists_sync(this.install_path)) {
|
if (fss.exists_sync(this.install_path)) {
|
||||||
|
@ -45,32 +47,37 @@ class Package {
|
||||||
logger.debug(`Making directory ${this.install_path}`);
|
logger.debug(`Making directory ${this.install_path}`);
|
||||||
await fs.mkdir(this.install_path, {recursive: true});
|
await fs.mkdir(this.install_path, {recursive: true});
|
||||||
|
|
||||||
|
logger.debug(`Downloading package from ${this.download} in to ${this.install_path}`);
|
||||||
|
const pkgpath = path.join(this.install_path, 'pkg.tar.gz');
|
||||||
|
const download = await fetch(this.download);
|
||||||
|
|
||||||
logger.debug(`Downloading package from ${this.download_url} in to ${this.install_path}`);
|
|
||||||
const pkgpath = path.join(this.install_path, "pkg.tar.gz");
|
|
||||||
const download = await fetch(this.download_url);
|
|
||||||
const file_stream = fss.create_write_stream(pkgpath);
|
const file_stream = fss.create_write_stream(pkgpath);
|
||||||
await new Promise((resolve, reject) => {
|
await new Promise((resolve, reject) => {
|
||||||
download.body.pipe(file_stream)
|
download.body.pipe(file_stream);
|
||||||
download.body.on("error", reject)
|
download.body.on('error', reject);
|
||||||
file_stream.on("finish", resolve)
|
|
||||||
|
file_stream.on('finish', resolve);
|
||||||
});
|
});
|
||||||
|
|
||||||
logger.debug('Validating checksums');
|
logger.debug('Validating checksums');
|
||||||
logger.debug(`Assert sha256(pkg.tar.gz) == ${this.checksum}`)
|
logger.debug(`Assert sha256(pkg.tar.gz) == ${this.checksum}`);
|
||||||
const cs = crypto.create_hash("sha256")
|
const cs = crypto.create_hash("sha256")
|
||||||
.update(fss.readFileSync(pkgpath))
|
.update(fss.readFileSync(pkgpath))
|
||||||
.digest('hex');
|
.digest('hex');
|
||||||
if(cs != this.checksum) throw new Error(`Checksum miss-match want: ${val} got: ${cs}`);
|
|
||||||
|
if (cs !== this.checksum) {
|
||||||
|
throw new Error(`Checksum miss-match want: ${val} got: ${cs}`);
|
||||||
|
}
|
||||||
|
|
||||||
logger.debug(`Extracting package files from archive ${pkgpath} in to ${this.install_path}`);
|
logger.debug(`Extracting package files from archive ${pkgpath} in to ${this.install_path}`);
|
||||||
|
|
||||||
await new Promise((resolve, reject) => {
|
await new Promise((resolve, reject) => {
|
||||||
const proc = cp.exec(`bash -c 'cd "${this.install_path}" && tar xzf ${pkgpath}'`);
|
const proc = cp.exec(`bash -c 'cd "${this.install_path}" && tar xzf ${pkgpath}'`);
|
||||||
|
|
||||||
proc.once('exit', (code, _) => {
|
proc.once('exit', (code, _) => {
|
||||||
if(code == 0) resolve();
|
code === 0 ? resolve() : reject();
|
||||||
else reject(new Error('Failed to extract package'));
|
|
||||||
});
|
});
|
||||||
|
|
||||||
proc.stdout.pipe(process.stdout);
|
proc.stdout.pipe(process.stdout);
|
||||||
proc.stderr.pipe(process.stderr);
|
proc.stderr.pipe(process.stderr);
|
||||||
|
|
||||||
|
@ -80,27 +87,34 @@ class Package {
|
||||||
logger.debug('Registering runtime');
|
logger.debug('Registering runtime');
|
||||||
new runtime.Runtime(this.install_path);
|
new runtime.Runtime(this.install_path);
|
||||||
|
|
||||||
|
|
||||||
logger.debug('Caching environment');
|
logger.debug('Caching environment');
|
||||||
const get_env_command = `cd ${this.install_path}; source environment; env`;
|
const get_env_command = `cd ${this.install_path}; source environment; env`;
|
||||||
|
|
||||||
const envout = await new Promise((resolve, reject) => {
|
const envout = await new Promise((resolve, reject) => {
|
||||||
var stdout = '';
|
let stdout = '';
|
||||||
const proc = cp.spawn('env',['-i','bash','-c',`${get_env_command}`], {
|
|
||||||
stdio: ['ignore', 'pipe', 'pipe']});
|
const proc = cp
|
||||||
|
.spawn(
|
||||||
|
'env',
|
||||||
|
['-i','bash','-c',`${get_env_command}`],
|
||||||
|
{
|
||||||
|
stdio: ['ignore', 'pipe', 'pipe']
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
proc.once('exit', (code, _) => {
|
proc.once('exit', (code, _) => {
|
||||||
if(code == 0) resolve(stdout);
|
code === 0 ? resolve() : reject();
|
||||||
else reject(new Error('Failed to cache environment'));
|
|
||||||
});
|
});
|
||||||
|
|
||||||
proc.stdout.on('data', (data)=>{
|
proc.stdout.on('data', data => {
|
||||||
stdout += data;
|
stdout += data;
|
||||||
});
|
});
|
||||||
|
|
||||||
proc.once('error', reject);
|
proc.once('error', reject);
|
||||||
});
|
});
|
||||||
|
|
||||||
const filtered_env = envout.split('\n')
|
const filtered_env = envout
|
||||||
|
.split('\n')
|
||||||
.filter(l => !['PWD','OLDPWD','_', 'SHLVL'].includes(l.split('=',2)[0]))
|
.filter(l => !['PWD','OLDPWD','_', 'SHLVL'].includes(l.split('=',2)[0]))
|
||||||
.join('\n');
|
.join('\n');
|
||||||
|
|
||||||
|
@ -116,7 +130,9 @@ class Package {
|
||||||
version: this.version.raw
|
version: this.version.raw
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
module.exports = {Package};
|
Package
|
||||||
|
};
|
||||||
|
|
|
@ -4,66 +4,98 @@ const fetch = require('node-fetch');
|
||||||
const config = require('../config');
|
const config = require('../config');
|
||||||
const { Package } = require('./package');
|
const { Package } = require('./package');
|
||||||
|
|
||||||
|
const get_package_list = async () => {
|
||||||
async function get_package_list(){
|
|
||||||
const repo_content = await fetch(config.repo_url).then(x => x.text());
|
const repo_content = await fetch(config.repo_url).then(x => x.text());
|
||||||
|
|
||||||
const entries = repo_content.split('\n').filter(x=>x.length > 0);
|
const entries = repo_content
|
||||||
|
.split('\n')
|
||||||
|
.filter(x => x.length > 0);
|
||||||
|
|
||||||
return entries.map(line => {
|
return entries.map(line => {
|
||||||
const [ language, version, checksum, download ] = line.split(',', 4);
|
const [ language, version, checksum, download ] = line.split(',', 4);
|
||||||
return new Package({language, version, checksum, download});
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
|
return new Package({
|
||||||
|
language,
|
||||||
|
version,
|
||||||
|
checksum,
|
||||||
|
download
|
||||||
|
});
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
async function get_package(lang, version){
|
const get_package async (lang, version) => {
|
||||||
const packages = await get_package_list();
|
const packages = await get_package_list();
|
||||||
const candidates = packages.filter(
|
|
||||||
pkg => pkg.language == lang && semver.satisfies(pkg.version, version)
|
const candidates = packages
|
||||||
);
|
.filter(pkg => {
|
||||||
return candidates.sort((a,b)=>semver.rcompare(a.version,b.version))[0] || null;
|
return pkg.language == lang && semver.satisfies(pkg.version, version)
|
||||||
}
|
});
|
||||||
|
|
||||||
|
candidates.sort((a, b) => semver.rcompare(a.version, b.version));
|
||||||
|
|
||||||
|
return candidates[0] || null;
|
||||||
|
};
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
|
|
||||||
async package_list(req, res){
|
|
||||||
// GET /packages
|
// GET /packages
|
||||||
|
async package_list(req, res) {
|
||||||
logger.debug('Request to list packages');
|
logger.debug('Request to list packages');
|
||||||
|
|
||||||
const packages = await get_package_list();
|
const packages = await get_package_list();
|
||||||
|
|
||||||
res.json_success({
|
packages = packages
|
||||||
packages: packages.map(pkg=>({
|
.map(pkg => {
|
||||||
|
return {
|
||||||
language: pkg.language,
|
language: pkg.language,
|
||||||
language_version: pkg.version.raw,
|
language_version: pkg.version.raw,
|
||||||
installed: pkg.installed
|
installed: pkg.installed
|
||||||
}))
|
};
|
||||||
});
|
});
|
||||||
|
|
||||||
|
return res
|
||||||
|
.status(200)
|
||||||
|
.send(packages);
|
||||||
},
|
},
|
||||||
async package_install(req,res){
|
|
||||||
// POST /packages/:language/:version
|
|
||||||
|
|
||||||
|
// POST /packages/:language/:version
|
||||||
|
async package_install(req, res) {
|
||||||
logger.debug('Request to install package');
|
logger.debug('Request to install package');
|
||||||
|
|
||||||
const pkg = await get_package(req.params.language, req.params.version);
|
const pkg = await get_package(req.params.language, req.params.version);
|
||||||
if(pkg == null) return res.json_error(`Requested package ${req.params.language}-${req.params.version} does not exist`, 404);
|
|
||||||
|
if (pkg == null) {
|
||||||
|
return res
|
||||||
|
.status(404)
|
||||||
|
.send({
|
||||||
|
message: `Requested package ${req.params.language}-${req.params.version} does not exist`
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const response = await pkg.install();
|
const response = await pkg.install();
|
||||||
return res.json_success(response);
|
|
||||||
}catch(err){
|
return res
|
||||||
logger.error(`Error while installing package ${pkg.language}-${pkg.version}:`, err.message);
|
.status(200)
|
||||||
res.json_error(err.message,500);
|
.send(response);
|
||||||
|
} catch(e) {
|
||||||
|
logger.error(`Error while installing package ${pkg.language}-${pkg.version}:`, e.message);
|
||||||
|
|
||||||
|
return res
|
||||||
|
.status(500)
|
||||||
|
.send({
|
||||||
|
message: e.message
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
},
|
},
|
||||||
async package_uninstall(req,res){
|
|
||||||
// DELETE /packages/:language/:version
|
|
||||||
|
|
||||||
//res.json(req.body); //TODO
|
// DELETE /packages/:language/:version
|
||||||
res.json_error('not implemented', 500);
|
async package_uninstall(req, res) {
|
||||||
|
return res
|
||||||
|
.status(500)
|
||||||
|
.send({
|
||||||
|
message: 'Not implemented'
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
};
|
};
|
|
@ -8,50 +8,57 @@ const path = require('path');
|
||||||
const runtimes = [];
|
const runtimes = [];
|
||||||
|
|
||||||
class Runtime {
|
class Runtime {
|
||||||
#env_vars
|
|
||||||
#compiled
|
|
||||||
constructor(package_dir){
|
constructor(package_dir){
|
||||||
const {language, version, author, build_platform, aliases} = JSON.parse(
|
let info = JSON.parse(
|
||||||
fss.read_file_sync(path.join(package_dir, 'pkg-info.json'))
|
fss.read_file_sync(path.join(package_dir, 'pkg-info.json'))
|
||||||
);
|
);
|
||||||
|
|
||||||
|
const { language, version, author, build_platform, aliases } = info;
|
||||||
|
|
||||||
this.pkgdir = package_dir;
|
this.pkgdir = package_dir;
|
||||||
this.language = language;
|
this.language = language;
|
||||||
this.version = semver.parse(version);
|
this.version = semver.parse(version);
|
||||||
this.author = author;
|
this.author = author;
|
||||||
this.aliases = aliases;
|
this.aliases = aliases;
|
||||||
|
|
||||||
if(build_platform != globals.platform){
|
if (build_platform !== globals.platform) {
|
||||||
logger.warn(`Package ${language}-${version} was built for platform ${build_platform}, but our platform is ${globals.platform}`);
|
logger.warn(
|
||||||
|
`Package ${language}-${version} was built for platform ${build_platform}, ` +
|
||||||
|
`but our platform is ${globals.platform}`
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.debug(`Package ${language}-${version} was loaded`);
|
logger.debug(`Package ${language}-${version} was loaded`);
|
||||||
|
|
||||||
runtimes.push(this);
|
runtimes.push(this);
|
||||||
}
|
}
|
||||||
|
|
||||||
get env_file_path(){
|
get compiled() {
|
||||||
return path.join(this.pkgdir, 'environment');
|
if (this.compiled === undefined) {
|
||||||
|
this.compiled = fss.exists_sync(path.join(this.pkgdir, 'compile'));
|
||||||
}
|
}
|
||||||
|
|
||||||
get compiled(){
|
return this.compiled;
|
||||||
if(this.#compiled === undefined) this.#compiled = fss.exists_sync(path.join(this.pkgdir, 'compile'));
|
|
||||||
return this.#compiled;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
get env_vars() {
|
get env_vars() {
|
||||||
if(!this.#env_vars){
|
if (!this.env_vars) {
|
||||||
const env_file = path.join(this.pkgdir, '.env');
|
const env_file = path.join(this.pkgdir, '.env');
|
||||||
const env_content = fss.read_file_sync(env_file).toString();
|
const env_content = fss.read_file_sync(env_file).toString();
|
||||||
this.#env_vars = {};
|
|
||||||
|
this.env_vars = {};
|
||||||
|
|
||||||
env_content
|
env_content
|
||||||
.trim()
|
.trim()
|
||||||
.split('\n')
|
.split('\n')
|
||||||
.map(line => line.split('=',2))
|
.map(line => line.split('=',2))
|
||||||
.forEach(([key,val]) => {
|
.forEach(([key,val]) => {
|
||||||
this.#env_vars[key.trim()] = val.trim();
|
this.env_vars[key.trim()] = val.trim();
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
return this.#env_vars;
|
|
||||||
|
return this.env_vars;
|
||||||
}
|
}
|
||||||
|
|
||||||
toString() {
|
toString() {
|
||||||
|
@ -68,4 +75,3 @@ module.exports.get_latest_runtime_matching_language_version = function(lang, ver
|
||||||
return module.exports.get_runtimes_matching_language_version(lang, ver)
|
return module.exports.get_runtimes_matching_language_version(lang, ver)
|
||||||
.sort((a,b) => semver.rcompare(a.version, b.version))[0];
|
.sort((a,b) => semver.rcompare(a.version, b.version))[0];
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue