Mascali
/
piston
mirror of https://github.com/engineer-man/piston.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

lint like rest of codebase

This commit is contained in:
Thomas Hobson 2021-05-08 12:30:40 +12:00
parent 1b7504a191
commit b3be57e0b4
No known key found for this signature in database
GPG Key ID: 9F1FD9D87950DB6F
10 changed files with 1893 additions and 1864 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
api/.prettierrc.yaml
View File

@ -1 +1,3 @@
singleQuote: true singleQuote: true
tabWidth: 4
arrowParens: avoid

2028
api/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

46
api/package.json
View File

@ -1,25 +1,25 @@
{ {
"name": "piston-api", "name": "piston-api",
"version": "3.0.0", "version": "3.0.0",
"description": "API for piston - a high performance code execution engine", "description": "API for piston - a high performance code execution engine",
"main": "src/index.js", "main": "src/index.js",
"dependencies": { "dependencies": {
"body-parser": "^1.19.0", "body-parser": "^1.19.0",
"chownr": "^2.0.0", "chownr": "^2.0.0",
"express": "^4.17.1", "express": "^4.17.1",
"is-docker": "^2.1.1", "is-docker": "^2.1.1",
"logplease": "^1.2.15", "logplease": "^1.2.15",
"nocamel": "HexF/nocamel#patch-1", "nocamel": "HexF/nocamel#patch-1",
"node-fetch": "^2.6.1", "node-fetch": "^2.6.1",
"semver": "^7.3.4", "semver": "^7.3.4",
"uuid": "^8.3.2", "uuid": "^8.3.2",
"waitpid": "git+https://github.com/HexF/node-waitpid.git" "waitpid": "git+https://github.com/HexF/node-waitpid.git"
}, },
"license": "MIT", "license": "MIT",
"scripts": { "scripts": {
"lint": "prettier . --write" "lint": "prettier . --write"
}, },
"devDependencies": { "devDependencies": {
"prettier": "2.2.1" "prettier": "2.2.1"
} }
} }

346
api/src/api/v2.js
View File

@ -8,217 +8,217 @@ const package = require('../package');
const logger = require('logplease').create('api/v2'); const logger = require('logplease').create('api/v2');
router.use((req, res, next) => { router.use((req, res, next) => {
if (['GET', 'HEAD', 'OPTIONS'].includes(req.method)) { if (['GET', 'HEAD', 'OPTIONS'].includes(req.method)) {
return next(); return next();
} }
if (!req.headers['content-type'].startsWith('application/json')) { if (!req.headers['content-type'].startsWith('application/json')) {
return res.status(415).send({ return res.status(415).send({
message: 'requests must be of type application/json', message: 'requests must be of type application/json',
}); });
} }
next(); next();
}); });
router.post('/execute', async (req, res) => { router.post('/execute', async (req, res) => {
const { const {
language, language,
version, version,
files, files,
stdin, stdin,
args, args,
run_timeout, run_timeout,
compile_timeout, compile_timeout,
compile_memory_limit, compile_memory_limit,
run_memory_limit, run_memory_limit,
} = req.body; } = req.body;
if (!language || typeof language !== 'string') { if (!language || typeof language !== 'string') {
return res.status(400).send({ return res.status(400).send({
message: 'language is required as a string', message: 'language is required as a string',
});
}
if (!version || typeof version !== 'string') {
return res.status(400).send({
message: 'version is required as a string',
});
}
if (!files || !Array.isArray(files)) {
return res.status(400).send({
message: 'files is required as an array',
});
}
for (const [i, file] of files.entries()) {
if (typeof file.content !== 'string') {
return res.status(400).send({
message: `files[${i}].content is required as a string`,
});
}
}
if (compile_memory_limit) {
if (typeof compile_memory_limit !== 'number') {
return res.status(400).send({
message: 'if specified, compile_memory_limit must be a number',
});
}
if (
config.compile_memory_limit >= 0 &&
(compile_memory_limit > config.compile_memory_limit ||
compile_memory_limit < 0)
) {
return res.status(400).send({
message:
'compile_memory_limit cannot exceed the configured limit of ' +
config.compile_memory_limit,
});
}
}
if (run_memory_limit) {
if (typeof run_memory_limit !== 'number') {
return res.status(400).send({
message: 'if specified, run_memory_limit must be a number',
});
}
if (
config.run_memory_limit >= 0 &&
(run_memory_limit > config.run_memory_limit || run_memory_limit < 0)
) {
return res.status(400).send({
message:
'run_memory_limit cannot exceed the configured limit of ' +
config.run_memory_limit,
});
}
}
const rt = runtime.get_latest_runtime_matching_language_version(
language,
version
);
if (rt === undefined) {
return res.status(400).send({
message: `${language}-${version} runtime is unknown`,
});
}
const job = new Job({
runtime: rt,
alias: language,
files: files,
args: args || [],
stdin: stdin || '',
timeouts: {
run: run_timeout || 3000,
compile: compile_timeout || 10000,
},
memory_limits: {
run: run_memory_limit || config.run_memory_limit,
compile: compile_memory_limit || config.compile_memory_limit,
},
}); });
}
if (!version || typeof version !== 'string') { await job.prime();
return res.status(400).send({
message: 'version is required as a string',
});
}
if (!files || !Array.isArray(files)) { const result = await job.execute();
return res.status(400).send({
message: 'files is required as an array',
});
}
for (const [i, file] of files.entries()) { await job.cleanup();
if (typeof file.content !== 'string') {
return res.status(400).send({
message: `files[${i}].content is required as a string`,
});
}
}
if (compile_memory_limit) { return res.status(200).send(result);
if (typeof compile_memory_limit !== 'number') {
return res.status(400).send({
message: 'if specified, compile_memory_limit must be a number',
});
}
if (
config.compile_memory_limit >= 0 &&
(compile_memory_limit > config.compile_memory_limit ||
compile_memory_limit < 0)
) {
return res.status(400).send({
message:
'compile_memory_limit cannot exceed the configured limit of ' +
config.compile_memory_limit,
});
}
}
if (run_memory_limit) {
if (typeof run_memory_limit !== 'number') {
return res.status(400).send({
message: 'if specified, run_memory_limit must be a number',
});
}
if (
config.run_memory_limit >= 0 &&
(run_memory_limit > config.run_memory_limit || run_memory_limit < 0)
) {
return res.status(400).send({
message:
'run_memory_limit cannot exceed the configured limit of ' +
config.run_memory_limit,
});
}
}
const rt = runtime.get_latest_runtime_matching_language_version(
language,
version
);
if (rt === undefined) {
return res.status(400).send({
message: `${language}-${version} runtime is unknown`,
});
}
const job = new Job({
runtime: rt,
alias: language,
files: files,
args: args || [],
stdin: stdin || '',
timeouts: {
run: run_timeout || 3000,
compile: compile_timeout || 10000,
},
memory_limits: {
run: run_memory_limit || config.run_memory_limit,
compile: compile_memory_limit || config.compile_memory_limit,
},
});
await job.prime();
const result = await job.execute();
await job.cleanup();
return res.status(200).send(result);
}); });
router.get('/runtimes', (req, res) => { router.get('/runtimes', (req, res) => {
const runtimes = runtime.map((rt) => { const runtimes = runtime.map(rt => {
return { return {
language: rt.language, language: rt.language,
version: rt.version.raw, version: rt.version.raw,
aliases: rt.aliases, aliases: rt.aliases,
runtime: rt.runtime, runtime: rt.runtime,
}; };
}); });
return res.status(200).send(runtimes); return res.status(200).send(runtimes);
}); });
router.get('/packages', async (req, res) => { router.get('/packages', async (req, res) => {
logger.debug('Request to list packages'); logger.debug('Request to list packages');
let packages = await package.get_package_list(); let packages = await package.get_package_list();
packages = packages.map((pkg) => { packages = packages.map(pkg => {
return { return {
language: pkg.language, language: pkg.language,
language_version: pkg.version.raw, language_version: pkg.version.raw,
installed: pkg.installed, installed: pkg.installed,
}; };
}); });
return res.status(200).send(packages); return res.status(200).send(packages);
}); });
router.post('/packages/:language/:version', async (req, res) => { router.post('/packages/:language/:version', async (req, res) => {
logger.debug('Request to install package'); logger.debug('Request to install package');
const { language, version } = req.params; const { language, version } = req.params;
const pkg = await package.get_package(language, version); const pkg = await package.get_package(language, version);
if (pkg == null) { if (pkg == null) {
return res.status(404).send({ return res.status(404).send({
message: `Requested package ${language}-${version} does not exist`, message: `Requested package ${language}-${version} does not exist`,
}); });
} }
try { try {
const response = await pkg.install(); const response = await pkg.install();
return res.status(200).send(response); return res.status(200).send(response);
} catch (e) { } catch (e) {
logger.error( logger.error(
`Error while installing package ${pkg.language}-${pkg.version}:`, `Error while installing package ${pkg.language}-${pkg.version}:`,
e.message e.message
); );
return res.status(500).send({ return res.status(500).send({
message: e.message, message: e.message,
}); });
} }
}); });
router.delete('/packages/:language/:version', async (req, res) => { router.delete('/packages/:language/:version', async (req, res) => {
logger.debug('Request to uninstall package'); logger.debug('Request to uninstall package');
const { language, version } = req.params; const { language, version } = req.params;
const pkg = await package.get_package(language, version); const pkg = await package.get_package(language, version);
if (pkg == null) { if (pkg == null) {
return res.status(404).send({ return res.status(404).send({
message: `Requested package ${language}-${version} does not exist`, message: `Requested package ${language}-${version} does not exist`,
}); });
} }
try { try {
const response = await pkg.uninstall(); const response = await pkg.uninstall();
return res.status(200).send(response); return res.status(200).send(response);
} catch (e) { } catch (e) {
logger.error( logger.error(
`Error while uninstalling package ${pkg.language}-${pkg.version}:`, `Error while uninstalling package ${pkg.language}-${pkg.version}:`,
e.message e.message
); );
return res.status(500).send({ return res.status(500).send({
message: e.message, message: e.message,
}); });
} }
}); });
module.exports = router; module.exports = router;

258
api/src/config.js
View File

@ -3,114 +3,117 @@ const Logger = require('logplease');
const logger = Logger.create('config'); const logger = Logger.create('config');
const options = [ const options = [
{ {
key: 'log_level', key: 'log_level',
desc: 'Level of data to log', desc: 'Level of data to log',
default: 'INFO', default: 'INFO',
options: Object.values(Logger.LogLevels), options: Object.values(Logger.LogLevels),
validators: [ validators: [
(x) => x =>
Object.values(Logger.LogLevels).includes(x) || Object.values(Logger.LogLevels).includes(x) ||
`Log level ${x} does not exist`, `Log level ${x} does not exist`,
], ],
}, },
{ {
key: 'bind_address', key: 'bind_address',
desc: 'Address to bind REST API on\nThank @Bones for the number', desc: 'Address to bind REST API on\nThank @Bones for the number',
default: '0.0.0.0:2000', default: '0.0.0.0:2000',
validators: [], validators: [],
}, },
{ {
key: 'data_directory', key: 'data_directory',
desc: 'Absolute path to store all piston related data at', desc: 'Absolute path to store all piston related data at',
default: '/piston', default: '/piston',
validators: [(x) => fss.exists_sync(x) || `Directory ${x} does not exist`], validators: [
}, x => fss.exists_sync(x) || `Directory ${x} does not exist`,
{ ],
key: 'runner_uid_min', },
desc: 'Minimum uid to use for runner', {
default: 1001, key: 'runner_uid_min',
parser: parse_int, desc: 'Minimum uid to use for runner',
validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`], default: 1001,
}, parser: parse_int,
{ validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
key: 'runner_uid_max', },
desc: 'Maximum uid to use for runner', {
default: 1500, key: 'runner_uid_max',
parser: parse_int, desc: 'Maximum uid to use for runner',
validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`], default: 1500,
}, parser: parse_int,
{ validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
key: 'runner_gid_min', },
desc: 'Minimum gid to use for runner', {
default: 1001, key: 'runner_gid_min',
parser: parse_int, desc: 'Minimum gid to use for runner',
validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`], default: 1001,
}, parser: parse_int,
{ validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
key: 'runner_gid_max', },
desc: 'Maximum gid to use for runner', {
default: 1500, key: 'runner_gid_max',
parser: parse_int, desc: 'Maximum gid to use for runner',
validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`], default: 1500,
}, parser: parse_int,
{ validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
key: 'disable_networking', },
desc: 'Set to true to disable networking', {
default: true, key: 'disable_networking',
parser: (x) => x === 'true', desc: 'Set to true to disable networking',
validators: [(x) => typeof x === 'boolean' || `${x} is not a boolean`], default: true,
}, parser: x => x === 'true',
{ validators: [x => typeof x === 'boolean' || `${x} is not a boolean`],
key: 'output_max_size', },
desc: 'Max size of each stdio buffer', {
default: 1024, key: 'output_max_size',
parser: parse_int, desc: 'Max size of each stdio buffer',
validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`], default: 1024,
}, parser: parse_int,
{ validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
key: 'max_process_count', },
desc: 'Max number of processes per job', {
default: 64, key: 'max_process_count',
parser: parse_int, desc: 'Max number of processes per job',
validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`], default: 64,
}, parser: parse_int,
{ validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
key: 'max_open_files', },
desc: 'Max number of open files per job', {
default: 2048, key: 'max_open_files',
parser: parse_int, desc: 'Max number of open files per job',
validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`], default: 2048,
}, parser: parse_int,
{ validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
key: 'max_file_size', },
desc: 'Max file size in bytes for a file', {
default: 10000000, //10MB key: 'max_file_size',
parser: parse_int, desc: 'Max file size in bytes for a file',
validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`], default: 10000000, //10MB
}, parser: parse_int,
{ validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
key: 'compile_memory_limit', },
desc: {
'Max memory usage for compile stage in bytes (set to -1 for no limit)', key: 'compile_memory_limit',
default: -1, // no limit desc:
parser: parse_int, 'Max memory usage for compile stage in bytes (set to -1 for no limit)',
validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`], default: -1, // no limit
}, parser: parse_int,
{ validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
key: 'run_memory_limit', },
desc: 'Max memory usage for run stage in bytes (set to -1 for no limit)', {
default: -1, // no limit key: 'run_memory_limit',
parser: parse_int, desc:
validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`], 'Max memory usage for run stage in bytes (set to -1 for no limit)',
}, default: -1, // no limit
{ parser: parse_int,
key: 'repo_url', validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
desc: 'URL of repo index', },
default: {
'https://github.com/engineer-man/piston/releases/download/pkgs/index', key: 'repo_url',
validators: [], desc: 'URL of repo index',
}, default:
'https://github.com/engineer-man/piston/releases/download/pkgs/index',
validators: [],
},
]; ];
logger.info(`Loading Configuration from environment`); logger.info(`Loading Configuration from environment`);
@ -119,34 +122,37 @@ let errored = false;
let config = {}; let config = {};
options.forEach((option) => { options.forEach(option => {
const env_key = 'PISTON_' + option.key.to_upper_case(); const env_key = 'PISTON_' + option.key.to_upper_case();
const parser = option.parser || ((x) => x); const parser = option.parser || (x => x);
const env_val = process.env[env_key]; const env_val = process.env[env_key];
const parsed_val = parser(env_val); const parsed_val = parser(env_val);
const value = env_val || option.default; const value = env_val || option.default;
option.validators.for_each((validator) => { option.validators.for_each(validator => {
let response = null; let response = null;
if (env_val) response = validator(parsed_val, env_val); if (env_val) response = validator(parsed_val, env_val);
else response = validator(value, value); else response = validator(value, value);
if (response !== true) { if (response !== true) {
errored = true; errored = true;
logger.error(`Config option ${option.key} failed validation:`, response); logger.error(
return; `Config option ${option.key} failed validation:`,
} response
}); );
return;
}
});
config[option.key] = value; config[option.key] = value;
}); });
if (errored) { if (errored) {
process.exit(1); process.exit(1);
} }
logger.info('Configuration successfully loaded'); logger.info('Configuration successfully loaded');

26
api/src/globals.js
View File

@ -2,19 +2,19 @@
const is_docker = require('is-docker'); const is_docker = require('is-docker');
const fs = require('fs'); const fs = require('fs');
const platform = `${is_docker() ? 'docker' : 'baremetal'}-${fs const platform = `${is_docker() ? 'docker' : 'baremetal'}-${fs
.read_file_sync('/etc/os-release') .read_file_sync('/etc/os-release')
.toString() .toString()
.split('\n') .split('\n')
.find((x) => x.startsWith('ID')) .find(x => x.startsWith('ID'))
.replace('ID=', '')}`; .replace('ID=', '')}`;
module.exports = { module.exports = {
data_directories: { data_directories: {
packages: 'packages', packages: 'packages',
jobs: 'jobs', jobs: 'jobs',
}, },
version: require('../package.json').version, version: require('../package.json').version,
platform, platform,
pkg_installed_file: '.ppman-installed', //Used as indication for if a package was installed pkg_installed_file: '.ppman-installed', //Used as indication for if a package was installed
clean_directories: ['/dev/shm', '/run/lock', '/tmp', '/var/tmp'], clean_directories: ['/dev/shm', '/run/lock', '/tmp', '/var/tmp'],
}; };

114
api/src/index.js
View File

@ -14,77 +14,77 @@ const logger = Logger.create('index');
const app = express(); const app = express();
(async () => { (async () => {
logger.info('Setting loglevel to', config.log_level); logger.info('Setting loglevel to', config.log_level);
Logger.setLogLevel(config.log_level); Logger.setLogLevel(config.log_level);
logger.debug('Ensuring data directories exist'); logger.debug('Ensuring data directories exist');
Object.values(globals.data_directories).for_each((dir) => { Object.values(globals.data_directories).for_each(dir => {
let data_path = path.join(config.data_directory, dir); let data_path = path.join(config.data_directory, dir);
logger.debug(`Ensuring ${data_path} exists`); logger.debug(`Ensuring ${data_path} exists`);
if (!fss.exists_sync(data_path)) { if (!fss.exists_sync(data_path)) {
logger.info(`${data_path} does not exist.. Creating..`); logger.info(`${data_path} does not exist.. Creating..`);
try { try {
fss.mkdir_sync(data_path); fss.mkdir_sync(data_path);
} catch (e) { } catch (e) {
logger.error(`Failed to create ${data_path}: `, e.message); logger.error(`Failed to create ${data_path}: `, e.message);
} }
} }
}); });
logger.info('Loading packages'); logger.info('Loading packages');
const pkgdir = path.join( const pkgdir = path.join(
config.data_directory, config.data_directory,
globals.data_directories.packages globals.data_directories.packages
);
const pkglist = await fs.readdir(pkgdir);
const languages = await Promise.all(
pkglist.map((lang) => {
return fs.readdir(path.join(pkgdir, lang)).then((x) => {
return x.map((y) => path.join(pkgdir, lang, y));
});
})
);
const installed_languages = languages
.flat()
.filter((pkg) =>
fss.exists_sync(path.join(pkg, globals.pkg_installed_file))
); );
installed_languages.for_each((pkg) => runtime.load_package(pkg)); const pkglist = await fs.readdir(pkgdir);
logger.info('Starting API Server'); const languages = await Promise.all(
logger.debug('Constructing Express App'); pkglist.map(lang => {
logger.debug('Registering middleware'); return fs.readdir(path.join(pkgdir, lang)).then(x => {
return x.map(y => path.join(pkgdir, lang, y));
});
})
);
app.use(body_parser.urlencoded({ extended: true })); const installed_languages = languages
app.use(body_parser.json()); .flat()
.filter(pkg =>
fss.exists_sync(path.join(pkg, globals.pkg_installed_file))
);
app.use((err, req, res, next) => { installed_languages.for_each(pkg => runtime.load_package(pkg));
return res.status(400).send({
stack: err.stack, logger.info('Starting API Server');
logger.debug('Constructing Express App');
logger.debug('Registering middleware');
app.use(body_parser.urlencoded({ extended: true }));
app.use(body_parser.json());
app.use((err, req, res, next) => {
return res.status(400).send({
stack: err.stack,
});
}); });
});
logger.debug('Registering Routes'); logger.debug('Registering Routes');
const api_v2 = require('./api/v2'); const api_v2 = require('./api/v2');
app.use('/api/v2', api_v2); app.use('/api/v2', api_v2);
app.use('/api/v2', api_v2); app.use('/api/v2', api_v2);
app.use((req, res, next) => { app.use((req, res, next) => {
return res.status(404).send({ message: 'Not Found' }); return res.status(404).send({ message: 'Not Found' });
}); });
logger.debug('Calling app.listen'); logger.debug('Calling app.listen');
const [address, port] = config.bind_address.split(':'); const [address, port] = config.bind_address.split(':');
app.listen(port, address, () => { app.listen(port, address, () => {
logger.info('API server started on', config.bind_address); logger.info('API server started on', config.bind_address);
}); });
})(); })();

448
api/src/job.js
View File

@ -8,268 +8,278 @@ const fs = require('fs/promises');
const wait_pid = require('waitpid'); const wait_pid = require('waitpid');
const job_states = { const job_states = {
READY: Symbol('Ready to be primed'), READY: Symbol('Ready to be primed'),
PRIMED: Symbol('Primed and ready for execution'), PRIMED: Symbol('Primed and ready for execution'),
EXECUTED: Symbol('Executed and ready for cleanup'), EXECUTED: Symbol('Executed and ready for cleanup'),
}; };
let uid = 0; let uid = 0;
let gid = 0; let gid = 0;
class Job { class Job {
constructor({ runtime, files, args, stdin, timeouts, memory_limits }) { constructor({ runtime, files, args, stdin, timeouts, memory_limits }) {
this.uuid = uuidv4(); this.uuid = uuidv4();
this.runtime = runtime; this.runtime = runtime;
this.files = files.map((file, i) => ({ this.files = files.map((file, i) => ({
name: file.name || `file${i}.code`, name: file.name || `file${i}.code`,
content: file.content, content: file.content,
})); }));
this.args = args; this.args = args;
this.stdin = stdin; this.stdin = stdin;
this.timeouts = timeouts; this.timeouts = timeouts;
this.memory_limits = memory_limits; this.memory_limits = memory_limits;
this.uid = config.runner_uid_min + uid; this.uid = config.runner_uid_min + uid;
this.gid = config.runner_gid_min + gid; this.gid = config.runner_gid_min + gid;
uid++; uid++;
gid++; gid++;
uid %= config.runner_uid_max - config.runner_uid_min + 1; uid %= config.runner_uid_max - config.runner_uid_min + 1;
gid %= config.runner_gid_max - config.runner_gid_min + 1; gid %= config.runner_gid_max - config.runner_gid_min + 1;
this.state = job_states.READY; this.state = job_states.READY;
this.dir = path.join( this.dir = path.join(
config.data_directory, config.data_directory,
globals.data_directories.jobs, globals.data_directories.jobs,
this.uuid this.uuid
); );
}
async prime() {
logger.info(`Priming job uuid=${this.uuid}`);
logger.debug('Writing files to job cache');
logger.debug(`Transfering ownership uid=${this.uid} gid=${this.gid}`);
await fs.mkdir(this.dir, { mode: 0o700 });
await fs.chown(this.dir, this.uid, this.gid);
for (const file of this.files) {
let file_path = path.join(this.dir, file.name);
await fs.write_file(file_path, file.content);
await fs.chown(file_path, this.uid, this.gid);
} }
this.state = job_states.PRIMED; async prime() {
logger.info(`Priming job uuid=${this.uuid}`);
logger.debug('Primed job'); logger.debug('Writing files to job cache');
}
async safe_call(file, args, timeout, memory_limit) { logger.debug(`Transfering ownership uid=${this.uid} gid=${this.gid}`);
return new Promise((resolve, reject) => {
const nonetwork = config.disable_networking ? ['nosocket'] : [];
const prlimit = [ await fs.mkdir(this.dir, { mode: 0o700 });
'prlimit', await fs.chown(this.dir, this.uid, this.gid);
'--nproc=' + config.max_process_count,
'--nofile=' + config.max_open_files,
'--fsize=' + config.max_file_size,
];
if (memory_limit >= 0) { for (const file of this.files) {
prlimit.push('--as=' + memory_limit); let file_path = path.join(this.dir, file.name);
}
const proc_call = [...prlimit, ...nonetwork, 'bash', file, ...args]; await fs.write_file(file_path, file.content);
await fs.chown(file_path, this.uid, this.gid);
var stdout = '';
var stderr = '';
var output = '';
const proc = cp.spawn(proc_call[0], proc_call.splice(1), {
env: {
...this.runtime.env_vars,
PISTON_LANGUAGE: this.runtime.language,
},
stdio: 'pipe',
cwd: this.dir,
uid: this.uid,
gid: this.gid,
detached: true, //give this process its own process group
});
proc.stdin.write(this.stdin);
proc.stdin.end();
proc.stdin.destroy();
const kill_timeout = set_timeout((_) => proc.kill('SIGKILL'), timeout);
proc.stderr.on('data', (data) => {
if (stderr.length > config.output_max_size) {
proc.kill('SIGKILL');
} else {
stderr += data;
output += data;
} }
});
proc.stdout.on('data', (data) => { this.state = job_states.PRIMED;
if (stdout.length > config.output_max_size) {
proc.kill('SIGKILL');
} else {
stdout += data;
output += data;
}
});
const exit_cleanup = () => { logger.debug('Primed job');
clear_timeout(kill_timeout);
proc.stderr.destroy();
proc.stdout.destroy();
};
proc.on('exit', (code, signal) => {
exit_cleanup();
resolve({ stdout, stderr, code, signal, output });
});
proc.on('error', (err) => {
exit_cleanup();
reject({ error: err, stdout, stderr, output });
});
});
}
async execute() {
if (this.state !== job_states.PRIMED) {
throw new Error(
'Job must be in primed state, current state: ' + this.state.toString()
);
} }
logger.info( async safe_call(file, args, timeout, memory_limit) {
`Executing job uuid=${this.uuid} uid=${this.uid} gid=${ return new Promise((resolve, reject) => {
this.gid const nonetwork = config.disable_networking ? ['nosocket'] : [];
} runtime=${this.runtime.toString()}`
);
logger.debug('Compiling'); const prlimit = [
'prlimit',
'--nproc=' + config.max_process_count,
'--nofile=' + config.max_open_files,
'--fsize=' + config.max_file_size,
];
let compile; if (memory_limit >= 0) {
prlimit.push('--as=' + memory_limit);
}
if (this.runtime.compiled) { const proc_call = [...prlimit, ...nonetwork, 'bash', file, ...args];
compile = await this.safe_call(
path.join(this.runtime.pkgdir, 'compile'),
this.files.map((x) => x.name),
this.timeouts.compile,
this.memory_limits.compile
);
}
logger.debug('Running'); var stdout = '';
var stderr = '';
var output = '';
const run = await this.safe_call( const proc = cp.spawn(proc_call[0], proc_call.splice(1), {
path.join(this.runtime.pkgdir, 'run'), env: {
[this.files[0].name, ...this.args], ...this.runtime.env_vars,
this.timeouts.run, PISTON_LANGUAGE: this.runtime.language,
this.memory_limits.run },
); stdio: 'pipe',
cwd: this.dir,
this.state = job_states.EXECUTED; uid: this.uid,
gid: this.gid,
return { detached: true, //give this process its own process group
compile,
run,
language: this.runtime.language,
version: this.runtime.version.raw,
};
}
async cleanup_processes() {
let processes = [1];
while (processes.length > 0) {
processes = await new Promise((resolve, reject) =>
cp.execFile('ps', ['awwxo', 'pid,ruid'], (err, stdout) => {
if (err === null) {
const lines = stdout.split('\n').slice(1); //Remove header with slice
const procs = lines.map((line) => {
const [pid, ruid] = line
.trim()
.split(/\s+/)
.map((n) => parseInt(n));
return { pid, ruid };
}); });
resolve(procs); proc.stdin.write(this.stdin);
} else { proc.stdin.end();
reject(error); proc.stdin.destroy();
}
})
);
processes = processes.filter((proc) => proc.ruid === this.uid); const kill_timeout = set_timeout(
_ => proc.kill('SIGKILL'),
timeout
);
for (const proc of processes) { proc.stderr.on('data', data => {
// First stop the processes, but keep their resources allocated so they cant re-fork if (stderr.length > config.output_max_size) {
try { proc.kill('SIGKILL');
process.kill(proc.pid, 'SIGSTOP'); } else {
} catch { stderr += data;
// Could already be dead output += data;
} }
} });
for (const proc of processes) { proc.stdout.on('data', data => {
// Then clear them out of the process tree if (stdout.length > config.output_max_size) {
try { proc.kill('SIGKILL');
process.kill(proc.pid, 'SIGKILL'); } else {
} catch { stdout += data;
// Could already be dead and just needs to be waited on output += data;
} }
});
wait_pid(proc.pid); const exit_cleanup = () => {
} clear_timeout(kill_timeout);
}
}
async cleanup_filesystem() { proc.stderr.destroy();
for (const clean_path of globals.clean_directories) { proc.stdout.destroy();
const contents = await fs.readdir(clean_path); };
for (const file of contents) { proc.on('exit', (code, signal) => {
const file_path = path.join(clean_path, file); exit_cleanup();
try { resolve({ stdout, stderr, code, signal, output });
const stat = await fs.stat(file_path); });
if (stat.uid === this.uid) { proc.on('error', err => {
await fs.rm(file_path, { recursive: true, force: true }); exit_cleanup();
}
} catch (e) { reject({ error: err, stdout, stderr, output });
// File was somehow deleted in the time that we read the dir to when we checked the file });
logger.warn(`Error removing file ${file_path}: ${e}`); });
}
}
} }
await fs.rm(this.dir, { recursive: true, force: true }); async execute() {
} if (this.state !== job_states.PRIMED) {
throw new Error(
'Job must be in primed state, current state: ' +
this.state.toString()
);
}
async cleanup() { logger.info(
logger.info(`Cleaning up job uuid=${this.uuid}`); `Executing job uuid=${this.uuid} uid=${this.uid} gid=${
this.gid
} runtime=${this.runtime.toString()}`
);
await Promise.all([this.cleanup_processes(), this.cleanup_filesystem()]); logger.debug('Compiling');
}
let compile;
if (this.runtime.compiled) {
compile = await this.safe_call(
path.join(this.runtime.pkgdir, 'compile'),
this.files.map(x => x.name),
this.timeouts.compile,
this.memory_limits.compile
);
}
logger.debug('Running');
const run = await this.safe_call(
path.join(this.runtime.pkgdir, 'run'),
[this.files[0].name, ...this.args],
this.timeouts.run,
this.memory_limits.run
);
this.state = job_states.EXECUTED;
return {
compile,
run,
language: this.runtime.language,
version: this.runtime.version.raw,
};
}
async cleanup_processes() {
let processes = [1];
while (processes.length > 0) {
processes = await new Promise((resolve, reject) =>
cp.execFile('ps', ['awwxo', 'pid,ruid'], (err, stdout) => {
if (err === null) {
const lines = stdout.split('\n').slice(1); //Remove header with slice
const procs = lines.map(line => {
const [pid, ruid] = line
.trim()
.split(/\s+/)
.map(n => parseInt(n));
return { pid, ruid };
});
resolve(procs);
} else {
reject(error);
}
})
);
processes = processes.filter(proc => proc.ruid === this.uid);
for (const proc of processes) {
// First stop the processes, but keep their resources allocated so they cant re-fork
try {
process.kill(proc.pid, 'SIGSTOP');
} catch {
// Could already be dead
}
}
for (const proc of processes) {
// Then clear them out of the process tree
try {
process.kill(proc.pid, 'SIGKILL');
} catch {
// Could already be dead and just needs to be waited on
}
wait_pid(proc.pid);
}
}
}
async cleanup_filesystem() {
for (const clean_path of globals.clean_directories) {
const contents = await fs.readdir(clean_path);
for (const file of contents) {
const file_path = path.join(clean_path, file);
try {
const stat = await fs.stat(file_path);
if (stat.uid === this.uid) {
await fs.rm(file_path, {
recursive: true,
force: true,
});
}
} catch (e) {
// File was somehow deleted in the time that we read the dir to when we checked the file
logger.warn(`Error removing file ${file_path}: ${e}`);
}
}
}
await fs.rm(this.dir, { recursive: true, force: true });
}
async cleanup() {
logger.info(`Cleaning up job uuid=${this.uuid}`);
await Promise.all([
this.cleanup_processes(),
this.cleanup_filesystem(),
]);
}
} }
module.exports = { module.exports = {
Job, Job,
}; };

315
api/src/package.js
View File

@ -13,197 +13,208 @@ const chownr = require('chownr');
const util = require('util'); const util = require('util');
class Package { class Package {
constructor({ language, version, download, checksum }) { constructor({ language, version, download, checksum }) {
this.language = language; this.language = language;
this.version = semver.parse(version); this.version = semver.parse(version);
this.checksum = checksum; this.checksum = checksum;
this.download = download; this.download = download;
}
get installed() {
return fss.exists_sync(
path.join(this.install_path, globals.pkg_installed_file)
);
}
get install_path() {
return path.join(
config.data_directory,
globals.data_directories.packages,
this.language,
this.version.raw
);
}
async install() {
if (this.installed) {
throw new Error('Already installed');
} }
logger.info(`Installing ${this.language}-${this.version.raw}`); get installed() {
return fss.exists_sync(
if (fss.exists_sync(this.install_path)) { path.join(this.install_path, globals.pkg_installed_file)
logger.warn( );
`${this.language}-${this.version.raw} has residual files. Removing them.`
);
await fs.rm(this.install_path, { recursive: true, force: true });
} }
logger.debug(`Making directory ${this.install_path}`); get install_path() {
await fs.mkdir(this.install_path, { recursive: true }); return path.join(
config.data_directory,
logger.debug( globals.data_directories.packages,
`Downloading package from ${this.download} in to ${this.install_path}` this.language,
); this.version.raw
const pkgpath = path.join(this.install_path, 'pkg.tar.gz'); );
const download = await fetch(this.download);
const file_stream = fss.create_write_stream(pkgpath);
await new Promise((resolve, reject) => {
download.body.pipe(file_stream);
download.body.on('error', reject);
file_stream.on('finish', resolve);
});
logger.debug('Validating checksums');
logger.debug(`Assert sha256(pkg.tar.gz) == ${this.checksum}`);
const cs = crypto
.create_hash('sha256')
.update(fss.readFileSync(pkgpath))
.digest('hex');
if (cs !== this.checksum) {
throw new Error(`Checksum miss-match want: ${val} got: ${cs}`);
} }
logger.debug( async install() {
`Extracting package files from archive ${pkgpath} in to ${this.install_path}` if (this.installed) {
); throw new Error('Already installed');
}
await new Promise((resolve, reject) => { logger.info(`Installing ${this.language}-${this.version.raw}`);
const proc = cp.exec(
`bash -c 'cd "${this.install_path}" && tar xzf ${pkgpath}'`
);
proc.once('exit', (code, _) => { if (fss.exists_sync(this.install_path)) {
code === 0 ? resolve() : reject(); logger.warn(
}); `${this.language}-${this.version.raw} has residual files. Removing them.`
);
await fs.rm(this.install_path, { recursive: true, force: true });
}
proc.stdout.pipe(process.stdout); logger.debug(`Making directory ${this.install_path}`);
proc.stderr.pipe(process.stderr); await fs.mkdir(this.install_path, { recursive: true });
proc.once('error', reject); logger.debug(
}); `Downloading package from ${this.download} in to ${this.install_path}`
);
const pkgpath = path.join(this.install_path, 'pkg.tar.gz');
const download = await fetch(this.download);
logger.debug('Registering runtime'); const file_stream = fss.create_write_stream(pkgpath);
runtime.load_package(this.install_path); await new Promise((resolve, reject) => {
download.body.pipe(file_stream);
download.body.on('error', reject);
logger.debug('Caching environment'); file_stream.on('finish', resolve);
const get_env_command = `cd ${this.install_path}; source environment; env`; });
const envout = await new Promise((resolve, reject) => { logger.debug('Validating checksums');
let stdout = ''; logger.debug(`Assert sha256(pkg.tar.gz) == ${this.checksum}`);
const cs = crypto
.create_hash('sha256')
.update(fss.readFileSync(pkgpath))
.digest('hex');
const proc = cp.spawn('env', ['-i', 'bash', '-c', `${get_env_command}`], { if (cs !== this.checksum) {
stdio: ['ignore', 'pipe', 'pipe'], throw new Error(`Checksum miss-match want: ${val} got: ${cs}`);
}); }
proc.once('exit', (code, _) => { logger.debug(
code === 0 ? resolve(stdout) : reject(); `Extracting package files from archive ${pkgpath} in to ${this.install_path}`
}); );
proc.stdout.on('data', (data) => { await new Promise((resolve, reject) => {
stdout += data; const proc = cp.exec(
}); `bash -c 'cd "${this.install_path}" && tar xzf ${pkgpath}'`
);
proc.once('error', reject); proc.once('exit', (code, _) => {
}); code === 0 ? resolve() : reject();
});
const filtered_env = envout proc.stdout.pipe(process.stdout);
.split('\n') proc.stderr.pipe(process.stderr);
.filter(
(l) => !['PWD', 'OLDPWD', '_', 'SHLVL'].includes(l.split('=', 2)[0])
)
.join('\n');
await fs.write_file(path.join(this.install_path, '.env'), filtered_env); proc.once('error', reject);
});
logger.debug('Changing Ownership of package directory'); logger.debug('Registering runtime');
await util.promisify(chownr)(this.install_path, 0, 0); runtime.load_package(this.install_path);
logger.debug('Writing installed state to disk'); logger.debug('Caching environment');
await fs.write_file( const get_env_command = `cd ${this.install_path}; source environment; env`;
path.join(this.install_path, globals.pkg_installed_file),
Date.now().toString()
);
logger.info(`Installed ${this.language}-${this.version.raw}`); const envout = await new Promise((resolve, reject) => {
let stdout = '';
return { const proc = cp.spawn(
language: this.language, 'env',
version: this.version.raw, ['-i', 'bash', '-c', `${get_env_command}`],
}; {
} stdio: ['ignore', 'pipe', 'pipe'],
}
);
async uninstall() { proc.once('exit', (code, _) => {
logger.info(`Uninstalling ${this.language}-${this.version.raw}`); code === 0 ? resolve(stdout) : reject();
});
logger.debug('Finding runtime'); proc.stdout.on('data', data => {
const found_runtime = runtime.get_runtime_by_name_and_version( stdout += data;
this.language, });
this.version.raw
);
if (!found_runtime) { proc.once('error', reject);
logger.error( });
`Uninstalling ${this.language}-${this.version.raw} failed: Not installed`
); const filtered_env = envout
throw new Error(`${this.language}-${this.version.raw} is not installed`); .split('\n')
.filter(
l =>
!['PWD', 'OLDPWD', '_', 'SHLVL'].includes(
l.split('=', 2)[0]
)
)
.join('\n');
await fs.write_file(path.join(this.install_path, '.env'), filtered_env);
logger.debug('Changing Ownership of package directory');
await util.promisify(chownr)(this.install_path, 0, 0);
logger.debug('Writing installed state to disk');
await fs.write_file(
path.join(this.install_path, globals.pkg_installed_file),
Date.now().toString()
);
logger.info(`Installed ${this.language}-${this.version.raw}`);
return {
language: this.language,
version: this.version.raw,
};
} }
logger.debug('Unregistering runtime'); async uninstall() {
found_runtime.unregister(); logger.info(`Uninstalling ${this.language}-${this.version.raw}`);
logger.debug('Cleaning files from disk'); logger.debug('Finding runtime');
await fs.rmdir(this.install_path, { recursive: true }); const found_runtime = runtime.get_runtime_by_name_and_version(
this.language,
this.version.raw
);
logger.info(`Uninstalled ${this.language}-${this.version.raw}`); if (!found_runtime) {
logger.error(
`Uninstalling ${this.language}-${this.version.raw} failed: Not installed`
);
throw new Error(
`${this.language}-${this.version.raw} is not installed`
);
}
return { logger.debug('Unregistering runtime');
language: this.language, found_runtime.unregister();
version: this.version.raw,
};
}
static async get_package_list() { logger.debug('Cleaning files from disk');
const repo_content = await fetch(config.repo_url).then((x) => x.text()); await fs.rmdir(this.install_path, { recursive: true });
const entries = repo_content.split('\n').filter((x) => x.length > 0); logger.info(`Uninstalled ${this.language}-${this.version.raw}`);
return entries.map((line) => { return {
const [language, version, checksum, download] = line.split(',', 4); language: this.language,
version: this.version.raw,
};
}
return new Package({ static async get_package_list() {
language, const repo_content = await fetch(config.repo_url).then(x => x.text());
version,
checksum,
download,
});
});
}
static async get_package(lang, version) { const entries = repo_content.split('\n').filter(x => x.length > 0);
const packages = await Package.get_package_list();
const candidates = packages.filter((pkg) => { return entries.map(line => {
return pkg.language == lang && semver.satisfies(pkg.version, version); const [language, version, checksum, download] = line.split(',', 4);
});
candidates.sort((a, b) => semver.rcompare(a.version, b.version)); return new Package({
language,
version,
checksum,
download,
});
});
}
return candidates[0] || null; static async get_package(lang, version) {
} const packages = await Package.get_package_list();
const candidates = packages.filter(pkg => {
return (
pkg.language == lang && semver.satisfies(pkg.version, version)
);
});
candidates.sort((a, b) => semver.rcompare(a.version, b.version));
return candidates[0] || null;
}
} }
module.exports = Package; module.exports = Package;

174
api/src/runtime.js
View File

@ -8,118 +8,118 @@ const path = require('path');
const runtimes = []; const runtimes = [];
class Runtime { class Runtime {
constructor({ language, version, aliases, pkgdir, runtime }) { constructor({ language, version, aliases, pkgdir, runtime }) {
this.language = language; this.language = language;
this.version = version; this.version = version;
this.aliases = aliases || []; this.aliases = aliases || [];
this.pkgdir = pkgdir; this.pkgdir = pkgdir;
this.runtime = runtime; this.runtime = runtime;
}
static load_package(package_dir) {
let info = JSON.parse(
fss.read_file_sync(path.join(package_dir, 'pkg-info.json'))
);
let { language, version, build_platform, aliases, provides } = info;
version = semver.parse(version);
if (build_platform !== globals.platform) {
logger.warn(
`Package ${language}-${version} was built for platform ${build_platform}, ` +
`but our platform is ${globals.platform}`
);
} }
if (provides) { static load_package(package_dir) {
// Multiple languages in 1 package let info = JSON.parse(
provides.forEach((lang) => { fss.read_file_sync(path.join(package_dir, 'pkg-info.json'))
runtimes.push(
new Runtime({
language: lang.language,
aliases: lang.aliases,
version,
pkgdir: package_dir,
runtime: language,
})
); );
});
} else { let { language, version, build_platform, aliases, provides } = info;
runtimes.push( version = semver.parse(version);
new Runtime({
language, if (build_platform !== globals.platform) {
version, logger.warn(
aliases, `Package ${language}-${version} was built for platform ${build_platform}, ` +
pkgdir: package_dir, `but our platform is ${globals.platform}`
}) );
); }
if (provides) {
// Multiple languages in 1 package
provides.forEach(lang => {
runtimes.push(
new Runtime({
language: lang.language,
aliases: lang.aliases,
version,
pkgdir: package_dir,
runtime: language,
})
);
});
} else {
runtimes.push(
new Runtime({
language,
version,
aliases,
pkgdir: package_dir,
})
);
}
logger.debug(`Package ${language}-${version} was loaded`);
} }
logger.debug(`Package ${language}-${version} was loaded`); get compiled() {
} if (this._compiled === undefined) {
this._compiled = fss.exists_sync(path.join(this.pkgdir, 'compile'));
}
get compiled() { return this._compiled;
if (this._compiled === undefined) {
this._compiled = fss.exists_sync(path.join(this.pkgdir, 'compile'));
} }
return this._compiled; get env_vars() {
} if (!this._env_vars) {
const env_file = path.join(this.pkgdir, '.env');
const env_content = fss.read_file_sync(env_file).toString();
get env_vars() { this._env_vars = {};
if (!this._env_vars) {
const env_file = path.join(this.pkgdir, '.env');
const env_content = fss.read_file_sync(env_file).toString();
this._env_vars = {}; env_content
.trim()
.split('\n')
.map(line => line.split('=', 2))
.forEach(([key, val]) => {
this._env_vars[key.trim()] = val.trim();
});
}
env_content return this._env_vars;
.trim()
.split('\n')
.map((line) => line.split('=', 2))
.forEach(([key, val]) => {
this._env_vars[key.trim()] = val.trim();
});
} }
return this._env_vars; toString() {
} return `${this.language}-${this.version.raw}`;
}
toString() { unregister() {
return `${this.language}-${this.version.raw}`; const index = runtimes.indexOf(this);
} runtimes.splice(index, 1); //Remove from runtimes list
}
unregister() {
const index = runtimes.indexOf(this);
runtimes.splice(index, 1); //Remove from runtimes list
}
} }
module.exports = runtimes; module.exports = runtimes;
module.exports.Runtime = Runtime; module.exports.Runtime = Runtime;
module.exports.get_runtimes_matching_language_version = function (lang, ver) { module.exports.get_runtimes_matching_language_version = function (lang, ver) {
return runtimes.filter( return runtimes.filter(
(rt) => rt =>
(rt.language == lang || rt.aliases.includes(lang)) && (rt.language == lang || rt.aliases.includes(lang)) &&
semver.satisfies(rt.version, ver) semver.satisfies(rt.version, ver)
); );
}; };
module.exports.get_latest_runtime_matching_language_version = function ( module.exports.get_latest_runtime_matching_language_version = function (
lang, lang,
ver ver
) { ) {
return module.exports return module.exports
.get_runtimes_matching_language_version(lang, ver) .get_runtimes_matching_language_version(lang, ver)
.sort((a, b) => semver.rcompare(a.version, b.version))[0]; .sort((a, b) => semver.rcompare(a.version, b.version))[0];
}; };
module.exports.get_runtime_by_name_and_version = function (runtime, ver) { module.exports.get_runtime_by_name_and_version = function (runtime, ver) {
return runtimes.find( return runtimes.find(
(rt) => rt =>
(rt.runtime == runtime || (rt.runtime == runtime ||
(rt.runtime === undefined && rt.language == runtime)) && (rt.runtime === undefined && rt.language == runtime)) &&
semver.satisfies(rt.version, ver) semver.satisfies(rt.version, ver)
); );
}; };
module.exports.load_package = Runtime.load_package; module.exports.load_package = Runtime.load_package;