Mascali
/
piston
mirror of https://github.com/engineer-man/piston.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

lint api

This commit is contained in:
Thomas Hobson 2021-05-08 12:20:21 +12:00
parent 2beb0abff7
commit 4259e89bb2
No known key found for this signature in database
GPG Key ID: 9F1FD9D87950DB6F
11 changed files with 1869 additions and 1857 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
api/.prettierignore Normal file
View File

@ -0,0 +1 @@
node_modules

1
api/.prettierrc.yaml Normal file
View File

@ -0,0 +1 @@
singleQuote: true

21
api/package-lock.json generated
View File

@ -19,6 +19,9 @@
"semver": "^7.3.4", "semver": "^7.3.4",
"uuid": "^8.3.2", "uuid": "^8.3.2",
"waitpid": "git+https://github.com/HexF/node-waitpid.git" "waitpid": "git+https://github.com/HexF/node-waitpid.git"
},
"devDependencies": {
"prettier": "2.2.1"
} }
}, },
"node_modules/accepts": { "node_modules/accepts": {
@ -391,6 +394,18 @@
"resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz",
"integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w="
}, },
"node_modules/prettier": {
"version": "2.2.1",
"resolved": "https://registry.npmjs.org/prettier/-/prettier-2.2.1.tgz",
"integrity": "sha512-PqyhM2yCjg/oKkFPtTGUojv7gnZAoG80ttl45O6x2Ug/rMJw4wcc9k6aaf2hibP7BGVCCM33gZoGjyvt9mm16Q==",
"dev": true,
"bin": {
"prettier": "bin-prettier.js"
},
"engines": {
"node": ">=10.13.0"
}
},
"node_modules/proxy-addr": { "node_modules/proxy-addr": {
"version": "2.0.6", "version": "2.0.6",
"resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.6.tgz", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.6.tgz",
@ -855,6 +870,12 @@
"resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz",
"integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w="
}, },
"prettier": {
"version": "2.2.1",
"resolved": "https://registry.npmjs.org/prettier/-/prettier-2.2.1.tgz",
"integrity": "sha512-PqyhM2yCjg/oKkFPtTGUojv7gnZAoG80ttl45O6x2Ug/rMJw4wcc9k6aaf2hibP7BGVCCM33gZoGjyvt9mm16Q==",
"dev": true
},
"proxy-addr": { "proxy-addr": {
"version": "2.0.6", "version": "2.0.6",
"resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.6.tgz", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.6.tgz",

8
api/package.json
View File

@ -15,5 +15,11 @@
"uuid": "^8.3.2", "uuid": "^8.3.2",
"waitpid": "git+https://github.com/HexF/node-waitpid.git" "waitpid": "git+https://github.com/HexF/node-waitpid.git"
}, },
"license": "MIT" "license": "MIT",
"scripts": {
"lint": "prettier . --write"
},
"devDependencies": {
"prettier": "2.2.1"
}
} }

172
api/src/api/v2.js
View File

@ -4,7 +4,7 @@ const router = express.Router();
const config = require('../config'); const config = require('../config');
const runtime = require('../runtime'); const runtime = require('../runtime');
const { Job } = require('../job'); const { Job } = require('../job');
const package = require('../package') const package = require('../package');
const logger = require('logplease').create('api/v2'); const logger = require('logplease').create('api/v2');
router.use((req, res, next) => { router.use((req, res, next) => {
@ -13,10 +13,8 @@ router.use((req, res, next) => {
} }
if (req.headers['content-type'] !== 'application/json') { if (req.headers['content-type'] !== 'application/json') {
return res return res.status(415).send({
.status(415) message: 'requests must be of type application/json',
.send({
message: 'requests must be of type application/json'
}); });
} }
@ -25,90 +23,90 @@ router.use((req, res, next) => {
router.post('/execute', async (req, res) => { router.post('/execute', async (req, res) => {
const { const {
language, version, language,
version,
files, files,
stdin, args, stdin,
run_timeout, compile_timeout, args,
compile_memory_limit, run_memory_limit run_timeout,
compile_timeout,
compile_memory_limit,
run_memory_limit,
} = req.body; } = req.body;
if (!language || typeof language !== 'string') { if (!language || typeof language !== 'string') {
return res return res.status(400).send({
.status(400) message: 'language is required as a string',
.send({
message: 'language is required as a string'
}); });
} }
if (!version || typeof version !== 'string') { if (!version || typeof version !== 'string') {
return res return res.status(400).send({
.status(400) message: 'version is required as a string',
.send({
message: 'version is required as a string'
}); });
} }
if (!files || !Array.isArray(files)) { if (!files || !Array.isArray(files)) {
return res return res.status(400).send({
.status(400) message: 'files is required as an array',
.send({
message: 'files is required as an array'
}); });
} }
for (const [i, file] of files.entries()) { for (const [i, file] of files.entries()) {
if (typeof file.content !== 'string') { if (typeof file.content !== 'string') {
return res return res.status(400).send({
.status(400) message: `files[${i}].content is required as a string`,
.send({
message: `files[${i}].content is required as a string`
}); });
} }
} }
if (compile_memory_limit) { if (compile_memory_limit) {
if (typeof compile_memory_limit !== 'number') { if (typeof compile_memory_limit !== 'number') {
return res return res.status(400).send({
.status(400) message: 'if specified, compile_memory_limit must be a number',
.send({ });
message: 'if specified, compile_memory_limit must be a number'
})
} }
if (config.compile_memory_limit >= 0 && (compile_memory_limit > config.compile_memory_limit || compile_memory_limit < 0)) { if (
return res config.compile_memory_limit >= 0 &&
.status(400) (compile_memory_limit > config.compile_memory_limit ||
.send({ compile_memory_limit < 0)
message: 'compile_memory_limit cannot exceed the configured limit of ' + config.compile_memory_limit ) {
}) return res.status(400).send({
message:
'compile_memory_limit cannot exceed the configured limit of ' +
config.compile_memory_limit,
});
} }
} }
if (run_memory_limit) { if (run_memory_limit) {
if (typeof run_memory_limit !== 'number') { if (typeof run_memory_limit !== 'number') {
return res return res.status(400).send({
.status(400) message: 'if specified, run_memory_limit must be a number',
.send({ });
message: 'if specified, run_memory_limit must be a number'
})
} }
if (config.run_memory_limit >= 0 && (run_memory_limit > config.run_memory_limit || run_memory_limit < 0)) { if (
return res config.run_memory_limit >= 0 &&
.status(400) (run_memory_limit > config.run_memory_limit || run_memory_limit < 0)
.send({ ) {
message: 'run_memory_limit cannot exceed the configured limit of ' + config.run_memory_limit return res.status(400).send({
}) message:
'run_memory_limit cannot exceed the configured limit of ' +
config.run_memory_limit,
});
} }
} }
const rt = runtime.get_latest_runtime_matching_language_version(language, version); const rt = runtime.get_latest_runtime_matching_language_version(
language,
version
);
if (rt === undefined) { if (rt === undefined) {
return res return res.status(400).send({
.status(400) message: `${language}-${version} runtime is unknown`,
.send({
message: `${language}-${version} runtime is unknown`
}); });
} }
@ -120,12 +118,12 @@ router.post('/execute', async (req, res) => {
stdin: stdin || '', stdin: stdin || '',
timeouts: { timeouts: {
run: run_timeout || 3000, run: run_timeout || 3000,
compile: compile_timeout || 10000 compile: compile_timeout || 10000,
}, },
memory_limits: { memory_limits: {
run: run_memory_limit || config.run_memory_limit, run: run_memory_limit || config.run_memory_limit,
compile: compile_memory_limit || config.compile_memory_limit compile: compile_memory_limit || config.compile_memory_limit,
} },
}); });
await job.prime(); await job.prime();
@ -134,43 +132,35 @@ router.post('/execute', async (req, res) => {
await job.cleanup(); await job.cleanup();
return res return res.status(200).send(result);
.status(200)
.send(result);
}); });
router.get('/runtimes', (req, res) => { router.get('/runtimes', (req, res) => {
const runtimes = runtime const runtimes = runtime.map((rt) => {
.map(rt => {
return { return {
language: rt.language, language: rt.language,
version: rt.version.raw, version: rt.version.raw,
aliases: rt.aliases, aliases: rt.aliases,
runtime: rt.runtime runtime: rt.runtime,
}; };
}); });
return res return res.status(200).send(runtimes);
.status(200)
.send(runtimes);
}); });
router.get('/packages', async (req, res) => { router.get('/packages', async (req, res) => {
logger.debug('Request to list packages'); logger.debug('Request to list packages');
let packages = await package.get_package_list(); let packages = await package.get_package_list();
packages = packages packages = packages.map((pkg) => {
.map(pkg => {
return { return {
language: pkg.language, language: pkg.language,
language_version: pkg.version.raw, language_version: pkg.version.raw,
installed: pkg.installed installed: pkg.installed,
}; };
}); });
return res return res.status(200).send(packages);
.status(200)
.send(packages);
}); });
router.post('/packages/:language/:version', async (req, res) => { router.post('/packages/:language/:version', async (req, res) => {
@ -181,26 +171,23 @@ router.post('/packages/:language/:version', async (req, res) => {
const pkg = await package.get_package(language, version); const pkg = await package.get_package(language, version);
if (pkg == null) { if (pkg == null) {
return res return res.status(404).send({
.status(404) message: `Requested package ${language}-${version} does not exist`,
.send({
message: `Requested package ${language}-${version} does not exist`
}); });
} }
try { try {
const response = await pkg.install(); const response = await pkg.install();
return res return res.status(200).send(response);
.status(200)
.send(response);
} catch (e) { } catch (e) {
logger.error(`Error while installing package ${pkg.language}-${pkg.version}:`, e.message); logger.error(
`Error while installing package ${pkg.language}-${pkg.version}:`,
e.message
);
return res return res.status(500).send({
.status(500) message: e.message,
.send({
message: e.message
}); });
} }
}); });
@ -213,26 +200,23 @@ router.delete('/packages/:language/:version', async (req, res) => {
const pkg = await package.get_package(language, version); const pkg = await package.get_package(language, version);
if (pkg == null) { if (pkg == null) {
return res return res.status(404).send({
.status(404) message: `Requested package ${language}-${version} does not exist`,
.send({
message: `Requested package ${language}-${version} does not exist`
}); });
} }
try { try {
const response = await pkg.uninstall(); const response = await pkg.uninstall();
return res return res.status(200).send(response);
.status(200)
.send(response);
} catch (e) { } catch (e) {
logger.error(`Error while uninstalling package ${pkg.language}-${pkg.version}:`, e.message); logger.error(
`Error while uninstalling package ${pkg.language}-${pkg.version}:`,
e.message
);
return res return res.status(500).send({
.status(500) message: e.message,
.send({
message: e.message
}); });
} }
}); });

80
api/src/config.js
View File

@ -9,126 +9,108 @@ const options = [
default: 'INFO', default: 'INFO',
options: Object.values(Logger.LogLevels), options: Object.values(Logger.LogLevels),
validators: [ validators: [
x => Object.values(Logger.LogLevels).includes(x) || `Log level ${x} does not exist` (x) =>
] Object.values(Logger.LogLevels).includes(x) ||
`Log level ${x} does not exist`,
],
}, },
{ {
key: 'bind_address', key: 'bind_address',
desc: 'Address to bind REST API on\nThank @Bones for the number', desc: 'Address to bind REST API on\nThank @Bones for the number',
default: '0.0.0.0:2000', default: '0.0.0.0:2000',
validators: [] validators: [],
}, },
{ {
key: 'data_directory', key: 'data_directory',
desc: 'Absolute path to store all piston related data at', desc: 'Absolute path to store all piston related data at',
default: '/piston', default: '/piston',
validators: [x=> fss.exists_sync(x) || `Directory ${x} does not exist`] validators: [(x) => fss.exists_sync(x) || `Directory ${x} does not exist`],
}, },
{ {
key: 'runner_uid_min', key: 'runner_uid_min',
desc: 'Minimum uid to use for runner', desc: 'Minimum uid to use for runner',
default: 1001, default: 1001,
parser: parse_int, parser: parse_int,
validators: [ validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
(x, raw) => !is_nan(x) || `${raw} is not a number`,
]
}, },
{ {
key: 'runner_uid_max', key: 'runner_uid_max',
desc: 'Maximum uid to use for runner', desc: 'Maximum uid to use for runner',
default: 1500, default: 1500,
parser: parse_int, parser: parse_int,
validators: [ validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
(x, raw) => !is_nan(x) || `${raw} is not a number`,
]
}, },
{ {
key: 'runner_gid_min', key: 'runner_gid_min',
desc: 'Minimum gid to use for runner', desc: 'Minimum gid to use for runner',
default: 1001, default: 1001,
parser: parse_int, parser: parse_int,
validators: [ validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
(x, raw) => !is_nan(x) || `${raw} is not a number`,
]
}, },
{ {
key: 'runner_gid_max', key: 'runner_gid_max',
desc: 'Maximum gid to use for runner', desc: 'Maximum gid to use for runner',
default: 1500, default: 1500,
parser: parse_int, parser: parse_int,
validators: [ validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
(x, raw) => !is_nan(x) || `${raw} is not a number`,
]
}, },
{ {
key: 'disable_networking', key: 'disable_networking',
desc: 'Set to true to disable networking', desc: 'Set to true to disable networking',
default: true, default: true,
parser: x => x === "true", parser: (x) => x === 'true',
validators: [ validators: [(x) => typeof x === 'boolean' || `${x} is not a boolean`],
x => typeof x === "boolean" || `${x} is not a boolean`
]
}, },
{ {
key: 'output_max_size', key: 'output_max_size',
desc: 'Max size of each stdio buffer', desc: 'Max size of each stdio buffer',
default: 1024, default: 1024,
parser: parse_int, parser: parse_int,
validators: [ validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
(x, raw) => !is_nan(x) || `${raw} is not a number`,
]
}, },
{ {
key: 'max_process_count', key: 'max_process_count',
desc: 'Max number of processes per job', desc: 'Max number of processes per job',
default: 64, default: 64,
parser: parse_int, parser: parse_int,
validators: [ validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
(x, raw) => !is_nan(x) || `${raw} is not a number`,
]
}, },
{ {
key: 'max_open_files', key: 'max_open_files',
desc: 'Max number of open files per job', desc: 'Max number of open files per job',
default: 2048, default: 2048,
parser: parse_int, parser: parse_int,
validators: [ validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
(x, raw) => !is_nan(x) || `${raw} is not a number`,
]
}, },
{ {
key: 'max_file_size', key: 'max_file_size',
desc: 'Max file size in bytes for a file', desc: 'Max file size in bytes for a file',
default: 10000000, //10MB default: 10000000, //10MB
parser: parse_int, parser: parse_int,
validators: [ validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
(x, raw) => !is_nan(x) || `${raw} is not a number`,
]
}, },
{ {
key: 'compile_memory_limit', key: 'compile_memory_limit',
desc: 'Max memory usage for compile stage in bytes (set to -1 for no limit)', desc:
'Max memory usage for compile stage in bytes (set to -1 for no limit)',
default: -1, // no limit default: -1, // no limit
parser: parse_int, parser: parse_int,
validators: [ validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
(x, raw) => !is_nan(x) || `${raw} is not a number`,
]
}, },
{ {
key: 'run_memory_limit', key: 'run_memory_limit',
desc: 'Max memory usage for run stage in bytes (set to -1 for no limit)', desc: 'Max memory usage for run stage in bytes (set to -1 for no limit)',
default: -1, // no limit default: -1, // no limit
parser: parse_int, parser: parse_int,
validators: [ validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
(x, raw) => !is_nan(x) || `${raw} is not a number`,
]
}, },
{ {
key: 'repo_url', key: 'repo_url',
desc: 'URL of repo index', desc: 'URL of repo index',
default: 'https://github.com/engineer-man/piston/releases/download/pkgs/index', default:
validators: [] 'https://github.com/engineer-man/piston/releases/download/pkgs/index',
} validators: [],
},
]; ];
logger.info(`Loading Configuration from environment`); logger.info(`Loading Configuration from environment`);
@ -137,10 +119,10 @@ let errored = false;
let config = {}; let config = {};
options.forEach(option => { options.forEach((option) => {
const env_key = "PISTON_" + option.key.to_upper_case(); const env_key = 'PISTON_' + option.key.to_upper_case();
const parser = option.parser || (x => x); const parser = option.parser || ((x) => x);
const env_val = process.env[env_key]; const env_val = process.env[env_key];
@ -148,12 +130,10 @@ options.forEach(option => {
const value = env_val || option.default; const value = env_val || option.default;
option.validators.for_each(validator => { option.validators.for_each((validator) => {
let response = null; let response = null;
if(env_val) if (env_val) response = validator(parsed_val, env_val);
response = validator(parsed_val, env_val); else response = validator(value, value);
else
response = validator(value, value);
if (response !== true) { if (response !== true) {
errored = true; errored = true;

18
api/src/globals.js
View File

@ -1,26 +1,20 @@
// Globals are things the user shouldn't change in config, but is good to not use inline constants for // Globals are things the user shouldn't change in config, but is good to not use inline constants for
const is_docker = require('is-docker'); const is_docker = require('is-docker');
const fs = require('fs'); const fs = require('fs');
const platform = `${is_docker() ? 'docker' : 'baremetal'}-${ const platform = `${is_docker() ? 'docker' : 'baremetal'}-${fs
fs.read_file_sync('/etc/os-release') .read_file_sync('/etc/os-release')
.toString() .toString()
.split('\n') .split('\n')
.find(x => x.startsWith('ID')) .find((x) => x.startsWith('ID'))
.replace('ID=','') .replace('ID=', '')}`;
}`;
module.exports = { module.exports = {
data_directories: { data_directories: {
packages: 'packages', packages: 'packages',
jobs: 'jobs' jobs: 'jobs',
}, },
version: require('../package.json').version, version: require('../package.json').version,
platform, platform,
pkg_installed_file: '.ppman-installed', //Used as indication for if a package was installed pkg_installed_file: '.ppman-installed', //Used as indication for if a package was installed
clean_directories: [ clean_directories: ['/dev/shm', '/run/lock', '/tmp', '/var/tmp'],
'/dev/shm',
'/run/lock',
'/tmp',
'/var/tmp'
]
}; };

33
api/src/index.js
View File

@ -18,7 +18,7 @@ const app = express();
Logger.setLogLevel(config.log_level); Logger.setLogLevel(config.log_level);
logger.debug('Ensuring data directories exist'); logger.debug('Ensuring data directories exist');
Object.values(globals.data_directories).for_each(dir => { Object.values(globals.data_directories).for_each((dir) => {
let data_path = path.join(config.data_directory, dir); let data_path = path.join(config.data_directory, dir);
logger.debug(`Ensuring ${data_path} exists`); logger.debug(`Ensuring ${data_path} exists`);
@ -35,25 +35,28 @@ const app = express();
}); });
logger.info('Loading packages'); logger.info('Loading packages');
const pkgdir = path.join(config.data_directory,globals.data_directories.packages); const pkgdir = path.join(
config.data_directory,
globals.data_directories.packages
);
const pkglist = await fs.readdir(pkgdir); const pkglist = await fs.readdir(pkgdir);
const languages = await Promise.all( const languages = await Promise.all(
pkglist.map(lang => { pkglist.map((lang) => {
return fs return fs.readdir(path.join(pkgdir, lang)).then((x) => {
.readdir(path.join(pkgdir,lang)) return x.map((y) => path.join(pkgdir, lang, y));
.then(x => {
return x.map(y => path.join(pkgdir, lang, y))
}); });
}) })
); );
const installed_languages = languages const installed_languages = languages
.flat() .flat()
.filter(pkg => fss.exists_sync(path.join(pkg, globals.pkg_installed_file))); .filter((pkg) =>
fss.exists_sync(path.join(pkg, globals.pkg_installed_file))
);
installed_languages.for_each(pkg => runtime.load_package(pkg)); installed_languages.for_each((pkg) => runtime.load_package(pkg));
logger.info('Starting API Server'); logger.info('Starting API Server');
logger.debug('Constructing Express App'); logger.debug('Constructing Express App');
@ -63,23 +66,19 @@ const app = express();
app.use(body_parser.json()); app.use(body_parser.json());
app.use((err, req, res, next) => { app.use((err, req, res, next) => {
return res return res.status(400).send({
.status(400) stack: err.stack,
.send({
stack: err.stack
}); });
}); });
logger.debug('Registering Routes'); logger.debug('Registering Routes');
const api_v2 = require('./api/v2') const api_v2 = require('./api/v2');
app.use('/api/v2', api_v2); app.use('/api/v2', api_v2);
app.use('/api/v2', api_v2); app.use('/api/v2', api_v2);
app.use((req, res, next) => { app.use((req, res, next) => {
return res return res.status(404).send({ message: 'Not Found' });
.status(404)
.send({message: 'Not Found'});
}); });
logger.debug('Calling app.listen'); logger.debug('Calling app.listen');

73
api/src/job.js
View File

@ -10,20 +10,19 @@ const wait_pid = require('waitpid');
const job_states = { const job_states = {
READY: Symbol('Ready to be primed'), READY: Symbol('Ready to be primed'),
PRIMED: Symbol('Primed and ready for execution'), PRIMED: Symbol('Primed and ready for execution'),
EXECUTED: Symbol('Executed and ready for cleanup') EXECUTED: Symbol('Executed and ready for cleanup'),
}; };
let uid = 0; let uid = 0;
let gid = 0; let gid = 0;
class Job { class Job {
constructor({ runtime, files, args, stdin, timeouts, memory_limits }) { constructor({ runtime, files, args, stdin, timeouts, memory_limits }) {
this.uuid = uuidv4(); this.uuid = uuidv4();
this.runtime = runtime; this.runtime = runtime;
this.files = files.map((file, i) => ({ this.files = files.map((file, i) => ({
name: file.name || `file${i}.code`, name: file.name || `file${i}.code`,
content: file.content content: file.content,
})); }));
this.args = args; this.args = args;
@ -37,11 +36,15 @@ class Job {
uid++; uid++;
gid++; gid++;
uid %= (config.runner_uid_max - config.runner_uid_min) + 1; uid %= config.runner_uid_max - config.runner_uid_min + 1;
gid %= (config.runner_gid_max - config.runner_gid_min) + 1; gid %= config.runner_gid_max - config.runner_gid_min + 1;
this.state = job_states.READY; this.state = job_states.READY;
this.dir = path.join(config.data_directory, globals.data_directories.jobs, this.uuid); this.dir = path.join(
config.data_directory,
globals.data_directories.jobs,
this.uuid
);
} }
async prime() { async prime() {
@ -74,19 +77,14 @@ class Job {
'prlimit', 'prlimit',
'--nproc=' + config.max_process_count, '--nproc=' + config.max_process_count,
'--nofile=' + config.max_open_files, '--nofile=' + config.max_open_files,
'--fsize=' + config.max_file_size '--fsize=' + config.max_file_size,
]; ];
if (memory_limit >= 0) { if (memory_limit >= 0) {
prlimit.push('--as=' + memory_limit); prlimit.push('--as=' + memory_limit);
} }
const proc_call = [ const proc_call = [...prlimit, ...nonetwork, 'bash', file, ...args];
...prlimit,
...nonetwork,
'bash',file,
...args
];
var stdout = ''; var stdout = '';
var stderr = ''; var stderr = '';
@ -95,22 +93,22 @@ class Job {
const proc = cp.spawn(proc_call[0], proc_call.splice(1), { const proc = cp.spawn(proc_call[0], proc_call.splice(1), {
env: { env: {
...this.runtime.env_vars, ...this.runtime.env_vars,
PISTON_LANGUAGE: this.runtime.language PISTON_LANGUAGE: this.runtime.language,
}, },
stdio: 'pipe', stdio: 'pipe',
cwd: this.dir, cwd: this.dir,
uid: this.uid, uid: this.uid,
gid: this.gid, gid: this.gid,
detached: true //give this process its own process group detached: true, //give this process its own process group
}); });
proc.stdin.write(this.stdin); proc.stdin.write(this.stdin);
proc.stdin.end(); proc.stdin.end();
proc.stdin.destroy(); proc.stdin.destroy();
const kill_timeout = set_timeout(_ => proc.kill('SIGKILL'), timeout); const kill_timeout = set_timeout((_) => proc.kill('SIGKILL'), timeout);
proc.stderr.on('data', data => { proc.stderr.on('data', (data) => {
if (stderr.length > config.output_max_size) { if (stderr.length > config.output_max_size) {
proc.kill('SIGKILL'); proc.kill('SIGKILL');
} else { } else {
@ -119,7 +117,7 @@ class Job {
} }
}); });
proc.stdout.on('data', data => { proc.stdout.on('data', (data) => {
if (stdout.length > config.output_max_size) { if (stdout.length > config.output_max_size) {
proc.kill('SIGKILL'); proc.kill('SIGKILL');
} else { } else {
@ -151,10 +149,16 @@ class Job {
async execute() { async execute() {
if (this.state !== job_states.PRIMED) { if (this.state !== job_states.PRIMED) {
throw new Error('Job must be in primed state, current state: ' + this.state.toString()); throw new Error(
'Job must be in primed state, current state: ' + this.state.toString()
);
} }
logger.info(`Executing job uuid=${this.uuid} uid=${this.uid} gid=${this.gid} runtime=${this.runtime.toString()}`); logger.info(
`Executing job uuid=${this.uuid} uid=${this.uid} gid=${
this.gid
} runtime=${this.runtime.toString()}`
);
logger.debug('Compiling'); logger.debug('Compiling');
@ -163,7 +167,7 @@ class Job {
if (this.runtime.compiled) { if (this.runtime.compiled) {
compile = await this.safe_call( compile = await this.safe_call(
path.join(this.runtime.pkgdir, 'compile'), path.join(this.runtime.pkgdir, 'compile'),
this.files.map(x => x.name), this.files.map((x) => x.name),
this.timeouts.compile, this.timeouts.compile,
this.memory_limits.compile this.memory_limits.compile
); );
@ -184,7 +188,7 @@ class Job {
compile, compile,
run, run,
language: this.runtime.language, language: this.runtime.language,
version: this.runtime.version.raw version: this.runtime.version.raw,
}; };
} }
@ -192,25 +196,27 @@ class Job {
let processes = [1]; let processes = [1];
while (processes.length > 0) { while (processes.length > 0) {
processes = await new Promise((resolve, reject) => cp.execFile('ps', ['awwxo', 'pid,ruid'], (err, stdout) => { processes = await new Promise((resolve, reject) =>
cp.execFile('ps', ['awwxo', 'pid,ruid'], (err, stdout) => {
if (err === null) { if (err === null) {
const lines = stdout.split('\n').slice(1); //Remove header with slice const lines = stdout.split('\n').slice(1); //Remove header with slice
const procs = lines.map(line => { const procs = lines.map((line) => {
const [pid, ruid] = line const [pid, ruid] = line
.trim() .trim()
.split(/\s+/) .split(/\s+/)
.map(n => parseInt(n)); .map((n) => parseInt(n));
return { pid, ruid } return { pid, ruid };
}); });
resolve(procs); resolve(procs);
} else { } else {
reject(error); reject(error);
} }
})); })
);
processes = processes.filter(proc => proc.ruid === this.uid); processes = processes.filter((proc) => proc.ruid === this.uid);
for (const proc of processes) { for (const proc of processes) {
// First stop the processes, but keep their resources allocated so they cant re-fork // First stop the processes, but keep their resources allocated so they cant re-fork
@ -221,7 +227,6 @@ class Job {
} }
} }
for (const proc of processes) { for (const proc of processes) {
// Then clear them out of the process tree // Then clear them out of the process tree
try { try {
@ -250,7 +255,7 @@ class Job {
} }
} catch (e) { } catch (e) {
// File was somehow deleted in the time that we read the dir to when we checked the file // File was somehow deleted in the time that we read the dir to when we checked the file
logger.warn(`Error removing file ${file_path}: ${e}`) logger.warn(`Error removing file ${file_path}: ${e}`);
} }
} }
} }
@ -261,14 +266,10 @@ class Job {
async cleanup() { async cleanup() {
logger.info(`Cleaning up job uuid=${this.uuid}`); logger.info(`Cleaning up job uuid=${this.uuid}`);
await Promise.all([ await Promise.all([this.cleanup_processes(), this.cleanup_filesystem()]);
this.cleanup_processes(),
this.cleanup_filesystem()
]);
} }
} }
module.exports = { module.exports = {
Job Job,
}; };

89
api/src/package.js
View File

@ -13,7 +13,6 @@ const chownr = require('chownr');
const util = require('util'); const util = require('util');
class Package { class Package {
constructor({ language, version, download, checksum }) { constructor({ language, version, download, checksum }) {
this.language = language; this.language = language;
this.version = semver.parse(version); this.version = semver.parse(version);
@ -22,7 +21,9 @@ class Package {
} }
get installed() { get installed() {
return fss.exists_sync(path.join(this.install_path, globals.pkg_installed_file)); return fss.exists_sync(
path.join(this.install_path, globals.pkg_installed_file)
);
} }
get install_path() { get install_path() {
@ -42,14 +43,18 @@ class Package {
logger.info(`Installing ${this.language}-${this.version.raw}`); logger.info(`Installing ${this.language}-${this.version.raw}`);
if (fss.exists_sync(this.install_path)) { if (fss.exists_sync(this.install_path)) {
logger.warn(`${this.language}-${this.version.raw} has residual files. Removing them.`); logger.warn(
`${this.language}-${this.version.raw} has residual files. Removing them.`
);
await fs.rm(this.install_path, { recursive: true, force: true }); await fs.rm(this.install_path, { recursive: true, force: true });
} }
logger.debug(`Making directory ${this.install_path}`); logger.debug(`Making directory ${this.install_path}`);
await fs.mkdir(this.install_path, { recursive: true }); await fs.mkdir(this.install_path, { recursive: true });
logger.debug(`Downloading package from ${this.download} in to ${this.install_path}`); logger.debug(
`Downloading package from ${this.download} in to ${this.install_path}`
);
const pkgpath = path.join(this.install_path, 'pkg.tar.gz'); const pkgpath = path.join(this.install_path, 'pkg.tar.gz');
const download = await fetch(this.download); const download = await fetch(this.download);
@ -63,7 +68,8 @@ class Package {
logger.debug('Validating checksums'); logger.debug('Validating checksums');
logger.debug(`Assert sha256(pkg.tar.gz) == ${this.checksum}`); logger.debug(`Assert sha256(pkg.tar.gz) == ${this.checksum}`);
const cs = crypto.create_hash("sha256") const cs = crypto
.create_hash('sha256')
.update(fss.readFileSync(pkgpath)) .update(fss.readFileSync(pkgpath))
.digest('hex'); .digest('hex');
@ -71,10 +77,14 @@ class Package {
throw new Error(`Checksum miss-match want: ${val} got: ${cs}`); throw new Error(`Checksum miss-match want: ${val} got: ${cs}`);
} }
logger.debug(`Extracting package files from archive ${pkgpath} in to ${this.install_path}`); logger.debug(
`Extracting package files from archive ${pkgpath} in to ${this.install_path}`
);
await new Promise((resolve, reject) => { await new Promise((resolve, reject) => {
const proc = cp.exec(`bash -c 'cd "${this.install_path}" && tar xzf ${pkgpath}'`); const proc = cp.exec(
`bash -c 'cd "${this.install_path}" && tar xzf ${pkgpath}'`
);
proc.once('exit', (code, _) => { proc.once('exit', (code, _) => {
code === 0 ? resolve() : reject(); code === 0 ? resolve() : reject();
@ -95,20 +105,15 @@ class Package {
const envout = await new Promise((resolve, reject) => { const envout = await new Promise((resolve, reject) => {
let stdout = ''; let stdout = '';
const proc = cp const proc = cp.spawn('env', ['-i', 'bash', '-c', `${get_env_command}`], {
.spawn( stdio: ['ignore', 'pipe', 'pipe'],
'env', });
['-i','bash','-c',`${get_env_command}`],
{
stdio: ['ignore', 'pipe', 'pipe']
}
);
proc.once('exit', (code, _) => { proc.once('exit', (code, _) => {
code === 0 ? resolve(stdout) : reject(); code === 0 ? resolve(stdout) : reject();
}); });
proc.stdout.on('data', data => { proc.stdout.on('data', (data) => {
stdout += data; stdout += data;
}); });
@ -117,7 +122,9 @@ class Package {
const filtered_env = envout const filtered_env = envout
.split('\n') .split('\n')
.filter(l => !['PWD','OLDPWD','_', 'SHLVL'].includes(l.split('=',2)[0])) .filter(
(l) => !['PWD', 'OLDPWD', '_', 'SHLVL'].includes(l.split('=', 2)[0])
)
.join('\n'); .join('\n');
await fs.write_file(path.join(this.install_path, '.env'), filtered_env); await fs.write_file(path.join(this.install_path, '.env'), filtered_env);
@ -126,58 +133,62 @@ class Package {
await util.promisify(chownr)(this.install_path, 0, 0); await util.promisify(chownr)(this.install_path, 0, 0);
logger.debug('Writing installed state to disk'); logger.debug('Writing installed state to disk');
await fs.write_file(path.join(this.install_path, globals.pkg_installed_file), Date.now().toString()); await fs.write_file(
path.join(this.install_path, globals.pkg_installed_file),
Date.now().toString()
);
logger.info(`Installed ${this.language}-${this.version.raw}`); logger.info(`Installed ${this.language}-${this.version.raw}`);
return { return {
language: this.language, language: this.language,
version: this.version.raw version: this.version.raw,
}; };
} }
async uninstall() { async uninstall() {
logger.info(`Uninstalling ${this.language}-${this.version.raw}`); logger.info(`Uninstalling ${this.language}-${this.version.raw}`);
logger.debug("Finding runtime") logger.debug('Finding runtime');
const found_runtime = runtime.get_runtime_by_name_and_version(this.language, this.version.raw); const found_runtime = runtime.get_runtime_by_name_and_version(
this.language,
this.version.raw
);
if (!found_runtime) { if (!found_runtime) {
logger.error(`Uninstalling ${this.language}-${this.version.raw} failed: Not installed`) logger.error(
throw new Error(`${this.language}-${this.version.raw} is not installed`) `Uninstalling ${this.language}-${this.version.raw} failed: Not installed`
);
throw new Error(`${this.language}-${this.version.raw} is not installed`);
} }
logger.debug("Unregistering runtime") logger.debug('Unregistering runtime');
found_runtime.unregister(); found_runtime.unregister();
logger.debug("Cleaning files from disk") logger.debug('Cleaning files from disk');
await fs.rmdir(this.install_path, {recursive: true}) await fs.rmdir(this.install_path, { recursive: true });
logger.info(`Uninstalled ${this.language}-${this.version.raw}`) logger.info(`Uninstalled ${this.language}-${this.version.raw}`);
return { return {
language: this.language, language: this.language,
version: this.version.raw version: this.version.raw,
}; };
} }
static async get_package_list() { static async get_package_list() {
const repo_content = await fetch(config.repo_url).then(x => x.text()); const repo_content = await fetch(config.repo_url).then((x) => x.text());
const entries = repo_content const entries = repo_content.split('\n').filter((x) => x.length > 0);
.split('\n')
.filter(x => x.length > 0);
return entries.map(line => { return entries.map((line) => {
const [language, version, checksum, download] = line.split(',', 4); const [language, version, checksum, download] = line.split(',', 4);
return new Package({ return new Package({
language, language,
version, version,
checksum, checksum,
download download,
}); });
}); });
} }
@ -185,16 +196,14 @@ class Package {
static async get_package(lang, version) { static async get_package(lang, version) {
const packages = await Package.get_package_list(); const packages = await Package.get_package_list();
const candidates = packages const candidates = packages.filter((pkg) => {
.filter(pkg => { return pkg.language == lang && semver.satisfies(pkg.version, version);
return pkg.language == lang && semver.satisfies(pkg.version, version)
}); });
candidates.sort((a, b) => semver.rcompare(a.version, b.version)); candidates.sort((a, b) => semver.rcompare(a.version, b.version));
return candidates[0] || null; return candidates[0] || null;
} }
} }
module.exports = Package; module.exports = Package;

44
api/src/runtime.js
View File

@ -8,7 +8,6 @@ const path = require('path');
const runtimes = []; const runtimes = [];
class Runtime { class Runtime {
constructor({ language, version, aliases, pkgdir, runtime }) { constructor({ language, version, aliases, pkgdir, runtime }) {
this.language = language; this.language = language;
this.version = version; this.version = version;
@ -34,22 +33,26 @@ class Runtime {
if (provides) { if (provides) {
// Multiple languages in 1 package // Multiple languages in 1 package
provides.forEach(lang => { provides.forEach((lang) => {
runtimes.push(new Runtime({ runtimes.push(
new Runtime({
language: lang.language, language: lang.language,
aliases: lang.aliases, aliases: lang.aliases,
version, version,
pkgdir: package_dir, pkgdir: package_dir,
runtime: language runtime: language,
})); })
);
}); });
} else { } else {
runtimes.push(new Runtime({ runtimes.push(
new Runtime({
language, language,
version, version,
aliases, aliases,
pkgdir: package_dir pkgdir: package_dir,
})) })
);
} }
logger.debug(`Package ${language}-${version} was loaded`); logger.debug(`Package ${language}-${version} was loaded`);
@ -73,7 +76,7 @@ class Runtime {
env_content env_content
.trim() .trim()
.split('\n') .split('\n')
.map(line => line.split('=',2)) .map((line) => line.split('=', 2))
.forEach(([key, val]) => { .forEach(([key, val]) => {
this._env_vars[key.trim()] = val.trim(); this._env_vars[key.trim()] = val.trim();
}); });
@ -95,15 +98,28 @@ class Runtime {
module.exports = runtimes; module.exports = runtimes;
module.exports.Runtime = Runtime; module.exports.Runtime = Runtime;
module.exports.get_runtimes_matching_language_version = function (lang, ver) { module.exports.get_runtimes_matching_language_version = function (lang, ver) {
return runtimes.filter(rt => (rt.language == lang || rt.aliases.includes(lang)) && semver.satisfies(rt.version, ver)); return runtimes.filter(
(rt) =>
(rt.language == lang || rt.aliases.includes(lang)) &&
semver.satisfies(rt.version, ver)
);
}; };
module.exports.get_latest_runtime_matching_language_version = function(lang, ver){ module.exports.get_latest_runtime_matching_language_version = function (
return module.exports.get_runtimes_matching_language_version(lang, ver) lang,
ver
) {
return module.exports
.get_runtimes_matching_language_version(lang, ver)
.sort((a, b) => semver.rcompare(a.version, b.version))[0]; .sort((a, b) => semver.rcompare(a.version, b.version))[0];
}; };
module.exports.get_runtime_by_name_and_version = function (runtime, ver) { module.exports.get_runtime_by_name_and_version = function (runtime, ver) {
return runtimes.find(rt => (rt.runtime == runtime || (rt.runtime === undefined && rt.language == runtime)) && semver.satisfies(rt.version, ver)); return runtimes.find(
} (rt) =>
(rt.runtime == runtime ||
(rt.runtime === undefined && rt.language == runtime)) &&
semver.satisfies(rt.version, ver)
);
};
module.exports.load_package = Runtime.load_package; module.exports.load_package = Runtime.load_package;