Mascali
/
piston
mirror of https://github.com/engineer-man/piston.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Piston lint

This commit is contained in:
Brikaa 2021-10-08 15:16:57 +02:00
parent d61fb8ec5b
commit f2c91acbe6
57 changed files with 1121 additions and 893 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.github/ISSUE_TEMPLATE/language-request.md vendored
View File

@ -4,7 +4,6 @@ about: Template for requesting language support
title: Add [insert language name here] title: Add [insert language name here]
labels: package labels: package
assignees: '' assignees: ''
--- ---
Provide links to different compilers/interpreters that could be used to implement this language, and discuss pros/cons of each. Provide links to different compilers/interpreters that could be used to implement this language, and discuss pros/cons of each.

19
.github/PULL_REQUEST_TEMPLATE/package.md vendored
View File

@ -1,10 +1,11 @@
Checklist: Checklist:
* [ ] The package builds locally with `./piston build-pkg [package] [version]`
* [ ] The package installs with `./piston ppman install [package]=[version]` - [ ] The package builds locally with `./piston build-pkg [package] [version]`
* [ ] The package runs the test code with `./piston run [package] -l [version] packages/[package]/[version]/test.*` - [ ] The package installs with `./piston ppman install [package]=[version]`
* [ ] Package files are placed in the correct directory - [ ] The package runs the test code with `./piston run [package] -l [version] packages/[package]/[version]/test.*`
* [ ] No old package versions are removed - [ ] Package files are placed in the correct directory
* [ ] All source files are deleted in the `build.sh` script - [ ] No old package versions are removed
* [ ] `metadata.json`'s `language` and `version` fields match the directory path - [ ] All source files are deleted in the `build.sh` script
* [ ] Any extensions the language may use are set as aliases - [ ] `metadata.json`'s `language` and `version` fields match the directory path
* [ ] Any alternative names the language is referred to are set as aliases. - [ ] Any extensions the language may use are set as aliases
- [ ] Any alternative names the language is referred to are set as aliases.

1
.github/workflows/api-push.yaml vendored
View File

@ -7,7 +7,6 @@ on:
paths: paths:
- api/** - api/**
jobs: jobs:
push_to_registry: push_to_registry:
runs-on: ubuntu-latest runs-on: ubuntu-latest

4
.github/workflows/package-pr.yaml vendored
View File

@ -1,4 +1,4 @@
name: "Package Pull Requests" name: 'Package Pull Requests'
on: on:
pull_request: pull_request:
@ -8,7 +8,7 @@ on:
- reopened - reopened
- synchronize - synchronize
paths: paths:
- "packages/**" - 'packages/**'
jobs: jobs:
check-pkg: check-pkg:

5
.github/workflows/package-push.yaml vendored
View File

@ -8,7 +8,6 @@ on:
paths: paths:
- packages/** - packages/**
jobs: jobs:
build-pkg: build-pkg:
name: Build package name: Build package
@ -51,9 +50,9 @@ jobs:
runs-on: ubuntu-latest runs-on: ubuntu-latest
needs: build-pkg needs: build-pkg
steps: steps:
- name: "Download all release assets" - name: 'Download all release assets'
run: curl -s https://api.github.com/repos/engineer-man/piston/releases/latest | jq '.assets[].browser_download_url' -r | xargs -L 1 curl -sLO run: curl -s https://api.github.com/repos/engineer-man/piston/releases/latest | jq '.assets[].browser_download_url' -r | xargs -L 1 curl -sLO
- name: "Generate index file" - name: 'Generate index file'
run: | run: |
echo "" > index echo "" > index
BASEURL=https://github.com/engineer-man/piston/releases/download/pkgs/ BASEURL=https://github.com/engineer-man/piston/releases/download/pkgs/

1
.gitignore vendored
View File

@ -1,2 +1,3 @@
data/ data/
.piston_env .piston_env
node_modules

12
.prettierignore Normal file
View File

@ -0,0 +1,12 @@
node_modules
data/
api/_piston
repo/build
packages/*/*/*
packages/*.pkg.tar.gz
!packages/*/*/metadata.json
!packages/*/*/build.sh
!packages/*/*/environment
!packages/*/*/run
!packages/*/*/compile
!packages/*/*/test.*

0
api/.prettierrc.yaml → .prettierrc.yaml
View File

1
api/.gitignore vendored
View File

@ -1,2 +1 @@
node_modules
_piston _piston

1
api/.prettierignore
View File

@ -1 +0,0 @@
node_modules

28
api/package-lock.json generated
View File

@ -1,12 +1,12 @@
{ {
"name": "piston-api", "name": "piston-api",
"version": "3.0.0", "version": "3.1.0",
"lockfileVersion": 2, "lockfileVersion": 2,
"requires": true, "requires": true,
"packages": { "packages": {
"": { "": {
"name": "piston-api", "name": "piston-api",
"version": "3.0.0", "version": "3.1.0",
"license": "MIT", "license": "MIT",
"dependencies": { "dependencies": {
"body-parser": "^1.19.0", "body-parser": "^1.19.0",
@ -20,9 +20,6 @@
"semver": "^7.3.4", "semver": "^7.3.4",
"uuid": "^8.3.2", "uuid": "^8.3.2",
"waitpid": "git+https://github.com/HexF/node-waitpid.git" "waitpid": "git+https://github.com/HexF/node-waitpid.git"
},
"devDependencies": {
"prettier": "2.2.1"
} }
}, },
"node_modules/accepts": { "node_modules/accepts": {
@ -409,18 +406,6 @@
"resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz",
"integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w="
}, },
"node_modules/prettier": {
"version": "2.2.1",
"resolved": "https://registry.npmjs.org/prettier/-/prettier-2.2.1.tgz",
"integrity": "sha512-PqyhM2yCjg/oKkFPtTGUojv7gnZAoG80ttl45O6x2Ug/rMJw4wcc9k6aaf2hibP7BGVCCM33gZoGjyvt9mm16Q==",
"dev": true,
"bin": {
"prettier": "bin-prettier.js"
},
"engines": {
"node": ">=10.13.0"
}
},
"node_modules/proxy-addr": { "node_modules/proxy-addr": {
"version": "2.0.6", "version": "2.0.6",
"resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.6.tgz", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.6.tgz",
@ -595,7 +580,8 @@
} }
}, },
"node_modules/waitpid": { "node_modules/waitpid": {
"resolved": "git+ssh://git@github.com/HexF/node-waitpid.git#a08d116a5d993a747624fe72ff890167be8c34aa" "resolved": "git+ssh://git@github.com/HexF/node-waitpid.git#a08d116a5d993a747624fe72ff890167be8c34aa",
"hasInstallScript": true
}, },
"node_modules/ws": { "node_modules/ws": {
"version": "7.5.3", "version": "7.5.3",
@ -913,12 +899,6 @@
"resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz",
"integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w="
}, },
"prettier": {
"version": "2.2.1",
"resolved": "https://registry.npmjs.org/prettier/-/prettier-2.2.1.tgz",
"integrity": "sha512-PqyhM2yCjg/oKkFPtTGUojv7gnZAoG80ttl45O6x2Ug/rMJw4wcc9k6aaf2hibP7BGVCCM33gZoGjyvt9mm16Q==",
"dev": true
},
"proxy-addr": { "proxy-addr": {
"version": "2.0.6", "version": "2.0.6",
"resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.6.tgz", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.6.tgz",

8
api/package.json
View File

@ -16,11 +16,5 @@
"uuid": "^8.3.2", "uuid": "^8.3.2",
"waitpid": "git+https://github.com/HexF/node-waitpid.git" "waitpid": "git+https://github.com/HexF/node-waitpid.git"
}, },
"license": "MIT", "license": "MIT"
"scripts": {
"lint": "prettier . --write"
},
"devDependencies": {
"prettier": "2.2.1"
}
} }

181
api/src/api/v2.js
View File

@ -8,10 +8,49 @@ const { Job } = require('../job');
const package = require('../package'); const package = require('../package');
const logger = require('logplease').create('api/v2'); const logger = require('logplease').create('api/v2');
const SIGNALS = ["SIGABRT","SIGALRM","SIGBUS","SIGCHLD","SIGCLD","SIGCONT","SIGEMT","SIGFPE","SIGHUP","SIGILL","SIGINFO","SIGINT","SIGIO","SIGIOT","SIGKILL","SIGLOST","SIGPIPE","SIGPOLL","SIGPROF","SIGPWR","SIGQUIT","SIGSEGV","SIGSTKFLT","SIGSTOP","SIGTSTP","SIGSYS","SIGTERM","SIGTRAP","SIGTTIN","SIGTTOU","SIGUNUSED","SIGURG","SIGUSR1","SIGUSR2","SIGVTALRM","SIGXCPU","SIGXFSZ","SIGWINCH"] const SIGNALS = [
'SIGABRT',
'SIGALRM',
'SIGBUS',
'SIGCHLD',
'SIGCLD',
'SIGCONT',
'SIGEMT',
'SIGFPE',
'SIGHUP',
'SIGILL',
'SIGINFO',
'SIGINT',
'SIGIO',
'SIGIOT',
'SIGKILL',
'SIGLOST',
'SIGPIPE',
'SIGPOLL',
'SIGPROF',
'SIGPWR',
'SIGQUIT',
'SIGSEGV',
'SIGSTKFLT',
'SIGSTOP',
'SIGTSTP',
'SIGSYS',
'SIGTERM',
'SIGTRAP',
'SIGTTIN',
'SIGTTOU',
'SIGUNUSED',
'SIGURG',
'SIGUSR1',
'SIGUSR2',
'SIGVTALRM',
'SIGXCPU',
'SIGXFSZ',
'SIGWINCH',
];
// ref: https://man7.org/linux/man-pages/man7/signal.7.html // ref: https://man7.org/linux/man-pages/man7/signal.7.html
function get_job(body){ function get_job(body) {
let { let {
language, language,
version, version,
@ -21,7 +60,7 @@ function get_job(body){
compile_memory_limit, compile_memory_limit,
run_memory_limit, run_memory_limit,
run_timeout, run_timeout,
compile_timeout compile_timeout,
} = body; } = body;
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
@ -68,7 +107,7 @@ function get_job(body){
} }
if (typeof constraint_value !== 'number') { if (typeof constraint_value !== 'number') {
return reject({ return reject({
message: `If specified, ${constraint_name} must be a number` message: `If specified, ${constraint_name} must be a number`,
}); });
} }
if (configured_limit <= 0) { if (configured_limit <= 0) {
@ -76,12 +115,12 @@ function get_job(body){
} }
if (constraint_value > configured_limit) { if (constraint_value > configured_limit) {
return reject({ return reject({
message: `${constraint_name} cannot exceed the configured limit of ${configured_limit}` message: `${constraint_name} cannot exceed the configured limit of ${configured_limit}`,
}); });
} }
if (constraint_value < 0) { if (constraint_value < 0) {
return reject({ return reject({
message: `${constraint_name} must be non-negative` message: `${constraint_name} must be non-negative`,
}); });
} }
} }
@ -91,10 +130,11 @@ function get_job(body){
run_timeout = run_timeout || rt.timeouts.run; run_timeout = run_timeout || rt.timeouts.run;
compile_memory_limit = compile_memory_limit || rt.memory_limits.compile; compile_memory_limit = compile_memory_limit || rt.memory_limits.compile;
run_timeout = run_timeout || rt.timeouts.run; run_timeout = run_timeout || rt.timeouts.run;
resolve(new Job({ resolve(
new Job({
runtime: rt, runtime: rt,
args: args || [], args: args || [],
stdin: stdin || "", stdin: stdin || '',
files, files,
timeouts: { timeouts: {
run: run_timeout, run: run_timeout,
@ -103,8 +143,9 @@ function get_job(body){
memory_limits: { memory_limits: {
run: run_memory_limit, run: run_memory_limit,
compile: compile_memory_limit, compile: compile_memory_limit,
} },
})); })
);
}); });
} }
@ -123,88 +164,104 @@ router.use((req, res, next) => {
}); });
router.ws('/connect', async (ws, req) => { router.ws('/connect', async (ws, req) => {
let job = null; let job = null;
let eventBus = new events.EventEmitter(); let eventBus = new events.EventEmitter();
eventBus.on("stdout", (data) => ws.send(JSON.stringify({type: "data", stream: "stdout", data: data.toString()}))) eventBus.on('stdout', data =>
eventBus.on("stderr", (data) => ws.send(JSON.stringify({type: "data", stream: "stderr", data: data.toString()}))) ws.send(
eventBus.on("stage", (stage)=> ws.send(JSON.stringify({type: "stage", stage}))) JSON.stringify({
eventBus.on("exit", (stage, status) => ws.send(JSON.stringify({type: "exit", stage, ...status}))) type: 'data',
stream: 'stdout',
data: data.toString(),
})
)
);
eventBus.on('stderr', data =>
ws.send(
JSON.stringify({
type: 'data',
stream: 'stderr',
data: data.toString(),
})
)
);
eventBus.on('stage', stage =>
ws.send(JSON.stringify({ type: 'stage', stage }))
);
eventBus.on('exit', (stage, status) =>
ws.send(JSON.stringify({ type: 'exit', stage, ...status }))
);
ws.on("message", async (data) => { ws.on('message', async data => {
try {
try{
const msg = JSON.parse(data); const msg = JSON.parse(data);
switch(msg.type){ switch (msg.type) {
case "init": case 'init':
if(job === null){ if (job === null) {
job = await get_job(msg); job = await get_job(msg);
await job.prime(); await job.prime();
ws.send(JSON.stringify({ ws.send(
type: "runtime", JSON.stringify({
type: 'runtime',
language: job.runtime.language, language: job.runtime.language,
version: job.runtime.version.raw version: job.runtime.version.raw,
})) })
);
await job.execute_interactive(eventBus); await job.execute_interactive(eventBus);
ws.close(4999, "Job Completed"); ws.close(4999, 'Job Completed');
} else {
}else{ ws.close(4000, 'Already Initialized');
ws.close(4000, "Already Initialized");
} }
break; break;
case "data": case 'data':
if(job !== null){ if (job !== null) {
if(msg.stream === "stdin"){ if (msg.stream === 'stdin') {
eventBus.emit("stdin", msg.data) eventBus.emit('stdin', msg.data);
}else{ } else {
ws.close(4004, "Can only write to stdin") ws.close(4004, 'Can only write to stdin');
} }
}else{ } else {
ws.close(4003, "Not yet initialized") ws.close(4003, 'Not yet initialized');
} }
break; break;
case "signal": case 'signal':
if(job !== null){ if (job !== null) {
if(SIGNALS.includes(msg.signal)){ if (SIGNALS.includes(msg.signal)) {
eventBus.emit("signal", msg.signal) eventBus.emit('signal', msg.signal);
}else{ } else {
ws.close(4005, "Invalid signal") ws.close(4005, 'Invalid signal');
} }
}else{ } else {
ws.close(4003, "Not yet initialized") ws.close(4003, 'Not yet initialized');
} }
break; break;
} }
} catch (error) {
}catch(error){ ws.send(JSON.stringify({ type: 'error', message: error.message }));
ws.send(JSON.stringify({type: "error", message: error.message})) ws.close(4002, 'Notified Error');
ws.close(4002, "Notified Error")
// ws.close message is limited to 123 characters, so we notify over WS then close. // ws.close message is limited to 123 characters, so we notify over WS then close.
} }
}) });
ws.on("close", async ()=>{ ws.on('close', async () => {
if(job !== null){ if (job !== null) {
await job.cleanup() await job.cleanup();
} }
}) });
setTimeout(()=>{ setTimeout(() => {
//Terminate the socket after 1 second, if not initialized. //Terminate the socket after 1 second, if not initialized.
if(job === null) if (job === null) ws.close(4001, 'Initialization Timeout');
ws.close(4001, "Initialization Timeout"); }, 1000);
}, 1000) });
})
router.post('/execute', async (req, res) => { router.post('/execute', async (req, res) => {
try {
try{
const job = await get_job(req.body); const job = await get_job(req.body);
await job.prime(); await job.prime();
@ -214,7 +271,7 @@ router.post('/execute', async (req, res) => {
await job.cleanup(); await job.cleanup();
return res.status(200).send(result); return res.status(200).send(result);
}catch(error){ } catch (error) {
return res.status(400).json(error); return res.status(400).json(error);
} }
}); });

49
api/src/config.js
View File

@ -5,8 +5,7 @@ const logger = Logger.create('config');
function parse_overrides(overrides) { function parse_overrides(overrides) {
try { try {
return JSON.parse(overrides); return JSON.parse(overrides);
} } catch (e) {
catch (e) {
return null; return null;
} }
} }
@ -16,15 +15,20 @@ function validate_overrides(overrides, options) {
for (let key in overrides[language]) { for (let key in overrides[language]) {
if ( if (
![ ![
'max_process_count', 'max_open_files', 'max_file_size', 'max_process_count',
'compile_memory_limit', 'run_memory_limit', 'compile_timeout', 'max_open_files',
'run_timeout', 'output_max_size' 'max_file_size',
'compile_memory_limit',
'run_memory_limit',
'compile_timeout',
'run_timeout',
'output_max_size',
].includes(key) ].includes(key)
) { ) {
logger.error(`Invalid overridden option: ${key}`); logger.error(`Invalid overridden option: ${key}`);
return false; return false;
} }
let option = options.find((o) => o.key === key); let option = options.find(o => o.key === key);
let parser = option.parser; let parser = option.parser;
let raw = overrides[language][key]; let raw = overrides[language][key];
let value = parser(raw); let value = parser(raw);
@ -32,14 +36,19 @@ function validate_overrides(overrides, options) {
for (let validator of validators) { for (let validator of validators) {
let response = validator(value, raw); let response = validator(value, raw);
if (response !== true) { if (response !== true) {
logger.error(`Failed to validate overridden option: ${key}`, response); logger.error(
`Failed to validate overridden option: ${key}`,
response
);
return false; return false;
} }
} }
overrides[language][key] = value; overrides[language][key] = value;
} }
// Modifies the reference // Modifies the reference
options[options.index_of(options.find((o) => o.key === 'limit_overrides'))] = overrides; options[
options.index_of(options.find(o => o.key === 'limit_overrides'))
] = overrides;
} }
return true; return true;
} }
@ -135,32 +144,28 @@ const options = [
}, },
{ {
key: 'compile_timeout', key: 'compile_timeout',
desc: desc: 'Max time allowed for compile stage in milliseconds',
'Max time allowed for compile stage in milliseconds',
default: 10000, // 10 seconds default: 10000, // 10 seconds
parser: parse_int, parser: parse_int,
validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`], validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
}, },
{ {
key: 'run_timeout', key: 'run_timeout',
desc: desc: 'Max time allowed for run stage in milliseconds',
'Max time allowed for run stage in milliseconds',
default: 3000, // 3 seconds default: 3000, // 3 seconds
parser: parse_int, parser: parse_int,
validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`], validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
}, },
{ {
key: 'compile_memory_limit', key: 'compile_memory_limit',
desc: desc: 'Max memory usage for compile stage in bytes (set to -1 for no limit)',
'Max memory usage for compile stage in bytes (set to -1 for no limit)',
default: -1, // no limit default: -1, // no limit
parser: parse_int, parser: parse_int,
validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`], validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
}, },
{ {
key: 'run_memory_limit', key: 'run_memory_limit',
desc: desc: 'Max memory usage for run stage in bytes (set to -1 for no limit)',
'Max memory usage for run stage in bytes (set to -1 for no limit)',
default: -1, // no limit default: -1, // no limit
parser: parse_int, parser: parse_int,
validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`], validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
@ -177,7 +182,7 @@ const options = [
desc: 'Maximum number of concurrent jobs to run at one time', desc: 'Maximum number of concurrent jobs to run at one time',
default: 64, default: 64,
parser: parse_int, parser: parse_int,
validators: [(x) => x > 0 || `${x} cannot be negative`] validators: [x => x > 0 || `${x} cannot be negative`],
}, },
{ {
key: 'limit_overrides', key: 'limit_overrides',
@ -187,10 +192,12 @@ const options = [
default: {}, default: {},
parser: parse_overrides, parser: parse_overrides,
validators: [ validators: [
(x) => !!x || `Invalid JSON format for the overrides\n${x}`, x => !!x || `Invalid JSON format for the overrides\n${x}`,
(overrides, _, options) => validate_overrides(overrides, options) || `Failed to validate the overrides` (overrides, _, options) =>
] validate_overrides(overrides, options) ||
} `Failed to validate the overrides`,
],
},
]; ];
logger.info(`Loading Configuration from environment`); logger.info(`Loading Configuration from environment`);

2
api/src/index.js
View File

@ -15,8 +15,6 @@ const logger = Logger.create('index');
const app = express(); const app = express();
expressWs(app); expressWs(app);
(async () => { (async () => {
logger.info('Setting loglevel to', config.log_level); logger.info('Setting loglevel to', config.log_level);
Logger.setLogLevel(config.log_level); Logger.setLogLevel(config.log_level);

162
api/src/job.js
View File

@ -19,15 +19,12 @@ let gid = 0;
let remainingJobSpaces = config.max_concurrent_jobs; let remainingJobSpaces = config.max_concurrent_jobs;
let jobQueue = []; let jobQueue = [];
setInterval(() => {
setInterval(()=>{
// Every 10ms try resolve a new job, if there is an available slot // Every 10ms try resolve a new job, if there is an available slot
if(jobQueue.length > 0 && remainingJobSpaces > 0){ if (jobQueue.length > 0 && remainingJobSpaces > 0) {
jobQueue.shift()() jobQueue.shift()();
} }
}, 10) }, 10);
class Job { class Job {
constructor({ runtime, files, args, stdin }) { constructor({ runtime, files, args, stdin }) {
@ -59,11 +56,11 @@ class Job {
} }
async prime() { async prime() {
if(remainingJobSpaces < 1){ if (remainingJobSpaces < 1) {
logger.info(`Awaiting job slot uuid=${this.uuid}`) logger.info(`Awaiting job slot uuid=${this.uuid}`);
await new Promise((resolve)=>{ await new Promise(resolve => {
jobQueue.push(resolve) jobQueue.push(resolve);
}) });
} }
logger.info(`Priming job uuid=${this.uuid}`); logger.info(`Priming job uuid=${this.uuid}`);
@ -79,10 +76,15 @@ class Job {
let file_path = path.join(this.dir, file.name); let file_path = path.join(this.dir, file.name);
const rel = path.relative(this.dir, file_path); const rel = path.relative(this.dir, file_path);
if(rel.startsWith("..")) if (rel.startsWith('..'))
throw Error(`File path "${file.name}" tries to escape parent directory: ${rel}`) throw Error(
`File path "${file.name}" tries to escape parent directory: ${rel}`
);
await fs.mkdir(path.dirname(file_path), {recursive: true, mode: 0o700}) await fs.mkdir(path.dirname(file_path), {
recursive: true,
mode: 0o700,
});
await fs.chown(path.dirname(file_path), this.uid, this.gid); await fs.chown(path.dirname(file_path), this.uid, this.gid);
await fs.write_file(file_path, file.content); await fs.write_file(file_path, file.content);
@ -127,34 +129,33 @@ class Job {
detached: true, //give this process its own process group detached: true, //give this process its own process group
}); });
if(eventBus === null){ if (eventBus === null) {
proc.stdin.write(this.stdin); proc.stdin.write(this.stdin);
proc.stdin.end(); proc.stdin.end();
proc.stdin.destroy(); proc.stdin.destroy();
}else{ } else {
eventBus.on("stdin", (data) => { eventBus.on('stdin', data => {
proc.stdin.write(data); proc.stdin.write(data);
}) });
eventBus.on("kill", (signal) => { eventBus.on('kill', signal => {
proc.kill(signal) proc.kill(signal);
}) });
} }
const kill_timeout = set_timeout( const kill_timeout = set_timeout(async _ => {
async _ => { logger.info(
logger.info(`Timeout exceeded timeout=${timeout} uuid=${this.uuid}`) `Timeout exceeded timeout=${timeout} uuid=${this.uuid}`
process.kill(proc.pid, 'SIGKILL')
},
timeout
); );
process.kill(proc.pid, 'SIGKILL');
}, timeout);
proc.stderr.on('data', async data => { proc.stderr.on('data', async data => {
if(eventBus !== null) { if (eventBus !== null) {
eventBus.emit("stderr", data); eventBus.emit('stderr', data);
} else if (stderr.length > this.runtime.output_max_size) { } else if (stderr.length > this.runtime.output_max_size) {
logger.info(`stderr length exceeded uuid=${this.uuid}`) logger.info(`stderr length exceeded uuid=${this.uuid}`);
process.kill(proc.pid, 'SIGKILL') process.kill(proc.pid, 'SIGKILL');
} else { } else {
stderr += data; stderr += data;
output += data; output += data;
@ -162,11 +163,11 @@ class Job {
}); });
proc.stdout.on('data', async data => { proc.stdout.on('data', async data => {
if(eventBus !== null){ if (eventBus !== null) {
eventBus.emit("stdout", data); eventBus.emit('stdout', data);
} else if (stdout.length > this.runtime.output_max_size) { } else if (stdout.length > this.runtime.output_max_size) {
logger.info(`stdout length exceeded uuid=${this.uuid}`) logger.info(`stdout length exceeded uuid=${this.uuid}`);
process.kill(proc.pid, 'SIGKILL') process.kill(proc.pid, 'SIGKILL');
} else { } else {
stdout += data; stdout += data;
output += data; output += data;
@ -179,14 +180,14 @@ class Job {
proc.stderr.destroy(); proc.stderr.destroy();
proc.stdout.destroy(); proc.stdout.destroy();
await this.cleanup_processes() await this.cleanup_processes();
logger.debug(`Finished exit cleanup uuid=${this.uuid}`) logger.debug(`Finished exit cleanup uuid=${this.uuid}`);
}; };
proc.on('exit', async (code, signal) => { proc.on('exit', async (code, signal) => {
await exit_cleanup(); await exit_cleanup();
resolve({stdout, stderr, code, signal, output }); resolve({ stdout, stderr, code, signal, output });
}); });
proc.on('error', async err => { proc.on('error', async err => {
@ -243,7 +244,7 @@ class Job {
}; };
} }
async execute_interactive(eventBus){ async execute_interactive(eventBus) {
if (this.state !== job_states.PRIMED) { if (this.state !== job_states.PRIMED) {
throw new Error( throw new Error(
'Job must be in primed state, current state: ' + 'Job must be in primed state, current state: ' +
@ -252,27 +253,27 @@ class Job {
} }
logger.info( logger.info(
`Interactively executing job uuid=${this.uuid} uid=${this.uid} gid=${ `Interactively executing job uuid=${this.uuid} uid=${
this.gid this.uid
} runtime=${this.runtime.toString()}` } gid=${this.gid} runtime=${this.runtime.toString()}`
); );
if(this.runtime.compiled){ if (this.runtime.compiled) {
eventBus.emit("stage", "compile") eventBus.emit('stage', 'compile');
const {error, code, signal} = await this.safe_call( const { error, code, signal } = await this.safe_call(
path.join(this.runtime.pkgdir, 'compile'), path.join(this.runtime.pkgdir, 'compile'),
this.files.map(x => x.name), this.files.map(x => x.name),
this.runtime.timeouts.compile, this.runtime.timeouts.compile,
this.runtime.memory_limits.compile, this.runtime.memory_limits.compile,
eventBus eventBus
) );
eventBus.emit("exit", "compile", {error, code, signal}) eventBus.emit('exit', 'compile', { error, code, signal });
} }
logger.debug('Running'); logger.debug('Running');
eventBus.emit("stage", "run") eventBus.emit('stage', 'run');
const {error, code, signal} = await this.safe_call( const { error, code, signal } = await this.safe_call(
path.join(this.runtime.pkgdir, 'run'), path.join(this.runtime.pkgdir, 'run'),
[this.files[0].name, ...this.args], [this.files[0].name, ...this.args],
this.runtime.timeouts.run, this.runtime.timeouts.run,
@ -280,47 +281,50 @@ class Job {
eventBus eventBus
); );
eventBus.emit("exit", "run", {error, code, signal}) eventBus.emit('exit', 'run', { error, code, signal });
this.state = job_states.EXECUTED; this.state = job_states.EXECUTED;
} }
async cleanup_processes(dont_wait = []) { async cleanup_processes(dont_wait = []) {
let processes = [1]; let processes = [1];
logger.debug(`Cleaning up processes uuid=${this.uuid}`) logger.debug(`Cleaning up processes uuid=${this.uuid}`);
while (processes.length > 0) { while (processes.length > 0) {
processes = [] processes = [];
const proc_ids = await fs.readdir('/proc');
const proc_ids = await fs.readdir("/proc"); processes = await Promise.all(
proc_ids.map(async proc_id => {
if (isNaN(proc_id)) return -1;
try {
const proc_status = await fs.read_file(
path.join('/proc', proc_id, 'status')
);
const proc_lines = proc_status.to_string().split('\n');
const uid_line = proc_lines.find(line =>
line.starts_with('Uid:')
);
const [_, ruid, euid, suid, fuid] =
uid_line.split(/\s+/);
if (ruid == this.uid || euid == this.uid)
processes = await Promise.all(proc_ids.map(async (proc_id) => { return parse_int(proc_id);
if(isNaN(proc_id)) return -1; } catch {
try{ return -1;
const proc_status = await fs.read_file(path.join("/proc",proc_id,"status"));
const proc_lines = proc_status.to_string().split("\n")
const uid_line = proc_lines.find(line=>line.starts_with("Uid:"))
const [_, ruid, euid, suid, fuid] = uid_line.split(/\s+/);
if(ruid == this.uid || euid == this.uid)
return parse_int(proc_id)
}catch{
return -1
} }
return -1 return -1;
})) })
);
processes = processes.filter(p => p > 0)
if(processes.length > 0)
logger.debug(`Got processes to kill: ${processes} uuid=${this.uuid}`)
processes = processes.filter(p => p > 0);
if (processes.length > 0)
logger.debug(
`Got processes to kill: ${processes} uuid=${this.uuid}`
);
for (const proc of processes) { for (const proc of processes) {
// First stop the processes, but keep their resources allocated so they cant re-fork // First stop the processes, but keep their resources allocated so they cant re-fork
@ -339,12 +343,11 @@ class Job {
// Could already be dead and just needs to be waited on // Could already be dead and just needs to be waited on
} }
if(!dont_wait.includes(proc)) if (!dont_wait.includes(proc)) wait_pid(proc);
wait_pid(proc);
} }
} }
logger.debug(`Cleaned up processes uuid=${this.uuid}`) logger.debug(`Cleaned up processes uuid=${this.uuid}`);
} }
async cleanup_filesystem() { async cleanup_filesystem() {
@ -382,7 +385,6 @@ class Job {
} }
} }
module.exports = { module.exports = {
Job, Job,
}; };

3
api/src/package.js
View File

@ -74,10 +74,9 @@ class Package {
await new Promise((resolve, reject) => { await new Promise((resolve, reject) => {
read_stream.on('data', chunk => hash.update(chunk)); read_stream.on('data', chunk => hash.update(chunk));
read_stream.on('end', () => resolve()); read_stream.on('end', () => resolve());
read_stream.on('error', error => reject(error)) read_stream.on('error', error => reject(error));
}); });
const cs = hash.digest('hex'); const cs = hash.digest('hex');
if (cs !== this.checksum) { if (cs !== this.checksum) {

101
api/src/runtime.js
View File

@ -9,8 +9,17 @@ const runtimes = [];
class Runtime { class Runtime {
constructor({ constructor({
language, version, aliases, pkgdir, runtime, timeouts, memory_limits, max_process_count, language,
max_open_files, max_file_size, output_max_size version,
aliases,
pkgdir,
runtime,
timeouts,
memory_limits,
max_process_count,
max_open_files,
max_file_size,
output_max_size,
}) { }) {
this.language = language; this.language = language;
this.version = version; this.version = version;
@ -25,37 +34,67 @@ class Runtime {
this.output_max_size = output_max_size; this.output_max_size = output_max_size;
} }
static compute_single_limit(language_name, limit_name, language_limit_overrides) { static compute_single_limit(
language_name,
limit_name,
language_limit_overrides
) {
return ( return (
config.limit_overrides[language_name] && config.limit_overrides[language_name][limit_name] (config.limit_overrides[language_name] &&
|| language_limit_overrides && language_limit_overrides[limit_name] config.limit_overrides[language_name][limit_name]) ||
|| config[limit_name] (language_limit_overrides &&
language_limit_overrides[limit_name]) ||
config[limit_name]
); );
} }
static compute_all_limits(language_name, language_limit_overrides) { static compute_all_limits(language_name, language_limit_overrides) {
return { return {
timeouts: { timeouts: {
compile: compile: this.compute_single_limit(
this.compute_single_limit(language_name, 'compile_timeout', language_limit_overrides), language_name,
run: 'compile_timeout',
this.compute_single_limit(language_name, 'run_timeout', language_limit_overrides) language_limit_overrides
),
run: this.compute_single_limit(
language_name,
'run_timeout',
language_limit_overrides
),
}, },
memory_limits: { memory_limits: {
compile: compile: this.compute_single_limit(
this.compute_single_limit(language_name, 'compile_memory_limit', language_limit_overrides), language_name,
run: 'compile_memory_limit',
this.compute_single_limit(language_name, 'run_memory_limit', language_limit_overrides) language_limit_overrides
),
run: this.compute_single_limit(
language_name,
'run_memory_limit',
language_limit_overrides
),
}, },
max_process_count: max_process_count: this.compute_single_limit(
this.compute_single_limit(language_name, 'max_process_count', language_limit_overrides), language_name,
max_open_files: 'max_process_count',
this.compute_single_limit(language_name, 'max_open_files', language_limit_overrides), language_limit_overrides
max_file_size: ),
this.compute_single_limit(language_name, 'max_file_size', language_limit_overrides), max_open_files: this.compute_single_limit(
output_max_size: language_name,
this.compute_single_limit(language_name, 'output_max_size', language_limit_overrides), 'max_open_files',
} language_limit_overrides
),
max_file_size: this.compute_single_limit(
language_name,
'max_file_size',
language_limit_overrides
),
output_max_size: this.compute_single_limit(
language_name,
'output_max_size',
language_limit_overrides
),
};
} }
static load_package(package_dir) { static load_package(package_dir) {
@ -63,7 +102,14 @@ class Runtime {
fss.read_file_sync(path.join(package_dir, 'pkg-info.json')) fss.read_file_sync(path.join(package_dir, 'pkg-info.json'))
); );
let { language, version, build_platform, aliases, provides, limit_overrides } = info; let {
language,
version,
build_platform,
aliases,
provides,
limit_overrides,
} = info;
version = semver.parse(version); version = semver.parse(version);
if (build_platform !== globals.platform) { if (build_platform !== globals.platform) {
@ -83,7 +129,10 @@ class Runtime {
version, version,
pkgdir: package_dir, pkgdir: package_dir,
runtime: language, runtime: language,
...Runtime.compute_all_limits(lang.language, lang.limit_overrides) ...Runtime.compute_all_limits(
lang.language,
lang.limit_overrides
),
}) })
); );
}); });
@ -94,7 +143,7 @@ class Runtime {
version, version,
aliases, aliases,
pkgdir: package_dir, pkgdir: package_dir,
...Runtime.compute_all_limits(language, limit_overrides) ...Runtime.compute_all_limits(language, limit_overrides),
}) })
); );
} }

1
cli/.gitignore vendored
View File

@ -1 +0,0 @@
node_modules

213
cli/commands/execute.js
View File

@ -3,8 +3,44 @@ const path = require('path');
const chalk = require('chalk'); const chalk = require('chalk');
const WebSocket = require('ws'); const WebSocket = require('ws');
const SIGNALS = ["SIGABRT","SIGALRM","SIGBUS","SIGCHLD","SIGCLD","SIGCONT","SIGEMT","SIGFPE","SIGHUP","SIGILL","SIGINFO","SIGINT","SIGIO","SIGIOT","SIGLOST","SIGPIPE","SIGPOLL","SIGPROF","SIGPWR","SIGQUIT","SIGSEGV","SIGSTKFLT","SIGTSTP","SIGSYS","SIGTERM","SIGTRAP","SIGTTIN","SIGTTOU","SIGUNUSED","SIGURG","SIGUSR1","SIGUSR2","SIGVTALRM","SIGXCPU","SIGXFSZ","SIGWINCH"] const SIGNALS = [
'SIGABRT',
'SIGALRM',
'SIGBUS',
'SIGCHLD',
'SIGCLD',
'SIGCONT',
'SIGEMT',
'SIGFPE',
'SIGHUP',
'SIGILL',
'SIGINFO',
'SIGINT',
'SIGIO',
'SIGIOT',
'SIGLOST',
'SIGPIPE',
'SIGPOLL',
'SIGPROF',
'SIGPWR',
'SIGQUIT',
'SIGSEGV',
'SIGSTKFLT',
'SIGTSTP',
'SIGSYS',
'SIGTERM',
'SIGTRAP',
'SIGTTIN',
'SIGTTOU',
'SIGUNUSED',
'SIGURG',
'SIGUSR1',
'SIGUSR2',
'SIGVTALRM',
'SIGXCPU',
'SIGXFSZ',
'SIGWINCH',
];
exports.command = ['execute <language> <file> [args..]']; exports.command = ['execute <language> <file> [args..]'];
exports.aliases = ['run']; exports.aliases = ['run'];
@ -15,18 +51,18 @@ exports.builder = {
string: true, string: true,
desc: 'Set the version of the language to use', desc: 'Set the version of the language to use',
alias: ['l'], alias: ['l'],
default: '*' default: '*',
}, },
stdin: { stdin: {
boolean: true, boolean: true,
desc: 'Read input from stdin and pass to executor', desc: 'Read input from stdin and pass to executor',
alias: ['i'] alias: ['i'],
}, },
run_timeout: { run_timeout: {
alias: ['rt', 'r'], alias: ['rt', 'r'],
number: true, number: true,
desc: 'Milliseconds before killing run process', desc: 'Milliseconds before killing run process',
default: 3000 default: 3000,
}, },
compile_timeout: { compile_timeout: {
alias: ['ct', 'c'], alias: ['ct', 'c'],
@ -42,117 +78,126 @@ exports.builder = {
interactive: { interactive: {
boolean: true, boolean: true,
alias: ['t'], alias: ['t'],
desc: 'Run interactively using WebSocket transport' desc: 'Run interactively using WebSocket transport',
}, },
status: { status: {
boolean: true, boolean: true,
alias: ['s'], alias: ['s'],
desc: 'Output additional status to stderr' desc: 'Output additional status to stderr',
} },
}; };
async function handle_interactive(files, argv){ async function handle_interactive(files, argv) {
const ws = new WebSocket(argv.pistonUrl.replace("http", "ws") + "/api/v2/connect") const ws = new WebSocket(
argv.pistonUrl.replace('http', 'ws') + '/api/v2/connect'
);
const log_message = (process.stderr.isTTY && argv.status) ? console.error : ()=>{}; const log_message =
process.stderr.isTTY && argv.status ? console.error : () => {};
process.on("exit", ()=>{ process.on('exit', () => {
ws.close(); ws.close();
process.stdin.end(); process.stdin.end();
process.stdin.destroy(); process.stdin.destroy();
process.exit(); process.exit();
}) });
for(const signal of SIGNALS){ for (const signal of SIGNALS) {
process.on(signal, ()=>{ process.on(signal, () => {
ws.send(JSON.stringify({type: 'signal', signal})) ws.send(JSON.stringify({ type: 'signal', signal }));
}) });
} }
ws.on('open', () => {
ws.on('open', ()=>{
const request = { const request = {
type: "init", type: 'init',
language: argv.language, language: argv.language,
version: argv['language_version'], version: argv['language_version'],
files: files, files: files,
args: argv.args, args: argv.args,
compile_timeout: argv.ct, compile_timeout: argv.ct,
run_timeout: argv.rt run_timeout: argv.rt,
} };
ws.send(JSON.stringify(request)) ws.send(JSON.stringify(request));
log_message(chalk.white.bold("Connected")) log_message(chalk.white.bold('Connected'));
process.stdin.resume(); process.stdin.resume();
process.stdin.on("data", (data) => { process.stdin.on('data', data => {
ws.send(JSON.stringify({ ws.send(
type: "data", JSON.stringify({
stream: "stdin", type: 'data',
data: data.toString() stream: 'stdin',
})) data: data.toString(),
})
}) })
);
});
});
ws.on("close", (code, reason)=>{ ws.on('close', (code, reason) => {
log_message( log_message(
chalk.white.bold("Disconnected: "), chalk.white.bold('Disconnected: '),
chalk.white.bold("Reason: "), chalk.white.bold('Reason: '),
chalk.yellow(`"${reason}"`), chalk.yellow(`"${reason}"`),
chalk.white.bold("Code: "), chalk.white.bold('Code: '),
chalk.yellow(`"${code}"`), chalk.yellow(`"${code}"`)
) );
process.stdin.pause() process.stdin.pause();
}) });
ws.on('message', function(data){ ws.on('message', function (data) {
const msg = JSON.parse(data); const msg = JSON.parse(data);
switch(msg.type){ switch (msg.type) {
case "runtime": case 'runtime':
log_message(chalk.bold.white("Runtime:"), chalk.yellow(`${msg.language} ${msg.version}`))
break;
case "stage":
log_message(chalk.bold.white("Stage:"), chalk.yellow(msg.stage))
break;
case "data":
if(msg.stream == "stdout") process.stdout.write(msg.data)
else if(msg.stream == "stderr") process.stderr.write(msg.data)
else log_message(chalk.bold.red(`(${msg.stream}) `), msg.data)
break;
case "exit":
if(msg.signal === null)
log_message( log_message(
chalk.white.bold("Stage"), chalk.bold.white('Runtime:'),
chalk.yellow(`${msg.language} ${msg.version}`)
);
break;
case 'stage':
log_message(
chalk.bold.white('Stage:'),
chalk.yellow(msg.stage)
);
break;
case 'data':
if (msg.stream == 'stdout') process.stdout.write(msg.data);
else if (msg.stream == 'stderr') process.stderr.write(msg.data);
else log_message(chalk.bold.red(`(${msg.stream}) `), msg.data);
break;
case 'exit':
if (msg.signal === null)
log_message(
chalk.white.bold('Stage'),
chalk.yellow(msg.stage), chalk.yellow(msg.stage),
chalk.white.bold("exited with code"), chalk.white.bold('exited with code'),
chalk.yellow(msg.code) chalk.yellow(msg.code)
) );
else else
log_message( log_message(
chalk.white.bold("Stage"), chalk.white.bold('Stage'),
chalk.yellow(msg.stage), chalk.yellow(msg.stage),
chalk.white.bold("exited with signal"), chalk.white.bold('exited with signal'),
chalk.yellow(msg.signal) chalk.yellow(msg.signal)
) );
break; break;
default: default:
log_message(chalk.red.bold("Unknown message:"), msg) log_message(chalk.red.bold('Unknown message:'), msg);
} }
}) });
} }
async function run_non_interactively(files, argv) { async function run_non_interactively(files, argv) {
const stdin =
(argv.stdin &&
const stdin = (argv.stdin && await new Promise((resolve, _) => { (await new Promise((resolve, _) => {
let data = ''; let data = '';
process.stdin.on('data', d => data += d); process.stdin.on('data', d => (data += d));
process.stdin.on('end', _ => resolve(data)); process.stdin.on('end', _ => resolve(data));
})) || ''; }))) ||
'';
const request = { const request = {
language: argv.language, language: argv.language,
@ -161,7 +206,7 @@ async function run_non_interactively(files, argv) {
args: argv.args, args: argv.args,
stdin, stdin,
compile_timeout: argv.ct, compile_timeout: argv.ct,
run_timeout: argv.rt run_timeout: argv.rt,
}; };
let { data: response } = await argv.axios.post('/api/v2/execute', request); let { data: response } = await argv.axios.post('/api/v2/execute', request);
@ -170,13 +215,13 @@ async function run_non_interactively(files, argv) {
console.log(chalk.bold(`== ${name} ==`)); console.log(chalk.bold(`== ${name} ==`));
if (ctx.stdout) { if (ctx.stdout) {
console.log(chalk.bold(`STDOUT`)) console.log(chalk.bold(`STDOUT`));
console.log(ctx.stdout.replace(/\n/g,'\n ')) console.log(ctx.stdout.replace(/\n/g, '\n '));
} }
if (ctx.stderr) { if (ctx.stderr) {
console.log(chalk.bold(`STDERR`)) console.log(chalk.bold(`STDERR`));
console.log(ctx.stderr.replace(/\n/g,'\n ')) console.log(ctx.stderr.replace(/\n/g, '\n '));
} }
if (ctx.code) { if (ctx.code) {
@ -187,12 +232,9 @@ async function run_non_interactively(files, argv) {
} }
if (ctx.signal) { if (ctx.signal) {
console.log( console.log(chalk.bold(`Signal:`), chalk.bold.yellow(ctx.signal));
chalk.bold(`Signal:`),
chalk.bold.yellow(ctx.signal)
);
}
} }
};
if (response.compile) { if (response.compile) {
step('Compile', response.compile); step('Compile', response.compile);
@ -201,17 +243,14 @@ async function run_non_interactively(files, argv) {
step('Run', response.run); step('Run', response.run);
} }
exports.handler = async (argv) => { exports.handler = async argv => {
const files = [...(argv.files || []),argv.file] const files = [...(argv.files || []), argv.file].map(file_path => {
.map(file_path => {
return { return {
name: path.basename(file_path), name: path.basename(file_path),
content: fs.readFileSync(file_path).toString() content: fs.readFileSync(file_path).toString(),
}; };
}); });
if(argv.interactive) await handle_interactive(files, argv); if (argv.interactive) await handle_interactive(files, argv);
else await run_non_interactively(files, argv); else await run_non_interactively(files, argv);
} };

4
cli/commands/ppman.js
View File

@ -2,6 +2,4 @@ exports.command = 'ppman';
exports.aliases = ['pkg']; exports.aliases = ['pkg'];
exports.describe = 'Package Manager'; exports.describe = 'Package Manager';
exports.builder = yargs => yargs exports.builder = yargs => yargs.commandDir('ppman_commands').demandCommand();
.commandDir('ppman_commands')
.demandCommand();

22
cli/commands/ppman_commands/install.js
View File

@ -4,30 +4,31 @@ exports.command = ['install <packages...>'];
exports.aliases = ['i']; exports.aliases = ['i'];
exports.describe = 'Installs the named package'; exports.describe = 'Installs the named package';
//Splits the package into it's language and version //Splits the package into it's language and version
function split_package(package) { function split_package(package) {
[language, language_version] = package.split("=") [language, language_version] = package.split('=');
res = { res = {
language: language, language: language,
version: language_version || "*" version: language_version || '*',
}; };
return res return res;
} }
const msg_format = { const msg_format = {
color: p => `${p.language ? chalk.green.bold('✓') : chalk.red.bold('❌')} Installation ${p.language ? 'succeeded' : 'failed: ' + p.message}`, color: p =>
monochrome: p => `Installation ${p.language ? 'succeeded' : 'failed: ' + p.message}`, `${
json: JSON.stringify p.language ? chalk.green.bold('✓') : chalk.red.bold('❌')
} Installation ${p.language ? 'succeeded' : 'failed: ' + p.message}`,
monochrome: p =>
`Installation ${p.language ? 'succeeded' : 'failed: ' + p.message}`,
json: JSON.stringify,
}; };
exports.handler = async ({ axios, packages }) => { exports.handler = async ({ axios, packages }) => {
const requests = packages.map(package => split_package(package)); const requests = packages.map(package => split_package(package));
for (request of requests) { for (request of requests) {
try { try {
const install = await axios.post(`/api/v2/packages`, request); const install = await axios.post(`/api/v2/packages`, request);
console.log(msg_format.color(install.data)); console.log(msg_format.color(install.data));
@ -35,5 +36,4 @@ exports.handler = async ({ axios, packages }) => {
console.error(response.data.message); console.error(response.data.message);
} }
} }
};
}

18
cli/commands/ppman_commands/list.js
View File

@ -5,17 +5,21 @@ exports.aliases = ['l'];
exports.describe = 'Lists all available packages'; exports.describe = 'Lists all available packages';
const msg_format = { const msg_format = {
color: p => `${chalk[p.installed ? 'green':'red']('•')} ${p.language} ${p.language_version}`, color: p =>
monochrome: p => `${p.language} ${p.language_version} ${p.installed ? '(INSTALLED)': ''}`, `${chalk[p.installed ? 'green' : 'red']('•')} ${p.language} ${
json: JSON.stringify p.language_version
}`,
monochrome: p =>
`${p.language} ${p.language_version} ${
p.installed ? '(INSTALLED)' : ''
}`,
json: JSON.stringify,
}; };
exports.handler = async ({ axios }) => { exports.handler = async ({ axios }) => {
const packages = await axios.get('/api/v2/packages'); const packages = await axios.get('/api/v2/packages');
const pkg_msg = packages.data const pkg_msg = packages.data.map(msg_format.color).join('\n');
.map(msg_format.color)
.join('\n');
console.log(pkg_msg); console.log(pkg_msg);
} };

167
cli/commands/ppman_commands/spec.js
View File

@ -1,49 +1,53 @@
const chalk = require('chalk'); const chalk = require('chalk');
const fs = require('fs/promises'); const fs = require('fs/promises');
const minimatch = require("minimatch"); const minimatch = require('minimatch');
const semver = require('semver'); const semver = require('semver');
exports.command = ['spec <specfile>']; exports.command = ['spec <specfile>'];
exports.aliases = ['s']; exports.aliases = ['s'];
exports.describe = 'Install the packages described in the spec file, uninstalling packages which aren\'t in the list' exports.describe =
"Install the packages described in the spec file, uninstalling packages which aren't in the list";
function does_match(package, rule){ function does_match(package, rule) {
const nameMatch = minimatch(package.language, rule.package_selector); const nameMatch = minimatch(package.language, rule.package_selector);
const versionMatch = semver.satisfies(package.language_version, rule.version_selector) const versionMatch = semver.satisfies(
package.language_version,
rule.version_selector
);
return nameMatch && versionMatch; return nameMatch && versionMatch;
} }
exports.handler = async ({axios, specfile}) => { exports.handler = async ({ axios, specfile }) => {
const spec_contents = await fs.readFile(specfile); const spec_contents = await fs.readFile(specfile);
const spec_lines = spec_contents.toString().split("\n"); const spec_lines = spec_contents.toString().split('\n');
const rules = []; const rules = [];
for(const line of spec_lines){ for (const line of spec_lines) {
const rule = { const rule = {
_raw: line.trim(), _raw: line.trim(),
comment: false, comment: false,
package_selector: null, package_selector: null,
version_selector: null, version_selector: null,
negate: false negate: false,
}; };
if(line.starts_with("#")){ if (line.starts_with('#')) {
rule.comment = true; rule.comment = true;
}else { } else {
let l = line.trim(); let l = line.trim();
if(line.starts_with("!")){ if (line.starts_with('!')) {
rule.negate = true; rule.negate = true;
l = line.slice(1).trim(); l = line.slice(1).trim();
} }
const [pkg, ver] = l.split(" ", 2); const [pkg, ver] = l.split(' ', 2);
rule.package_selector = pkg; rule.package_selector = pkg;
rule.version_selector = ver; rule.version_selector = ver;
} }
if(rule._raw.length != 0) rules.push(rule); if (rule._raw.length != 0) rules.push(rule);
} }
const packages_req = await axios.get('/api/v2/packages'); const packages_req = await axios.get('/api/v2/packages');
@ -53,108 +57,127 @@ exports.handler = async ({axios, specfile}) => {
let ensure_packages = []; let ensure_packages = [];
for(const rule of rules){ for (const rule of rules) {
if(rule.comment) continue; if (rule.comment) continue;
const matches = []; const matches = [];
if(!rule.negate){ if (!rule.negate) {
for(const package of packages){ for (const package of packages) {
if(does_match(package, rule)) if (does_match(package, rule)) matches.push(package);
matches.push(package)
} }
const latest_matches = matches.filter( const latest_matches = matches.filter(pkg => {
pkg => {
const versions = matches const versions = matches
.filter(x=>x.language == pkg.language) .filter(x => x.language == pkg.language)
.map(x=>x.language_version).sort(semver.rcompare); .map(x => x.language_version)
return versions[0] == pkg.language_version .sort(semver.rcompare);
} return versions[0] == pkg.language_version;
); });
for(const match of latest_matches){ for (const match of latest_matches) {
if(!ensure_packages.find(pkg => pkg.language == match.language && pkg.language_version == match.language_version)) if (
ensure_packages.push(match) !ensure_packages.find(
pkg =>
pkg.language == match.language &&
pkg.language_version == match.language_version
)
)
ensure_packages.push(match);
} }
}else{ } else {
ensure_packages = ensure_packages.filter( ensure_packages = ensure_packages.filter(
pkg => !does_match(pkg, rule) pkg => !does_match(pkg, rule)
) );
} }
} }
const operations = []; const operations = [];
for(const package of ensure_packages){ for (const package of ensure_packages) {
if(!package.installed) if (!package.installed)
operations.push({ operations.push({
type: "install", type: 'install',
package: package.language, package: package.language,
version: package.language_version version: package.language_version,
}); });
} }
for(const installed_package of installed){ for (const installed_package of installed) {
if(!ensure_packages.find( if (
pkg => pkg.language == installed_package.language && !ensure_packages.find(
pkg =>
pkg.language == installed_package.language &&
pkg.language_version == installed_package.language_version pkg.language_version == installed_package.language_version
)) )
)
operations.push({ operations.push({
type: "uninstall", type: 'uninstall',
package: installed_package.language, package: installed_package.language,
version: installed_package.language_version version: installed_package.language_version,
}) });
} }
console.log(chalk.bold.yellow("Actions")) console.log(chalk.bold.yellow('Actions'));
for(const op of operations){ for (const op of operations) {
console.log((op.type == "install" ? chalk.green("Install") : chalk.red("Uninstall")) + ` ${op.package} ${op.version}`) console.log(
(op.type == 'install'
? chalk.green('Install')
: chalk.red('Uninstall')) + ` ${op.package} ${op.version}`
);
} }
if(operations.length == 0){ if (operations.length == 0) {
console.log(chalk.gray("None")) console.log(chalk.gray('None'));
} }
for(const op of operations){ for (const op of operations) {
if(op.type == "install"){ if (op.type == 'install') {
try{ try {
const install = await axios.post(`/api/v2/packages`, { const install = await axios.post(`/api/v2/packages`, {
language: op.package, language: op.package,
version: op.version version: op.version,
}); });
if(!install.data.language) if (!install.data.language)
throw new Error(install.data.message); // Go to exception handler throw new Error(install.data.message); // Go to exception handler
console.log(chalk.bold.green("Installed"), op.package, op.version) console.log(
chalk.bold.green('Installed'),
}catch(e){ op.package,
console.log(chalk.bold.red("Failed to install") + ` ${op.package} ${op.version}:`, e.message) op.version
);
} catch (e) {
console.log(
chalk.bold.red('Failed to install') +
` ${op.package} ${op.version}:`,
e.message
);
} }
} } else if (op.type == 'uninstall') {
else if(op.type == "uninstall"){ try {
try{
const install = await axios.delete(`/api/v2/packages`, { const install = await axios.delete(`/api/v2/packages`, {
data: { data: {
language: op.package, language: op.package,
version: op.version version: op.version,
} },
}); });
if(!install.data.language) if (!install.data.language)
throw new Error(install.data.message); // Go to exception handler throw new Error(install.data.message); // Go to exception handler
console.log(chalk.bold.green("Uninstalled"), op.package, op.version) console.log(
chalk.bold.green('Uninstalled'),
}catch(e){ op.package,
console.log(chalk.bold.red("Failed to uninstall") + ` ${op.package} ${op.version}:`, e.message) op.version
);
} catch (e) {
console.log(
chalk.bold.red('Failed to uninstall') +
` ${op.package} ${op.version}:`,
e.message
);
} }
} }
} }
};
}

25
cli/commands/ppman_commands/uninstall.js
View File

@ -6,31 +6,36 @@ exports.describe = 'Uninstalls the named package';
//Splits the package into it's language and version //Splits the package into it's language and version
function split_package(package) { function split_package(package) {
[language, language_version] = package.split("=") [language, language_version] = package.split('=');
res = { res = {
language: language, language: language,
version: language_version || "*" version: language_version || '*',
}; };
return res return res;
} }
const msg_format = { const msg_format = {
color: p => `${p.language ? chalk.green.bold('✓') : chalk.red.bold('❌')} Uninstallation ${p.language ? 'succeeded' : 'failed: ' + p.message}`, color: p =>
monochrome: p => `Uninstallation ${p.language ? 'succeeded' : 'failed: ' + p.message}`, `${
json: JSON.stringify p.language ? chalk.green.bold('✓') : chalk.red.bold('❌')
} Uninstallation ${p.language ? 'succeeded' : 'failed: ' + p.message}`,
monochrome: p =>
`Uninstallation ${p.language ? 'succeeded' : 'failed: ' + p.message}`,
json: JSON.stringify,
}; };
exports.handler = async ({ axios, packages }) => { exports.handler = async ({ axios, packages }) => {
const requests = packages.map(package => split_package(package)); const requests = packages.map(package => split_package(package));
for (request of requests) { for (request of requests) {
try { try {
const uninstall = await axios.delete(`/api/v2/packages`, {
const uninstall = await axios.delete(`/api/v2/packages`, { data: request }); data: request,
});
console.log(msg_format.color(uninstall.data)); console.log(msg_format.color(uninstall.data));
} catch ({ response }) { } catch ({ response }) {
console.error(response.data.message) console.error(response.data.message);
} }
} }
} };

9
cli/index.js
View File

@ -6,8 +6,8 @@ const axios_instance = argv => {
argv.axios = axios.create({ argv.axios = axios.create({
baseURL: argv['piston-url'], baseURL: argv['piston-url'],
headers: { headers: {
'Content-Type': 'application/json' 'Content-Type': 'application/json',
} },
}); });
return argv; return argv;
@ -18,12 +18,11 @@ require('yargs')(process.argv.slice(2))
alias: ['u'], alias: ['u'],
default: 'http://127.0.0.1:2000', default: 'http://127.0.0.1:2000',
desc: 'Piston API URL', desc: 'Piston API URL',
string: true string: true,
}) })
.middleware(axios_instance) .middleware(axios_instance)
.scriptName('piston') .scriptName('piston')
.commandDir('commands') .commandDir('commands')
.demandCommand() .demandCommand()
.help() .help()
.wrap(72) .wrap(72).argv;
.argv;

4
docker-compose.dev.yaml
View File

@ -1,4 +1,4 @@
version: "3.2" version: '3.2'
services: services:
api: api:
@ -19,6 +19,6 @@ services:
repo: # Local testing of packages repo: # Local testing of packages
build: repo build: repo
container_name: piston_repo container_name: piston_repo
command: ["--no-build"] # Don't build anything command: ['--no-build'] # Don't build anything
volumes: volumes:
- .:/piston - .:/piston

4
docs/configuration.md
View File

@ -124,6 +124,7 @@ Maximum size for a singular file written to disk.
Resists against large file writes to exhaust disk space. Resists against large file writes to exhaust disk space.
## Compile/Run timeouts ## Compile/Run timeouts
```yaml ```yaml
key: key:
- PISTON_COMPILE_TIMEOUT - PISTON_COMPILE_TIMEOUT
@ -133,6 +134,7 @@ key:
- PISTON_RUN_TIMEOUT - PISTON_RUN_TIMEOUT
default: 3000 default: 3000
``` ```
The maximum time that is allowed to be taken by a stage in milliseconds. The maximum time that is allowed to be taken by a stage in milliseconds.
Use -1 for unlimited time. Use -1 for unlimited time.
@ -177,7 +179,9 @@ default: {}
Per-language overrides/exceptions for the each of `max_process_count`, `max_open_files`, `max_file_size`, Per-language overrides/exceptions for the each of `max_process_count`, `max_open_files`, `max_file_size`,
`compile_memory_limit`, `run_memory_limit`, `compile_timeout`, `run_timeout`, `output_max_size`. Defined as follows: `compile_memory_limit`, `run_memory_limit`, `compile_timeout`, `run_timeout`, `output_max_size`. Defined as follows:
``` ```
PISTON_LIMIT_OVERRIDES={"c++":{"max_process_count":128}} PISTON_LIMIT_OVERRIDES={"c++":{"max_process_count":128}}
``` ```
This will give `c++` a max_process_count of 128 regardless of the configuration. This will give `c++` a max_process_count of 128 regardless of the configuration.

28
package-lock.json generated
View File

@ -2,5 +2,31 @@
"name": "piston", "name": "piston",
"lockfileVersion": 2, "lockfileVersion": 2,
"requires": true, "requires": true,
"packages": {} "packages": {
"": {
"devDependencies": {
"prettier": "2.4.1"
}
},
"node_modules/prettier": {
"version": "2.4.1",
"resolved": "https://registry.npmjs.org/prettier/-/prettier-2.4.1.tgz",
"integrity": "sha512-9fbDAXSBcc6Bs1mZrDYb3XKzDLm4EXXL9sC1LqKP5rZkT6KRr/rf9amVUcODVXgguK/isJz0d0hP72WeaKWsvA==",
"dev": true,
"bin": {
"prettier": "bin-prettier.js"
},
"engines": {
"node": ">=10.13.0"
}
}
},
"dependencies": {
"prettier": {
"version": "2.4.1",
"resolved": "https://registry.npmjs.org/prettier/-/prettier-2.4.1.tgz",
"integrity": "sha512-9fbDAXSBcc6Bs1mZrDYb3XKzDLm4EXXL9sC1LqKP5rZkT6KRr/rf9amVUcODVXgguK/isJz0d0hP72WeaKWsvA==",
"dev": true
}
}
} }

6
package.json
View File

@ -1 +1,5 @@
{} {
"devDependencies": {
"prettier": "2.4.1"
}
}

2
packages/deno/1.7.5/metadata.json vendored
View File

@ -4,7 +4,7 @@
"provides": [ "provides": [
{ {
"language": "typescript", "language": "typescript",
"aliases": ["deno-ts","deno"] "aliases": ["deno-ts", "deno"]
}, },
{ {
"language": "javascript", "language": "javascript",

2
packages/deno/1.7.5/test.deno.ts vendored
View File

@ -1 +1 @@
console.log("OK") console.log('OK');

2
packages/gcc/10.2.0/metadata.json vendored
View File

@ -3,7 +3,7 @@
"version": "10.2.0", "version": "10.2.0",
"provides": [ "provides": [
{ {
"language":"c", "language": "c",
"aliases": ["gcc"] "aliases": ["gcc"]
}, },
{ {

8
packages/mono/6.12.0/metadata.json vendored
View File

@ -8,7 +8,13 @@
}, },
{ {
"language": "basic", "language": "basic",
"aliases": ["vb", "mono-vb", "mono-basic", "visual-basic", "visual basic"] "aliases": [
"vb",
"mono-vb",
"mono-basic",
"visual-basic",
"visual basic"
]
} }
] ]
} }

2
packages/node/15.10.0/test.js vendored
View File

@ -1 +1 @@
console.log("OK") console.log('OK');

2
packages/node/16.3.0/test.js vendored
View File

@ -1 +1 @@
console.log("OK") console.log('OK');

2
packages/typescript/4.2.3/test.ts vendored
View File

@ -1 +1 @@
console.log("OK") console.log('OK');

12
piston
View File

@ -38,6 +38,7 @@ case $1 in
echo " clean-repo Remove all packages from local repo" echo " clean-repo Remove all packages from local repo"
echo " build-pkg <package> <version> Build a package" echo " build-pkg <package> <version> Build a package"
echo " rebuild Build and restart the docker container" echo " rebuild Build and restart the docker container"
echo " lint Lint the codebase using prettier"
else else
@ -53,7 +54,11 @@ case $1 in
logs) docker_compose logs -f ;; logs) docker_compose logs -f ;;
restart) docker_compose restart ;; restart) docker_compose restart ;;
start) docker_compose up -d ;; start)
rm -f .git/hooks/pre-commit
ln -s $(realpath $(dirname "$0"))/pre-commit .git/hooks/pre-commit
docker_compose up -d
;;
stop) docker_compose down ;; stop) docker_compose down ;;
bash) docker_compose exec api /bin/bash ;; bash) docker_compose exec api /bin/bash ;;
@ -75,6 +80,11 @@ case $1 in
docker build repo -t piston-repo-builder docker build repo -t piston-repo-builder
docker run --rm -v "$(realpath $(dirname "$0")):/piston" piston-repo-builder --no-server $PKGSLUG docker run --rm -v "$(realpath $(dirname "$0")):/piston" piston-repo-builder --no-server $PKGSLUG
;; ;;
lint)
npm install
npx prettier --ignore-unknown --write .
;;
*) *)
cd cli cd cli
npm i > /dev/null npm i > /dev/null

15
pre-commit Executable file
View File

@ -0,0 +1,15 @@
#!/bin/sh
echo "Linting staged files..."
npm install
FILES=$(git diff --cached --name-only --diff-filter=ACMR | sed 's| |\\ |g')
[ -z "$FILES" ] && exit 0
# Prettify all selected files
echo "$FILES" | xargs npx prettier --ignore-unknown --write
# Add back the modified/prettified files to staging
echo "$FILES" | xargs git add
exit 0

7
readme.md
View File

@ -47,9 +47,10 @@ While we are accepting pull requests for Hacktoberfest, we will reject any low-q
If we see PR abuse for Hacktoberfest, we will stop providing Hacktoberfest approval for pull requests. If we see PR abuse for Hacktoberfest, we will stop providing Hacktoberfest approval for pull requests.
We are accepting PRs for: We are accepting PRs for:
* Packages - updating package versions, adding new packages
* Documentation updates - Packages - updating package versions, adding new packages
* CLI/API improvements - please discuss these with us in the Discord first - Documentation updates
- CLI/API improvements - please discuss these with us in the Discord first
Any queries or concerns, ping @HexF#0015 in the Discord. Any queries or concerns, ping @HexF#0015 in the Discord.