Mascali
/
piston
mirror of https://github.com/engineer-man/piston.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge branch 'master' of github.com:engineer-man/piston

This commit is contained in:
Thomas Hobson 2021-05-08 12:08:30 +12:00
parent b51350489f 106d5c72c9
commit 2beb0abff7
No known key found for this signature in database
GPG Key ID: 9F1FD9D87950DB6F
15 changed files with 275 additions and 271 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.github/workflows/package-pr.yaml vendored
View File

@ -71,7 +71,7 @@ jobs:
docker run -v $(pwd)'/repo:/piston/repo' -v $(pwd)'/packages:/piston/packages' -d --name repo docker.pkg.github.com/engineer-man/piston/repo-builder --no-build
docker run --network container:repo -v $(pwd)'/data:/piston' -e PISTON_LOG_LEVEL=DEBUG -e 'PISTON_REPO_URL=http://localhost:8000/index' -d --name api docker.pkg.github.com/engineer-man/piston/api
echo Waiting for API to start..
docker run --network container:api appropriate/curl -s --retry 10 --retry-connrefused http://localhost:2000/api/v1/runtimes
docker run --network container:api appropriate/curl -s --retry 10 --retry-connrefused http://localhost:2000/api/v2/runtimes
echo Waiting for Index to start..
docker run --network container:repo appropriate/curl -s --retry 999 --retry-max-time 0 --retry-connrefused http://localhost:8000/index
@ -80,7 +80,7 @@ jobs:
sed -i 's/repo/localhost/g' repo/index
echo Listing Packages
PACKAGES_JSON=$(docker run --network container:api appropriate/curl -s http://localhost:2000/api/v1/packages)
PACKAGES_JSON=$(docker run --network container:api appropriate/curl -s http://localhost:2000/api/v2/packages)
echo $PACKAGES_JSON
echo Getting CLI ready
@ -94,7 +94,7 @@ jobs:
PKG_VERSION=$(awk -F- '{ print $2 }' <<< $package)
echo "Installing..."
docker run --network container:api appropriate/curl -sXPOST http://localhost:2000/api/v1/packages/$PKG_PATH
docker run --network container:api appropriate/curl -sXPOST http://localhost:2000/api/v2/packages/$PKG_PATH
TEST_SCRIPTS=packages/$PKG_PATH/test.*
echo "Tests: $TEST_SCRIPTS"

1
api/Dockerfile
View File

@ -27,4 +27,3 @@ RUN make -C ./src/nosocket/ all && make -C ./src/nosocket/ install
CMD [ "node", "src"]
EXPOSE 2000/tcp

290
api/src/api/v2.js
View File

@ -3,166 +3,180 @@ const router = express.Router();
const config = require('../config');
const runtime = require('../runtime');
const {Job} = require("../job");
const { Job } = require('../job');
const package = require('../package')
const logger = require('logplease').create('api/v1');
const logger = require('logplease').create('api/v2');
router.use(function(req, res, next){
if(req.method == "POST" && !req.headers['content-type'].startsWith("application/json"))
router.use((req, res, next) => {
if (['GET', 'HEAD', 'OPTIONS'].includes(req.method)) {
return next();
}
if (req.headers['content-type'] !== 'application/json') {
return res
.status(415)
.send({
message: "requests must be of type application/json"
})
message: 'requests must be of type application/json'
});
}
next();
})
router.post('/execute', async function(req, res){
const {language, version, files, stdin, args, run_timeout, compile_timeout, compile_memory_limit, run_memory_limit} = req.body;
if(!language || typeof language !== "string")
{
return res
.status(400)
.send({
message: "language is required as a string"
});
}
if(!version || typeof version !== "string")
{
return res
.status(400)
.send({
message: "version is required as a string"
});
}
if(!files || !Array.isArray(files))
{
return res
.status(400)
.send({
message: "files is required as an array"
});
}
for (const [i,file] of files.entries()) {
if(typeof file.content !== "string"){
return res
.status(400)
.send({
message: `files[${i}].content is required as a string`
});
}
}
if (compile_memory_limit) {
if (typeof compile_memory_limit !== "number") {
return res
.status(400)
.send({
message: "if specified, compile_memory_limit must be a number"
})
} else if (config.compile_memory_limit >= 0 && (compile_memory_limit > config.compile_memory_limit || compile_memory_limit < 0)) {
return res
.status(400)
.send({
message: "compile_memory_limit cannot exceed the configured limit of " + config.compile_memory_limit
})
}
}
if (run_memory_limit) {
if (typeof run_memory_limit !== "number") {
return res
.status(400)
.send({
message: "if specified, run_memory_limit must be a number"
})
} else if (config.run_memory_limit >= 0 && (run_memory_limit > config.run_memory_limit || run_memory_limit < 0)) {
return res
.status(400)
.send({
message: "run_memory_limit cannot exceed the configured limit of " + config.run_memory_limit
})
}
}
const rt = runtime.get_latest_runtime_matching_language_version(language, version);
if (rt === undefined) {
return res
.status(400)
.send({
message: `${language}-${version} runtime is unknown`
});
}
const job = new Job({
runtime: rt,
alias: language,
files: files,
args: args || [],
stdin: stdin || "",
timeouts: {
run: run_timeout || 3000,
compile: compile_timeout || 10000
},
memory_limits: {
run: run_memory_limit || config.run_memory_limit,
compile: compile_memory_limit || config.compile_memory_limit
}
});
await job.prime();
const result = await job.execute();
await job.cleanup();
return res
.status(200)
.send(result);
});
router.get('/runtimes', function(req, res){
const runtimes = runtime.map(rt => ({
language: rt.language,
version: rt.version.raw,
aliases: rt.aliases,
runtime: rt.runtime
}));
router.post('/execute', async (req, res) => {
const {
language, version,
files,
stdin, args,
run_timeout, compile_timeout,
compile_memory_limit, run_memory_limit
} = req.body;
if (!language || typeof language !== 'string') {
return res
.status(400)
.send({
message: 'language is required as a string'
});
}
if (!version || typeof version !== 'string') {
return res
.status(400)
.send({
message: 'version is required as a string'
});
}
if (!files || !Array.isArray(files)) {
return res
.status(400)
.send({
message: 'files is required as an array'
});
}
for (const [i, file] of files.entries()) {
if (typeof file.content !== 'string') {
return res
.status(400)
.send({
message: `files[${i}].content is required as a string`
});
}
}
if (compile_memory_limit) {
if (typeof compile_memory_limit !== 'number') {
return res
.status(400)
.send({
message: 'if specified, compile_memory_limit must be a number'
})
}
if (config.compile_memory_limit >= 0 && (compile_memory_limit > config.compile_memory_limit || compile_memory_limit < 0)) {
return res
.status(400)
.send({
message: 'compile_memory_limit cannot exceed the configured limit of ' + config.compile_memory_limit
})
}
}
if (run_memory_limit) {
if (typeof run_memory_limit !== 'number') {
return res
.status(400)
.send({
message: 'if specified, run_memory_limit must be a number'
})
}
if (config.run_memory_limit >= 0 && (run_memory_limit > config.run_memory_limit || run_memory_limit < 0)) {
return res
.status(400)
.send({
message: 'run_memory_limit cannot exceed the configured limit of ' + config.run_memory_limit
})
}
}
const rt = runtime.get_latest_runtime_matching_language_version(language, version);
if (rt === undefined) {
return res
.status(400)
.send({
message: `${language}-${version} runtime is unknown`
});
}
const job = new Job({
runtime: rt,
alias: language,
files: files,
args: args || [],
stdin: stdin || '',
timeouts: {
run: run_timeout || 3000,
compile: compile_timeout || 10000
},
memory_limits: {
run: run_memory_limit || config.run_memory_limit,
compile: compile_memory_limit || config.compile_memory_limit
}
});
await job.prime();
const result = await job.execute();
await job.cleanup();
return res
.status(200)
.send(result);
});
router.get('/runtimes', (req, res) => {
const runtimes = runtime
.map(rt => {
return {
language: rt.language,
version: rt.version.raw,
aliases: rt.aliases,
runtime: rt.runtime
};
});
return res
.status(200)
.send(runtimes);
});
router.get('/packages', async function(req, res){
router.get('/packages', async (req, res) => {
logger.debug('Request to list packages');
let packages = await package.get_package_list();
packages = packages
.map(pkg => {
return {
language: pkg.language,
language_version: pkg.version.raw,
installed: pkg.installed
};
});
return {
language: pkg.language,
language_version: pkg.version.raw,
installed: pkg.installed
};
});
return res
.status(200)
.send(packages);
});
router.post('/packages/:language/:version', async function(req, res){
router.post('/packages/:language/:version', async (req, res) => {
logger.debug('Request to install package');
const {language, version} = req.params;
const { language, version } = req.params;
const pkg = await package.get_package(language, version);
@ -180,7 +194,7 @@ router.post('/packages/:language/:version', async function(req, res){
return res
.status(200)
.send(response);
} catch(e) {
} catch (e) {
logger.error(`Error while installing package ${pkg.language}-${pkg.version}:`, e.message);
return res
@ -191,7 +205,7 @@ router.post('/packages/:language/:version', async function(req, res){
}
});
router.delete('/packages/:language/:version', async function(req, res){
router.delete('/packages/:language/:version', async (req, res) => {
logger.debug('Request to uninstall package');
const {language, version} = req.params;
@ -212,7 +226,7 @@ router.delete('/packages/:language/:version', async function(req, res){
return res
.status(200)
.send(response);
} catch(e) {
} catch (e) {
logger.error(`Error while uninstalling package ${pkg.language}-${pkg.version}:`, e.message);
return res
@ -223,8 +237,4 @@ router.delete('/packages/:language/:version', async function(req, res){
}
});
module.exports = router;

26
api/src/config.js
View File

@ -2,7 +2,6 @@ const fss = require('fs');
const Logger = require('logplease');
const logger = Logger.create('config');
const options = [
{
key: 'log_level',
@ -31,7 +30,7 @@ const options = [
default: 1001,
parser: parse_int,
validators: [
(x,raw) => !isNaN(x) || `${raw} is not a number`,
(x, raw) => !is_nan(x) || `${raw} is not a number`,
]
},
{
@ -40,7 +39,7 @@ const options = [
default: 1500,
parser: parse_int,
validators: [
(x,raw) => !isNaN(x) || `${raw} is not a number`,
(x, raw) => !is_nan(x) || `${raw} is not a number`,
]
},
{
@ -49,7 +48,7 @@ const options = [
default: 1001,
parser: parse_int,
validators: [
(x,raw) => !isNaN(x) || `${raw} is not a number`,
(x, raw) => !is_nan(x) || `${raw} is not a number`,
]
},
{
@ -58,7 +57,7 @@ const options = [
default: 1500,
parser: parse_int,
validators: [
(x,raw) => !isNaN(x) || `${raw} is not a number`,
(x, raw) => !is_nan(x) || `${raw} is not a number`,
]
},
{
@ -76,7 +75,7 @@ const options = [
default: 1024,
parser: parse_int,
validators: [
(x,raw) => !isNaN(x) || `${raw} is not a number`,
(x, raw) => !is_nan(x) || `${raw} is not a number`,
]
},
{
@ -85,7 +84,7 @@ const options = [
default: 64,
parser: parse_int,
validators: [
(x,raw) => !isNaN(x) || `${raw} is not a number`,
(x, raw) => !is_nan(x) || `${raw} is not a number`,
]
},
{
@ -94,7 +93,7 @@ const options = [
default: 2048,
parser: parse_int,
validators: [
(x,raw) => !isNaN(x) || `${raw} is not a number`,
(x, raw) => !is_nan(x) || `${raw} is not a number`,
]
},
{
@ -103,7 +102,7 @@ const options = [
default: 10000000, //10MB
parser: parse_int,
validators: [
(x,raw) => !isNaN(x) || `${raw} is not a number`,
(x, raw) => !is_nan(x) || `${raw} is not a number`,
]
},
{
@ -112,7 +111,7 @@ const options = [
default: -1, // no limit
parser: parse_int,
validators: [
(x,raw) => !isNaN(x) || `${raw} is not a number`,
(x, raw) => !is_nan(x) || `${raw} is not a number`,
]
},
{
@ -121,7 +120,7 @@ const options = [
default: -1, // no limit
parser: parse_int,
validators: [
(x,raw) => !isNaN(x) || `${raw} is not a number`,
(x, raw) => !is_nan(x) || `${raw} is not a number`,
]
},
{
@ -141,16 +140,14 @@ let config = {};
options.forEach(option => {
const env_key = "PISTON_" + option.key.to_upper_case();
const parser = option.parser || (x=>x);
const parser = option.parser || (x => x);
const env_val = process.env[env_key];
const parsed_val = parser(env_val);
const value = env_val || option.default;
option.validators.for_each(validator => {
let response = null;
if(env_val)
@ -174,5 +171,4 @@ if (errored) {
logger.info('Configuration successfully loaded');
module.exports = config;

12
api/src/globals.js
View File

@ -1,8 +1,8 @@
// Globals are things the user shouldn't change in config, but is good to not use inline constants for
const is_docker = require('is-docker');
const fss = require('fs');
const fs = require('fs');
const platform = `${is_docker() ? 'docker' : 'baremetal'}-${
fss.read_file_sync('/etc/os-release')
fs.read_file_sync('/etc/os-release')
.toString()
.split('\n')
.find(x => x.startsWith('ID'))
@ -18,9 +18,9 @@ module.exports = {
platform,
pkg_installed_file: '.ppman-installed', //Used as indication for if a package was installed
clean_directories: [
"/dev/shm",
"/run/lock",
"/tmp",
"/var/tmp"
'/dev/shm',
'/run/lock',
'/tmp',
'/var/tmp'
]
};

28
api/src/index.js
View File

@ -40,16 +40,20 @@ const app = express();
const pkglist = await fs.readdir(pkgdir);
const languages = await Promise.all(
pkglist.map(lang=>
fs.readdir(path.join(pkgdir,lang))
.then(x=>x.map(y=>path.join(pkgdir, lang, y)))
));
pkglist.map(lang => {
return fs
.readdir(path.join(pkgdir,lang))
.then(x => {
return x.map(y => path.join(pkgdir, lang, y))
});
})
);
const installed_languages = languages
.flat()
.filter(pkg => fss.exists_sync(path.join(pkg, globals.pkg_installed_file)));
installed_languages.forEach(pkg => runtime.load_package(pkg));
installed_languages.for_each(pkg => runtime.load_package(pkg));
logger.info('Starting API Server');
logger.debug('Constructing Express App');
@ -58,22 +62,24 @@ const app = express();
app.use(body_parser.urlencoded({ extended: true }));
app.use(body_parser.json());
app.use(function (err, req, res, next) {
app.use((err, req, res, next) => {
return res
.status(400)
.send({
stack: err.stack
})
})
});
});
logger.debug('Registering Routes');
const api_v2 = require('./api/v2')
app.use('/api/v1', api_v2);
app.use('/api/v2', api_v2);
app.use('/api/v2', api_v2);
app.use(function (req,res,next){
return res.status(404).send({message: 'Not Found'});
app.use((req, res, next) => {
return res
.status(404)
.send({message: 'Not Found'});
});
logger.debug('Calling app.listen');

51
api/src/job.js
View File

@ -1,5 +1,5 @@
const logger = require('logplease').create('job');
const {v4: uuidv4} = require('uuid');
const { v4: uuidv4 } = require('uuid');
const cp = require('child_process');
const path = require('path');
const config = require('./config');
@ -40,7 +40,6 @@ class Job {
uid %= (config.runner_uid_max - config.runner_uid_min) + 1;
gid %= (config.runner_gid_max - config.runner_gid_min) + 1;
this.state = job_states.READY;
this.dir = path.join(config.data_directory, globals.data_directories.jobs, this.uuid);
}
@ -189,12 +188,12 @@ class Job {
};
}
async cleanup_processes(){
async cleanup_processes() {
let processes = [1];
while(processes.length > 0){
processes = await new Promise((resolve, reject) => cp.execFile('ps', ['awwxo', 'pid,ruid'], function(err, stdout) {
if(err === null){
while (processes.length > 0) {
processes = await new Promise((resolve, reject) => cp.execFile('ps', ['awwxo', 'pid,ruid'], (err, stdout) => {
if (err === null) {
const lines = stdout.split('\n').slice(1); //Remove header with slice
const procs = lines.map(line => {
const [pid, ruid] = line
@ -203,55 +202,57 @@ class Job {
.map(n => parseInt(n));
return { pid, ruid }
})
resolve(procs)
}
else{
reject(error)
});
resolve(procs);
} else {
reject(error);
}
}));
processes = processes.filter(proc => proc.ruid == this.uid);
processes = processes.filter(proc => proc.ruid === this.uid);
for(const proc of processes){
for (const proc of processes) {
// First stop the processes, but keep their resources allocated so they cant re-fork
try{
try {
process.kill(proc.pid, 'SIGSTOP');
}catch{
} catch {
// Could already be dead
}
}
for(const proc of processes){
for (const proc of processes) {
// Then clear them out of the process tree
try{
try {
process.kill(proc.pid, 'SIGKILL');
}catch{
} catch {
// Could already be dead and just needs to be waited on
}
wait_pid(proc.pid);
}
}
}
async cleanup_filesystem(){
async cleanup_filesystem() {
for (const clean_path of globals.clean_directories) {
const contents = await fs.readdir(clean_path);
for (const file of contents) {
const file_path = path.join(clean_path, file);
try{
try {
const stat = await fs.stat(file_path);
if(stat.uid == this.uid)
if (stat.uid === this.uid) {
await fs.rm(file_path, { recursive: true, force: true });
}catch(e){
}
} catch (e) {
// File was somehow deleted in the time that we read the dir to when we checked the file
logger.warn(`Error removing file ${file_path}: ${e}`)
}
}
}
await fs.rm(this.dir, { recursive: true, force: true });

2
api/src/package.js
View File

@ -182,7 +182,7 @@ class Package {
});
}
static async get_package (lang, version) {
static async get_package(lang, version) {
const packages = await Package.get_package_list();
const candidates = packages

10
api/src/runtime.js
View File

@ -9,7 +9,7 @@ const runtimes = [];
class Runtime {
constructor({language, version, aliases, pkgdir, runtime}){
constructor({language, version, aliases, pkgdir, runtime}) {
this.language = language;
this.version = version;
this.aliases = aliases || [];
@ -17,7 +17,7 @@ class Runtime {
this.runtime = runtime;
}
static load_package(package_dir){
static load_package(package_dir) {
let info = JSON.parse(
fss.read_file_sync(path.join(package_dir, 'pkg-info.json'))
);
@ -32,7 +32,7 @@ class Runtime {
);
}
if(provides){
if (provides) {
// Multiple languages in 1 package
provides.forEach(lang => {
runtimes.push(new Runtime({
@ -43,7 +43,7 @@ class Runtime {
runtime: language
}));
});
}else{
} else {
runtimes.push(new Runtime({
language,
version,
@ -53,8 +53,6 @@ class Runtime {
}
logger.debug(`Package ${language}-${version} was loaded`);
}
get compiled() {

6
cli/commands/execute.js
View File

@ -8,7 +8,7 @@ exports.aliases = ['run'];
exports.describe = 'Executes file with the specified runner';
exports.builder = {
languageVersion: {
version: {
string: true,
desc: 'Set the version of the language to use',
alias: ['l'],
@ -38,7 +38,7 @@ exports.builder = {
}
};
exports.handler = async function(argv) {
exports.handler = async (argv) => {
const files = [...(argv.files || []),argv.file]
.map(file_path => {
return {
@ -63,7 +63,7 @@ exports.handler = async function(argv) {
run_timeout: argv.rt
};
let { data: response } = await argv.axios.post('/api/v1/execute', request);
let { data: response } = await argv.axios.post('/api/v2/execute', request);
const step = (name, ctx) => {
console.log(chalk.bold(`== ${name} ==`));

8
cli/commands/ppman.js
View File

@ -1,7 +1,7 @@
exports.command = 'ppman'
exports.aliases = ['pkg']
exports.describe = 'Package Manager'
exports.command = 'ppman';
exports.aliases = ['pkg'];
exports.describe = 'Package Manager';
exports.builder = yargs => yargs
.commandDir('ppman_commands')
.demandCommand()
.demandCommand();

24
cli/commands/ppman_commands/install.js
View File

@ -1,23 +1,21 @@
const chalk = require('chalk');
exports.command = ['install <language> [language-version]']
exports.aliases = ['i']
exports.describe = 'Installs the named package'
exports.command = ['install <language> [language-version]'];
exports.aliases = ['i'];
exports.describe = 'Installs the named package';
const msg_format = {
'color': p => `${p.language ? chalk.green.bold('✓') : chalk.red.bold('❌')} Installation ${p.language ? "succeeded" : "failed: " + p.message}`,
'monochrome': p => `Installation ${p.language ? "succeeded" : "failed: " + p.message}`,
'json': JSON.stringify
color: p => `${p.language ? chalk.green.bold('✓') : chalk.red.bold('❌')} Installation ${p.language ? 'succeeded' : 'failed: ' + p.message}`,
monochrome: p => `Installation ${p.language ? 'succeeded' : 'failed: ' + p.message}`,
json: JSON.stringify
};
}
exports.handler = async function({axios, language, languageVersion}){
try{
const install = await axios.post(`/api/v1/packages/${language}/${languageVersion || '*'}`)
exports.handler = async ({ axios, language, version }) => {
try {
const install = await axios.post(`/api/v2/packages/${language}/${version || '*'}`);
console.log(msg_format.color(install.data));
}catch({response}){
} catch ({ response }) {
console.error(response.data.message)
}
}

23
cli/commands/ppman_commands/list.js
View File

@ -1,22 +1,17 @@
//const fetch = require('node-fetch');
const chalk = require('chalk');
exports.command = ['list']
exports.aliases = ['l']
exports.describe = 'Lists all available packages'
exports.command = ['list'];
exports.aliases = ['l'];
exports.describe = 'Lists all available packages';
const msg_format = {
'color': p => `${chalk[p.installed ? "green":"red"]("•")} ${p.language} ${p.language_version}`,
'monochrome': p => `${p.language} ${p.language_version} ${p.installed ? "(INSTALLED)": ""}`,
'json': JSON.stringify
}
exports.handler = async function({axios}){
const packages = await axios.get('/api/v1/packages');
color: p => `${chalk[p.installed ? 'green':'red']('•')} ${p.language} ${p.language_version}`,
monochrome: p => `${p.language} ${p.language_version} ${p.installed ? '(INSTALLED)': ''}`,
json: JSON.stringify
};
exports.handler = async ({ axios }) => {
const packages = await axios.get('/api/v2/packages');
const pkg_msg = packages.data
.map(msg_format.color)

24
cli/commands/ppman_commands/uninstall.js
View File

@ -1,23 +1,21 @@
const chalk = require('chalk');
exports.command = ['uninstall <language> [language-version]']
exports.aliases = ['u']
exports.describe = 'Uninstalls the named package'
exports.command = ['uninstall <language> [language-version]'];
exports.aliases = ['u'];
exports.describe = 'Uninstalls the named package';
const msg_format = {
'color': p => `${p.language ? chalk.green.bold('✓') : chalk.red.bold('❌')} Uninstallation ${p.language ? "succeeded" : "failed: " + p.message}`,
'monochrome': p => `Uninstallation ${p.language ? "succeeded" : "failed: " + p.message}`,
'json': JSON.stringify
color: p => `${p.language ? chalk.green.bold('✓') : chalk.red.bold('❌')} Uninstallation ${p.language ? 'succeeded' : 'failed: ' + p.message}`,
monochrome: p => `Uninstallation ${p.language ? 'succeeded' : 'failed: ' + p.message}`,
json: JSON.stringify
};
}
exports.handler = async function({axios, language, languageVersion}){
try{
const uninstall = await axios.delete(`/api/v1/packages/${language}/${languageVersion || '*'}`)
exports.handler = async ({ axios, language, version }) => {
try {
const uninstall = await axios.delete(`/api/v2/packages/${language}/${version || '*'}`)
console.log(msg_format.color(uninstall.data));
}catch({response}){
} catch ({ response }) {
console.error(response.data.message)
}
}

9
cli/index.js
View File

@ -2,9 +2,12 @@
const axios = require('axios').default;
const axios_instance = function(argv){
const axios_instance = argv => {
argv.axios = axios.create({
baseURL: argv['piston-url']
baseURL: argv['piston-url'],
headers: {
'Content-Type': 'application/json'
}
});
return argv;
@ -18,7 +21,7 @@ require('yargs')(process.argv.slice(2))
string: true
})
.middleware(axios_instance)
.scriptName("piston")
.scriptName('piston')
.commandDir('commands')
.demandCommand()
.help()