From 291cbe8c50b26e55f7135e0a2207d920023b4d65 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sun, 21 Feb 2021 03:29:13 +1300 Subject: [PATCH 01/17] pkg: fix secondary rules --- packages/secondary.mk | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/secondary.mk b/packages/secondary.mk index a748fa6..2def001 100644 --- a/packages/secondary.mk +++ b/packages/secondary.mk @@ -15,8 +15,9 @@ cleanup: $(patsubst %,%/cleanup,${VERSIONS}) %/${LANGUAGE}-%.pkg.tar.gz: %/Makefile $(MAKE) -C $(shell dirname $<) + %/Makefile: @mkdir -p $(shell dirname $@) @echo 'VERSION=$(patsubst %/Makefile,%,$@)' > $@ - @echo 'NAME=${LANGUAGE}' > $@ + @echo 'NAME=${LANGUAGE}' >> $@ @echo 'include ../base.mk' >> $@ From f1c082bfa135f3078bdedf79a16d5ec2909f5d86 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sun, 21 Feb 2021 03:29:32 +1300 Subject: [PATCH 02/17] fix(python *): fix python rules --- packages/python/base.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/python/base.mk b/packages/python/base.mk index 8c3b746..722f8d5 100644 --- a/packages/python/base.mk +++ b/packages/python/base.mk @@ -15,5 +15,5 @@ ${NAME}-${VERSION}/: Python-${VERSION}/ $(MAKE) -j$(or ${MAKE_JOBS},64) -C $< DESTDIR=../$@ $(MAKE) -j$(or ${MAKE_JOBS},64) -C $< altinstall || true -${NAME}-${VERSION}.tgz: +Python-${VERSION}.tgz: curl "https://www.python.org/ftp/python/${VERSION}/$@" -o $@ \ No newline at end of file From 216451d1aa8830d4b4010f87235078887fa506d8 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sun, 21 Feb 2021 03:29:47 +1300 Subject: [PATCH 03/17] pkg: add tar.gz unpack rule --- packages/common.mk | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/common.mk b/packages/common.mk index 08fc488..3b3d201 100644 --- a/packages/common.mk +++ b/packages/common.mk @@ -31,6 +31,9 @@ pkg-info.jq: %/: %.tgz tar xzf $< +%/: %.tar.gz + tar xzf $< + .PHONY: clean clean: rm -rf $(filter-out Makefile, $(wildcard *)) From 60c004eea9f646ee61f0d6ee23a828998b465685 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sun, 21 Feb 2021 11:39:03 +1300 Subject: [PATCH 04/17] api: lint **everything** --- api/.eslintrc.json | 5 +- api/package.json | 1 + api/src/cache.js | 55 ++++++----- api/src/config.js | 158 ++++++++++++++--------------- api/src/executor/job.js | 147 ++++++++++++++------------- api/src/executor/routes.js | 34 +++---- api/src/globals.js | 32 +++--- api/src/helpers.js | 25 +++-- api/src/index.js | 124 +++++++++++------------ api/src/ppman/package.js | 198 ++++++++++++++++++------------------- api/src/ppman/repo.js | 70 ++++++------- api/src/ppman/routes.js | 118 +++++++++++----------- api/src/runtime.js | 86 ++++++++-------- api/src/state.js | 40 ++++---- api/yarn.lock | 152 +++++++++++++++++++++++++++- docker-compose.yaml | 20 ++++ packages/common.mk | 2 +- repo/.gitignore | 3 + repo/Dockerfile | 8 ++ repo/README.MD | 3 + repo/make.sh | 7 ++ repo/mkindex.sh | 26 +++++ 22 files changed, 764 insertions(+), 550 deletions(-) create mode 100644 docker-compose.yaml create mode 100644 repo/.gitignore create mode 100644 repo/Dockerfile create mode 100644 repo/README.MD create mode 100644 repo/make.sh create mode 100755 repo/mkindex.sh diff --git a/api/.eslintrc.json b/api/.eslintrc.json index 4af7bf3..579bcfc 100644 --- a/api/.eslintrc.json +++ b/api/.eslintrc.json @@ -8,6 +8,7 @@ "snakecasejs" ], "extends": "eslint:recommended", + "parser": "babel-eslint", "parserOptions": { "ecmaVersion": 12 }, @@ -27,11 +28,11 @@ ], "quotes": [ "error", - "double" + "single" ], "semi": [ "error", - "never" + "always" ], "no-unused-vars": ["error", { "argsIgnorePattern": "^_"}], "snakecasejs/snakecasejs": "warn" diff --git a/api/package.json b/api/package.json index bee046b..a530298 100644 --- a/api/package.json +++ b/api/package.json @@ -16,6 +16,7 @@ "yargs": "^16.2.0" }, "devDependencies": { + "babel-eslint": "^10.1.0", "eslint": "^7.20.0", "eslint-plugin-snakecasejs": "^2.2.0" } diff --git a/api/src/cache.js b/api/src/cache.js index 97b31a1..469623a 100644 --- a/api/src/cache.js +++ b/api/src/cache.js @@ -1,54 +1,55 @@ -const globals = require("./globals") -const logger = require("logplease").create("cache") -const fs = require("fs"), path = require("path") -const util = require("util") +const globals = require('./globals'); +const logger = require('logplease').create('cache'); +const fs = require('fs/promises'), + fss = require('fs'), + path = require('path'); -const cache = new Map() +const cache = new Map(); module.exports = { - cache_key: (context, key) => Buffer.from(`${context}-${key}`).toString("base64"), + cache_key: (context, key) => Buffer.from(`${context}-${key}`).toString('base64'), has(key){ - return cache.has(key) && cache.get(key).expiry > Date.now() + return cache.has(key) && cache.get(key).expiry > Date.now(); }, async get(key, callback, ttl=globals.cache_ttl){ - logger.debug("get:", key) + logger.debug('get:', key); if(module.exports.has(key)){ - logger.debug("hit:",key) - return cache.get(key).data + logger.debug('hit:',key); + return cache.get(key).data; } - logger.debug("miss:", key) - var data = await callback() - cache.set(key, {data, expiry: Date.now() + ttl}) - return data + logger.debug('miss:', key); + var data = await callback(); + cache.set(key, {data, expiry: Date.now() + ttl}); + return data; }, async flush(cache_dir){ - logger.info("Flushing cache") + logger.info('Flushing cache'); cache.forEach((v,k)=>{ - var file_path = path.join(cache_dir, k) + var file_path = path.join(cache_dir, k); if(v.expiry < Date.now()){ //remove from cache - cache.delete(k) + cache.delete(k); fs.stat(file_path, (err, stats)=>{ - if(err) return //ignore - probably hasn't been flushed yet + if(err) return; //ignore - probably hasn't been flushed yet if(stats.is_file()) fs.rm(file_path, (err)=>{ - if(err) logger.warn(`Couldn't clean up on-disk cache file ${k}`) - }) - }) + if(err) logger.warn(`Couldn't clean up on-disk cache file ${k}`); + }); + }); }else{ //flush to disk - fs.write_file(file_path, JSON.stringify(v),()=>{}) + fs.write_file(file_path, JSON.stringify(v),()=>{}); } - }) + }); }, async load(cache_dir){ - return util.promisify(fs.readdir)(cache_dir) + return fs.readdir(cache_dir) .then(files => Promise.all(files.map( async file => { - cache.set(file, JSON.parse(fs.read_file_sync(path.join(cache_dir,file)).toString())) + cache.set(file, JSON.parse(fss.read_file_sync(path.join(cache_dir,file)).toString())); } - ))) + ))); } -} \ No newline at end of file +}; \ No newline at end of file diff --git a/api/src/config.js b/api/src/config.js index be4dae5..a2503bf 100644 --- a/api/src/config.js +++ b/api/src/config.js @@ -1,9 +1,9 @@ -const fs = require("fs") -const yargs = require("yargs") -const hide_bin = require("yargs/helpers").hideBin //eslint-disable-line snakecasejs/snakecasejs -const Logger = require("logplease") -const logger = Logger.create("config") -const yaml = require("js-yaml") +const fss = require('fs'); +const yargs = require('yargs'); +const hide_bin = require('yargs/helpers').hideBin; //eslint-disable-line snakecasejs/snakecasejs +const Logger = require('logplease'); +const logger = Logger.create('config'); +const yaml = require('js-yaml'); const header = `# # ____ _ _ @@ -16,151 +16,151 @@ const header = `# # github.com/engineer-man/piston # -` +`; const argv = yargs(hide_bin(process.argv)) - .usage("Usage: $0 -c [config]") - .demandOption("c") //eslint-disable-line snakecasejs/snakecasejs - .option("config", { - alias: "c", - describe: "config file to load from", - default: "/piston/config.yaml" + .usage('Usage: $0 -c [config]') + .demandOption('c') //eslint-disable-line snakecasejs/snakecasejs + .option('config', { + alias: 'c', + describe: 'config file to load from', + default: '/piston/config.yaml' }) - .option("make-config", { - alias: "m", - type: "boolean", - describe: "create config file and populate defaults if it does not already exist" - }).argv + .option('make-config', { + alias: 'm', + type: 'boolean', + describe: 'create config file and populate defaults if it does not already exist' + }).argv; const options = [ { - key: "log_level", - desc: "Level of data to log", - default: "INFO", + key: 'log_level', + desc: 'Level of data to log', + default: 'INFO', /* eslint-disable snakecasejs/snakecasejs */ options: Object.values(Logger.LogLevels), validators: [x=>Object.values(Logger.LogLevels).includes(x) || `Log level ${x} does not exist`] /* eslint-enable snakecasejs/snakecasejs */ }, { - key: "bind_address", - desc: "Address to bind REST API on\nThank @Bones for the number", - default: "0.0.0.0:6969", + key: 'bind_address', + desc: 'Address to bind REST API on\nThank @Bones for the number', + default: '0.0.0.0:6969', validators: [] }, { - key: "data_directory", - desc: "Absolute path to store all piston related data at", - default: "/piston", - validators: [x=> fs.exists_sync(x) || `Directory ${x} does not exist`] + key: 'data_directory', + desc: 'Absolute path to store all piston related data at', + default: '/piston', + validators: [x=> fss.exists_sync(x) || `Directory ${x} does not exist`] }, { - key: "cache_ttl", - desc: "Time in milliseconds to keep data in cache for at a maximum", + key: 'cache_ttl', + desc: 'Time in milliseconds to keep data in cache for at a maximum', default: 60 * 60 * 1000, validators: [] }, { - key: "cache_flush_time", - desc: "Interval in milliseconds to flush cache to disk at", + key: 'cache_flush_time', + desc: 'Interval in milliseconds to flush cache to disk at', default: 90 * 60 * 1000, //90 minutes validators: [] }, { - key: "state_flush_time", - desc: "Interval in milliseconds to flush state to disk at", + key: 'state_flush_time', + desc: 'Interval in milliseconds to flush state to disk at', default: 5000, // 5 seconds (file is tiny) validators: [] }, { - key: "runner_uid_min", - desc: "Minimum uid to use for runner", + key: 'runner_uid_min', + desc: 'Minimum uid to use for runner', default: 1000, validators: [] }, { - key: "runner_uid_max", - desc: "Maximum uid to use for runner", + key: 'runner_uid_max', + desc: 'Maximum uid to use for runner', default: 1500, validators: [] }, { - key: "runner_gid_min", - desc: "Minimum gid to use for runner", + key: 'runner_gid_min', + desc: 'Minimum gid to use for runner', default: 1000, validators: [] }, { - key: "runner_gid_max", - desc: "Maximum gid to use for runner", + key: 'runner_gid_max', + desc: 'Maximum gid to use for runner', default: 1500, validators: [] } -] +]; const default_config = [ - ...header.split("\n"), + ...header.split('\n'), ...options.map(option => ` ${[ - ...option.desc.split("\n"), - option.options?("Options: " + option.options.join(", ")):"" - ].filter(x=>x.length>0).map(x=>`# ${x}`).join("\n")} + ...option.desc.split('\n'), + option.options?('Options: ' + option.options.join(', ')):'' + ].filter(x=>x.length>0).map(x=>`# ${x}`).join('\n')} ${option.key}: ${option.default} - `)].join("\n") + `)].join('\n'); -logger.info(`Loading Configuration from ${argv.config}`) -!!argv["make-config"] && logger.debug("Make configuration flag is set") +logger.info(`Loading Configuration from ${argv.config}`); +!!argv['make-config'] && logger.debug('Make configuration flag is set'); -if(!!argv["make-config"] && !fs.exists_sync(argv.config)){ - logger.info("Writing default configuration...") +if(!!argv['make-config'] && !fss.exists_sync(argv.config)){ + logger.info('Writing default configuration...'); try { - fs.write_file_sync(argv.config, default_config) + fss.write_file_sync(argv.config, default_config); } catch (err) { - logger.error("Error writing default configuration:", err.message) - process.exit(1) + logger.error('Error writing default configuration:', err.message); + process.exit(1); } } -var config = {} -logger.debug("Reading config file") +var config = {}; +logger.debug('Reading config file'); try{ - const cfg_content = fs.read_file_sync(argv.config) + const cfg_content = fss.read_file_sync(argv.config); try{ - config = yaml.load(cfg_content) + config = yaml.load(cfg_content); }catch(err){ - logger.error("Error parsing configuration file:", err.message) - process.exit(1) + logger.error('Error parsing configuration file:', err.message); + process.exit(1); } }catch(err){ - logger.error("Error reading configuration from disk:", err.message) - process.exit(1) + logger.error('Error reading configuration from disk:', err.message); + process.exit(1); } -logger.debug("Validating config entries") -var errored=false +logger.debug('Validating config entries'); +var errored=false; options.forEach(opt => { - logger.debug("Checking key",opt.key) - var cfg_val = config[opt.key] + logger.debug('Checking key',opt.key); + var cfg_val = config[opt.key]; if(cfg_val == undefined){ - errored = true - logger.error(`Config key ${opt.key} does not exist on currently loaded configuration`) - return + errored = true; + logger.error(`Config key ${opt.key} does not exist on currently loaded configuration`); + return; } opt.validators.forEach(validator => { - var response = validator(cfg_val) + var response = validator(cfg_val); if(response !== true){ - errored = true - logger.error(`Config key ${opt.key} failed validation:`, response) - return + errored = true; + logger.error(`Config key ${opt.key} failed validation:`, response); + return; } - }) -}) + }); +}); -if(errored) process.exit(1) +if(errored) process.exit(1); -logger.info("Configuration successfully loaded") +logger.info('Configuration successfully loaded'); -module.exports = config +module.exports = config; diff --git a/api/src/executor/job.js b/api/src/executor/job.js index f56f3a7..b10e3ab 100644 --- a/api/src/executor/job.js +++ b/api/src/executor/job.js @@ -1,146 +1,145 @@ -const logger = require("logplease").create("executor/job") -const { v4: uuidv4 } = require("uuid") -const cp = require("child_process") -const path = require("path") -const config = require("../config"); -const globals = require("../globals"); -const fs = require("fs"); -const util = require("util"); +const logger = require('logplease').create('executor/job'); +const { v4: uuidv4 } = require('uuid'); +const cp = require('child_process'); +const path = require('path'); +const config = require('../config'); +const globals = require('../globals'); +const fs = require('fs/promises'); + const job_states = { - READY: Symbol("Ready to be primed"), - PRIMED: Symbol("Primed and ready for execution"), - EXECUTED: Symbol("Executed and ready for cleanup") -} + READY: Symbol('Ready to be primed'), + PRIMED: Symbol('Primed and ready for execution'), + EXECUTED: Symbol('Executed and ready for cleanup') +}; var uid=0; var gid=0; class Job { constructor(runtime, files, args, stdin, timeouts, main){ - this.uuid = uuidv4() - this.runtime = runtime - this.files = files - this.args = args - this.stdin = stdin - this.timeouts = timeouts - this.main = main + this.uuid = uuidv4(); + this.runtime = runtime; + this.files = files; + this.args = args; + this.stdin = stdin; + this.timeouts = timeouts; + this.main = main; - if(!Object.keys(this.files).includes(this.main)) - throw new Error(`Main file "${this.main}" will not be written to disk`) + if(!this.files.map(f=>f.name).includes(this.main)) + throw new Error(`Main file "${this.main}" will not be written to disk`); this.uid = config.runner_uid_min + uid; this.gid = config.runner_gid_min + gid; - uid++ - gid++ + uid++; + gid++; - uid %= (config.runner_uid_max - config.runner_uid_min) + 1 - gid %= (config.runner_gid_max - config.runner_gid_min) + 1 + uid %= (config.runner_uid_max - config.runner_uid_min) + 1; + gid %= (config.runner_gid_max - config.runner_gid_min) + 1; this.state = job_states.READY; this.dir = path.join(config.data_directory, globals.data_directories.jobs, this.uuid); } async prime(){ - logger.info(`Priming job uuid=${this.uuid}`) + logger.info(`Priming job uuid=${this.uuid}`); - logger.debug("Writing files to job cache") + logger.debug('Writing files to job cache'); - await util.promisify(fs.mkdir)(this.dir, {mode:0o700}) + await fs.mkdir(this.dir, {mode:0o700}); - const files = Object.keys(this.files).map(fileName => { - var content = this.files[fileName]; - return util.promisify(fs.writeFile)(path.join(this.dir, fileName), content) - }) + const files = this.files.map(({name: file_name, content}) => { + return fs.write_file(path.join(this.dir, file_name), content); + }); - await Promise.all(files) + await Promise.all(files); - logger.debug(`Transfering ownership uid=${this.uid} gid=${this.gid}`) - await util.promisify(fs.chown)(this.dir, this.uid, this.gid) + logger.debug(`Transfering ownership uid=${this.uid} gid=${this.gid}`); + await fs.chown(this.dir, this.uid, this.gid); - const chowns = Object.keys(this.files).map(fileName => { - return util.promisify(fs.chown)(path.join(this.dir, fileName), this.uid, this.gid) - }) + const chowns = this.files.map(({name:file_name}) => { + return fs.chown(path.join(this.dir, file_name), this.uid, this.gid); + }); - await Promise.all(chowns) + await Promise.all(chowns); this.state = job_states.PRIMED; - logger.debug("Primed job") + logger.debug('Primed job'); } async execute(){ - if(this.state != job_states.PRIMED) throw new Error("Job must be in primed state, current state: " + this.state.toString()) - logger.info(`Executing job uuid=${this.uuid} uid=${this.uid} gid=${this.gid} runtime=${this.runtime.toString()}`) - logger.debug(`Compiling`) + if(this.state != job_states.PRIMED) throw new Error('Job must be in primed state, current state: ' + this.state.toString()); + logger.info(`Executing job uuid=${this.uuid} uid=${this.uid} gid=${this.gid} runtime=${this.runtime.toString()}`); + logger.debug('Compiling'); const compile = this.runtime.compiled && await new Promise((resolve, reject) => { - var stderr, stdout = ""; - const proc = cp.spawn(this.runtime.pkgdir, [this.main, ...this.args] ,{ + var stderr, stdout = ''; + const proc = cp.spawn(path.join(this.runtime.pkgdir, 'compile'), [this.main, ...this.args] ,{ env: this.runtime.env_vars, stdio: ['pipe', 'pipe', 'pipe'], cwd: this.dir, uid: this.uid, gid: this.gid - }) + }); - const killTimeout = setTimeout(proc.kill, this.timeouts.compile, "SIGKILL") + const kill_timeout = setTimeout(proc.kill, this.timeouts.compile, 'SIGKILL'); - proc.stderr.on('data', d=>stderr += d) - proc.stdout.on('data', d=>stdout += d) + proc.stderr.on('data', d=>stderr += d); + proc.stdout.on('data', d=>stdout += d); proc.on('exit', (code, signal)=>{ - clearTimeout(killTimeout); - resolve({stdout, stderr, code, signal}) - }) + clearTimeout(kill_timeout); + resolve({stdout, stderr, code, signal}); + }); proc.on('error', (code, signal) => { - clearTimeout(killTimeout); - reject({stdout, stderr, code, signal}) - }) - }) + clearTimeout(kill_timeout); + reject({stdout, stderr, code, signal}); + }); + }); - logger.debug("Running") + logger.debug('Running'); const run = await new Promise((resolve, reject) => { - var stderr, stdout = ""; - const proc = cp.spawn('bash', [path.join(this.runtime.pkgdir, "run"), this.main, ...this.args] ,{ + var stderr, stdout = ''; + const proc = cp.spawn(path.join(this.runtime.pkgdir, 'run'), [this.main, ...this.args] ,{ env: this.runtime.env_vars, stdio: ['pipe', 'pipe', 'pipe'], cwd: this.dir, uid: this.uid, gid: this.gid - }) + }); - const killTimeout = setTimeout(proc.kill, this.timeouts.run, "SIGKILL") + const kill_timeout = setTimeout(proc.kill, this.timeouts.run, 'SIGKILL'); - proc.stderr.on('data', d=>stderr += d) - proc.stdout.on('data', d=>stdout += d) + proc.stderr.on('data', d=>stderr += d); + proc.stdout.on('data', d=>stdout += d); proc.on('exit', (code, signal)=>{ - clearTimeout(killTimeout); - resolve({stdout, stderr, code, signal}) - }) + clearTimeout(kill_timeout); + resolve({stdout, stderr, code, signal}); + }); proc.on('error', (code, signal) => { - clearTimeout(killTimeout); - reject({stdout, stderr, code, signal}) - }) - }) + clearTimeout(kill_timeout); + reject({stdout, stderr, code, signal}); + }); + }); this.state = job_states.EXECUTED; return { compile, run - } + }; } async cleanup(){ - logger.info(`Cleaning up job uuid=${this.uuid}`) - await util.promisify(fs.rm)(this.dir, {recursive: true, force: true}) + logger.info(`Cleaning up job uuid=${this.uuid}`); + await fs.rm(this.dir, {recursive: true, force: true}); } } -module.exports = {Job} \ No newline at end of file +module.exports = {Job}; \ No newline at end of file diff --git a/api/src/executor/routes.js b/api/src/executor/routes.js index 7bd0ef3..07f7ace 100644 --- a/api/src/executor/routes.js +++ b/api/src/executor/routes.js @@ -1,34 +1,34 @@ // {"language":"python","version":"3.9.1","files":{"code.py":"print('hello world')"},"args":[],"stdin":"","compile_timeout":10, "run_timeout":3, "main": "code.py"} // {"success":true, "run":{"stdout":"hello world", "stderr":"", "error_code":0},"compile":{"stdout":"","stderr":"","error_code":0}} -const { get_latest_runtime_matching_language_version } = require("../runtime"); -const { Job } = require("./job"); +const { get_latest_runtime_matching_language_version } = require('../runtime'); +const { Job } = require('./job'); module.exports = { async run_job(req, res){ // POST /jobs var errored = false; - ["language", "version", - "files", "main", - "args", "stdin", - "compile_timeout", "run_timeout", - ].forEach(key => { - if(req.body[key] == undefined) errored = errored || res.json_error(`${key} is required`, 400) - }) - if(errored) return errored; + ['language', 'version', + 'files', 'main', + 'args', 'stdin', + 'compile_timeout', 'run_timeout', + ].forEach(key => { + if(req.body[key] == undefined) errored = errored || res.json_error(`${key} is required`, 400); + }); + if(errored) return errored; const runtime = get_latest_runtime_matching_language_version(req.body.language, req.body.version); - if(runtime == undefined) return res.json_error(`${req.body.language}-${req.body.version} runtime is unknown`, 400) + if(runtime == undefined) return res.json_error(`${req.body.language}-${req.body.version} runtime is unknown`, 400); - const job = new Job(runtime, req.body.files, req.body.args, req.body.stdin, {run: req.body.run_timeout, compile: req.body.compile_timeout}, req.body.main) - await job.prime() + const job = new Job(runtime, req.body.files, req.body.args, req.body.stdin, {run: req.body.run_timeout, compile: req.body.compile_timeout}, req.body.main); + await job.prime(); - const result = await job.execute() - res.json_success(result) + const result = await job.execute(); + res.json_success(result); - await job.cleanup() + await job.cleanup(); } -} \ No newline at end of file +}; \ No newline at end of file diff --git a/api/src/globals.js b/api/src/globals.js index 55f376d..09ccb2e 100644 --- a/api/src/globals.js +++ b/api/src/globals.js @@ -1,25 +1,25 @@ // Globals are things the user shouldn't change in config, but is good to not use inline constants for -const is_docker = require("is-docker") -const fs = require("fs") -const platform = `${is_docker() ? "docker" : "baremetal"}-${ - fs.read_file_sync("/etc/os-release") +const is_docker = require('is-docker'); +const fss = require('fs'); +const platform = `${is_docker() ? 'docker' : 'baremetal'}-${ + fss.read_file_sync('/etc/os-release') .toString() - .split("\n") - .find(x=>x.startsWith("ID")) - .replace("ID=","") -}` + .split('\n') + .find(x=>x.startsWith('ID')) + .replace('ID=','') +}`; module.exports = { data_directories: { - cache: "cache", - packages: "packages", - runtimes: "runtimes", - jobs: "jobs" + cache: 'cache', + packages: 'packages', + runtimes: 'runtimes', + jobs: 'jobs' }, data_files:{ - state: "state.json" + state: 'state.json' }, - version: require("../package.json").version, + version: require('../package.json').version, platform, - pkg_installed_file: ".ppman-installed" //Used as indication for if a package was installed -} \ No newline at end of file + pkg_installed_file: '.ppman-installed' //Used as indication for if a package was installed +}; \ No newline at end of file diff --git a/api/src/helpers.js b/api/src/helpers.js index 61f6483..0a67c5b 100644 --- a/api/src/helpers.js +++ b/api/src/helpers.js @@ -1,34 +1,33 @@ -const fs = require("fs"), - path= require("path"), - util = require("util"), - fetch = require("node-fetch"), - urlp = require("url") +const fs = require('fs/promises'), + path= require('path'), + fetch = require('node-fetch'), + urlp = require('url'); module.exports = { async buffer_from_u_r_l(url){ if(!(url instanceof URL)) - url = new URL(url) - if(url.protocol == "file:"){ + url = new URL(url); + if(url.protocol == 'file:'){ //eslint-disable-next-line snakecasejs/snakecasejs - return await util.promisify(fs.read_file)(urlp.fileURLToPath(url)) + return await fs.read_file(urlp.fileURLToPath(url)); }else{ return await fetch({ url: url.toString() - }) + }); } }, add_url_base_if_required(url, base){ try{ - return new URL(url) + return new URL(url); }catch{ //Assume this is a file name - return new URL(url, base + "/") + return new URL(url, base + '/'); } }, url_basename(url){ - return path.basename(url.pathname) + return path.basename(url.pathname); }, -} \ No newline at end of file +}; \ No newline at end of file diff --git a/api/src/index.js b/api/src/index.js index b9d8fca..873b66a 100644 --- a/api/src/index.js +++ b/api/src/index.js @@ -1,106 +1,106 @@ #!/usr/bin/env node -require("nocamel") -const Logger = require("logplease") -const express = require("express") -const globals = require("./globals") -const config = require("./config") -const cache = require("./cache") -const state = require("./state") -const path = require("path") -const fs = require("fs") -const util = require("util") -const body_parser = require("body-parser") -const runtime = require("./runtime") +require('nocamel'); +const Logger = require('logplease'); +const express = require('express'); +const globals = require('./globals'); +const config = require('./config'); +const cache = require('./cache'); +const state = require('./state'); +const path = require('path'); +const fs = require('fs/promises'); +const fss = require('fs'); +const body_parser = require('body-parser'); +const runtime = require('./runtime'); -const logger = Logger.create("index") +const logger = Logger.create('index'); const app = express(); (async () => { - logger.info("Setting loglevel to",config.log_level) - Logger.setLogLevel(config.log_level) //eslint-disable-line snakecasejs/snakecasejs + logger.info('Setting loglevel to',config.log_level); + Logger.setLogLevel(config.log_level); //eslint-disable-line snakecasejs/snakecasejs - logger.debug("Ensuring data directories exist") + logger.debug('Ensuring data directories exist'); Object.values(globals.data_directories).forEach(dir => { - var data_path = path.join(config.data_directory, dir) - logger.debug(`Ensuring ${data_path} exists`) - if(!fs.exists_sync(data_path)){ - logger.info(`${data_path} does not exist.. Creating..`) + var data_path = path.join(config.data_directory, dir); + logger.debug(`Ensuring ${data_path} exists`); + if(!fss.exists_sync(data_path)){ + logger.info(`${data_path} does not exist.. Creating..`); try{ - fs.mkdir_sync(data_path) + fss.mkdir_sync(data_path); }catch(err){ - logger.error(`Failed to create ${data_path}: `, err.message) + logger.error(`Failed to create ${data_path}: `, err.message); } } - }) + }); - logger.info("Loading state") - await state.load(path.join(config.data_directory,globals.data_files.state)) + logger.info('Loading state'); + await state.load(path.join(config.data_directory,globals.data_files.state)); - logger.info("Loading cache") - await cache.load(path.join(config.data_directory,globals.data_directories.cache)) + logger.info('Loading cache'); + await cache.load(path.join(config.data_directory,globals.data_directories.cache)); - logger.info("Loading packages") - const pkgdir = path.join(config.data_directory,globals.data_directories.packages) - await util.promisify(fs.readdir)(pkgdir) + logger.info('Loading packages'); + const pkgdir = path.join(config.data_directory,globals.data_directories.packages); + await fs.readdir(pkgdir) .then(langs => Promise.all( langs.map(lang=> - util.promisify(fs.readdir)(path.join(pkgdir,lang)) + fs.readdir(path.join(pkgdir,lang)) .then(x=>x.map(y=>path.join(pkgdir, lang, y))) ))) //eslint-disable-next-line snakecasejs/snakecasejs .then(pkgs=>pkgs.flat().filter(pkg=>fs.existsSync(path.join(pkg, globals.pkg_installed_file)))) - .then(pkgs=>pkgs.forEach(pkg => new runtime.Runtime(pkg))) + .then(pkgs=>pkgs.forEach(pkg => new runtime.Runtime(pkg))); - logger.info("Starting API Server") + logger.info('Starting API Server'); - logger.debug("Constructing Express App") + logger.debug('Constructing Express App'); - logger.debug("Registering middleware") + logger.debug('Registering middleware'); - app.use(body_parser.urlencoded({extended: true})) - app.use(body_parser.json()) + app.use(body_parser.urlencoded({extended: true})); + app.use(body_parser.json()); - logger.debug("Registering custom message wrappers") + logger.debug('Registering custom message wrappers'); express.response.json_error = function(message, code) { - this.status(code) - return this.json({success: false, message, code}) - } + this.status(code); + return this.json({success: false, message, code}); + }; express.response.json_success = function(obj) { - return this.json({success: true, data: obj}) - } + return this.json({success: true, data: obj}); + }; - logger.debug("Registering Routes") + logger.debug('Registering Routes'); - const ppman_routes = require("./ppman/routes") + const ppman_routes = require('./ppman/routes'); - app.get ("/repos", ppman_routes.repo_list) - app.post ("/repos", ppman_routes.repo_add) - app.get ("/repos/:repo_slug", ppman_routes.repo_info) - app.get ("/repos/:repo_slug/packages", ppman_routes.repo_packages) - app.get ("/repos/:repo_slug/packages/:language/:version", ppman_routes.package_info) - app.post ("/repos/:repo_slug/packages/:language/:version", ppman_routes.package_install) - app.delete("/repos/:repo_slug/packages/:language/:version", ppman_routes.package_uninstall) //TODO + app.get ('/repos', ppman_routes.repo_list); + app.post ('/repos', ppman_routes.repo_add); + app.get ('/repos/:repo_slug', ppman_routes.repo_info); + app.get ('/repos/:repo_slug/packages', ppman_routes.repo_packages); + app.get ('/repos/:repo_slug/packages/:language/:version', ppman_routes.package_info); + app.post ('/repos/:repo_slug/packages/:language/:version', ppman_routes.package_install); + app.delete('/repos/:repo_slug/packages/:language/:version', ppman_routes.package_uninstall); //TODO - const executor_routes = require('./executor/routes') - app.post ("/jobs", executor_routes.run_job) + const executor_routes = require('./executor/routes'); + app.post ('/jobs', executor_routes.run_job); - logger.debug("Calling app.listen") - const [address,port] = config.bind_address.split(":") + logger.debug('Calling app.listen'); + const [address,port] = config.bind_address.split(':'); app.listen(port, address, ()=>{ - logger.info("API server started on", config.bind_address) - }) + logger.info('API server started on', config.bind_address); + }); - logger.debug("Setting up flush timers") - setInterval(cache.flush,config.cache_flush_time,path.join(config.data_directory,globals.data_directories.cache)) - setInterval(state.save,config.state_flush_time,path.join(config.data_directory,globals.data_files.state)) -})() \ No newline at end of file + logger.debug('Setting up flush timers'); + setInterval(cache.flush,config.cache_flush_time,path.join(config.data_directory,globals.data_directories.cache)); + setInterval(state.save,config.state_flush_time,path.join(config.data_directory,globals.data_files.state)); +})(); \ No newline at end of file diff --git a/api/src/ppman/package.js b/api/src/ppman/package.js index e3e69cc..5ccdf4d 100644 --- a/api/src/ppman/package.js +++ b/api/src/ppman/package.js @@ -1,170 +1,170 @@ -const logger = require("logplease").create("ppman/package") -const semver = require("semver") -const config = require("../config") -const globals = require("../globals") -const helpers = require("../helpers") -const path = require("path") -const fs = require("fs") -const util = require("util") -const cp = require("child_process") -const crypto = require("crypto") -const runtime = require("../runtime") +const logger = require('logplease').create('ppman/package'); +const semver = require('semver'); +const config = require('../config'); +const globals = require('../globals'); +const helpers = require('../helpers'); +const path = require('path'); +const fs = require('fs/promises'); +const fss = require('fs'); +const cp = require('child_process'); +const crypto = require('crypto'); +const runtime = require('../runtime'); class Package { constructor(repo, {author, language, version, checksums, dependencies, size, buildfile, download, signature}){ - this.author = author - this.language = language - this.version = semver.parse(version) - this.checksums = checksums - this.dependencies = dependencies - this.size = size - this.buildfile = buildfile - this.download = download - this.signature = signature + this.author = author; + this.language = language; + this.version = semver.parse(version); + this.checksums = checksums; + this.dependencies = dependencies; + this.size = size; + this.buildfile = buildfile; + this.download = download; + this.signature = signature; - this.repo = repo + this.repo = repo; } get installed(){ - return fs.exists_sync(path.join(this.install_path, globals.pkg_installed_file)) + return fss.exists_sync(path.join(this.install_path, globals.pkg_installed_file)); } get download_url(){ - return helpers.add_url_base_if_required(this.download, this.repo.base_u_r_l) + return helpers.add_url_base_if_required(this.download, this.repo.base_u_r_l); } get install_path(){ return path.join(config.data_directory, globals.data_directories.packages, this.language, - this.version.raw) + this.version.raw); } async install(){ - if(this.installed) throw new Error("Already installed") - logger.info(`Installing ${this.language}-${this.version.raw}`) + if(this.installed) throw new Error('Already installed'); + logger.info(`Installing ${this.language}-${this.version.raw}`); - if(fs.exists_sync(this.install_path)){ - logger.warn(`${this.language}-${this.version.raw} has residual files. Removing them.`) - await util.promisify(fs.rm)(this.install_path, {recursive: true, force: true}) + if(fss.exists_sync(this.install_path)){ + logger.warn(`${this.language}-${this.version.raw} has residual files. Removing them.`); + await fs.rm(this.install_path, {recursive: true, force: true}); } - logger.debug(`Making directory ${this.install_path}`) - await util.promisify(fs.mkdir)(this.install_path, {recursive: true}) + logger.debug(`Making directory ${this.install_path}`); + await fs.mkdir(this.install_path, {recursive: true}); - logger.debug(`Downloading package from ${this.download_url} in to ${this.install_path}`) - const pkgfile = helpers.url_basename(this.download_url) - const pkgpath = path.join(this.install_path, pkgfile) + logger.debug(`Downloading package from ${this.download_url} in to ${this.install_path}`); + const pkgfile = helpers.url_basename(this.download_url); + const pkgpath = path.join(this.install_path, pkgfile); await helpers.buffer_from_u_r_l(this.download_url) - .then(buf=> util.promisify(fs.write_file)(pkgpath, buf)) + .then(buf=> fs.write_file(pkgpath, buf)); - logger.debug("Validating checksums") + logger.debug('Validating checksums'); Object.keys(this.checksums).forEach(algo => { - var val = this.checksums[algo] - logger.debug(`Assert ${algo}(${pkgpath}) == ${val}`) + var val = this.checksums[algo]; + logger.debug(`Assert ${algo}(${pkgpath}) == ${val}`); var cs = crypto.create_hash(algo) - .update(fs.read_file_sync(pkgpath)) - .digest("hex") - if(cs != val) throw new Error(`Checksum miss-match want: ${val} got: ${cs}`) - }) + .update(fss.read_file_sync(pkgpath)) + .digest('hex'); + if(cs != val) throw new Error(`Checksum miss-match want: ${val} got: ${cs}`); + }); - await this.repo.importKeys() + await this.repo.import_keys(); - logger.debug("Validating signatutes") + logger.debug('Validating signatutes'); await new Promise((resolve,reject)=>{ - const gpgspawn = cp.spawn("gpg", ["--verify", "-", pkgpath], { - stdio: ["pipe", "ignore", "ignore"] - }) + const gpgspawn = cp.spawn('gpg', ['--verify', '-', pkgpath], { + stdio: ['pipe', 'ignore', 'ignore'] + }); - gpgspawn.once("exit", (code, _) => { - if(code == 0) resolve() - else reject(new Error("Invalid signature")) - }) + gpgspawn.once('exit', (code, _) => { + if(code == 0) resolve(); + else reject(new Error('Invalid signature')); + }); - gpgspawn.once("error", reject) + gpgspawn.once('error', reject); - gpgspawn.stdin.write(this.signature) - gpgspawn.stdin.end() + gpgspawn.stdin.write(this.signature); + gpgspawn.stdin.end(); - }) + }); - logger.debug(`Extracting package files from archive ${pkgfile} in to ${this.install_path}`) + logger.debug(`Extracting package files from archive ${pkgfile} in to ${this.install_path}`); await new Promise((resolve, reject)=>{ - const proc = cp.exec(`bash -c 'cd "${this.install_path}" && tar xzf ${pkgfile}'`) - proc.once("exit", (code,_)=>{ - if(code == 0) resolve() - else reject(new Error("Failed to extract package")) - }) - proc.stdout.pipe(process.stdout) - proc.stderr.pipe(process.stderr) + const proc = cp.exec(`bash -c 'cd "${this.install_path}" && tar xzf ${pkgfile}'`); + proc.once('exit', (code,_)=>{ + if(code == 0) resolve(); + else reject(new Error('Failed to extract package')); + }); + proc.stdout.pipe(process.stdout); + proc.stderr.pipe(process.stderr); - proc.once("error", reject) - }) + proc.once('error', reject); + }); - logger.debug("Ensuring binary files exist for package") - const pkgbin = path.join(this.install_path, `${this.language}-${this.version.raw}`) + logger.debug('Ensuring binary files exist for package'); + const pkgbin = path.join(this.install_path, `${this.language}-${this.version.raw}`); try{ - const pkgbinstat = await util.promisify(fs.stat)(pkgbin) + const pkgbinstat = await fs.stat(pkgbin); //eslint-disable-next-line snakecasejs/snakecasejs - if(!pkgbinstat.isDirectory()) throw new Error() + if(!pkgbinstat.isDirectory()) throw new Error(); }catch(err){ - throw new Error(`Invalid package: could not find ${this.language}-${this.version.raw}/ contained within package files`) + throw new Error(`Invalid package: could not find ${this.language}-${this.version.raw}/ contained within package files`); } - logger.debug("Symlinking into runtimes") - await util.promisify(fs.symlink)( + logger.debug('Symlinking into runtimes'); + await fs.symlink( pkgbin, path.join(config.data_directory, globals.data_directories.runtimes, `${this.language}-${this.version.raw}`) - ).catch((err)=>err) //catch + ).catch((err)=>err); //catch - logger.debug("Registering runtime") - const pkgruntime = new runtime.Runtime(this.install_path) + logger.debug('Registering runtime'); + const pkgruntime = new runtime.Runtime(this.install_path); - logger.debug("Caching environment") - const required_pkgs = [pkgruntime, ...pkgruntime.get_all_dependencies()] + logger.debug('Caching environment'); + const required_pkgs = [pkgruntime, ...pkgruntime.get_all_dependencies()]; const get_env_command = [...required_pkgs.map(p=>`cd "${p.runtime_dir}"; source environment; `), - "env" ].join(" ") + 'env' ].join(' '); const envout = await new Promise((resolve, reject)=>{ - var stdout = "" - const proc = cp.spawn("env",["-i","bash","-c",`${get_env_command}`], { - stdio: ["ignore", "pipe", "pipe"]}) - proc.once("exit", (code,_)=>{ - if(code == 0) resolve(stdout) - else reject(new Error("Failed to cache environment")) - }) + var stdout = ''; + const proc = cp.spawn('env',['-i','bash','-c',`${get_env_command}`], { + stdio: ['ignore', 'pipe', 'pipe']}); + proc.once('exit', (code,_)=>{ + if(code == 0) resolve(stdout); + else reject(new Error('Failed to cache environment')); + }); - proc.stdout.on("data", (data)=>{ - stdout += data - }) + proc.stdout.on('data', (data)=>{ + stdout += data; + }); - proc.once("error", reject) - }) + proc.once('error', reject); + }); - const filtered_env = envout.split("\n") - .filter(l=>!["PWD","OLDPWD","_", "SHLVL"].includes(l.split("=",2)[0])) - .join("\n") + const filtered_env = envout.split('\n') + .filter(l=>!['PWD','OLDPWD','_', 'SHLVL'].includes(l.split('=',2)[0])) + .join('\n'); - await util.promisify(fs.write_file)(path.join(this.install_path, ".env"), filtered_env) + await fs.write_file(path.join(this.install_path, '.env'), filtered_env); - logger.debug("Writing installed state to disk") - await util.promisify(fs.write_file)(path.join(this.install_path, globals.pkg_installed_file), Date.now().toString()) + logger.debug('Writing installed state to disk'); + await fs.write_file(path.join(this.install_path, globals.pkg_installed_file), Date.now().toString()); - logger.info(`Installed ${this.language}-${this.version.raw}`) + logger.info(`Installed ${this.language}-${this.version.raw}`); return { language: this.language, version: this.version.raw - } + }; } } -module.exports = {Package} \ No newline at end of file +module.exports = {Package}; \ No newline at end of file diff --git a/api/src/ppman/repo.js b/api/src/ppman/repo.js index 77fb8e3..fc6378c 100644 --- a/api/src/ppman/repo.js +++ b/api/src/ppman/repo.js @@ -1,66 +1,66 @@ -const logger = require("logplease").create("ppman/repo") -const cache = require("../cache") -const CACHE_CONTEXT = "repo" +const logger = require('logplease').create('ppman/repo'); +const cache = require('../cache'); +const CACHE_CONTEXT = 'repo'; -const cp = require("child_process") -const yaml = require("js-yaml") -const { Package } = require("./package") -const helpers = require("../helpers") +const cp = require('child_process'); +const yaml = require('js-yaml'); +const { Package } = require('./package'); +const helpers = require('../helpers'); class Repository { constructor(slug, url){ - this.slug = slug - this.url = new URL(url) - this.keys = [] - this.packages = [] - this.base_u_r_l="" - logger.debug(`Created repo slug=${this.slug} url=${this.url}`) + this.slug = slug; + this.url = new URL(url); + this.keys = []; + this.packages = []; + this.base_u_r_l=''; + logger.debug(`Created repo slug=${this.slug} url=${this.url}`); } get cache_key(){ - return cache.cache_key(CACHE_CONTEXT, this.slug) + return cache.cache_key(CACHE_CONTEXT, this.slug); } async load(){ try{ var index = await cache.get(this.cache_key,async ()=>{ - return helpers.buffer_from_u_r_l(this.url) - }) + return helpers.buffer_from_u_r_l(this.url); + }); - var repo = yaml.load(index) - if(repo.schema != "ppman-repo-1"){ - throw new Error("YAML Schema unknown") + var repo = yaml.load(index); + if(repo.schema != 'ppman-repo-1'){ + throw new Error('YAML Schema unknown'); } - this.keys = repo.keys - this.packages = repo.packages.map(pkg => new Package(this, pkg)) - this.base_u_r_l = repo.baseurl + this.keys = repo.keys; + this.packages = repo.packages.map(pkg => new Package(this, pkg)); + this.base_u_r_l = repo.baseurl; }catch(err){ - logger.error(`Failed to load repository ${this.slug}:`,err.message) + logger.error(`Failed to load repository ${this.slug}:`,err.message); } } - async importKeys(){ + async import_keys(){ await this.load(); - logger.info(`Importing keys for repo ${this.slug}`) + logger.info(`Importing keys for repo ${this.slug}`); await new Promise((resolve,reject)=>{ - const gpgspawn = cp.spawn("gpg", ['--receive-keys', this.keys], { - stdio: ["ignore", "ignore", "ignore"] - }) + const gpgspawn = cp.spawn('gpg', ['--receive-keys', this.keys], { + stdio: ['ignore', 'ignore', 'ignore'] + }); - gpgspawn.once("exit", (code, _) => { - if(code == 0) resolve() - else reject(new Error("Failed to import keys")) - }) + gpgspawn.once('exit', (code, _) => { + if(code == 0) resolve(); + else reject(new Error('Failed to import keys')); + }); - gpgspawn.once("error", reject) + gpgspawn.once('error', reject); - }) + }); } } -module.exports = {Repository} \ No newline at end of file +module.exports = {Repository}; \ No newline at end of file diff --git a/api/src/ppman/routes.js b/api/src/ppman/routes.js index 13818f4..c376172 100644 --- a/api/src/ppman/routes.js +++ b/api/src/ppman/routes.js @@ -1,82 +1,82 @@ -const repos = new Map() -const state = require("../state") -const logger = require("logplease").create("ppman/routes") -const {Repository} = require("./repo") -const semver = require("semver") +const repos = new Map(); +const state = require('../state'); +const logger = require('logplease').create('ppman/routes'); +const {Repository} = require('./repo'); +const semver = require('semver'); async function get_or_construct_repo(slug){ - if(repos.has(slug))return repos.get(slug) - if(state.state.get("repositories").has(slug)){ - const repo_url = state.state.get("repositories").get(slug) - const repo = new Repository(slug, repo_url) - await repo.load() - repos.set(slug, repo) - return repo + if(repos.has(slug))return repos.get(slug); + if(state.state.get('repositories').has(slug)){ + const repo_url = state.state.get('repositories').get(slug); + const repo = new Repository(slug, repo_url); + await repo.load(); + repos.set(slug, repo); + return repo; } - logger.warn(`Requested repo ${slug} does not exist`) - return null + logger.warn(`Requested repo ${slug} does not exist`); + return null; } async function get_package(repo, lang, version){ var candidates = repo.packages.filter( pkg => pkg.language == lang && semver.satisfies(pkg.version, version) - ) - return candidates.sort((a,b)=>semver.rcompare(a.version,b.version))[0] || null + ); + return candidates.sort((a,b)=>semver.rcompare(a.version,b.version))[0] || null; } module.exports = { async repo_list(req,res){ // GET /repos - logger.debug("Request for repoList") + logger.debug('Request for repoList'); res.json_success({ repos: (await Promise.all( - [...state.state.get("repositories").keys()].map( async slug => await get_or_construct_repo(slug)) + [...state.state.get('repositories').keys()].map( async slug => await get_or_construct_repo(slug)) )).map(repo=>({ slug: repo.slug, url: repo.url, packages: repo.packages.length })) - }) + }); }, async repo_add(req, res){ // POST /repos - logger.debug(`Request for repoAdd slug=${req.body.slug} url=${req.body.url}`) + logger.debug(`Request for repoAdd slug=${req.body.slug} url=${req.body.url}`); if(!req.body.slug) - return res.json_error("slug is missing from request body", 400) + return res.json_error('slug is missing from request body', 400); if(!req.body.url) - return res.json_error("url is missing from request body", 400) + return res.json_error('url is missing from request body', 400); - const repo_state = state.state.get("repositories") + const repo_state = state.state.get('repositories'); - if(repo_state.has(req.body.slug)) return res.json_error(`repository ${req.body.slug} already exists`, 409) + if(repo_state.has(req.body.slug)) return res.json_error(`repository ${req.body.slug} already exists`, 409); - repo_state.set(req.body.slug, req.body.url) - logger.info(`Repository ${req.body.slug} added url=${req.body.url}`) + repo_state.set(req.body.slug, req.body.url); + logger.info(`Repository ${req.body.slug} added url=${req.body.url}`); - return res.json_success(req.body.slug) + return res.json_success(req.body.slug); }, async repo_info(req, res){ // GET /repos/:slug - logger.debug(`Request for repoInfo for ${req.params.repo_slug}`) - const repo = await get_or_construct_repo(req.params.repo_slug) + logger.debug(`Request for repoInfo for ${req.params.repo_slug}`); + const repo = await get_or_construct_repo(req.params.repo_slug); - if(repo == null) return res.json_error(`Requested repo ${req.params.repo_slug} does not exist`, 404) + if(repo == null) return res.json_error(`Requested repo ${req.params.repo_slug} does not exist`, 404); res.json_success({ slug: repo.slug, url: repo.url, packages: repo.packages.length - }) + }); }, async repo_packages(req, res){ // GET /repos/:slug/packages - logger.debug("Request to repoPackages") + logger.debug('Request to repoPackages'); - const repo = await get_or_construct_repo(req.params.repo_slug) - if(repo == null) return res.json_error(`Requested repo ${req.params.repo_slug} does not exist`, 404) + const repo = await get_or_construct_repo(req.params.repo_slug); + if(repo == null) return res.json_error(`Requested repo ${req.params.repo_slug} does not exist`, 404); res.json_success({ packages: repo.packages.map(pkg=>({ @@ -84,46 +84,46 @@ module.exports = { language_version: pkg.version.raw, installed: pkg.installed })) - }) + }); }, async package_info(req, res){ // GET /repos/:slug/packages/:language/:version - logger.debug("Request to packageInfo") + logger.debug('Request to packageInfo'); - const repo = await get_or_construct_repo(req.params.repo_slug) - if(repo == null) return res.json_error(`Requested repo ${req.params.repo_slug} does not exist`, 404) + const repo = await get_or_construct_repo(req.params.repo_slug); + if(repo == null) return res.json_error(`Requested repo ${req.params.repo_slug} does not exist`, 404); - const package = await get_package(repo, req.params.language, req.params.version) - if(package == null) return res.json_error(`Requested package ${req.params.language}-${req.params.version} does not exist`, 404) + const pkg = await get_package(repo, req.params.language, req.params.version); + if(pkg == null) return res.json_error(`Requested package ${req.params.language}-${req.params.version} does not exist`, 404); res.json_success({ - language: package.language, - language_version: package.version.raw, - author: package.author, - buildfile: package.buildfile, - size: package.size, - dependencies: package.dependencies, - installed: package.installed - }) + language: pkg.language, + language_version: pkg.version.raw, + author: pkg.author, + buildfile: pkg.buildfile, + size: pkg.size, + dependencies: pkg.dependencies, + installed: pkg.installed + }); }, async package_install(req,res){ // POST /repos/:slug/packages/:language/:version - logger.debug("Request to packageInstall") + logger.debug('Request to packageInstall'); - const repo = await get_or_construct_repo(req.params.repo_slug) - if(repo == null) return res.json_error(`Requested repo ${req.params.repo_slug} does not exist`, 404) + const repo = await get_or_construct_repo(req.params.repo_slug); + if(repo == null) return res.json_error(`Requested repo ${req.params.repo_slug} does not exist`, 404); - const package = await get_package(repo, req.params.language, req.params.version) - if(package == null) return res.json_error(`Requested package ${req.params.language}-${req.params.version} does not exist`, 404) + const pkg = await get_package(repo, req.params.language, req.params.version); + if(pkg == null) return res.json_error(`Requested package ${req.params.language}-${req.params.version} does not exist`, 404); try{ - const response = await package.install() - return res.json_success(response) + const response = await pkg.install(); + return res.json_success(response); }catch(err){ - logger.error(`Error while installing package ${package.language}-${package.version}:`, err.message) - res.json_error(err.message,500) + logger.error(`Error while installing package ${pkg.language}-${pkg.version}:`, err.message); + res.json_error(err.message,500); } @@ -131,6 +131,6 @@ module.exports = { async package_uninstall(req,res){ // DELETE /repos/:slug/packages/:language/:version - res.json(req.body) //TODO + res.json(req.body); //TODO } -} \ No newline at end of file +}; \ No newline at end of file diff --git a/api/src/runtime.js b/api/src/runtime.js index 4686d06..f32140b 100644 --- a/api/src/runtime.js +++ b/api/src/runtime.js @@ -1,85 +1,85 @@ -const logger = require("logplease").create("runtime") -const semver = require("semver") -const config = require("./config") -const globals = require("./globals") -const fs = require("fs") -const path = require("path") +const logger = require('logplease').create('runtime'); +const semver = require('semver'); +const config = require('./config'); +const globals = require('./globals'); +const fss = require('fs'); +const path = require('path'); -const runtimes = [] +const runtimes = []; class Runtime { #env_vars #compiled constructor(package_dir){ const {language, version, author, dependencies, build_platform} = JSON.parse( - fs.read_file_sync(path.join(package_dir, "pkg-info.json")) - ) + fss.read_file_sync(path.join(package_dir, 'pkg-info.json')) + ); - this.pkgdir = package_dir - this.language = language - this.version = semver.parse(version) - this.author = author - this.dependencies = dependencies + this.pkgdir = package_dir; + this.language = language; + this.version = semver.parse(version); + this.author = author; + this.dependencies = dependencies; if(build_platform != globals.platform){ - logger.warn(`Package ${language}-${version} was built for platform ${build_platform}, but our platform is ${globals.platform}`) + logger.warn(`Package ${language}-${version} was built for platform ${build_platform}, but our platform is ${globals.platform}`); } - logger.debug(`Package ${language}-${version} was loaded`) - runtimes.push(this) + logger.debug(`Package ${language}-${version} was loaded`); + runtimes.push(this); } get env_file_path(){ - return path.join(this.runtime_dir, "environment") + return path.join(this.runtime_dir, 'environment'); } get runtime_dir(){ - return path.join(config.data_directory,globals.data_directories.runtimes, this.toString()) + return path.join(config.data_directory,globals.data_directories.runtimes, this.toString()); } get_all_dependencies(){ - const res = [] + const res = []; Object.keys(this.dependencies).forEach(dep => { - const selector = this.dependencies[dep] - const lang = module.exports.get_latest_runtime_matching_language_version(dep, selector) - res.push(lang) - res.concat(lang.get_all_dependencies(lang)) - }) - return res + const selector = this.dependencies[dep]; + const lang = module.exports.get_latest_runtime_matching_language_version(dep, selector); + res.push(lang); + res.concat(lang.get_all_dependencies(lang)); + }); + return res; } get compile(){ - if(this.#compiled === undefined) this.#compiled = fs.existsSync(path.join(this.pkgdir, "compile")) - return this.#compiled + if(this.#compiled === undefined) this.#compiled = fss.exists_sync(path.join(this.pkgdir, 'compile')); + return this.#compiled; } get env_vars(){ if(!this.#env_vars){ - const env_file = path.join(this.pkgdir, ".env") - const env_content = fs.read_file_sync(env_file).toString() - this.#env_vars = {} + const env_file = path.join(this.pkgdir, '.env'); + const env_content = fss.read_file_sync(env_file).toString(); + this.#env_vars = {}; env_content - .split("\n") - .map(line => line.split("=",2)) + .split('\n') + .map(line => line.split('=',2)) .forEach(([key,val]) => { - this.#env_vars[key] = val - }) + this.#env_vars[key] = val; + }); } - return this.#env_vars + return this.#env_vars; } toString(){ - return `${this.language}-${this.version.raw}` + return `${this.language}-${this.version.raw}`; } } -module.exports = runtimes -module.exports.Runtime = Runtime +module.exports = runtimes; +module.exports.Runtime = Runtime; module.exports.get_runtimes_matching_language_version = function(lang, ver){ - return runtimes.filter(rt => rt.language == lang && semver.satisfies(rt.version, ver)) -} + return runtimes.filter(rt => rt.language == lang && semver.satisfies(rt.version, ver)); +}; module.exports.get_latest_runtime_matching_language_version = function(lang, ver){ return module.exports.get_runtimes_matching_language_version(lang, ver) - .sort((a,b) => semver.rcompare(a.version, b.version))[0] -} + .sort((a,b) => semver.rcompare(a.version, b.version))[0]; +}; diff --git a/api/src/state.js b/api/src/state.js index 1122a3b..6dc555f 100644 --- a/api/src/state.js +++ b/api/src/state.js @@ -1,45 +1,45 @@ -const fs = require("fs") -const util = require("util") +const fs = require('fs/promises'); +const fss = require('fs'); -const logger = require("logplease").create("state") -const state = new Map() +const logger = require('logplease').create('state'); +const state = new Map(); function replacer(key, value) { if(value instanceof Map) { return { - data_type: "Map", + data_type: 'Map', value: Array.from(value.entries()), - } + }; } else { - return value + return value; } } function reviver(key, value) { - if(typeof value === "object" && value !== null) { - if (value.data_type === "Map") { - return new Map(value.value) + if(typeof value === 'object' && value !== null) { + if (value.data_type === 'Map') { + return new Map(value.value); } } - return value + return value; } module.exports = { state, async load(data_file){ - if(fs.exists_sync(data_file)){ - logger.info("Loading state from file") - var content = await util.promisify(fs.read_file)(data_file) + if(fss.exists_sync(data_file)){ + logger.info('Loading state from file'); + var content = await fs.read_file(data_file); var obj = JSON.parse(content.toString(), reviver); - [...obj.keys()].forEach(k => state.set(k, obj.get(k))) + [...obj.keys()].forEach(k => state.set(k, obj.get(k))); }else{ - logger.info("Creating new statefile") - state.set("repositories", new Map().set("offical", "https://repo.pistonee.org/index.yaml")) + logger.info('Creating new statefile'); + state.set('repositories', new Map().set('offical', 'https://repo.pistonee.org/index.yaml')); } }, async save(data_file){ - logger.info("Saving state to disk") - await util.promisify(fs.write_file)(data_file, JSON.stringify(state, replacer)) + logger.info('Saving state to disk'); + await fs.write_file(data_file, JSON.stringify(state, replacer)); } -} \ No newline at end of file +}; \ No newline at end of file diff --git a/api/yarn.lock b/api/yarn.lock index 7f679dd..7221bfc 100644 --- a/api/yarn.lock +++ b/api/yarn.lock @@ -9,12 +9,51 @@ dependencies: "@babel/highlight" "^7.10.4" +"@babel/code-frame@^7.0.0", "@babel/code-frame@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.12.13.tgz#dcfc826beef65e75c50e21d3837d7d95798dd658" + integrity sha512-HV1Cm0Q3ZrpCR93tkWOYiuYIgLxZXZFVG2VgK+MBWjUqZTundupbfx2aXarXuw5Ko5aMcjtJgbSs4vUGBS5v6g== + dependencies: + "@babel/highlight" "^7.12.13" + +"@babel/generator@^7.12.17": + version "7.12.17" + resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.12.17.tgz#9ef1dd792d778b32284411df63f4f668a9957287" + integrity sha512-DSA7ruZrY4WI8VxuS1jWSRezFnghEoYEFrZcw9BizQRmOZiUsiHl59+qEARGPqPikwA/GPTyRCi7isuCK/oyqg== + dependencies: + "@babel/types" "^7.12.17" + jsesc "^2.5.1" + source-map "^0.5.0" + +"@babel/helper-function-name@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/helper-function-name/-/helper-function-name-7.12.13.tgz#93ad656db3c3c2232559fd7b2c3dbdcbe0eb377a" + integrity sha512-TZvmPn0UOqmvi5G4vvw0qZTpVptGkB1GL61R6lKvrSdIxGm5Pky7Q3fpKiIkQCAtRCBUwB0PaThlx9vebCDSwA== + dependencies: + "@babel/helper-get-function-arity" "^7.12.13" + "@babel/template" "^7.12.13" + "@babel/types" "^7.12.13" + +"@babel/helper-get-function-arity@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/helper-get-function-arity/-/helper-get-function-arity-7.12.13.tgz#bc63451d403a3b3082b97e1d8b3fe5bd4091e583" + integrity sha512-DjEVzQNz5LICkzN0REdpD5prGoidvbdYk1BVgRUOINaWJP2t6avB27X1guXK1kXNrX0WMfsrm1A/ZBthYuIMQg== + dependencies: + "@babel/types" "^7.12.13" + +"@babel/helper-split-export-declaration@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.12.13.tgz#e9430be00baf3e88b0e13e6f9d4eaf2136372b05" + integrity sha512-tCJDltF83htUtXx5NLcaDqRmknv652ZWCHyoTETf1CXYJdPC7nohZohjUgieXhv0hTJdRf2FjDueFehdNucpzg== + dependencies: + "@babel/types" "^7.12.13" + "@babel/helper-validator-identifier@^7.12.11": version "7.12.11" resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.12.11.tgz#c9a1f021917dcb5ccf0d4e453e399022981fc9ed" integrity sha512-np/lG3uARFybkoHokJUmf1QfEvRVCPbmQeUQpKow5cQ3xWrV9i3rUHodKDJPQfTVX61qKi+UdYk8kik84n7XOw== -"@babel/highlight@^7.10.4": +"@babel/highlight@^7.10.4", "@babel/highlight@^7.12.13": version "7.12.13" resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.12.13.tgz#8ab538393e00370b26271b01fa08f7f27f2e795c" integrity sha512-kocDQvIbgMKlWxXe9fof3TQ+gkIPOUSEYhJjqUjvKMez3krV7vbzYCDq39Oj11UAVK7JqPVGQPlgE85dPNlQww== @@ -23,6 +62,44 @@ chalk "^2.0.0" js-tokens "^4.0.0" +"@babel/parser@^7.12.13", "@babel/parser@^7.12.17", "@babel/parser@^7.7.0": + version "7.12.17" + resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.12.17.tgz#bc85d2d47db38094e5bb268fc761716e7d693848" + integrity sha512-r1yKkiUTYMQ8LiEI0UcQx5ETw5dpTLn9wijn9hk6KkTtOK95FndDN10M+8/s6k/Ymlbivw0Av9q4SlgF80PtHg== + +"@babel/template@^7.12.13": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.12.13.tgz#530265be8a2589dbb37523844c5bcb55947fb327" + integrity sha512-/7xxiGA57xMo/P2GVvdEumr8ONhFOhfgq2ihK3h1e6THqzTAkHbkXgB0xI9yeTfIUoH3+oAeHhqm/I43OTbbjA== + dependencies: + "@babel/code-frame" "^7.12.13" + "@babel/parser" "^7.12.13" + "@babel/types" "^7.12.13" + +"@babel/traverse@^7.7.0": + version "7.12.17" + resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.12.17.tgz#40ec8c7ffb502c4e54c7f95492dc11b88d718619" + integrity sha512-LGkTqDqdiwC6Q7fWSwQoas/oyiEYw6Hqjve5KOSykXkmFJFqzvGMb9niaUEag3Rlve492Mkye3gLw9FTv94fdQ== + dependencies: + "@babel/code-frame" "^7.12.13" + "@babel/generator" "^7.12.17" + "@babel/helper-function-name" "^7.12.13" + "@babel/helper-split-export-declaration" "^7.12.13" + "@babel/parser" "^7.12.17" + "@babel/types" "^7.12.17" + debug "^4.1.0" + globals "^11.1.0" + lodash "^4.17.19" + +"@babel/types@^7.12.13", "@babel/types@^7.12.17", "@babel/types@^7.7.0": + version "7.12.17" + resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.12.17.tgz#9d711eb807e0934c90b8b1ca0eb1f7230d150963" + integrity sha512-tNMDjcv/4DIcHxErTgwB9q2ZcYyN0sUfgGKUK/mm1FJK7Wz+KstoEekxrl/tBiNDgLK1HGi+sppj1An/1DR4fQ== + dependencies: + "@babel/helper-validator-identifier" "^7.12.11" + lodash "^4.17.19" + to-fast-properties "^2.0.0" + "@eslint/eslintrc@^0.3.0": version "0.3.0" resolved "https://registry.yarnpkg.com/@eslint/eslintrc/-/eslintrc-0.3.0.tgz#d736d6963d7003b6514e6324bec9c602ac340318" @@ -123,6 +200,18 @@ astral-regex@^2.0.0: resolved "https://registry.yarnpkg.com/astral-regex/-/astral-regex-2.0.0.tgz#483143c567aeed4785759c0865786dc77d7d2e31" integrity sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ== +babel-eslint@^10.1.0: + version "10.1.0" + resolved "https://registry.yarnpkg.com/babel-eslint/-/babel-eslint-10.1.0.tgz#6968e568a910b78fb3779cdd8b6ac2f479943232" + integrity sha512-ifWaTHQ0ce+448CYop8AdrQiBsGrnC+bMgfyKFdi6EsPLTAWG+QfyDeM6OH+FmWnKvEq5NnBMLvlBUPKQZoDSg== + dependencies: + "@babel/code-frame" "^7.0.0" + "@babel/parser" "^7.7.0" + "@babel/traverse" "^7.7.0" + "@babel/types" "^7.7.0" + eslint-visitor-keys "^1.0.0" + resolve "^1.12.0" + balanced-match@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" @@ -255,7 +344,7 @@ debug@2.6.9: dependencies: ms "2.0.0" -debug@^4.0.1, debug@^4.1.1: +debug@^4.0.1, debug@^4.1.0, debug@^4.1.1: version "4.3.1" resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.1.tgz#f0d229c505e0c6d8c49ac553d1b13dc183f6b2ee" integrity sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ== @@ -341,7 +430,7 @@ eslint-utils@^2.1.0: dependencies: eslint-visitor-keys "^1.1.0" -eslint-visitor-keys@^1.1.0, eslint-visitor-keys@^1.3.0: +eslint-visitor-keys@^1.0.0, eslint-visitor-keys@^1.1.0, eslint-visitor-keys@^1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz#30ebd1ef7c2fdff01c3a4f151044af25fab0523e" integrity sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ== @@ -541,6 +630,11 @@ fs.realpath@^1.0.0: resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" integrity sha1-FQStJSMVjKpA20onh8sBQRmU6k8= +function-bind@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.1.tgz#a56899d3ea3c9bab874bb9773b7c5ede92f4895d" + integrity sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A== + functional-red-black-tree@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz#1b0ab3bd553b2a0d6399d29c0e3ea0b252078327" @@ -570,6 +664,11 @@ glob@^7.1.3: once "^1.3.0" path-is-absolute "^1.0.0" +globals@^11.1.0: + version "11.12.0" + resolved "https://registry.yarnpkg.com/globals/-/globals-11.12.0.tgz#ab8795338868a0babd8525758018c2a7eb95c42e" + integrity sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA== + globals@^12.1.0: version "12.4.0" resolved "https://registry.yarnpkg.com/globals/-/globals-12.4.0.tgz#a18813576a41b00a24a97e7f815918c2e19925f8" @@ -587,6 +686,13 @@ has-flag@^4.0.0: resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-4.0.0.tgz#944771fd9c81c81265c4d6941860da06bb59479b" integrity sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ== +has@^1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/has/-/has-1.0.3.tgz#722d7cbfc1f6aa8241f16dd814e011e1f41e8796" + integrity sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw== + dependencies: + function-bind "^1.1.1" + http-errors@1.7.2: version "1.7.2" resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-1.7.2.tgz#4f5029cf13239f31036e5b2e55292bcfbcc85c8f" @@ -657,6 +763,13 @@ ipaddr.js@1.9.1: resolved "https://registry.yarnpkg.com/ipaddr.js/-/ipaddr.js-1.9.1.tgz#bff38543eeb8984825079ff3a2a8e6cbd46781b3" integrity sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g== +is-core-module@^2.2.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.2.0.tgz#97037ef3d52224d85163f5597b2b63d9afed981a" + integrity sha512-XRAfAdyyY5F5cOXn7hYQDqh2Xmii+DEfIcQGxK/uNwMHhIkPWO0g8msXcbzLe+MpGoR951MlqM/2iIlU4vKDdQ== + dependencies: + has "^1.0.3" + is-docker@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/is-docker/-/is-docker-2.1.1.tgz#4125a88e44e450d384e09047ede71adc2d144156" @@ -704,6 +817,11 @@ js-yaml@^4.0.0: dependencies: argparse "^2.0.1" +jsesc@^2.5.1: + version "2.5.2" + resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-2.5.2.tgz#80564d2e483dacf6e8ef209650a67df3f0c283a4" + integrity sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA== + json-schema-traverse@^0.4.1: version "0.4.1" resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz#69f6a87d9513ab8bb8fe63bdb0979c448e684660" @@ -727,6 +845,11 @@ levn@^0.4.1: prelude-ls "^1.2.1" type-check "~0.4.0" +lodash@^4.17.19: + version "4.17.21" + resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" + integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== + lodash@^4.17.20: version "4.17.20" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.20.tgz#b44a9b6297bcb698f1c51a3545a2b3b368d59c52" @@ -866,6 +989,11 @@ path-key@^3.1.0: resolved "https://registry.yarnpkg.com/path-key/-/path-key-3.1.1.tgz#581f6ade658cbba65a0d3380de7753295054f375" integrity sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q== +path-parse@^1.0.6: + version "1.0.6" + resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.6.tgz#d62dbb5679405d72c4737ec58600e9ddcf06d24c" + integrity sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw== + path-to-regexp@0.1.7: version "0.1.7" resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-0.1.7.tgz#df604178005f522f15eb4490e7247a1bfaa67f8c" @@ -934,6 +1062,14 @@ resolve-from@^4.0.0: resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-4.0.0.tgz#4abcd852ad32dd7baabfe9b40e00a36db5f392e6" integrity sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g== +resolve@^1.12.0: + version "1.20.0" + resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.20.0.tgz#629a013fb3f70755d6f0b7935cc1c2c5378b1975" + integrity sha512-wENBPt4ySzg4ybFQW2TT1zMQucPK95HSh/nq2CFTZVOGut2+pQvSsgtda4d26YrYcr067wjbmzOG8byDPBX63A== + dependencies: + is-core-module "^2.2.0" + path-parse "^1.0.6" + rimraf@^3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-3.0.2.tgz#f1a5402ba6220ad52cc1282bac1ae3aa49fd061a" @@ -1013,6 +1149,11 @@ slice-ansi@^4.0.0: astral-regex "^2.0.0" is-fullwidth-code-point "^3.0.0" +source-map@^0.5.0: + version "0.5.7" + resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.5.7.tgz#8a039d2d1021d22d1ea14c80d8ea468ba2ef3fcc" + integrity sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w= + sprintf-js@~1.0.2: version "1.0.3" resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" @@ -1073,6 +1214,11 @@ text-table@^0.2.0: resolved "https://registry.yarnpkg.com/text-table/-/text-table-0.2.0.tgz#7f5ee823ae805207c00af2df4a84ec3fcfa570b4" integrity sha1-f17oI66AUgfACvLfSoTsP8+lcLQ= +to-fast-properties@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/to-fast-properties/-/to-fast-properties-2.0.0.tgz#dc5e698cbd079265bc73e0377681a4e4e83f616e" + integrity sha1-3F5pjL0HkmW8c+A3doGk5Og/YW4= + toidentifier@1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/toidentifier/-/toidentifier-1.0.0.tgz#7e1be3470f1e77948bc43d94a3c8f4d7752ba553" diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..356f800 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,20 @@ +version: '3.8' + +services: + piston_api: + build: api + restart: always + ports: + - 6969:6969 + volumes: + - ./data/piston:/piston + - ./repo:/repo + tmpfs: + - /piston/cache + - /piston/jobs + + piston_fs_repo: #Temporary solution until CI works + build: repo + volumes: + - ./repo:/repo + - ./packages:/packages \ No newline at end of file diff --git a/packages/common.mk b/packages/common.mk index 3b3d201..90773cc 100644 --- a/packages/common.mk +++ b/packages/common.mk @@ -26,7 +26,7 @@ pkg-info.jq: $(foreach dep, ${LANG_DEPS}, echo '.dependencies.$(word 1,$(subst =, ,${dep}))="$(word 2,$(subst =, ,${dep}))"' >> pkg-info.jq) %.asc: % - gpg --detach-sig --armor --output $@ $< + gpg --detach-sig --armor --batch --output $@ $< %/: %.tgz tar xzf $< diff --git a/repo/.gitignore b/repo/.gitignore new file mode 100644 index 0000000..fd572dc --- /dev/null +++ b/repo/.gitignore @@ -0,0 +1,3 @@ +*.pkg.tar.gz +index.yaml +*.key \ No newline at end of file diff --git a/repo/Dockerfile b/repo/Dockerfile new file mode 100644 index 0000000..beeb9d0 --- /dev/null +++ b/repo/Dockerfile @@ -0,0 +1,8 @@ +FROM alpine:3.13 + +RUN apk add --no-cache gnupg jq zlib zlib-dev cmake cmake-doc extra-cmake-modules extra-cmake-modules-doc build-base gcc abuild binutils binutils-doc gcc-doc yq bash coreutils util-linux pciutils usbutils coreutils binutils findutils grep && \ + ln -sf /bin/bash /bin/sh && \ + wget https://github.com/mikefarah/yq/releases/download/2.4.1/yq_linux_amd64 -O /usr/bin/yq &&\ + chmod +x /usr/bin/yq + +CMD [ "bash", "/repo/make.sh" ] \ No newline at end of file diff --git a/repo/README.MD b/repo/README.MD new file mode 100644 index 0000000..43d66dd --- /dev/null +++ b/repo/README.MD @@ -0,0 +1,3 @@ +# Piston Filesystem Repo Builder + +This is just a simple POC for a repository tool to run locally. diff --git a/repo/make.sh b/repo/make.sh new file mode 100644 index 0000000..f380e4f --- /dev/null +++ b/repo/make.sh @@ -0,0 +1,7 @@ +#!/bin/bash +cd /repo +cat password.key | gpg --batch --import private.key +pushd ../packages/python +cat password.key | make sign VERSIONS=3.9.1 && make cleanup +popd +./mkindex.sh \ No newline at end of file diff --git a/repo/mkindex.sh b/repo/mkindex.sh new file mode 100755 index 0000000..4f2ac42 --- /dev/null +++ b/repo/mkindex.sh @@ -0,0 +1,26 @@ +echo "schema: ppman-repo-1" > index.yaml +echo "baseurl: file://$PWD" >> index.yaml +echo "keys: []" >> index.yaml +echo "packages: []" >> index.yaml + +yq -yi '.keys[0] = "0x107DA02C7AE97B084746564B9F1FD9D87950DB6F"' index.yaml + +i=-1 + +for pkg in $(find ../packages -type f -name "*.pkg.tar.gz") +do + ((i=i+1)) + cp $pkg . + PKGFILE=$(basename $pkg) + PKGFILENAME=$(echo $PKGFILE | sed 's/\.pkg\.tar\.gz//g') + PKGNAME=$(echo $PKGFILENAME | grep -oP '^\K.+(?=-)') + PKGVERSION=$(echo $PKGFILENAME | grep -oP '^.+-\K.+') + BUILDFILE=https://github.com/engineer-man/piston/tree/v3/packages/python/ + SIZE=$(tar tzvf $PKGFILE | sed 's/ \+/ /g' | cut -f3 -d' ' | sed '2,$s/^/+ /' | paste -sd' ' | bc) + + tar xzf $PKGFILE pkg-info.json + + yq -yi ".packages[$i] = {} | .packages[$i].signature = \"$(cat ${pkg}.asc)\" | .packages[$i].buildfile = \"$BUILDFILE\" | .packages[$i].size = $SIZE | .packages[$i].download = \"$PKGFILE\" | .packages[$i].dependencies = $(jq .dependencies -r pkg-info.json) | .packages[$i].author = $(jq .author pkg-info.json) | .packages[$i].language =\"$PKGNAME\" | .packages[$i].version = \"$PKGVERSION\" | .packages[$i].checksums = {} | .packages[$i].checksums.sha256 = \"$(sha256sum $PKGFILE | awk '{print $1}')\"" index.yaml + + rm pkg-info.json +done \ No newline at end of file From 8ad62ec9838686c6cc78cc908c963907884e4b43 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sun, 21 Feb 2021 12:06:20 +1300 Subject: [PATCH 05/17] api: use patched nocamel for fs/promises --- api/package.json | 2 +- api/src/index.js | 3 +-- api/yarn.lock | 7 +++---- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/api/package.json b/api/package.json index a530298..74072e5 100644 --- a/api/package.json +++ b/api/package.json @@ -9,7 +9,7 @@ "is-docker": "^2.1.1", "js-yaml": "^4.0.0", "logplease": "^1.2.15", - "nocamel": "*", + "nocamel": "HexF/nocamel#patch-1", "node-fetch": "^2.6.1", "semver": "^7.3.4", "uuid": "^8.3.2", diff --git a/api/src/index.js b/api/src/index.js index 873b66a..7b295b3 100644 --- a/api/src/index.js +++ b/api/src/index.js @@ -49,8 +49,7 @@ const app = express(); fs.readdir(path.join(pkgdir,lang)) .then(x=>x.map(y=>path.join(pkgdir, lang, y))) ))) - //eslint-disable-next-line snakecasejs/snakecasejs - .then(pkgs=>pkgs.flat().filter(pkg=>fs.existsSync(path.join(pkg, globals.pkg_installed_file)))) + .then(pkgs=>pkgs.flat().filter(pkg=>fss.exists_sync(path.join(pkg, globals.pkg_installed_file)))) .then(pkgs=>pkgs.forEach(pkg => new runtime.Runtime(pkg))); diff --git a/api/yarn.lock b/api/yarn.lock index 7221bfc..ec163b5 100644 --- a/api/yarn.lock +++ b/api/yarn.lock @@ -931,10 +931,9 @@ negotiator@0.6.2: resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.2.tgz#feacf7ccf525a77ae9634436a64883ffeca346fb" integrity sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw== -nocamel@*: - version "1.0.2" - resolved "https://registry.yarnpkg.com/nocamel/-/nocamel-1.0.2.tgz#13ff04ffacd5487ba65555c0dcafcf8c95c918ba" - integrity sha512-CRkRSRLChj+H6e4lHS851QS6YGCoTETnSG/z+XGanxLSsTbBkvEeIWaIYMKzuBznFwWM0YcLGXsFyXg4xWYnWA== +nocamel@HexF/nocamel#patch-1: + version "1.1.0" + resolved "https://codeload.github.com/HexF/nocamel/tar.gz/89a5bfbbd07c72c302d968b967d0f4fe54846544" node-fetch@^2.6.1: version "2.6.1" From b20f853ef161feea921c5cda794d09dbd65a709c Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sun, 21 Feb 2021 12:56:35 +1300 Subject: [PATCH 06/17] api: fix function name + allow unsigned packages --- api/src/helpers.js | 2 +- api/src/ppman/package.js | 36 +++++++++++++++++++----------------- api/src/ppman/repo.js | 2 +- 3 files changed, 21 insertions(+), 19 deletions(-) diff --git a/api/src/helpers.js b/api/src/helpers.js index 0a67c5b..ba3ef56 100644 --- a/api/src/helpers.js +++ b/api/src/helpers.js @@ -6,7 +6,7 @@ const fs = require('fs/promises'), module.exports = { - async buffer_from_u_r_l(url){ + async buffer_from_url(url){ if(!(url instanceof URL)) url = new URL(url); if(url.protocol == 'file:'){ diff --git a/api/src/ppman/package.js b/api/src/ppman/package.js index 5ccdf4d..e45877a 100644 --- a/api/src/ppman/package.js +++ b/api/src/ppman/package.js @@ -57,7 +57,7 @@ class Package { logger.debug(`Downloading package from ${this.download_url} in to ${this.install_path}`); const pkgfile = helpers.url_basename(this.download_url); const pkgpath = path.join(this.install_path, pkgfile); - await helpers.buffer_from_u_r_l(this.download_url) + await helpers.buffer_from_url(this.download_url) .then(buf=> fs.write_file(pkgpath, buf)); logger.debug('Validating checksums'); @@ -73,23 +73,25 @@ class Package { await this.repo.import_keys(); logger.debug('Validating signatutes'); - await new Promise((resolve,reject)=>{ - const gpgspawn = cp.spawn('gpg', ['--verify', '-', pkgpath], { - stdio: ['pipe', 'ignore', 'ignore'] + if(this.signature != "") + await new Promise((resolve,reject)=>{ + const gpgspawn = cp.spawn('gpg', ['--verify', '-', pkgpath], { + stdio: ['pipe', 'ignore', 'ignore'] + }); + + gpgspawn.once('exit', (code, _) => { + if(code == 0) resolve(); + else reject(new Error('Invalid signature')); + }); + + gpgspawn.once('error', reject); + + gpgspawn.stdin.write(this.signature); + gpgspawn.stdin.end(); + }); - - gpgspawn.once('exit', (code, _) => { - if(code == 0) resolve(); - else reject(new Error('Invalid signature')); - }); - - gpgspawn.once('error', reject); - - gpgspawn.stdin.write(this.signature); - gpgspawn.stdin.end(); - - }); - + else + logger.warn("Package does not contain a signature - allowing install, but proceed with caution") logger.debug(`Extracting package files from archive ${pkgfile} in to ${this.install_path}`); await new Promise((resolve, reject)=>{ diff --git a/api/src/ppman/repo.js b/api/src/ppman/repo.js index fc6378c..4c6deab 100644 --- a/api/src/ppman/repo.js +++ b/api/src/ppman/repo.js @@ -24,7 +24,7 @@ class Repository { async load(){ try{ var index = await cache.get(this.cache_key,async ()=>{ - return helpers.buffer_from_u_r_l(this.url); + return helpers.buffer_from_url(this.url); }); var repo = yaml.load(index); From cdc65d6605911e1c29c1978b1902cb05afdc8c14 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sun, 21 Feb 2021 12:57:02 +1300 Subject: [PATCH 07/17] api: use bash to call run/compile script --- api/src/executor/job.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/api/src/executor/job.js b/api/src/executor/job.js index b10e3ab..12946ab 100644 --- a/api/src/executor/job.js +++ b/api/src/executor/job.js @@ -72,10 +72,9 @@ class Job { if(this.state != job_states.PRIMED) throw new Error('Job must be in primed state, current state: ' + this.state.toString()); logger.info(`Executing job uuid=${this.uuid} uid=${this.uid} gid=${this.gid} runtime=${this.runtime.toString()}`); logger.debug('Compiling'); - const compile = this.runtime.compiled && await new Promise((resolve, reject) => { var stderr, stdout = ''; - const proc = cp.spawn(path.join(this.runtime.pkgdir, 'compile'), [this.main, ...this.args] ,{ + const proc = cp.spawn('bash', [path.join(this.runtime.pkgdir, 'compile'),this.main, ...this.args] ,{ env: this.runtime.env_vars, stdio: ['pipe', 'pipe', 'pipe'], cwd: this.dir, @@ -103,7 +102,7 @@ class Job { const run = await new Promise((resolve, reject) => { var stderr, stdout = ''; - const proc = cp.spawn(path.join(this.runtime.pkgdir, 'run'), [this.main, ...this.args] ,{ + const proc = cp.spawn('bash', [path.join(this.runtime.pkgdir, 'run'),this.main, ...this.args] ,{ env: this.runtime.env_vars, stdio: ['pipe', 'pipe', 'pipe'], cwd: this.dir, From 233fb9bf2629f2e7defc44534613cbb2f5da2467 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sun, 21 Feb 2021 12:57:20 +1300 Subject: [PATCH 08/17] api: trim whitespace off env vars --- api/src/runtime.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/src/runtime.js b/api/src/runtime.js index f32140b..6e71837 100644 --- a/api/src/runtime.js +++ b/api/src/runtime.js @@ -59,10 +59,11 @@ class Runtime { const env_content = fss.read_file_sync(env_file).toString(); this.#env_vars = {}; env_content + .trim() .split('\n') .map(line => line.split('=',2)) .forEach(([key,val]) => { - this.#env_vars[key] = val; + this.#env_vars[key.trim()] = val.trim(); }); } return this.#env_vars; From 816efaff3b8c4a636d60a13328c5a63241de0dd7 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sun, 21 Feb 2021 12:57:40 +1300 Subject: [PATCH 09/17] pkg(python *): correct environment --- packages/python/base.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/python/base.mk b/packages/python/base.mk index 722f8d5..3a19952 100644 --- a/packages/python/base.mk +++ b/packages/python/base.mk @@ -8,7 +8,7 @@ run: echo 'python$(shell grep -oP "\d+.\d+"<<<${VERSION}) $$*' > run ${NAME}-${VERSION}/environment: - echo 'export PATH=$$PWD/${NAME}-${VERSION}/bin:$$PATH' > $@ + echo 'export PATH=$$PWD/bin:$$PATH' > $@ ${NAME}-${VERSION}/: Python-${VERSION}/ cd $< && ./configure --prefix / From 60b258f57c6e0c3f2af71c48bee755ee1bcef285 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sun, 21 Feb 2021 12:58:18 +1300 Subject: [PATCH 10/17] repo: Automated local repository builder --- repo/Dockerfile | 5 ++--- repo/README.MD | 4 ++++ repo/make.sh | 12 +++++++++--- 3 files changed, 15 insertions(+), 6 deletions(-) mode change 100644 => 100755 repo/make.sh diff --git a/repo/Dockerfile b/repo/Dockerfile index beeb9d0..d49182e 100644 --- a/repo/Dockerfile +++ b/repo/Dockerfile @@ -1,8 +1,7 @@ FROM alpine:3.13 -RUN apk add --no-cache gnupg jq zlib zlib-dev cmake cmake-doc extra-cmake-modules extra-cmake-modules-doc build-base gcc abuild binutils binutils-doc gcc-doc yq bash coreutils util-linux pciutils usbutils coreutils binutils findutils grep && \ +RUN apk add --no-cache python3 py3-pip gnupg jq zlib zlib-dev cmake cmake-doc extra-cmake-modules extra-cmake-modules-doc build-base gcc abuild binutils binutils-doc gcc-doc yq bash coreutils util-linux pciutils usbutils coreutils binutils findutils grep && \ ln -sf /bin/bash /bin/sh && \ - wget https://github.com/mikefarah/yq/releases/download/2.4.1/yq_linux_amd64 -O /usr/bin/yq &&\ - chmod +x /usr/bin/yq + pip3 install 'yq==2.12.0' CMD [ "bash", "/repo/make.sh" ] \ No newline at end of file diff --git a/repo/README.MD b/repo/README.MD index 43d66dd..28f833f 100644 --- a/repo/README.MD +++ b/repo/README.MD @@ -1,3 +1,7 @@ # Piston Filesystem Repo Builder This is just a simple POC for a repository tool to run locally. + +This only demonstrates building an unsigned python-3.9.1 package, however if it finds an `asc` file it will include it as the signature. + +Mount this whole directory into `/repo` in your API container if you wish to use it. \ No newline at end of file diff --git a/repo/make.sh b/repo/make.sh old mode 100644 new mode 100755 index f380e4f..e89dc1e --- a/repo/make.sh +++ b/repo/make.sh @@ -1,7 +1,13 @@ -#!/bin/bash +#!/bin/bash -e + cd /repo -cat password.key | gpg --batch --import private.key + +# Make packages pushd ../packages/python -cat password.key | make sign VERSIONS=3.9.1 && make cleanup +make build VERSIONS=3.9.1 popd + + +# Make repo index + ./mkindex.sh \ No newline at end of file From 7b2305f30c629d5efd8e967003d226e86171017e Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sun, 21 Feb 2021 13:15:11 +1300 Subject: [PATCH 11/17] api: add licence to package.json --- api/package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/package.json b/api/package.json index 74072e5..5c494a7 100644 --- a/api/package.json +++ b/api/package.json @@ -19,5 +19,6 @@ "babel-eslint": "^10.1.0", "eslint": "^7.20.0", "eslint-plugin-snakecasejs": "^2.2.0" - } + }, + "license": "MIT" } From f957019710a4252eb29ff1af0150bd2e82f81669 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sun, 21 Feb 2021 13:15:27 +1300 Subject: [PATCH 12/17] deploy: docker compose file --- .gitignore | 1 + docker-compose.yaml | 5 +++++ 2 files changed, 6 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..adbb97d --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +data/ \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml index 356f800..5cd8ede 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -15,6 +15,11 @@ services: piston_fs_repo: #Temporary solution until CI works build: repo + command: > + bash -c '/repo/make.sh && + curl http://piston_api:6969/repos -XPOST -d "slug=local&url=file:///repo/index.yaml"; + echo -e "\nAn error here is fine, it just means its already added it. Perhaps you restarted this container" + ' volumes: - ./repo:/repo - ./packages:/packages \ No newline at end of file From ac46c1b5bb08f218adc045d8a7357a29e4c71a8c Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sun, 21 Feb 2021 13:32:35 +1300 Subject: [PATCH 13/17] api: read both stdout and stderr --- api/src/executor/job.js | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/api/src/executor/job.js b/api/src/executor/job.js index 12946ab..52792ac 100644 --- a/api/src/executor/job.js +++ b/api/src/executor/job.js @@ -73,8 +73,9 @@ class Job { logger.info(`Executing job uuid=${this.uuid} uid=${this.uid} gid=${this.gid} runtime=${this.runtime.toString()}`); logger.debug('Compiling'); const compile = this.runtime.compiled && await new Promise((resolve, reject) => { - var stderr, stdout = ''; - const proc = cp.spawn('bash', [path.join(this.runtime.pkgdir, 'compile'),this.main, ...this.args] ,{ + var stdout = ''; + var stderr = ''; + const proc = cp.spawn('bash', [path.join(this.runtime.pkgdir, 'compile'),this.main, ...this.files] ,{ env: this.runtime.env_vars, stdio: ['pipe', 'pipe', 'pipe'], cwd: this.dir, @@ -92,16 +93,17 @@ class Job { resolve({stdout, stderr, code, signal}); }); - proc.on('error', (code, signal) => { + proc.on('error', (err) => { clearTimeout(kill_timeout); - reject({stdout, stderr, code, signal}); + reject({error: err, stdout, stderr}); }); }); logger.debug('Running'); const run = await new Promise((resolve, reject) => { - var stderr, stdout = ''; + var stdout = ''; + var stderr = ''; const proc = cp.spawn('bash', [path.join(this.runtime.pkgdir, 'run'),this.main, ...this.args] ,{ env: this.runtime.env_vars, stdio: ['pipe', 'pipe', 'pipe'], @@ -120,9 +122,9 @@ class Job { resolve({stdout, stderr, code, signal}); }); - proc.on('error', (code, signal) => { + proc.on('error', (err) => { clearTimeout(kill_timeout); - reject({stdout, stderr, code, signal}); + reject({error: err, stdout, stderr}); }); }); From 72f57ef1ce38413e1d6839a8a4da586d192edd09 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sun, 21 Feb 2021 13:37:21 +1300 Subject: [PATCH 14/17] docs: readme --- README.MD | 238 ++++++++++++++++++++++++++++++++++++ docs/images/icon_circle.svg | 32 +++++ 2 files changed, 270 insertions(+) create mode 100644 README.MD create mode 100644 docs/images/icon_circle.svg diff --git a/README.MD b/README.MD new file mode 100644 index 0000000..44f9298 --- /dev/null +++ b/README.MD @@ -0,0 +1,238 @@ +

+ engineer-man piston + Piston +

+ +

A high performance general purpose code execution engine.

+
+ +

+ + GitHub last commit + + GitHub issues + + GitHub pull requests +

+ +--- + +

+ About • + Public API • + Getting Started • + Usage • + Supported Languages • + Principles • + Security • + License +

+ +--- +
+ +# About + +

+Piston is a high performance general purpose code execution engine. It excels at running untrusted and +possibly malicious code without fear from any harmful effects. +

+
+ +It's used in numerous places including: +* [EMKC Challenges](https://emkc.org/challenges), +* [EMKC Weekly Contests](https://emkc.org/contests), +* [Engineer Man Discord Server](https://discord.gg/engineerman), +* [I Run Code (Discord Bot)](https://github.com/engineer-man/piston-bot) bot as well as 1300+ other servers +and 100+ direct integrations. + +To get it in your own server, go here: https://emkc.org/run. + +
+ +# Public API + +- Requires no installation and you can use it immediately. +- Reference the Versions/Execute sections below to learn about the request and response formats. + +
+ +When using the public Piston API, use the base URL: + +``` +https://emkc.org/api/v1/piston +``` + +#### GET +``` +https://emkc.org/api/v1/piston/versions +``` +#### POST +``` +https://emkc.org/api/v1/piston/execute +``` + +> Important Note: The Piston API is rate limited to 5 requests per second. If you have a need for more requests than that +and it's for a good cause, please reach out to me (EngineerMan#0001) on [Discord](https://discord.gg/engineerman) +so we can discuss potentially getting you an unlimited key. + +
+ +# Getting Started + +### Host System Package Dependencies + +- Docker +- Docker Compose +- Node JS + +#### After system dependencies are installed, clone this repository: + +```sh +# clone and enter repo +git clone https://github.com/engineer-man/piston +``` + +#### Installation + +- docker-compose up + +#### CLI Usage +- `cli/execute [language] [file path] [args]` +
+ +# Usage + +### CLI + +```sh +lxc/execute [language] [file path] [args] +``` + +### API +To use the API, it must first be started. Please note that if root is required to access +LXC then the API must also be running as root. To start the API, run the following: + +``` +cd api +./start +``` + +For your own local installation, the API is available at: + +``` +http://127.0.0.1:2000 +``` + +#### Versions Endpoint +`GET /versions` +This endpoint will return the supported languages along with the current version and aliases. To execute +code for a particular language using the `/execute` endpoint, either the name or one of the aliases must +be provided. +```json +HTTP/1.1 200 OK +Content-Type: application/json + +[ + { + "name": "awk", + "aliases": ["awk"], + "version": "1.3.3" + }, + { + "name": "bash", + "aliases": ["bash"], + "version": "4.4.20" + }, + { + "name": "c", + "aliases": ["c"], + "version": "7.5.0" + } +] +``` + +#### Execute Endpoint +`POST /execute` +This endpoint requests execution of some arbitrary code. +- `language` (**required**) The language to use for execution, must be a string and supported by Piston (see list below). +- `source` (**required**) The source code to execute, must be a string. +- `stdin` (*optional*) The text to pass as stdin to the program. Must be a string or left out of the request. +- `args` (*optional*) The arguments to pass to the program. Must be an array or left out of the request. +```json +{ + "language": "js", + "source": "console.log(process.argv)", + "stdin": "", + "args": [ + "1", + "2", + "3" + ] +} +``` +A typical response upon successful execution will contain the `language`, `version`, `output` which +is a combination of both `stdout` and `stderr` but in chronological order according to program output, +as well as separate `stdout` and `stderr`. +```json +HTTP/1.1 200 OK +Content-Type: application/json + +{ + "ran": true, + "language": "js", + "version": "12.13.0", + "output": "[ '/usr/bin/node',\n '/tmp/code.code',\n '1',\n '2',\n '3' ]", + "stdout": "[ '/usr/bin/node',\n '/tmp/code.code',\n '1',\n '2',\n '3' ]", + "stderr": "" +} +``` +If a problem exists with the request, a `400` status code is returned and the reason in the `message` key. +```json +HTTP/1.1 400 Bad Request +Content-Type: application/json + +{ + "message": "Supplied language is not supported by Piston" +} +``` + +
+ +# Supported Languages + +`python`, + + +
+ +
+ + +# License +Piston is licensed under the MIT license. \ No newline at end of file diff --git a/docs/images/icon_circle.svg b/docs/images/icon_circle.svg new file mode 100644 index 0000000..7acfd39 --- /dev/null +++ b/docs/images/icon_circle.svg @@ -0,0 +1,32 @@ + + + From 5ac128553426b2d2e9d6ee1dc179ed02dc63595b Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sun, 21 Feb 2021 14:15:48 +1300 Subject: [PATCH 15/17] api: lint --- api/src/ppman/package.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/src/ppman/package.js b/api/src/ppman/package.js index e45877a..ebccd7f 100644 --- a/api/src/ppman/package.js +++ b/api/src/ppman/package.js @@ -73,7 +73,7 @@ class Package { await this.repo.import_keys(); logger.debug('Validating signatutes'); - if(this.signature != "") + if(this.signature != '') await new Promise((resolve,reject)=>{ const gpgspawn = cp.spawn('gpg', ['--verify', '-', pkgpath], { stdio: ['pipe', 'ignore', 'ignore'] @@ -91,7 +91,7 @@ class Package { }); else - logger.warn("Package does not contain a signature - allowing install, but proceed with caution") + logger.warn('Package does not contain a signature - allowing install, but proceed with caution'); logger.debug(`Extracting package files from archive ${pkgfile} in to ${this.install_path}`); await new Promise((resolve, reject)=>{ From 2f64f238962809264d4177d6aa30a7ce4584fac4 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sun, 21 Feb 2021 14:25:03 +1300 Subject: [PATCH 16/17] api: container hardening --- api/Dockerfile | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/api/Dockerfile b/api/Dockerfile index 1d32806..faa6b8b 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -1,5 +1,13 @@ FROM node:15.8.0-alpine3.13 -RUN apk add --no-cache gnupg tar bash coreutils +RUN apk add --no-cache gnupg tar bash coreutils shadow +RUN for i in $(seq 1000 1500); do \ + groupadd -g $i runner$i && \ + useradd -M runner$i -g $i -u $i && \ + echo "runner$i soft nproc 64" >> /etc/security/limits.conf && \ + echo "runner$i hard nproc 64" >> /etc/security/limits.conf && \ + echo "runner$i soft nofile 2048" >> /etc/security/limits.conf && \ + echo "runner$i hard nofile 2048" >> /etc/security/limits.conf ;\ + done ENV NODE_ENV=production WORKDIR /piston_api From 8727a545c635ed7ac174cf2c089a7e34b5a412ef Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sun, 21 Feb 2021 14:56:07 +1300 Subject: [PATCH 17/17] api: disable networking during execute --- api/src/executor/job.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/src/executor/job.js b/api/src/executor/job.js index 52792ac..b28589e 100644 --- a/api/src/executor/job.js +++ b/api/src/executor/job.js @@ -75,7 +75,7 @@ class Job { const compile = this.runtime.compiled && await new Promise((resolve, reject) => { var stdout = ''; var stderr = ''; - const proc = cp.spawn('bash', [path.join(this.runtime.pkgdir, 'compile'),this.main, ...this.files] ,{ + const proc = cp.spawn('unshare', ['-n', 'bash', path.join(this.runtime.pkgdir, 'compile'),this.main, ...this.files] ,{ env: this.runtime.env_vars, stdio: ['pipe', 'pipe', 'pipe'], cwd: this.dir, @@ -104,7 +104,7 @@ class Job { const run = await new Promise((resolve, reject) => { var stdout = ''; var stderr = ''; - const proc = cp.spawn('bash', [path.join(this.runtime.pkgdir, 'run'),this.main, ...this.args] ,{ + const proc = cp.spawn('unshare', ['-n', 'bash', path.join(this.runtime.pkgdir, 'run'),this.main, ...this.args] ,{ env: this.runtime.env_vars, stdio: ['pipe', 'pipe', 'pipe'], cwd: this.dir,