Mascali/piston
1
0
Fork 0
mirror of https://github.com/engineer-man/piston.git synced 2025-09-06 03:50:06 +02:00
Code Issues Projects Releases Wiki Activity

api: remove repos from ppman

Browse source
This commit is contained in:
Thomas Hobson 2021-03-06 19:17:56 +13:00
parent 22dcad0dd9
commit 812069cc3f
No known key found for this signature in database
GPG key ID: 9F1FD9D87950DB6F
10 changed files with 66 additions and 511 deletions
Download patch file Download diff file Expand all files Collapse all files

101
api/src/ppman/package.js
View file

@ -2,7 +2,7 @@ const logger = require('logplease').create('ppman/package');
const semver = require('semver');
const config = require('../config');
const globals = require('../globals');
const helpers = require('../helpers');
const fetch = require('node-fetch');
const path = require('path');
const fs = require('fs/promises');
const fss = require('fs');
@ -11,19 +11,11 @@ const crypto = require('crypto');
const runtime = require('../runtime');
class Package {
constructor(repo, {author, language, version, checksums, dependencies, size, buildfile, download, signature}){
this.author = author;
constructor({language, version, download, checksum}){
this.language = language;
this.version = semver.parse(version);
this.checksums = checksums;
this.dependencies = dependencies;
this.size = size;
this.buildfile = buildfile;
this.checksum = checksum;
this.download = download;
this.signature = signature;
this.repo = repo;
}
get installed(){
@ -31,7 +23,7 @@ class Package {
}
get download_url(){
return helpers.add_url_base_if_required(this.download, this.repo.base_u_r_l);
return this.download;
}
get install_path(){
@ -55,51 +47,26 @@ class Package {
logger.debug(`Downloading package from ${this.download_url} in to ${this.install_path}`);
const pkgfile = helpers.url_basename(this.download_url);
const pkgpath = path.join(this.install_path, pkgfile);
await helpers.buffer_from_url(this.download_url)
.then(buf=> fs.write_file(pkgpath, buf));
logger.debug('Validating checksums');
Object.keys(this.checksums).forEach(algo => {
var val = this.checksums[algo];
logger.debug(`Assert ${algo}(${pkgpath}) == ${val}`);
var cs = crypto.create_hash(algo)
.update(fss.read_file_sync(pkgpath))
.digest('hex');
if(cs != val) throw new Error(`Checksum miss-match want: ${val} got: ${cs}`);
const pkgpath = path.join(this.install_path, "pkg.tar.gz");
const download = await fetch(this.download_url);
const file_stream = fss.create_write_stream(pkgpath);
await new Promise((resolve, reject) => {
download.body.pipe(file_stream)
download.body.on("error", reject)
file_stream.on("finish", resolve)
});
await this.repo.import_keys();
logger.debug('Validating checksums');
logger.debug(`Assert sha256(pkg.tar.gz) == ${this.checksum}`)
const cs = crypto.create_hash("sha256")
.update(fss.readFileSync(pkgpath))
.digest('hex');
if(cs != this.checksum) throw new Error(`Checksum miss-match want: ${val} got: ${cs}`);
logger.debug('Validating signatures');
if(this.signature != '')
await new Promise((resolve,reject)=>{
const gpgspawn = cp.spawn('gpg', ['--verify', '-', pkgpath], {
stdio: ['pipe', 'ignore', 'ignore']
});
gpgspawn.once('exit', (code, _) => {
if(code == 0) resolve();
else reject(new Error('Invalid signature'));
});
gpgspawn.once('error', reject);
gpgspawn.stdin.write(this.signature);
gpgspawn.stdin.end();
});
else
logger.warn('Package does not contain a signature - allowing install, but proceed with caution');
logger.debug(`Extracting package files from archive ${pkgfile} in to ${this.install_path}`);
logger.debug(`Extracting package files from archive ${pkgpath} in to ${this.install_path}`);
await new Promise((resolve, reject)=>{
const proc = cp.exec(`bash -c 'cd "${this.install_path}" && tar xzf ${pkgfile}'`);
const proc = cp.exec(`bash -c 'cd "${this.install_path}" && tar xzf ${pkgpath}'`);
proc.once('exit', (code,_)=>{
if(code == 0) resolve();
else reject(new Error('Failed to extract package'));
@ -110,38 +77,12 @@ class Package {
proc.once('error', reject);
});
logger.debug('Ensuring binary files exist for package');
const pkgbin = path.join(this.install_path, `${this.language}-${this.version.raw}`);
try{
const pkgbin_stat = await fs.stat(pkgbin);
//eslint-disable-next-line snakecasejs/snakecasejs
if(!pkgbin_stat.isDirectory()) throw new Error();
// Throw a blank error here, so it will be caught by the following catch, and output the correct error message
// The catch is used to catch fs.stat
}catch(err){
throw new Error(`Invalid package: could not find ${this.language}-${this.version.raw}/ contained within package files`);
}
logger.debug('Symlinking into runtimes');
await fs.symlink(
pkgbin,
path.join(config.data_directory,
globals.data_directories.runtimes,
`${this.language}-${this.version.raw}`)
).catch((err)=>err); //Ignore if we fail - probably means its already been installed and not cleaned up right
logger.debug('Registering runtime');
const pkg_runtime = new runtime.Runtime(this.install_path);
new runtime.Runtime(this.install_path);
logger.debug('Caching environment');
const required_pkgs = [pkg_runtime, ...pkg_runtime.get_all_dependencies()];
const get_env_command = [
...required_pkgs.map(pkg=>`cd "${pkg.runtime_dir}"; source environment; `),
'env'
].join(' ');
const get_env_command = `cd ${this.install_path}; source environment; env`;
const envout = await new Promise((resolve, reject)=>{
var stdout = '';

65
api/src/ppman/repo.js
View file

@ -1,65 +0,0 @@
const logger = require('logplease').create('ppman/repo');
const cache = require('../cache');
const CACHE_CONTEXT = 'repo';
const cp = require('child_process');
const yaml = require('js-yaml');
const { Package } = require('./package');
const helpers = require('../helpers');
class Repository {
constructor(slug, url){
this.slug = slug;
this.url = new URL(url);
this.keys = [];
this.packages = [];
this.base_u_r_l='';
logger.debug(`Created repo slug=${this.slug} url=${this.url}`);
}
get cache_key(){
return cache.cache_key(CACHE_CONTEXT, this.slug);
}
async load(){
try{
var index = await cache.get(this.cache_key,async ()=>{
return helpers.buffer_from_url(this.url);
});
var repo = yaml.load(index);
if(repo.schema != 'ppman-repo-1'){
throw new Error('YAML Schema unknown');
}
this.keys = repo.keys;
this.packages = repo.packages.map(pkg => new Package(this, pkg));
this.base_u_r_l = repo.baseurl;
}catch(err){
logger.error(`Failed to load repository ${this.slug}:`,err.message);
}
}
async import_keys(){
await this.load();
logger.info(`Importing keys for repo ${this.slug}`);
await new Promise((resolve,reject)=>{
const gpgspawn = cp.spawn('gpg', ['--receive-keys', ...this.keys], {
stdio: ['ignore', 'ignore', 'ignore']
});
gpgspawn.once('exit', (code, _) => {
if(code == 0) resolve();
else reject(new Error('Failed to import keys'));
});
gpgspawn.once('error', reject);
});
}
}
module.exports = {Repository};

151
api/src/ppman/routes.js
View file

@ -1,150 +1,53 @@
const repos = new Map();
const state = require('../state');
const logger = require('logplease').create('ppman/routes');
const {Repository} = require('./repo');
const semver = require('semver');
const { body, param } = require('express-validator');
const fetch = require('node-fetch');
const config = require('../config');
const { Package } = require('./package');
async function get_or_construct_repo(slug){
if(repos.has(slug))return repos.get(slug);
if(state.state.get('repositories').has(slug)){
const repo_url = state.state.get('repositories').get(slug);
const repo = new Repository(slug, repo_url);
await repo.load();
repos.set(slug, repo);
return repo;
}
logger.warn(`Requested repo ${slug} does not exist`);
return null;
async function get_package_list(){
const repo_content = await fetch(config.repo_url).then(x=>x.text());
const entries = repo_content.split('\n').filter(x=>x.length > 0);
return entries.map(line => {
const [language, version, checksum, download] = line.split(',',4);
return new Package({language, version, checksum, download});
})
}
async function get_package(repo, lang, version){
var candidates = repo.packages.filter(
async function get_package(lang, version){
const packages = await get_package_list();
const candidates = packages.filter(
pkg => pkg.language == lang && semver.satisfies(pkg.version, version)
);
return candidates.sort((a,b)=>semver.rcompare(a.version,b.version))[0] || null;
}
module.exports = {
async repo_list(req,res){
// GET /repos
async package_list(req, res){
// GET /packages
logger.debug('Request to list packages');
logger.debug('Request for repoList');
res.json_success({
repos: (await Promise.all(
[...state.state.get('repositories').keys()].map( async slug => await get_or_construct_repo(slug))
)).map(repo=>({
slug: repo.slug,
url: repo.url,
packages: repo.packages.length
}))
});
},
repo_add_validators: [
body('slug')
.notEmpty() // eslint-disable-line snakecasejs/snakecasejs
.bail()
.isSlug() // eslint-disable-line snakecasejs/snakecasejs
.bail()
.not()
.custom(value=>state.state.get('repositories').keys().includes(value))
.withMessage('slug is already in use'), // eslint-disable-line snakecasejs/snakecasejs
body('url')
.notEmpty() // eslint-disable-line snakecasejs/snakecasejs
.bail()
.isURL({require_host: false, require_protocol: true, protocols: ['http','https','file']}) // eslint-disable-line snakecasejs/snakecasejs
],
async repo_add(req, res){
// POST /repos
logger.debug(`Request for repoAdd slug=${req.body.slug} url=${req.body.url}`);
const repo_state = state.state.get('repositories');
repo_state.set(req.body.slug, req.body.url);
logger.info(`Repository ${req.body.slug} added url=${req.body.url}`);
return res.json_success(req.body.slug);
},
repo_info_validators: [
param('repo_slug')
.isSlug() // eslint-disable-line snakecasejs/snakecasejs
.bail()
.custom(value=>state.state.get('repositories').has(value))
.withMessage('repository does not exist') // eslint-disable-line snakecasejs/snakecasejs
.bail()
],
async repo_info(req, res){
// GET /repos/:slug
logger.debug(`Request for repoInfo for ${req.params.repo_slug}`);
const repo = await get_or_construct_repo(req.params.repo_slug);
res.json_success({
slug: repo.slug,
url: repo.url,
packages: repo.packages.length
});
},
repo_packages_validators: [
param('repo_slug')
.isSlug() // eslint-disable-line snakecasejs/snakecasejs
.bail()
.custom(value=>state.state.get('repositories').has(value))
.withMessage('repository does not exist') // eslint-disable-line snakecasejs/snakecasejs
.bail()
],
async repo_packages(req, res){
// GET /repos/:slug/packages
logger.debug('Request to repoPackages');
const repo = await get_or_construct_repo(req.params.repo_slug);
if(repo == null) return res.json_error(`Requested repo ${req.params.repo_slug} does not exist`, 404);
const packages = await get_package_list();
res.json_success({
packages: repo.packages.map(pkg=>({
packages: packages.map(pkg=>({
language: pkg.language,
language_version: pkg.version.raw,
installed: pkg.installed
}))
});
},
package_info_validators: [
param('repo_slug')
.isSlug() // eslint-disable-line snakecasejs/snakecasejs
.bail()
.custom(value=>state.state.get('repositories').has(value))
.withMessage('repository does not exist') // eslint-disable-line snakecasejs/snakecasejs
.bail()
],
async package_info(req, res){
// GET /repos/:slug/packages/:language/:version
logger.debug('Request to packageInfo');
const repo = await get_or_construct_repo(req.params.repo_slug);
const pkg = await get_package(repo, req.params.language, req.params.version);
if(pkg == null) return res.json_error(`Requested package ${req.params.language}-${req.params.version} does not exist`, 404);
res.json_success({
language: pkg.language,
language_version: pkg.version.raw,
author: pkg.author,
buildfile: pkg.buildfile,
size: pkg.size,
dependencies: pkg.dependencies,
installed: pkg.installed
});
},
async package_install(req,res){
// POST /repos/:slug/packages/:language/:version
// POST /packages/:language/:version
logger.debug('Request to packageInstall');
logger.debug('Request to install package');
const repo = await get_or_construct_repo(req.params.repo_slug);
const pkg = await get_package(repo, req.params.language, req.params.version);
const pkg = await get_package(req.params.language, req.params.version);
if(pkg == null) return res.json_error(`Requested package ${req.params.language}-${req.params.version} does not exist`, 404);
try{
@ -158,7 +61,7 @@ module.exports = {
},
async package_uninstall(req,res){
// DELETE /repos/:slug/packages/:language/:version
// DELETE /packages/:language/:version
//res.json(req.body); //TODO
res.json_error('not implemented', 500);