Mascali
/
piston
mirror of https://github.com/engineer-man/piston.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

BREAKING: replace custom build scripts with nix 

General:
- Switched to yarn to better work with nix-based tooling
- Switched package system to use nix. This stops double dependencies and slow cloud compile times, while providing more compile/runtime support to the Nix project
- Removed container builder in favor of internal container tooling
- Package versions no-longer need to be SemVer compliant
- Removed "piston package spec" files, replaced with nix-flake based runtimes
- Exported nosocket and piston-api as packages within the nix-flake
- Removed repo container
- Switched docker building to nix-based container outputting
- Removed docker compose as this is a single container
- Removed package commands from CLI

Packages:
- Move bash, clojure, cobol, node, python2, python3 to new format
- Remainder of packages still need to be moved

v2 API:
- Removed "version" specifier. To select specific versions, use the v3 api
- Removed "/package" endpoints as this doesn't work with the new nix-based system

v3 API:
- Duplicate of v2 API, except instead of passing in a language name an ID is used intead.
This commit is contained in:
Thomas Hobson 2022-01-30 18:41:24 +13:00
parent e06b59d82c
commit 564da5a7eb
No known key found for this signature in database
GPG Key ID: 9F1FD9D87950DB6F
110 changed files with 2293 additions and 2793 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -1,3 +1,4 @@
data/
.piston_env
node_modules
result

37
api/Dockerfile
View File

@ -1,37 +0,0 @@
FROM node:15.10.0-buster-slim
ENV DEBIAN_FRONTEND=noninteractive
RUN dpkg-reconfigure -p critical dash
RUN for i in $(seq 1001 1500); do \
groupadd -g $i runner$i && \
useradd -M runner$i -g $i -u $i ; \
done
RUN apt-get update && \
apt-get install -y libxml2 gnupg tar coreutils util-linux libc6-dev \
binutils build-essential locales libpcre3-dev libevent-dev libgmp3-dev \
libncurses6 libncurses5 libedit-dev libseccomp-dev rename procps python3 \
libreadline-dev libblas-dev liblapack-dev libpcre3-dev libarpack2-dev \
libfftw3-dev libglpk-dev libqhull-dev libqrupdate-dev libsuitesparse-dev \
libsundials-dev libpcre2-dev curl sudo && \
rm -rf /var/lib/apt/lists/*
RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen
RUN mkdir -m 0755 /nix && chown node /nix && touch /nix/piston_detected
RUN runuser -l node -c 'curl -L https://nixos.org/nix/install | sh -s -- --no-daemon '
RUN runuser -l node -c 'source ~/.profile; nix-env -iA nixpkgs.nixUnstable'
RUN runuser -l node -c 'mkdir -p /home/node/.config/nix/; echo "experimental-features = nix-command flakes" >> /home/node/.config/nix/nix.conf'
RUN cp -r /nix /var/nix
WORKDIR /piston_api
COPY ["package.json", "package-lock.json", "./"]
RUN npm install
COPY ./src ./src
RUN make -C ./src/nosocket/ all && make -C ./src/nosocket/ install
COPY ./entrypoint.sh .
CMD [ "./entrypoint.sh"]
EXPOSE 2000/tcp

28
api/default.nix Normal file
View File

@ -0,0 +1,28 @@
{pkgs, ...}:
with pkgs; {
package = mkYarnPackage {
name = "piston";
src = ./.;
yarnPreBuild = ''
mkdir -p $HOME/.node-gyp/${nodejs.version}
echo 9 > $HOME/.node-gyp/${nodejs.version}/installVersion
ln -sfv ${nodejs}/include $HOME/.node-gyp/${nodejs.version}
export npm_config_nodedir=${nodejs}
'';
pkgConfig = {
waitpid = {
buildInputs = [
gcc
gnumake
python3
];
postInstall = ''
yarn --offline run install
'';
};
};
};
}

7
api/entrypoint.sh
View File

@ -4,8 +4,11 @@ echo "Starting Piston API"
echo "Checking presense of nix store"
if [[ ! -f "/nix/piston_detected" ]]; then
echo "Nix Store is not loaded, assuming /nix has been mounted - copying contents"
cp -r /var/nix /nix
cp -rp /var/nix/* /nix
fi
echo "Adding nix to env"
. ~/.profile
echo "Launching Piston API"
node src
node src

1048
api/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

24
api/package.json
View File

@ -1,20 +1,28 @@
{
"name": "piston-api",
"version": "3.1.0",
"version": "4.0.0",
"description": "API for piston - a high performance code execution engine",
"main": "src/index.js",
"main": "src/pistond.js",
"dependencies": {
"body-parser": "^1.19.0",
"chownr": "^2.0.0",
"express": "^4.17.1",
"express-ws": "^5.0.2",
"is-docker": "^2.1.1",
"logplease": "^1.2.15",
"nocamel": "HexF/nocamel#patch-1",
"node-fetch": "^2.6.1",
"semver": "^7.3.4",
"uuid": "^8.3.2",
"waitpid": "git+https://github.com/HexF/node-waitpid.git"
"nocamel": "git://github.com/HexF/nocamel.git#patch-1",
"waitpid": "git://github.com/HexF/node-waitpid.git"
},
"license": "MIT"
"license": "MIT",
"scripts": {
"lint": "prettier . --write",
"prepack": "yarn2nix > yarn.nix"
},
"devDependencies": {
"node2nix": "^1.6.0",
"prettier": "2.2.1"
},
"bin": {
"pistond": "./src/pistond.js"
}
}

131
api/src/api/v2.js
View File

@ -5,8 +5,7 @@ const events = require('events');
const runtime = require('../runtime');
const { Job } = require('../job');
const package = require('../package');
const logger = require('logplease').create('api/v2');
const logger = require('logplease').create('api/v3');
const SIGNALS = [
'SIGABRT',
@ -53,7 +52,6 @@ const SIGNALS = [
function get_job(body) {
let {
language,
version,
args,
stdin,
files,
@ -69,11 +67,7 @@ function get_job(body) {
message: 'language is required as a string',
});
}
if (!version || typeof version !== 'string') {
return reject({
message: 'version is required as a string',
});
}
if (!files || !Array.isArray(files)) {
return reject({
message: 'files is required as an array',
@ -87,10 +81,50 @@ function get_job(body) {
}
}
const rt = runtime.get_latest_runtime_matching_language_version(
language,
version
);
if (compile_memory_limit) {
if (typeof compile_memory_limit !== 'number') {
return reject({
message: 'if specified, compile_memory_limit must be a number',
});
}
if (
config.compile_memory_limit >= 0 &&
(compile_memory_limit > config.compile_memory_limit ||
compile_memory_limit < 0)
) {
return reject({
message:
'compile_memory_limit cannot exceed the configured limit of ' +
config.compile_memory_limit,
});
}
}
if (run_memory_limit) {
if (typeof run_memory_limit !== 'number') {
return reject({
message: 'if specified, run_memory_limit must be a number',
});
}
if (
config.run_memory_limit >= 0 &&
(run_memory_limit > config.run_memory_limit || run_memory_limit < 0)
) {
return reject({
message:
'run_memory_limit cannot exceed the configured limit of ' +
config.run_memory_limit,
});
}
}
const rt = runtime.find(rt => [
...rt.aliases,
rt.language
].includes(rt.language))
if (rt === undefined) {
return reject({
message: `${language}-${version} runtime is unknown`,
@ -298,77 +332,4 @@ router.get('/runtimes', (req, res) => {
return res.status(200).send(runtimes);
});
router.get('/packages', async (req, res) => {
logger.debug('Request to list packages');
let packages = await package.get_package_list();
packages = packages.map(pkg => {
return {
language: pkg.language,
language_version: pkg.version.raw,
installed: pkg.installed,
};
});
return res.status(200).send(packages);
});
router.post('/packages', async (req, res) => {
logger.debug('Request to install package');
const { language, version } = req.body;
const pkg = await package.get_package(language, version);
if (pkg == null) {
return res.status(404).send({
message: `Requested package ${language}-${version} does not exist`,
});
}
try {
const response = await pkg.install();
return res.status(200).send(response);
} catch (e) {
logger.error(
`Error while installing package ${pkg.language}-${pkg.version}:`,
e.message
);
return res.status(500).send({
message: e.message,
});
}
});
router.delete('/packages', async (req, res) => {
logger.debug('Request to uninstall package');
const { language, version } = req.body;
const pkg = await package.get_package(language, version);
if (pkg == null) {
return res.status(404).send({
message: `Requested package ${language}-${version} does not exist`,
});
}
try {
const response = await pkg.uninstall();
return res.status(200).send(response);
} catch (e) {
logger.error(
`Error while uninstalling package ${pkg.language}-${pkg.version}:`,
e.message
);
return res.status(500).send({
message: e.message,
});
}
});
module.exports = router;

236
api/src/api/v3.js Normal file
View File

@ -0,0 +1,236 @@
const express = require('express');
const router = express.Router();
const events = require('events');
const config = require('../config');
const runtime = require('../runtime');
const { Job } = require('../job');
const logger = require('logplease').create('api/v3');
const SIGNALS = ["SIGABRT","SIGALRM","SIGBUS","SIGCHLD","SIGCLD","SIGCONT","SIGEMT","SIGFPE","SIGHUP","SIGILL","SIGINFO","SIGINT","SIGIO","SIGIOT","SIGKILL","SIGLOST","SIGPIPE","SIGPOLL","SIGPROF","SIGPWR","SIGQUIT","SIGSEGV","SIGSTKFLT","SIGSTOP","SIGTSTP","SIGSYS","SIGTERM","SIGTRAP","SIGTTIN","SIGTTOU","SIGUNUSED","SIGURG","SIGUSR1","SIGUSR2","SIGVTALRM","SIGXCPU","SIGXFSZ","SIGWINCH"]
// ref: https://man7.org/linux/man-pages/man7/signal.7.html
function get_job(body){
const {
runtime_id,
args,
stdin,
files,
compile_memory_limit,
run_memory_limit,
run_timeout,
compile_timeout
} = body;
return new Promise((resolve, reject) => {
if (typeof runtime_id !== 'number') {
return reject({
message: 'runtime_id is required as a number'
});
}
if (!Array.isArray(files)) {
return reject({
message: 'files is required as an array',
});
}
for (const [i, file] of files.entries()) {
if (typeof file.content !== 'string') {
return reject({
message: `files[${i}].content is required as a string`,
});
}
}
if (compile_memory_limit) {
if (typeof compile_memory_limit !== 'number') {
return reject({
message: 'if specified, compile_memory_limit must be a number',
});
}
if (
config.compile_memory_limit >= 0 &&
(compile_memory_limit > config.compile_memory_limit ||
compile_memory_limit < 0)
) {
return reject({
message:
'compile_memory_limit cannot exceed the configured limit of ' +
config.compile_memory_limit,
});
}
}
if (run_memory_limit) {
if (typeof run_memory_limit !== 'number') {
return reject({
message: 'if specified, run_memory_limit must be a number',
});
}
if (
config.run_memory_limit >= 0 &&
(run_memory_limit > config.run_memory_limit || run_memory_limit < 0)
) {
return reject({
message:
'run_memory_limit cannot exceed the configured limit of ' +
config.run_memory_limit,
});
}
}
const rt = runtime[runtime_id];
if (rt === undefined) {
return reject({
message: `Runtime #${runtime_id} is unknown`,
});
}
resolve(new Job({
runtime: rt,
args: args || [],
stdin: stdin || "",
files,
timeouts: {
run: run_timeout || 3000,
compile: compile_timeout || 10000,
},
memory_limits: {
run: run_memory_limit || config.run_memory_limit,
compile: compile_memory_limit || config.compile_memory_limit,
}
}));
})
}
router.use((req, res, next) => {
if (['GET', 'HEAD', 'OPTIONS'].includes(req.method)) {
return next();
}
if (!req.headers['content-type'].startsWith('application/json')) {
return res.status(415).send({
message: 'requests must be of type application/json',
});
}
next();
});
router.ws('/connect', async (ws, req) => {
let job = null;
let eventBus = new events.EventEmitter();
eventBus.on("stdout", (data) => ws.send(JSON.stringify({type: "data", stream: "stdout", data: data.toString()})))
eventBus.on("stderr", (data) => ws.send(JSON.stringify({type: "data", stream: "stderr", data: data.toString()})))
eventBus.on("stage", (stage)=> ws.send(JSON.stringify({type: "stage", stage})))
eventBus.on("exit", (stage, status) => ws.send(JSON.stringify({type: "exit", stage, ...status})))
ws.on("message", async (data) => {
try{
const msg = JSON.parse(data);
switch(msg.type){
case "init":
if(job === null){
job = await get_job(msg);
await job.prime();
ws.send(JSON.stringify({
type: "runtime",
language: job.runtime.language,
version: job.runtime.version.raw
}))
await job.execute_interactive(eventBus);
ws.close(4999, "Job Completed");
}else{
ws.close(4000, "Already Initialized");
}
break;
case "data":
if(job !== null){
if(msg.stream === "stdin"){
eventBus.emit("stdin", msg.data)
}else{
ws.close(4004, "Can only write to stdin")
}
}else{
ws.close(4003, "Not yet initialized")
}
break;
case "signal":
if(job !== null){
if(SIGNALS.includes(msg.signal)){
eventBus.emit("signal", msg.signal)
}else{
ws.close(4005, "Invalid signal")
}
}else{
ws.close(4003, "Not yet initialized")
}
break;
}
}catch(error){
ws.send(JSON.stringify({type: "error", message: error.message}))
ws.close(4002, "Notified Error")
// ws.close message is limited to 123 characters, so we notify over WS then close.
}
})
ws.on("close", async ()=>{
if(job !== null){
await job.cleanup()
}
})
setTimeout(()=>{
//Terminate the socket after 1 second, if not initialized.
if(job === null)
ws.close(4001, "Initialization Timeout");
}, 1000)
})
router.post('/execute', async (req, res) => {
try{
const job = await get_job(req.body);
await job.prime();
const result = await job.execute();
await job.cleanup();
return res.status(200).send(result);
}catch(error){
return res.status(400).json(error);
}
});
router.get('/runtimes', (req, res) => {
const runtimes = runtime.map(rt => {
return {
language: rt.language,
version: rt.version.raw,
aliases: rt.aliases,
runtime: rt.runtime,
id: rt.id
};
});
return res.status(200).send(runtimes);
});
module.exports = router;

7
api/src/config.js
View File

@ -171,10 +171,9 @@ const options = [
validators: [(x, raw) => !is_nan(x) || `${raw} is not a number`],
},
{
key: 'repo_url',
desc: 'URL of repo index',
default:
'https://github.com/engineer-man/piston/releases/download/pkgs/index',
key: 'flake_path',
desc: 'Path to nix flake defining runtimes to install',
default: 'github:engineer-man/piston?directory=packages',
validators: [],
},
{

11
api/src/globals.js
View File

@ -1,20 +1,9 @@
// Globals are things the user shouldn't change in config, but is good to not use inline constants for
const is_docker = require('is-docker');
const fs = require('fs');
const platform = `${is_docker() ? 'docker' : 'baremetal'}-${fs
.read_file_sync('/etc/os-release')
.toString()
.split('\n')
.find(x => x.startsWith('ID'))
.replace('ID=', '')}`;
module.exports = {
data_directories: {
packages: 'packages',
jobs: 'jobs',
},
version: require('../package.json').version,
platform,
pkg_installed_file: '.ppman-installed', //Used as indication for if a package was installed
clean_directories: ['/dev/shm', '/run/lock', '/tmp', '/var/tmp'],
};

6
api/src/job.js
View File

@ -240,8 +240,8 @@ class Job {
if (this.runtime.compiled) {
compile = await this.safe_call(
path.join(this.runtime.pkgdir, 'compile'),
code_files.map(x => x.name),
this.runtime.compile,
this.files.map(x => x.name),
this.timeouts.compile,
this.memory_limits.compile
);
@ -262,7 +262,7 @@ class Job {
compile,
run,
language: this.runtime.language,
version: this.runtime.version.raw,
version: this.runtime.version,
};
}

226
api/src/package.js
View File

@ -1,226 +0,0 @@
const logger = require('logplease').create('package');
const semver = require('semver');
const config = require('./config');
const globals = require('./globals');
const fetch = require('node-fetch');
const path = require('path');
const fs = require('fs/promises');
const fss = require('fs');
const cp = require('child_process');
const crypto = require('crypto');
const runtime = require('./runtime');
const chownr = require('chownr');
const util = require('util');
class Package {
constructor({ language, version, download, checksum }) {
this.language = language;
this.version = semver.parse(version);
this.checksum = checksum;
this.download = download;
}
get installed() {
return fss.exists_sync(
path.join(this.install_path, globals.pkg_installed_file)
);
}
get install_path() {
return path.join(
config.data_directory,
globals.data_directories.packages,
this.language,
this.version.raw
);
}
async install() {
if (this.installed) {
throw new Error('Already installed');
}
logger.info(`Installing ${this.language}-${this.version.raw}`);
if (fss.exists_sync(this.install_path)) {
logger.warn(
`${this.language}-${this.version.raw} has residual files. Removing them.`
);
await fs.rm(this.install_path, { recursive: true, force: true });
}
logger.debug(`Making directory ${this.install_path}`);
await fs.mkdir(this.install_path, { recursive: true });
logger.debug(
`Downloading package from ${this.download} in to ${this.install_path}`
);
const pkgpath = path.join(this.install_path, 'pkg.tar.gz');
const download = await fetch(this.download);
const file_stream = fss.create_write_stream(pkgpath);
await new Promise((resolve, reject) => {
download.body.pipe(file_stream);
download.body.on('error', reject);
file_stream.on('finish', resolve);
});
logger.debug('Validating checksums');
logger.debug(`Assert sha256(pkg.tar.gz) == ${this.checksum}`);
const hash = crypto.create_hash('sha256');
const read_stream = fss.create_read_stream(pkgpath);
await new Promise((resolve, reject) => {
read_stream.on('data', chunk => hash.update(chunk));
read_stream.on('end', () => resolve());
read_stream.on('error', error => reject(error));
});
const cs = hash.digest('hex');
if (cs !== this.checksum) {
throw new Error(`Checksum miss-match want: ${val} got: ${cs}`);
}
logger.debug(
`Extracting package files from archive ${pkgpath} in to ${this.install_path}`
);
await new Promise((resolve, reject) => {
const proc = cp.exec(
`bash -c 'cd "${this.install_path}" && tar xzf ${pkgpath}'`
);
proc.once('exit', (code, _) => {
code === 0 ? resolve() : reject();
});
proc.stdout.pipe(process.stdout);
proc.stderr.pipe(process.stderr);
proc.once('error', reject);
});
logger.debug('Registering runtime');
runtime.load_package(this.install_path);
logger.debug('Caching environment');
const get_env_command = `cd ${this.install_path}; source environment; env`;
const envout = await new Promise((resolve, reject) => {
let stdout = '';
const proc = cp.spawn(
'env',
['-i', 'bash', '-c', `${get_env_command}`],
{
stdio: ['ignore', 'pipe', 'pipe'],
}
);
proc.once('exit', (code, _) => {
code === 0 ? resolve(stdout) : reject();
});
proc.stdout.on('data', data => {
stdout += data;
});
proc.once('error', reject);
});
const filtered_env = envout
.split('\n')
.filter(
l =>
!['PWD', 'OLDPWD', '_', 'SHLVL'].includes(
l.split('=', 2)[0]
)
)
.join('\n');
await fs.write_file(path.join(this.install_path, '.env'), filtered_env);
logger.debug('Changing Ownership of package directory');
await util.promisify(chownr)(this.install_path, 0, 0);
logger.debug('Writing installed state to disk');
await fs.write_file(
path.join(this.install_path, globals.pkg_installed_file),
Date.now().toString()
);
logger.info(`Installed ${this.language}-${this.version.raw}`);
return {
language: this.language,
version: this.version.raw,
};
}
async uninstall() {
logger.info(`Uninstalling ${this.language}-${this.version.raw}`);
logger.debug('Finding runtime');
const found_runtime = runtime.get_runtime_by_name_and_version(
this.language,
this.version.raw
);
if (!found_runtime) {
logger.error(
`Uninstalling ${this.language}-${this.version.raw} failed: Not installed`
);
throw new Error(
`${this.language}-${this.version.raw} is not installed`
);
}
logger.debug('Unregistering runtime');
found_runtime.unregister();
logger.debug('Cleaning files from disk');
await fs.rmdir(this.install_path, { recursive: true });
logger.info(`Uninstalled ${this.language}-${this.version.raw}`);
return {
language: this.language,
version: this.version.raw,
};
}
static async get_package_list() {
const repo_content = await fetch(config.repo_url).then(x => x.text());
const entries = repo_content.split('\n').filter(x => x.length > 0);
return entries.map(line => {
const [language, version, checksum, download] = line.split(',', 4);
return new Package({
language,
version,
checksum,
download,
});
});
}
static async get_package(lang, version) {
const packages = await Package.get_package_list();
const candidates = packages.filter(pkg => {
return (
pkg.language == lang && semver.satisfies(pkg.version, version)
);
});
candidates.sort((a, b) => semver.rcompare(a.version, b.version));
return candidates[0] || null;
}
}
module.exports = Package;

29
api/src/index.js → api/src/pistond.js Normal file → Executable file
View File

@ -5,6 +5,7 @@ const express = require('express');
const expressWs = require('express-ws');
const globals = require('./globals');
const config = require('./config');
const cp = require('child_process');
const path = require('path');
const fs = require('fs/promises');
const fss = require('fs');
@ -37,28 +38,11 @@ expressWs(app);
});
logger.info('Loading packages');
const pkgdir = path.join(
config.data_directory,
globals.data_directories.packages
);
const pkglist = await fs.readdir(pkgdir);
const languages = await Promise.all(
pkglist.map(lang => {
return fs.readdir(path.join(pkgdir, lang)).then(x => {
return x.map(y => path.join(pkgdir, lang, y));
});
})
);
const installed_languages = languages
.flat()
.filter(pkg =>
fss.exists_sync(path.join(pkg, globals.pkg_installed_file))
);
installed_languages.for_each(pkg => runtime.load_package(pkg));
const runtimes_data = cp.execSync(`nix eval --json ${config.flake_path}#pistonRuntimes --apply builtins.attrNames`).toString();
const runtimes = JSON.parse(runtimes_data);
runtimes.for_each(pkg => runtime.load_runtime(pkg));
logger.info('Starting API Server');
logger.debug('Constructing Express App');
@ -76,8 +60,9 @@ expressWs(app);
logger.debug('Registering Routes');
const api_v2 = require('./api/v2');
const api_v3 = require('./api/v3');
app.use('/api/v2', api_v2);
app.use('/api/v2', api_v2);
app.use('/api/v3', api_v3);
app.use((req, res, next) => {
return res.status(404).send({ message: 'Not Found' });

161
api/src/runtime.js
View File

@ -1,5 +1,5 @@
const logger = require('logplease').create('runtime');
const semver = require('semver');
const cp = require('child_process');
const config = require('./config');
const globals = require('./globals');
const fss = require('fs');
@ -7,32 +7,44 @@ const path = require('path');
const runtimes = [];
class Runtime {
constructor({
language,
version,
aliases,
pkgdir,
runtime,
run,
compile,
packageSupport,
flake_key,
timeouts,
memory_limits,
max_process_count,
max_open_files,
max_file_size,
output_max_size,
output_max_size
}) {
this.language = language;
this.version = version;
this.aliases = aliases || [];
this.pkgdir = pkgdir;
this.runtime = runtime;
this.timeouts = timeouts;
this.memory_limits = memory_limits;
this.max_process_count = max_process_count;
this.max_open_files = max_open_files;
this.max_file_size = max_file_size;
this.output_max_size = output_max_size;
this.aliases = aliases;
this.version = version;
this.run = run;
this.compile = compile;
this.flake_key = flake_key;
this.package_support = packageSupport;
}
static compute_single_limit(
language_name,
@ -97,122 +109,61 @@ class Runtime {
};
}
static load_package(package_dir) {
let info = JSON.parse(
fss.read_file_sync(path.join(package_dir, 'pkg-info.json'))
);
ensure_built(){
logger.info(`Ensuring ${this} is built`);
let {
language,
version,
build_platform,
aliases,
provides,
limit_overrides,
} = info;
version = semver.parse(version);
const flake_key = this.flake_key;
if (build_platform !== globals.platform) {
logger.warn(
`Package ${language}-${version} was built for platform ${build_platform}, ` +
`but our platform is ${globals.platform}`
);
function _ensure_built(key){
const command = `nix build ${config.flake_path}#pistonRuntimes.${flake_key}.metadata.${key} --no-link`;
cp.execSync(command, {stdio: "pipe"})
}
if (provides) {
// Multiple languages in 1 package
provides.forEach(lang => {
runtimes.push(
new Runtime({
language: lang.language,
aliases: lang.aliases,
version,
pkgdir: package_dir,
runtime: language,
...Runtime.compute_all_limits(
lang.language,
lang.limit_overrides
),
})
);
});
} else {
runtimes.push(
new Runtime({
language,
version,
aliases,
pkgdir: package_dir,
...Runtime.compute_all_limits(language, limit_overrides),
})
);
}
_ensure_built("run");
if(this.compiled) _ensure_built("compile");
logger.debug(`Finished ensuring ${this} is installed`)
}
static load_runtime(flake_key){
logger.info(`Loading ${flake_key}`)
const metadata_command = `nix eval --json ${config.flake_path}#pistonRuntimes.${flake_key}.metadata`;
const metadata = JSON.parse(cp.execSync(metadata_command));
const this_runtime = new Runtime({
...metadata,
...Runtime.compute_all_limits(
metadata.language,
metadata.limit_overrides
),
flake_key
});
this_runtime.ensure_built();
runtimes.push(this_runtime);
logger.debug(`Package ${flake_key} was loaded`);
logger.debug(`Package ${language}-${version} was loaded`);
}
get compiled() {
if (this._compiled === undefined) {
this._compiled = fss.exists_sync(path.join(this.pkgdir, 'compile'));
}
return this._compiled;
return this.compile !== null;
}
get env_vars() {
if (!this._env_vars) {
const env_file = path.join(this.pkgdir, '.env');
const env_content = fss.read_file_sync(env_file).toString();
this._env_vars = {};
env_content
.trim()
.split('\n')
.map(line => line.split('=', 2))
.forEach(([key, val]) => {
this._env_vars[key.trim()] = val.trim();
});
}
return this._env_vars;
get id(){
return runtimes.indexOf(this);
}
toString() {
return `${this.language}-${this.version.raw}`;
return `${this.language}-${this.version}`;
}
unregister() {
const index = runtimes.indexOf(this);
runtimes.splice(index, 1); //Remove from runtimes list
}
}
module.exports = runtimes;
module.exports.Runtime = Runtime;
module.exports.get_runtimes_matching_language_version = function (lang, ver) {
return runtimes.filter(
rt =>
(rt.language == lang || rt.aliases.includes(lang)) &&
semver.satisfies(rt.version, ver)
);
};
module.exports.get_latest_runtime_matching_language_version = function (
lang,
ver
) {
return module.exports
.get_runtimes_matching_language_version(lang, ver)
.sort((a, b) => semver.rcompare(a.version, b.version))[0];
};
module.exports.load_runtime = Runtime.load_runtime;
module.exports.get_runtime_by_name_and_version = function (runtime, ver) {
return runtimes.find(
rt =>
(rt.runtime == runtime ||
(rt.runtime === undefined && rt.language == runtime)) &&
semver.satisfies(rt.version, ver)
);
};
module.exports.load_package = Runtime.load_package;

1406
api/yarn.lock Normal file
View File

File diff suppressed because it is too large Load Diff

1
builder/.gitignore vendored
View File

@ -1 +0,0 @@
build

2
builder/Dockerfile
View File

@ -1,2 +0,0 @@
FROM ghcr.io/engineer-man/piston:latest
ADD . /piston/packages/

61
builder/build.sh
View File

@ -1,61 +0,0 @@
#!/usr/bin/env bash
# Build a container using the spec file provided
help_msg(){
echo "Usage: $0 [specfile] [tag]"
echo
echo "$1"
exit 1
}
cleanup(){
echo "Exiting..."
docker stop builder_piston_instance && docker rm builder_piston_instance
}
fetch_packages(){
local port=$((5535 + $RANDOM % 60000))
mkdir build
# Start a piston container
docker run \
-v "$PWD/build":'/piston/packages' \
--tmpfs /piston/jobs \
-dit \
-p $port:2000 \
--name builder_piston_instance \
ghcr.io/engineer-man/piston
# Ensure the CLI is installed
cd ../cli
npm i
cd -
# Evalulate the specfile
../cli/index.js -u "http://127.0.0.1:$port" ppman spec $1
}
build_container(){
docker build -t $1 -f "$(dirname $0)/Dockerfile" "$PWD/build"
}
SPEC_FILE=$1
TAG=$2
[ -z "$SPEC_FILE" ] && help_msg "specfile is required"
[ -z "$TAG" ] && help_msg "tag is required"
[ -f "$SPEC_FILE" ] || help_msg "specfile does not exist"
which node || help_msg "nodejs is required"
which npm || help_msg "npm is required"
trap cleanup EXIT
fetch_packages $SPEC_FILE
build_container $TAG
echo "Start your custom piston container with"
echo "$ docker run --tmpfs /piston/jobs -dit -p 2000:2000 $TAG"

5
cli/commands/ppman.js
View File

@ -1,5 +0,0 @@
exports.command = 'ppman';
exports.aliases = ['pkg'];
exports.describe = 'Package Manager';
exports.builder = yargs => yargs.commandDir('ppman_commands').demandCommand();

39
cli/commands/ppman_commands/install.js
View File

@ -1,39 +0,0 @@
const chalk = require('chalk');
exports.command = ['install <packages...>'];
exports.aliases = ['i'];
exports.describe = 'Installs the named package';
//Splits the package into it's language and version
function split_package(package) {
[language, language_version] = package.split('=');
res = {
language: language,
version: language_version || '*',
};
return res;
}
const msg_format = {
color: p =>
`${
p.language ? chalk.green.bold('✓') : chalk.red.bold('❌')
} Installation ${p.language ? 'succeeded' : 'failed: ' + p.message}`,
monochrome: p =>
`Installation ${p.language ? 'succeeded' : 'failed: ' + p.message}`,
json: JSON.stringify,
};
exports.handler = async ({ axios, packages }) => {
const requests = packages.map(package => split_package(package));
for (request of requests) {
try {
const install = await axios.post(`/api/v2/packages`, request);
console.log(msg_format.color(install.data));
} catch ({ response }) {
console.error(response.data.message);
}
}
};

25
cli/commands/ppman_commands/list.js
View File

@ -1,25 +0,0 @@
const chalk = require('chalk');
exports.command = ['list'];
exports.aliases = ['l'];
exports.describe = 'Lists all available packages';
const msg_format = {
color: p =>
`${chalk[p.installed ? 'green' : 'red']('•')} ${p.language} ${
p.language_version
}`,
monochrome: p =>
`${p.language} ${p.language_version} ${
p.installed ? '(INSTALLED)' : ''
}`,
json: JSON.stringify,
};
exports.handler = async ({ axios }) => {
const packages = await axios.get('/api/v2/packages');
const pkg_msg = packages.data.map(msg_format.color).join('\n');
console.log(pkg_msg);
};

183
cli/commands/ppman_commands/spec.js
View File

@ -1,183 +0,0 @@
const chalk = require('chalk');
const fs = require('fs/promises');
const minimatch = require('minimatch');
const semver = require('semver');
exports.command = ['spec <specfile>'];
exports.aliases = ['s'];
exports.describe =
"Install the packages described in the spec file, uninstalling packages which aren't in the list";
function does_match(package, rule) {
const nameMatch = minimatch(package.language, rule.package_selector);
const versionMatch = semver.satisfies(
package.language_version,
rule.version_selector
);
return nameMatch && versionMatch;
}
exports.handler = async ({ axios, specfile }) => {
const spec_contents = await fs.readFile(specfile);
const spec_lines = spec_contents.toString().split('\n');
const rules = [];
for (const line of spec_lines) {
const rule = {
_raw: line.trim(),
comment: false,
package_selector: null,
version_selector: null,
negate: false,
};
if (line.starts_with('#')) {
rule.comment = true;
} else {
let l = line.trim();
if (line.starts_with('!')) {
rule.negate = true;
l = line.slice(1).trim();
}
const [pkg, ver] = l.split(' ', 2);
rule.package_selector = pkg;
rule.version_selector = ver;
}
if (rule._raw.length != 0) rules.push(rule);
}
const packages_req = await axios.get('/api/v2/packages');
const packages = packages_req.data;
const installed = packages.filter(pkg => pkg.installed);
let ensure_packages = [];
for (const rule of rules) {
if (rule.comment) continue;
const matches = [];
if (!rule.negate) {
for (const package of packages) {
if (does_match(package, rule)) matches.push(package);
}
const latest_matches = matches.filter(pkg => {
const versions = matches
.filter(x => x.language == pkg.language)
.map(x => x.language_version)
.sort(semver.rcompare);
return versions[0] == pkg.language_version;
});
for (const match of latest_matches) {
if (
!ensure_packages.find(
pkg =>
pkg.language == match.language &&
pkg.language_version == match.language_version
)
)
ensure_packages.push(match);
}
} else {
ensure_packages = ensure_packages.filter(
pkg => !does_match(pkg, rule)
);
}
}
const operations = [];
for (const package of ensure_packages) {
if (!package.installed)
operations.push({
type: 'install',
package: package.language,
version: package.language_version,
});
}
for (const installed_package of installed) {
if (
!ensure_packages.find(
pkg =>
pkg.language == installed_package.language &&
pkg.language_version == installed_package.language_version
)
)
operations.push({
type: 'uninstall',
package: installed_package.language,
version: installed_package.language_version,
});
}
console.log(chalk.bold.yellow('Actions'));
for (const op of operations) {
console.log(
(op.type == 'install'
? chalk.green('Install')
: chalk.red('Uninstall')) + ` ${op.package} ${op.version}`
);
}
if (operations.length == 0) {
console.log(chalk.gray('None'));
}
for (const op of operations) {
if (op.type == 'install') {
try {
const install = await axios.post(`/api/v2/packages`, {
language: op.package,
version: op.version,
});
if (!install.data.language)
throw new Error(install.data.message); // Go to exception handler
console.log(
chalk.bold.green('Installed'),
op.package,
op.version
);
} catch (e) {
console.log(
chalk.bold.red('Failed to install') +
` ${op.package} ${op.version}:`,
e.message
);
}
} else if (op.type == 'uninstall') {
try {
const install = await axios.delete(`/api/v2/packages`, {
data: {
language: op.package,
version: op.version,
},
});
if (!install.data.language)
throw new Error(install.data.message); // Go to exception handler
console.log(
chalk.bold.green('Uninstalled'),
op.package,
op.version
);
} catch (e) {
console.log(
chalk.bold.red('Failed to uninstall') +
` ${op.package} ${op.version}:`,
e.message
);
}
}
}
};

41
cli/commands/ppman_commands/uninstall.js
View File

@ -1,41 +0,0 @@
const chalk = require('chalk');
exports.command = ['uninstall <packages...>'];
exports.aliases = ['u'];
exports.describe = 'Uninstalls the named package';
//Splits the package into it's language and version
function split_package(package) {
[language, language_version] = package.split('=');
res = {
language: language,
version: language_version || '*',
};
return res;
}
const msg_format = {
color: p =>
`${
p.language ? chalk.green.bold('✓') : chalk.red.bold('❌')
} Uninstallation ${p.language ? 'succeeded' : 'failed: ' + p.message}`,
monochrome: p =>
`Uninstallation ${p.language ? 'succeeded' : 'failed: ' + p.message}`,
json: JSON.stringify,
};
exports.handler = async ({ axios, packages }) => {
const requests = packages.map(package => split_package(package));
for (request of requests) {
try {
const uninstall = await axios.delete(`/api/v2/packages`, {
data: request,
});
console.log(msg_format.color(uninstall.data));
} catch ({ response }) {
console.error(response.data.message);
}
}
};

8
dev.pps
View File

@ -1,8 +0,0 @@
#!/usr/bin/env -S piston ppman spec
# Development Piston Packages
# Defines packages to be installed by developers
# All packages, latest version
# Don't use this when connected to public repo, in excess of 10GB
* *

0
docker-compose.dev.yaml
View File

26
flake.lock Normal file
View File

@ -0,0 +1,26 @@
{
"nodes": {
"nixpkgs": {
"locked": {
"lastModified": 1643456774,
"narHash": "sha256-abP2nVe3bsndDQgkGxoLdBqHRzisYJSO6cwdEi+AMVc=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "5cf5cad0da6244da30be1b6da2ff3d44b6f3ebe5",
"type": "github"
},
"original": {
"owner": "NixOS",
"repo": "nixpkgs",
"type": "github"
}
},
"root": {
"inputs": {
"nixpkgs": "nixpkgs"
}
}
},
"root": "root",
"version": 7
}

118
flake.nix Normal file
View File

@ -0,0 +1,118 @@
{
description = "Piston packages repo";
inputs.nixpkgs.url = "github:NixOS/nixpkgs";
outputs = { self, nixpkgs }:
let
system = "x86_64-linux";
pkgs = nixpkgs.legacyPackages.${system};
args = {
inherit pkgs;
piston = {
mkRuntime = {
language,
version,
runtime? null,
run,
compile? null,
packages? null,
aliases? [],
tests
}: let
compileFile = if compile != null then
pkgs.writeShellScript "compile" compile
else null;
runFile = pkgs.writeShellScript "run" run;
metadata = {
inherit language version runtime aliases;
run = runFile;
compile = compileFile;
packageSupport = packages != null;
};
in {
inherit packages metadata;
tests = if (builtins.length tests) > 0 then
tests
else abort "Language ${language} doesn't provide any tests";
};
mkTest = {
files,
args? [],
stdin? "",
packages? [],
main? null
}: {
inherit files args stdin packages;
main = if main == null then
(
if (builtins.length (builtins.attrNames files)) == 1 then
(builtins.head (builtins.attrNames files))
else abort "Could not determine the main file for test - specify it using the 'main' parameter"
)
else main;
};
};
};
allRuntimes = import ./runtimes args;
in {
piston = args.piston;
pistonRuntimes = {
"bash" = allRuntimes.bash;
};
legacyPackages."${system}" = {
piston = (import ./api { inherit pkgs; }).package;
nosocket = (import ./nosocket { inherit pkgs; }).package;
};
containerImage = pkgs.dockerTools.buildLayeredImageWithNixDb {
name = "piston";
tag = "latest";
contents = with pkgs; [
self.legacyPackages."${system}".piston
self.legacyPackages."${system}".nosocket
bash
nixFlakes
coreutils-full
cacert.out
git
gnutar
gzip
gnugrep
util-linux
];
extraCommands = ''
mkdir -p piston/jobs etc/nix {,var/}tmp run/lock
echo -e "experimental-features = nix-command flakes" >> etc/nix/nix.conf
echo "nixbld:x:30000:nixbld1,nixbld10,nixbld11,nixbld12,nixbld13,nixbld14,nixbld15,nixbld16,nixbld17,nixbld18,nixbld19,nixbld2,nixbld20,nixbld21,nixbld22,nixbld23,nixbld24,nixbld25,nixbld26,nixbld27,nixbld28,nixbld29,nixbld3,nixbld30,nixbld31,nixbld32,nixbld4,nixbld5,nixbld6,nixbld7,nixbld8,nixbld9" >> etc/group
for i in $(seq 1 32)
do
echo "nixbld$i:x:$(( $i + 30000 )):30000:Nix build user $i:/var/empty:/run/current-system/sw/bin/nologin" >> etc/passwd
done
'';
config = {
Cmd = [
"${self.legacyPackages."${system}".piston}/bin/pistond"
];
Env = [
"NIX_PAGER=cat"
"USER=nobody"
"SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt"
"GIT_SSL_CAINFO=/etc/ssl/certs/ca-bundle.crt"
"NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt"
];
ExposedPorts = {
"2000/tcp" = {};
};
};
};
};
}

0
api/src/nosocket/Makefile → nosocket/Makefile
View File

24
nosocket/default.nix Normal file
View File

@ -0,0 +1,24 @@
{pkgs, ...}:
with pkgs; {
package = stdenv.mkDerivation {
name = "nosocket-1.0.0";
dontUnpack = true;
src = ./nosocket.c;
buildInputs = [
libseccomp
];
buildPhase = ''
gcc $src -O2 -Wall -lseccomp -o nosocket
'';
installPhase = ''
mkdir -p $out/bin
cp nosocket $out/bin
'';
};
}

0
api/src/nosocket/nosocket.c → nosocket/nosocket.c
View File

32
package-lock.json generated
View File

@ -1,32 +0,0 @@
{
"name": "piston",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"devDependencies": {
"prettier": "2.4.1"
}
},
"node_modules/prettier": {
"version": "2.4.1",
"resolved": "https://registry.npmjs.org/prettier/-/prettier-2.4.1.tgz",
"integrity": "sha512-9fbDAXSBcc6Bs1mZrDYb3XKzDLm4EXXL9sC1LqKP5rZkT6KRr/rf9amVUcODVXgguK/isJz0d0hP72WeaKWsvA==",
"dev": true,
"bin": {
"prettier": "bin-prettier.js"
},
"engines": {
"node": ">=10.13.0"
}
}
},
"dependencies": {
"prettier": {
"version": "2.4.1",
"resolved": "https://registry.npmjs.org/prettier/-/prettier-2.4.1.tgz",
"integrity": "sha512-9fbDAXSBcc6Bs1mZrDYb3XKzDLm4EXXL9sC1LqKP5rZkT6KRr/rf9amVUcODVXgguK/isJz0d0hP72WeaKWsvA==",
"dev": true
}
}
}

5
package.json
View File

@ -1,5 +0,0 @@
{
"devDependencies": {
"prettier": "2.4.1"
}
}

8
packages/.gitignore vendored
View File

@ -1,8 +0,0 @@
*/*/*
*.pkg.tar.gz
!*/*/metadata.json
!*/*/build.sh
!*/*/environment
!*/*/run
!*/*/compile
!*/*/test.*

100
packages/CONTRIBUTING.MD
View File

@ -1,100 +0,0 @@
# Contributing packages to the Piston Repository
## Naming Languages
Languages should be named after their interpreters, and the command line binaries you call. The language version should use semantic versioning.
For example, the full name of the standard python interpreter is `CPython`, however we would name it `python`, after the main binary which it provides.
In the example of NodeJS, we would call this `node`, after the main binary.
## Creating new languages
See [deno/1.7.5/](deno/1.7.5/) or any other directory for examples.
1. Create a new branch on your fork of engineer-man/piston
2. Create directories named `[language]/[version]`. See Naming Languages for how to determine the name for your language
3. Create a file named `build.sh`, adding a shebang for bash `#!/bin/bash` on the first line.
In this file put any steps to compile the specified langauge.
This script should download sources, compile sources and output binaries. They should be dumped into the current working directory, removing any files which aren't required in the process.
4. Create a file named `run`, containing bash script to run the interpreter.
The first argument given to this script (`$1`) is the name of the main file, with the remaining ones as program arguments.
STDIN is piped directly into the run file, and as such nothing special is required to deal with STDIN, except leaving it open.
5. Create a file named `compile`, containing bash script to compile sources into binaries. This is only required if the language requires a compling stage.
The first argument is always the main file, followed the names of the other files as additional arguements. If the language does not require a compile stage, don't create a compile file.
6. Create a file named `environment`, containing `export` statements which edit the environment variables accordingly. The `$PWD` variable should be used, and is set inside the package directory when running on the target system.
7. Create a test script starting with test, with the file extension of the language. This script should simply output the phrase `OK`. For example, for mono we would create `test.cs` with the content:
```cs
using System;
public class Test
{
public static void Main(string[] args)
{
Console.WriteLine("OK");
}
}
```
8. Create a `metadata.json` file which contains metadata about the language and interpreter. This simply contains the language name, as in the folder name, the version as in the folder name, aliases that can be used to call this package, limit overrides (if any) that can be used to override the default constraints and finally a dependencies map.
The dependencies map contains the keys as language names, and the values as semver selectors for packages.
```json
{
"language": "deno",
"version": "1.7.5",
"dependencies": {},
"aliases": ["deno-ts", "deno-js"]
}
```
If the interpreter/compiler provides multiple languages, then the provides property should be used:
```json
{
"language": "dotnet",
"version": "5.0.201",
"provides": [
{
"language": "basic.net",
"aliases": [
"basic",
"visual-basic",
"visual-basic.net",
"vb",
"vb.net",
"vb-dotnet",
"dotnet-vb",
"basic-dotnet",
"dotnet-basic"
],
"limit_overrides": { "max_process_count": 128 }
},
{
"language": "fsi",
"aliases": [
"fsx",
"fsharp-interactive",
"f#-interactive",
"dotnet-fsi",
"fsi-dotnet",
"fsi.net"
]
}
]
}
```
9. Test your package builds with running `make [language]-[version].pkg.tar.gz`.
If it all goes to plan, you should have a file named `[language]-[version].pkg.tar.gz`, in this case you're good to go, albeit it is preferable to test the package locally as follows
```shell
./piston build-pkg [package] [version]
./piston ppman install [package]=[version]
./piston run [package] -l [version] packages/[package]/[version]/test.*
```
10. Commit your changes, using message format of `pkg([language]-[version]): Added [language] [version]`
Any additional commits regarding this package should start with `pkg([language]-[version]): `
11. Create a pull request (currently to v3 branch), referencing an Issue number (if there is one associated).

25
packages/Makefile
View File

@ -1,25 +0,0 @@
PACKAGES=$(subst /,-,$(shell find * -maxdepth 1 -mindepth 1 -type d))
BUILD_PLATFORM=$(or ${PLATFORM},baremetal-$(shell grep -oP "^ID=\K.+" /etc/os-release))
help:
@echo "You probably don't want to build all package"
@echo "If you do run $`make build-all$`"
@echo
@echo "Run $`make [language]-[version].pkg.tar.gz$` to build a specific language"
build build-all: $(addsuffix .pkg.tar.gz, ${PACKAGES})
define PKG_RULE
$(1).pkg.tar.gz: $(subst -,/,$(1)) $(subst -,/,$(1))/pkg-info.json
cd $$< && chmod +x ./build.sh && ./build.sh
rm -f $$@
tar czf $$@ -C $$< .
endef
$(foreach pkg,$(PACKAGES),$(eval $(call PKG_RULE,$(pkg))))
%/pkg-info.json: %/metadata.json
jq '.build_platform="${BUILD_PLATFORM}"' $< > $@

7
packages/README.MD
View File

@ -1,7 +0,0 @@
# Piston Package Build Scripts
## Building
```bash
make build-[name]-[version]
```

21
packages/bash/5.1.0/build.sh vendored
View File

@ -1,21 +0,0 @@
#!/usr/bin/env bash
# Put instructions to build your package in here
PREFIX=$(realpath $(dirname $0))
mkdir -p build
cd build
curl "https://ftp.gnu.org/gnu/bash/bash-5.1.tar.gz" -o bash.tar.gz
tar xzf bash.tar.gz --strip-components=1
# === autoconf based ===
./configure --prefix "$PREFIX"
make -j$(nproc)
make install -j$(nproc)
cd ../
rm -rf build

4
packages/bash/5.1.0/environment vendored
View File

@ -1,4 +0,0 @@
#!/usr/bin/env bash
# Put 'export' statements here for environment variables
export PATH=$PWD/bin:$PATH

5
packages/bash/5.1.0/metadata.json vendored
View File

@ -1,5 +0,0 @@
{
"language": "bash",
"version": "5.1.0",
"aliases": ["sh"]
}

4
packages/bash/5.1.0/run vendored
View File

@ -1,4 +0,0 @@
#!/usr/bin/env bash
# Put instructions to run the runtime
bash "$@"

1
packages/bash/5.1.0/test.bash.sh vendored
View File

@ -1 +0,0 @@
echo "OK"

43
packages/clojure/1.10.3/build.sh vendored
View File

@ -1,43 +0,0 @@
#!/usr/bin/env bash
# Installation location
PREFIX=$(realpath $(dirname $0))
# Clojure depends on Java (build and runtime)
mkdir -p java
cd java
curl "https://download.java.net/java/GA/jdk15.0.2/0d1cfde4252546c6931946de8db48ee2/7/GPL/openjdk-15.0.2_linux-x64_bin.tar.gz" -o java.tar.gz
tar xzf java.tar.gz --strip-components=1
rm java.tar.gz
cd ..
# Clojure depends on Maven (build)
mkdir -p maven
cd maven
curl "https://apache.claz.org/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz" -o maven.tar.gz
tar xzf maven.tar.gz --strip-components=1
rm maven.tar.gz
cd ..
# Adding java and maven to the path for building
export PATH=$PWD/java/bin:$PWD/maven/bin:$PATH
export JAVA_HOME=$PWD/java
# Clojure download
mkdir -p build
cd build
git clone -q "https://github.com/clojure/clojure.git" .
git checkout -b clojure-1.10.3 aaf73b12467df80f5db3e086550a33fee0e1b39e # commit for 1.10.3 release
# Build using maven
mvn -Plocal -Dmaven.test.skip=true package
# Get ridda that m2 bloat from Maven and remove Maven itself
cd ../
rm -rf ~/.m2
rm -rf maven/
# Move the jar for easier reference and cleanup
mkdir -p bin
mv build/clojure.jar bin
rm -rf build

6
packages/clojure/1.10.3/environment vendored
View File

@ -1,6 +0,0 @@
#!/usr/bin/env bash
# Clojure requires JAVA_HOME to be set and java binary to be in the path
export JAVA_HOME=$PWD/java
export CLOJURE_PATH=$PWD/bin
export PATH=$PWD/java/bin:$PATH

5
packages/clojure/1.10.3/metadata.json vendored
View File

@ -1,5 +0,0 @@
{
"language": "clojure",
"version": "1.10.3",
"aliases": ["clojure", "clj"]
}

4
packages/clojure/1.10.3/run vendored
View File

@ -1,4 +0,0 @@
#!/usr/bin/env bash
# Run clojure with Java referencing the clojure jar location
java -jar $CLOJURE_PATH/clojure.jar "$@"

5
packages/clojure/1.10.3/test.clj vendored
View File

@ -1,5 +0,0 @@
(ns clojure.examples.main
(:gen-class))
(defn main []
(println "OK"))
(main)

20
packages/cobol/3.1.2/build.sh vendored
View File

@ -1,20 +0,0 @@
#!/usr/bin/env bash
# Put instructions to build your package in here
PREFIX=$(realpath $(dirname $0))
mkdir -p build
cd build
curl -OL "https://downloads.sourceforge.net/project/gnucobol/gnucobol/3.1/gnucobol-3.1.2.tar.xz"
tar xf gnucobol-3.1.2.tar.xz --strip-components=1
# === autoconf based ===
./configure --prefix "$PREFIX" --without-db
make -j$(nproc)
make install -j$(nproc)
cd ../
rm -rf build

4
packages/cobol/3.1.2/compile vendored
View File

@ -1,4 +0,0 @@
#!/usr/bin/env bash
cobc -o binary --free -x -L lib "$@"
chmod +x binary

5
packages/cobol/3.1.2/environment vendored
View File

@ -1,5 +0,0 @@
#!/usr/bin/env bash
export PATH=$PWD/bin:$PATH
export LD_LIBRARY_PATH=$PWD/lib

5
packages/cobol/3.1.2/metadata.json vendored
View File

@ -1,5 +0,0 @@
{
"language": "cobol",
"version": "3.1.2",
"aliases": ["cob"]
}

5
packages/cobol/3.1.2/run vendored
View File

@ -1,5 +0,0 @@
#!/usr/bin/env bash
shift
./binary "$@"

8
packages/cobol/3.1.2/test.cob vendored
View File

@ -1,8 +0,0 @@
*> Test Program
identification division.
program-id. ok-test.
procedure division.
display "OK"
goback.
end program ok-test.

41
packages/flake.lock
View File

@ -1,41 +0,0 @@
{
"nodes": {
"flake-utils": {
"locked": {
"lastModified": 1631561581,
"narHash": "sha256-3VQMV5zvxaVLvqqUrNz3iJelLw30mIVSfZmAaauM3dA=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "7e5bf3925f6fbdfaf50a2a7ca0be2879c4261d19",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
"nixpkgs": {
"locked": {
"lastModified": 1633356775,
"narHash": "sha256-UBhRo1qy8xpOGTrjf7r2SFcULkFM+Wn4kchxN1ToDxs=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "38501bec61c1e9447aa4ffc01ba07c40f4515327",
"type": "github"
},
"original": {
"id": "nixpkgs",
"type": "indirect"
}
},
"root": {
"inputs": {
"flake-utils": "flake-utils",
"nixpkgs": "nixpkgs"
}
}
},
"root": "root",
"version": 7
}

47
packages/flake.nix
View File

@ -1,47 +0,0 @@
{
description = "Piston packages repo";
inputs.flake-utils.url = "github:numtide/flake-utils";
outputs = { self, nixpkgs, flake-utils }:
let
system = "x86_64-linux";
pkgs = nixpkgs.legacyPackages.${system};
args = {
inherit pkgs;
piston = {
mkRuntime = {
language,
version,
runtime,
run,
compile? null,
aliases? []
}: let
packageName = "${runtime}-${language}";
compileFile = if compile != null then
pkgs.writeShellScript "compile" compile
else null;
runFile = pkgs.writeShellScript "run" run;
metadataFile = pkgs.writeText "metadata.json" (builtins.toJSON {
inherit language version runtime aliases;
});
in pkgs.runCommandNoCC packageName {}
(
''
mkdir -p $out/piston
ln -s ${runFile} $out/piston/run
ln -s ${metadataFile} $out/piston/metadata.json
'' + (
if compileFile != null then
''
ln -s ${compileFile} $out/piston/compile
'' else "")
);
};
};
in {
piston = {
"node-javascript" = import ./node-javascript.nix args;
};
};
}

64
packages/init
View File

@ -1,64 +0,0 @@
#!/usr/bin/env bash
if [[ $# -lt 3 ]]; then
echo "Usage: $0 [name] [version] [source]"
echo ""
echo "Initializes an empty package"
exit 1
fi
NAME=$1
VERSION=$2
SOURCE=$3
DIR=$NAME/$VERSION
mkdir -p $DIR
build_instructions(){
echo 'PREFIX=$(realpath $(dirname $0))'
echo
echo 'mkdir -p build'
echo
echo 'cd build'
echo
echo "curl \"$SOURCE\" -o $NAME.tar.gz"
echo
echo "tar xzf $NAME.tar.gz --strip-components=1"
echo
echo "# === autoconf based ==="
echo './configure --prefix "$PREFIX"'
echo
echo 'make -j$(nproc)'
echo 'make install -j$(nproc)'
echo 'cd ../'
echo 'rm -rf build'
}
cd $DIR
for name in build.sh environment run compile; do
echo "#!/usr/bin/env bash" > "$name"
echo "" >> "$name"
done
echo "# Put instructions to build your package in here" >> build.sh
echo ""
build_instructions >> build.sh
echo "# Put 'export' statements here for environment variables" >> environment
echo "export PATH=\$PWD/bin:\$PATH" >> environment
echo "# Put instructions to run the runtime" >> run
echo "$NAME-$VERSION \"\$@\"" >> run
echo "# Put instructions to compile source code, remove this file if the language does not require this stage" >> compile
jq '.language = "'$NAME'" | .version = "'$VERSION'" | .aliases = []' <<< "{}" > metadata.json
cd - > /dev/null
echo $DIR

17
packages/node-javascript.nix
View File

@ -1,17 +0,0 @@
{pkgs, piston}:
piston.mkRuntime {
language = "javascript";
version = pkgs.nodejs.version;
runtime = "node";
aliases = [
"node-js"
"node-javascript"
"js"
];
run = ''
${pkgs.nodejs}/bin/node "$@"
'';
}

4
packages/node/15.10.0/build.sh vendored
View File

@ -1,4 +0,0 @@
#!/bin/bash
curl "https://nodejs.org/dist/v15.10.0/node-v15.10.0-linux-x64.tar.xz" -o node.tar.xz
tar xf node.tar.xz --strip-components=1
rm node.tar.xz

1
packages/node/15.10.0/environment vendored
View File

@ -1 +0,0 @@
export PATH=$PWD/bin:$PATH

10
packages/node/15.10.0/metadata.json vendored
View File

@ -1,10 +0,0 @@
{
"language": "node",
"version": "15.10.0",
"provides": [
{
"language": "javascript",
"aliases": ["node-javascript", "node-js", "javascript", "js"]
}
]
}

3
packages/node/15.10.0/run vendored
View File

@ -1,3 +0,0 @@
#!/bin/bash
node "$@"

1
packages/node/15.10.0/test.js vendored
View File

@ -1 +0,0 @@
console.log('OK');

4
packages/node/16.3.0/build.sh vendored
View File

@ -1,4 +0,0 @@
#!/bin/bash
curl "https://nodejs.org/dist/v16.3.0/node-v16.3.0-linux-x64.tar.xz" -o node.tar.xz
tar xf node.tar.xz --strip-components=1
rm node.tar.xz

1
packages/node/16.3.0/environment vendored
View File

@ -1 +0,0 @@
export PATH=$PWD/bin:$PATH

10
packages/node/16.3.0/metadata.json vendored
View File

@ -1,10 +0,0 @@
{
"language": "node",
"version": "16.3.0",
"provides": [
{
"language": "javascript",
"aliases": ["node-javascript", "node-js", "javascript", "js"]
}
]
}

3
packages/node/16.3.0/run vendored
View File

@ -1,3 +0,0 @@
#!/bin/bash
node "$@"

1
packages/node/16.3.0/test.js vendored
View File

@ -1 +0,0 @@
console.log('OK');

23
packages/python/2.7.18/build.sh vendored
View File

@ -1,23 +0,0 @@
#!/bin/bash
PREFIX=$(realpath $(dirname $0))
mkdir -p build
cd build
curl "https://www.python.org/ftp/python/2.7.18/Python-2.7.18.tgz" -o python.tar.gz
tar xzf python.tar.gz --strip-components=1
rm python.tar.gz
./configure --prefix "$PREFIX" --with-ensurepip=install
make -j$(nproc)
make install -j$(nproc)
cd ..
rm -rf build
bin/pip2 install -U pip==20.3.*
# Upgrade pip to latest supported version
bin/pip2 install numpy scipy pycrypto whoosh bcrypt passlib

1
packages/python/2.7.18/environment vendored
View File

@ -1 +0,0 @@
export PATH=$PWD/bin:$PATH

5
packages/python/2.7.18/metadata.json vendored
View File

@ -1,5 +0,0 @@
{
"language": "python2",
"version": "2.7.18",
"aliases": ["py2", "python2"]
}

3
packages/python/2.7.18/run vendored
View File

@ -1,3 +0,0 @@
#!/bin/bash
python2.7 "$@"

1
packages/python/2.7.18/test.py vendored
View File

@ -1 +0,0 @@
print "OK"

22
packages/python/3.10.0-alpha.7/build.sh vendored
View File

@ -1,22 +0,0 @@
#!/bin/bash
PREFIX=$(realpath $(dirname $0))
mkdir -p build
cd build
curl "https://www.python.org/ftp/python/3.10.0/Python-3.10.0a7.tgz" -o python.tar.gz
tar xzf python.tar.gz --strip-components=1
rm python.tar.gz
./configure --prefix "$PREFIX" --with-ensurepip=install
make -j$(nproc)
make install -j$(nproc)
cd ..
rm -rf build
# This is alpha version, hence most of the libraries are not compatible with python3.10.0
# bin/pip3 install numpy scipy pandas pycrypto whoosh bcrypt passlib

1
packages/python/3.10.0-alpha.7/environment vendored
View File

@ -1 +0,0 @@
export PATH=$PWD/bin:$PATH

5
packages/python/3.10.0-alpha.7/metadata.json vendored
View File

@ -1,5 +0,0 @@
{
"language": "python",
"version": "3.10.0-alpha.7",
"aliases": ["py", "py3", "python3"]
}

3
packages/python/3.10.0-alpha.7/run vendored
View File

@ -1,3 +0,0 @@
#!/bin/bash
python3.10 "$@"

7
packages/python/3.10.0-alpha.7/test.py vendored
View File

@ -1,7 +0,0 @@
working = True
match working:
case True:
print("OK")
case False:
print()

21
packages/python/3.5.10/build.sh vendored
View File

@ -1,21 +0,0 @@
#!/bin/bash
PREFIX=$(realpath $(dirname $0))
mkdir -p build
cd build
curl "https://www.python.org/ftp/python/3.5.10/Python-3.5.10.tgz" -o python.tar.gz
tar xzf python.tar.gz --strip-components=1
rm python.tar.gz
./configure --prefix "$PREFIX" --with-ensurepip=install
make -j$(nproc)
make install -j$(nproc)
cd ..
rm -rf build
bin/pip3 install numpy scipy pandas pycrypto whoosh bcrypt passlib

1
packages/python/3.5.10/environment vendored
View File

@ -1 +0,0 @@
export PATH=$PWD/bin:$PATH

5
packages/python/3.5.10/metadata.json vendored
View File

@ -1,5 +0,0 @@
{
"language": "python",
"version": "3.5.10",
"aliases": ["py", "py3", "python3"]
}

3
packages/python/3.5.10/run vendored
View File

@ -1,3 +0,0 @@
#!/bin/bash
python3.5 "$@"

1
packages/python/3.5.10/test.py vendored
View File

@ -1 +0,0 @@
print("OK")

21
packages/python/3.9.1/build.sh vendored
View File

@ -1,21 +0,0 @@
#!/bin/bash
PREFIX=$(realpath $(dirname $0))
mkdir -p build
cd build
curl "https://www.python.org/ftp/python/3.9.1/Python-3.9.1.tgz" -o python.tar.gz
tar xzf python.tar.gz --strip-components=1
rm python.tar.gz
./configure --prefix "$PREFIX" --with-ensurepip=install
make -j$(nproc)
make install -j$(nproc)
cd ..
rm -rf build
bin/pip3 install numpy scipy pandas pycrypto whoosh bcrypt passlib

1
packages/python/3.9.1/environment vendored
View File

@ -1 +0,0 @@
export PATH=$PWD/bin:$PATH

5
packages/python/3.9.1/metadata.json vendored
View File

@ -1,5 +0,0 @@
{
"language": "python",
"version": "3.9.1",
"aliases": ["py", "py3", "python3"]
}

3
packages/python/3.9.1/run vendored
View File

@ -1,3 +0,0 @@
#!/bin/bash
python3.9 "$@"

1
packages/python/3.9.1/test.py vendored
View File

@ -1 +0,0 @@
print("OK")

21
packages/python/3.9.4/build.sh vendored
View File

@ -1,21 +0,0 @@
#!/bin/bash
PREFIX=$(realpath $(dirname $0))
mkdir -p build
cd build
curl "https://www.python.org/ftp/python/3.9.4/Python-3.9.4.tgz" -o python.tar.gz
tar xzf python.tar.gz --strip-components=1
rm python.tar.gz
./configure --prefix "$PREFIX" --with-ensurepip=install
make -j$(nproc)
make install -j$(nproc)
cd ..
rm -rf build
bin/pip3 install numpy scipy pandas pycrypto whoosh bcrypt passlib sympy

1
packages/python/3.9.4/environment vendored
View File

@ -1 +0,0 @@
export PATH=$PWD/bin:$PATH

5
packages/python/3.9.4/metadata.json vendored
View File

@ -1,5 +0,0 @@
{
"language": "python",
"version": "3.9.4",
"aliases": ["py", "py3", "python3"]
}

3
packages/python/3.9.4/run vendored
View File

@ -1,3 +0,0 @@
#!/bin/bash
python3.9 "$@"

18
packages/python/3.9.4/test.py vendored
View File

@ -1,18 +0,0 @@
execute = (execute_with := lambda *a, **k: lambda f: f(*a, **k))()
@int
@execute
class n: __int__ = lambda _: 69
@execute
class cout: __lshift__ = print
@execute_with(n)
def output(n):
return "OK"
cout << output

21
packages/test.sh
View File

@ -1,21 +0,0 @@
#!/usr/bin/env bash
AUTH_HEADER="Authorization: $API_KEY"
for test_file in */*/test.*
do
IFS='/' read -ra test_parts <<< "$test_file"
IFS='.' read -ra file_parts <<< "$(basename $test_file)"
language=${file_parts[1]}
lang_ver=${test_parts[1]}
test_src=$(python3 -c "import json; print(json.dumps(open('$test_file').read()))")
json='{"language":"'$language'","version":"'$lang_ver'","files":[{"content":'$test_src'}]}'
result=$(curl -s -XPOST -H "Content-Type: application/json" -d "$json" https://emkc.org/api/v2/piston/execute -H $AUTH_HEADER)
echo "==$test_file: $language-$lang_ver=="
#jq '.' <<<"$result"
jq -r 'if (.run.stdout | contains("OK") ) then (.run.stdout) else (.compile.output + .run.output) end' <<<$result
done

14
public.pps
View File

@ -1,14 +0,0 @@
#!/usr/bin/env -S piston ppman spec
# Public Piston Packages
# Defines packages to be installed on the public piston installation
# All packages, latest version
* *
# Except python
!python *
# Install python 3.* and 2.*
python 3.*
python 2.*

51
readme.md
View File

@ -215,6 +215,7 @@ Content-Type: application/json
[
{
"id": 1,
"language": "bash",
"version": "5.1.0",
"aliases": [
@ -222,6 +223,7 @@ Content-Type: application/json
]
},
{
"id": 2,
"language": "brainfuck",
"version": "2.7.3",
"aliases": [
@ -237,35 +239,32 @@ Content-Type: application/json
`POST /api/v2/execute`
This endpoint requests execution of some arbitrary code.
- `language` (**required**) The language to use for execution, must be a string and must be installed.
- `version` (**required**) The version of the language to use for execution, must be a string containing a SemVer selector for the version or the specific version number to use.
- `files` (**required**) An array of files containing code or other data that should be used for execution. The first file in this array is considered the main file.
- `files[].name` (_optional_) The name of the file to upload, must be a string containing no path or left out.
- `files[].content` (**required**) The content of the files to upload, must be a string containing text to write.
- `files[].encoding` (_optional_) The encoding scheme used for the file content. One of `base64`, `hex` or `utf8`. Defaults to `utf8`.
- `stdin` (_optional_) The text to pass as stdin to the program. Must be a string or left out. Defaults to blank string.
- `args` (_optional_) The arguments to pass to the program. Must be an array or left out. Defaults to `[]`.
- `compile_timeout` (_optional_) The maximum time allowed for the compile stage to finish before bailing out in milliseconds. Must be a number or left out. Defaults to `10000` (10 seconds).
- `run_timeout` (_optional_) The maximum time allowed for the run stage to finish before bailing out in milliseconds. Must be a number or left out. Defaults to `3000` (3 seconds).
- `compile_memory_limit` (_optional_) The maximum amount of memory the compile stage is allowed to use in bytes. Must be a number or left out. Defaults to `-1` (no limit)
- `run_memory_limit` (_optional_) The maximum amount of memory the run stage is allowed to use in bytes. Must be a number or left out. Defaults to `-1` (no limit)
- `language` (**required**) The language to use for execution, must be a string and must be installed.
- `files` (**required**) An array of files containing code or other data that should be used for execution. The first file in this array is considered the main file.
- `files[].name` (_optional_) The name of the file to upload, must be a string containing no path or left out.
- `files[].content` (**required**) The content of the files to upload, must be a string containing text to write.
- `stdin` (_optional_) The text to pass as stdin to the program. Must be a string or left out. Defaults to blank string.
- `args` (_optional_) The arguments to pass to the program. Must be an array or left out. Defaults to `[]`.
- `compile_timeout` (_optional_) The maximum time allowed for the compile stage to finish before bailing out in milliseconds. Must be a number or left out. Defaults to `10000` (10 seconds).
- `run_timeout` (_optional_) The maximum time allowed for the run stage to finish before bailing out in milliseconds. Must be a number or left out. Defaults to `3000` (3 seconds).
- `compile_memory_limit` (_optional_) The maximum amount of memory the compile stage is allowed to use in bytes. Must be a number or left out. Defaults to `-1` (no limit)
- `run_memory_limit` (_optional_) The maximum amount of memory the run stage is allowed to use in bytes. Must be a number or left out. Defaults to `-1` (no limit)
```json
{
"language": "js",
"version": "15.10.0",
"files": [
{
"name": "my_cool_code.js",
"content": "console.log(process.argv)"
}
],
"stdin": "",
"args": ["1", "2", "3"],
"compile_timeout": 10000,
"run_timeout": 3000,
"compile_memory_limit": -1,
"run_memory_limit": -1
"language": "js",
"files": [
{
"name": "my_cool_code.js",
"content": "console.log(process.argv)"
}
],
"stdin": "",
"args": ["1", "2", "3"],
"compile_timeout": 10000,
"run_timeout": 3000,
"compile_memory_limit": -1,
"run_memory_limit": -1
}
```

1
repo/.dockerignore
View File

@ -1 +0,0 @@
*.pkg.tar.gz

2
repo/.gitignore vendored
View File

@ -1,2 +0,0 @@
*.pkg.tar.gz
index

20
repo/Dockerfile
View File

@ -1,20 +0,0 @@
FROM debian:buster-slim
ENV DEBIAN_FRONTEND noninteractive
RUN apt-get update && apt-get install -y unzip autoconf build-essential libssl-dev \
pkg-config zlib1g-dev libargon2-dev libsodium-dev libcurl4-openssl-dev \
sqlite3 libsqlite3-dev libonig-dev libxml2 libxml2-dev bc curl git \
linux-headers-amd64 perl xz-utils python3 python3-pip gnupg jq zlib1g-dev \
cmake cmake-doc extra-cmake-modules build-essential gcc binutils bash coreutils \
util-linux pciutils usbutils coreutils binutils findutils grep libncurses5-dev \
libncursesw5-dev python3-pip libgmp-dev libmpfr-dev python2 libffi-dev gfortran\
libreadline-dev libblas-dev liblapack-dev libpcre3-dev libarpack2-dev libfftw3-dev \
libglpk-dev libqhull-dev libqrupdate-dev libsuitesparse-dev libsundials-dev \
libbz2-dev liblzma-dev libpcre2-dev gperf bison flex g++ && \
ln -sf /bin/bash /bin/sh && \
rm -rf /var/lib/apt/lists/* && \
update-alternatives --install /usr/bin/python python /usr/bin/python3.7 2
ADD entrypoint.sh mkindex.sh /
ENTRYPOINT ["bash","/entrypoint.sh"]
CMD ["--no-build"]

59
repo/entrypoint.sh
View File

@ -1,59 +0,0 @@
cd /piston/packages
SERVER=1
BUILD=1
CI=0
echo "Running through arguments.."
for pkg in "$@"
do
shift
if [[ "$pkg" = "--no-server" ]]; then
echo "Not starting index server after builds"
SERVER=0
elif [[ "$pkg" = "--no-build" ]]; then
echo "Building no more package"
BUILD=0
elif [[ "$pkg" = "--ci" ]]; then
echo "Running in CI mode, --no-build, --no-server"
BUILD=0
SERVER=0
CI=1
else
if [[ $BUILD -eq 1 ]]; then
echo "Building package $pkg"
make -j16 $pkg.pkg.tar.gz PLATFORM=docker-debian
echo "Done with package $pkg"
elif [[ $CI -eq 1 ]]; then
echo "Commit SHA: $pkg"
cd ..
echo "Changed files:"
git diff --name-only $pkg^1 $pkg
PACKAGES=$(git diff --name-only $pkg^1 $pkg | awk -F/ '{ print $2 "-" $3 }' | sort -u)
cd packages
echo "Building packages: $PACKAGES"
for package in "$PACKAGES"; do
make -j16 $package.pkg.tar.gz PLATFORM=docker-debian
done
else
echo "Building was disabled, skipping $pkg build=$BUILD ci=$CI"
fi
fi
done
cd /piston/repo
echo "Creating index"
./mkindex.sh
echo "Index created"
if [[ $SERVER -eq 1 ]]; then
echo "Starting index server.."
python3 -m http.server
else
echo "Skipping starting index server"
fi
exit 0

23
repo/mkindex.sh
View File

@ -1,23 +0,0 @@
BASEURL=http://repo:8000/
i=0
echo "" > index
for pkg in $(find ../packages -type f -name "*.pkg.tar.gz")
do
cp $pkg .
PKGFILE=$(basename $pkg)
PKGFILENAME=$(echo $PKGFILE | sed 's/\.pkg\.tar\.gz//g')
PKGNAME=$(echo $PKGFILENAME | grep -oP '^\K.+(?=-)')
PKGVERSION=$(echo $PKGFILENAME | grep -oP '^.+-\K.+')
PKGCHECKSUM=$(sha256sum $PKGFILE | awk '{print $1}')
echo "$PKGNAME,$PKGVERSION,$PKGCHECKSUM,$BASEURL$PKGFILE" >> index
echo "Adding package $PKGNAME-$PKGVERSION"
((i=i+1))
done

1
result Symbolic link
View File

@ -0,0 +1 @@
/nix/store/1idnmsddirm1hpxw40r17wsg9p9dscb9-piston.tar.gz

Some files were not shown because too many files have changed in this diff Show More