Mascali
/
piston
mirror of https://github.com/engineer-man/piston.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

refactor: typescript api

This commit is contained in:
Endercheif 2023-03-08 17:59:43 -08:00
parent 234530ed60
commit 996929f053
No known key found for this signature in database
GPG Key ID: 7767459A0C8BEE00
16 changed files with 341 additions and 321 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
api/.gitignore vendored
View File

@ -1 +1,2 @@
_piston
api/dist

15
api/Dockerfile
View File

@ -18,14 +18,19 @@ RUN apt-get update && \
RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen
RUN curl -fsSL https://get.pnpm.io/install.sh | sh -
RUN npm install --global pnpm
WORKDIR /piston_api
COPY ["package.json", "pnpm-lock.yaml", "./"]
RUN pnpm install
COPY ["package.json", "pnpm-lock.yaml", "tsconfig.json", "./"]
COPY ./src ./src
RUN make -C ./src/nosocket/ all && make -C ./src/nosocket/ install
RUN pnpm install
RUN pnpm build
CMD [ "pnpm", "run"]
COPY ./src/nosocket ./dist/nosocket
RUN make -C ./dist/nosocket/ all && make -C ./dist/nosocket/ install
CMD [ "pnpm", "api"]
EXPOSE 2000/tcp

9
api/package.json
View File

@ -2,10 +2,11 @@
"name": "piston-api",
"version": "3.2.0",
"description": "API for piston - a high performance code execution engine",
"module": "src/index.js",
"module": "./dist/index.js",
"type": "module",
"scripts": {
"run": "node src/index.js"
"api": "node ./dist/index.js",
"build": "npx tsc"
},
"dependencies": {
"body-parser": "^1.19.0",
@ -21,6 +22,8 @@
},
"license": "MIT",
"devDependencies": {
"@types/express": "^4.17.17"
"@types/express": "^4.17.17",
"@types/node-fetch": "^2.6.2",
"typescript": "^4.9.5"
}
}

52
api/pnpm-lock.yaml
View File

@ -2,6 +2,7 @@ lockfileVersion: 5.4
specifiers:
'@types/express': ^4.17.17
'@types/node-fetch': ^2.6.2
body-parser: ^1.19.0
chownr: ^2.0.0
express: ^4.17.1
@ -10,6 +11,7 @@ specifiers:
logplease: github:Endercheif/logplease#feature/quality
node-fetch: ^2.6.7
semver: ^7.3.4
typescript: ^4.9.5
uuid: ^8.3.2
waitpid: git+https://github.com/HexF/node-waitpid.git
@ -19,7 +21,7 @@ dependencies:
express: 4.18.2
express-ws: 5.0.2_express@4.18.2
is-docker: 2.2.1
logplease: github.com/Endercheif/logplease/e900d3ace383484f9d8de8276788ab889bf4f8cc
logplease: github.com/Endercheif/logplease/808583a0f24b2d6625d0d30da5b4164cc7bbf23a
node-fetch: 2.6.9
semver: 7.3.8
uuid: 8.3.2
@ -27,6 +29,8 @@ dependencies:
devDependencies:
'@types/express': 4.17.17
'@types/node-fetch': 2.6.2
typescript: 4.9.5
packages:
@ -64,6 +68,13 @@ packages:
resolution: {integrity: sha512-Y4XFY5VJAuw0FgAqPNd6NNoV44jbq9Bz2L7Rh/J6jLTiHBSBJa9fxqQIvkIld4GsoDOcCbvzOUAbLPsSKKg+uA==}
dev: true
/@types/node-fetch/2.6.2:
resolution: {integrity: sha512-DHqhlq5jeESLy19TYhLakJ07kNumXWjcDdxXsLUMJZ6ue8VZJj4kLPQVE/2mdHh3xZziNF1xppu5lwmS53HR+A==}
dependencies:
'@types/node': 18.14.1
form-data: 3.0.1
dev: true
/@types/node/18.14.1:
resolution: {integrity: sha512-QH+37Qds3E0eDlReeboBxfHbX9omAcBCXEzswCu6jySP642jiM3cYSIkU/REqwhCUqXdonHFuBfJDiAJxMNhaQ==}
dev: true
@ -95,6 +106,10 @@ packages:
resolution: {integrity: sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==}
dev: false
/asynckit/0.4.0:
resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==}
dev: true
/body-parser/1.20.1:
resolution: {integrity: sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw==}
engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16}
@ -152,6 +167,13 @@ packages:
engines: {node: '>=10'}
dev: false
/combined-stream/1.0.8:
resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==}
engines: {node: '>= 0.8'}
dependencies:
delayed-stream: 1.0.0
dev: true
/content-disposition/0.5.4:
resolution: {integrity: sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==}
engines: {node: '>= 0.6'}
@ -184,6 +206,11 @@ packages:
ms: 2.0.0
dev: false
/delayed-stream/1.0.0:
resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==}
engines: {node: '>=0.4.0'}
dev: true
/depd/2.0.0:
resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==}
engines: {node: '>= 0.8'}
@ -279,6 +306,15 @@ packages:
- supports-color
dev: false
/form-data/3.0.1:
resolution: {integrity: sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==}
engines: {node: '>= 6'}
dependencies:
asynckit: 0.4.0
combined-stream: 1.0.8
mime-types: 2.1.35
dev: true
/forwarded/0.2.0:
resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==}
engines: {node: '>= 0.6'}
@ -370,14 +406,12 @@ packages:
/mime-db/1.52.0:
resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==}
engines: {node: '>= 0.6'}
dev: false
/mime-types/2.1.35:
resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==}
engines: {node: '>= 0.6'}
dependencies:
mime-db: 1.52.0
dev: false
/mime/1.6.0:
resolution: {integrity: sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==}
@ -553,6 +587,12 @@ packages:
mime-types: 2.1.35
dev: false
/typescript/4.9.5:
resolution: {integrity: sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==}
engines: {node: '>=4.2.0'}
hasBin: true
dev: true
/unpipe/1.0.0:
resolution: {integrity: sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==}
engines: {node: '>= 0.8'}
@ -601,10 +641,10 @@ packages:
resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==}
dev: false
github.com/Endercheif/logplease/e900d3ace383484f9d8de8276788ab889bf4f8cc:
resolution: {tarball: https://codeload.github.com/Endercheif/logplease/tar.gz/e900d3ace383484f9d8de8276788ab889bf4f8cc}
github.com/Endercheif/logplease/808583a0f24b2d6625d0d30da5b4164cc7bbf23a:
resolution: {tarball: https://codeload.github.com/Endercheif/logplease/tar.gz/808583a0f24b2d6625d0d30da5b4164cc7bbf23a}
name: logplease
version: 1.3.0
version: 1.3.5
dev: false
github.com/HexF/node-waitpid/a08d116a5d993a747624fe72ff890167be8c34aa:

23
api/src/api/v2.js → api/src/api/v2.ts
View File

@ -1,11 +1,15 @@
import { Router } from 'express';
import { EventEmitter } from 'events';
import { EventEmitter } from 'node:events';
import { get_latest_runtime_matching_language_version, runtimes as _runtimes } from '../runtime';
import {
get_latest_runtime_matching_language_version,
runtimes as _runtimes,
} from '../runtime.js';
import Job from '../job.js';
import package_ from '../package.js';
import { create } from 'logplease'
import { create } from 'logplease';
import { RequestBody } from '../types.js';
const logger = create('api/v2', {});
const router = Router();
@ -49,10 +53,10 @@ const SIGNALS = [
'SIGXCPU',
'SIGXFSZ',
'SIGWINCH',
];
] as const;
// ref: https://man7.org/linux/man-pages/man7/signal.7.html
function get_job(body) {
function get_job(body: RequestBody): Promise<Job> {
let {
language,
version,
@ -65,7 +69,7 @@ function get_job(body) {
compile_timeout,
} = body;
return new Promise((resolve, reject) => {
return new Promise<Job>((resolve, reject) => {
if (!language || typeof language !== 'string') {
return reject({
message: 'language is required as a string',
@ -173,7 +177,7 @@ router.use((req, res, next) => {
next();
});
// @ts-ignore
router.ws('/connect', async (ws, req) => {
let job = null;
let eventBus = new EventEmitter();
@ -205,8 +209,7 @@ router.ws('/connect', async (ws, req) => {
ws.on('message', async data => {
try {
// @ts-ignore
const msg = JSON.parse(data);
const msg = JSON.parse(data.toString());
switch (msg.type) {
case 'init':
@ -302,6 +305,8 @@ router.get('/runtimes', (req, res) => {
});
router.get('/packages', async (req, res) => {
console.log({req, res});
logger.debug('Request to list packages');
let packages = await package_.get_package_list();

34
api/src/config.js → api/src/config.ts
View File

@ -1,11 +1,12 @@
import { existsSync } from 'fs';
import { create, LogLevels } from 'logplease';
import { create, type LogLevel, LogLevels } from 'logplease';
import { Limit, Limits, ObjectType } from './types.js';
const logger = create('config', {});
const options = {
log_level: {
desc: 'Level of data to log',
default: 'INFO',
default: 'INFO' as LogLevel,
validators: [
x =>
Object.values(LogLevels).includes(x) ||
@ -116,7 +117,7 @@ const options = {
desc: 'Per-language exceptions in JSON format for each of:\
max_process_count, max_open_files, max_file_size, compile_memory_limit,\
run_memory_limit, compile_timeout, run_timeout, output_max_size',
default: {},
default: {} as Record<string, Limits>,
parser: parse_overrides,
validators: [
x => !!x || `Failed to parse the overrides\n${x}`,
@ -127,7 +128,7 @@ const options = {
Object.freeze(options);
function apply_validators(validators, validator_parameters) {
function apply_validators(validators: Array<(...args: unknown[]) => true | string>, validator_parameters: unknown[]) {
for (const validator of validators) {
const validation_response = validator(...validator_parameters);
if (validation_response !== true) {
@ -137,8 +138,8 @@ function apply_validators(validators, validator_parameters) {
return true;
}
function parse_overrides(overrides_string) {
function get_parsed_json_or_null(overrides) {
function parse_overrides(overrides_string: string): Record<string, Limits> {
function get_parsed_json_or_null(overrides: string): Record<string, Partial<Limits>> | null {
try {
return JSON.parse(overrides);
} catch (e) {
@ -150,7 +151,7 @@ function parse_overrides(overrides_string) {
if (overrides === null) {
return null;
}
const parsed_overrides = {};
const parsed_overrides: Record<string, Partial<Limits>> = {};
for (const language in overrides) {
parsed_overrides[language] = {};
for (const key in overrides[language]) {
@ -169,21 +170,22 @@ function parse_overrides(overrides_string) {
return null;
}
// Find the option for the override
const option = options[key];
const option = options[key as Limit];
const parser = option.parser;
const raw_value = overrides[language][key];
const raw_value = overrides[language][key as Limit];
// @ts-ignore: lgtm
const parsed_value = parser(raw_value);
parsed_overrides[language][key] = parsed_value;
}
}
return parsed_overrides;
return parsed_overrides as Record<string, Limits>;
}
function validate_overrides(overrides) {
function validate_overrides(overrides: Record<string, Limits>) {
for (const language in overrides) {
for (const key in overrides[language]) {
const value = overrides[language][key];
const option = options[key];
const value = overrides[language][key as Limit];
const option = options[key as Limit];
const validators = option.validators;
const validation_response = apply_validators(validators, [
value,
@ -199,14 +201,12 @@ function validate_overrides(overrides) {
logger.info(`Loading Configuration from environment`);
/** @type {import('./types').ObjectType<typeof options, "default">} */
// @ts-ignore
let config = {};
let config = {} as ObjectType<typeof options, "default">;
for (const option_name in options) {
const env_key = 'PISTON_' + option_name.toUpperCase();
const option = options[option_name];
const parser = option.parser || ((/** @type {any} */ x) => x);
const parser = option.parser || ((x: unknown) => x);
const env_val = process.env[env_key];
const parsed_val = parser(env_val);
const value = env_val === undefined ? option.default : parsed_val;

20
api/src/globals.js → api/src/globals.ts
View File

@ -1,20 +1,24 @@
// Globals are things the user shouldn't change in config, but is good to not use inline constants for
import is_docker from 'is-docker';
import { readFileSync } from 'fs';
import { readFileSync } from 'node:fs';
import { createRequire } from 'node:module';
export const platform = `${is_docker() ? 'docker' : 'baremetal'}-${readFileSync('/etc/os-release')
const require = createRequire(import.meta.url);
export const platform = `${is_docker() ? 'docker' : 'baremetal'}-${readFileSync(
'/etc/os-release'
)
.toString()
.split('\n')
.find(x => x.startsWith('ID'))
.replace('ID=', '')}`;
export const data_directories = {
packages: 'packages',
jobs: 'jobs',
}
export const version = require('../package.json').version
export const pkg_installed_file = '.ppman-installed' //Used as indication for if a package was installed
export const clean_directories = ['/dev/shm', '/run/lock', '/tmp', '/var/tmp']
};
// @ts-ignore
export const version: string = require('../package.json').version;
export const pkg_installed_file = '.ppman-installed'; //Used as indication for if a package was installed
export const clean_directories = ['/dev/shm', '/run/lock', '/tmp', '/var/tmp'];

39
api/src/index.js → api/src/index.ts
View File

@ -5,9 +5,9 @@ import expressWs from 'express-ws';
import * as globals from './globals.js';
import config from './config.js';
import { join } from 'path';
import { readdir } from 'fs/promises';
import { existsSync, mkdirSync, chmodSync } from 'fs';
import { urlencoded, json } from 'body-parser';
import { readdir } from 'node:fs/promises';
import { existsSync, mkdirSync, chmodSync } from 'node:fs';
import bodyParser from 'body-parser';
import { load_package } from './runtime.js';
const logger = create('index', {});
@ -16,7 +16,6 @@ expressWs(app);
(async () => {
logger.info('Setting loglevel to');
// @ts-ignore
setLogLevel(config.log_level);
logger.debug('Ensuring data directories exist');
@ -35,7 +34,10 @@ expressWs(app);
}
}
});
chmodSync(join(config.data_directory, globals.data_directories.jobs), 0o711)
chmodSync(
join(config.data_directory, globals.data_directories.jobs),
0o711
);
logger.info('Loading packages');
const pkgdir = join(
@ -54,9 +56,7 @@ expressWs(app);
const installed_languages = languages
.flat()
.filter(pkg =>
existsSync(join(pkg, globals.pkg_installed_file))
);
.filter(pkg => existsSync(join(pkg, globals.pkg_installed_file)));
installed_languages.forEach(pkg => load_package(pkg));
@ -64,18 +64,25 @@ expressWs(app);
logger.debug('Constructing Express App');
logger.debug('Registering middleware');
app.use(urlencoded({ extended: true }));
app.use(json());
app.use(bodyParser.urlencoded({ extended: true }));
app.use(bodyParser.json());
app.use((err, req, res, next) => {
return res.status(400).send({
stack: err.stack,
});
});
app.use(
(
err: Error,
req: express.Request,
res: express.Response,
next: express.NextFunction
) => {
return res.status(400).send({
stack: err.stack,
});
}
);
logger.debug('Registering Routes');
const api_v2 = require('./api/v2').default;
const api_v2 = (await import('./api/v2.js')).default;
app.use('/api/v2', api_v2);
app.use((req, res, next) => {

74
api/src/job.js → api/src/job.ts
View File

@ -1,20 +1,30 @@
import { create } from 'logplease';
// @ts-ignore
import { create, type Logger } from 'logplease';
const logger = create('job');
import { v4 as uuidv4 } from 'uuid';
import { spawn } from 'child_process';
import { join, relative, dirname } from 'path';
import { join, relative, dirname } from 'node:path';
import config from './config.js';
import * as globals from './globals.js';
import { mkdir, chown, writeFile, readdir, stat as _stat, rm } from 'fs/promises';
import { readdirSync, readFileSync } from 'fs';
import {
mkdir,
chown,
writeFile,
readdir,
stat as _stat,
rm,
} from 'node:fs/promises';
import { readdirSync, readFileSync } from 'node:fs';
import wait_pid from 'waitpid';
import EventEmitter from 'events';
import { File, ResponseBody } from './types.js';
const job_states = {
READY: Symbol('Ready to be primed'),
PRIMED: Symbol('Primed and ready for execution'),
EXECUTED: Symbol('Executed and ready for cleanup'),
};
} as const;
let uid = 0;
let gid = 0;
@ -30,13 +40,39 @@ setInterval(() => {
}, 10);
export default class Job {
constructor({ runtime, files, args, stdin, timeouts, memory_limits }) {
uuid: string;
logger: Logger;
runtime: any;
files: File[];
args: string[];
stdin: string;
timeouts: { compile: number; run: number };
memory_limits: { compile: number; run: number };
uid: number;
gid: number;
state: symbol;
dir: string;
constructor({
runtime,
files,
args,
stdin,
timeouts,
memory_limits,
}: {
runtime: unknown;
files: File[];
args: string[];
stdin: string;
timeouts: { compile: number; run: number };
memory_limits: { compile: number; run: number };
}) {
this.uuid = uuidv4();
this.logger = create(`job/${this.uuid}`, {});
this.runtime = runtime;
this.files = files.map((file, i) => ({
this.files = files.map((file: File, i: number) => ({
name: file.name || `file${i}.code`,
content: file.content,
encoding: ['base64', 'hex', 'utf8'].includes(file.encoding)
@ -111,7 +147,13 @@ export default class Job {
this.logger.debug('Primed job');
}
async safe_call(file, args, timeout, memory_limit, eventBus = null) {
async safe_call(
file: string,
args: string[],
timeout: number,
memory_limit: string | number,
eventBus: EventEmitter = null
): Promise<ResponseBody['run'] & { error?: Error }> {
return new Promise((resolve, reject) => {
const nonetwork = config.disable_networking ? ['nosocket'] : [];
@ -141,7 +183,7 @@ export default class Job {
'bash',
file,
...args,
];
] as Array<string>;
var stdout = '';
var stderr = '';
@ -164,11 +206,11 @@ export default class Job {
proc.stdin.end();
proc.stdin.destroy();
} else {
eventBus.on('stdin', data => {
eventBus.on('stdin', (data: any) => {
proc.stdin.write(data);
});
eventBus.on('kill', signal => {
eventBus.on('kill', (signal: NodeJS.Signals | number) => {
proc.kill(signal);
});
}
@ -241,11 +283,11 @@ export default class Job {
const code_files =
(this.runtime.language === 'file' && this.files) ||
this.files.filter(file => file.encoding == 'utf8');
this.files.filter((file: File) => file.encoding == 'utf8');
this.logger.debug('Compiling');
let compile;
let compile: unknown;
if (this.runtime.compiled) {
compile = await this.safe_call(
@ -275,7 +317,7 @@ export default class Job {
};
}
async execute_interactive(eventBus) {
async execute_interactive(eventBus: EventEmitter) {
if (this.state !== job_states.PRIMED) {
throw new Error(
'Job must be in primed state, current state: ' +
@ -320,7 +362,7 @@ export default class Job {
}
cleanup_processes(dont_wait = []) {
let processes = [1];
let processes: number[] = [1];
const to_wait = [];
this.logger.debug(`Cleaning up processes`);

52
api/src/package.js → api/src/package.ts
View File

@ -1,20 +1,34 @@
import { create } from 'logplease'
import { parse, satisfies, rcompare } from 'semver';
import config from './config';
import * as globals from './globals';
import { create } from 'logplease';
import { parse, satisfies, rcompare, type SemVer } from 'semver';
import config from './config.js';
import * as globals from './globals.js';
import fetch from 'node-fetch';
import { join } from 'path';
import { rm, mkdir, writeFile, rmdir } from 'fs/promises';
import { existsSync, createWriteStream, createReadStream } from 'fs';
import { exec, spawn } from 'child_process';
import { createHash } from 'crypto';
import { load_package, get_runtime_by_name_and_version } from './runtime';
import { load_package, get_runtime_by_name_and_version } from './runtime.js';
import chownr from 'chownr';
import { promisify } from 'util';
const logger = create('package', {});
class Package {
constructor({ language, version, download, checksum }) {
language: string;
version: SemVer;
checksum: string;
download: string;
constructor({
language,
version,
download,
checksum,
}: {
language: string;
version: string;
checksum: string;
download: string;
}) {
this.language = language;
this.version = parse(version);
this.checksum = checksum;
@ -22,9 +36,7 @@ class Package {
}
get installed() {
return existsSync(
join(this.install_path, globals.pkg_installed_file)
);
return existsSync(join(this.install_path, globals.pkg_installed_file));
}
get install_path() {
@ -57,7 +69,8 @@ class Package {
`Downloading package from ${this.download} in to ${this.install_path}`
);
const pkgpath = join(this.install_path, 'pkg.tar.gz');
const download = await fetch(this.download);
// @ts-ignore
const download = (await fetch(this.download)) as fetch.Response;
const file_stream = createWriteStream(pkgpath);
await new Promise((resolve, reject) => {
@ -72,7 +85,7 @@ class Package {
const hash = createHash('sha256');
const read_stream = createReadStream(pkgpath);
await new Promise((resolve, reject) => {
await new Promise<void>((resolve, reject) => {
read_stream.on('data', chunk => hash.update(chunk));
read_stream.on('end', () => resolve());
read_stream.on('error', error => reject(error));
@ -90,7 +103,7 @@ class Package {
`Extracting package files from archive ${pkgpath} in to ${this.install_path}`
);
await new Promise((resolve, reject) => {
await new Promise<void>((resolve, reject) => {
const proc = exec(
`bash -c 'cd "${this.install_path}" && tar xzf ${pkgpath}'`
);
@ -111,7 +124,7 @@ class Package {
logger.debug('Caching environment');
const get_env_command = `cd ${this.install_path}; source environment; env`;
const envout = await new Promise((resolve, reject) => {
const envout = await new Promise<string>((resolve, reject) => {
let stdout = '';
const proc = spawn(
@ -162,7 +175,7 @@ class Package {
};
}
async uninstall() {
async uninstall(): Promise<{ language: string; version: string }> {
logger.info(`Uninstalling ${this.language}-${this.version.raw}`);
logger.debug('Finding runtime');
@ -195,7 +208,10 @@ class Package {
}
static async get_package_list() {
const repo_content = await fetch(config.repo_url).then(x => x.text());
// @ts-ignore
const repo_content: string = await fetch(config.repo_url).then(
(x: fetch.Response) => x.text()
);
const entries = repo_content.split('\n').filter(x => x.length > 0);
@ -211,13 +227,11 @@ class Package {
});
}
static async get_package(lang, version) {
static async get_package(lang: string, version: string) {
const packages = await Package.get_package_list();
const candidates = packages.filter(pkg => {
return (
pkg.language == lang && satisfies(pkg.version, version)
);
return pkg.language == lang && satisfies(pkg.version, version);
});
candidates.sort((a, b) => rcompare(a.version, b.version));

114
api/src/runtime.js → api/src/runtime.ts
View File

@ -1,47 +1,60 @@
import { create } from 'logplease';
import { parse, satisfies, rcompare } from 'semver';
import { parse, satisfies, rcompare, type SemVer } from 'semver';
import config from './config.js';
import { platform } from './globals.js';
import { readFileSync, existsSync } from 'fs';
import { join } from 'path';
import { Limit, Limits, PackageInfo } from './types.js';
const logger = create('runtime', {});
/** @type {Array<Runtime>} */
const runtimes = [];
export const runtimes: Runtime[] = [];
class Runtime {
constructor({
language,
version,
aliases,
pkgdir,
runtime,
timeouts,
memory_limits,
max_process_count,
max_open_files,
max_file_size,
output_max_size,
export class Runtime {
language: string;
version: SemVer;
aliases: string[];
pkgdir: string;
runtime?: any;
timeouts: { run: number; compile: number };
memory_limits: { run: number; compile: number };
max_process_count: number;
max_open_files: number;
max_file_size: number;
output_max_size: number;
_compiled?: boolean;
_env_vars?: Record<string, any>;
constructor(o: {
language: string;
version: SemVer;
aliases: string[];
pkgdir: string;
runtime?: any;
timeouts: { run: number; compile: number };
memory_limits: { run: number; compile: number };
max_process_count: number;
max_open_files: number;
max_file_size: number;
output_max_size: number;
}) {
this.language = language;
this.version = version;
this.aliases = aliases || [];
this.pkgdir = pkgdir;
this.runtime = runtime;
this.timeouts = timeouts;
this.memory_limits = memory_limits;
this.max_process_count = max_process_count;
this.max_open_files = max_open_files;
this.max_file_size = max_file_size;
this.output_max_size = output_max_size;
this.language = o.language;
this.version = o.version;
this.aliases = o.aliases || [];
this.pkgdir = o.pkgdir;
this.runtime = o.runtime;
this.timeouts = o.timeouts;
this.memory_limits = o.memory_limits;
this.max_process_count = o.max_process_count;
this.max_open_files = o.max_open_files;
this.max_file_size = o.max_file_size;
this.output_max_size = o.output_max_size;
}
static compute_single_limit(
language_name,
limit_name,
language_limit_overrides
) {
language_name: string,
limit_name: Limit,
language_limit_overrides: Limits
): number {
return (
(config.limit_overrides[language_name] &&
config.limit_overrides[language_name][limit_name]) ||
@ -51,7 +64,10 @@ class Runtime {
);
}
static compute_all_limits(language_name, language_limit_overrides) {
static compute_all_limits(
language_name: string,
language_limit_overrides: Limits
) {
return {
timeouts: {
compile: this.compute_single_limit(
@ -100,26 +116,25 @@ class Runtime {
};
}
static load_package(package_dir) {
static load_package(package_dir: string) {
let info = JSON.parse(
// @ts-ignore
readFileSync(join(package_dir, 'pkg-info.json'))
);
readFileSync(join(package_dir, 'pkg-info.json'), 'utf8')
) as PackageInfo;
let {
language,
version,
version: _version,
build_platform,
aliases,
provides,
limit_overrides,
} = info;
version = parse(version);
const version = parse(_version);
if (build_platform !== platform) {
logger.warn(
`Package ${language}-${version} was built for platform ${build_platform}, ` +
`but our platform is ${platform}`
`but our platform is ${platform}`
);
}
@ -142,7 +157,6 @@ class Runtime {
});
} else {
runtimes.push(
// @ts-ignore
new Runtime({
language,
version,
@ -193,11 +207,10 @@ class Runtime {
}
}
const _runtimes = runtimes;
export { _runtimes as runtimes };
const _Runtime = Runtime;
export { _Runtime as Runtime };
export function get_runtimes_matching_language_version(lang, ver) {
export function get_runtimes_matching_language_version(
lang: string,
ver: string | import('semver/classes/range.js')
) {
return runtimes.filter(
rt =>
(rt.language == lang || rt.aliases.includes(lang)) &&
@ -205,14 +218,15 @@ export function get_runtimes_matching_language_version(lang, ver) {
);
}
export function get_latest_runtime_matching_language_version(
lang,
ver
lang: string,
ver: string
) {
return get_runtimes_matching_language_version(lang, ver)
.sort((a, b) => rcompare(a.version, b.version))[0];
return get_runtimes_matching_language_version(lang, ver).sort((a, b) =>
rcompare(a.version, b.version)
)[0];
}
export function get_runtime_by_name_and_version(runtime, ver) {
export function get_runtime_by_name_and_version(runtime: string, ver: string) {
return runtimes.find(
rt =>
(rt.runtime == runtime ||

56
api/src/types.ts
View File

@ -1,21 +1,63 @@
export interface Metadata {
language: string;
version: string;
aliases?: string[];
dependencies?: Record<string, string>;
provides: {
language: string;
aliases: string[];
limit_overrides: Limits;
}[];
limit_overrides?: Limits;
}
export type Limit =
| 'compile_timeout'
| 'compile_memory_limit'
| 'max_process_count'
| 'max_open_files'
| 'max_file_size'
| 'output_max_size'
| 'run_memory_limit'
| 'run_timeout';
export type Limits = Record<Limit, number>;
export type LanguageMetadata = {
language: string;
version: string;
};
export type PackageInfo = Metadata & { build_platform: string };
export type File = {
content: string;
name?: string;
encoding?: 'base64' | 'hex' | 'utf8';
};
export interface Body {
export type RequestBody = {
language: string;
version: string;
files: Array<File>;
stdin?: string;
args?: Array<string>;
run_timeout?: number;
compile_timeout?: number;
compile_memory_limit?: number;
run_memory_limit?: number;
} & Partial<Limits>;
export interface ResponseBody {
language: string;
version: string;
run: {
stdout: string;
stderr: string;
output: string;
code: number;
signal?: NodeJS.Signals;
};
}
export type ObjectType<TObject extends Record<any, Record<Key, any>>, Key extends string> = {
export type ObjectType<
TObject extends Record<any, Record<Key, any>>,
Key extends string
> = {
[K in keyof TObject]: TObject[K][Key];
};

10
api/tsconfig.json
View File

@ -4,9 +4,13 @@
"checkJs": true,
"resolveJsonModule": true,
"target": "ES2022",
"module": "ES2022",
"moduleResolution": "nodenext",
"allowSyntheticDefaultImports": true,
// we dont have any thing to compile just yet
"noEmit": true
}
"outDir": "./dist",
"rootDir": "src",
// "declaration": true
},
"include": ["src"],
"exclude": ["dist", "node_modules"]
}

2
cli/commands/ppman_commands/install.js
View File

@ -27,7 +27,7 @@ const msg_format = {
exports.handler = async ({ axios, packages }) => {
const requests = packages.map(package => split_package(package));
for (request of requests) {
for (const request of requests) {
try {
const install = await axios.post(`/api/v2/packages`, request);

7
package.json
View File

@ -1,12 +1,5 @@
{
"devDependencies": {
"@types/chownr": "^1.0.0",
"@types/express-ws": "^3.0.1",
"@types/node-fetch": "^2.6.2",
"@types/uuid": "^9.0.1",
"prettier": "2.4.1"
},
"dependencies": {
"logplease": "github:Endercheif/logplease#feature/quality"
}
}

154
pnpm-lock.yaml
View File

@ -1,154 +0,0 @@
lockfileVersion: 5.4
specifiers:
'@types/chownr': ^1.0.0
'@types/express-ws': ^3.0.1
'@types/node-fetch': ^2.6.2
'@types/uuid': ^9.0.1
logplease: github:Endercheif/logplease#feature/quality
prettier: 2.4.1
dependencies:
logplease: github.com/Endercheif/logplease/3cb7bb615d2c20402662a4ea37c812d767382a37
devDependencies:
'@types/chownr': 1.0.0
'@types/express-ws': 3.0.1
'@types/node-fetch': 2.6.2
'@types/uuid': 9.0.1
prettier: 2.4.1
packages:
/@types/body-parser/1.19.2:
resolution: {integrity: sha512-ALYone6pm6QmwZoAgeyNksccT9Q4AWZQ6PvfwR37GT6r6FWUPguq6sUmNGSMV2Wr761oQoBxwGGa6DR5o1DC9g==}
dependencies:
'@types/connect': 3.4.35
'@types/node': 18.14.2
dev: true
/@types/chownr/1.0.0:
resolution: {integrity: sha512-T6sN7Nzh1l4Dhvzbq3h5LHumB9LP/vw0MSyGbZKFGRmmx54/rQMYgEGlKvAT9Y1NZwqSAnQFPTOPUntWwyZthQ==}
dependencies:
'@types/node': 18.14.2
dev: true
/@types/connect/3.4.35:
resolution: {integrity: sha512-cdeYyv4KWoEgpBISTxWvqYsVy444DOqehiF3fM3ne10AmJ62RSyNkUnxMJXHQWRQQX2eR94m5y1IZyDwBjV9FQ==}
dependencies:
'@types/node': 18.14.2
dev: true
/@types/express-serve-static-core/4.17.33:
resolution: {integrity: sha512-TPBqmR/HRYI3eC2E5hmiivIzv+bidAfXofM+sbonAGvyDhySGw9/PQZFt2BLOrjUUR++4eJVpx6KnLQK1Fk9tA==}
dependencies:
'@types/node': 18.14.2
'@types/qs': 6.9.7
'@types/range-parser': 1.2.4
dev: true
/@types/express-ws/3.0.1:
resolution: {integrity: sha512-VguRXzcpPBF0IggIGpUoM65cZJDfMQxoc6dKoCz1yLzcwcXW7ft60yhq3ygKhyEhEIQFtLrWjyz4AJ1qjmzCFw==}
dependencies:
'@types/express': 4.17.17
'@types/express-serve-static-core': 4.17.33
'@types/ws': 8.5.4
dev: true
/@types/express/4.17.17:
resolution: {integrity: sha512-Q4FmmuLGBG58btUnfS1c1r/NQdlp3DMfGDGig8WhfpA2YRUtEkxAjkZb0yvplJGYdF1fsQ81iMDcH24sSCNC/Q==}
dependencies:
'@types/body-parser': 1.19.2
'@types/express-serve-static-core': 4.17.33
'@types/qs': 6.9.7
'@types/serve-static': 1.15.1
dev: true
/@types/mime/3.0.1:
resolution: {integrity: sha512-Y4XFY5VJAuw0FgAqPNd6NNoV44jbq9Bz2L7Rh/J6jLTiHBSBJa9fxqQIvkIld4GsoDOcCbvzOUAbLPsSKKg+uA==}
dev: true
/@types/node-fetch/2.6.2:
resolution: {integrity: sha512-DHqhlq5jeESLy19TYhLakJ07kNumXWjcDdxXsLUMJZ6ue8VZJj4kLPQVE/2mdHh3xZziNF1xppu5lwmS53HR+A==}
dependencies:
'@types/node': 18.14.2
form-data: 3.0.1
dev: true
/@types/node/18.14.2:
resolution: {integrity: sha512-1uEQxww3DaghA0RxqHx0O0ppVlo43pJhepY51OxuQIKHpjbnYLA7vcdwioNPzIqmC2u3I/dmylcqjlh0e7AyUA==}
dev: true
/@types/qs/6.9.7:
resolution: {integrity: sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw==}
dev: true
/@types/range-parser/1.2.4:
resolution: {integrity: sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==}
dev: true
/@types/serve-static/1.15.1:
resolution: {integrity: sha512-NUo5XNiAdULrJENtJXZZ3fHtfMolzZwczzBbnAeBbqBwG+LaG6YaJtuwzwGSQZ2wsCrxjEhNNjAkKigy3n8teQ==}
dependencies:
'@types/mime': 3.0.1
'@types/node': 18.14.2
dev: true
/@types/uuid/9.0.1:
resolution: {integrity: sha512-rFT3ak0/2trgvp4yYZo5iKFEPsET7vKydKF+VRCxlQ9bpheehyAJH89dAkaLEq/j/RZXJIqcgsmPJKUP1Z28HA==}
dev: true
/@types/ws/8.5.4:
resolution: {integrity: sha512-zdQDHKUgcX/zBc4GrwsE/7dVdAD8JR4EuiAXiiUhhfyIJXXb2+PrGshFyeXWQPMmmZ2XxgaqclgpIC7eTXc1mg==}
dependencies:
'@types/node': 18.14.2
dev: true
/asynckit/0.4.0:
resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==}
dev: true
/combined-stream/1.0.8:
resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==}
engines: {node: '>= 0.8'}
dependencies:
delayed-stream: 1.0.0
dev: true
/delayed-stream/1.0.0:
resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==}
engines: {node: '>=0.4.0'}
dev: true
/form-data/3.0.1:
resolution: {integrity: sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==}
engines: {node: '>= 6'}
dependencies:
asynckit: 0.4.0
combined-stream: 1.0.8
mime-types: 2.1.35
dev: true
/mime-db/1.52.0:
resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==}
engines: {node: '>= 0.6'}
dev: true
/mime-types/2.1.35:
resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==}
engines: {node: '>= 0.6'}
dependencies:
mime-db: 1.52.0
dev: true
/prettier/2.4.1:
resolution: {integrity: sha512-9fbDAXSBcc6Bs1mZrDYb3XKzDLm4EXXL9sC1LqKP5rZkT6KRr/rf9amVUcODVXgguK/isJz0d0hP72WeaKWsvA==}
engines: {node: '>=10.13.0'}
hasBin: true
dev: true
github.com/Endercheif/logplease/3cb7bb615d2c20402662a4ea37c812d767382a37:
resolution: {tarball: https://codeload.github.com/Endercheif/logplease/tar.gz/3cb7bb615d2c20402662a4ea37c812d767382a37}
name: logplease
version: 1.3.3
dev: false