From 996929f053badec1a189481b4a2edf488a07afbd Mon Sep 17 00:00:00 2001 From: Endercheif Date: Wed, 8 Mar 2023 17:59:43 -0800 Subject: [PATCH] refactor: typescript api --- api/.gitignore | 1 + api/Dockerfile | 15 ++- api/package.json | 9 +- api/pnpm-lock.yaml | 52 ++++++++- api/src/api/{v2.js => v2.ts} | 23 ++-- api/src/{config.js => config.ts} | 34 +++--- api/src/{globals.js => globals.ts} | 20 ++-- api/src/{index.js => index.ts} | 39 ++++--- api/src/{job.js => job.ts} | 74 +++++++++--- api/src/{package.js => package.ts} | 52 ++++++--- api/src/{runtime.js => runtime.ts} | 114 ++++++++++-------- api/src/types.ts | 56 +++++++-- api/tsconfig.json | 10 +- cli/commands/ppman_commands/install.js | 2 +- package.json | 7 -- pnpm-lock.yaml | 154 ------------------------- 16 files changed, 341 insertions(+), 321 deletions(-) rename api/src/api/{v2.js => v2.ts} (95%) rename api/src/{config.js => config.ts} (86%) rename api/src/{globals.js => globals.ts} (53%) rename api/src/{index.js => index.ts} (73%) rename api/src/{job.js => job.ts} (89%) rename api/src/{package.js => package.ts} (83%) rename api/src/{runtime.js => runtime.ts} (68%) delete mode 100644 pnpm-lock.yaml diff --git a/api/.gitignore b/api/.gitignore index 4b5a9b8..9639bab 100644 --- a/api/.gitignore +++ b/api/.gitignore @@ -1 +1,2 @@ _piston +api/dist diff --git a/api/Dockerfile b/api/Dockerfile index 995a4ed..82d8410 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -18,14 +18,19 @@ RUN apt-get update && \ RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen -RUN curl -fsSL https://get.pnpm.io/install.sh | sh - +RUN npm install --global pnpm WORKDIR /piston_api -COPY ["package.json", "pnpm-lock.yaml", "./"] -RUN pnpm install +COPY ["package.json", "pnpm-lock.yaml", "tsconfig.json", "./"] COPY ./src ./src -RUN make -C ./src/nosocket/ all && make -C ./src/nosocket/ install +RUN pnpm install +RUN pnpm build -CMD [ "pnpm", "run"] +COPY ./src/nosocket ./dist/nosocket + +RUN make -C ./dist/nosocket/ all && make -C ./dist/nosocket/ install + + +CMD [ "pnpm", "api"] EXPOSE 2000/tcp diff --git a/api/package.json b/api/package.json index cda3309..5642d41 100644 --- a/api/package.json +++ b/api/package.json @@ -2,10 +2,11 @@ "name": "piston-api", "version": "3.2.0", "description": "API for piston - a high performance code execution engine", - "module": "src/index.js", + "module": "./dist/index.js", "type": "module", "scripts": { - "run": "node src/index.js" + "api": "node ./dist/index.js", + "build": "npx tsc" }, "dependencies": { "body-parser": "^1.19.0", @@ -21,6 +22,8 @@ }, "license": "MIT", "devDependencies": { - "@types/express": "^4.17.17" + "@types/express": "^4.17.17", + "@types/node-fetch": "^2.6.2", + "typescript": "^4.9.5" } } diff --git a/api/pnpm-lock.yaml b/api/pnpm-lock.yaml index 47d8177..700f98b 100644 --- a/api/pnpm-lock.yaml +++ b/api/pnpm-lock.yaml @@ -2,6 +2,7 @@ lockfileVersion: 5.4 specifiers: '@types/express': ^4.17.17 + '@types/node-fetch': ^2.6.2 body-parser: ^1.19.0 chownr: ^2.0.0 express: ^4.17.1 @@ -10,6 +11,7 @@ specifiers: logplease: github:Endercheif/logplease#feature/quality node-fetch: ^2.6.7 semver: ^7.3.4 + typescript: ^4.9.5 uuid: ^8.3.2 waitpid: git+https://github.com/HexF/node-waitpid.git @@ -19,7 +21,7 @@ dependencies: express: 4.18.2 express-ws: 5.0.2_express@4.18.2 is-docker: 2.2.1 - logplease: github.com/Endercheif/logplease/e900d3ace383484f9d8de8276788ab889bf4f8cc + logplease: github.com/Endercheif/logplease/808583a0f24b2d6625d0d30da5b4164cc7bbf23a node-fetch: 2.6.9 semver: 7.3.8 uuid: 8.3.2 @@ -27,6 +29,8 @@ dependencies: devDependencies: '@types/express': 4.17.17 + '@types/node-fetch': 2.6.2 + typescript: 4.9.5 packages: @@ -64,6 +68,13 @@ packages: resolution: {integrity: sha512-Y4XFY5VJAuw0FgAqPNd6NNoV44jbq9Bz2L7Rh/J6jLTiHBSBJa9fxqQIvkIld4GsoDOcCbvzOUAbLPsSKKg+uA==} dev: true + /@types/node-fetch/2.6.2: + resolution: {integrity: sha512-DHqhlq5jeESLy19TYhLakJ07kNumXWjcDdxXsLUMJZ6ue8VZJj4kLPQVE/2mdHh3xZziNF1xppu5lwmS53HR+A==} + dependencies: + '@types/node': 18.14.1 + form-data: 3.0.1 + dev: true + /@types/node/18.14.1: resolution: {integrity: sha512-QH+37Qds3E0eDlReeboBxfHbX9omAcBCXEzswCu6jySP642jiM3cYSIkU/REqwhCUqXdonHFuBfJDiAJxMNhaQ==} dev: true @@ -95,6 +106,10 @@ packages: resolution: {integrity: sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==} dev: false + /asynckit/0.4.0: + resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} + dev: true + /body-parser/1.20.1: resolution: {integrity: sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw==} engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16} @@ -152,6 +167,13 @@ packages: engines: {node: '>=10'} dev: false + /combined-stream/1.0.8: + resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} + engines: {node: '>= 0.8'} + dependencies: + delayed-stream: 1.0.0 + dev: true + /content-disposition/0.5.4: resolution: {integrity: sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==} engines: {node: '>= 0.6'} @@ -184,6 +206,11 @@ packages: ms: 2.0.0 dev: false + /delayed-stream/1.0.0: + resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} + engines: {node: '>=0.4.0'} + dev: true + /depd/2.0.0: resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==} engines: {node: '>= 0.8'} @@ -279,6 +306,15 @@ packages: - supports-color dev: false + /form-data/3.0.1: + resolution: {integrity: sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==} + engines: {node: '>= 6'} + dependencies: + asynckit: 0.4.0 + combined-stream: 1.0.8 + mime-types: 2.1.35 + dev: true + /forwarded/0.2.0: resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==} engines: {node: '>= 0.6'} @@ -370,14 +406,12 @@ packages: /mime-db/1.52.0: resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} engines: {node: '>= 0.6'} - dev: false /mime-types/2.1.35: resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} engines: {node: '>= 0.6'} dependencies: mime-db: 1.52.0 - dev: false /mime/1.6.0: resolution: {integrity: sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==} @@ -553,6 +587,12 @@ packages: mime-types: 2.1.35 dev: false + /typescript/4.9.5: + resolution: {integrity: sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==} + engines: {node: '>=4.2.0'} + hasBin: true + dev: true + /unpipe/1.0.0: resolution: {integrity: sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==} engines: {node: '>= 0.8'} @@ -601,10 +641,10 @@ packages: resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==} dev: false - github.com/Endercheif/logplease/e900d3ace383484f9d8de8276788ab889bf4f8cc: - resolution: {tarball: https://codeload.github.com/Endercheif/logplease/tar.gz/e900d3ace383484f9d8de8276788ab889bf4f8cc} + github.com/Endercheif/logplease/808583a0f24b2d6625d0d30da5b4164cc7bbf23a: + resolution: {tarball: https://codeload.github.com/Endercheif/logplease/tar.gz/808583a0f24b2d6625d0d30da5b4164cc7bbf23a} name: logplease - version: 1.3.0 + version: 1.3.5 dev: false github.com/HexF/node-waitpid/a08d116a5d993a747624fe72ff890167be8c34aa: diff --git a/api/src/api/v2.js b/api/src/api/v2.ts similarity index 95% rename from api/src/api/v2.js rename to api/src/api/v2.ts index 2fa475f..8ebf527 100644 --- a/api/src/api/v2.js +++ b/api/src/api/v2.ts @@ -1,11 +1,15 @@ import { Router } from 'express'; -import { EventEmitter } from 'events'; +import { EventEmitter } from 'node:events'; -import { get_latest_runtime_matching_language_version, runtimes as _runtimes } from '../runtime'; +import { + get_latest_runtime_matching_language_version, + runtimes as _runtimes, +} from '../runtime.js'; import Job from '../job.js'; import package_ from '../package.js'; -import { create } from 'logplease' +import { create } from 'logplease'; +import { RequestBody } from '../types.js'; const logger = create('api/v2', {}); const router = Router(); @@ -49,10 +53,10 @@ const SIGNALS = [ 'SIGXCPU', 'SIGXFSZ', 'SIGWINCH', -]; +] as const; // ref: https://man7.org/linux/man-pages/man7/signal.7.html -function get_job(body) { +function get_job(body: RequestBody): Promise { let { language, version, @@ -65,7 +69,7 @@ function get_job(body) { compile_timeout, } = body; - return new Promise((resolve, reject) => { + return new Promise((resolve, reject) => { if (!language || typeof language !== 'string') { return reject({ message: 'language is required as a string', @@ -173,7 +177,7 @@ router.use((req, res, next) => { next(); }); - +// @ts-ignore router.ws('/connect', async (ws, req) => { let job = null; let eventBus = new EventEmitter(); @@ -205,8 +209,7 @@ router.ws('/connect', async (ws, req) => { ws.on('message', async data => { try { - // @ts-ignore - const msg = JSON.parse(data); + const msg = JSON.parse(data.toString()); switch (msg.type) { case 'init': @@ -302,6 +305,8 @@ router.get('/runtimes', (req, res) => { }); router.get('/packages', async (req, res) => { + console.log({req, res}); + logger.debug('Request to list packages'); let packages = await package_.get_package_list(); diff --git a/api/src/config.js b/api/src/config.ts similarity index 86% rename from api/src/config.js rename to api/src/config.ts index 80f41ac..0bd1442 100644 --- a/api/src/config.js +++ b/api/src/config.ts @@ -1,11 +1,12 @@ import { existsSync } from 'fs'; -import { create, LogLevels } from 'logplease'; +import { create, type LogLevel, LogLevels } from 'logplease'; +import { Limit, Limits, ObjectType } from './types.js'; const logger = create('config', {}); const options = { log_level: { desc: 'Level of data to log', - default: 'INFO', + default: 'INFO' as LogLevel, validators: [ x => Object.values(LogLevels).includes(x) || @@ -116,7 +117,7 @@ const options = { desc: 'Per-language exceptions in JSON format for each of:\ max_process_count, max_open_files, max_file_size, compile_memory_limit,\ run_memory_limit, compile_timeout, run_timeout, output_max_size', - default: {}, + default: {} as Record, parser: parse_overrides, validators: [ x => !!x || `Failed to parse the overrides\n${x}`, @@ -127,7 +128,7 @@ const options = { Object.freeze(options); -function apply_validators(validators, validator_parameters) { +function apply_validators(validators: Array<(...args: unknown[]) => true | string>, validator_parameters: unknown[]) { for (const validator of validators) { const validation_response = validator(...validator_parameters); if (validation_response !== true) { @@ -137,8 +138,8 @@ function apply_validators(validators, validator_parameters) { return true; } -function parse_overrides(overrides_string) { - function get_parsed_json_or_null(overrides) { +function parse_overrides(overrides_string: string): Record { + function get_parsed_json_or_null(overrides: string): Record> | null { try { return JSON.parse(overrides); } catch (e) { @@ -150,7 +151,7 @@ function parse_overrides(overrides_string) { if (overrides === null) { return null; } - const parsed_overrides = {}; + const parsed_overrides: Record> = {}; for (const language in overrides) { parsed_overrides[language] = {}; for (const key in overrides[language]) { @@ -169,21 +170,22 @@ function parse_overrides(overrides_string) { return null; } // Find the option for the override - const option = options[key]; + const option = options[key as Limit]; const parser = option.parser; - const raw_value = overrides[language][key]; + const raw_value = overrides[language][key as Limit]; + // @ts-ignore: lgtm const parsed_value = parser(raw_value); parsed_overrides[language][key] = parsed_value; } } - return parsed_overrides; + return parsed_overrides as Record; } -function validate_overrides(overrides) { +function validate_overrides(overrides: Record) { for (const language in overrides) { for (const key in overrides[language]) { - const value = overrides[language][key]; - const option = options[key]; + const value = overrides[language][key as Limit]; + const option = options[key as Limit]; const validators = option.validators; const validation_response = apply_validators(validators, [ value, @@ -199,14 +201,12 @@ function validate_overrides(overrides) { logger.info(`Loading Configuration from environment`); -/** @type {import('./types').ObjectType} */ -// @ts-ignore -let config = {}; +let config = {} as ObjectType; for (const option_name in options) { const env_key = 'PISTON_' + option_name.toUpperCase(); const option = options[option_name]; - const parser = option.parser || ((/** @type {any} */ x) => x); + const parser = option.parser || ((x: unknown) => x); const env_val = process.env[env_key]; const parsed_val = parser(env_val); const value = env_val === undefined ? option.default : parsed_val; diff --git a/api/src/globals.js b/api/src/globals.ts similarity index 53% rename from api/src/globals.js rename to api/src/globals.ts index edc3d3a..25cdfe6 100644 --- a/api/src/globals.js +++ b/api/src/globals.ts @@ -1,20 +1,24 @@ // Globals are things the user shouldn't change in config, but is good to not use inline constants for import is_docker from 'is-docker'; -import { readFileSync } from 'fs'; +import { readFileSync } from 'node:fs'; +import { createRequire } from 'node:module'; -export const platform = `${is_docker() ? 'docker' : 'baremetal'}-${readFileSync('/etc/os-release') +const require = createRequire(import.meta.url); + +export const platform = `${is_docker() ? 'docker' : 'baremetal'}-${readFileSync( + '/etc/os-release' +) .toString() .split('\n') .find(x => x.startsWith('ID')) .replace('ID=', '')}`; - export const data_directories = { packages: 'packages', jobs: 'jobs', -} -export const version = require('../package.json').version - -export const pkg_installed_file = '.ppman-installed' //Used as indication for if a package was installed -export const clean_directories = ['/dev/shm', '/run/lock', '/tmp', '/var/tmp'] +}; +// @ts-ignore +export const version: string = require('../package.json').version; +export const pkg_installed_file = '.ppman-installed'; //Used as indication for if a package was installed +export const clean_directories = ['/dev/shm', '/run/lock', '/tmp', '/var/tmp']; diff --git a/api/src/index.js b/api/src/index.ts similarity index 73% rename from api/src/index.js rename to api/src/index.ts index 62b07fd..2e432ed 100644 --- a/api/src/index.js +++ b/api/src/index.ts @@ -5,9 +5,9 @@ import expressWs from 'express-ws'; import * as globals from './globals.js'; import config from './config.js'; import { join } from 'path'; -import { readdir } from 'fs/promises'; -import { existsSync, mkdirSync, chmodSync } from 'fs'; -import { urlencoded, json } from 'body-parser'; +import { readdir } from 'node:fs/promises'; +import { existsSync, mkdirSync, chmodSync } from 'node:fs'; +import bodyParser from 'body-parser'; import { load_package } from './runtime.js'; const logger = create('index', {}); @@ -16,7 +16,6 @@ expressWs(app); (async () => { logger.info('Setting loglevel to'); - // @ts-ignore setLogLevel(config.log_level); logger.debug('Ensuring data directories exist'); @@ -35,7 +34,10 @@ expressWs(app); } } }); - chmodSync(join(config.data_directory, globals.data_directories.jobs), 0o711) + chmodSync( + join(config.data_directory, globals.data_directories.jobs), + 0o711 + ); logger.info('Loading packages'); const pkgdir = join( @@ -54,9 +56,7 @@ expressWs(app); const installed_languages = languages .flat() - .filter(pkg => - existsSync(join(pkg, globals.pkg_installed_file)) - ); + .filter(pkg => existsSync(join(pkg, globals.pkg_installed_file))); installed_languages.forEach(pkg => load_package(pkg)); @@ -64,18 +64,25 @@ expressWs(app); logger.debug('Constructing Express App'); logger.debug('Registering middleware'); - app.use(urlencoded({ extended: true })); - app.use(json()); + app.use(bodyParser.urlencoded({ extended: true })); + app.use(bodyParser.json()); - app.use((err, req, res, next) => { - return res.status(400).send({ - stack: err.stack, - }); - }); + app.use( + ( + err: Error, + req: express.Request, + res: express.Response, + next: express.NextFunction + ) => { + return res.status(400).send({ + stack: err.stack, + }); + } + ); logger.debug('Registering Routes'); - const api_v2 = require('./api/v2').default; + const api_v2 = (await import('./api/v2.js')).default; app.use('/api/v2', api_v2); app.use((req, res, next) => { diff --git a/api/src/job.js b/api/src/job.ts similarity index 89% rename from api/src/job.js rename to api/src/job.ts index 7ef5be1..db5d77c 100644 --- a/api/src/job.js +++ b/api/src/job.ts @@ -1,20 +1,30 @@ -import { create } from 'logplease'; -// @ts-ignore +import { create, type Logger } from 'logplease'; + const logger = create('job'); import { v4 as uuidv4 } from 'uuid'; import { spawn } from 'child_process'; -import { join, relative, dirname } from 'path'; +import { join, relative, dirname } from 'node:path'; import config from './config.js'; import * as globals from './globals.js'; -import { mkdir, chown, writeFile, readdir, stat as _stat, rm } from 'fs/promises'; -import { readdirSync, readFileSync } from 'fs'; +import { + mkdir, + chown, + writeFile, + readdir, + stat as _stat, + rm, +} from 'node:fs/promises'; +import { readdirSync, readFileSync } from 'node:fs'; import wait_pid from 'waitpid'; +import EventEmitter from 'events'; + +import { File, ResponseBody } from './types.js'; const job_states = { READY: Symbol('Ready to be primed'), PRIMED: Symbol('Primed and ready for execution'), EXECUTED: Symbol('Executed and ready for cleanup'), -}; +} as const; let uid = 0; let gid = 0; @@ -30,13 +40,39 @@ setInterval(() => { }, 10); export default class Job { - constructor({ runtime, files, args, stdin, timeouts, memory_limits }) { + uuid: string; + logger: Logger; + runtime: any; + files: File[]; + args: string[]; + stdin: string; + timeouts: { compile: number; run: number }; + memory_limits: { compile: number; run: number }; + uid: number; + gid: number; + state: symbol; + dir: string; + constructor({ + runtime, + files, + args, + stdin, + timeouts, + memory_limits, + }: { + runtime: unknown; + files: File[]; + args: string[]; + stdin: string; + timeouts: { compile: number; run: number }; + memory_limits: { compile: number; run: number }; + }) { this.uuid = uuidv4(); this.logger = create(`job/${this.uuid}`, {}); this.runtime = runtime; - this.files = files.map((file, i) => ({ + this.files = files.map((file: File, i: number) => ({ name: file.name || `file${i}.code`, content: file.content, encoding: ['base64', 'hex', 'utf8'].includes(file.encoding) @@ -111,7 +147,13 @@ export default class Job { this.logger.debug('Primed job'); } - async safe_call(file, args, timeout, memory_limit, eventBus = null) { + async safe_call( + file: string, + args: string[], + timeout: number, + memory_limit: string | number, + eventBus: EventEmitter = null + ): Promise { return new Promise((resolve, reject) => { const nonetwork = config.disable_networking ? ['nosocket'] : []; @@ -141,7 +183,7 @@ export default class Job { 'bash', file, ...args, - ]; + ] as Array; var stdout = ''; var stderr = ''; @@ -164,11 +206,11 @@ export default class Job { proc.stdin.end(); proc.stdin.destroy(); } else { - eventBus.on('stdin', data => { + eventBus.on('stdin', (data: any) => { proc.stdin.write(data); }); - eventBus.on('kill', signal => { + eventBus.on('kill', (signal: NodeJS.Signals | number) => { proc.kill(signal); }); } @@ -241,11 +283,11 @@ export default class Job { const code_files = (this.runtime.language === 'file' && this.files) || - this.files.filter(file => file.encoding == 'utf8'); + this.files.filter((file: File) => file.encoding == 'utf8'); this.logger.debug('Compiling'); - let compile; + let compile: unknown; if (this.runtime.compiled) { compile = await this.safe_call( @@ -275,7 +317,7 @@ export default class Job { }; } - async execute_interactive(eventBus) { + async execute_interactive(eventBus: EventEmitter) { if (this.state !== job_states.PRIMED) { throw new Error( 'Job must be in primed state, current state: ' + @@ -320,7 +362,7 @@ export default class Job { } cleanup_processes(dont_wait = []) { - let processes = [1]; + let processes: number[] = [1]; const to_wait = []; this.logger.debug(`Cleaning up processes`); diff --git a/api/src/package.js b/api/src/package.ts similarity index 83% rename from api/src/package.js rename to api/src/package.ts index fa38a04..1f8fb03 100644 --- a/api/src/package.js +++ b/api/src/package.ts @@ -1,20 +1,34 @@ -import { create } from 'logplease' -import { parse, satisfies, rcompare } from 'semver'; -import config from './config'; -import * as globals from './globals'; +import { create } from 'logplease'; +import { parse, satisfies, rcompare, type SemVer } from 'semver'; +import config from './config.js'; +import * as globals from './globals.js'; import fetch from 'node-fetch'; import { join } from 'path'; import { rm, mkdir, writeFile, rmdir } from 'fs/promises'; import { existsSync, createWriteStream, createReadStream } from 'fs'; import { exec, spawn } from 'child_process'; import { createHash } from 'crypto'; -import { load_package, get_runtime_by_name_and_version } from './runtime'; +import { load_package, get_runtime_by_name_and_version } from './runtime.js'; import chownr from 'chownr'; import { promisify } from 'util'; const logger = create('package', {}); class Package { - constructor({ language, version, download, checksum }) { + language: string; + version: SemVer; + checksum: string; + download: string; + constructor({ + language, + version, + download, + checksum, + }: { + language: string; + version: string; + checksum: string; + download: string; + }) { this.language = language; this.version = parse(version); this.checksum = checksum; @@ -22,9 +36,7 @@ class Package { } get installed() { - return existsSync( - join(this.install_path, globals.pkg_installed_file) - ); + return existsSync(join(this.install_path, globals.pkg_installed_file)); } get install_path() { @@ -57,7 +69,8 @@ class Package { `Downloading package from ${this.download} in to ${this.install_path}` ); const pkgpath = join(this.install_path, 'pkg.tar.gz'); - const download = await fetch(this.download); + // @ts-ignore + const download = (await fetch(this.download)) as fetch.Response; const file_stream = createWriteStream(pkgpath); await new Promise((resolve, reject) => { @@ -72,7 +85,7 @@ class Package { const hash = createHash('sha256'); const read_stream = createReadStream(pkgpath); - await new Promise((resolve, reject) => { + await new Promise((resolve, reject) => { read_stream.on('data', chunk => hash.update(chunk)); read_stream.on('end', () => resolve()); read_stream.on('error', error => reject(error)); @@ -90,7 +103,7 @@ class Package { `Extracting package files from archive ${pkgpath} in to ${this.install_path}` ); - await new Promise((resolve, reject) => { + await new Promise((resolve, reject) => { const proc = exec( `bash -c 'cd "${this.install_path}" && tar xzf ${pkgpath}'` ); @@ -111,7 +124,7 @@ class Package { logger.debug('Caching environment'); const get_env_command = `cd ${this.install_path}; source environment; env`; - const envout = await new Promise((resolve, reject) => { + const envout = await new Promise((resolve, reject) => { let stdout = ''; const proc = spawn( @@ -162,7 +175,7 @@ class Package { }; } - async uninstall() { + async uninstall(): Promise<{ language: string; version: string }> { logger.info(`Uninstalling ${this.language}-${this.version.raw}`); logger.debug('Finding runtime'); @@ -195,7 +208,10 @@ class Package { } static async get_package_list() { - const repo_content = await fetch(config.repo_url).then(x => x.text()); + // @ts-ignore + const repo_content: string = await fetch(config.repo_url).then( + (x: fetch.Response) => x.text() + ); const entries = repo_content.split('\n').filter(x => x.length > 0); @@ -211,13 +227,11 @@ class Package { }); } - static async get_package(lang, version) { + static async get_package(lang: string, version: string) { const packages = await Package.get_package_list(); const candidates = packages.filter(pkg => { - return ( - pkg.language == lang && satisfies(pkg.version, version) - ); + return pkg.language == lang && satisfies(pkg.version, version); }); candidates.sort((a, b) => rcompare(a.version, b.version)); diff --git a/api/src/runtime.js b/api/src/runtime.ts similarity index 68% rename from api/src/runtime.js rename to api/src/runtime.ts index 45e8938..ae2c445 100644 --- a/api/src/runtime.js +++ b/api/src/runtime.ts @@ -1,47 +1,60 @@ import { create } from 'logplease'; -import { parse, satisfies, rcompare } from 'semver'; +import { parse, satisfies, rcompare, type SemVer } from 'semver'; import config from './config.js'; import { platform } from './globals.js'; import { readFileSync, existsSync } from 'fs'; import { join } from 'path'; +import { Limit, Limits, PackageInfo } from './types.js'; const logger = create('runtime', {}); -/** @type {Array} */ -const runtimes = []; +export const runtimes: Runtime[] = []; -class Runtime { - constructor({ - language, - version, - aliases, - pkgdir, - runtime, - timeouts, - memory_limits, - max_process_count, - max_open_files, - max_file_size, - output_max_size, +export class Runtime { + language: string; + version: SemVer; + aliases: string[]; + pkgdir: string; + runtime?: any; + timeouts: { run: number; compile: number }; + memory_limits: { run: number; compile: number }; + max_process_count: number; + max_open_files: number; + max_file_size: number; + output_max_size: number; + _compiled?: boolean; + _env_vars?: Record; + constructor(o: { + language: string; + version: SemVer; + aliases: string[]; + pkgdir: string; + runtime?: any; + timeouts: { run: number; compile: number }; + memory_limits: { run: number; compile: number }; + max_process_count: number; + max_open_files: number; + max_file_size: number; + output_max_size: number; }) { - this.language = language; - this.version = version; - this.aliases = aliases || []; - this.pkgdir = pkgdir; - this.runtime = runtime; - this.timeouts = timeouts; - this.memory_limits = memory_limits; - this.max_process_count = max_process_count; - this.max_open_files = max_open_files; - this.max_file_size = max_file_size; - this.output_max_size = output_max_size; + this.language = o.language; + this.version = o.version; + this.aliases = o.aliases || []; + this.pkgdir = o.pkgdir; + this.runtime = o.runtime; + this.timeouts = o.timeouts; + this.memory_limits = o.memory_limits; + this.max_process_count = o.max_process_count; + this.max_open_files = o.max_open_files; + this.max_file_size = o.max_file_size; + this.output_max_size = o.output_max_size; } static compute_single_limit( - language_name, - limit_name, - language_limit_overrides - ) { + language_name: string, + limit_name: Limit, + language_limit_overrides: Limits + ): number { return ( (config.limit_overrides[language_name] && config.limit_overrides[language_name][limit_name]) || @@ -51,7 +64,10 @@ class Runtime { ); } - static compute_all_limits(language_name, language_limit_overrides) { + static compute_all_limits( + language_name: string, + language_limit_overrides: Limits + ) { return { timeouts: { compile: this.compute_single_limit( @@ -100,26 +116,25 @@ class Runtime { }; } - static load_package(package_dir) { + static load_package(package_dir: string) { let info = JSON.parse( - // @ts-ignore - readFileSync(join(package_dir, 'pkg-info.json')) - ); + readFileSync(join(package_dir, 'pkg-info.json'), 'utf8') + ) as PackageInfo; let { language, - version, + version: _version, build_platform, aliases, provides, limit_overrides, } = info; - version = parse(version); + const version = parse(_version); if (build_platform !== platform) { logger.warn( `Package ${language}-${version} was built for platform ${build_platform}, ` + - `but our platform is ${platform}` + `but our platform is ${platform}` ); } @@ -142,7 +157,6 @@ class Runtime { }); } else { runtimes.push( - // @ts-ignore new Runtime({ language, version, @@ -193,11 +207,10 @@ class Runtime { } } -const _runtimes = runtimes; -export { _runtimes as runtimes }; -const _Runtime = Runtime; -export { _Runtime as Runtime }; -export function get_runtimes_matching_language_version(lang, ver) { +export function get_runtimes_matching_language_version( + lang: string, + ver: string | import('semver/classes/range.js') +) { return runtimes.filter( rt => (rt.language == lang || rt.aliases.includes(lang)) && @@ -205,14 +218,15 @@ export function get_runtimes_matching_language_version(lang, ver) { ); } export function get_latest_runtime_matching_language_version( - lang, - ver + lang: string, + ver: string ) { - return get_runtimes_matching_language_version(lang, ver) - .sort((a, b) => rcompare(a.version, b.version))[0]; + return get_runtimes_matching_language_version(lang, ver).sort((a, b) => + rcompare(a.version, b.version) + )[0]; } -export function get_runtime_by_name_and_version(runtime, ver) { +export function get_runtime_by_name_and_version(runtime: string, ver: string) { return runtimes.find( rt => (rt.runtime == runtime || diff --git a/api/src/types.ts b/api/src/types.ts index 1504e45..68eb36a 100644 --- a/api/src/types.ts +++ b/api/src/types.ts @@ -1,21 +1,63 @@ +export interface Metadata { + language: string; + version: string; + aliases?: string[]; + dependencies?: Record; + provides: { + language: string; + aliases: string[]; + limit_overrides: Limits; + }[]; + limit_overrides?: Limits; +} + +export type Limit = + | 'compile_timeout' + | 'compile_memory_limit' + | 'max_process_count' + | 'max_open_files' + | 'max_file_size' + | 'output_max_size' + | 'run_memory_limit' + | 'run_timeout'; + +export type Limits = Record; + +export type LanguageMetadata = { + language: string; + version: string; +}; + +export type PackageInfo = Metadata & { build_platform: string }; + export type File = { content: string; name?: string; encoding?: 'base64' | 'hex' | 'utf8'; }; -export interface Body { +export type RequestBody = { language: string; version: string; files: Array; stdin?: string; args?: Array; - run_timeout?: number; - compile_timeout?: number; - compile_memory_limit?: number; - run_memory_limit?: number; +} & Partial; + +export interface ResponseBody { + language: string; + version: string; + run: { + stdout: string; + stderr: string; + output: string; + code: number; + signal?: NodeJS.Signals; + }; } - -export type ObjectType>, Key extends string> = { +export type ObjectType< + TObject extends Record>, + Key extends string +> = { [K in keyof TObject]: TObject[K][Key]; }; diff --git a/api/tsconfig.json b/api/tsconfig.json index 681bc70..ca4d1bd 100644 --- a/api/tsconfig.json +++ b/api/tsconfig.json @@ -4,9 +4,13 @@ "checkJs": true, "resolveJsonModule": true, "target": "ES2022", + "module": "ES2022", "moduleResolution": "nodenext", "allowSyntheticDefaultImports": true, - // we dont have any thing to compile just yet - "noEmit": true - } + "outDir": "./dist", + "rootDir": "src", + // "declaration": true + }, + "include": ["src"], + "exclude": ["dist", "node_modules"] } diff --git a/cli/commands/ppman_commands/install.js b/cli/commands/ppman_commands/install.js index a47665d..a4254b6 100644 --- a/cli/commands/ppman_commands/install.js +++ b/cli/commands/ppman_commands/install.js @@ -27,7 +27,7 @@ const msg_format = { exports.handler = async ({ axios, packages }) => { const requests = packages.map(package => split_package(package)); - for (request of requests) { + for (const request of requests) { try { const install = await axios.post(`/api/v2/packages`, request); diff --git a/package.json b/package.json index 7d0d6de..8f07606 100644 --- a/package.json +++ b/package.json @@ -1,12 +1,5 @@ { "devDependencies": { - "@types/chownr": "^1.0.0", - "@types/express-ws": "^3.0.1", - "@types/node-fetch": "^2.6.2", - "@types/uuid": "^9.0.1", "prettier": "2.4.1" - }, - "dependencies": { - "logplease": "github:Endercheif/logplease#feature/quality" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml deleted file mode 100644 index 8291940..0000000 --- a/pnpm-lock.yaml +++ /dev/null @@ -1,154 +0,0 @@ -lockfileVersion: 5.4 - -specifiers: - '@types/chownr': ^1.0.0 - '@types/express-ws': ^3.0.1 - '@types/node-fetch': ^2.6.2 - '@types/uuid': ^9.0.1 - logplease: github:Endercheif/logplease#feature/quality - prettier: 2.4.1 - -dependencies: - logplease: github.com/Endercheif/logplease/3cb7bb615d2c20402662a4ea37c812d767382a37 - -devDependencies: - '@types/chownr': 1.0.0 - '@types/express-ws': 3.0.1 - '@types/node-fetch': 2.6.2 - '@types/uuid': 9.0.1 - prettier: 2.4.1 - -packages: - - /@types/body-parser/1.19.2: - resolution: {integrity: sha512-ALYone6pm6QmwZoAgeyNksccT9Q4AWZQ6PvfwR37GT6r6FWUPguq6sUmNGSMV2Wr761oQoBxwGGa6DR5o1DC9g==} - dependencies: - '@types/connect': 3.4.35 - '@types/node': 18.14.2 - dev: true - - /@types/chownr/1.0.0: - resolution: {integrity: sha512-T6sN7Nzh1l4Dhvzbq3h5LHumB9LP/vw0MSyGbZKFGRmmx54/rQMYgEGlKvAT9Y1NZwqSAnQFPTOPUntWwyZthQ==} - dependencies: - '@types/node': 18.14.2 - dev: true - - /@types/connect/3.4.35: - resolution: {integrity: sha512-cdeYyv4KWoEgpBISTxWvqYsVy444DOqehiF3fM3ne10AmJ62RSyNkUnxMJXHQWRQQX2eR94m5y1IZyDwBjV9FQ==} - dependencies: - '@types/node': 18.14.2 - dev: true - - /@types/express-serve-static-core/4.17.33: - resolution: {integrity: sha512-TPBqmR/HRYI3eC2E5hmiivIzv+bidAfXofM+sbonAGvyDhySGw9/PQZFt2BLOrjUUR++4eJVpx6KnLQK1Fk9tA==} - dependencies: - '@types/node': 18.14.2 - '@types/qs': 6.9.7 - '@types/range-parser': 1.2.4 - dev: true - - /@types/express-ws/3.0.1: - resolution: {integrity: sha512-VguRXzcpPBF0IggIGpUoM65cZJDfMQxoc6dKoCz1yLzcwcXW7ft60yhq3ygKhyEhEIQFtLrWjyz4AJ1qjmzCFw==} - dependencies: - '@types/express': 4.17.17 - '@types/express-serve-static-core': 4.17.33 - '@types/ws': 8.5.4 - dev: true - - /@types/express/4.17.17: - resolution: {integrity: sha512-Q4FmmuLGBG58btUnfS1c1r/NQdlp3DMfGDGig8WhfpA2YRUtEkxAjkZb0yvplJGYdF1fsQ81iMDcH24sSCNC/Q==} - dependencies: - '@types/body-parser': 1.19.2 - '@types/express-serve-static-core': 4.17.33 - '@types/qs': 6.9.7 - '@types/serve-static': 1.15.1 - dev: true - - /@types/mime/3.0.1: - resolution: {integrity: sha512-Y4XFY5VJAuw0FgAqPNd6NNoV44jbq9Bz2L7Rh/J6jLTiHBSBJa9fxqQIvkIld4GsoDOcCbvzOUAbLPsSKKg+uA==} - dev: true - - /@types/node-fetch/2.6.2: - resolution: {integrity: sha512-DHqhlq5jeESLy19TYhLakJ07kNumXWjcDdxXsLUMJZ6ue8VZJj4kLPQVE/2mdHh3xZziNF1xppu5lwmS53HR+A==} - dependencies: - '@types/node': 18.14.2 - form-data: 3.0.1 - dev: true - - /@types/node/18.14.2: - resolution: {integrity: sha512-1uEQxww3DaghA0RxqHx0O0ppVlo43pJhepY51OxuQIKHpjbnYLA7vcdwioNPzIqmC2u3I/dmylcqjlh0e7AyUA==} - dev: true - - /@types/qs/6.9.7: - resolution: {integrity: sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw==} - dev: true - - /@types/range-parser/1.2.4: - resolution: {integrity: sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==} - dev: true - - /@types/serve-static/1.15.1: - resolution: {integrity: sha512-NUo5XNiAdULrJENtJXZZ3fHtfMolzZwczzBbnAeBbqBwG+LaG6YaJtuwzwGSQZ2wsCrxjEhNNjAkKigy3n8teQ==} - dependencies: - '@types/mime': 3.0.1 - '@types/node': 18.14.2 - dev: true - - /@types/uuid/9.0.1: - resolution: {integrity: sha512-rFT3ak0/2trgvp4yYZo5iKFEPsET7vKydKF+VRCxlQ9bpheehyAJH89dAkaLEq/j/RZXJIqcgsmPJKUP1Z28HA==} - dev: true - - /@types/ws/8.5.4: - resolution: {integrity: sha512-zdQDHKUgcX/zBc4GrwsE/7dVdAD8JR4EuiAXiiUhhfyIJXXb2+PrGshFyeXWQPMmmZ2XxgaqclgpIC7eTXc1mg==} - dependencies: - '@types/node': 18.14.2 - dev: true - - /asynckit/0.4.0: - resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} - dev: true - - /combined-stream/1.0.8: - resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} - engines: {node: '>= 0.8'} - dependencies: - delayed-stream: 1.0.0 - dev: true - - /delayed-stream/1.0.0: - resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} - engines: {node: '>=0.4.0'} - dev: true - - /form-data/3.0.1: - resolution: {integrity: sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==} - engines: {node: '>= 6'} - dependencies: - asynckit: 0.4.0 - combined-stream: 1.0.8 - mime-types: 2.1.35 - dev: true - - /mime-db/1.52.0: - resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} - engines: {node: '>= 0.6'} - dev: true - - /mime-types/2.1.35: - resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} - engines: {node: '>= 0.6'} - dependencies: - mime-db: 1.52.0 - dev: true - - /prettier/2.4.1: - resolution: {integrity: sha512-9fbDAXSBcc6Bs1mZrDYb3XKzDLm4EXXL9sC1LqKP5rZkT6KRr/rf9amVUcODVXgguK/isJz0d0hP72WeaKWsvA==} - engines: {node: '>=10.13.0'} - hasBin: true - dev: true - - github.com/Endercheif/logplease/3cb7bb615d2c20402662a4ea37c812d767382a37: - resolution: {tarball: https://codeload.github.com/Endercheif/logplease/tar.gz/3cb7bb615d2c20402662a4ea37c812d767382a37} - name: logplease - version: 1.3.3 - dev: false