From d5392eafb83625ae3feb46813c8d23f1385c906e Mon Sep 17 00:00:00 2001 From: Omar Brikaa Date: Fri, 11 Feb 2022 22:04:36 +0200 Subject: [PATCH 1/2] Naming refactor, remove id getter and use index --- api/src/api/v3.js | 4 ++-- api/src/bin/pistond.js | 12 ++++++++---- api/src/runtime.js | 12 ++++-------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/api/src/api/v3.js b/api/src/api/v3.js index aee5772..fb1b764 100644 --- a/api/src/api/v3.js +++ b/api/src/api/v3.js @@ -277,13 +277,13 @@ router.post('/execute', async (req, res) => { }); router.get('/runtimes', (req, res) => { - const runtimes = runtime.map(rt => { + const runtimes = runtime.map((rt, index) => { return { language: rt.language, version: rt.version.raw, aliases: rt.aliases, runtime: rt.runtime, - id: rt.id, + id: index, }; }); diff --git a/api/src/bin/pistond.js b/api/src/bin/pistond.js index c56cc99..9e1fca7 100755 --- a/api/src/bin/pistond.js +++ b/api/src/bin/pistond.js @@ -39,10 +39,14 @@ expressWs(app); logger.info('Loading packages'); - const runtimes_data = cp.execSync(`nix eval --json ${config.flake_path}#pistonRuntimeSets.${config.runtime_set} --apply builtins.attrNames`).toString(); - const runtimes = JSON.parse(runtimes_data); - - runtimes.for_each(pkg => runtime.load_runtime(pkg)); + const runtimes_data = cp + .execSync( + `nix eval --json ${config.flake_path}#pistonRuntimeSets.${config.runtime_set} --apply builtins.attrNames` + ) + .toString(); + const language_names = JSON.parse(runtimes_data); + + language_names.for_each(language_name => runtime.load_runtime(language_name)); logger.info('Starting API Server'); logger.debug('Constructing Express App'); diff --git a/api/src/runtime.js b/api/src/runtime.js index 554feed..a2cf0d9 100644 --- a/api/src/runtime.js +++ b/api/src/runtime.js @@ -120,9 +120,9 @@ class Runtime { logger.debug(`Finished ensuring ${this} is installed`); } - static load_runtime(flake_key) { - logger.info(`Loading ${flake_key}`); - const flake_path = `${config.flake_path}#pistonRuntimeSets.${config.runtime_set}.${flake_key}`; + static load_runtime(language_name) { + logger.info(`Loading ${language_name}`); + const flake_path = `${config.flake_path}#pistonRuntimeSets.${config.runtime_set}.${language_name}`; const metadata_command = `nix eval --json ${flake_path}.metadata`; const metadata = JSON.parse(cp.execSync(metadata_command)); @@ -139,17 +139,13 @@ class Runtime { runtimes.push(this_runtime); - logger.debug(`Package ${flake_key} was loaded`); + logger.debug(`Package ${language_name} was loaded`); } get compiled() { return this.compile !== null; } - get id() { - return runtimes.indexOf(this); - } - toString() { return `${this.language}-${this.version}`; } From cb834cff3e283cb96667aeb5b31fe21b58b6ebc6 Mon Sep 17 00:00:00 2001 From: Omar Brikaa Date: Fri, 11 Feb 2022 23:03:46 +0200 Subject: [PATCH 2/2] Refactor runtimes: no globally shared state --- api/src/api/v2.js | 14 ++-- api/src/api/v3.js | 15 ++-- api/src/bin/pistond.js | 14 +++- api/src/bin/test.js | 22 ++---- api/src/runtime.js | 170 ++++++++++++++++++++--------------------- 5 files changed, 111 insertions(+), 124 deletions(-) diff --git a/api/src/api/v2.js b/api/src/api/v2.js index 0ccbbf0..fede616 100644 --- a/api/src/api/v2.js +++ b/api/src/api/v2.js @@ -48,7 +48,7 @@ const SIGNALS = [ ]; // ref: https://man7.org/linux/man-pages/man7/signal.7.html -function get_job(body) { +function get_job(job_info, available_runtimes) { let { language, args, @@ -58,7 +58,7 @@ function get_job(body) { run_memory_limit, run_timeout, compile_timeout, - } = body; + } = job_info; return new Promise((resolve, reject) => { if (!language || typeof language !== 'string') { @@ -80,7 +80,7 @@ function get_job(body) { } } - const rt = runtime.find(rt => + const rt = available_runtimes.find(rt => [...rt.aliases, rt.language].includes(rt.language) ); @@ -102,7 +102,7 @@ function get_job(body) { for (const constraint of ['memory_limit', 'timeout']) { for (const type of ['compile', 'run']) { const constraint_name = `${type}_${constraint}`; - const constraint_value = body[constraint_name]; + const constraint_value = job_info[constraint_name]; const configured_limit = rt[`${constraint}s`][type]; if (!constraint_value) { continue; @@ -201,7 +201,7 @@ router.ws('/connect', async (ws, req) => { switch (msg.type) { case 'init': if (job === null) { - job = await get_job(msg); + job = await get_job(msg, req.app.locals.runtimes); await job.prime(); @@ -264,7 +264,7 @@ router.ws('/connect', async (ws, req) => { router.post('/execute', async (req, res) => { try { - const job = await get_job(req.body); + const job = await get_job(req.body, req.app.locals.runtimes); await job.prime(); @@ -279,7 +279,7 @@ router.post('/execute', async (req, res) => { }); router.get('/runtimes', (req, res) => { - const runtimes = runtime.map(rt => { + const runtimes = req.app.locals.runtimes.map(rt => { return { language: rt.language, version: rt.version.raw, diff --git a/api/src/api/v3.js b/api/src/api/v3.js index fb1b764..08b4b35 100644 --- a/api/src/api/v3.js +++ b/api/src/api/v3.js @@ -3,7 +3,6 @@ const router = express.Router(); const events = require('events'); -const runtime = require('../runtime'); const { Job } = require('../job'); const SIGNALS = [ @@ -48,7 +47,7 @@ const SIGNALS = [ ]; // ref: https://man7.org/linux/man-pages/man7/signal.7.html -function get_job(body) { +function get_job(job_info, available_runtimes) { const { runtime_id, args, @@ -58,7 +57,7 @@ function get_job(body) { run_memory_limit, run_timeout, compile_timeout, - } = body; + } = job_info; return new Promise((resolve, reject) => { if (typeof runtime_id !== 'number') { @@ -73,7 +72,7 @@ function get_job(body) { }); } - const rt = runtime[runtime_id]; + const rt = available_runtimes[runtime_id]; if (rt === undefined) { return reject({ @@ -99,7 +98,7 @@ function get_job(body) { for (const constraint of ['memory_limit', 'timeout']) { for (const type of ['compile', 'run']) { const constraint_name = `${type}_${constraint}`; - const constraint_value = body[constraint_name]; + const constraint_value = job_info[constraint_name]; const configured_limit = rt[`${constraint}s`][type]; if (!constraint_value) { continue; @@ -199,7 +198,7 @@ router.ws('/connect', async (ws, req) => { switch (msg.type) { case 'init': if (job === null) { - job = await get_job(msg); + job = await get_job(msg, req.app.locals.runtimes); await job.prime(); @@ -262,7 +261,7 @@ router.ws('/connect', async (ws, req) => { router.post('/execute', async (req, res) => { try { - const job = await get_job(req.body); + const job = await get_job(req.body, req.app.locals.runtimes); await job.prime(); @@ -277,7 +276,7 @@ router.post('/execute', async (req, res) => { }); router.get('/runtimes', (req, res) => { - const runtimes = runtime.map((rt, index) => { + const runtimes = req.app.locals.runtimes.map((rt, index) => { return { language: rt.language, version: rt.version.raw, diff --git a/api/src/bin/pistond.js b/api/src/bin/pistond.js index 9e1fca7..1712c36 100755 --- a/api/src/bin/pistond.js +++ b/api/src/bin/pistond.js @@ -7,7 +7,6 @@ const globals = require('../globals'); const config = require('../config'); const cp = require('child_process'); const path = require('path'); -const fs = require('fs/promises'); const fss = require('fs'); const body_parser = require('body-parser'); const runtime = require('../runtime'); @@ -44,9 +43,18 @@ expressWs(app); `nix eval --json ${config.flake_path}#pistonRuntimeSets.${config.runtime_set} --apply builtins.attrNames` ) .toString(); - const language_names = JSON.parse(runtimes_data); + const runtime_names = JSON.parse(runtimes_data); - language_names.for_each(language_name => runtime.load_runtime(language_name)); + logger.info('Loading the runtimes from the flakes'); + const runtimes = runtime_names.map(runtime_name => { + logger.info(`Loading ${runtime_name}`); + return runtime.get_runtime_from_flakes(runtime_name); + }); + logger.info('Ensuring all of the runtimes are built'); + runtimes.for_each(r => r.ensure_built()); + + Object.freeze(runtimes); + app.locals.runtimes = runtimes; logger.info('Starting API Server'); logger.debug('Constructing Express App'); diff --git a/api/src/bin/test.js b/api/src/bin/test.js index 16de55d..fdaa788 100755 --- a/api/src/bin/test.js +++ b/api/src/bin/test.js @@ -30,29 +30,17 @@ const { Job } = require('../job'); const runtime_path = `${config.flake_path}#pistonRuntimes.${runtime_name}`; logger.info(`Testing runtime ${runtime_path}`); - logger.debug(`Loading runtime metadata`); - const metadata = JSON.parse( - cp.execSync(`nix eval --json ${runtime_path}.metadata --json`) - ); + logger.debug(`Loading runtime`); + + const testable_runtime = runtime.get_runtime_from_flakes(runtime_name); + + testable_runtime.ensure_built(); logger.debug(`Loading runtime tests`); const tests = JSON.parse( cp.execSync(`nix eval --json ${runtime_path}.tests --json`) ); - logger.debug(`Loading runtime`); - - const testable_runtime = new runtime.Runtime({ - ...metadata, - ...runtime.Runtime.compute_all_limits( - metadata.language, - metadata.limitOverrides - ), - flake_path: runtime_path, - }); - - testable_runtime.ensure_built(); - logger.info(`Running tests`); for (const test of tests) { diff --git a/api/src/runtime.js b/api/src/runtime.js index a2cf0d9..ba1ab87 100644 --- a/api/src/runtime.js +++ b/api/src/runtime.js @@ -2,8 +2,6 @@ const logger = require('logplease').create('runtime'); const cp = require('child_process'); const config = require('./config'); -const runtimes = []; - class Runtime { constructor({ language, @@ -41,69 +39,6 @@ class Runtime { this.package_support = packageSupport; } - static compute_single_limit( - language_name, - limit_name, - language_limit_overrides - ) { - return ( - (config.limit_overrides[language_name] && - config.limit_overrides[language_name][limit_name]) || - (language_limit_overrides && - language_limit_overrides[limit_name]) || - config[limit_name] - ); - } - - static compute_all_limits(language_name, language_limit_overrides) { - return { - timeouts: { - compile: this.compute_single_limit( - language_name, - 'compile_timeout', - language_limit_overrides - ), - run: this.compute_single_limit( - language_name, - 'run_timeout', - language_limit_overrides - ), - }, - memory_limits: { - compile: this.compute_single_limit( - language_name, - 'compile_memory_limit', - language_limit_overrides - ), - run: this.compute_single_limit( - language_name, - 'run_memory_limit', - language_limit_overrides - ), - }, - max_process_count: this.compute_single_limit( - language_name, - 'max_process_count', - language_limit_overrides - ), - max_open_files: this.compute_single_limit( - language_name, - 'max_open_files', - language_limit_overrides - ), - max_file_size: this.compute_single_limit( - language_name, - 'max_file_size', - language_limit_overrides - ), - output_max_size: this.compute_single_limit( - language_name, - 'output_max_size', - language_limit_overrides - ), - }; - } - ensure_built() { logger.info(`Ensuring ${this} is built`); @@ -120,28 +55,6 @@ class Runtime { logger.debug(`Finished ensuring ${this} is installed`); } - static load_runtime(language_name) { - logger.info(`Loading ${language_name}`); - const flake_path = `${config.flake_path}#pistonRuntimeSets.${config.runtime_set}.${language_name}`; - const metadata_command = `nix eval --json ${flake_path}.metadata`; - const metadata = JSON.parse(cp.execSync(metadata_command)); - - const this_runtime = new Runtime({ - ...metadata, - ...Runtime.compute_all_limits( - metadata.language, - metadata.limitOverrides - ), - flake_path, - }); - - this_runtime.ensure_built(); - - runtimes.push(this_runtime); - - logger.debug(`Package ${language_name} was loaded`); - } - get compiled() { return this.compile !== null; } @@ -151,6 +64,85 @@ class Runtime { } } -module.exports = runtimes; +function compute_single_limit( + language_name, + limit_name, + language_limit_overrides +) { + return ( + (config.limit_overrides[language_name] && + config.limit_overrides[language_name][limit_name]) || + (language_limit_overrides && + language_limit_overrides[limit_name]) || + config[limit_name] + ); +} + +function compute_all_limits(language_name, language_limit_overrides) { + return { + timeouts: { + compile: compute_single_limit( + language_name, + 'compile_timeout', + language_limit_overrides + ), + run: compute_single_limit( + language_name, + 'run_timeout', + language_limit_overrides + ), + }, + memory_limits: { + compile: compute_single_limit( + language_name, + 'compile_memory_limit', + language_limit_overrides + ), + run: compute_single_limit( + language_name, + 'run_memory_limit', + language_limit_overrides + ), + }, + max_process_count: compute_single_limit( + language_name, + 'max_process_count', + language_limit_overrides + ), + max_open_files: compute_single_limit( + language_name, + 'max_open_files', + language_limit_overrides + ), + max_file_size: compute_single_limit( + language_name, + 'max_file_size', + language_limit_overrides + ), + output_max_size: compute_single_limit( + language_name, + 'output_max_size', + language_limit_overrides + ), + }; +} + +function get_runtime_from_flakes(runtime_name) { + const flake_path = `${config.flake_path}#pistonRuntimeSets.${config.runtime_set}.${runtime_name}`; + const metadata_command = `nix eval --json ${flake_path}.metadata`; + const metadata = JSON.parse(cp.execSync(metadata_command)); + + const this_runtime = new Runtime({ + ...metadata, + ...compute_all_limits( + metadata.language, + metadata.limitOverrides + ), + flake_path, + }); + + return this_runtime +} + module.exports.Runtime = Runtime; -module.exports.load_runtime = Runtime.load_runtime; +module.exports.get_runtime_from_flakes = get_runtime_from_flakes;