From 24a352699da305e353b885560504bf8b9d3717b3 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Thu, 14 Oct 2021 00:46:49 +1300 Subject: [PATCH 1/2] Support for uploading files in base64/hex format --- api/src/job.js | 18 +++++++++++------- docs/api-v2.md | 1 + readme.md | 1 + 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/api/src/job.js b/api/src/job.js index 8948be1..bf7870b 100644 --- a/api/src/job.js +++ b/api/src/job.js @@ -16,12 +16,12 @@ const job_states = { let uid = 0; let gid = 0; -let remainingJobSpaces = config.max_concurrent_jobs; +let remaining_job_spaces = config.max_concurrent_jobs; let jobQueue = []; setInterval(() => { // Every 10ms try resolve a new job, if there is an available slot - if (jobQueue.length > 0 && remainingJobSpaces > 0) { + if (jobQueue.length > 0 && remaining_job_spaces > 0) { jobQueue.shift()(); } }, 10); @@ -33,6 +33,9 @@ class Job { this.files = files.map((file, i) => ({ name: file.name || `file${i}.code`, content: file.content, + encoding: ['base64', 'hex', 'utf8'].includes(file.encoding) + ? file.encoding + : 'utf8', })); this.args = args; @@ -59,7 +62,7 @@ class Job { } async prime() { - if (remainingJobSpaces < 1) { + if (remaining_job_spaces < 1) { logger.info(`Awaiting job slot uuid=${this.uuid}`); await new Promise(resolve => { jobQueue.push(resolve); @@ -67,7 +70,7 @@ class Job { } logger.info(`Priming job uuid=${this.uuid}`); - remainingJobSpaces--; + remaining_job_spaces--; logger.debug('Writing files to job cache'); logger.debug(`Transfering ownership uid=${this.uid} gid=${this.gid}`); @@ -76,8 +79,9 @@ class Job { await fs.chown(this.dir, this.uid, this.gid); for (const file of this.files) { - let file_path = path.join(this.dir, file.name); + const file_path = path.join(this.dir, file.name); const rel = path.relative(this.dir, file_path); + const file_content = Buffer.from(file.content, file.encoding); if (rel.startsWith('..')) throw Error( @@ -90,7 +94,7 @@ class Job { }); await fs.chown(path.dirname(file_path), this.uid, this.gid); - await fs.write_file(file_path, file.content); + await fs.write_file(file_path, file_content); await fs.chown(file_path, this.uid, this.gid); } @@ -387,7 +391,7 @@ class Job { await this.cleanup_filesystem(); - remainingJobSpaces++; + remaining_job_spaces++; } } diff --git a/docs/api-v2.md b/docs/api-v2.md index d5cb486..b25e142 100644 --- a/docs/api-v2.md +++ b/docs/api-v2.md @@ -60,6 +60,7 @@ Runs the given code, using the given runtime and arguments, returning the result - `files`: An array of files which should be uploaded into the job context - `files[].name` (_optional_): Name of file to be written, if none a random name is picked - `files[].content`: Content of file to be written +- `files[].encoding` (_optional_): The encoding scheme used for the file content. One of `base64`, `hex` or `utf8`. Defaults to `utf8`. - `stdin` (_optional_): Text to pass into stdin of the program. Defaults to blank string. - `args` (_optional_): Arguments to pass to the program. Defaults to none - `run_timeout` (_optional_): The maximum allowed time in milliseconds for the compile stage to finish before bailing out. Must be a number, less than or equal to the configured maximum timeout. diff --git a/readme.md b/readme.md index a2f3afa..d2f2e75 100644 --- a/readme.md +++ b/readme.md @@ -256,6 +256,7 @@ This endpoint requests execution of some arbitrary code. - `files` (**required**) An array of files containing code or other data that should be used for execution. The first file in this array is considered the main file. - `files[].name` (_optional_) The name of the file to upload, must be a string containing no path or left out. - `files[].content` (**required**) The content of the files to upload, must be a string containing text to write. +- `files[].encoding` (_optional_) The encoding scheme used for the file content. One of `base64`, `hex` or `utf8`. Defaults to `utf8`. - `stdin` (_optional_) The text to pass as stdin to the program. Must be a string or left out. Defaults to blank string. - `args` (_optional_) The arguments to pass to the program. Must be an array or left out. Defaults to `[]`. - `compile_timeout` (_optional_) The maximum time allowed for the compile stage to finish before bailing out in milliseconds. Must be a number or left out. Defaults to `10000` (10 seconds). From 0faea205dbdc05429087dc7cab804bae667f5ff3 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Thu, 14 Oct 2021 01:36:29 +1300 Subject: [PATCH 2/2] Only compile/run files in utf8 encoding --- api/src/job.js | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/api/src/job.js b/api/src/job.js index bf7870b..7acff22 100644 --- a/api/src/job.js +++ b/api/src/job.js @@ -222,6 +222,8 @@ class Job { } runtime=${this.runtime.toString()}` ); + const code_files = this.files.filter(file => file.encoding == 'utf8'); + logger.debug('Compiling'); let compile; @@ -229,7 +231,7 @@ class Job { if (this.runtime.compiled) { compile = await this.safe_call( path.join(this.runtime.pkgdir, 'compile'), - this.files.map(x => x.name), + code_files.map(x => x.name), this.timeouts.compile, this.memory_limits.compile ); @@ -239,7 +241,7 @@ class Job { const run = await this.safe_call( path.join(this.runtime.pkgdir, 'run'), - [this.files[0].name, ...this.args], + [code_files[0].name, ...this.args], this.timeouts.run, this.memory_limits.run ); @@ -268,11 +270,13 @@ class Job { } gid=${this.gid} runtime=${this.runtime.toString()}` ); + const code_files = this.files.filter(file => file.encoding == 'utf8'); + if (this.runtime.compiled) { eventBus.emit('stage', 'compile'); const { error, code, signal } = await this.safe_call( path.join(this.runtime.pkgdir, 'compile'), - this.files.map(x => x.name), + code_files.map(x => x.name), this.timeouts.compile, this.memory_limits.compile, eventBus @@ -285,7 +289,7 @@ class Job { eventBus.emit('stage', 'run'); const { error, code, signal } = await this.safe_call( path.join(this.runtime.pkgdir, 'run'), - [this.files[0].name, ...this.args], + [code_files[0].name, ...this.args], this.timeouts.run, this.memory_limits.run, eventBus