Merge pull request #373 from engineer-man/binary

Support for uploading files in base64/hex format

api/src/job.js

@@ -16,12 +16,12 @@ const job_states = {
 let uid = 0;
 let gid = 0;
 
-let remainingJobSpaces = config.max_concurrent_jobs;
+let remaining_job_spaces = config.max_concurrent_jobs;
 let jobQueue = [];
 
 setInterval(() => {
     // Every 10ms try resolve a new job, if there is an available slot
-    if (jobQueue.length > 0 && remainingJobSpaces > 0) {
+    if (jobQueue.length > 0 && remaining_job_spaces > 0) {
         jobQueue.shift()();
     }
 }, 10);
@@ -33,6 +33,9 @@ class Job {
         this.files = files.map((file, i) => ({
             name: file.name || `file${i}.code`,
             content: file.content,
+            encoding: ['base64', 'hex', 'utf8'].includes(file.encoding)
+                ? file.encoding
+                : 'utf8',
         }));
 
         this.args = args;
@@ -59,7 +62,7 @@ class Job {
     }
 
     async prime() {
-        if (remainingJobSpaces < 1) {
+        if (remaining_job_spaces < 1) {
             logger.info(`Awaiting job slot uuid=${this.uuid}`);
             await new Promise(resolve => {
                 jobQueue.push(resolve);
@@ -67,7 +70,7 @@ class Job {
         }
 
         logger.info(`Priming job uuid=${this.uuid}`);
-        remainingJobSpaces--;
+        remaining_job_spaces--;
         logger.debug('Writing files to job cache');
 
         logger.debug(`Transfering ownership uid=${this.uid} gid=${this.gid}`);
@@ -76,8 +79,9 @@ class Job {
         await fs.chown(this.dir, this.uid, this.gid);
 
         for (const file of this.files) {
-            let file_path = path.join(this.dir, file.name);
+            const file_path = path.join(this.dir, file.name);
             const rel = path.relative(this.dir, file_path);
+            const file_content = Buffer.from(file.content, file.encoding);
 
             if (rel.startsWith('..'))
                 throw Error(
@@ -90,7 +94,7 @@ class Job {
             });
             await fs.chown(path.dirname(file_path), this.uid, this.gid);
 
-            await fs.write_file(file_path, file.content);
+            await fs.write_file(file_path, file_content);
             await fs.chown(file_path, this.uid, this.gid);
         }
 
@@ -218,6 +222,8 @@ class Job {
             } runtime=${this.runtime.toString()}`
         );
 
+        const code_files = this.files.filter(file => file.encoding == 'utf8');
+
         logger.debug('Compiling');
 
         let compile;
@@ -225,7 +231,7 @@ class Job {
         if (this.runtime.compiled) {
             compile = await this.safe_call(
                 path.join(this.runtime.pkgdir, 'compile'),
-                this.files.map(x => x.name),
+                code_files.map(x => x.name),
                 this.timeouts.compile,
                 this.memory_limits.compile
             );
@@ -235,7 +241,7 @@ class Job {
 
         const run = await this.safe_call(
             path.join(this.runtime.pkgdir, 'run'),
-            [this.files[0].name, ...this.args],
+            [code_files[0].name, ...this.args],
             this.timeouts.run,
             this.memory_limits.run
         );
@@ -264,11 +270,13 @@ class Job {
             } gid=${this.gid} runtime=${this.runtime.toString()}`
         );
 
+        const code_files = this.files.filter(file => file.encoding == 'utf8');
+
         if (this.runtime.compiled) {
             eventBus.emit('stage', 'compile');
             const { error, code, signal } = await this.safe_call(
                 path.join(this.runtime.pkgdir, 'compile'),
-                this.files.map(x => x.name),
+                code_files.map(x => x.name),
                 this.timeouts.compile,
                 this.memory_limits.compile,
                 eventBus
@@ -281,7 +289,7 @@ class Job {
         eventBus.emit('stage', 'run');
         const { error, code, signal } = await this.safe_call(
             path.join(this.runtime.pkgdir, 'run'),
-            [this.files[0].name, ...this.args],
+            [code_files[0].name, ...this.args],
             this.timeouts.run,
             this.memory_limits.run,
             eventBus
@@ -387,7 +395,7 @@ class Job {
 
         await this.cleanup_filesystem();
 
-        remainingJobSpaces++;
+        remaining_job_spaces++;
     }
 }
 

docs/api-v2.md

@@ -60,6 +60,7 @@ Runs the given code, using the given runtime and arguments, returning the result
 -   `files`: An array of files which should be uploaded into the job context
 -   `files[].name` (_optional_): Name of file to be written, if none a random name is picked
 -   `files[].content`: Content of file to be written
+-   `files[].encoding` (_optional_): The encoding scheme used for the file content. One of `base64`, `hex` or `utf8`. Defaults to `utf8`.
 -   `stdin` (_optional_): Text to pass into stdin of the program. Defaults to blank string.
 -   `args` (_optional_): Arguments to pass to the program. Defaults to none
 -   `run_timeout` (_optional_): The maximum allowed time in milliseconds for the compile stage to finish before bailing out. Must be a number, less than or equal to the configured maximum timeout.

readme.md

@@ -256,6 +256,7 @@ This endpoint requests execution of some arbitrary code.
 -   `files` (**required**) An array of files containing code or other data that should be used for execution. The first file in this array is considered the main file.
 -   `files[].name` (_optional_) The name of the file to upload, must be a string containing no path or left out.
 -   `files[].content` (**required**) The content of the files to upload, must be a string containing text to write.
+-   `files[].encoding` (_optional_) The encoding scheme used for the file content. One of `base64`, `hex` or `utf8`. Defaults to `utf8`.
 -   `stdin` (_optional_) The text to pass as stdin to the program. Must be a string or left out. Defaults to blank string.
 -   `args` (_optional_) The arguments to pass to the program. Must be an array or left out. Defaults to `[]`.
 -   `compile_timeout` (_optional_) The maximum time allowed for the compile stage to finish before bailing out in milliseconds. Must be a number or left out. Defaults to `10000` (10 seconds).