api: maximum concurrent jobs and potential fix for gcc

This commit is contained in:
Thomas Hobson 2021-10-01 20:28:54 +13:00 committed by Brikaa
parent c24937aeb7
commit 5d392effcc
3 changed files with 47 additions and 7 deletions

View File

@ -114,6 +114,13 @@ const options = [
'https://github.com/engineer-man/piston/releases/download/pkgs/index', 'https://github.com/engineer-man/piston/releases/download/pkgs/index',
validators: [], validators: [],
}, },
{
key: 'max_concurrent_jobs',
desc: 'Maximum number of concurrent jobs to run at one time',
default: 64,
parser: parse_int,
validators: [(x) => x > 0 || `${x} cannot be negative`]
}
]; ];
logger.info(`Loading Configuration from environment`); logger.info(`Loading Configuration from environment`);

View File

@ -16,6 +16,19 @@ const job_states = {
let uid = 0; let uid = 0;
let gid = 0; let gid = 0;
let remainingJobSpaces = config.max_concurrent_jobs;
let jobQueue = [];
setInterval(()=>{
// Every 10ms try resolve a new job, if there is an available slot
if(jobQueue.length > 0 && remainingJobSpaces > 0){
jobQueue.shift()()
}
}, 10)
class Job { class Job {
constructor({ runtime, files, args, stdin, timeouts, memory_limits }) { constructor({ runtime, files, args, stdin, timeouts, memory_limits }) {
this.uuid = uuidv4(); this.uuid = uuidv4();
@ -48,8 +61,15 @@ class Job {
} }
async prime() { async prime() {
logger.info(`Priming job uuid=${this.uuid}`); if(remainingJobSpaces < 1){
logger.info(`Awaiting job slot uuid=${this.uuid}`)
await new Promise((resolve)=>{
jobQueue.push(resolve)
})
}
logger.info(`Priming job uuid=${this.uuid}`);
remainingJobSpaces--;
logger.debug('Writing files to job cache'); logger.debug('Writing files to job cache');
logger.debug(`Transfering ownership uid=${this.uid} gid=${this.gid}`); logger.debug(`Transfering ownership uid=${this.uid} gid=${this.gid}`);
@ -152,21 +172,23 @@ class Job {
} }
}); });
const exit_cleanup = () => { const exit_cleanup = async () => {
clear_timeout(kill_timeout); clear_timeout(kill_timeout);
proc.stderr.destroy(); proc.stderr.destroy();
proc.stdout.destroy(); proc.stdout.destroy();
await this.cleanup_processes()
}; };
proc.on('exit', (code, signal) => { proc.on('exit', async (code, signal) => {
exit_cleanup(); await exit_cleanup();
resolve({stdout, stderr, code, signal, output }); resolve({stdout, stderr, code, signal, output });
}); });
proc.on('error', err => { proc.on('error', async err => {
exit_cleanup(); await exit_cleanup();
reject({ error: err, stdout, stderr, output }); reject({ error: err, stdout, stderr, output });
}); });
@ -339,11 +361,13 @@ class Job {
async cleanup() { async cleanup() {
logger.info(`Cleaning up job uuid=${this.uuid}`); logger.info(`Cleaning up job uuid=${this.uuid}`);
await this.cleanup_processes();
await this.cleanup_filesystem(); await this.cleanup_filesystem();
remainingJobSpaces++;
} }
} }
module.exports = { module.exports = {
Job, Job,
}; };

View File

@ -145,3 +145,12 @@ default: https://github.com/engineer-man/piston/releases/download/pkgs/index
``` ```
URL for repository index, where packages will be downloaded from. URL for repository index, where packages will be downloaded from.
## Maximum Concurrent Jobs
```yaml
key: PISTON_MAX_CONCURRENT_JOBS
default: 64
```
Maximum number of jobs to run concurrently.