From d8af1ee301abf1eff0a008f7625600b6b743b5b8 Mon Sep 17 00:00:00 2001
From: Shubham Sahai <shubham@codaveri.com>
Date: Mon, 30 Oct 2023 20:09:01 +0800
Subject: [PATCH] Try-Catch process kills to handle dead processes

---
 api/src/job.js | 33 ++++++++++++++++++++++++++++++---
 1 file changed, 30 insertions(+), 3 deletions(-)

diff --git a/api/src/job.js b/api/src/job.js
index 00ee14f..46efe27 100644
--- a/api/src/job.js
+++ b/api/src/job.js
@@ -204,7 +204,16 @@ class Job {
                 (timeout >= 0 &&
                     set_timeout(async _ => {
                         this.logger.info(`Timeout exceeded timeout=${timeout}`);
-                        process.kill(proc.pid, 'SIGKILL');
+                        try {
+                            process.kill(proc.pid, 'SIGKILL');
+                        }
+                        catch (e) {
+                            // Could already be dead and just needs to be waited on
+                            this.logger.debug(
+                                `Got error while SIGKILLing process ${proc}:`,
+                                e
+                            );
+                        }
                     }, timeout)) ||
                 null;
             this.#active_timeouts.push(kill_timeout);
@@ -214,7 +223,16 @@ class Job {
                     event_bus.emit('stderr', data);
                 } else if (stderr.length > this.runtime.output_max_size) {
                     this.logger.info(`stderr length exceeded`);
-                    process.kill(proc.pid, 'SIGKILL');
+                    try {
+                        process.kill(proc.pid, 'SIGKILL');
+                    }
+                    catch (e) {
+                        // Could already be dead and just needs to be waited on
+                        this.logger.debug(
+                            `Got error while SIGKILLing process ${proc}:`,
+                            e
+                        );
+                    }
                 } else {
                     stderr += data;
                     output += data;
@@ -226,7 +244,16 @@ class Job {
                     event_bus.emit('stdout', data);
                 } else if (stdout.length > this.runtime.output_max_size) {
                     this.logger.info(`stdout length exceeded`);
-                    process.kill(proc.pid, 'SIGKILL');
+                    try {
+                        process.kill(proc.pid, 'SIGKILL');
+                    }
+                    catch (e) {
+                        // Could already be dead and just needs to be waited on
+                        this.logger.debug(
+                            `Got error while SIGKILLing process ${proc}:`,
+                            e
+                        );
+                    }
                 } else {
                     stdout += data;
                     output += data;