Try-Catch process kills to handle dead processes

engineer-man · Oct 30, 2023 · d8af1ee · d8af1ee
1 parent dc4bb29
commit d8af1ee
Showing 1 changed file with 30 additions and 3 deletions.
diff --git a/api/src/job.js b/api/src/job.js
@@ -204,7 +204,16 @@ class Job {
                 (timeout >= 0 &&
                     set_timeout(async _ => {
                         this.logger.info(`Timeout exceeded timeout=${timeout}`);
-                        process.kill(proc.pid, 'SIGKILL');
+                        try {
+                            process.kill(proc.pid, 'SIGKILL');
+                        }
+                        catch (e) {
+                            // Could already be dead and just needs to be waited on
+                            this.logger.debug(
+                                `Got error while SIGKILLing process ${proc}:`,
+                                e
+                            );
+                        }
                     }, timeout)) ||
                 null;
             this.#active_timeouts.push(kill_timeout);
@@ -214,7 +223,16 @@ class Job {
                     event_bus.emit('stderr', data);
                 } else if (stderr.length > this.runtime.output_max_size) {
                     this.logger.info(`stderr length exceeded`);
-                    process.kill(proc.pid, 'SIGKILL');
+                    try {
+                        process.kill(proc.pid, 'SIGKILL');
+                    }
+                    catch (e) {
+                        // Could already be dead and just needs to be waited on
+                        this.logger.debug(
+                            `Got error while SIGKILLing process ${proc}:`,
+                            e
+                        );
+                    }
                 } else {
                     stderr += data;
                     output += data;
@@ -226,7 +244,16 @@ class Job {
                     event_bus.emit('stdout', data);
                 } else if (stdout.length > this.runtime.output_max_size) {
                     this.logger.info(`stdout length exceeded`);
-                    process.kill(proc.pid, 'SIGKILL');
+                    try {
+                        process.kill(proc.pid, 'SIGKILL');
+                    }
+                    catch (e) {
+                        // Could already be dead and just needs to be waited on
+                        this.logger.debug(
+                            `Got error while SIGKILLing process ${proc}:`,
+                            e
+                        );
+                    }
                 } else {
                     stdout += data;
                     output += data;