From 9dca272f7a9a97088fc7eb7598cabd9a17bb6d65 Mon Sep 17 00:00:00 2001
From: Harshad Reddy Nalla <hnalla@redhat.com>
Date: Fri, 21 Jun 2024 04:28:55 -0400
Subject: [PATCH] consolidate logs of python script into container logs (#21)

Signed-off-by: Harshad Reddy Nalla <hnalla@redhat.com>
---
 elyra/kfp/bootstrapper.py | 52 +++++++++++++++++++++++++--------------
 1 file changed, 33 insertions(+), 19 deletions(-)

diff --git a/elyra/kfp/bootstrapper.py b/elyra/kfp/bootstrapper.py
index 6e039ca94..23e159054 100644
--- a/elyra/kfp/bootstrapper.py
+++ b/elyra/kfp/bootstrapper.py
@@ -472,7 +472,7 @@ def execute(self) -> None:
         """Execute the Python script and upload results to object storage"""
         python_script = os.path.basename(self.filepath)
         python_script_name = python_script.replace(".py", "")
-        python_script_output = f"{python_script_name}.log"
+        # python_script_output = f"{python_script_name}.log"
 
         try:
             OpUtil.log_operation_info(
@@ -485,28 +485,30 @@ def execute(self) -> None:
                 self.set_parameters_in_env()
             
             logger.info("----------------------Python logs start----------------------")
-            with open(python_script_output, "w") as log_file:
-                process = subprocess.Popen(run_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-                for line in iter(process.stdout.readline, b''):
-                    sys.stdout.write(line.decode())
-                    log_file.write(line.decode())
-                    
-                process.stdout.close()
-                return_code = process.wait()
-                logger.info("----------------------Python logs ends----------------------")
-                if return_code:
-                    raise subprocess.CalledProcessError(return_code, run_args)
+            # Removing support for the s3 storage of python script logs
+            # with open(python_script_output, "w") as log_file:
+            #     process = subprocess.Popen(run_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+            process = subprocess.Popen(run_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+
+            for line in iter(process.stdout.readline, b''):
+                sys.stdout.write(line.decode())
+                
+            process.stdout.close()
+            return_code = process.wait()
+            logger.info("----------------------Python logs ends----------------------")
+            if return_code:
+                raise subprocess.CalledProcessError(return_code, run_args)
             duration = time.time() - t0
             OpUtil.log_operation_info("python script execution completed", duration)
 
-            self.put_file_to_object_storage(python_script_output, python_script_output)
+            # self.put_file_to_object_storage(python_script_output, python_script_output)
             self.process_outputs()
         except Exception as ex:
             # log in case of errors
             logger.error(f"Unexpected error: {sys.exc_info()[0]}")
             logger.error(f"Error details: {ex}")
 
-            self.put_file_to_object_storage(python_script_output, python_script_output)
+            # self.put_file_to_object_storage(python_script_output, python_script_output)
             raise ex
 
 
@@ -517,7 +519,7 @@ def execute(self) -> None:
         """Execute the R script and upload results to object storage"""
         r_script = os.path.basename(self.filepath)
         r_script_name = r_script.replace(".r", "")
-        r_script_output = f"{r_script_name}.log"
+        # r_script_output = f"{r_script_name}.log"
 
         try:
             OpUtil.log_operation_info(f"executing R script using 'Rscript {r_script}' to '{r_script_output}'")
@@ -527,20 +529,32 @@ def execute(self) -> None:
             if self.parameter_pass_method == "env":
                 self.set_parameters_in_env()
 
-            with open(r_script_output, "w") as log_file:
-                subprocess.run(run_args, stdout=log_file, stderr=subprocess.STDOUT, check=True)
+            logger.info("----------------------R script logs start----------------------")
+            # Removing support for the s3 storage of R script logs
+            # with open(r_script_output, "w") as log_file:
+            #     process = subprocess.Popen(run_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+            process = subprocess.Popen(run_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+
+            for line in iter(process.stdout.readline, b''):
+                sys.stdout.write(line.decode())
+                
+            process.stdout.close()
+            return_code = process.wait()
+            logger.info("----------------------R script logs ends----------------------")
+            if return_code:
+                raise subprocess.CalledProcessError(return_code, run_args)
 
             duration = time.time() - t0
             OpUtil.log_operation_info("R script execution completed", duration)
 
-            self.put_file_to_object_storage(r_script_output, r_script_output)
+            # self.put_file_to_object_storage(r_script_output, r_script_output)
             self.process_outputs()
         except Exception as ex:
             # log in case of errors
             logger.error(f"Unexpected error: {sys.exc_info()[0]}")
             logger.error(f"Error details: {ex}")
 
-            self.put_file_to_object_storage(r_script_output, r_script_output)
+            # self.put_file_to_object_storage(r_script_output, r_script_output)
             raise ex