MINIFICPP-2606 Improve logging for python virtualenv initialization

extensions/python/PythonDependencyInstaller.cpp

@@ -17,11 +17,14 @@
  */
 #include "PythonDependencyInstaller.h"
 
+#include <cstdio>
+
 #include "PythonScriptException.h"
 #include "PythonInterpreter.h"
 #include "PyException.h"
 #include "types/Types.h"
 #include "utils/OptionalUtils.h"
+#include "utils/ConfigurationUtils.h"
 
 namespace org::apache::nifi::minifi::extensions::python {
 
@@ -49,6 +52,44 @@ std::string encapsulateCommandInQuotesIfNeeded(const std::string& command) {
 #endif
 }
 
+#ifdef WIN32
+#define popen _popen
+#define pclose _pclose
+#endif
+
+struct CommandResult {
+  int exit_code;
+  std::string output;
+};
+
+CommandResult executeProcess(const std::string& command) {
+  std::array<char, utils::configuration::DEFAULT_BUFFER_SIZE> buffer{};
+
+  FILE* pipe = popen(encapsulateCommandInQuotesIfNeeded(command).c_str(), "r");
+  if (!pipe) {
+    return {1, fmt::format("Failed to open pipe for command: {}", command)};
+  }
+
+  std::ostringstream result;
+  while (fgets(buffer.data(), gsl::narrow<int>(buffer.size()), pipe) != nullptr) {
+    result << buffer.data();
+  }
+
+  int status = pclose(pipe);
+#ifdef WIN32
+  int exit_code = status;
+#else
+  int exit_code = -1;
+  if (WIFEXITED(status)) {
+    exit_code = WEXITSTATUS(status);
+  } else if (WIFSIGNALED(status)) {
+    exit_code = -WTERMSIG(status);
+  }
+#endif
+
+  return {exit_code, result.str()};
+}
+
 }  // namespace
 
 PythonDependencyInstaller::PythonDependencyInstaller(const std::shared_ptr<Configure> &configuration) {
@@ -93,25 +134,31 @@ void PythonDependencyInstaller::createVirtualEnvIfSpecified() const {
   }
   if (!std::filesystem::exists(virtualenv_path_) || std::filesystem::is_empty(virtualenv_path_)) {
     logger_->log_info("Creating python virtual env at: {}", virtualenv_path_.string());
-    auto venv_command = "\"" + python_binary_ + "\" -m venv \"" + virtualenv_path_.string() + "\"";
-    auto return_value = std::system(encapsulateCommandInQuotesIfNeeded(venv_command).c_str());
-    if (return_value != 0) {
-      throw PythonScriptException(fmt::format("The following command creating python virtual env failed: '{}'", venv_command));
+    auto venv_command = "\"" + python_binary_ + "\" -m venv \"" + virtualenv_path_.string() + "\" 2>&1";
+    auto result = executeProcess(venv_command);
+    if (result.exit_code != 0) {
+      logger_->log_error("The following command creating python virtual env failed: '{}'\nSetup process output:\n{}", venv_command, result.output);
+      throw PythonScriptException(fmt::format("The following command creating python virtual env failed: '{}'\nSetup process output:\n{}", venv_command, result.output));
     }
   }
 }
 
 void PythonDependencyInstaller::runInstallCommandInVirtualenv(const std::string& install_command) const {
   std::string command_with_virtualenv;
 #if WIN32
-  command_with_virtualenv.append("\"").append((virtualenv_path_ / "Scripts" / "activate.bat").string()).append("\" && ");
+  command_with_virtualenv.append("\"").append((virtualenv_path_ / "Scripts" / "activate.bat").string()).append("\" 2>&1 && ");
 #else
-  command_with_virtualenv.append(". \"").append((virtualenv_path_ / "bin" / "activate").string()).append("\" && ");
+  command_with_virtualenv.append(". \"").append((virtualenv_path_ / "bin" / "activate").string()).append("\" 2>&1 && ");
 #endif
   command_with_virtualenv.append(install_command);
-  auto return_value = std::system(encapsulateCommandInQuotesIfNeeded(command_with_virtualenv).c_str());
-  if (return_value != 0) {
-    throw PythonScriptException(fmt::format("The following command to install python packages failed: '{}'", command_with_virtualenv));
+  command_with_virtualenv.append(" 2>&1");
+
+  auto result = executeProcess(command_with_virtualenv);
+  if (result.exit_code != 0) {
+    logger_->log_error("Failed to install python packages to virtualenv with command: {}\nInstall process output:\n{}", command_with_virtualenv, result.output);
+    throw PythonScriptException(fmt::format("Failed to install python packages to virtualenv with command: {}\nInstall process output:\n{}", command_with_virtualenv, result.output));
+  } else {
+    logger_->log_info("Python packages installed successfully with command: '{}'.\nInstall process output:\n{}", command_with_virtualenv, result.output);
   }
 }
 

extensions/python/pythonprocessors/nifi_python_processors/utils/dependency_installer.py

@@ -25,7 +25,7 @@ def visit_ClassDef(self, node):
                         for elt in detail.value.elts:
                             # Check if the element is a string constant and add it to the dependencies list
                             if isinstance(elt, ast.Constant):
-                                self.dependencies.append(elt.s)
+                                self.dependencies.append(elt.value)
                         break
                 break
 
@@ -48,7 +48,7 @@ def extract_dependencies(file_path):
     print("Installing dependencies for MiNiFi python processors...")
 
     # --no-cache-dir is used to be in line with NiFi's dependency install behavior
-    command = [sys.executable, "-m", "pip", "install", "--no-cache-dir"]
+    command = [sys.executable, "-m", "pip", "install", "--no-cache-dir", "--progress-bar", "off"]
     dependencies_found = False
     for i in range(1, len(sys.argv)):
         if "requirements.txt" in sys.argv[i]:
@@ -63,4 +63,8 @@ def extract_dependencies(file_path):
                 command += dependencies
 
     if dependencies_found:
-        subprocess.check_call(command)
+        try:
+            subprocess.check_call(command)
+        except subprocess.CalledProcessError as e:
+            print("Error occurred while installing dependencies: {}".format(e))
+            sys.exit(1)