AMBARI-26000: Make Ambari supports Python3 environment (apache#3751)

* AMBARI-26000 support python3 on trunk
* remove yum since it's not supported in py3
* 1.fix shell decode error
2.fix sudo encoding error
3.remove redunant get_encoded_string
* remove duplicate import
* fix bugs
* update ambari metrics to python3
* fix review comment

Co-authored-by: jialiang <[email protected]>
Co-authored-by: jialiang <[email protected]>

Signed-off-by: Mohammad Arshad <[email protected]>

ambari-agent/conf/unix/agent-multiplier.py

@@ -17,7 +17,7 @@
 import sys
 import os
 import re
-from ambari_commons import subprocess32
+import subprocess
 import shutil
 from optparse import OptionParser
 
@@ -76,18 +76,18 @@ def __init__(self, args):
     self.parse_configs()
 
     if len(args) != 2:
-      print "Sample Usage: python agent_multiplier.py [action]\n" \
-            "actions: start|stop|restart|status"
+      print("Sample Usage: python agent_multiplier.py [action]\n" \
+            "actions: start|stop|restart|status")
     self.command = args[1]
 
     # Validate configs
     self.validate()
 
-    print "*** Params ***"
-    print "Start: %d" % self.start
-    print "Num: %d" % self.num
-    print "Prefix: %s" % self.prefix
-    print "Command: %s" % self.command
+    print("*** Params ***")
+    print("Start: %d" % self.start)
+    print("Num: %d" % self.num)
+    print("Prefix: %s" % self.prefix)
+    print("Command: %s" % self.command)
 
     # All hostnames that will be managed by Ambari Agents on this host
     self.hosts = []
@@ -104,7 +104,7 @@ def parse_configs(self):
     Parse the configuration file to set the config params.
     """
     if not os.path.exists(self.CONFIG_FILE):
-      print "Did not find Agent Multiplier config file: %s" % str(self.CONFIG_FILE)
+      print("Did not find Agent Multiplier config file: %s" % str(self.CONFIG_FILE))
       sys.exit(-1)
 
     params = {}
@@ -147,8 +147,8 @@ def validate(self):
       errors.append("Command must be one of %s" % ", ".join(valid_commands))
 
     if len(errors) > 0:
-      print "Error:"
-      print "\n".join(errors)
+      print("Error:")
+      print("\n".join(errors))
       sys.exit(-1)
 
   def bootstrap(self):
@@ -166,28 +166,28 @@ def bootstrap(self):
       host_cache_dir = host_home_dir + self.cache_dir
 
       if self.verbose:
-        print "Analyzing host %s with port %d" % (host_name, host.ping_port)
+        print("Analyzing host %s with port %d" % (host_name, host.ping_port))
 
       for dir in [host_home_dir, host_log_dir, host_config_dir, host_pid_dir, host_prefix, host_cache_dir]:
         if not os.path.isdir(dir):
-          print "Creating dir %s" % (dir)
+          print("Creating dir %s" % (dir))
           os.makedirs(dir)
 
       # Copy config file
       host_config_file = os.path.join(host_config_dir, "ambari-agent.ini")
       if not os.path.isfile(host_config_file):
-        print "Copying config file %s" % str(host_config_file)
+        print("Copying config file %s" % str(host_config_file))
         shutil.copyfile(self.source_config_file, host_config_file)
 
       # Copy version file
       version_file = os.path.join(host_prefix, "version")
       if not os.path.isfile(version_file):
-        print "Copying version file %s" % str(version_file)
+        print("Copying version file %s" % str(version_file))
         shutil.copyfile(self.source_version_file, version_file)
 
       # Copy cache dir content
       if not os.path.isdir(os.path.join(host_cache_dir, "stacks")):
-        print "Copying cache directory content %s" % str(host_cache_dir)
+        print("Copying cache directory content %s" % str(host_cache_dir))
         self.copytree(self.cache_dir, host_cache_dir)
 
       # Create hostname.sh script to use custom FQDN for each agent.
@@ -226,7 +226,7 @@ def create_host_name_script(self, host_name, host_name_script):
                "echo HOSTNAME"
     with open(str(host_name_script), "w+") as f:
       f.writelines(template.replace("HOSTNAME", host_name))
-    subprocess32.call("chmod +x %s" % host_name_script, shell=True)
+    subprocess.call("chmod +x %s" % host_name_script, shell=True)
 
   def change_config(self, config_file, config_dict):
     """
@@ -237,7 +237,7 @@ def change_config(self, config_file, config_dict):
     # TODO, allow appending configs to [AGENT] section.
 
     if not os.path.exists(config_file):
-      print "ERROR. Did not file config file: %s" % config_file
+      print("ERROR. Did not file config file: %s" % config_file)
       return
 
     lines = []
@@ -249,7 +249,7 @@ def change_config(self, config_file, config_dict):
     configs_found = set()
     configs_changed = set()
     for line in lines:
-      for config, value in config_dict.iteritems():
+      for config, value in config_dict.items():
         p = re.compile(config + "\s?=")
         if p.match(line):
           configs_found.add(config)
@@ -264,11 +264,11 @@ def change_config(self, config_file, config_dict):
     # TODO, if can append configs, then this is not needed.
     if len(configs_found) < len(config_dict.keys()):
       missing_configs = set(config_dict.keys()) - configs_found
-      print "ERROR: Did not find all required configs. Missing: %s" % ", ".join(missing_configs)
+      print("ERROR: Did not find all required configs. Missing: %s" % ", ".join(missing_configs))
       sys.exit(-1)
 
     if len(configs_changed) > 0:
-      print "Making changes to file %s" % config_file
+      print("Making changes to file %s" % config_file)
       with open(config_file, "w") as f:
         f.writelines(new_lines)
 
@@ -279,7 +279,7 @@ def modify_etc_hosts_file(self):
     """
     etc_hosts = "/etc/hosts"
     if not os.path.isfile(etc_hosts):
-      print "ERROR. Did not find file %s" % etc_hosts
+      print("ERROR. Did not find file %s" % etc_hosts)
       return
 
     lines = []
@@ -299,7 +299,7 @@ def modify_etc_hosts_file(self):
       new_lines.append(line)
 
     if line_changed:
-      print "Making changes to %s" % etc_hosts
+      print("Making changes to %s" % etc_hosts)
       with open(etc_hosts, "w") as f:
         f.writelines(new_lines)
 
@@ -317,42 +317,42 @@ def run(self):
       self.cmd_status()
 
   def cmd_start(self):
-    print "Starting %d host(s)" % len(self.hosts)
+    print("Starting %d host(s)" % len(self.hosts))
     for host in self.hosts:
       cmd = "ambari-agent start --home %s" % (host.home_dir)
       os.environ['AMBARI_AGENT_CONF_DIR'] = os.path.join(host.home_dir, "etc/ambari-agent/conf")
-      subprocess32.call(cmd, shell=True, env=os.environ)
+      subprocess.call(cmd, shell=True, env=os.environ)
 
   def cmd_stop(self):
-    print "Stopping %d host(s)" % len(self.hosts)
+    print("Stopping %d host(s)" % len(self.hosts))
     for host in self.hosts:
       cmd = "ambari-agent stop --home %s" % (host.home_dir)
       os.environ['AMBARI_AGENT_CONF_DIR'] = os.path.join(host.home_dir, "etc/ambari-agent/conf")
-      subprocess32.call(cmd, shell=True, env=os.environ)
+      subprocess.call(cmd, shell=True, env=os.environ)
 
   def cmd_restart(self):
-    print "Restarting %d host(s)" % len(self.hosts)
+    print("Restarting %d host(s)" % len(self.hosts))
     for host in self.hosts:
       cmd = "ambari-agent restart --home %s" % (host.home_dir)
       os.environ['AMBARI_AGENT_CONF_DIR'] = os.path.join(host.home_dir, "etc/ambari-agent/conf")
-      subprocess32.call(cmd, shell=True, env=os.environ)
+      subprocess.call(cmd, shell=True, env=os.environ)
 
   def cmd_status(self):
-    print "Summary of Agent Status:"
-    print "Total agents: %d\n" % len(self.hosts)
+    print("Summary of Agent Status:")
+    print("Total agents: %d\n" % len(self.hosts))
     (running_hosts, unknown_hosts, stopped_hosts) = self.aggregate_status()
 
-    print "Running agents: %d" % len(running_hosts)
+    print("Running agents: %d" % len(running_hosts))
     if self.verbose and len(running_hosts):
-      print "(%s)\n" % (", ".join(running_hosts))
+      print("(%s)\n" % (", ".join(running_hosts)))
 
-    print "Unknown agents: %d" % len(unknown_hosts)
+    print("Unknown agents: %d" % len(unknown_hosts))
     if self.verbose and len(unknown_hosts):
-      print "(%s)\n" % (", ".join(unknown_hosts))
+      print("(%s)\n" % (", ".join(unknown_hosts)))
 
-    print "Stopped agents: %d" % len(stopped_hosts)
+    print("Stopped agents: %d" % len(stopped_hosts))
     if self.verbose and len(stopped_hosts):
-      print "(%s)\n" % (", ".join(stopped_hosts))
+      print("(%s)\n" % (", ".join(stopped_hosts)))
 
   def aggregate_status(self):
     """

ambari-agent/conf/unix/install-helper.sh

@@ -119,10 +119,10 @@ install_autostart(){
 }
 
 locate_python(){
-  local python_binaries="/usr/bin/python;/usr/bin/python2;/usr/bin/python2.7"
+  local python_binaries="/usr/bin/python;/usr/bin/python3;/usr/bin/python3.9"
 
   echo ${python_binaries}| tr ';' '\n' | while read python_binary; do
-    ${python_binary} -c "import sys ; ver = sys.version_info ; sys.exit(not (ver >= (2,7) and ver<(3,0)))" 1>>${LOG_FILE} 2>/dev/null
+    ${python_binary} -c "import sys ; ver = sys.version_info ; sys.exit(not (ver >= (3,0)))" 1>>${LOG_FILE} 2>/dev/null
 
     if [ $? -eq 0 ]; then
       echo "${python_binary}"

ambari-agent/conf/unix/upgrade_agent_configs.py

@@ -18,7 +18,7 @@
 '''
 
 import os
-import ConfigParser
+import configparser
 
 PROPERTIES_TO_REWRITE = [
   ('heartbeat', 'dirs'),
@@ -33,27 +33,27 @@
 
 if os.path.isfile(CONFIG_FILE_BACKUP):
   if os.path.isfile(CONFIG_FILE):
-    print "Upgrading configs in {0}".format(CONFIG_FILE)
-    print "Values will be updated from {0} except the following list: {1}, {2}".format(CONFIG_FILE_BACKUP, PROPERTIES_TO_REWRITE, SECTIONS_TO_REMOVE)
+    print("Upgrading configs in {0}".format(CONFIG_FILE))
+    print("Values will be updated from {0} except the following list: {1}, {2}".format(CONFIG_FILE_BACKUP, PROPERTIES_TO_REWRITE, SECTIONS_TO_REMOVE))
 
-    agent_config_backup = ConfigParser.ConfigParser()
+    agent_config_backup = configparser.ConfigParser()
     agent_config_backup.read(CONFIG_FILE_BACKUP)
 
-    agent_config = ConfigParser.ConfigParser()
+    agent_config = configparser.ConfigParser()
     agent_config.read(CONFIG_FILE)
 
     for section in agent_config_backup.sections():
       for (property_name, property_val) in agent_config_backup.items(section):
         if section not in SECTIONS_TO_REMOVE and (section, property_name) not in PROPERTIES_TO_REWRITE:
           try:
             agent_config.set(section, property_name, property_val)
-          except ConfigParser.NoSectionError:
+          except configparser.NoSectionError:
             agent_config.add_section(section)
             agent_config.set(section, property_name, property_val)
 
-    with (open(CONFIG_FILE, "wb")) as new_agent_config:
+    with (open(CONFIG_FILE, "w")) as new_agent_config:
       agent_config.write(new_agent_config)
   else:
-    print "Values are not updated, configs {0} is not found".format(CONFIG_FILE)
+    print("Values are not updated, configs {0} is not found".format(CONFIG_FILE))
 else:
-  print "Values are not updated, backup {0} is not found".format(CONFIG_FILE_BACKUP)
+  print("Values are not updated, backup {0} is not found".format(CONFIG_FILE_BACKUP))

ambari-agent/conf/windows/service_wrapper.py

@@ -15,7 +15,7 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 '''
-import ConfigParser
+import configparser
 import os
 import optparse
 import sys
@@ -51,13 +51,13 @@ def parse_options():
       name, value = line[4:].split("=")
       os.environ[name] = value.rstrip()
   # checking env variables, and fallback to working dir if no env var was founded
-  if not os.environ.has_key("AMBARI_AGENT_CONF_DIR"):
+  if "AMBARI_AGENT_CONF_DIR" not in os.environ:
     os.environ["AMBARI_AGENT_CONF_DIR"] = os.getcwd()
-  if not os.environ.has_key("AMBARI_AGENT_LOG_DIR"):
+  if "AMBARI_AGENT_LOG_DIR" not in os.environ:
     os.environ["AMBARI_AGENT_LOG_DIR"] = os.path.join("\\", "var", "log", "ambari-agent")
   if not os.path.exists(os.environ["AMBARI_AGENT_LOG_DIR"]):
     os.makedirs(os.environ["AMBARI_AGENT_LOG_DIR"])
-  if not os.environ.has_key("PYTHON_EXE"):
+  if "PYTHON_EXE" not in os.environ:
     os.environ["PYTHON_EXE"] = find_in_path("python.exe")
 
 
@@ -162,7 +162,7 @@ def svcstatus(options):
   options.exit_message = None
 
   statusStr = AmbariAgentService.QueryStatus()
-  print "Ambari Agent is " + statusStr
+  print("Ambari Agent is " + statusStr)
 
 
 def svcdebug(options):
@@ -201,7 +201,7 @@ def agent_main():
   options.warnings = []
 
   if len(args) == 0:
-    print parser.print_help()
+    print(parser.print_help())
     parser.error("No action entered")
 
   action = args[0]
@@ -212,7 +212,7 @@ def agent_main():
     matches += int(len(args) == args_number_required)
 
   if matches == 0:
-    print parser.print_help()
+    print(parser.print_help())
     possible_args = ' or '.join(str(x) for x in possible_args_numbers)
     parser.error("Invalid number of arguments. Entered: " + str(len(args)) + ", required: " + possible_args)
 
@@ -247,7 +247,7 @@ def agent_main():
       print_warning_msg(e.reason)
 
   if options.exit_message is not None:
-    print options.exit_message
+    print(options.exit_message)
 
 
 if __name__ == '__main__':

ambari-agent/pom.xml

@@ -715,7 +715,7 @@
       <dirsep>\</dirsep>
       <pathsep>;</pathsep>
       <stack.distribution>HDPWIN</stack.distribution>
-      <executable.python>python</executable.python>
+      <executable.python>python3</executable.python>
       <executable.shell>cmd</executable.shell>
       <fileextension.shell>cmd</fileextension.shell>
       <fileextension.dot.shell-default>.cmd</fileextension.dot.shell-default>

ambari-agent/src/main/package/dependencies.properties

@@ -28,5 +28,5 @@
 # Such a format is respected by install_ambari_tarball.py by default, 
 # however should be encouraged manually in pom.xml.
 
-rpm.dependency.list=openssl,\nRequires: rpm-python,\nRequires: zlib,\nRequires: python >= 2.6
+rpm.dependency.list=openssl,\nRequires: python3-rpm,\nRequires: zlib
 deb.dependency.list=openssl, python (>= 2.6)