Merge lp:~maddevelopers/mg5amcnlo/HEPToolsInstallers_dev into lp:~maddevelopers/mg5amcnlo/HEPToolsInstallers

[Olivier Mattelaer]

[Olivier Mattelaer (olivier-mattelaer)]

He he,

I learned that you can merge multiple times the same branch...
I guess I will merge this quickly any objection?

Olivier

PS: I will auto-review the change ;-)

[Valentin Hirschi (valentin-hirschi)]

> He he,
>
> I learned that you can merge multiple times the same branch...
> I guess I will merge this quickly any objection?
>
> Olivier
>
> PS: I will auto-review the change ;-)

There seem to be *a lot* of modifications which are however probably just some re-structuring an reshuffling of the various bits of HEPToolsInstaller.py.
Would you mind describing a bit your changes? Thanks!

[Olivier Mattelaer (olivier-mattelaer)]

Note that revision 109 change a lot but at the same time,
it actually is mainly identation and adding
if "__main__" == __name__
at couple of place.
The reason is that I want to be able to do
import HEPToolsInstaller
and not run the code.
So this is in principle harmless.

[Valentin Hirschi (valentin-hirschi)]

Ok, that's a very positive change. I like the idea of being able to use this tool both as a standalone and a python module. Thanks.

review: Approve

=== modified file 'HEPToolInstaller.py'
--- HEPToolInstaller.py	2018-04-13 09:53:15 +0000
+++ HEPToolInstaller.py	2019-02-05 10:15:18 +0000
@@ -263,9 +263,6 @@
 # List of tools for which the tarballs have been specified
 _tarballs_specified = []
 
-if len(sys.argv)>1 and sys.argv[1].lower() not in _HepTools.keys():
-    logger.warning("HEPToolInstaller does not support the installation of %s" , sys.argv[1])
-    sys.argv[1] = 'help'
 
 logger_level = [a[10:] for a in sys.argv if '--logging=' in a]
 if not logger_level:
@@ -276,15 +273,22 @@
       logger_level = int(logger_level)
    else:
       try:
-        logger_level = getattr(logging,logger_level)
+         logger_level = getattr(logging,logger_level)
       except AttributeError:
-        print "Logging level %s not reckognized."%logger_level
-        sys.exit(9)
-        
-logging.basicConfig(format='%(message)s',level=logger_level)
-
-if len(sys.argv)<2 or sys.argv[1]=='help': 
-    print """
+         print "Logging level %s not reckognized."%logger_level
+         sys.exit(9)
+logging.basicConfig(format='%(message)s',level=logger_level)         
+
+if "__main__" == __name__:
+
+   if len(sys.argv)>1 and sys.argv[1].lower() not in _HepTools.keys():
+      logger.warning("HEPToolInstaller does not support the installation of %s" , sys.argv[1])
+      sys.argv[1] = 'help'
+
+
+
+   if len(sys.argv)<2 or sys.argv[1]=='help': 
+      print """
 ./HEPToolInstaller <target> <options>"
      Possible values and meaning for the various <options> are:
            
@@ -312,16 +316,17 @@
 Example of usage:
     ./HEPToolInstaller.py pythia8 --prefix=~/MyTools --with_lhapdf6=OFF --pythia8_tarball=~/MyTarball.tar.gz
 """%(', '.join(_HepTools.keys()),', '.join(_non_installable_dependencies.keys()))
-    sys.exit(9)
-
-target_tool = sys.argv[1].lower()
+      sys.exit(9)
+
+   target_tool = sys.argv[1].lower()
+
 
 # Make sure to set the install location of all other tools than the target to 'default'. Meaning that they
 # will be installed if not found.
-for tool in _HepTools:
-    if tool==target_tool:
-        continue
-    _HepTools[tool]['install_path']='default'
+   for tool in _HepTools:
+      if tool==target_tool:
+         continue
+      _HepTools[tool]['install_path']='default'
 
 
 # For compatibility reason, map some names
@@ -336,9 +341,10 @@
                     ['--veto_%s'%tool for tool in _HepTools.keys()+_non_installable_dependencies.keys()]+\
                     ['--logging','--source', '--version']
 
-# Recall input command for logfiles
-logger.debug("Installer HEPToolInstaller.py is now processing the following command:")
-logger.debug("   %s"%' '.join(sys.argv))
+if '__main__' == __name__:
+   # Recall input command for logfiles
+   logger.debug("Installer HEPToolInstaller.py is now processing the following command:")
+   logger.debug("   %s"%' '.join(sys.argv))
 
 def with_option_parser(with_option):
     if with_option=='ON':
@@ -363,140 +369,141 @@
                     tool_options['tarball'] = tarball_specifier
                     break
 
-_version = None
-# Now parse the options
-for user_option in sys.argv[2:]:
-    try:
-        option, value = user_option.split('=')
-    except:
-        option = user_option
-        value  = None
-    if option not in available_options:
-        logger.error("Option '%s' not reckognized." , option)
-        sys.exit(9)
-    if option=='--force':
-        _overwrite_existing_installation = True
-    if option=='--update':
-        supported_tools = ['madanalysis5']
-        if target_tool not in supported_tools:
-            logger.error("HEPToolsInstaller.py supports the option '--update' only for the following tools:")
-            logger.error("  %s"%supported_tools)
-            sys.exit(9)
-        _keep_existing_installation = True
-    if option=='--keep_source':
-        _keep_source = True
-    if option=='--prefix':
-        if not os.path.isdir(value):
-            logger.info("Creating root directory '%s'.", os.path.abspath(value))
-            os.mkdir(os.path.abspath(value))
-        _prefix = os.path.abspath(value)
-    elif option=='--fortran_compiler':
-        _gfortran = value
-    elif option=='--cpp_compiler':
-        _cpp = value
-    elif option=='--no_MA5_further_install':
-        _HepTools['madanalysis5']['MA5_further_install']=False
-    elif option=='--no_root_in_MA5':
-        _HepTools['madanalysis5']['use_root_if_available']=False        
-    elif option=='--cpp_standard_lib':
-        if value not in ['-lc++','-lstdc++']:
-            logger.error( "ERROR: Option '--cpp_standard_lib' must be either '-lc++' or '-libstdc++', not '%s'.", value)
-            sys.exit(9)
-        _cpp_standard_lib = value
-    elif option=='--mg5_path':
-        _mg5_path = value
-        # Try to gather MG5_version
+if '__main__' == __name__:
+    _version = None
+    # Now parse the options
+    for user_option in sys.argv[2:]:
         try:
-            _mg5_version = None
-            for line in open(pjoin(_mg5_path,'VERSION'),'r').read().split('\n'):
-                if line.startswith('version ='):
-                    _mg5_version = LooseVersion(line[9:].strip())
-                    break
+            option, value = user_option.split('=')
         except:
-            raise
-            _mg5_version = None
-    elif option.startswith('--with_'):
-        dependency_name = _dependency_names_map[option[7:]] if option[7:] in _dependency_names_map else option[7:]
-        value           = with_option_parser(value)
-        if dependency_name in _HepTools:
-            _HepTools[dependency_name]['install_path'] = value
-        else:
-            # Special treatment for dependencies that cannot be directly installed in this installer
-            _non_installable_dependencies[dependency_name]['path']   = value if (value not in ['Default',False, None]) else None 
-            _non_installable_dependencies[dependency_name]['active'] = (value!=False)
-    elif option.startswith('--veto_'):
-        dependency_name = option[7:]
-        if dependency_name in _HepTools:
-            _HepTools[dependency_name]['install_path'] = None 
-        else:
-            # Special treatment for dependencies that cannot be directly installed in this installer
-            _non_installable_dependencies[dependency_name]['path']   = None 
-            _non_installable_dependencies[dependency_name]['active'] = False
-
-    elif option.endswith('_tarball'):
-        access_mode = 'online' if '//' in value else 'local'
-        if access_mode=='local':
-            value = os.path.abspath(value)
-        _HepTools[option[2:-8]]['tarball'] = [access_mode, value]
-        # Flag the tarball of this tool as specified
-        _tarballs_specified.append(option[2:-8])       
-    elif option.startswith('--version='):
-       _version = value
-
-
-# Adapt paths according to MG5 version specified
-if _mg5_version:
-    adapt_tarball_paths_according_to_MG5_version(_mg5_version)
-
-# force code version if request by the user
-if _version:
-   if 'format_version' in _HepTools[target_tool]:
-      _version = _HepTools[target_tool]['format_version'](_version)
-   if '%(version)s' in _HepTools[target_tool]['tarball'][1]:
-      _HepTools[target_tool]['tarball'][1]=_HepTools[target_tool]['tarball'][1]%{'version':_version}
-   else:
-      raise Exception, 'fail to specify version for this tools.'
-
-# Apply substitutions if necessary:
-
-for tool in _HepTools:
-    if _HepTools[tool]['install_path']:
-       _HepTools[tool]['install_path']=_HepTools[tool]['install_path']%\
-                                       {'prefix':_prefix, 'mg5_path': '' if _mg5_path is None else _mg5_path }
-    if _HepTools[tool]['tarball'][0]=='local':
-       _HepTools[tool]['tarball'][1]=_HepTools[tool]['tarball'][1]%\
-                                       {'prefix':_prefix, 'mg5_path': '' if _mg5_path is None else _mg5_path }
-    
-    if _HepTools[tool]['tarball'][0]=='online':
-       version = _HepTools[tool]['version']
-       if 'format_version' in _HepTools[tool]:
-          version = _HepTools[tool]['format_version'](version)
-       _HepTools[tool]['tarball'][1]=_HepTools[tool]['tarball'][1]%{'version':version}
-    
-    new_libs = []
-    for lib in _HepTools[tool]['libraries']:
-        for libext in _lib_extensions:
-            if lib%{'libextension':libext} not in new_libs:
-                new_libs.append(lib%{'libextension':libext})
-    _HepTools[tool]['libraries'] = new_libs
-
-
-# Make sure it is not already installed, but if the directory is empty, then remove it
-if os.path.isdir(pjoin(_prefix,target_tool)):
-    if os.listdir(pjoin(_prefix,target_tool)) in [[],['%s_install.log'%target_tool]]:
-        shutil.rmtree(pjoin(_prefix,target_tool))
-        _keep_existing_installation = False
+            option = user_option
+            value  = None
+        if option not in available_options:
+            logger.error("Option '%s' not reckognized." , option)
+            sys.exit(9)
+        if option=='--force':
+            _overwrite_existing_installation = True
+        if option=='--update':
+            supported_tools = ['madanalysis5']
+            if target_tool not in supported_tools:
+                logger.error("HEPToolsInstaller.py supports the option '--update' only for the following tools:")
+                logger.error("  %s"%supported_tools)
+                sys.exit(9)
+            _keep_existing_installation = True
+        if option=='--keep_source':
+            _keep_source = True
+        if option=='--prefix':
+            if not os.path.isdir(value):
+                logger.info("Creating root directory '%s'.", os.path.abspath(value))
+                os.mkdir(os.path.abspath(value))
+            _prefix = os.path.abspath(value)
+        elif option=='--fortran_compiler':
+            _gfortran = value
+        elif option=='--cpp_compiler':
+            _cpp = value
+        elif option=='--no_MA5_further_install':
+            _HepTools['madanalysis5']['MA5_further_install']=False
+        elif option=='--no_root_in_MA5':
+            _HepTools['madanalysis5']['use_root_if_available']=False        
+        elif option=='--cpp_standard_lib':
+            if value not in ['-lc++','-lstdc++']:
+                logger.error( "ERROR: Option '--cpp_standard_lib' must be either '-lc++' or '-libstdc++', not '%s'.", value)
+                sys.exit(9)
+            _cpp_standard_lib = value
+        elif option=='--mg5_path':
+            _mg5_path = value
+            # Try to gather MG5_version
+            try:
+                _mg5_version = None
+                for line in open(pjoin(_mg5_path,'VERSION'),'r').read().split('\n'):
+                    if line.startswith('version ='):
+                        _mg5_version = LooseVersion(line[9:].strip())
+                        break
+            except:
+                raise
+                _mg5_version = None
+        elif option.startswith('--with_'):
+            dependency_name = _dependency_names_map[option[7:]] if option[7:] in _dependency_names_map else option[7:]
+            value           = with_option_parser(value)
+            if dependency_name in _HepTools:
+                _HepTools[dependency_name]['install_path'] = value
+            else:
+                # Special treatment for dependencies that cannot be directly installed in this installer
+                _non_installable_dependencies[dependency_name]['path']   = value if (value not in ['Default',False, None]) else None 
+                _non_installable_dependencies[dependency_name]['active'] = (value!=False)
+        elif option.startswith('--veto_'):
+            dependency_name = option[7:]
+            if dependency_name in _HepTools:
+                _HepTools[dependency_name]['install_path'] = None 
+            else:
+                # Special treatment for dependencies that cannot be directly installed in this installer
+                _non_installable_dependencies[dependency_name]['path']   = None 
+                _non_installable_dependencies[dependency_name]['active'] = False
+
+        elif option.endswith('_tarball'):
+            access_mode = 'online' if '//' in value else 'local'
+            if access_mode=='local':
+                value = os.path.abspath(value)
+            _HepTools[option[2:-8]]['tarball'] = [access_mode, value]
+            # Flag the tarball of this tool as specified
+            _tarballs_specified.append(option[2:-8])       
+        elif option.startswith('--version='):
+           _version = value
+
+
+    # Adapt paths according to MG5 version specified
+    if _mg5_version:
+        adapt_tarball_paths_according_to_MG5_version(_mg5_version)
+
+    # force code version if request by the user
+    if _version:
+       if 'format_version' in _HepTools[target_tool]:
+          _version = _HepTools[target_tool]['format_version'](_version)
+       if '%(version)s' in _HepTools[target_tool]['tarball'][1]:
+          _HepTools[target_tool]['tarball'][1]=_HepTools[target_tool]['tarball'][1]%{'version':_version}
+       else:
+          raise Exception, 'fail to specify version for this tools.'
+
+    # Apply substitutions if necessary:
+
+    for tool in _HepTools:
+        if _HepTools[tool]['install_path']:
+           _HepTools[tool]['install_path']=_HepTools[tool]['install_path']%\
+                                           {'prefix':_prefix, 'mg5_path': '' if _mg5_path is None else _mg5_path }
+        if _HepTools[tool]['tarball'][0]=='local':
+           _HepTools[tool]['tarball'][1]=_HepTools[tool]['tarball'][1]%\
+                                           {'prefix':_prefix, 'mg5_path': '' if _mg5_path is None else _mg5_path }
+
+        if _HepTools[tool]['tarball'][0]=='online':
+           version = _HepTools[tool]['version']
+           if 'format_version' in _HepTools[tool]:
+              version = _HepTools[tool]['format_version'](version)
+           _HepTools[tool]['tarball'][1]=_HepTools[tool]['tarball'][1]%{'version':version}
+
+        new_libs = []
+        for lib in _HepTools[tool]['libraries']:
+            for libext in _lib_extensions:
+                if lib%{'libextension':libext} not in new_libs:
+                    new_libs.append(lib%{'libextension':libext})
+        _HepTools[tool]['libraries'] = new_libs
+
+
+    # Make sure it is not already installed, but if the directory is empty, then remove it
+    if os.path.isdir(pjoin(_prefix,target_tool)):
+        if os.listdir(pjoin(_prefix,target_tool)) in [[],['%s_install.log'%target_tool]]:
+            shutil.rmtree(pjoin(_prefix,target_tool))
+            _keep_existing_installation = False
+        else:
+            if not _keep_existing_installation:
+               if not _overwrite_existing_installation:
+                  logger.warning( "The specified path '%s' already contains an installation of tool '%s'.", _prefix, target_tool)
+                  logger.warning( "Rerun the HEPToolInstaller.py script again with the option '--force' if you want to overwrite it.")
+                  sys.exit(66)
+               else:
+                  logger.info("Removing existing installation of tool '%s' in '%s'.", target_tool, _prefix)
+                  shutil.rmtree(pjoin(_prefix,target_tool))
     else:
-        if not _keep_existing_installation:
-           if not _overwrite_existing_installation:
-              logger.warning( "The specified path '%s' already contains an installation of tool '%s'.", _prefix, target_tool)
-              logger.warning( "Rerun the HEPToolInstaller.py script again with the option '--force' if you want to overwrite it.")
-              sys.exit(66)
-           else:
-              logger.info("Removing existing installation of tool '%s' in '%s'.", target_tool, _prefix)
-              shutil.rmtree(pjoin(_prefix,target_tool))
-else:
-   _keep_existing_installation = False
+       _keep_existing_installation = False
 
 # TMP_directory (designed to work as with statement) and go to it
 class TMP_directory(object):
@@ -681,11 +688,17 @@
 def install_lhapdf6(tmp_path):
     """Installation operations for lhapdf6"""
     lhapdf6_log = open(pjoin(_HepTools['lhapdf6']['install_path'],"lhapdf6_install.log"), "w")
+    cxx_flags = '-O'
+    for flag in ['-static-libstdc++']:
+       if test_cpp_compiler([flag]):
+            cxx_flags = flag
+
     subprocess.call([pjoin(_installers_path,'installLHAPDF6.sh'),
                      _HepTools['boost']['install_path'],
                      _HepTools['lhapdf6']['install_path'],
                      _HepTools['lhapdf6']['version'],
-                     _HepTools['lhapdf6']['tarball'][1]],
+                     _HepTools['lhapdf6']['tarball'][1],
+                     cxx_flags],
                     stdout=lhapdf6_log,
                     stderr=lhapdf6_log)
     lhapdf6_log.close()
@@ -765,7 +778,7 @@
 
     # Run the installation script
     mg5amc_py8_interface_log = open(pjoin(_HepTools['mg5amc_py8_interface']['install_path'],"mg5amc_py8_interface_install.log"), "w")
-    subprocess.call([pjoin(sys.executable, _HepTools['mg5amc_py8_interface']['install_path'],'compile.py')]+options, 
+    subprocess.call([sys.executable, pjoin(_HepTools['mg5amc_py8_interface']['install_path'],'compile.py')]+options, 
                     stdout=mg5amc_py8_interface_log,
                     stderr=mg5amc_py8_interface_log)
     mg5amc_py8_interface_log.close()
@@ -1307,34 +1320,36 @@
                 logger.info("    > Now aborting installation of tool '%s'."%target_tool)
                 sys.exit(9)
 
-_environ = dict(os.environ)
-try:   
-    os.environ["CXX"]     = _cpp
-    os.environ["FC"]      = _gfortran
-    # Also add the bin directory of the HEPTools install location, as we might need some of the executables installed there like cmake
-    os.environ["PATH"]    = pjoin(_prefix,'bin')+os.pathsep+os.environ["PATH"]
-    install_with_dependencies(target_tool,is_main_target=True)
-except ZeroDivisionError as e:
+
+if "__main__" == __name__:
+    _environ = dict(os.environ)
+    try:   
+        os.environ["CXX"]     = _cpp
+        os.environ["FC"]      = _gfortran
+        # Also add the bin directory of the HEPTools install location, as we might need some of the executables installed there like cmake
+        os.environ["PATH"]    = pjoin(_prefix,'bin')+os.pathsep+os.environ["PATH"]
+        install_with_dependencies(target_tool,is_main_target=True)
+    except ZeroDivisionError as e:
+        os.environ.clear()
+        os.environ.update(_environ)
+        logger.critical("The following error occured during the installation of '%s' (and its dependencies):\n%s"%(target_tool,repr(e)))
+        sys.exit(9)
+
     os.environ.clear()
     os.environ.update(_environ)
-    logger.critical("The following error occured during the installation of '%s' (and its dependencies):\n%s"%(target_tool,repr(e)))
-    sys.exit(9)
-
-os.environ.clear()
-os.environ.update(_environ)
-
-if check_successful_installation(target_tool):
-    # Successful installation, now copy the installed components directly under HEPTools
-    finalize_installation(target_tool)
-    logger.info("Successful installation of '%s' in '%s'."%(target_tool,_prefix))
-    logger.debug("See installation log at '%s'."%pjoin(_HepTools[target_tool]['install_path'],'%s_install.log'%target_tool))
-    sys.exit(0)
-else:
-    logger.warning("A problem occured during the installation of '%s'.", target_tool)
-    try:
-      logger.warning("Content of the installation log file '%s':\n\n%s"%(\
-        pjoin(_HepTools[target_tool]['install_path'],'%s_install.log'%target_tool),
-        open(pjoin(_HepTools[target_tool]['install_path'],'%s_install.log'%target_tool),'r').read()))
-    except IOError:
-      logger.warning("No additional information on the installation problem available.")
-    sys.exit(9)
+
+    if check_successful_installation(target_tool):
+        # Successful installation, now copy the installed components directly under HEPTools
+        finalize_installation(target_tool)
+        logger.info("Successful installation of '%s' in '%s'."%(target_tool,_prefix))
+        logger.debug("See installation log at '%s'."%pjoin(_HepTools[target_tool]['install_path'],'%s_install.log'%target_tool))
+        sys.exit(0)
+    else:
+        logger.warning("A problem occured during the installation of '%s'.", target_tool)
+        try:
+          logger.warning("Content of the installation log file '%s':\n\n%s"%(\
+            pjoin(_HepTools[target_tool]['install_path'],'%s_install.log'%target_tool),
+            open(pjoin(_HepTools[target_tool]['install_path'],'%s_install.log'%target_tool),'r').read()))
+        except IOError:
+          logger.warning("No additional information on the installation problem available.")
+        sys.exit(9)

=== modified file 'installLHAPDF6.sh'
--- installLHAPDF6.sh	2016-04-20 19:10:54 +0000
+++ installLHAPDF6.sh	2019-02-05 10:15:18 +0000
@@ -10,6 +10,7 @@
   INSTALLD="$2"
   VERSION="$3"
   TARBALL="$4"
+  CXXFLAGS="$5"
   LOCAL=$INSTALLD
 
   # set SLC5 platform name:
@@ -32,7 +33,7 @@
   cd LHAPDF-${VERSION}/
 
   echo " Configure LHAPDF"
-  LIBRARY_PATH=$LD_LIBRARY_PATH ./configure CXXFLAGS="-static-libstdc++" --prefix=$LOCAL --bindir=$LOCAL/bin --datadir=$LOCAL/share --libdir=$LOCAL/lib --with-boost=$BOOST --enable-static
+  LIBRARY_PATH=$LD_LIBRARY_PATH ./configure CXXFLAGS="$CXXFLAGS" --prefix=$LOCAL --bindir=$LOCAL/bin --datadir=$LOCAL/share --libdir=$LOCAL/lib --with-boost=$BOOST --enable-static
 
   echo " Compile LHAPDF6"
   LIBRARY_PATH=$LD_LIBRARY_PATH make