Merge lp:~maddevelopers/mg5amcnlo/HEPToolsInstallers_dev into lp:~maddevelopers/mg5amcnlo/HEPToolsInstallers

Proposed by Olivier Mattelaer on 2019-02-05
Status: Merged
Merged at revision: 109
Proposed branch: lp:~maddevelopers/mg5amcnlo/HEPToolsInstallers_dev
Merge into: lp:~maddevelopers/mg5amcnlo/HEPToolsInstallers
Diff against target: 471 lines (+200/-184)
2 files modified
HEPToolInstaller.py (+198/-183)
installLHAPDF6.sh (+2/-1)
To merge this branch: bzr merge lp:~maddevelopers/mg5amcnlo/HEPToolsInstallers_dev
Reviewer Review Type Date Requested Status
Valentin Hirschi 2019-02-05 Approve on 2019-02-05
Review via email: mp+362720@code.launchpad.net
To post a comment you must log in.

He he,

I learned that you can merge multiple times the same branch...
I guess I will merge this quickly any objection?

Olivier

PS: I will auto-review the change ;-)

Valentin Hirschi (valentin-hirschi) wrote :

> He he,
>
> I learned that you can merge multiple times the same branch...
> I guess I will merge this quickly any objection?
>
> Olivier
>
> PS: I will auto-review the change ;-)

There seem to be *a lot* of modifications which are however probably just some re-structuring an reshuffling of the various bits of HEPToolsInstaller.py.
Would you mind describing a bit your changes? Thanks!

Note that revision 109 change a lot but at the same time,
it actually is mainly identation and adding
if "__main__" == __name__
at couple of place.
The reason is that I want to be able to do
import HEPToolsInstaller
and not run the code.
So this is in principle harmless.

Valentin Hirschi (valentin-hirschi) wrote :

Ok, that's a very positive change. I like the idea of being able to use this tool both as a standalone and a python module. Thanks.

review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'HEPToolInstaller.py'
2--- HEPToolInstaller.py 2018-04-13 09:53:15 +0000
3+++ HEPToolInstaller.py 2019-02-05 10:15:18 +0000
4@@ -263,9 +263,6 @@
5 # List of tools for which the tarballs have been specified
6 _tarballs_specified = []
7
8-if len(sys.argv)>1 and sys.argv[1].lower() not in _HepTools.keys():
9- logger.warning("HEPToolInstaller does not support the installation of %s" , sys.argv[1])
10- sys.argv[1] = 'help'
11
12 logger_level = [a[10:] for a in sys.argv if '--logging=' in a]
13 if not logger_level:
14@@ -276,15 +273,22 @@
15 logger_level = int(logger_level)
16 else:
17 try:
18- logger_level = getattr(logging,logger_level)
19+ logger_level = getattr(logging,logger_level)
20 except AttributeError:
21- print "Logging level %s not reckognized."%logger_level
22- sys.exit(9)
23-
24-logging.basicConfig(format='%(message)s',level=logger_level)
25-
26-if len(sys.argv)<2 or sys.argv[1]=='help':
27- print """
28+ print "Logging level %s not reckognized."%logger_level
29+ sys.exit(9)
30+logging.basicConfig(format='%(message)s',level=logger_level)
31+
32+if "__main__" == __name__:
33+
34+ if len(sys.argv)>1 and sys.argv[1].lower() not in _HepTools.keys():
35+ logger.warning("HEPToolInstaller does not support the installation of %s" , sys.argv[1])
36+ sys.argv[1] = 'help'
37+
38+
39+
40+ if len(sys.argv)<2 or sys.argv[1]=='help':
41+ print """
42 ./HEPToolInstaller <target> <options>"
43 Possible values and meaning for the various <options> are:
44
45@@ -312,16 +316,17 @@
46 Example of usage:
47 ./HEPToolInstaller.py pythia8 --prefix=~/MyTools --with_lhapdf6=OFF --pythia8_tarball=~/MyTarball.tar.gz
48 """%(', '.join(_HepTools.keys()),', '.join(_non_installable_dependencies.keys()))
49- sys.exit(9)
50-
51-target_tool = sys.argv[1].lower()
52+ sys.exit(9)
53+
54+ target_tool = sys.argv[1].lower()
55+
56
57 # Make sure to set the install location of all other tools than the target to 'default'. Meaning that they
58 # will be installed if not found.
59-for tool in _HepTools:
60- if tool==target_tool:
61- continue
62- _HepTools[tool]['install_path']='default'
63+ for tool in _HepTools:
64+ if tool==target_tool:
65+ continue
66+ _HepTools[tool]['install_path']='default'
67
68
69 # For compatibility reason, map some names
70@@ -336,9 +341,10 @@
71 ['--veto_%s'%tool for tool in _HepTools.keys()+_non_installable_dependencies.keys()]+\
72 ['--logging','--source', '--version']
73
74-# Recall input command for logfiles
75-logger.debug("Installer HEPToolInstaller.py is now processing the following command:")
76-logger.debug(" %s"%' '.join(sys.argv))
77+if '__main__' == __name__:
78+ # Recall input command for logfiles
79+ logger.debug("Installer HEPToolInstaller.py is now processing the following command:")
80+ logger.debug(" %s"%' '.join(sys.argv))
81
82 def with_option_parser(with_option):
83 if with_option=='ON':
84@@ -363,140 +369,141 @@
85 tool_options['tarball'] = tarball_specifier
86 break
87
88-_version = None
89-# Now parse the options
90-for user_option in sys.argv[2:]:
91- try:
92- option, value = user_option.split('=')
93- except:
94- option = user_option
95- value = None
96- if option not in available_options:
97- logger.error("Option '%s' not reckognized." , option)
98- sys.exit(9)
99- if option=='--force':
100- _overwrite_existing_installation = True
101- if option=='--update':
102- supported_tools = ['madanalysis5']
103- if target_tool not in supported_tools:
104- logger.error("HEPToolsInstaller.py supports the option '--update' only for the following tools:")
105- logger.error(" %s"%supported_tools)
106- sys.exit(9)
107- _keep_existing_installation = True
108- if option=='--keep_source':
109- _keep_source = True
110- if option=='--prefix':
111- if not os.path.isdir(value):
112- logger.info("Creating root directory '%s'.", os.path.abspath(value))
113- os.mkdir(os.path.abspath(value))
114- _prefix = os.path.abspath(value)
115- elif option=='--fortran_compiler':
116- _gfortran = value
117- elif option=='--cpp_compiler':
118- _cpp = value
119- elif option=='--no_MA5_further_install':
120- _HepTools['madanalysis5']['MA5_further_install']=False
121- elif option=='--no_root_in_MA5':
122- _HepTools['madanalysis5']['use_root_if_available']=False
123- elif option=='--cpp_standard_lib':
124- if value not in ['-lc++','-lstdc++']:
125- logger.error( "ERROR: Option '--cpp_standard_lib' must be either '-lc++' or '-libstdc++', not '%s'.", value)
126- sys.exit(9)
127- _cpp_standard_lib = value
128- elif option=='--mg5_path':
129- _mg5_path = value
130- # Try to gather MG5_version
131+if '__main__' == __name__:
132+ _version = None
133+ # Now parse the options
134+ for user_option in sys.argv[2:]:
135 try:
136- _mg5_version = None
137- for line in open(pjoin(_mg5_path,'VERSION'),'r').read().split('\n'):
138- if line.startswith('version ='):
139- _mg5_version = LooseVersion(line[9:].strip())
140- break
141+ option, value = user_option.split('=')
142 except:
143- raise
144- _mg5_version = None
145- elif option.startswith('--with_'):
146- dependency_name = _dependency_names_map[option[7:]] if option[7:] in _dependency_names_map else option[7:]
147- value = with_option_parser(value)
148- if dependency_name in _HepTools:
149- _HepTools[dependency_name]['install_path'] = value
150- else:
151- # Special treatment for dependencies that cannot be directly installed in this installer
152- _non_installable_dependencies[dependency_name]['path'] = value if (value not in ['Default',False, None]) else None
153- _non_installable_dependencies[dependency_name]['active'] = (value!=False)
154- elif option.startswith('--veto_'):
155- dependency_name = option[7:]
156- if dependency_name in _HepTools:
157- _HepTools[dependency_name]['install_path'] = None
158- else:
159- # Special treatment for dependencies that cannot be directly installed in this installer
160- _non_installable_dependencies[dependency_name]['path'] = None
161- _non_installable_dependencies[dependency_name]['active'] = False
162-
163- elif option.endswith('_tarball'):
164- access_mode = 'online' if '//' in value else 'local'
165- if access_mode=='local':
166- value = os.path.abspath(value)
167- _HepTools[option[2:-8]]['tarball'] = [access_mode, value]
168- # Flag the tarball of this tool as specified
169- _tarballs_specified.append(option[2:-8])
170- elif option.startswith('--version='):
171- _version = value
172-
173-
174-# Adapt paths according to MG5 version specified
175-if _mg5_version:
176- adapt_tarball_paths_according_to_MG5_version(_mg5_version)
177-
178-# force code version if request by the user
179-if _version:
180- if 'format_version' in _HepTools[target_tool]:
181- _version = _HepTools[target_tool]['format_version'](_version)
182- if '%(version)s' in _HepTools[target_tool]['tarball'][1]:
183- _HepTools[target_tool]['tarball'][1]=_HepTools[target_tool]['tarball'][1]%{'version':_version}
184- else:
185- raise Exception, 'fail to specify version for this tools.'
186-
187-# Apply substitutions if necessary:
188-
189-for tool in _HepTools:
190- if _HepTools[tool]['install_path']:
191- _HepTools[tool]['install_path']=_HepTools[tool]['install_path']%\
192- {'prefix':_prefix, 'mg5_path': '' if _mg5_path is None else _mg5_path }
193- if _HepTools[tool]['tarball'][0]=='local':
194- _HepTools[tool]['tarball'][1]=_HepTools[tool]['tarball'][1]%\
195- {'prefix':_prefix, 'mg5_path': '' if _mg5_path is None else _mg5_path }
196-
197- if _HepTools[tool]['tarball'][0]=='online':
198- version = _HepTools[tool]['version']
199- if 'format_version' in _HepTools[tool]:
200- version = _HepTools[tool]['format_version'](version)
201- _HepTools[tool]['tarball'][1]=_HepTools[tool]['tarball'][1]%{'version':version}
202-
203- new_libs = []
204- for lib in _HepTools[tool]['libraries']:
205- for libext in _lib_extensions:
206- if lib%{'libextension':libext} not in new_libs:
207- new_libs.append(lib%{'libextension':libext})
208- _HepTools[tool]['libraries'] = new_libs
209-
210-
211-# Make sure it is not already installed, but if the directory is empty, then remove it
212-if os.path.isdir(pjoin(_prefix,target_tool)):
213- if os.listdir(pjoin(_prefix,target_tool)) in [[],['%s_install.log'%target_tool]]:
214- shutil.rmtree(pjoin(_prefix,target_tool))
215- _keep_existing_installation = False
216+ option = user_option
217+ value = None
218+ if option not in available_options:
219+ logger.error("Option '%s' not reckognized." , option)
220+ sys.exit(9)
221+ if option=='--force':
222+ _overwrite_existing_installation = True
223+ if option=='--update':
224+ supported_tools = ['madanalysis5']
225+ if target_tool not in supported_tools:
226+ logger.error("HEPToolsInstaller.py supports the option '--update' only for the following tools:")
227+ logger.error(" %s"%supported_tools)
228+ sys.exit(9)
229+ _keep_existing_installation = True
230+ if option=='--keep_source':
231+ _keep_source = True
232+ if option=='--prefix':
233+ if not os.path.isdir(value):
234+ logger.info("Creating root directory '%s'.", os.path.abspath(value))
235+ os.mkdir(os.path.abspath(value))
236+ _prefix = os.path.abspath(value)
237+ elif option=='--fortran_compiler':
238+ _gfortran = value
239+ elif option=='--cpp_compiler':
240+ _cpp = value
241+ elif option=='--no_MA5_further_install':
242+ _HepTools['madanalysis5']['MA5_further_install']=False
243+ elif option=='--no_root_in_MA5':
244+ _HepTools['madanalysis5']['use_root_if_available']=False
245+ elif option=='--cpp_standard_lib':
246+ if value not in ['-lc++','-lstdc++']:
247+ logger.error( "ERROR: Option '--cpp_standard_lib' must be either '-lc++' or '-libstdc++', not '%s'.", value)
248+ sys.exit(9)
249+ _cpp_standard_lib = value
250+ elif option=='--mg5_path':
251+ _mg5_path = value
252+ # Try to gather MG5_version
253+ try:
254+ _mg5_version = None
255+ for line in open(pjoin(_mg5_path,'VERSION'),'r').read().split('\n'):
256+ if line.startswith('version ='):
257+ _mg5_version = LooseVersion(line[9:].strip())
258+ break
259+ except:
260+ raise
261+ _mg5_version = None
262+ elif option.startswith('--with_'):
263+ dependency_name = _dependency_names_map[option[7:]] if option[7:] in _dependency_names_map else option[7:]
264+ value = with_option_parser(value)
265+ if dependency_name in _HepTools:
266+ _HepTools[dependency_name]['install_path'] = value
267+ else:
268+ # Special treatment for dependencies that cannot be directly installed in this installer
269+ _non_installable_dependencies[dependency_name]['path'] = value if (value not in ['Default',False, None]) else None
270+ _non_installable_dependencies[dependency_name]['active'] = (value!=False)
271+ elif option.startswith('--veto_'):
272+ dependency_name = option[7:]
273+ if dependency_name in _HepTools:
274+ _HepTools[dependency_name]['install_path'] = None
275+ else:
276+ # Special treatment for dependencies that cannot be directly installed in this installer
277+ _non_installable_dependencies[dependency_name]['path'] = None
278+ _non_installable_dependencies[dependency_name]['active'] = False
279+
280+ elif option.endswith('_tarball'):
281+ access_mode = 'online' if '//' in value else 'local'
282+ if access_mode=='local':
283+ value = os.path.abspath(value)
284+ _HepTools[option[2:-8]]['tarball'] = [access_mode, value]
285+ # Flag the tarball of this tool as specified
286+ _tarballs_specified.append(option[2:-8])
287+ elif option.startswith('--version='):
288+ _version = value
289+
290+
291+ # Adapt paths according to MG5 version specified
292+ if _mg5_version:
293+ adapt_tarball_paths_according_to_MG5_version(_mg5_version)
294+
295+ # force code version if request by the user
296+ if _version:
297+ if 'format_version' in _HepTools[target_tool]:
298+ _version = _HepTools[target_tool]['format_version'](_version)
299+ if '%(version)s' in _HepTools[target_tool]['tarball'][1]:
300+ _HepTools[target_tool]['tarball'][1]=_HepTools[target_tool]['tarball'][1]%{'version':_version}
301+ else:
302+ raise Exception, 'fail to specify version for this tools.'
303+
304+ # Apply substitutions if necessary:
305+
306+ for tool in _HepTools:
307+ if _HepTools[tool]['install_path']:
308+ _HepTools[tool]['install_path']=_HepTools[tool]['install_path']%\
309+ {'prefix':_prefix, 'mg5_path': '' if _mg5_path is None else _mg5_path }
310+ if _HepTools[tool]['tarball'][0]=='local':
311+ _HepTools[tool]['tarball'][1]=_HepTools[tool]['tarball'][1]%\
312+ {'prefix':_prefix, 'mg5_path': '' if _mg5_path is None else _mg5_path }
313+
314+ if _HepTools[tool]['tarball'][0]=='online':
315+ version = _HepTools[tool]['version']
316+ if 'format_version' in _HepTools[tool]:
317+ version = _HepTools[tool]['format_version'](version)
318+ _HepTools[tool]['tarball'][1]=_HepTools[tool]['tarball'][1]%{'version':version}
319+
320+ new_libs = []
321+ for lib in _HepTools[tool]['libraries']:
322+ for libext in _lib_extensions:
323+ if lib%{'libextension':libext} not in new_libs:
324+ new_libs.append(lib%{'libextension':libext})
325+ _HepTools[tool]['libraries'] = new_libs
326+
327+
328+ # Make sure it is not already installed, but if the directory is empty, then remove it
329+ if os.path.isdir(pjoin(_prefix,target_tool)):
330+ if os.listdir(pjoin(_prefix,target_tool)) in [[],['%s_install.log'%target_tool]]:
331+ shutil.rmtree(pjoin(_prefix,target_tool))
332+ _keep_existing_installation = False
333+ else:
334+ if not _keep_existing_installation:
335+ if not _overwrite_existing_installation:
336+ logger.warning( "The specified path '%s' already contains an installation of tool '%s'.", _prefix, target_tool)
337+ logger.warning( "Rerun the HEPToolInstaller.py script again with the option '--force' if you want to overwrite it.")
338+ sys.exit(66)
339+ else:
340+ logger.info("Removing existing installation of tool '%s' in '%s'.", target_tool, _prefix)
341+ shutil.rmtree(pjoin(_prefix,target_tool))
342 else:
343- if not _keep_existing_installation:
344- if not _overwrite_existing_installation:
345- logger.warning( "The specified path '%s' already contains an installation of tool '%s'.", _prefix, target_tool)
346- logger.warning( "Rerun the HEPToolInstaller.py script again with the option '--force' if you want to overwrite it.")
347- sys.exit(66)
348- else:
349- logger.info("Removing existing installation of tool '%s' in '%s'.", target_tool, _prefix)
350- shutil.rmtree(pjoin(_prefix,target_tool))
351-else:
352- _keep_existing_installation = False
353+ _keep_existing_installation = False
354
355 # TMP_directory (designed to work as with statement) and go to it
356 class TMP_directory(object):
357@@ -681,11 +688,17 @@
358 def install_lhapdf6(tmp_path):
359 """Installation operations for lhapdf6"""
360 lhapdf6_log = open(pjoin(_HepTools['lhapdf6']['install_path'],"lhapdf6_install.log"), "w")
361+ cxx_flags = '-O'
362+ for flag in ['-static-libstdc++']:
363+ if test_cpp_compiler([flag]):
364+ cxx_flags = flag
365+
366 subprocess.call([pjoin(_installers_path,'installLHAPDF6.sh'),
367 _HepTools['boost']['install_path'],
368 _HepTools['lhapdf6']['install_path'],
369 _HepTools['lhapdf6']['version'],
370- _HepTools['lhapdf6']['tarball'][1]],
371+ _HepTools['lhapdf6']['tarball'][1],
372+ cxx_flags],
373 stdout=lhapdf6_log,
374 stderr=lhapdf6_log)
375 lhapdf6_log.close()
376@@ -765,7 +778,7 @@
377
378 # Run the installation script
379 mg5amc_py8_interface_log = open(pjoin(_HepTools['mg5amc_py8_interface']['install_path'],"mg5amc_py8_interface_install.log"), "w")
380- subprocess.call([pjoin(sys.executable, _HepTools['mg5amc_py8_interface']['install_path'],'compile.py')]+options,
381+ subprocess.call([sys.executable, pjoin(_HepTools['mg5amc_py8_interface']['install_path'],'compile.py')]+options,
382 stdout=mg5amc_py8_interface_log,
383 stderr=mg5amc_py8_interface_log)
384 mg5amc_py8_interface_log.close()
385@@ -1307,34 +1320,36 @@
386 logger.info(" > Now aborting installation of tool '%s'."%target_tool)
387 sys.exit(9)
388
389-_environ = dict(os.environ)
390-try:
391- os.environ["CXX"] = _cpp
392- os.environ["FC"] = _gfortran
393- # Also add the bin directory of the HEPTools install location, as we might need some of the executables installed there like cmake
394- os.environ["PATH"] = pjoin(_prefix,'bin')+os.pathsep+os.environ["PATH"]
395- install_with_dependencies(target_tool,is_main_target=True)
396-except ZeroDivisionError as e:
397+
398+if "__main__" == __name__:
399+ _environ = dict(os.environ)
400+ try:
401+ os.environ["CXX"] = _cpp
402+ os.environ["FC"] = _gfortran
403+ # Also add the bin directory of the HEPTools install location, as we might need some of the executables installed there like cmake
404+ os.environ["PATH"] = pjoin(_prefix,'bin')+os.pathsep+os.environ["PATH"]
405+ install_with_dependencies(target_tool,is_main_target=True)
406+ except ZeroDivisionError as e:
407+ os.environ.clear()
408+ os.environ.update(_environ)
409+ logger.critical("The following error occured during the installation of '%s' (and its dependencies):\n%s"%(target_tool,repr(e)))
410+ sys.exit(9)
411+
412 os.environ.clear()
413 os.environ.update(_environ)
414- logger.critical("The following error occured during the installation of '%s' (and its dependencies):\n%s"%(target_tool,repr(e)))
415- sys.exit(9)
416-
417-os.environ.clear()
418-os.environ.update(_environ)
419-
420-if check_successful_installation(target_tool):
421- # Successful installation, now copy the installed components directly under HEPTools
422- finalize_installation(target_tool)
423- logger.info("Successful installation of '%s' in '%s'."%(target_tool,_prefix))
424- logger.debug("See installation log at '%s'."%pjoin(_HepTools[target_tool]['install_path'],'%s_install.log'%target_tool))
425- sys.exit(0)
426-else:
427- logger.warning("A problem occured during the installation of '%s'.", target_tool)
428- try:
429- logger.warning("Content of the installation log file '%s':\n\n%s"%(\
430- pjoin(_HepTools[target_tool]['install_path'],'%s_install.log'%target_tool),
431- open(pjoin(_HepTools[target_tool]['install_path'],'%s_install.log'%target_tool),'r').read()))
432- except IOError:
433- logger.warning("No additional information on the installation problem available.")
434- sys.exit(9)
435+
436+ if check_successful_installation(target_tool):
437+ # Successful installation, now copy the installed components directly under HEPTools
438+ finalize_installation(target_tool)
439+ logger.info("Successful installation of '%s' in '%s'."%(target_tool,_prefix))
440+ logger.debug("See installation log at '%s'."%pjoin(_HepTools[target_tool]['install_path'],'%s_install.log'%target_tool))
441+ sys.exit(0)
442+ else:
443+ logger.warning("A problem occured during the installation of '%s'.", target_tool)
444+ try:
445+ logger.warning("Content of the installation log file '%s':\n\n%s"%(\
446+ pjoin(_HepTools[target_tool]['install_path'],'%s_install.log'%target_tool),
447+ open(pjoin(_HepTools[target_tool]['install_path'],'%s_install.log'%target_tool),'r').read()))
448+ except IOError:
449+ logger.warning("No additional information on the installation problem available.")
450+ sys.exit(9)
451
452=== modified file 'installLHAPDF6.sh'
453--- installLHAPDF6.sh 2016-04-20 19:10:54 +0000
454+++ installLHAPDF6.sh 2019-02-05 10:15:18 +0000
455@@ -10,6 +10,7 @@
456 INSTALLD="$2"
457 VERSION="$3"
458 TARBALL="$4"
459+ CXXFLAGS="$5"
460 LOCAL=$INSTALLD
461
462 # set SLC5 platform name:
463@@ -32,7 +33,7 @@
464 cd LHAPDF-${VERSION}/
465
466 echo " Configure LHAPDF"
467- LIBRARY_PATH=$LD_LIBRARY_PATH ./configure CXXFLAGS="-static-libstdc++" --prefix=$LOCAL --bindir=$LOCAL/bin --datadir=$LOCAL/share --libdir=$LOCAL/lib --with-boost=$BOOST --enable-static
468+ LIBRARY_PATH=$LD_LIBRARY_PATH ./configure CXXFLAGS="$CXXFLAGS" --prefix=$LOCAL --bindir=$LOCAL/bin --datadir=$LOCAL/share --libdir=$LOCAL/lib --with-boost=$BOOST --enable-static
469
470 echo " Compile LHAPDF6"
471 LIBRARY_PATH=$LD_LIBRARY_PATH make

Subscribers

People subscribed via source and target branches

to all changes: