Merge lp:~maddevelopers/mg5amcnlo/1.5.14 into lp:~madteam/mg5amcnlo/trunk

[Olivier Mattelaer]

- Improve submission to various cluster (condor/SLURM/PBS). Thanks Dorival and Shu for the testing.
- Add a warning that ./bin/newprocess_mg5 will NOT be available in 2.0.0

This is a technical merge, mainly important for the adding of that warning.
So I would like to push this version tomorrow. This is what I will do if nobody complains about it.

Cheers,

Olivier

[lp:~maddevelopers/mg5amcnlo/1.5.14]

259. By Olivier Mattelaer on 2013-11-26

UpdateNotes.txt

=== modified file 'Template/Source/dsample.f'
--- Template/Source/dsample.f	2013-08-01 05:18:06 +0000
+++ Template/Source/dsample.f	2013-11-26 22:31:29 +0000
@@ -104,9 +104,7 @@
       itminx = itmin
       if (nsteps .lt. 1) nsteps=1
       nwrite = itmax*ncall/nsteps
-c      open(unit=66,file='.sample_warn',status='unknown')
-c      write(66,*) 'Warnings from sample run.',itmax,ncall
-c      close(66)
+
       call sample_init(ndim,ncall,itmax,ninvar,nconfigs)
       call graph_init
       do i=1,itmax
@@ -213,13 +211,15 @@
       do i=cur_it-itsum,cur_it-1
          write(66,'(i4,5e15.5)') i,xmean(i),xsigma(i),xeff(i),xwmax(i),xrmean(i)
       enddo
-      close(66)
+      flush(66)
+      close(66, status='KEEP')
       else
          open(unit=66,file='results.dat',status='unknown')
          write(66,'(3e12.5,2i9,i5,i9,3e10.3)')0.,0.,0.,kevent,nw,
      &     1,0,0.,0.,0.
          write(66,'(i4,5e15.5)') 1,0.,0.,0.,0.,0.
-         close(66)
+         flush(66)
+         close(66, status='KEEP')
 
       endif
 c
@@ -373,13 +373,15 @@
       do i=cur_it-itsum,cur_it-1
          write(66,'(i4,5e15.5)') i,xmean(i),xsigma(i),xeff(i),xwmax(i),xrmean(i)
       enddo
-      close(66)
+      flush(66)
+      close(66, status='KEEP')
       else
          open(unit=66,file='results.dat',status='unknown')
          write(66,'(3e12.5,2i9,i5,i9,3e10.3)')0.,0.,0.,kevent,nw,
      &     1,0,0.,0.,0.
          write(66,'(i4,5e15.5)') 1,0.,0.,0.,0.,0.
-         close(66)
+         flush(66)
+         close(66, status='KEEP')
 
       endif      
 
@@ -1863,7 +1865,8 @@
       write(66,'(3e12.5,2i9,i5,i9,3e10.3)')0.,0.,0.,0,0,
      &     0,1,0.,0.,0.
       write(66,'(i4,5e15.5)') 1,0.,0.,0.,0.,0.
-      close(66)
+      flush(66)
+      close(66, status='KEEP')
 
 c     Remove file events.lhe (otherwise event combination gets screwed up)
       write(*,*) 'Deleting file events.lhe'

=== modified file 'Template/Source/gen_ximprove.f'
--- Template/Source/gen_ximprove.f	2012-11-08 04:48:28 +0000
+++ Template/Source/gen_ximprove.f	2013-11-26 22:31:29 +0000
@@ -354,9 +354,19 @@
      &        '"  >> input_sg.txt' !Helicity 
          write(26,'(5x,3a)')'echo "',gn(io(i))(2:ip-1),
      $        '" >>input_sg.txt'
+      write(26,20) 'for try in $(seq 1 10);'
+      write(26,20) 'do'
          write(26,20) '../madevent >> $k <input_sg.txt'
+         write(26,25) 'if [ -s $k ]'
+         write(26,25) 'then'
+         write(26,25) '    break'
+         write(26,25) 'else'
+         write(26,25) '    echo $try > fail.log '
+         write(26,25) 'fi'
+         write(26,25) 'done'
          write(26,20) 'rm ftn25 ftn26'
          write(26,20) 'cat $k >> log.txt'
+         write(26,20) 'echo "" >> $k; echo "ls status:" >> $k; ls >> $k'
          write(26,20) 'cd ../'
       endif
       enddo  !Loop over diagrams
@@ -681,7 +691,17 @@
      &        ' " >> input_sg.txt' !Helicity 0=exact
          write(26,'(9x,3a)')'echo "',gn(io(np))(2:ip-1),
      $        '" >>input_sg.txt'
+         write(26,25) 'for try in $(seq 1 10);'
+         write(26,25) 'do'
          write(26,25) '../madevent >> $k <input_sg.txt'
+         write(26,25) 'if [ -s $k ]'
+         write(26,25) 'then'
+         write(26,25) '    break'
+         write(26,25) 'else'
+         write(26,25) '    echo $try > fail.log '
+         write(26,25) 'fi'
+         write(26,25) 'done'
+         write(26,20) 'echo "" >> $k; echo "ls status:" >> $k; ls >> $k'
          write(26,25) 'cat $k >> log.txt'
          write(26,25) 'if [[ -e ftn26 ]]; then'
          write(26,25) '     cp ftn26 ftn25'
@@ -718,7 +738,17 @@
          write(26,25) 'if [[ -e ftn26 ]]; then'
          write(26,25) '     cp ftn26 ftn25'
          write(26,25) 'fi'
+                  write(26,25) 'for try in $(seq 1 10);'
+         write(26,25) 'do'
          write(26,25) '../madevent >> $k <input_sg.txt'
+         write(26,25) 'if [ -s $k ]'
+         write(26,25) 'then'
+         write(26,25) '    break'
+         write(26,25) 'else'
+         write(26,25) '    echo $try > fail.log '
+         write(26,25) 'fi'
+         write(26,25) 'done'
+         write(26,20) 'echo "" >> $k; echo "ls status:" >> $k; ls >> $k'
          write(26,25) 'cat $k >> log.txt'
          write(26,20) 'fi'
          write(26,20) 'cd ../'
@@ -848,7 +878,17 @@
      &           ' " >> input_sg.txt' !Helicity 0=exact
             write(26,'(9x,3a)')'echo "',gn(i)(2:ip-1),
      $           '" >>input_sg.txt'
+                  write(26,25) 'for try in $(seq 1 10);'
+         write(26,25) 'do'
             write(26,25) '../madevent >> $k <input_sg.txt'
+         write(26,25) 'if [ -s $k ]'
+         write(26,25) 'then'
+         write(26,25) '    break'
+         write(26,25) 'else'
+         write(26,25) '    echo $try > fail.log '
+         write(26,25) 'fi'
+         write(26,25) 'done'
+            write(26,20) 'echo "" >> $k; echo "ls status:" >> $k; ls >> $k'
             write(26,25) 'cat $k >> log.txt'
             write(26,25) 'if [[ -e ftn26 ]]; then'
             write(26,25) '     cp ftn26 ftn25'
@@ -876,7 +916,17 @@
             write(26,25) 'if [[ -e ftn26 ]]; then'
             write(26,25) '     cp ftn26 ftn25'
             write(26,25) 'fi'
+                     write(26,25) 'for try in $(seq 1 10);'
+         write(26,25) 'do'
             write(26,25) '../madevent >> $k <input_sg.txt'
+                     write(26,25) 'if [ -s $k ]'
+         write(26,25) 'then'
+         write(26,25) '    break'
+         write(26,25) 'else'
+         write(26,25) '    echo $try > fail.log '
+         write(26,25) 'fi'
+         write(26,25) 'done'
+            write(26,20) 'echo "" >> $k; echo "ls status:" >> $k; ls >> $k'
             write(26,25) 'cat $k >> log.txt'
             write(26,20) 'fi'
             write(26,20) 'cd ../'

=== modified file 'Template/bin/internal/run_combine'
--- Template/bin/internal/run_combine	2011-08-31 15:58:51 +0000
+++ Template/bin/internal/run_combine	2013-11-26 22:31:29 +0000
@@ -1,4 +1,15 @@
 #!/bin/bash
 
-../bin/internal/combine_events 
+rm fail_combine.log &> /dev/null
+for try in $(seq 1 10)
+do
+    ../bin/internal/combine_events 2> fail_combine.log
+    if grep -F "combine_events: No such file or directory" fail_combine.log
+    then
+        sleep 1
+    else
+     	rm fail_combine.log &> /dev/null
+        break
+    fi
+done
 rm -f scratch

=== modified file 'Template/bin/newprocess_mg5'
--- Template/bin/newprocess_mg5	2013-03-15 04:03:57 +0000
+++ Template/bin/newprocess_mg5	2013-11-26 22:31:29 +0000
@@ -10,6 +10,9 @@
         exit
     fi
 fi
+echo -e  "\033[1;31m WARNING This out-dated method of using MG5 will be remove in version 2.0\033[0m"
+echo -e "\033[1;31m A proc_card example is present in the main directory of MG5 \033[0m"
+echo -e "\033[1;31m you can run it like ./bin/mg5 proc_card.dat \033[0m"
 
 if [[ -e SubProcesses/subproc.mg ]]; then
     echo "Error: newprocess_mg5 can only be used in a clean copy of Template."

=== modified file 'UpdateNotes.txt'
--- UpdateNotes.txt	2013-11-04 08:13:31 +0000
+++ UpdateNotes.txt	2013-11-26 22:31:29 +0000
@@ -1,5 +1,8 @@
 Update notes for MadGraph 5 (in reverse time order)
 
+1.5.14 (26/11/13) OM: Add warning about the fact that newprocess_mg5 is going to be remove in MG5_aMC_V2.0.0
+                  OM: Improved cluster submision/re-submition control. 
+
 1.5.13 (04/11/13) OM: Implement a function which check if jobs submitted to cluster are correctly runned.
                       In case of failure, you can re-submitted the failing jobs automatically. The maximal 
                       number of re-submission for a job can be parametrize (default 1) and how long you have to

=== modified file 'madgraph/interface/madevent_interface.py'
--- madgraph/interface/madevent_interface.py	2013-11-01 18:33:23 +0000
+++ madgraph/interface/madevent_interface.py	2013-11-26 22:31:29 +0000
@@ -1795,7 +1795,8 @@
             args.remove(arg)
 
         if args and args[0] in ["run_mode", "cluster_mode", "cluster_queue", 
-                                "cluster_temp_path", "nb_core"]:
+                                "cluster_temp_path", "nb_core", "cluster_nb_retry",
+                                "cluster_retry_wait"]:
             return args
 
         if self.cluster_mode == 2 and not self.nb_core:
@@ -1807,6 +1808,8 @@
             cluster_name = opt['cluster_type']
             self.cluster = cluster.from_name[cluster_name](opt['cluster_queue'],
                                                         opt['cluster_temp_path'])
+            self.cluster.nb_retry = opt['cluster_nb_retry']
+            self.cluster.cluster_retry_wait = int(opt['cluster_retry_wait'])
         return args
     
     ############################################################################            
@@ -1906,6 +1909,11 @@
                         continue
                 self.options[key] = None
             elif key.startswith('cluster'):
+                if key in ('cluster_nb_retry','cluster_wait_retry'):
+                    self.options[key] = int(self.options[key])
+                if hasattr(self,'cluster'):
+                    del self.cluster
+
                 pass              
             elif key == 'automatic_html_opening':
                 if self.options[key] in ['False', 'True']:
@@ -2199,13 +2207,13 @@
             self.cluster = cluster.from_name[opt['cluster_type']](\
                                  opt['cluster_queue'], opt['cluster_temp_path'])
             self.cluster.nb_retry = self.options['cluster_nb_retry']
-            self.cluster_retry_wait = self.options['cluster_retry_wait']
+            self.cluster.cluster_retry_wait = int(self.options['cluster_retry_wait'])
         elif args[0] in ['cluster_nb_retry', 'cluster_retry_wait']:
             self.options[args[0]] = int(args[1])
             if args[0] == 'cluster_nb_retry':
                 self.cluster.nb_retry = int(args[1])
             else:
-                self.cluster_retry_wait = int(args[1])
+                self.cluster.cluster_retry_wait = int(args[1])
         elif args[0] == 'nb_core':
             if args[1] == 'None':
                 import multiprocessing
@@ -2845,7 +2853,8 @@
         if self.cluster_mode == 1:
             self.cluster.launch_and_wait('../bin/internal/run_combine', 
                                         cwd=pjoin(self.me_dir,'SubProcesses'),
-                                        stdout=pjoin(self.me_dir,'SubProcesses', 'combine.log'))
+                                        stdout=pjoin(self.me_dir,'SubProcesses', 'combine.log'),
+                                        required_output=[pjoin(self.me_dir,'SubProcesses', 'combine.log')])
         else:
             misc.call(['../bin/internal/run_combine'],
                          cwd=pjoin(self.me_dir,'SubProcesses'), 
@@ -3071,16 +3080,20 @@
             
         if not self.run_name:
             self.check_pythia(args)
-            self.configure_directory()
+            self.configure_directory(html_opening =False)
         else:
             # initialize / remove lhapdf mode        
-            self.configure_directory()
+            self.configure_directory(html_opening =False)
             self.check_pythia(args)        
         
         # the args are modify and the last arg is always the mode 
         if not no_default:
             self.ask_pythia_run_configuration(args[-1])
 
+        if self.options['automatic_html_opening']:
+            misc.open_file(os.path.join(self.me_dir, 'crossx.html'))
+            self.options['automatic_html_opening'] = False
+
         # Update the banner with the pythia card
         if not self.banner:
             self.banner = banner_mod.recover_banner(self.results, 'pythia')
@@ -3903,7 +3916,7 @@
         return name % (max(data+[0])+1) 
 
     ############################################################################   
-    def configure_directory(self):
+    def configure_directory(self, html_opening=True):
         """ All action require before any type of run """   
 
 
@@ -3919,7 +3932,7 @@
         else:
             self.configured = time.time()
         self.update_status('compile directory', level=None)
-        if self.options['automatic_html_opening']:
+        if self.options['automatic_html_opening'] and html_opening:
             misc.open_file(os.path.join(self.me_dir, 'crossx.html'))
             self.options['automatic_html_opening'] = False
             #open only once the web page

=== modified file 'madgraph/interface/madgraph_interface.py'
--- madgraph/interface/madgraph_interface.py	2013-11-01 18:33:23 +0000
+++ madgraph/interface/madgraph_interface.py	2013-11-26 22:31:29 +0000
@@ -3152,6 +3152,7 @@
                 text = open(path).read()
                 text = text.replace('FC=g77','FC=gfortran')
                 open(path, 'w').writelines(text)
+            os.environ['FC'] = compiler
                             
         if logger.level <= logging.INFO:
             devnull = open(os.devnull,'w') 
@@ -3354,6 +3355,11 @@
                     break
                 print 'apply patch %s' % (i+1)
                 text = filetext.read()
+                # track rename since patch fail to apply those correctly.
+                pattern = re.compile(r'''=== renamed file \'(?P<orig>[^\']*)\' => \'(?P<new>[^\']*)\'''')
+                #=== renamed file 'Template/SubProcesses/addmothers.f' => 'madgraph/iolibs/template_files/addmothers.f'
+                for orig, new in pattern.findall(text):
+                    files.cp(pjoin(MG5DIR, orig), pjoin(MG5DIR, new))
                 p= subprocess.Popen(['patch', '-p1'], stdin=subprocess.PIPE, 
                                                                   cwd=MG5DIR)
                 p.communicate(text)

=== modified file 'madgraph/iolibs/template_files/madevent_symmetry.f'
--- madgraph/iolibs/template_files/madevent_symmetry.f	2013-08-06 19:08:29 +0000
+++ madgraph/iolibs/template_files/madevent_symmetry.f	2013-11-26 22:31:29 +0000
@@ -651,7 +651,7 @@
 c
 c     Now write the commands
 c      
-      write(lun,20) 'echo $i >& run.$script'
+c      write(lun,20) 'echo $i >& run.$script'
       write(lun,20) 'j=G$i'
       write(lun,20) 'if [[ ! -e $j ]]; then'
       write(lun,25) 'mkdir $j'
@@ -661,9 +661,20 @@
       write(lun,20) 'rm -f $k'
       write(lun,20) 'cat ../input_app.txt >& input_app.txt'
       write(lun,20) 'echo $i >> input_app.txt'
+      write(lun,20) 'for try in $(seq 1 10);'
+      write(lun,20) 'do'
       write(lun,20) '../madevent > $k <input_app.txt'
+      write(lun,20) 'if [ -s $k ]'
+      write(lun,20) 'then'
+      write(lun,20) '    break'
+      write(lun,20) 'else'
+      write(lun,20) 'sleep 1'
+c      write(lun,20) 'rm -rf $k; ../madevent > $k <input_app.txt'
+      write(lun,20) 'fi'
+      write(lun,20) 'done'
       write(lun,20) 'rm -f ftn25 ftn99'
       if(.not.gridpack) write(lun,20) 'rm -f ftn26'
+      write(lun,20) 'echo "" >> $k; echo "ls status:" >> $k; ls >> $k'
       write(lun,20) 'cp $k log.txt'
       write(lun,20) 'cd ../'
       write(lun,15) 'done'

=== modified file 'madgraph/various/cluster.py'
--- madgraph/various/cluster.py	2013-11-01 18:33:23 +0000
+++ madgraph/various/cluster.py	2013-11-26 22:31:29 +0000
@@ -48,7 +48,10 @@
             try:
                 return f(self, *args, **opt)
             except error:
-                self.remove(*args, **opt)
+                try:
+                    self.remove(*args, **opt)
+                except Exception:
+                    pass
                 raise error
         return deco_f_interupt
     return deco_interupt
@@ -59,7 +62,7 @@
         def deco_f_store(self, prog, argument=[], cwd=None, stdout=None, stderr=None, log=None,
                 input_files=[], output_files=[], required_output=[], nb_submit=0):
             frame = inspect.currentframe()
-            args, i, j, values = inspect.getargvalues(frame)
+            args, _, _, values = inspect.getargvalues(frame)
             args = dict([(i, values[i]) for i in args if i != 'self'])
             id = f(self, **args)
             if self.nb_retry > 0:
@@ -83,6 +86,7 @@
         # attribute to relaunch jobs if they failed to produce expected data
         self.nb_retry = 1
         self.retry_args = {}
+        self.cluster_retry_wait = 300
     
     def submit(self, prog, argument=[], cwd=None, stdout=None, stderr=None, 
                log=None, required_output=[], nb_submit=0):
@@ -95,10 +99,11 @@
         """How to make one submission. Return status id on the cluster.
         NO SHARE DISK"""
         
-        if not hasattr(self, 'temp_dir') or not self.temp_dir:
+        if not hasattr(self, 'temp_dir') or not self.temp_dir or \
+            (input_files == [] == output_files):
             return self.submit(prog, argument, cwd, stdout, stderr, log, 
                                required_output=required_output, nb_submit=nb_submit)
-        print self.temp_dir
+
         if cwd is None:
             cwd = os.getcwd()
         if not os.path.exists(prog):
@@ -183,6 +188,7 @@
     def check_termination(self, job_id):
         """Check the termination of the jobs with job_id and relaunch it if needed."""
         
+
         if job_id not in self.retry_args:
             return True
 
@@ -206,12 +212,11 @@
             return 'done'
         
         if time_check == 0:
-            logger.warning('''Job %s failed to produce expected output. Waiting for filesystem update.\nMissing file:\n%s''' % (job_id,path))
+            logger.debug('''Job %s: missing output:%s''' % (job_id,path))
             args['time_check'] = time.time()
             return 'wait'
-        elif self.cluster_retry_wait < time.time() - time_check:    
+        elif self.cluster_retry_wait > time.time() - time_check:    
             return 'wait'
-        
 
         #jobs failed to be completed even after waiting time!!
         if self.nb_retry < 0:
@@ -224,14 +229,14 @@
             with option: %s
             file missing: %s.
             Stopping all runs.''' % (job_id, args, path))
-            self.remove()
+            #self.remove()
         elif args['nb_submit'] >= self.nb_retry:
             logger.critical('''Fail to run correctly job %s.
             with option: %s
             file missing: %s
             Fails %s times
             No resubmition. ''' % (job_id, args, path, args['nb_submit']))
-            self.remove()
+            #self.remove()
         else:
             args['nb_submit'] += 1            
             logger.warning('resubmit job (for the %s times)' % args['nb_submit'])
@@ -247,7 +252,8 @@
             
     @check_interupt()
     def launch_and_wait(self, prog, argument=[], cwd=None, stdout=None, 
-                                                         stderr=None, log=None):
+                        stderr=None, log=None, required_output=[], nb_submit=0,
+                        input_files=[], output_files=[]):
         """launch one job on the cluster and wait for it"""
         
         special_output = False # tag for concatenate the error with the output.
@@ -255,14 +261,41 @@
             #We are suppose to send the output to stdout
             special_output = True
             stderr = stdout + '.err'
-        id = self.submit(prog, argument, cwd, stdout, stderr, log)
-        while 1:        
+        id = self.submit2(prog, argument, cwd, stdout, stderr, log,
+                          required_output=required_output, input_files=input_files,
+                          output_files=output_files)
+        
+        frame = inspect.currentframe()
+        args, _, _, values = inspect.getargvalues(frame)
+        args = dict([(i, values[i]) for i in args if i != 'self'])        
+        self.retry_args[id] = args
+        
+        nb_wait=0
+        while 1: 
+            nb_wait+=1
             status = self.control_one_job(id)
             if not status in ['R','I']:
+                status = self.check_termination(id)
+                if status in ['wait']:
+                    time.sleep(30)
+                    continue
+                elif status in ['resubmit']:
+                    id = self.submitted_ids[0]
+                    time.sleep(30)
+                    continue
+                #really stop!
                 time.sleep(30) #security to ensure that the file are really written on the disk
                 break
             time.sleep(30)
         
+        if required_output:
+            status = self.check_termination(id)
+            if status == 'wait':
+                run += 1
+            elif status == 'resubmit':
+                idle += 1
+        
+        
         if special_output:
             # combine the stdout and the stderr
             #wait up to 50 s to see if those files exists
@@ -281,7 +314,7 @@
                         return
                 time.sleep(10)
                         
-    def remove(self, *args):
+    def remove(self, *args, **opts):
         """ """
         logger.warning("""This cluster didn't support job removal, 
     the jobs are still running on the cluster.""")
@@ -362,6 +395,10 @@
            input/output file should be give relative to cwd
         """
         
+        if (input_files == [] == output_files):
+            return self.submit(prog, argument, cwd, stdout, stderr, log, 
+                               required_output=required_output, nb_submit=nb_submit)
+        
         text = """Executable = %(prog)s
                   output = %(stdout)s
                   error = %(stderr)s
@@ -483,7 +520,7 @@
                     fail += 1
 
         for id in list(self.submitted_ids):
-            if id not in ongoing:
+            if int(id) not in ongoing:
                 status = self.check_termination(id)
                 if status == 'wait':
                     run += 1
@@ -493,7 +530,7 @@
         return idle, run, self.submitted - (idle+run+fail), fail
     
     @multiple_try()
-    def remove(self, *args):
+    def remove(self, *args, **opts):
         """Clean the jobson the cluster"""
         
         if not self.submitted_ids:
@@ -623,7 +660,7 @@
         return idle, run, self.submitted - (idle+run+fail), fail
 
     @multiple_try()
-    def remove(self, *args):
+    def remove(self, *args, **opts):
         """Clean the jobs on the cluster"""
         
         if not self.submitted_ids:
@@ -773,7 +810,7 @@
     
     
     @multiple_try()
-    def remove(self, *args):
+    def remove(self, *args, **opts):
         """Clean the jobs on the cluster"""
         
         if not self.submitted_ids:
@@ -899,7 +936,7 @@
         return idle, run, self.submitted - (idle+run+fail), fail
 
     @multiple_try()
-    def remove(self, *args):
+    def remove(self, *args,**opts):
         """Clean the jobs on the cluster"""
         
         if not self.submitted_ids:
@@ -1023,7 +1060,7 @@
         return idle, run, self.submitted - idle - run - fail, fail
 
     @multiple_try()
-    def remove(self, *args):
+    def remove(self, *args, **opts):
         """Clean the jobs on the cluster"""
         
         if not self.submitted_ids:
@@ -1113,22 +1150,38 @@
                   me_dir = 'a' + me_dir[1:]
 
         idle, run, fail = 0, 0, 0
+        ongoing=[]
         for line in status.stdout:
             if me_dir in line:
-                status = line.split()[4]
+                id, _, _,_ , status,_ = line.split(None,5)
+                ongoing.append(id)
                 if status in self.idle_tag:
                     idle += 1
                 elif status in self.running_tag:
                     run += 1
                 elif status in self.complete_tag:
-                    continue
+                    status = self.check_termination(id)
+                    if status == 'wait':
+                        run += 1
+                    elif status == 'resubmit':
+                        idle += 1                    
                 else:
                     fail += 1
-
+        
+        #control other finished job
+        for id in list(self.submitted_ids):
+            if id not in ongoing:
+                status = self.check_termination(id)
+                if status == 'wait':
+                    run += 1
+                elif status == 'resubmit':
+                    idle += 1
+                    
+        
         return idle, run, self.submitted - (idle+run+fail), fail
 
     @multiple_try()
-    def remove(self, *args):
+    def remove(self, *args, **opts):
         """Clean the jobs on the cluster"""
         
         if not self.submitted_ids:

=== added file 'proc_card.dat'
--- proc_card.dat	1970-01-01 00:00:00 +0000
+++ proc_card.dat	2013-11-26 22:31:29 +0000
@@ -0,0 +1,40 @@
+#************************************************************
+#*                        MadGraph 5                        *
+#*                                                          *
+#*                *                       *                 *
+#*                  *        * *        *                   *
+#*                    * * * * 5 * * * *                     *
+#*                  *        * *        *                   *
+#*                *                       *                 *
+#*                                                          *
+#*                                                          *
+#*    The MadGraph Development Team - Please visit us at    *
+#*    https://server06.fynu.ucl.ac.be/projects/madgraph     *
+#*                                                          *
+#************************************************************
+#*                                                          *
+#*               Command File for MadGraph 5                *
+#*                                                          *
+#*     run as ./bin/mg5  filename                           *
+#*                                                          *
+#************************************************************
+import model sm
+# Define multiparticle labels
+define p = g u c d s u~ c~ d~ s~
+define j = g u c d s u~ c~ d~ s~
+define l+ = e+ mu+
+define l- = e- mu-
+define vl = ve vm vt
+define vl~ = ve~ vm~ vt~
+# Specify process(es) to run
+generate p p > e- ve~ @1
+add process p p > e- ve~ j @2
+add process p p > t t~ @3
+# Output processes to MadEvent directory
+output 
+# This will create a directory PROC_$MODELNAME_$X
+# If you want to specify the path/name of the directory use
+# output PATH
+
+# To generate events, you can go to the created directory and 
+# run ./bin/generate_events
\ No newline at end of file