Merge lp:~fluidity-core/fluidity/zoltan into lp:fluidity

[Paul Woodhams]

One of the MPI_AllReduces is only necessary when using additional adapt iterations. This commit fixes the code so that it only does the collective communication when the additional adapt iterations option is being used.

Also adds some more logging messages detailing the selection of partitioners.

[Jon Hill (jon-hill)]

Looks good.

review: Approve

[lp:~fluidity-core/fluidity/zoltan]

3449. By Jon Hill on 2012-04-03

Merge from trunk

=== modified file 'assemble/Adapt_State.F90'
--- assemble/Adapt_State.F90	2011-09-28 13:38:45 +0000
+++ assemble/Adapt_State.F90	2012-04-03 09:59:20 +0000
@@ -1003,13 +1003,9 @@
     call get_option("/mesh_adaptivity/hr_adaptivity/zoltan_options/element_quality_cutoff", &
        & quality_tolerance, default = 0.6)
 
-    if (max_adapt_iteration .ne. 1) then
-       zoltan_min_adapt_iterations = adapt_iterations()
-       zoltan_max_adapt_iterations = zoltan_min_adapt_iterations + zoltan_additional_adapt_iterations
-    else
-       zoltan_min_adapt_iterations = adapt_iterations()
-       zoltan_max_adapt_iterations = zoltan_min_adapt_iterations
-    end if
+    zoltan_min_adapt_iterations = adapt_iterations()
+    zoltan_max_adapt_iterations = zoltan_min_adapt_iterations + zoltan_additional_adapt_iterations
+
 #endif
 
     finished_adapting = .false.
@@ -1238,7 +1234,11 @@
           ! call zoltan now but we need to pass in both the 2d metric (metric) and the 3d full metric (full_metric)
           ! the first is needed to define the element qualities while the second must be interpolated to the newly
           ! decomposed mesh
-          call zoltan_drive(states, final_adapt_iteration, global_min_quality, metric = metric, full_metric = full_metric)
+          if (zoltan_additional_adapt_iterations .gt. 0) then
+             call zoltan_drive(states, final_adapt_iteration, global_min_quality = global_min_quality, metric = metric, full_metric = full_metric)
+          else
+             call zoltan_drive(states, final_adapt_iteration, metric = metric, full_metric = full_metric)
+          end if
           default_stat%zoltan_drive_call=.true.
 
           ! now we can deallocate the horizontal metric and point metric back at the full metric again
@@ -1246,7 +1246,11 @@
           metric = full_metric
         else
 
-          call zoltan_drive(states, final_adapt_iteration, global_min_quality, metric = metric)
+          if (zoltan_additional_adapt_iterations .gt. 0) then
+             call zoltan_drive(states, final_adapt_iteration, global_min_quality = global_min_quality, metric = metric)
+          else
+             call zoltan_drive(states, final_adapt_iteration, metric = metric)
+          end if
           default_stat%zoltan_drive_call=.true.
 
         end if
@@ -1303,10 +1307,12 @@
                ewrite(2,*) "The next iteration will be final adapt iteration else we'll go over the maximum adapt iterations."
             end if
 
-            if (global_min_quality .le. quality_tolerance) then
-               ewrite(-1,*) "Mesh contains elements with quality below element quality tolerance. May need to increase number of adapt iterations to ensure good quality mesh."
-               ewrite(-1,*) "min_quality = ", global_min_quality
-               ewrite(-1,*) "quality_tolerance = ", quality_tolerance
+            if (zoltan_additional_adapt_iterations .gt. 0) then
+               if (global_min_quality .le. quality_tolerance) then
+                  ewrite(-1,*) "Mesh contains elements with quality below element quality tolerance. May need to increase number of adapt iterations to ensure good quality mesh."
+                  ewrite(-1,*) "min_quality = ", global_min_quality
+                  ewrite(-1,*) "quality_tolerance = ", quality_tolerance
+               end if
             end if
             
             final_adapt_iteration = .true.
 
=== modified file 'assemble/Zoltan_integration.F90'
--- assemble/Zoltan_integration.F90	2011-09-26 17:04:31 +0000
+++ assemble/Zoltan_integration.F90	2012-04-03 09:59:20 +0000
@@ -60,7 +60,7 @@
 
     type(state_type), dimension(:), intent(inout), target :: states
     logical, intent(in) :: final_adapt_iteration
-    real, intent(out) :: global_min_quality
+    real, intent(out), optional :: global_min_quality
     ! the metric is the metric we base the quality functions on
     type(tensor_field), intent(inout), optional :: metric
     ! the full_metric is the metric we need to interpolate
@@ -152,23 +152,27 @@
        & p1_num_export, p1_export_global_ids, p1_export_local_ids, p1_export_procs, &
        & load_imbalance_tolerance, flredecomp, flredecomp_input_procs, flredecomp_target_procs)
 
-    if (.NOT. final_adapt_iteration) then
 
-       if (.NOT. zoltan_global_calculated_local_min_quality) then
-          FLAbort("Minimum element quality was not calculated during the load balance call.")
+    ! Only calculate the global minimum element quality if additional adapt iterations are being used
+    if (present(global_min_quality)) then
+       if (.NOT. final_adapt_iteration) then
+          
+          if (.NOT. zoltan_global_calculated_local_min_quality) then
+             FLAbort("Minimum element quality was not calculated during the load balance call.")
+          end if
+          
+          ! calculate the global minimum element quality
+          call mpi_allreduce(zoltan_global_local_min_quality, global_min_quality, 1, getPREAL(), &
+               & MPI_MIN, MPI_COMM_FEMTOOLS, ierr)
+          assert(ierr == MPI_SUCCESS)
+          
+          ewrite(1,*) "local minimum element quality = ", zoltan_global_local_min_quality
+          ewrite(1,*) "global minimum element quality = ", global_min_quality
+          
+       else
+          ! On final iteration we do not calculate the minimum element quality
+          global_min_quality = 1.0
        end if
-
-       ! calculate the global minimum element quality
-       call mpi_allreduce(zoltan_global_local_min_quality, global_min_quality, 1, getPREAL(), &
-          & MPI_MIN, MPI_COMM_FEMTOOLS, ierr)
-       assert(ierr == MPI_SUCCESS)
-       
-       ewrite(1,*) "local minimum element quality = ", zoltan_global_local_min_quality
-       ewrite(1,*) "global minimum element quality = ", global_min_quality
-       
-    else
-       ! On final iteration we do not calculate the minimum element quality
-       global_min_quality = 1.0
     end if
 
     if (changes .eqv. .false.) then
@@ -524,6 +528,7 @@
           if (have_option(trim(zoltan_global_base_option_path) // "/partitioner/metis"))  then
              ierr = Zoltan_Set_Param(zz, "LB_METHOD", "GRAPH"); assert(ierr == ZOLTAN_OK)
              ierr = Zoltan_Set_Param(zz, "GRAPH_PACKAGE", "PARMETIS"); assert(ierr == ZOLTAN_OK)
+             ewrite(3,*) "Setting the partitioner to be ParMETIS."
              ! turn off graph checking unless debugging, this was filling the error file with Zoltan warnings
              if (have_option(trim(zoltan_global_base_option_path) // "/zoltan_debug/graph_checking")) then
                 call get_option(trim(zoltan_global_base_option_path) // "/zoltan_debug/graph_checking", graph_checking_level)
@@ -540,9 +545,11 @@
              if (trim(method) == "graph") then
                 ierr = Zoltan_Set_Param(zz, "LB_METHOD", "GRAPH"); assert(ierr == ZOLTAN_OK)
                 ierr = Zoltan_Set_Param(zz, "GRAPH_PACKAGE", "PHG"); assert(ierr == ZOLTAN_OK)
+                ewrite(3,*) "Setting the partitioner to be Zoltan-Graph."
              else if (trim(method) == "hypergraph") then
                 ierr = Zoltan_Set_Param(zz, "LB_METHOD", "HYPERGRAPH"); assert(ierr == ZOLTAN_OK)
                 ierr = Zoltan_Set_Param(zz, "HYPERGRAPH_PACKAGE", "PHG"); assert(ierr == ZOLTAN_OK)
+                ewrite(3,*) "Setting the partitioner to be Zoltan-Hypergraph."
              end if
              
           end if
@@ -550,6 +557,7 @@
           if (have_option(trim(zoltan_global_base_option_path) // "/partitioner/scotch")) then
              ierr = Zoltan_Set_Param(zz, "LB_METHOD", "GRAPH"); assert(ierr == ZOLTAN_OK)
              ierr = Zoltan_Set_Param(zz, "GRAPH_PACKAGE", "SCOTCH"); assert(ierr == ZOLTAN_OK)
+             ewrite(3,*) "Setting the partitioner to be Scotch."
              ! Probably not going to want graph checking unless debugging
              if (have_option(trim(zoltan_global_base_option_path) // "/zoltan_debug/graph_checking")) then
                 call get_option(trim(zoltan_global_base_option_path) // "/zoltan_debug/graph_checking", graph_checking_level)
@@ -563,6 +571,7 @@
           ! Use the Zoltan graph partitioner by default
           ierr = Zoltan_Set_Param(zz, "LB_METHOD", "GRAPH"); assert(ierr == ZOLTAN_OK)
           ierr = Zoltan_Set_Param(zz, "GRAPH_PACKAGE", "PHG"); assert(ierr == ZOLTAN_OK)
+          ewrite(3,*) "No partitioner option set, defaulting to using Zoltan-Graph."
        end if
 
     else
@@ -572,6 +581,7 @@
           if (have_option(trim(zoltan_global_base_option_path) // "/final_partitioner/metis"))  then
              ierr = Zoltan_Set_Param(zz, "LB_METHOD", "GRAPH"); assert(ierr == ZOLTAN_OK)
              ierr = Zoltan_Set_Param(zz, "GRAPH_PACKAGE", "PARMETIS"); assert(ierr == ZOLTAN_OK)
+             ewrite(3,*) "Setting the final partitioner to be ParMETIS."
              ! turn off graph checking unless debugging, this was filling the error file with Zoltan warnings
              if (have_option(trim(zoltan_global_base_option_path) // "/zoltan_debug/graph_checking")) then
                 call get_option(trim(zoltan_global_base_option_path) // "/zoltan_debug/graph_checking", graph_checking_level)
@@ -588,9 +598,11 @@
              if (trim(method) == "graph") then
                 ierr = Zoltan_Set_Param(zz, "LB_METHOD", "GRAPH"); assert(ierr == ZOLTAN_OK)
                 ierr = Zoltan_Set_Param(zz, "GRAPH_PACKAGE", "PHG"); assert(ierr == ZOLTAN_OK)
+                ewrite(3,*) "Setting the final partitioner to be Zoltan-Graph."
              else if (trim(method) == "hypergraph") then
                 ierr = Zoltan_Set_Param(zz, "LB_METHOD", "HYPERGRAPH"); assert(ierr == ZOLTAN_OK)
                 ierr = Zoltan_Set_Param(zz, "HYPERGRAPH_PACKAGE", "PHG"); assert(ierr == ZOLTAN_OK)
+                ewrite(3,*) "Setting the final partitioner to be Zoltan-Hypergraph."
              end if
              
           end if
@@ -598,6 +610,7 @@
           if (have_option(trim(zoltan_global_base_option_path) // "/final_partitioner/scotch")) then
              ierr = Zoltan_Set_Param(zz, "LB_METHOD", "GRAPH"); assert(ierr == ZOLTAN_OK)
              ierr = Zoltan_Set_Param(zz, "GRAPH_PACKAGE", "SCOTCH"); assert(ierr == ZOLTAN_OK)
+                ewrite(3,*) "Setting the final partitioner to be Scotch."
              ! Probably not going to want graph checking unless debugging
              if (have_option(trim(zoltan_global_base_option_path) // "/zoltan_debug/graph_checking")) then
                 call get_option(trim(zoltan_global_base_option_path) // "/zoltan_debug/graph_checking", graph_checking_level)
@@ -611,6 +624,7 @@
           ! Use ParMETIS by default on the final adapt iteration
           ierr = Zoltan_Set_Param(zz, "LB_METHOD", "GRAPH"); assert(ierr == ZOLTAN_OK)
           ierr = Zoltan_Set_Param(zz, "GRAPH_PACKAGE", "PARMETIS"); assert(ierr == ZOLTAN_OK)
+          ewrite(3,*) "No final partitioner option set, defaulting to using ParMETIS."
        end if
 
     end if
@@ -620,16 +634,20 @@
     ! iteration to produce a load balanced partitioning
     if (final_adapt_iteration) then
        ierr = Zoltan_Set_Param(zz, "LB_APPROACH", "PARTITION"); assert(ierr == ZOLTAN_OK)
+       ewrite(3,*) "Setting partitioning approach to PARTITION."
        if (have_option(trim(zoltan_global_base_option_path) // "/final_partitioner/metis") .OR. &
           & (.NOT.(have_option(trim(zoltan_global_base_option_path) // "/final_partitioner")))) then
           ! chosen to match what Sam uses
           ierr = Zoltan_Set_Param(zz, "PARMETIS_METHOD", "PartKway"); assert(ierr == ZOLTAN_OK)
+          ewrite(3,*) "Setting ParMETIS method to PartKway."
        end if
     else
        ierr = Zoltan_Set_Param(zz, "LB_APPROACH", "REPARTITION"); assert(ierr == ZOLTAN_OK)
+       ewrite(3,*) "Setting partitioning approach to REPARTITION."
        if (have_option(trim(zoltan_global_base_option_path) // "/partitioner/metis"))  then
           ! chosen to match what Sam uses
           ierr = Zoltan_Set_Param(zz, "PARMETIS_METHOD", "AdaptiveRepart"); assert(ierr == ZOLTAN_OK)
+          ewrite(3,*) "Setting ParMETIS method to AdaptiveRepart."
           ierr = Zoltan_Set_Param(zz, "PARMETIS_ITR", "100000.0"); assert(ierr == ZOLTAN_OK)
        end if
     end if
 
=== modified file 'tools/Flredecomp.F90'
--- tools/Flredecomp.F90	2011-09-20 13:54:07 +0000
+++ tools/Flredecomp.F90	2012-04-03 09:59:20 +0000
@@ -76,7 +76,6 @@
 #ifdef HAVE_ZOLTAN
   real(zoltan_float) :: ver
   integer(zoltan_int) :: ierr
-  real :: global_min_quality
 
   ierr = Zoltan_Initialize(ver)  
   assert(ierr == ZOLTAN_OK)
@@ -170,7 +169,7 @@
   no_active_processes = target_nprocs
   
 #ifdef HAVE_ZOLTAN
-  call zoltan_drive(state, .true., global_min_quality, initialise_fields=.true., ignore_extrusion=skip_initial_extrusion, &
+  call zoltan_drive(state, .true., initialise_fields=.true., ignore_extrusion=skip_initial_extrusion, &
      & flredecomping=.true., input_procs = input_nprocs, target_procs = target_nprocs)
 #else
   call strip_level_2_halo(state, initialise_fields=.true.)