Fluidity

Merge lp:~fluidity-core/fluidity/zoltan into lp:fluidity

  1. zoltan
  2. Merge into dev-trunk
Proposed by Paul Woodhams
Status: Merged
Merged at revision: 3973
Proposed branch: lp:~fluidity-core/fluidity/zoltan
Merge into: lp:fluidity
Diff against target: 233 lines (+54/-31)
3 files modified
assemble/Adapt_State.F90 (+19/-13)
assemble/Zoltan_integration.F90 (+34/-16)
tools/Flredecomp.F90 (+1/-2)
To merge this branch: bzr merge lp:~fluidity-core/fluidity/zoltan
Reviewer Review Type Date Requested Status
Jon Hill Approve
Review via email: mp+85829@code.launchpad.net

Description of the change

One of the MPI_AllReduces is only necessary when using additional adapt iterations. This commit fixes the code so that it only does the collective communication when the additional adapt iterations option is being used.

Also adds some more logging messages detailing the selection of partitioners.

To post a comment you must log in.
Revision history for this message
Jon Hill (jon-hill) wrote :

Looks good.

review: Approve
lp:~fluidity-core/fluidity/zoltan updated
3449. By Jon Hill

Merge from trunk

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'assemble/Adapt_State.F90'
2--- assemble/Adapt_State.F90 2011-09-28 13:38:45 +0000
3+++ assemble/Adapt_State.F90 2012-04-03 09:59:20 +0000
4@@ -1003,13 +1003,9 @@
5 call get_option("/mesh_adaptivity/hr_adaptivity/zoltan_options/element_quality_cutoff", &
6 & quality_tolerance, default = 0.6)
7
8- if (max_adapt_iteration .ne. 1) then
9- zoltan_min_adapt_iterations = adapt_iterations()
10- zoltan_max_adapt_iterations = zoltan_min_adapt_iterations + zoltan_additional_adapt_iterations
11- else
12- zoltan_min_adapt_iterations = adapt_iterations()
13- zoltan_max_adapt_iterations = zoltan_min_adapt_iterations
14- end if
15+ zoltan_min_adapt_iterations = adapt_iterations()
16+ zoltan_max_adapt_iterations = zoltan_min_adapt_iterations + zoltan_additional_adapt_iterations
17+
18 #endif
19
20 finished_adapting = .false.
21@@ -1238,7 +1234,11 @@
22 ! call zoltan now but we need to pass in both the 2d metric (metric) and the 3d full metric (full_metric)
23 ! the first is needed to define the element qualities while the second must be interpolated to the newly
24 ! decomposed mesh
25- call zoltan_drive(states, final_adapt_iteration, global_min_quality, metric = metric, full_metric = full_metric)
26+ if (zoltan_additional_adapt_iterations .gt. 0) then
27+ call zoltan_drive(states, final_adapt_iteration, global_min_quality = global_min_quality, metric = metric, full_metric = full_metric)
28+ else
29+ call zoltan_drive(states, final_adapt_iteration, metric = metric, full_metric = full_metric)
30+ end if
31 default_stat%zoltan_drive_call=.true.
32
33 ! now we can deallocate the horizontal metric and point metric back at the full metric again
34@@ -1246,7 +1246,11 @@
35 metric = full_metric
36 else
37
38- call zoltan_drive(states, final_adapt_iteration, global_min_quality, metric = metric)
39+ if (zoltan_additional_adapt_iterations .gt. 0) then
40+ call zoltan_drive(states, final_adapt_iteration, global_min_quality = global_min_quality, metric = metric)
41+ else
42+ call zoltan_drive(states, final_adapt_iteration, metric = metric)
43+ end if
44 default_stat%zoltan_drive_call=.true.
45
46 end if
47@@ -1303,10 +1307,12 @@
48 ewrite(2,*) "The next iteration will be final adapt iteration else we'll go over the maximum adapt iterations."
49 end if
50
51- if (global_min_quality .le. quality_tolerance) then
52- ewrite(-1,*) "Mesh contains elements with quality below element quality tolerance. May need to increase number of adapt iterations to ensure good quality mesh."
53- ewrite(-1,*) "min_quality = ", global_min_quality
54- ewrite(-1,*) "quality_tolerance = ", quality_tolerance
55+ if (zoltan_additional_adapt_iterations .gt. 0) then
56+ if (global_min_quality .le. quality_tolerance) then
57+ ewrite(-1,*) "Mesh contains elements with quality below element quality tolerance. May need to increase number of adapt iterations to ensure good quality mesh."
58+ ewrite(-1,*) "min_quality = ", global_min_quality
59+ ewrite(-1,*) "quality_tolerance = ", quality_tolerance
60+ end if
61 end if
62
63 final_adapt_iteration = .true.
64
65=== modified file 'assemble/Zoltan_integration.F90'
66--- assemble/Zoltan_integration.F90 2011-09-26 17:04:31 +0000
67+++ assemble/Zoltan_integration.F90 2012-04-03 09:59:20 +0000
68@@ -60,7 +60,7 @@
69
70 type(state_type), dimension(:), intent(inout), target :: states
71 logical, intent(in) :: final_adapt_iteration
72- real, intent(out) :: global_min_quality
73+ real, intent(out), optional :: global_min_quality
74 ! the metric is the metric we base the quality functions on
75 type(tensor_field), intent(inout), optional :: metric
76 ! the full_metric is the metric we need to interpolate
77@@ -152,23 +152,27 @@
78 & p1_num_export, p1_export_global_ids, p1_export_local_ids, p1_export_procs, &
79 & load_imbalance_tolerance, flredecomp, flredecomp_input_procs, flredecomp_target_procs)
80
81- if (.NOT. final_adapt_iteration) then
82
83- if (.NOT. zoltan_global_calculated_local_min_quality) then
84- FLAbort("Minimum element quality was not calculated during the load balance call.")
85+ ! Only calculate the global minimum element quality if additional adapt iterations are being used
86+ if (present(global_min_quality)) then
87+ if (.NOT. final_adapt_iteration) then
88+
89+ if (.NOT. zoltan_global_calculated_local_min_quality) then
90+ FLAbort("Minimum element quality was not calculated during the load balance call.")
91+ end if
92+
93+ ! calculate the global minimum element quality
94+ call mpi_allreduce(zoltan_global_local_min_quality, global_min_quality, 1, getPREAL(), &
95+ & MPI_MIN, MPI_COMM_FEMTOOLS, ierr)
96+ assert(ierr == MPI_SUCCESS)
97+
98+ ewrite(1,*) "local minimum element quality = ", zoltan_global_local_min_quality
99+ ewrite(1,*) "global minimum element quality = ", global_min_quality
100+
101+ else
102+ ! On final iteration we do not calculate the minimum element quality
103+ global_min_quality = 1.0
104 end if
105-
106- ! calculate the global minimum element quality
107- call mpi_allreduce(zoltan_global_local_min_quality, global_min_quality, 1, getPREAL(), &
108- & MPI_MIN, MPI_COMM_FEMTOOLS, ierr)
109- assert(ierr == MPI_SUCCESS)
110-
111- ewrite(1,*) "local minimum element quality = ", zoltan_global_local_min_quality
112- ewrite(1,*) "global minimum element quality = ", global_min_quality
113-
114- else
115- ! On final iteration we do not calculate the minimum element quality
116- global_min_quality = 1.0
117 end if
118
119 if (changes .eqv. .false.) then
120@@ -524,6 +528,7 @@
121 if (have_option(trim(zoltan_global_base_option_path) // "/partitioner/metis")) then
122 ierr = Zoltan_Set_Param(zz, "LB_METHOD", "GRAPH"); assert(ierr == ZOLTAN_OK)
123 ierr = Zoltan_Set_Param(zz, "GRAPH_PACKAGE", "PARMETIS"); assert(ierr == ZOLTAN_OK)
124+ ewrite(3,*) "Setting the partitioner to be ParMETIS."
125 ! turn off graph checking unless debugging, this was filling the error file with Zoltan warnings
126 if (have_option(trim(zoltan_global_base_option_path) // "/zoltan_debug/graph_checking")) then
127 call get_option(trim(zoltan_global_base_option_path) // "/zoltan_debug/graph_checking", graph_checking_level)
128@@ -540,9 +545,11 @@
129 if (trim(method) == "graph") then
130 ierr = Zoltan_Set_Param(zz, "LB_METHOD", "GRAPH"); assert(ierr == ZOLTAN_OK)
131 ierr = Zoltan_Set_Param(zz, "GRAPH_PACKAGE", "PHG"); assert(ierr == ZOLTAN_OK)
132+ ewrite(3,*) "Setting the partitioner to be Zoltan-Graph."
133 else if (trim(method) == "hypergraph") then
134 ierr = Zoltan_Set_Param(zz, "LB_METHOD", "HYPERGRAPH"); assert(ierr == ZOLTAN_OK)
135 ierr = Zoltan_Set_Param(zz, "HYPERGRAPH_PACKAGE", "PHG"); assert(ierr == ZOLTAN_OK)
136+ ewrite(3,*) "Setting the partitioner to be Zoltan-Hypergraph."
137 end if
138
139 end if
140@@ -550,6 +557,7 @@
141 if (have_option(trim(zoltan_global_base_option_path) // "/partitioner/scotch")) then
142 ierr = Zoltan_Set_Param(zz, "LB_METHOD", "GRAPH"); assert(ierr == ZOLTAN_OK)
143 ierr = Zoltan_Set_Param(zz, "GRAPH_PACKAGE", "SCOTCH"); assert(ierr == ZOLTAN_OK)
144+ ewrite(3,*) "Setting the partitioner to be Scotch."
145 ! Probably not going to want graph checking unless debugging
146 if (have_option(trim(zoltan_global_base_option_path) // "/zoltan_debug/graph_checking")) then
147 call get_option(trim(zoltan_global_base_option_path) // "/zoltan_debug/graph_checking", graph_checking_level)
148@@ -563,6 +571,7 @@
149 ! Use the Zoltan graph partitioner by default
150 ierr = Zoltan_Set_Param(zz, "LB_METHOD", "GRAPH"); assert(ierr == ZOLTAN_OK)
151 ierr = Zoltan_Set_Param(zz, "GRAPH_PACKAGE", "PHG"); assert(ierr == ZOLTAN_OK)
152+ ewrite(3,*) "No partitioner option set, defaulting to using Zoltan-Graph."
153 end if
154
155 else
156@@ -572,6 +581,7 @@
157 if (have_option(trim(zoltan_global_base_option_path) // "/final_partitioner/metis")) then
158 ierr = Zoltan_Set_Param(zz, "LB_METHOD", "GRAPH"); assert(ierr == ZOLTAN_OK)
159 ierr = Zoltan_Set_Param(zz, "GRAPH_PACKAGE", "PARMETIS"); assert(ierr == ZOLTAN_OK)
160+ ewrite(3,*) "Setting the final partitioner to be ParMETIS."
161 ! turn off graph checking unless debugging, this was filling the error file with Zoltan warnings
162 if (have_option(trim(zoltan_global_base_option_path) // "/zoltan_debug/graph_checking")) then
163 call get_option(trim(zoltan_global_base_option_path) // "/zoltan_debug/graph_checking", graph_checking_level)
164@@ -588,9 +598,11 @@
165 if (trim(method) == "graph") then
166 ierr = Zoltan_Set_Param(zz, "LB_METHOD", "GRAPH"); assert(ierr == ZOLTAN_OK)
167 ierr = Zoltan_Set_Param(zz, "GRAPH_PACKAGE", "PHG"); assert(ierr == ZOLTAN_OK)
168+ ewrite(3,*) "Setting the final partitioner to be Zoltan-Graph."
169 else if (trim(method) == "hypergraph") then
170 ierr = Zoltan_Set_Param(zz, "LB_METHOD", "HYPERGRAPH"); assert(ierr == ZOLTAN_OK)
171 ierr = Zoltan_Set_Param(zz, "HYPERGRAPH_PACKAGE", "PHG"); assert(ierr == ZOLTAN_OK)
172+ ewrite(3,*) "Setting the final partitioner to be Zoltan-Hypergraph."
173 end if
174
175 end if
176@@ -598,6 +610,7 @@
177 if (have_option(trim(zoltan_global_base_option_path) // "/final_partitioner/scotch")) then
178 ierr = Zoltan_Set_Param(zz, "LB_METHOD", "GRAPH"); assert(ierr == ZOLTAN_OK)
179 ierr = Zoltan_Set_Param(zz, "GRAPH_PACKAGE", "SCOTCH"); assert(ierr == ZOLTAN_OK)
180+ ewrite(3,*) "Setting the final partitioner to be Scotch."
181 ! Probably not going to want graph checking unless debugging
182 if (have_option(trim(zoltan_global_base_option_path) // "/zoltan_debug/graph_checking")) then
183 call get_option(trim(zoltan_global_base_option_path) // "/zoltan_debug/graph_checking", graph_checking_level)
184@@ -611,6 +624,7 @@
185 ! Use ParMETIS by default on the final adapt iteration
186 ierr = Zoltan_Set_Param(zz, "LB_METHOD", "GRAPH"); assert(ierr == ZOLTAN_OK)
187 ierr = Zoltan_Set_Param(zz, "GRAPH_PACKAGE", "PARMETIS"); assert(ierr == ZOLTAN_OK)
188+ ewrite(3,*) "No final partitioner option set, defaulting to using ParMETIS."
189 end if
190
191 end if
192@@ -620,16 +634,20 @@
193 ! iteration to produce a load balanced partitioning
194 if (final_adapt_iteration) then
195 ierr = Zoltan_Set_Param(zz, "LB_APPROACH", "PARTITION"); assert(ierr == ZOLTAN_OK)
196+ ewrite(3,*) "Setting partitioning approach to PARTITION."
197 if (have_option(trim(zoltan_global_base_option_path) // "/final_partitioner/metis") .OR. &
198 & (.NOT.(have_option(trim(zoltan_global_base_option_path) // "/final_partitioner")))) then
199 ! chosen to match what Sam uses
200 ierr = Zoltan_Set_Param(zz, "PARMETIS_METHOD", "PartKway"); assert(ierr == ZOLTAN_OK)
201+ ewrite(3,*) "Setting ParMETIS method to PartKway."
202 end if
203 else
204 ierr = Zoltan_Set_Param(zz, "LB_APPROACH", "REPARTITION"); assert(ierr == ZOLTAN_OK)
205+ ewrite(3,*) "Setting partitioning approach to REPARTITION."
206 if (have_option(trim(zoltan_global_base_option_path) // "/partitioner/metis")) then
207 ! chosen to match what Sam uses
208 ierr = Zoltan_Set_Param(zz, "PARMETIS_METHOD", "AdaptiveRepart"); assert(ierr == ZOLTAN_OK)
209+ ewrite(3,*) "Setting ParMETIS method to AdaptiveRepart."
210 ierr = Zoltan_Set_Param(zz, "PARMETIS_ITR", "100000.0"); assert(ierr == ZOLTAN_OK)
211 end if
212 end if
213
214=== modified file 'tools/Flredecomp.F90'
215--- tools/Flredecomp.F90 2011-09-20 13:54:07 +0000
216+++ tools/Flredecomp.F90 2012-04-03 09:59:20 +0000
217@@ -76,7 +76,6 @@
218 #ifdef HAVE_ZOLTAN
219 real(zoltan_float) :: ver
220 integer(zoltan_int) :: ierr
221- real :: global_min_quality
222
223 ierr = Zoltan_Initialize(ver)
224 assert(ierr == ZOLTAN_OK)
225@@ -170,7 +169,7 @@
226 no_active_processes = target_nprocs
227
228 #ifdef HAVE_ZOLTAN
229- call zoltan_drive(state, .true., global_min_quality, initialise_fields=.true., ignore_extrusion=skip_initial_extrusion, &
230+ call zoltan_drive(state, .true., initialise_fields=.true., ignore_extrusion=skip_initial_extrusion, &
231 & flredecomping=.true., input_procs = input_nprocs, target_procs = target_nprocs)
232 #else
233 call strip_level_2_halo(state, initialise_fields=.true.)