deal.II version 9.7.0
\(\newcommand{\dealvcentcolon}{\mathrel{\mathop{:}}}\) \(\newcommand{\dealcoloneq}{\dealvcentcolon\mathrel{\mkern-1.2mu}=}\) \(\newcommand{\jump}[1]{\left[\!\left[ #1 \right]\!\right]}\) \(\newcommand{\average}[1]{\left\{\!\left\{ #1 \right\}\!\right\}}\)
Loading...
Searching...
No Matches
task_info.cc
Go to the documentation of this file.
1// ------------------------------------------------------------------------
2//
3// SPDX-License-Identifier: LGPL-2.1-or-later
4// Copyright (C) 2018 - 2025 by the deal.II authors
5//
6// This file is part of the deal.II library.
7//
8// Part of the source code is dual licensed under Apache-2.0 WITH
9// LLVM-exception OR LGPL-2.1-or-later. Detailed license information
10// governing the source code and code contributions can be found in
11// LICENSE.md and CONTRIBUTING.md at the top level directory of deal.II.
12//
13// ------------------------------------------------------------------------
14
15
16#include <deal.II/base/conditional_ostream.h>
17#include <deal.II/base/memory_consumption.h>
18#include <deal.II/base/mpi.h>
19#include <deal.II/base/multithread_info.h>
20#include <deal.II/base/parallel.h>
21#include <deal.II/base/utilities.h>
22
23#include <deal.II/lac/dynamic_sparsity_pattern.h>
24
25#include <deal.II/matrix_free/task_info.h>
26#include <deal.II/matrix_free/util.h>
27
28
29#ifdef DEAL_II_WITH_TBB
30# include <tbb/blocked_range.h>
31# include <tbb/parallel_for.h>
32# include <tbb/task.h>
33# ifndef DEAL_II_TBB_WITH_ONEAPI
34# include <tbb/task_scheduler_init.h>
35# endif
36#endif
37
38#include <iostream>
39#include <set>
40
41//
42// TBB with oneAPI API has deprecated and removed the
43// <code>tbb::tasks</code> backend. With this it is no longer possible to
44// compile the following code that builds a directed acyclic graph (DAG) of
45// (thread parallel) tasks without a major porting effort. It turned out
46// that such a dynamic handling of dependencies and structures is not as
47// competitive as initially assumed. Consequently, this part of the matrix
48// free infrastructure has seen less attention than the rest over the last
49// years and is (presumably) not used that often.
50//
51// In case of detected oneAPI backend we simply disable threading in the
52// matrix free backend for now.
53//
54// Matthias Maier, Martin Kronbichler, 2021
55//
56
57DEAL_II_NAMESPACE_OPEN
58
59
60
61/*-------------------- Implementation of the matrix-free loop --------------*/
62namespace internal
63{
64 namespace MatrixFreeFunctions
65 {
66#if defined(DEAL_II_WITH_TBB) && !defined(DEAL_II_TBB_WITH_ONEAPI)
67
68 // This defines the TBB data structures that are needed to schedule the
69 // partition-partition variant
70
71 namespace partition
72 {
73 class ActualCellWork
74 {
75 public:
76 ActualCellWork(MFWorkerInterface **worker_pointer,
77 const unsigned int partition,
78 const TaskInfo &task_info)
79 : worker(nullptr)
80 , worker_pointer(worker_pointer)
81 , partition(partition)
82 , task_info(task_info)
83 {}
84
85 ActualCellWork(MFWorkerInterface &worker,
86 const unsigned int partition,
87 const TaskInfo &task_info)
88 : worker(&worker)
89 , worker_pointer(nullptr)
90 , partition(partition)
91 , task_info(task_info)
92 {}
93
94 void
95 operator()() const
96 {
97 MFWorkerInterface *used_worker =
98 worker != nullptr ? worker : *worker_pointer;
99 Assert(used_worker != nullptr, ExcInternalError());
100 used_worker->cell(partition);
101
102 if (task_info.face_partition_data.empty() == false)
103 {
104 used_worker->face(partition);
105 used_worker->boundary(partition);
106 }
107 }
108
109 private:
110 MFWorkerInterface *worker;
111 MFWorkerInterface **worker_pointer;
112 const unsigned int partition;
113 const TaskInfo &task_info;
114 };
115
116 class CellWork : public tbb::task
117 {
118 public:
119 CellWork(MFWorkerInterface &worker,
120 const unsigned int partition,
121 const TaskInfo &task_info,
122 const bool is_blocked)
123 : dummy(nullptr)
124 , work(worker, partition, task_info)
125 , is_blocked(is_blocked)
126 {}
127
128 tbb::task *
129 execute() override
130 {
131 work();
132
133 if (is_blocked == true)
134 tbb::empty_task::spawn(*dummy);
135 return nullptr;
136 }
137
138 tbb::empty_task *dummy;
139
140 private:
141 ActualCellWork work;
142 const bool is_blocked;
143 };
144
145
146
147 class PartitionWork : public tbb::task
148 {
149 public:
150 PartitionWork(MFWorkerInterface &function_in,
151 const unsigned int partition_in,
152 const TaskInfo &task_info_in,
153 const bool is_blocked_in = false)
154 : dummy(nullptr)
155 , function(function_in)
156 , partition(partition_in)
157 , task_info(task_info_in)
158 , is_blocked(is_blocked_in)
159 {}
160
161 tbb::task *
162 execute() override
163 {
164 tbb::empty_task *root =
165 new (tbb::task::allocate_root()) tbb::empty_task;
166 const unsigned int evens = task_info.partition_evens[partition];
167 const unsigned int odds = task_info.partition_odds[partition];
168 const unsigned int n_blocked_workers =
169 task_info.partition_n_blocked_workers[partition];
170 const unsigned int n_workers =
171 task_info.partition_n_workers[partition];
172 std::vector<CellWork *> worker(n_workers);
173 std::vector<CellWork *> blocked_worker(n_blocked_workers);
174
175 root->set_ref_count(evens + 1);
176 for (unsigned int j = 0; j < evens; ++j)
177 {
178 worker[j] = new (root->allocate_child())
179 CellWork(function,
180 task_info.partition_row_index[partition] + 2 * j,
181 task_info,
182 false);
183 if (j > 0)
184 {
185 worker[j]->set_ref_count(2);
186 blocked_worker[j - 1]->dummy =
187 new (worker[j]->allocate_child()) tbb::empty_task;
188 tbb::task::spawn(*blocked_worker[j - 1]);
189 }
190 else
191 worker[j]->set_ref_count(1);
192 if (j < evens - 1)
193 {
194 blocked_worker[j] = new (worker[j]->allocate_child())
195 CellWork(function,
196 task_info.partition_row_index[partition] + 2 * j +
197 1,
198 task_info,
199 true);
200 }
201 else
202 {
203 if (odds == evens)
204 {
205 worker[evens] = new (worker[j]->allocate_child())
206 CellWork(function,
207 task_info.partition_row_index[partition] +
208 2 * j + 1,
209 task_info,
210 false);
211 tbb::task::spawn(*worker[evens]);
212 }
213 else
214 {
215 tbb::empty_task *child =
216 new (worker[j]->allocate_child()) tbb::empty_task();
217 tbb::task::spawn(*child);
218 }
219 }
220 }
221
222 root->wait_for_all();
223 root->destroy(*root);
224 if (is_blocked == true)
225 tbb::empty_task::spawn(*dummy);
226 return nullptr;
227 }
228
229 tbb::empty_task *dummy;
230
231 private:
232 MFWorkerInterface &function;
233 const unsigned int partition;
234 const TaskInfo &task_info;
235 const bool is_blocked;
236 };
237
238 } // end of namespace partition
239
240
241
242 namespace color
243 {
244 class CellWork
245 {
246 public:
247 CellWork(MFWorkerInterface &worker_in,
248 const TaskInfo &task_info_in,
249 const unsigned int partition_in)
250 : worker(worker_in)
251 , task_info(task_info_in)
252 , partition(partition_in)
253 {}
254
255 void
256 operator()(const tbb::blocked_range<unsigned int> &r) const
257 {
258 const unsigned int start_index =
259 task_info.cell_partition_data[partition] +
260 task_info.block_size * r.begin();
261 const unsigned int end_index =
262 std::min(start_index + task_info.block_size * (r.end() - r.begin()),
263 task_info.cell_partition_data[partition + 1]);
264 worker.cell(std::make_pair(start_index, end_index));
265
266 if (task_info.face_partition_data.empty() == false)
267 {
268 AssertThrow(false, ExcNotImplemented());
269 }
270 }
271
272 private:
273 MFWorkerInterface &worker;
274 const TaskInfo &task_info;
275 const unsigned int partition;
276 };
277
278
279
280 class PartitionWork : public tbb::task
281 {
282 public:
283 PartitionWork(MFWorkerInterface &worker_in,
284 const unsigned int partition_in,
285 const TaskInfo &task_info_in,
286 const bool is_blocked_in)
287 : dummy(nullptr)
288 , worker(worker_in)
289 , partition(partition_in)
290 , task_info(task_info_in)
291 , is_blocked(is_blocked_in)
292 {}
293
294 tbb::task *
295 execute() override
296 {
297 const unsigned int n_chunks =
298 (task_info.cell_partition_data[partition + 1] -
299 task_info.cell_partition_data[partition] + task_info.block_size -
300 1) /
301 task_info.block_size;
302 parallel_for(tbb::blocked_range<unsigned int>(0, n_chunks, 1),
303 CellWork(worker, task_info, partition));
304 if (is_blocked == true)
305 tbb::empty_task::spawn(*dummy);
306 return nullptr;
307 }
308
309 tbb::empty_task *dummy;
310
311 private:
312 MFWorkerInterface &worker;
313 const unsigned int partition;
314 const TaskInfo &task_info;
315 const bool is_blocked;
316 };
317
318 } // end of namespace color
319
320
321
322 class MPICommunication : public tbb::task
323 {
324 public:
325 MPICommunication(MFWorkerInterface &worker_in, const bool do_compress)
326 : worker(worker_in)
327 , do_compress(do_compress)
328 {}
329
330 tbb::task *
331 execute() override
332 {
333 if (do_compress == false)
334 worker.vector_update_ghosts_finish();
335 else
336 worker.vector_compress_start();
337 return nullptr;
338 }
339
340 private:
341 MFWorkerInterface &worker;
342 const bool do_compress;
343 };
344
345#endif // DEAL_II_WITH_TBB
346
347
348
349 void
350 TaskInfo::loop(MFWorkerInterface &funct) const
351 {
352 // If we use thread parallelism, we do not currently support to schedule
353 // pieces of updates within the loop, so this index will collect all
354 // calls in that case and work like a single complete loop over all
355 // cells
356 if (scheme != none)
357 funct.cell_loop_pre_range(numbers::invalid_unsigned_int);
358 else
359 funct.cell_loop_pre_range(
360 partition_row_index[partition_row_index.size() - 2]);
361
362 funct.vector_update_ghosts_start();
363
364#if defined(DEAL_II_WITH_TBB) && !defined(DEAL_II_TBB_WITH_ONEAPI)
365
366 if (scheme != none)
367 {
368 funct.zero_dst_vector_range(numbers::invalid_unsigned_int);
369 if (scheme == partition_partition && evens > 0)
370 {
371 tbb::empty_task *root =
372 new (tbb::task::allocate_root()) tbb::empty_task;
373 root->set_ref_count(evens + 1);
374 std::vector<partition::PartitionWork *> worker(n_workers);
375 std::vector<partition::PartitionWork *> blocked_worker(
376 n_blocked_workers);
377 MPICommunication *worker_compr =
378 new (root->allocate_child()) MPICommunication(funct, true);
379 worker_compr->set_ref_count(1);
380 for (unsigned int j = 0; j < evens; ++j)
381 {
382 if (j > 0)
383 {
384 worker[j] = new (root->allocate_child())
385 partition::PartitionWork(funct, 2 * j, *this, false);
386 worker[j]->set_ref_count(2);
387 blocked_worker[j - 1]->dummy =
388 new (worker[j]->allocate_child()) tbb::empty_task;
389 tbb::task::spawn(*blocked_worker[j - 1]);
390 }
391 else
392 {
393 worker[j] = new (worker_compr->allocate_child())
394 partition::PartitionWork(funct, 2 * j, *this, false);
395 worker[j]->set_ref_count(2);
396 MPICommunication *worker_dist =
397 new (worker[j]->allocate_child())
398 MPICommunication(funct, false);
399 tbb::task::spawn(*worker_dist);
400 }
401 if (j < evens - 1)
402 {
403 blocked_worker[j] = new (worker[j]->allocate_child())
404 partition::PartitionWork(funct, 2 * j + 1, *this, true);
405 }
406 else
407 {
408 if (odds == evens)
409 {
410 worker[evens] = new (worker[j]->allocate_child())
411 partition::PartitionWork(funct,
412 2 * j + 1,
413 *this,
414 false);
415 tbb::task::spawn(*worker[evens]);
416 }
417 else
418 {
419 tbb::empty_task *child =
420 new (worker[j]->allocate_child()) tbb::empty_task();
421 tbb::task::spawn(*child);
422 }
423 }
424 }
425
426 root->wait_for_all();
427 root->destroy(*root);
428 }
429 else if (scheme == partition_partition)
430 {
431 // catch the case of empty partition list: we still need to call
432 // the vector communication routines to clean up and initiate
433 // things
434 funct.vector_update_ghosts_finish();
435 funct.vector_compress_start();
436 }
437 else // end of partition-partition, start of partition-color
438 {
439 // check whether there is only one partition. if not, build up the
440 // tree of partitions
441 if (odds > 0)
442 {
443 tbb::empty_task *root =
444 new (tbb::task::allocate_root()) tbb::empty_task;
445 root->set_ref_count(evens + 1);
446 const unsigned int n_blocked_workers =
447 odds - (odds + evens + 1) % 2;
448 const unsigned int n_workers =
449 cell_partition_data.size() - 1 - n_blocked_workers;
450 std::vector<color::PartitionWork *> worker(n_workers);
451 std::vector<color::PartitionWork *> blocked_worker(
452 n_blocked_workers);
453 unsigned int worker_index = 0, slice_index = 0;
454 int spawn_index_child = -2;
455 MPICommunication *worker_compr =
456 new (root->allocate_child()) MPICommunication(funct, true);
457 worker_compr->set_ref_count(1);
458 for (unsigned int part = 0;
459 part < partition_row_index.size() - 1;
460 part++)
461 {
462 if (part == 0)
463 worker[worker_index] =
464 new (worker_compr->allocate_child())
465 color::PartitionWork(funct,
466 slice_index,
467 *this,
468 false);
469 else
470 worker[worker_index] = new (root->allocate_child())
471 color::PartitionWork(funct,
472 slice_index,
473 *this,
474 false);
475 ++slice_index;
476 for (; slice_index < partition_row_index[part + 1];
477 slice_index++)
478 {
479 worker[worker_index]->set_ref_count(1);
480 ++worker_index;
481 worker[worker_index] =
482 new (worker[worker_index - 1]->allocate_child())
483 color::PartitionWork(funct,
484 slice_index,
485 *this,
486 false);
487 }
488 worker[worker_index]->set_ref_count(2);
489 if (part > 0)
490 {
491 blocked_worker[(part - 1) / 2]->dummy =
492 new (worker[worker_index]->allocate_child())
493 tbb::empty_task;
494 ++worker_index;
495 if (spawn_index_child == -1)
496 tbb::task::spawn(*blocked_worker[(part - 1) / 2]);
497 else
498 {
499 Assert(spawn_index_child >= 0,
500 ExcInternalError());
501 tbb::task::spawn(*worker[spawn_index_child]);
502 }
503 spawn_index_child = -2;
504 }
505 else
506 {
507 MPICommunication *worker_dist =
508 new (worker[worker_index]->allocate_child())
509 MPICommunication(funct, false);
510 tbb::task::spawn(*worker_dist);
511 ++worker_index;
512 }
513 part += 1;
514 if (part < partition_row_index.size() - 1)
515 {
516 if (part < partition_row_index.size() - 2)
517 {
518 blocked_worker[part / 2] =
519 new (worker[worker_index - 1]->allocate_child())
520 color::PartitionWork(funct,
521 slice_index,
522 *this,
523 true);
524 ++slice_index;
525 if (slice_index < partition_row_index[part + 1])
526 {
527 blocked_worker[part / 2]->set_ref_count(1);
528 worker[worker_index] = new (
529 blocked_worker[part / 2]->allocate_child())
530 color::PartitionWork(funct,
531 slice_index,
532 *this,
533 false);
534 ++slice_index;
535 }
536 else
537 {
538 spawn_index_child = -1;
539 continue;
540 }
541 }
542 for (; slice_index < partition_row_index[part + 1];
543 slice_index++)
544 {
545 if (slice_index > partition_row_index[part])
546 {
547 worker[worker_index]->set_ref_count(1);
548 ++worker_index;
549 }
550 worker[worker_index] =
551 new (worker[worker_index - 1]->allocate_child())
552 color::PartitionWork(funct,
553 slice_index,
554 *this,
555 false);
556 }
557 spawn_index_child = worker_index;
558 ++worker_index;
559 }
560 else
561 {
562 tbb::empty_task *final =
563 new (worker[worker_index - 1]->allocate_child())
564 tbb::empty_task;
565 tbb::task::spawn(*final);
566 spawn_index_child = worker_index - 1;
567 }
568 }
569 if (evens == odds)
570 {
571 Assert(spawn_index_child >= 0, ExcInternalError());
572 tbb::task::spawn(*worker[spawn_index_child]);
573 }
574 root->wait_for_all();
575 root->destroy(*root);
576 }
577 // case when we only have one partition: this is the usual
578 // coloring scheme, and we just schedule a parallel for loop for
579 // each color
580 else
581 {
582 Assert(evens <= 1, ExcInternalError());
583 funct.vector_update_ghosts_finish();
584
585 for (unsigned int color = 0; color < partition_row_index[1];
586 ++color)
587 {
588 tbb::empty_task *root =
589 new (tbb::task::allocate_root()) tbb::empty_task;
590 root->set_ref_count(2);
591 color::PartitionWork *worker =
592 new (root->allocate_child())
593 color::PartitionWork(funct, color, *this, false);
594 tbb::empty_task::spawn(*worker);
595 root->wait_for_all();
596 root->destroy(*root);
597 }
598
599 funct.vector_compress_start();
600 }
601 }
602 }
603 else
604#endif
605 // serial loop, go through up to three times and do the MPI transfer at
606 // the beginning/end of the second part
607 {
608 for (unsigned int part = 0; part < partition_row_index.size() - 2;
609 ++part)
610 {
611 if (part == 1)
612 funct.vector_update_ghosts_finish();
613
614 for (unsigned int i = partition_row_index[part];
615 i < partition_row_index[part + 1];
616 ++i)
617 {
618 funct.cell_loop_pre_range(i);
619 funct.zero_dst_vector_range(i);
620 AssertIndexRange(i + 1, cell_partition_data.size());
621 if (cell_partition_data[i + 1] > cell_partition_data[i])
622 {
623 funct.cell(i);
624 }
625
626 if (face_partition_data.empty() == false)
627 {
628 if (face_partition_data[i + 1] > face_partition_data[i])
629 funct.face(i);
630 if (boundary_partition_data[i + 1] >
631 boundary_partition_data[i])
632 funct.boundary(i);
633 }
634 funct.cell_loop_post_range(i);
635 }
636
637 if (part == 1)
638 funct.vector_compress_start();
639 }
640 }
641 funct.vector_compress_finish();
642
643 if (scheme != none)
644 funct.cell_loop_post_range(numbers::invalid_unsigned_int);
645 else
646 funct.cell_loop_post_range(
647 partition_row_index[partition_row_index.size() - 2]);
648 }
649
650
651
652 TaskInfo::TaskInfo()
653 {
654 clear();
655 }
656
657
658
659 void
660 TaskInfo::clear()
661 {
662 n_active_cells = 0;
663 n_ghost_cells = 0;
664 vectorization_length = 1;
665 block_size = 0;
666 n_blocks = 0;
667 scheme = none;
668 partition_row_index.clear();
669 partition_row_index.resize(2);
670 cell_partition_data.clear();
671 face_partition_data.clear();
672 boundary_partition_data.clear();
673 evens = 0;
674 odds = 0;
675 n_blocked_workers = 0;
676 n_workers = 0;
677 partition_evens.clear();
678 partition_odds.clear();
679 partition_n_blocked_workers.clear();
680 partition_n_workers.clear();
681 communicator = MPI_COMM_SELF;
682 my_pid = 0;
683 n_procs = 1;
684 }
685
686
687
688 template <typename StreamType>
689 void
690 TaskInfo::print_memory_statistics(StreamType &out,
691 const std::size_t data_length) const
692 {
693 Utilities::MPI::MinMaxAvg memory_c =
694 Utilities::MPI::min_max_avg(1e-6 * data_length, communicator);
695 if (n_procs < 2)
696 out << memory_c.min;
697 else
698 out << memory_c.min << "/" << memory_c.avg << "/" << memory_c.max;
699 out << " MB" << std::endl;
700 }
701
702
703
704 std::size_t
718
719
720
721 void
722 TaskInfo::make_boundary_cells_divisible(
723 std::vector<unsigned int> &boundary_cells)
724 {
725 // try to make the number of boundary cells divisible by the number of
726 // vectors in vectorization
727 unsigned int fillup_needed =
728 (vectorization_length - boundary_cells.size() % vectorization_length) %
729 vectorization_length;
730 if (fillup_needed > 0 && boundary_cells.size() < n_active_cells)
731 {
732 // fill additional cells into the list of boundary cells to get a
733 // balanced number. Go through the indices successively until we
734 // found enough indices
735 std::vector<unsigned int> new_boundary_cells;
736 new_boundary_cells.reserve(boundary_cells.size());
737
738 unsigned int next_free_slot = 0, bound_index = 0;
739 while (fillup_needed > 0 && bound_index < boundary_cells.size())
740 {
741 if (next_free_slot < boundary_cells[bound_index])
742 {
743 // check if there are enough cells to fill with in the
744 // current slot
745 if (next_free_slot + fillup_needed <=
746 boundary_cells[bound_index])
747 {
748 for (unsigned int j =
749 boundary_cells[bound_index] - fillup_needed;
750 j < boundary_cells[bound_index];
751 ++j)
752 new_boundary_cells.push_back(j);
753 fillup_needed = 0;
754 }
755 // ok, not enough indices, so just take them all up to the
756 // next boundary cell
757 else
758 {
759 for (unsigned int j = next_free_slot;
760 j < boundary_cells[bound_index];
761 ++j)
762 new_boundary_cells.push_back(j);
763 fillup_needed -=
764 boundary_cells[bound_index] - next_free_slot;
765 }
766 }
767 new_boundary_cells.push_back(boundary_cells[bound_index]);
768 next_free_slot = boundary_cells[bound_index] + 1;
769 ++bound_index;
770 }
771 while (fillup_needed > 0 &&
772 (new_boundary_cells.empty() ||
773 new_boundary_cells.back() < n_active_cells - 1))
774 new_boundary_cells.push_back(new_boundary_cells.back() + 1);
775 while (bound_index < boundary_cells.size())
776 new_boundary_cells.push_back(boundary_cells[bound_index++]);
777
778 boundary_cells.swap(new_boundary_cells);
779 }
780
781 // set the number of cells
782 std::sort(boundary_cells.begin(), boundary_cells.end());
783
784 // check that number of boundary cells is divisible by
785 // vectorization_length or that it contains all cells
786 Assert(boundary_cells.size() % vectorization_length == 0 ||
787 boundary_cells.size() == n_active_cells,
788 ExcInternalError());
789 }
790
791
792
793 void
794 TaskInfo::create_blocks_serial(
795 const std::vector<unsigned int> &cells_with_comm,
796 const unsigned int dofs_per_cell,
797 const bool categories_are_hp,
798 const std::vector<unsigned int> &cell_vectorization_categories,
799 const bool cell_vectorization_categories_strict,
800 const std::vector<unsigned int> &parent_relation,
801 std::vector<unsigned int> &renumbering,
802 std::vector<unsigned char> &incompletely_filled_vectorization)
803 {
804 Assert(dofs_per_cell > 0, ExcInternalError());
805 // This function is decomposed into several steps to determine a good
806 // ordering that satisfies the following constraints:
807 // a. Only cells belonging to the same category (or next higher if the
808 // cell_vectorization_categories_strict is false) can be grouped into
809 // the same SIMD batch
810 // b. hp-adaptive computations must form contiguous ranges for the same
811 // degree (category) in cell_partition_data
812 // c. We want to group the cells with the same parent in the same SIMD
813 // lane if possible
814 // d. The cell order should be similar to the initial one
815 // e. Form sets without MPI communication and those with to overlap
816 // communication with computation
817 //
818 // These constraints are satisfied by first grouping by the categories
819 // and, within the groups, to distinguish between cells with a parent
820 // and those without. All of this is set up with batches of cells (with
821 // padding if the size does not match). Then we define a vector of
822 // arrays where we define sorting criteria for the cell batches to
823 // satisfy the items b and d together, split by different parts to
824 // satisfy item e.
825
826 // Give the compiler a chance to detect that vectorization_length is a
827 // power of two, which allows it to replace integer divisions by shifts
828 const unsigned int n_lanes = indicate_power_of_two(vectorization_length);
829
830 // Step 1: find tight map of categories for not taking exceeding amounts
831 // of memory below. Sort the new categories by the numbers in the
832 // old one to ensure we respect the given rules
833 unsigned int n_categories = 1;
834 std::vector<unsigned int> tight_category_map(n_active_cells, 0);
835 if (cell_vectorization_categories.empty() == false)
836 {
837 AssertDimension(cell_vectorization_categories.size(),
838 n_active_cells + n_ghost_cells);
839
840 std::set<unsigned int> used_categories;
841 for (unsigned int i = 0; i < n_active_cells; ++i)
842 used_categories.insert(cell_vectorization_categories[i]);
843 std::vector<unsigned int> used_categories_vector(
844 used_categories.size());
845 n_categories = 0;
846 for (const auto &it : used_categories)
847 used_categories_vector[n_categories++] = it;
848 for (unsigned int i = 0; i < n_active_cells; ++i)
849 {
850 const unsigned int index =
851 std::lower_bound(used_categories_vector.begin(),
852 used_categories_vector.end(),
853 cell_vectorization_categories[i]) -
854 used_categories_vector.begin();
855 AssertIndexRange(index, used_categories_vector.size());
856 tight_category_map[i] = index;
857 }
858 }
859
860 // Step 2: Sort the cells by the category. If we want to fill up the
861 // ranges in vectorization, promote some of the cells to a higher
862 // category
863 std::vector<std::vector<unsigned int>> renumbering_category(n_categories);
864 for (unsigned int i = 0; i < n_active_cells; ++i)
865 renumbering_category[tight_category_map[i]].push_back(i);
866
867 if (cell_vectorization_categories_strict == false && n_categories > 1)
868 for (unsigned int j = n_categories - 1; j > 0; --j)
869 {
870 unsigned int lower_index = j - 1;
871 while ((renumbering_category[j].size() % n_lanes) != 0u)
872 {
873 while (((renumbering_category[j].size() % n_lanes) != 0u) &&
874 !renumbering_category[lower_index].empty())
875 {
876 renumbering_category[j].push_back(
877 renumbering_category[lower_index].back());
878 renumbering_category[lower_index].pop_back();
879 }
880 if (lower_index == 0)
881 break;
882 else
883 --lower_index;
884 }
885 }
886
887 // Step 3: Use the parent relation to find a good grouping of cells. To
888 // do this, we first put cells of each category defined above into two
889 // bins, those which we know can be grouped together by the given parent
890 // relation and those which cannot
891 std::vector<unsigned int> temporary_numbering;
892 temporary_numbering.reserve(n_active_cells +
893 (n_lanes - 1) * n_categories);
894 const unsigned int n_cells_per_parent =
895 std::count(parent_relation.begin(), parent_relation.end(), 0);
896 std::vector<unsigned int> category_size;
897 for (unsigned int j = 0; j < n_categories; ++j)
898 {
899 std::vector<std::pair<unsigned int, unsigned int>> grouped_cells;
900 std::vector<unsigned int> other_cells;
901 for (const unsigned int cell : renumbering_category[j])
902 if (parent_relation.empty() ||
903 parent_relation[cell] == numbers::invalid_unsigned_int)
904 other_cells.push_back(cell);
905 else
906 grouped_cells.emplace_back(parent_relation[cell], cell);
907
908 // Compute the number of cells per group
909 std::sort(grouped_cells.begin(), grouped_cells.end());
910 std::vector<unsigned int> n_cells_per_group;
911 unsigned int length = 0;
912 for (unsigned int i = 0; i < grouped_cells.size(); ++i, ++length)
913 if (i > 0 && grouped_cells[i].first != grouped_cells[i - 1].first)
914 {
915 n_cells_per_group.push_back(length);
916 length = 0;
917 }
918 if (length > 0)
919 n_cells_per_group.push_back(length);
920
921 // Move groups that do not have the complete size (due to
922 // categories) to the 'other_cells'. The cells with correct group
923 // size are immediately appended to the temporary cell numbering
924 auto group_it = grouped_cells.begin();
925 for (const unsigned int length : n_cells_per_group)
926 if (length < n_cells_per_parent)
927 for (unsigned int j = 0; j < length; ++j)
928 other_cells.push_back((group_it++)->second);
929 else
930 {
931 // we should not have more cells in a group than in the first
932 // check we did above
933 AssertDimension(length, n_cells_per_parent);
934 for (unsigned int j = 0; j < length; ++j)
935 temporary_numbering.push_back((group_it++)->second);
936 }
937
938 // Sort the remaining cells and append them as well
939 std::sort(other_cells.begin(), other_cells.end());
940 temporary_numbering.insert(temporary_numbering.end(),
941 other_cells.begin(),
942 other_cells.end());
943
944 while (temporary_numbering.size() % n_lanes != 0)
945 temporary_numbering.push_back(numbers::invalid_unsigned_int);
946
947 category_size.push_back(temporary_numbering.size());
948 }
949
950 // Step 4: Identify the batches with cells marked as "comm"
951 std::vector<bool> batch_with_comm(temporary_numbering.size() / n_lanes,
952 false);
953 std::vector<unsigned int> temporary_numbering_inverse(n_active_cells);
954 for (unsigned int i = 0; i < temporary_numbering.size(); ++i)
955 if (temporary_numbering[i] != numbers::invalid_unsigned_int)
956 temporary_numbering_inverse[temporary_numbering[i]] = i;
957 for (const unsigned int cell : cells_with_comm)
958 batch_with_comm[temporary_numbering_inverse[cell] / n_lanes] = true;
959
960 // Step 5: Sort the batches of cells by their last cell index to get
961 // good locality, assuming that the initial cell order is of good
962 // locality. In case we have hp-calculations with categories, we need to
963 // sort also by the category.
964 std::vector<std::array<unsigned int, 3>> batch_order;
965 std::vector<std::array<unsigned int, 3>> batch_order_comm;
966 for (unsigned int i = 0; i < temporary_numbering.size(); i += n_lanes)
967 {
968 unsigned int max_index = 0;
969 for (unsigned int j = 0; j < n_lanes; ++j)
970 if (temporary_numbering[i + j] < numbers::invalid_unsigned_int)
971 max_index = std::max(temporary_numbering[i + j], max_index);
972 const unsigned int category_hp =
973 categories_are_hp ?
974 std::upper_bound(category_size.begin(), category_size.end(), i) -
975 category_size.begin() :
976 0;
977 const std::array<unsigned int, 3> next{{category_hp, max_index, i}};
978 if (batch_with_comm[i / n_lanes])
979 batch_order_comm.emplace_back(next);
980 else
981 batch_order.emplace_back(next);
982 }
983
984 std::sort(batch_order.begin(), batch_order.end());
985 std::sort(batch_order_comm.begin(), batch_order_comm.end());
986
987 // Step 6: Put the cells with communication in the middle of the cell
988 // range. For the MPI case, we need three groups to enable overlap for
989 // communication and computation (part before comm, part with comm, part
990 // after comm), whereas we need one for the other case. And in each
991 // case, we allow for a slot of "ghosted" cells.
992 std::vector<unsigned int> blocks;
993 if (n_procs == 1)
994 {
995 if (batch_order.empty())
996 std::swap(batch_order_comm, batch_order);
997 Assert(batch_order_comm.empty(), ExcInternalError());
998 partition_row_index.resize(3);
999 blocks = {0, static_cast<unsigned int>(batch_order.size())};
1000 }
1001 else
1002 {
1003 partition_row_index.resize(5);
1004 const unsigned int comm_begin = batch_order.size() / 2;
1005 batch_order.insert(batch_order.begin() + comm_begin,
1006 batch_order_comm.begin(),
1007 batch_order_comm.end());
1008 const unsigned int comm_end = comm_begin + batch_order_comm.size();
1009 const unsigned int end = batch_order.size();
1010 blocks = {0, comm_begin, comm_end, end};
1011 }
1012
1013 // Step 7: sort ghost cells according to the category
1014 std::vector<std::array<unsigned int, 2>> tight_category_map_ghost;
1015
1016 if (cell_vectorization_categories.empty() == false)
1017 {
1018 tight_category_map_ghost.reserve(n_ghost_cells);
1019
1020 std::set<unsigned int> used_categories;
1021 for (unsigned int i = 0; i < n_ghost_cells; ++i)
1022 used_categories.insert(
1023 cell_vectorization_categories[i + n_active_cells]);
1024
1025 std::vector<unsigned int> used_categories_vector(
1026 used_categories.size());
1027 n_categories = 0;
1028 for (const auto &it : used_categories)
1029 used_categories_vector[n_categories++] = it;
1030
1031 std::vector<unsigned int> counters(n_categories, 0);
1032
1033 for (unsigned int i = 0; i < n_ghost_cells; ++i)
1034 {
1035 const unsigned int index =
1036 std::lower_bound(
1037 used_categories_vector.begin(),
1038 used_categories_vector.end(),
1039 cell_vectorization_categories[i + n_active_cells]) -
1040 used_categories_vector.begin();
1041 AssertIndexRange(index, used_categories_vector.size());
1042 tight_category_map_ghost.emplace_back(
1043 std::array<unsigned int, 2>{{index, i}});
1044
1045 // account for padding in the hp and strict case
1046 if (categories_are_hp || cell_vectorization_categories_strict)
1047 counters[index]++;
1048 }
1049
1050 // insert padding
1051 for (unsigned int i = 0; i < counters.size(); ++i)
1052 if (counters[i] % n_lanes != 0)
1053 for (unsigned int j = counters[i] % n_lanes; j < n_lanes; ++j)
1054 tight_category_map_ghost.emplace_back(
1055 std::array<unsigned int, 2>{
1056 {i, numbers::invalid_unsigned_int}});
1057
1058 std::sort(tight_category_map_ghost.begin(),
1059 tight_category_map_ghost.end());
1060 }
1061
1062 // Step 8: Fill in the data by batches for the locally owned cells.
1063 const unsigned int n_cell_batches = batch_order.size();
1064 const unsigned int n_ghost_batches =
1065 ((tight_category_map_ghost.empty() ? n_ghost_cells :
1066 tight_category_map_ghost.size()) +
1067 n_lanes - 1) /
1068 n_lanes;
1069 incompletely_filled_vectorization.resize(n_cell_batches +
1070 n_ghost_batches);
1071
1072 cell_partition_data.clear();
1073 cell_partition_data.resize(1, 0);
1074
1075 renumbering.clear();
1076 renumbering.resize(n_active_cells + n_ghost_cells,
1077 numbers::invalid_unsigned_int);
1078
1079 unsigned int counter = 0;
1080 for (unsigned int block = 0; block < blocks.size() - 1; ++block)
1081 {
1082 const unsigned int grain_size =
1083 std::max((2048U / dofs_per_cell) / 8 * 4, 2U);
1084 for (unsigned int k = blocks[block]; k < blocks[block + 1];
1085 k += grain_size)
1086 cell_partition_data.push_back(
1087 std::min(k + grain_size, blocks[block + 1]));
1088 partition_row_index[block + 1] = cell_partition_data.size() - 1;
1089
1090 // Set the numbering according to the reordered temporary one
1091 for (unsigned int k = blocks[block]; k < blocks[block + 1]; ++k)
1092 {
1093 const unsigned int pos = batch_order[k][2];
1094 unsigned int j = 0;
1095 for (; j < n_lanes && temporary_numbering[pos + j] !=
1096 numbers::invalid_unsigned_int;
1097 ++j)
1098 renumbering[counter++] = temporary_numbering[pos + j];
1099 if (j < n_lanes)
1100 incompletely_filled_vectorization[k] = j;
1101 }
1102 }
1103 AssertDimension(counter, n_active_cells);
1104
1105 // Step 9: Treat the ghost cells
1106 if (tight_category_map_ghost.empty())
1107 {
1108 for (unsigned int cell = 0; cell < n_ghost_cells; ++cell)
1109 renumbering[n_active_cells + cell] = n_active_cells + cell;
1110
1111 if ((n_ghost_cells % n_lanes) != 0u)
1112 incompletely_filled_vectorization.back() = n_ghost_cells % n_lanes;
1113 }
1114 else
1115 {
1116 for (unsigned int k = 0, ptr = 0; k < n_ghost_batches;
1117 ++k, ptr += n_lanes)
1118 {
1119 unsigned int j = 0;
1120
1121 for (;
1122 j < n_lanes && (ptr + j < tight_category_map_ghost.size()) &&
1123 (tight_category_map_ghost[ptr + j][1] !=
1124 numbers::invalid_unsigned_int);
1125 ++j)
1126 renumbering[counter++] =
1127 n_active_cells + tight_category_map_ghost[ptr + j][1];
1128
1129 if (j < n_lanes)
1130 incompletely_filled_vectorization[n_cell_batches + k] = j;
1131 }
1132
1133 AssertDimension(counter, n_active_cells + n_ghost_cells);
1134 }
1135
1136 cell_partition_data.push_back(n_cell_batches + n_ghost_batches);
1137 partition_row_index.back() = cell_partition_data.size() - 1;
1138
1139 if constexpr (running_in_debug_mode())
1140 {
1141 std::vector<unsigned int> renumber_cpy(renumbering);
1142 std::sort(renumber_cpy.begin(), renumber_cpy.end());
1143 for (unsigned int i = 0; i < renumber_cpy.size(); ++i)
1144 AssertDimension(i, renumber_cpy[i]);
1145 }
1146 }
1147
1148
1149
1150 void
1151 TaskInfo::initial_setup_blocks_tasks(
1152 const std::vector<unsigned int> &boundary_cells,
1153 std::vector<unsigned int> &renumbering,
1154 std::vector<unsigned char> &incompletely_filled_vectorization)
1155 {
1156 const unsigned int n_cell_batches =
1157 (n_active_cells + vectorization_length - 1) / vectorization_length;
1158 const unsigned int n_ghost_slots =
1159 (n_ghost_cells + vectorization_length - 1) / vectorization_length;
1160 incompletely_filled_vectorization.resize(n_cell_batches + n_ghost_slots);
1161 if (n_cell_batches * vectorization_length > n_active_cells)
1162 incompletely_filled_vectorization[n_cell_batches - 1] =
1163 vectorization_length -
1164 (n_cell_batches * vectorization_length - n_active_cells);
1165 if (n_ghost_slots * vectorization_length > n_ghost_cells)
1166 incompletely_filled_vectorization[n_cell_batches + n_ghost_slots - 1] =
1167 vectorization_length -
1168 (n_ghost_slots * vectorization_length - n_ghost_cells);
1169
1170 std::vector<unsigned int> reverse_numbering(
1171 n_active_cells, numbers::invalid_unsigned_int);
1172 for (unsigned int j = 0; j < boundary_cells.size(); ++j)
1173 reverse_numbering[boundary_cells[j]] = j;
1174 unsigned int counter = boundary_cells.size();
1175 for (unsigned int j = 0; j < n_active_cells; ++j)
1176 if (reverse_numbering[j] == numbers::invalid_unsigned_int)
1177 reverse_numbering[j] = counter++;
1178
1179 AssertDimension(counter, n_active_cells);
1180 renumbering = Utilities::invert_permutation(reverse_numbering);
1181
1182 for (unsigned int j = n_active_cells; j < n_active_cells + n_ghost_cells;
1183 ++j)
1184 renumbering.push_back(j);
1185
1186 // TODO: might be able to simplify this code by not relying on the cell
1187 // partition data while computing the thread graph
1188 cell_partition_data.clear();
1189 cell_partition_data.push_back(0);
1190 if (n_procs > 1)
1191 {
1192 const unsigned int n_macro_boundary_cells =
1193 (boundary_cells.size() + vectorization_length - 1) /
1194 vectorization_length;
1195 cell_partition_data.push_back(
1196 (n_cell_batches - n_macro_boundary_cells) / 2);
1197 cell_partition_data.push_back(cell_partition_data[1] +
1198 n_macro_boundary_cells);
1199 }
1200 else
1201 AssertDimension(boundary_cells.size(), 0);
1202 cell_partition_data.push_back(n_cell_batches);
1203 cell_partition_data.push_back(cell_partition_data.back() + n_ghost_slots);
1204 partition_row_index.resize(n_procs > 1 ? 4 : 2);
1205 partition_row_index[0] = 0;
1206 partition_row_index[1] = 1;
1207 if (n_procs > 1)
1208 {
1209 partition_row_index[2] = 2;
1210 partition_row_index[3] = 3;
1211 }
1212 }
1213
1214
1215
1216 void
1217 TaskInfo::guess_block_size(const unsigned int dofs_per_cell)
1218 {
1219 // user did not say a positive number, so we have to guess
1220 if (block_size == 0)
1221 {
1222 // we would like to have enough work to do, so as first guess, try
1223 // to get 16 times as many chunks as we have threads on the system.
1224 block_size = n_active_cells / (MultithreadInfo::n_threads() * 16 *
1225 vectorization_length);
1226
1227 // if there are too few degrees of freedom per cell, need to
1228 // increase the block size
1229 const unsigned int minimum_parallel_grain_size = 200;
1230 if (dofs_per_cell * block_size < minimum_parallel_grain_size)
1231 block_size = (minimum_parallel_grain_size / dofs_per_cell + 1);
1232 if (dofs_per_cell * block_size > 10000)
1233 block_size /= 4;
1234
1235 block_size =
1236 1 << static_cast<unsigned int>(std::log2(block_size + 1));
1237 }
1238 if (block_size > n_active_cells)
1239 block_size = std::max(1U, n_active_cells);
1240 }
1241
1242
1243
1244 void
1245 TaskInfo::make_thread_graph_partition_color(
1246 DynamicSparsityPattern &connectivity_large,
1247 std::vector<unsigned int> &renumbering,
1248 std::vector<unsigned char> &irregular_cells,
1249 const bool)
1250 {
1251 const unsigned int n_cell_batches = *(cell_partition_data.end() - 2);
1252 if (n_cell_batches == 0)
1253 return;
1254
1255 Assert(vectorization_length > 0, ExcInternalError());
1256
1257 unsigned int partition = 0, counter = 0;
1258
1259 // Create connectivity graph for blocks based on connectivity graph for
1260 // cells.
1261 DynamicSparsityPattern connectivity(n_blocks, n_blocks);
1262 make_connectivity_cells_to_blocks(irregular_cells,
1263 connectivity_large,
1264 connectivity);
1265
1266 // Create cell-block partitioning.
1267
1268 // For each block of cells, this variable saves to which partitions the
1269 // block belongs. Initialize all to -1 to mark them as not yet assigned
1270 // a partition.
1271 std::vector<unsigned int> cell_partition(n_blocks,
1272 numbers::invalid_unsigned_int);
1273
1274 // In element j of this variable, one puts the old number of the block
1275 // that should be the jth block in the new numeration.
1276 std::vector<unsigned int> partition_list(n_blocks, 0);
1277 std::vector<unsigned int> partition_color_list(n_blocks, 0);
1278
1279 // This vector points to the start of each partition.
1280 std::vector<unsigned int> partition_size(2, 0);
1281
1282 // blocking_connectivity = true;
1283
1284 // The cluster_size in make_partitioning defines that the no. of cells
1285 // in each partition should be a multiple of cluster_size.
1286 unsigned int cluster_size = 1;
1287
1288 // Make the partitioning of the first layer of the blocks of cells.
1289 make_partitioning(connectivity,
1290 cluster_size,
1291 cell_partition,
1292 partition_list,
1293 partition_size,
1294 partition);
1295
1296 // Color the cells within each partition
1297 make_coloring_within_partitions_pre_blocked(connectivity,
1298 partition,
1299 cell_partition,
1300 partition_list,
1301 partition_size,
1302 partition_color_list);
1303
1304 partition_list = renumbering;
1305
1306 if constexpr (running_in_debug_mode())
1307 {
1308 // in debug mode, check that the partition color list is one-to-one
1309 {
1310 std::vector<unsigned int> sorted_pc_list(partition_color_list);
1311 std::sort(sorted_pc_list.begin(), sorted_pc_list.end());
1312 for (unsigned int i = 0; i < sorted_pc_list.size(); ++i)
1313 Assert(sorted_pc_list[i] == i, ExcInternalError());
1314 }
1315 }
1316
1317 // set the start list for each block and compute the renumbering of
1318 // cells
1319 std::vector<unsigned int> block_start(n_cell_batches + 1);
1320 std::vector<unsigned char> irregular(n_cell_batches);
1321
1322 unsigned int mcell_start = 0;
1323 block_start[0] = 0;
1324 for (unsigned int block = 0; block < n_blocks; ++block)
1325 {
1326 block_start[block + 1] = block_start[block];
1327 for (unsigned int mcell = mcell_start;
1328 mcell < std::min(mcell_start + block_size, n_cell_batches);
1329 ++mcell)
1330 {
1331 unsigned int n_comp = (irregular_cells[mcell] > 0) ?
1332 irregular_cells[mcell] :
1333 vectorization_length;
1334 block_start[block + 1] += n_comp;
1335 ++counter;
1336 }
1337 mcell_start += block_size;
1338 }
1339 counter = 0;
1340 unsigned int counter_macro = 0;
1341 unsigned int block_size_last =
1342 n_cell_batches - block_size * (n_blocks - 1);
1343 if (block_size_last == 0)
1344 block_size_last = block_size;
1345
1346 unsigned int tick = 0;
1347 for (unsigned int block = 0; block < n_blocks; ++block)
1348 {
1349 unsigned int present_block = partition_color_list[block];
1350 for (unsigned int cell = block_start[present_block];
1351 cell < block_start[present_block + 1];
1352 ++cell)
1353 renumbering[counter++] = partition_list[cell];
1354 unsigned int this_block_size =
1355 (present_block == n_blocks - 1) ? block_size_last : block_size;
1356
1357 // Also re-compute the content of cell_partition_data to
1358 // contain the numbers of cells, not blocks
1359 if (cell_partition_data[tick] == block)
1360 cell_partition_data[tick++] = counter_macro;
1361
1362 for (unsigned int j = 0; j < this_block_size; ++j)
1363 irregular[counter_macro++] =
1364 irregular_cells[present_block * block_size + j];
1365 }
1366 AssertDimension(tick + 1, cell_partition_data.size());
1367 cell_partition_data.back() = counter_macro;
1368
1369 irregular_cells.swap(irregular);
1370 AssertDimension(counter, n_active_cells);
1371 AssertDimension(counter_macro, n_cell_batches);
1372
1373 // check that the renumbering is one-to-one
1374 if constexpr (running_in_debug_mode())
1375 {
1376 {
1377 std::vector<unsigned int> sorted_renumbering(renumbering);
1378 std::sort(sorted_renumbering.begin(), sorted_renumbering.end());
1379 for (unsigned int i = 0; i < sorted_renumbering.size(); ++i)
1380 Assert(sorted_renumbering[i] == i, ExcInternalError());
1381 }
1382 }
1383
1384
1385 update_task_info(
1386 partition); // Actually sets too much for partition color case
1387
1388 AssertDimension(cell_partition_data.back(), n_cell_batches);
1389 }
1390
1391
1392
1393 void
1394 TaskInfo::make_thread_graph(
1395 const std::vector<unsigned int> &cell_active_fe_index,
1396 DynamicSparsityPattern &connectivity,
1397 std::vector<unsigned int> &renumbering,
1398 std::vector<unsigned char> &irregular_cells,
1399 const bool hp_bool)
1400 {
1401 const unsigned int n_cell_batches = *(cell_partition_data.end() - 2);
1402 if (n_cell_batches == 0)
1403 return;
1404
1405 Assert(vectorization_length > 0, ExcInternalError());
1406
1407 // if we want to block before partitioning, create connectivity graph
1408 // for blocks based on connectivity graph for cells.
1409 DynamicSparsityPattern connectivity_blocks(n_blocks, n_blocks);
1410 make_connectivity_cells_to_blocks(irregular_cells,
1411 connectivity,
1412 connectivity_blocks);
1413
1414 unsigned int n_blocks = 0;
1415 if (scheme == partition_color ||
1416 scheme == color) // blocking_connectivity == true
1417 n_blocks = this->n_blocks;
1418 else
1419 n_blocks = n_active_cells;
1420
1421 // For each block of cells, this variable saves to which partitions the
1422 // block belongs. Initialize all to -1 to mark them as not yet assigned
1423 // a partition.
1424 std::vector<unsigned int> cell_partition(n_blocks,
1425 numbers::invalid_unsigned_int);
1426
1427 // In element j of this variable, one puts the old number (but after
1428 // renumbering according to the input renumbering) of the block that
1429 // should be the jth block in the new numeration.
1430 std::vector<unsigned int> partition_list(n_blocks, 0);
1431 std::vector<unsigned int> partition_2layers_list(n_blocks, 0);
1432
1433 // This vector points to the start of each partition.
1434 std::vector<unsigned int> partition_size(2, 0);
1435
1436 unsigned int partition = 0;
1437
1438 // Within the partitions we want to be able to block for the case that
1439 // we do not block already in the connectivity. The cluster_size in
1440 // make_partitioning defines that the no. of cells in each partition
1441 // should be a multiple of cluster_size.
1442 unsigned int cluster_size = 1;
1443 if (scheme == partition_partition)
1444 cluster_size = block_size * vectorization_length;
1445
1446 // Make the partitioning of the first layer of the blocks of cells.
1447 if (scheme == partition_color || scheme == color)
1448 make_partitioning(connectivity_blocks,
1449 cluster_size,
1450 cell_partition,
1451 partition_list,
1452 partition_size,
1453 partition);
1454 else
1455 make_partitioning(connectivity,
1456 cluster_size,
1457 cell_partition,
1458 partition_list,
1459 partition_size,
1460 partition);
1461
1462 // Partition or color second layer
1463 if (scheme == partition_partition)
1464
1465 {
1466 // Partition within partitions.
1467 make_partitioning_within_partitions_post_blocked(
1468 connectivity,
1469 cell_active_fe_index,
1470 partition,
1471 cluster_size,
1472 hp_bool,
1473 cell_partition,
1474 partition_list,
1475 partition_size,
1476 partition_2layers_list,
1477 irregular_cells);
1478 }
1479 else if (scheme == partition_color || scheme == color)
1480 {
1481 make_coloring_within_partitions_pre_blocked(connectivity_blocks,
1482 partition,
1483 cell_partition,
1484 partition_list,
1485 partition_size,
1486 partition_2layers_list);
1487 }
1488
1489 // in debug mode, check that the partition_2layers_list is one-to-one
1490 if constexpr (running_in_debug_mode())
1491 {
1492 {
1493 std::vector<unsigned int> sorted_pc_list(partition_2layers_list);
1494 std::sort(sorted_pc_list.begin(), sorted_pc_list.end());
1495 for (unsigned int i = 0; i < sorted_pc_list.size(); ++i)
1496 Assert(sorted_pc_list[i] == i, ExcInternalError());
1497 }
1498 }
1499
1500 // Set the new renumbering
1501 std::vector<unsigned int> renumbering_in(n_active_cells, 0);
1502 renumbering_in.swap(renumbering);
1503 if (scheme == partition_partition) // blocking_connectivity == false
1504 {
1505 // This is the simple case. The renumbering is just a combination of
1506 // the renumbering that we were given as an input and the
1507 // renumbering of partition/coloring given in partition_2layers_list
1508 for (unsigned int j = 0; j < renumbering.size(); ++j)
1509 renumbering[j] = renumbering_in[partition_2layers_list[j]];
1510 // Account for the ghost cells, finally.
1511 for (unsigned int i = 0; i < n_ghost_cells; ++i)
1512 renumbering.push_back(i + n_active_cells);
1513 }
1514 else
1515 {
1516 // set the start list for each block and compute the renumbering of
1517 // cells
1518 std::vector<unsigned int> block_start(n_cell_batches + 1);
1519 std::vector<unsigned char> irregular(n_cell_batches);
1520
1521 unsigned int counter = 0;
1522 unsigned int mcell_start = 0;
1523 block_start[0] = 0;
1524 for (unsigned int block = 0; block < n_blocks; ++block)
1525 {
1526 block_start[block + 1] = block_start[block];
1527 for (unsigned int mcell = mcell_start;
1528 mcell < std::min(mcell_start + block_size, n_cell_batches);
1529 ++mcell)
1530 {
1531 unsigned int n_comp = (irregular_cells[mcell] > 0) ?
1532 irregular_cells[mcell] :
1533 vectorization_length;
1534 block_start[block + 1] += n_comp;
1535 ++counter;
1536 }
1537 mcell_start += block_size;
1538 }
1539 counter = 0;
1540 unsigned int counter_macro = 0;
1541 unsigned int block_size_last =
1542 n_cell_batches - block_size * (n_blocks - 1);
1543 if (block_size_last == 0)
1544 block_size_last = block_size;
1545
1546 unsigned int tick = 0;
1547 for (unsigned int block = 0; block < n_blocks; ++block)
1548 {
1549 unsigned int present_block = partition_2layers_list[block];
1550 for (unsigned int cell = block_start[present_block];
1551 cell < block_start[present_block + 1];
1552 ++cell)
1553 renumbering[counter++] = renumbering_in[cell];
1554 unsigned int this_block_size =
1555 (present_block == n_blocks - 1) ? block_size_last : block_size;
1556
1557 // Also re-compute the content of cell_partition_data to
1558 // contain the numbers of cells, not blocks
1559 if (cell_partition_data[tick] == block)
1560 cell_partition_data[tick++] = counter_macro;
1561
1562 for (unsigned int j = 0; j < this_block_size; ++j)
1563 irregular[counter_macro++] =
1564 irregular_cells[present_block * block_size + j];
1565 }
1566 AssertDimension(tick + 1, cell_partition_data.size());
1567 cell_partition_data.back() = counter_macro;
1568
1569 irregular_cells.swap(irregular);
1570 AssertDimension(counter, n_active_cells);
1571 AssertDimension(counter_macro, n_cell_batches);
1572 // check that the renumbering is one-to-one
1573 if constexpr (running_in_debug_mode())
1574 {
1575 {
1576 std::vector<unsigned int> sorted_renumbering(renumbering);
1577 std::sort(sorted_renumbering.begin(), sorted_renumbering.end());
1578 for (unsigned int i = 0; i < sorted_renumbering.size(); ++i)
1579 Assert(sorted_renumbering[i] == i, ExcInternalError());
1580 }
1581 }
1582 }
1583
1584 // Update the task_info with the more information for the thread graph.
1585 update_task_info(partition);
1586 }
1587
1588
1589
1590 void
1591 TaskInfo::make_thread_graph_partition_partition(
1592 const std::vector<unsigned int> &cell_active_fe_index,
1593 DynamicSparsityPattern &connectivity,
1594 std::vector<unsigned int> &renumbering,
1595 std::vector<unsigned char> &irregular_cells,
1596 const bool hp_bool)
1597 {
1598 const unsigned int n_cell_batches = *(cell_partition_data.end() - 2);
1599 if (n_cell_batches == 0)
1600 return;
1601
1602 const unsigned int cluster_size = block_size * vectorization_length;
1603
1604 // Create cell-block partitioning.
1605
1606 // For each block of cells, this variable saves to which partitions the
1607 // block belongs. Initialize all to n_cell_batches to mark them as not
1608 // yet assigned a partition.
1609 std::vector<unsigned int> cell_partition(n_active_cells,
1610 numbers::invalid_unsigned_int);
1611
1612
1613 // In element j of this variable, one puts the old number of the block
1614 // that should be the jth block in the new numeration.
1615 std::vector<unsigned int> partition_list(n_active_cells, 0);
1616 std::vector<unsigned int> partition_partition_list(n_active_cells, 0);
1617
1618 // This vector points to the start of each partition.
1619 std::vector<unsigned int> partition_size(2, 0);
1620
1621 unsigned int partition = 0;
1622 // Here, we do not block inside the connectivity graph
1623 // blocking_connectivity = false;
1624
1625 // Make the partitioning of the first layer of the blocks of cells.
1626 make_partitioning(connectivity,
1627 cluster_size,
1628 cell_partition,
1629 partition_list,
1630 partition_size,
1631 partition);
1632
1633 // Partition within partitions.
1634 make_partitioning_within_partitions_post_blocked(connectivity,
1635 cell_active_fe_index,
1636 partition,
1637 cluster_size,
1638 hp_bool,
1639 cell_partition,
1640 partition_list,
1641 partition_size,
1642 partition_partition_list,
1643 irregular_cells);
1644
1645 partition_list.swap(renumbering);
1646
1647 for (unsigned int j = 0; j < renumbering.size(); ++j)
1648 renumbering[j] = partition_list[partition_partition_list[j]];
1649
1650 for (unsigned int i = 0; i < n_ghost_cells; ++i)
1651 renumbering.push_back(i + n_active_cells);
1652
1653 update_task_info(partition);
1654 }
1655
1656
1657
1658 void
1659 TaskInfo::make_connectivity_cells_to_blocks(
1660 const std::vector<unsigned char> &irregular_cells,
1661 const DynamicSparsityPattern &connectivity_cells,
1662 DynamicSparsityPattern &connectivity_blocks) const
1663 {
1664 std::vector<std::vector<unsigned int>> cell_blocks(n_blocks);
1665 std::vector<unsigned int> touched_cells(n_active_cells);
1666 unsigned int cell = 0;
1667 for (unsigned int i = 0, mcell = 0; i < n_blocks; ++i)
1668 {
1669 for (unsigned int c = 0;
1670 c < block_size && mcell < *(cell_partition_data.end() - 2);
1671 ++c, ++mcell)
1672 {
1673 unsigned int ncomp = (irregular_cells[mcell] > 0) ?
1674 irregular_cells[mcell] :
1675 vectorization_length;
1676 for (unsigned int c = 0; c < ncomp; ++c, ++cell)
1677 {
1678 cell_blocks[i].push_back(cell);
1679 touched_cells[cell] = i;
1680 }
1681 }
1682 }
1683 AssertDimension(cell, n_active_cells);
1684 for (unsigned int i = 0; i < cell_blocks.size(); ++i)
1685 for (unsigned int col = 0; col < cell_blocks[i].size(); ++col)
1686 {
1687 for (DynamicSparsityPattern::iterator it =
1688 connectivity_cells.begin(cell_blocks[i][col]);
1689 it != connectivity_cells.end(cell_blocks[i][col]);
1690 ++it)
1691 {
1692 if (touched_cells[it->column()] != i)
1693 connectivity_blocks.add(i, touched_cells[it->column()]);
1694 }
1695 }
1696 }
1697
1698
1699
1700 // Function to create partitioning on the second layer within each
1701 // partition. Version without preblocking.
1702 void
1703 TaskInfo::make_partitioning_within_partitions_post_blocked(
1704 const DynamicSparsityPattern &connectivity,
1705 const std::vector<unsigned int> &cell_active_fe_index,
1706 const unsigned int partition,
1707 const unsigned int cluster_size,
1708 const bool hp_bool,
1709 const std::vector<unsigned int> &cell_partition,
1710 const std::vector<unsigned int> &partition_list,
1711 const std::vector<unsigned int> &partition_size,
1712 std::vector<unsigned int> &partition_partition_list,
1713 std::vector<unsigned char> &irregular_cells)
1714 {
1715 const unsigned int n_cell_batches = *(cell_partition_data.end() - 2);
1716 const unsigned int n_ghost_slots =
1717 *(cell_partition_data.end() - 1) - n_cell_batches;
1718
1719 // List of cells in previous partition
1720 std::vector<unsigned int> neighbor_list;
1721 // List of cells in current partition for use as neighbors in next
1722 // partition
1723 std::vector<unsigned int> neighbor_neighbor_list;
1724
1725 std::vector<unsigned int> renumbering(n_active_cells);
1726
1727 irregular_cells.back() = 0;
1728 irregular_cells.resize(n_active_cells + n_ghost_slots);
1729
1730 unsigned int max_fe_index = 0;
1731 for (const unsigned int fe_index : cell_active_fe_index)
1732 max_fe_index = std::max(fe_index, max_fe_index);
1733
1734 Assert(!hp_bool || cell_active_fe_index.size() == n_active_cells,
1735 ExcInternalError());
1736
1737 {
1738 unsigned int n_cell_batches_before = 0;
1739 // Create partitioning within partitions.
1740
1741 // For each block of cells, this variable saves to which partitions
1742 // the block belongs. Initialize all to n_cell_batches to mark them as
1743 // not yet assigned a partition.
1744 std::vector<unsigned int> cell_partition_l2(
1745 n_active_cells, numbers::invalid_unsigned_int);
1746 partition_row_index.clear();
1747 partition_row_index.resize(partition + 1, 0);
1748 cell_partition_data.resize(1, 0);
1749
1750 unsigned int counter = 0;
1751 unsigned int missing_macros;
1752 for (unsigned int part = 0; part < partition; ++part)
1753 {
1754 neighbor_neighbor_list.resize(0);
1755 neighbor_list.resize(0);
1756 bool work = true;
1757 unsigned int partition_l2 = 0;
1758 unsigned int start_up = partition_size[part];
1759 unsigned int partition_counter = 0;
1760 while (work)
1761 {
1762 if (neighbor_list.empty())
1763 {
1764 work = false;
1765 partition_counter = 0;
1766 for (unsigned int j = start_up;
1767 j < partition_size[part + 1];
1768 ++j)
1769 if (cell_partition[partition_list[j]] == part &&
1770 cell_partition_l2[partition_list[j]] ==
1771 numbers::invalid_unsigned_int)
1772 {
1773 start_up = j;
1774 work = true;
1775 partition_counter = 1;
1776 // To start up, set the start_up cell to partition
1777 // and list all its neighbors.
1778 AssertIndexRange(start_up, partition_size[part + 1]);
1779 cell_partition_l2[partition_list[start_up]] =
1780 partition_l2;
1781 neighbor_neighbor_list.push_back(
1782 partition_list[start_up]);
1783 partition_partition_list[counter++] =
1784 partition_list[start_up];
1785 ++start_up;
1786 break;
1787 }
1788 }
1789 else
1790 {
1791 partition_counter = 0;
1792 for (const unsigned int neighbor : neighbor_list)
1793 {
1794 Assert(cell_partition[neighbor] == part,
1795 ExcInternalError());
1796 Assert(cell_partition_l2[neighbor] == partition_l2 - 1,
1797 ExcInternalError());
1798 auto neighbor_it = connectivity.begin(neighbor);
1799 const auto end_it = connectivity.end(neighbor);
1800 for (; neighbor_it != end_it; ++neighbor_it)
1801 {
1802 if (cell_partition[neighbor_it->column()] == part &&
1803 cell_partition_l2[neighbor_it->column()] ==
1804 numbers::invalid_unsigned_int)
1805 {
1806 cell_partition_l2[neighbor_it->column()] =
1807 partition_l2;
1808 neighbor_neighbor_list.push_back(
1809 neighbor_it->column());
1810 partition_partition_list[counter++] =
1811 neighbor_it->column();
1812 ++partition_counter;
1813 }
1814 }
1815 }
1816 }
1817 if (partition_counter > 0)
1818 {
1819 int index_before = neighbor_neighbor_list.size(),
1820 index = index_before;
1821 {
1822 // put the cells into separate lists for each FE index
1823 // within one partition-partition
1824 missing_macros = 0;
1825 std::vector<unsigned int> remaining_per_cell_batch(
1826 max_fe_index + 1);
1827 std::vector<std::vector<unsigned int>>
1828 renumbering_fe_index;
1829 unsigned int cell;
1830 bool filled = true;
1831 if (hp_bool == true)
1832 {
1833 renumbering_fe_index.resize(max_fe_index + 1);
1834 for (cell = counter - partition_counter;
1835 cell < counter;
1836 ++cell)
1837 {
1838 renumbering_fe_index
1839 [cell_active_fe_index.empty() ?
1840 0 :
1841 cell_active_fe_index
1842 [partition_partition_list[cell]]]
1843 .push_back(partition_partition_list[cell]);
1844 }
1845 // check how many more cells are needed in the lists
1846 for (unsigned int j = 0; j < max_fe_index + 1; ++j)
1847 {
1848 remaining_per_cell_batch[j] =
1849 renumbering_fe_index[j].size() %
1850 vectorization_length;
1851 if (remaining_per_cell_batch[j] != 0)
1852 filled = false;
1853 missing_macros +=
1854 ((renumbering_fe_index[j].size() +
1855 vectorization_length - 1) /
1856 vectorization_length);
1857 }
1858 }
1859 else
1860 {
1861 remaining_per_cell_batch.resize(1);
1862 remaining_per_cell_batch[0] =
1863 partition_counter % vectorization_length;
1864 missing_macros =
1865 partition_counter / vectorization_length;
1866 if (remaining_per_cell_batch[0] != 0)
1867 {
1868 filled = false;
1869 ++missing_macros;
1870 }
1871 }
1872 missing_macros =
1873 cluster_size - (missing_macros % cluster_size);
1874
1875 // now we realized that there are some cells missing.
1876 while (missing_macros > 0 || filled == false)
1877 {
1878 if (index == 0)
1879 {
1880 index = neighbor_neighbor_list.size();
1881 if (index == index_before)
1882 {
1883 if (missing_macros != 0)
1884 {
1885 neighbor_neighbor_list.resize(0);
1886 }
1887 start_up--;
1888 break; // not connected - start again
1889 }
1890 index_before = index;
1891 }
1892 index--;
1893 unsigned int additional =
1894 neighbor_neighbor_list[index];
1895
1896 // go through the neighbors of the last cell in the
1897 // current partition and check if we find some to
1898 // fill up with.
1899 DynamicSparsityPattern::iterator neighbor =
1900 connectivity.begin(
1901 additional),
1902 end =
1903 connectivity.end(
1904 additional);
1905 for (; neighbor != end; ++neighbor)
1906 {
1907 if (cell_partition[neighbor->column()] == part &&
1908 cell_partition_l2[neighbor->column()] ==
1909 numbers::invalid_unsigned_int)
1910 {
1911 unsigned int this_index = 0;
1912 if (hp_bool == true)
1913 this_index =
1914 cell_active_fe_index.empty() ?
1915 0 :
1916 cell_active_fe_index[neighbor
1917 ->column()];
1918
1919 // Only add this cell if we need more macro
1920 // cells in the current block or if there is
1921 // a macro cell with the FE index that is
1922 // not yet fully populated
1923 if (missing_macros > 0 ||
1924 remaining_per_cell_batch[this_index] > 0)
1925 {
1926 cell_partition_l2[neighbor->column()] =
1927 partition_l2;
1928 neighbor_neighbor_list.push_back(
1929 neighbor->column());
1930 if (hp_bool == true)
1931 renumbering_fe_index[this_index]
1932 .push_back(neighbor->column());
1933 partition_partition_list[counter] =
1934 neighbor->column();
1935 ++counter;
1936 ++partition_counter;
1937 if (remaining_per_cell_batch
1938 [this_index] == 0 &&
1939 missing_macros > 0)
1940 missing_macros--;
1941 remaining_per_cell_batch[this_index]++;
1942 if (remaining_per_cell_batch
1943 [this_index] ==
1944 vectorization_length)
1945 {
1946 remaining_per_cell_batch[this_index] =
1947 0;
1948 }
1949 if (missing_macros == 0)
1950 {
1951 filled = true;
1952 for (unsigned int fe_ind = 0;
1953 fe_ind < max_fe_index + 1;
1954 ++fe_ind)
1955 if (remaining_per_cell_batch
1956 [fe_ind] != 0)
1957 filled = false;
1958 }
1959 if (filled == true)
1960 break;
1961 }
1962 }
1963 }
1964 }
1965 if (hp_bool == true)
1966 {
1967 // set the renumbering according to their active FE
1968 // index within one partition-partition which was
1969 // implicitly assumed above
1970 cell = counter - partition_counter;
1971 for (unsigned int j = 0; j < max_fe_index + 1; ++j)
1972 {
1973 for (const unsigned int jj :
1974 renumbering_fe_index[j])
1975 renumbering[cell++] = jj;
1976 if (renumbering_fe_index[j].size() %
1977 vectorization_length !=
1978 0)
1979 irregular_cells[renumbering_fe_index[j].size() /
1980 vectorization_length +
1981 n_cell_batches_before] =
1982 renumbering_fe_index[j].size() %
1983 vectorization_length;
1984 n_cell_batches_before +=
1985 (renumbering_fe_index[j].size() +
1986 vectorization_length - 1) /
1987 vectorization_length;
1988 renumbering_fe_index[j].resize(0);
1989 }
1990 }
1991 else
1992 {
1993 n_cell_batches_before +=
1994 partition_counter / vectorization_length;
1995 if (partition_counter % vectorization_length != 0)
1996 {
1997 irregular_cells[n_cell_batches_before] =
1998 partition_counter % vectorization_length;
1999 ++n_cell_batches_before;
2000 }
2001 }
2002 }
2003 cell_partition_data.push_back(n_cell_batches_before);
2004 partition_l2++;
2005 }
2006 neighbor_list = neighbor_neighbor_list;
2007 neighbor_neighbor_list.resize(0);
2008 }
2009 partition_row_index[part + 1] =
2010 partition_row_index[part] + partition_l2;
2011 }
2012 }
2013 if (hp_bool == true)
2014 {
2015 partition_partition_list.swap(renumbering);
2016 }
2017 }
2018
2019
2020
2021 // Function to create coloring on the second layer within each partition.
2022 // Version assumes preblocking.
2023 void
2024 TaskInfo::make_coloring_within_partitions_pre_blocked(
2025 const DynamicSparsityPattern &connectivity,
2026 const unsigned int partition,
2027 const std::vector<unsigned int> &cell_partition,
2028 const std::vector<unsigned int> &partition_list,
2029 const std::vector<unsigned int> &partition_size,
2030 std::vector<unsigned int> &partition_color_list)
2031 {
2032 const unsigned int n_cell_batches = *(cell_partition_data.end() - 2);
2033 std::vector<unsigned int> cell_color(n_blocks, n_cell_batches);
2034 std::vector<bool> color_finder;
2035
2036 partition_row_index.resize(partition + 1);
2037 cell_partition_data.clear();
2038 unsigned int color_counter = 0, index_counter = 0;
2039 for (unsigned int part = 0; part < partition; ++part)
2040 {
2041 partition_row_index[part] = index_counter;
2042 unsigned int max_color = 0;
2043 for (unsigned int k = partition_size[part];
2044 k < partition_size[part + 1];
2045 k++)
2046 {
2047 unsigned int cell = partition_list[k];
2048 unsigned int n_neighbors = connectivity.row_length(cell);
2049
2050 // In the worst case, each neighbor has a different color. So we
2051 // find at least one available color between 0 and n_neighbors.
2052 color_finder.resize(n_neighbors + 1);
2053 for (unsigned int j = 0; j <= n_neighbors; ++j)
2054 color_finder[j] = true;
2055 DynamicSparsityPattern::iterator neighbor =
2056 connectivity.begin(cell),
2057 end = connectivity.end(cell);
2058 for (; neighbor != end; ++neighbor)
2059 {
2060 // Mark the color that a neighbor within the partition has
2061 // as taken
2062 if (cell_partition[neighbor->column()] == part &&
2063 cell_color[neighbor->column()] <= n_neighbors)
2064 color_finder[cell_color[neighbor->column()]] = false;
2065 }
2066 // Choose the smallest color that is not taken for the block
2067 cell_color[cell] = 0;
2068 while (color_finder[cell_color[cell]] == false)
2069 cell_color[cell]++;
2070 if (cell_color[cell] > max_color)
2071 max_color = cell_color[cell];
2072 }
2073 // Reorder within partition: First, all blocks that belong the 0 and
2074 // then so on until those with color max (Note that the smaller the
2075 // number the larger the partition)
2076 for (unsigned int color = 0; color <= max_color; ++color)
2077 {
2078 cell_partition_data.push_back(color_counter);
2079 ++index_counter;
2080 for (unsigned int k = partition_size[part];
2081 k < partition_size[part + 1];
2082 k++)
2083 {
2084 unsigned int cell = partition_list[k];
2085 if (cell_color[cell] == color)
2086 {
2087 partition_color_list[color_counter++] = cell;
2088 }
2089 }
2090 }
2091 }
2092 cell_partition_data.push_back(n_blocks);
2093 partition_row_index[partition] = index_counter;
2094 AssertDimension(color_counter, n_blocks);
2095 }
2096
2097
2098 // Function to create partitioning on the first layer.
2099 void
2100 TaskInfo::make_partitioning(const DynamicSparsityPattern &connectivity,
2101 const unsigned int cluster_size,
2102 std::vector<unsigned int> &cell_partition,
2103 std::vector<unsigned int> &partition_list,
2104 std::vector<unsigned int> &partition_size,
2105 unsigned int &partition) const
2106
2107 {
2108 // For each block of cells, this variable saves to which partitions the
2109 // block belongs. Initialize all to n_cell_batches to mark them as not
2110 // yet assigned a partition.
2111 // std::vector<unsigned int> cell_partition (n_active_cells,
2112 // numbers::invalid_unsigned_int);
2113 // List of cells in previous partition
2114 std::vector<unsigned int> neighbor_list;
2115 // List of cells in current partition for use as neighbors in next
2116 // partition
2117 std::vector<unsigned int> neighbor_neighbor_list;
2118
2119 // In element j of this variable, one puts the old number of the block
2120 // that should be the jth block in the new numeration.
2121 // std::vector<unsigned int> partition_list(n_active_cells,0);
2122
2123 // This vector points to the start of each partition.
2124 // std::vector<unsigned int> partition_size(2,0);
2125
2126 partition = 0;
2127 unsigned int counter = 0;
2128 unsigned int start_nonboundary =
2129 cell_partition_data.size() == 5 ?
2130 vectorization_length *
2131 (cell_partition_data[2] - cell_partition_data[1]) :
2132 0;
2133
2134 const unsigned int n_cell_batches = *(cell_partition_data.end() - 2);
2135 if (n_cell_batches == 0)
2136 return;
2137 if (scheme == color)
2138 start_nonboundary = n_cell_batches;
2139 if (scheme == partition_color ||
2140 scheme == color) // blocking_connectivity == true
2141 start_nonboundary = ((start_nonboundary + block_size - 1) / block_size);
2142 unsigned int n_blocks;
2143 if (scheme == partition_color ||
2144 scheme == color) // blocking_connectivity == true
2145 n_blocks = this->n_blocks;
2146 else
2147 n_blocks = n_active_cells;
2148
2149 if (start_nonboundary > n_blocks)
2150 start_nonboundary = n_blocks;
2151
2152
2153 unsigned int start_up = 0;
2154 bool work = true;
2155 unsigned int remainder = cluster_size;
2156
2157 // this performs a classical breath-first search in the connectivity
2158 // graph of the cells under the restriction that the size of the
2159 // partitions should be a multiple of the given block size
2160 while (work)
2161 {
2162 // put the cells with neighbors on remote MPI processes up front
2163 if (start_nonboundary > 0)
2164 {
2165 for (unsigned int cell = 0; cell < start_nonboundary; ++cell)
2166 {
2167 const unsigned int cell_nn = cell;
2168 cell_partition[cell_nn] = partition;
2169 neighbor_list.push_back(cell_nn);
2170 partition_list[counter++] = cell_nn;
2171 partition_size.back()++;
2172 }
2173 start_nonboundary = 0;
2174 remainder -= (start_nonboundary % cluster_size);
2175 if (remainder == cluster_size)
2176 remainder = 0;
2177 }
2178 else
2179 {
2180 // To start up, set the start_up cell to partition and list all
2181 // its neighbors.
2182 cell_partition[start_up] = partition;
2183 neighbor_list.push_back(start_up);
2184 partition_list[counter++] = start_up;
2185 partition_size.back()++;
2186 ++start_up;
2187 remainder--;
2188 if (remainder == cluster_size)
2189 remainder = 0;
2190 }
2191 int index_before = neighbor_list.size(), index = index_before,
2192 index_stop = 0;
2193 while (remainder > 0)
2194 {
2195 if (index == index_stop)
2196 {
2197 index = neighbor_list.size();
2198 if (index == index_before)
2199 {
2200 neighbor_list.resize(0);
2201 goto not_connect;
2202 }
2203 index_stop = index_before;
2204 index_before = index;
2205 }
2206 index--;
2207 unsigned int additional = neighbor_list[index];
2208 DynamicSparsityPattern::iterator neighbor =
2209 connectivity.begin(additional),
2210 end =
2211 connectivity.end(additional);
2212 for (; neighbor != end; ++neighbor)
2213 {
2214 if (cell_partition[neighbor->column()] ==
2215 numbers::invalid_unsigned_int)
2216 {
2217 partition_size.back()++;
2218 cell_partition[neighbor->column()] = partition;
2219 neighbor_list.push_back(neighbor->column());
2220 partition_list[counter++] = neighbor->column();
2221 remainder--;
2222 if (remainder == 0)
2223 break;
2224 }
2225 }
2226 }
2227
2228 while (neighbor_list.size() > 0)
2229 {
2230 ++partition;
2231
2232 // counter for number of cells so far in current partition
2233 unsigned int partition_counter = 0;
2234
2235 // Mark the start of the new partition
2236 partition_size.push_back(partition_size.back());
2237
2238 // Loop through the list of cells in previous partition and put
2239 // all their neighbors in current partition
2240 for (const unsigned int cell : neighbor_list)
2241 {
2242 Assert(cell_partition[cell] == partition - 1,
2243 ExcInternalError());
2244 auto neighbor = connectivity.begin(cell);
2245 const auto end = connectivity.end(cell);
2246 for (; neighbor != end; ++neighbor)
2247 {
2248 if (cell_partition[neighbor->column()] ==
2249 numbers::invalid_unsigned_int)
2250 {
2251 partition_size.back()++;
2252 cell_partition[neighbor->column()] = partition;
2253
2254 // collect the cells of the current partition for
2255 // use as neighbors in next partition
2256 neighbor_neighbor_list.push_back(neighbor->column());
2257 partition_list[counter++] = neighbor->column();
2258 ++partition_counter;
2259 }
2260 }
2261 }
2262 remainder = cluster_size - (partition_counter % cluster_size);
2263 if (remainder == cluster_size)
2264 remainder = 0;
2265 int index_stop = 0;
2266 int index_before = neighbor_neighbor_list.size(),
2267 index = index_before;
2268 while (remainder > 0)
2269 {
2270 if (index == index_stop)
2271 {
2272 index = neighbor_neighbor_list.size();
2273 if (index == index_before)
2274 {
2275 neighbor_neighbor_list.resize(0);
2276 break;
2277 }
2278 index_stop = index_before;
2279 index_before = index;
2280 }
2281 index--;
2282 unsigned int additional = neighbor_neighbor_list[index];
2283 DynamicSparsityPattern::iterator neighbor =
2284 connectivity.begin(
2285 additional),
2286 end = connectivity.end(
2287 additional);
2288 for (; neighbor != end; ++neighbor)
2289 {
2290 if (cell_partition[neighbor->column()] ==
2291 numbers::invalid_unsigned_int)
2292 {
2293 partition_size.back()++;
2294 cell_partition[neighbor->column()] = partition;
2295 neighbor_neighbor_list.push_back(neighbor->column());
2296 partition_list[counter++] = neighbor->column();
2297 remainder--;
2298 if (remainder == 0)
2299 break;
2300 }
2301 }
2302 }
2303
2304 neighbor_list = neighbor_neighbor_list;
2305 neighbor_neighbor_list.resize(0);
2306 }
2307 not_connect:
2308 // One has to check if the graph is not connected so we have to find
2309 // another partition.
2310 work = false;
2311 for (unsigned int j = start_up; j < n_blocks; ++j)
2312 if (cell_partition[j] == numbers::invalid_unsigned_int)
2313 {
2314 start_up = j;
2315 work = true;
2316 if (remainder == 0)
2317 remainder = cluster_size;
2318 break;
2319 }
2320 }
2321 if (remainder != 0)
2322 ++partition;
2323
2324 AssertDimension(partition_size[partition], n_blocks);
2325 }
2326
2327
2328 void
2329 TaskInfo::update_task_info(const unsigned int partition)
2330 {
2331 evens = (partition + 1) / 2;
2332 odds = partition / 2;
2333 n_blocked_workers = odds - (odds + evens + 1) % 2;
2334 n_workers = evens + odds - n_blocked_workers;
2335 // From here only used for partition partition option.
2336 partition_evens.resize(partition);
2337 partition_odds.resize(partition);
2338 partition_n_blocked_workers.resize(partition);
2339 partition_n_workers.resize(partition);
2340 for (unsigned int part = 0; part < partition; ++part)
2341 {
2342 partition_evens[part] =
2343 (partition_row_index[part + 1] - partition_row_index[part] + 1) / 2;
2344 partition_odds[part] =
2345 (partition_row_index[part + 1] - partition_row_index[part]) / 2;
2346 partition_n_blocked_workers[part] =
2347 partition_odds[part] -
2348 (partition_odds[part] + partition_evens[part] + 1) % 2;
2349 partition_n_workers[part] = partition_evens[part] +
2350 partition_odds[part] -
2351 partition_n_blocked_workers[part];
2352 }
2353 }
2354 } // namespace MatrixFreeFunctions
2355} // namespace internal
2356
2357
2358
2359// explicit instantiations of template functions
2360template void
2361internal::MatrixFreeFunctions::TaskInfo::print_memory_statistics<std::ostream>(
2362 std::ostream &,
2363 const std::size_t) const;
2364template void
2365internal::MatrixFreeFunctions::TaskInfo::print_memory_statistics<
2366 ConditionalOStream>(ConditionalOStream &, const std::size_t) const;
2367
2368
2369DEAL_II_NAMESPACE_CLOSE
DynamicSparsityPattern::iterator
DynamicSparsityPatternIterators::Iterator iterator
Definition dynamic_sparsity_pattern.h:336
DynamicSparsityPattern::row_length
size_type row_length(const size_type row) const
Definition dynamic_sparsity_pattern.h:1041
DynamicSparsityPattern::add
void add(const size_type i, const size_type j)
Definition dynamic_sparsity_pattern.h:1001
MultithreadInfo::n_threads
static unsigned int n_threads()
Definition multithread_info.cc:129
conditional_ostream.h
DEAL_II_NAMESPACE_OPEN
#define DEAL_II_NAMESPACE_OPEN
Definition config.h:40
running_in_debug_mode
constexpr bool running_in_debug_mode()
Definition config.h:78
DEAL_II_NAMESPACE_CLOSE
#define DEAL_II_NAMESPACE_CLOSE
Definition config.h:41
dynamic_sparsity_pattern.h
StandardExceptions::ExcNotImplemented
static ::ExceptionBase & ExcNotImplemented()
Assert
#define Assert(cond, exc)
Definition exception_macros.h:559
AssertDimension
#define AssertDimension(dim1, dim2)
Definition exception_macros.h:851
AssertIndexRange
#define AssertIndexRange(index, range)
Definition exception_macros.h:923
StandardExceptions::ExcInternalError
static ::ExceptionBase & ExcInternalError()
AssertThrow
#define AssertThrow(cond, exc)
Definition exception_macros.h:656
memory_consumption.h
multithread_info.h
MemoryConsumption::memory_consumption
std::enable_if_t< std::is_fundamental_v< T >, std::size_t > memory_consumption(const T &t)
Definition memory_consumption.h:273
SparsityTools::partition
void partition(const SparsityPattern &sparsity_pattern, const unsigned int n_partitions, std::vector< unsigned int > &partition_indices, const Partitioner partitioner=Partitioner::metis)
Definition sparsity_tools.cc:396
Utilities::MPI::min_max_avg
MinMaxAvg min_max_avg(const double my_value, const MPI_Comm mpi_communicator)
Definition mpi.cc:79
Utilities::invert_permutation
std::vector< Integer > invert_permutation(const std::vector< Integer > &permutation)
Definition utilities.h:1700
internal::MatrixFreeFunctions::indicate_power_of_two
unsigned int indicate_power_of_two(const unsigned int vectorization_length)
Definition util.h:192
numbers::invalid_unsigned_int
constexpr unsigned int invalid_unsigned_int
Definition types.h:238
parallel::internal::parallel_for
void parallel_for(Iterator x_begin, Iterator x_end, const Functor &functor, const unsigned int grainsize)
Definition parallel.h:122
std::min
::VectorizedArray< Number, width > min(const ::VectorizedArray< Number, width > &, const ::VectorizedArray< Number, width > &)
Definition vectorization.h:6961
std::max
::VectorizedArray< Number, width > max(const ::VectorizedArray< Number, width > &, const ::VectorizedArray< Number, width > &)
Definition vectorization.h:6944
internal::MFWorkerInterface::face
virtual void face(const unsigned int range_index)=0
internal::MFWorkerInterface::zero_dst_vector_range
virtual void zero_dst_vector_range(const unsigned int range_index)=0
internal::MFWorkerInterface::cell
virtual void cell(const std::pair< unsigned int, unsigned int > &cell_range)=0
internal::MFWorkerInterface::boundary
virtual void boundary(const unsigned int range_index)=0
internal::MFWorkerInterface::vector_compress_start
virtual void vector_compress_start()=0
Starts the communication for the vector compress operation.
internal::MFWorkerInterface::cell_loop_post_range
virtual void cell_loop_post_range(const unsigned int range_index)=0
internal::MFWorkerInterface::vector_update_ghosts_start
virtual void vector_update_ghosts_start()=0
Starts the communication for the update ghost values operation.
internal::MFWorkerInterface::cell_loop_pre_range
virtual void cell_loop_pre_range(const unsigned int range_index)=0
internal::MFWorkerInterface::vector_update_ghosts_finish
virtual void vector_update_ghosts_finish()=0
Finishes the communication for the update ghost values operation.
internal::MFWorkerInterface::vector_compress_finish
virtual void vector_compress_finish()=0
Finishes the communication for the vector compress operation.
internal::MatrixFreeFunctions::TaskInfo::boundary_partition_data
std::vector< unsigned int > boundary_partition_data
Definition task_info.h:518
internal::MatrixFreeFunctions::TaskInfo::loop
void loop(MFWorkerInterface &worker) const
Definition task_info.cc:350
internal::MatrixFreeFunctions::TaskInfo::make_thread_graph_partition_color
void make_thread_graph_partition_color(DynamicSparsityPattern &connectivity, std::vector< unsigned int > &renumbering, std::vector< unsigned char > &irregular_cells, const bool hp_bool)
Definition task_info.cc:1245
internal::MatrixFreeFunctions::TaskInfo::partition_n_workers
std::vector< unsigned int > partition_n_workers
Definition task_info.h:576
internal::MatrixFreeFunctions::TaskInfo::create_blocks_serial
void create_blocks_serial(const std::vector< unsigned int > &cells_with_comm, const unsigned int dofs_per_cell, const bool categories_are_hp, const std::vector< unsigned int > &cell_vectorization_categories, const bool cell_vectorization_categories_strict, const std::vector< unsigned int > &parent_relation, std::vector< unsigned int > &renumbering, std::vector< unsigned char > &incompletely_filled_vectorization)
Definition task_info.cc:794
internal::MatrixFreeFunctions::TaskInfo::make_connectivity_cells_to_blocks
void make_connectivity_cells_to_blocks(const std::vector< unsigned char > &irregular_cells, const DynamicSparsityPattern &connectivity_cells, DynamicSparsityPattern &connectivity_blocks) const
Definition task_info.cc:1659
internal::MatrixFreeFunctions::TaskInfo::make_partitioning_within_partitions_post_blocked
void make_partitioning_within_partitions_post_blocked(const DynamicSparsityPattern &connectivity, const std::vector< unsigned int > &cell_active_fe_index, const unsigned int partition, const unsigned int cluster_size, const bool hp_bool, const std::vector< unsigned int > &cell_partition, const std::vector< unsigned int > &partition_list, const std::vector< unsigned int > &partition_size, std::vector< unsigned int > &partition_partition_list, std::vector< unsigned char > &irregular_cells)
Definition task_info.cc:1703
internal::MatrixFreeFunctions::TaskInfo::initial_setup_blocks_tasks
void initial_setup_blocks_tasks(const std::vector< unsigned int > &boundary_cells, std::vector< unsigned int > &renumbering, std::vector< unsigned char > &incompletely_filled_vectorization)
Definition task_info.cc:1151
internal::MatrixFreeFunctions::TaskInfo::print_memory_statistics
void print_memory_statistics(StreamType &out, std::size_t data_length) const
Definition task_info.cc:690
internal::MatrixFreeFunctions::TaskInfo::make_coloring_within_partitions_pre_blocked
void make_coloring_within_partitions_pre_blocked(const DynamicSparsityPattern &connectivity, const unsigned int partition, const std::vector< unsigned int > &cell_partition, const std::vector< unsigned int > &partition_list, const std::vector< unsigned int > &partition_size, std::vector< unsigned int > &partition_color_list)
Definition task_info.cc:2024
internal::MatrixFreeFunctions::TaskInfo::partition_row_index
std::vector< unsigned int > partition_row_index
Definition task_info.h:466
internal::MatrixFreeFunctions::TaskInfo::make_thread_graph_partition_partition
void make_thread_graph_partition_partition(const std::vector< unsigned int > &cell_active_fe_index, DynamicSparsityPattern &connectivity, std::vector< unsigned int > &renumbering, std::vector< unsigned char > &irregular_cells, const bool hp_bool)
Definition task_info.cc:1591
internal::MatrixFreeFunctions::TaskInfo::partition_evens
std::vector< unsigned int > partition_evens
Definition task_info.h:558
internal::MatrixFreeFunctions::TaskInfo::make_thread_graph
void make_thread_graph(const std::vector< unsigned int > &cell_active_fe_index, DynamicSparsityPattern &connectivity, std::vector< unsigned int > &renumbering, std::vector< unsigned char > &irregular_cells, const bool hp_bool)
Definition task_info.cc:1394
internal::MatrixFreeFunctions::TaskInfo::partition_n_blocked_workers
std::vector< unsigned int > partition_n_blocked_workers
Definition task_info.h:570
internal::MatrixFreeFunctions::TaskInfo::cell_partition_data
std::vector< unsigned int > cell_partition_data
Definition task_info.h:474
internal::MatrixFreeFunctions::TaskInfo::make_partitioning
void make_partitioning(const DynamicSparsityPattern &connectivity, const unsigned int cluster_size, std::vector< unsigned int > &cell_partition, std::vector< unsigned int > &partition_list, std::vector< unsigned int > &partition_size, unsigned int &partition) const
Definition task_info.cc:2100
internal::MatrixFreeFunctions::TaskInfo::update_task_info
void update_task_info(const unsigned int partition)
Definition task_info.cc:2329
internal::MatrixFreeFunctions::TaskInfo::make_boundary_cells_divisible
void make_boundary_cells_divisible(std::vector< unsigned int > &boundary_cells)
Definition task_info.cc:722
internal::MatrixFreeFunctions::TaskInfo::guess_block_size
void guess_block_size(const unsigned int dofs_per_cell)
Definition task_info.cc:1217
internal::MatrixFreeFunctions::TaskInfo::partition_odds
std::vector< unsigned int > partition_odds
Definition task_info.h:564
internal::MatrixFreeFunctions::TaskInfo::face_partition_data
std::vector< unsigned int > face_partition_data
Definition task_info.h:496
task_info.h