10#ifndef TPETRA_DISTRIBUTOR_HPP
11#define TPETRA_DISTRIBUTOR_HPP
13#include "Tpetra_Details_DistributorActor.hpp"
17#include "Teuchos_as.hpp"
18#include "Teuchos_Describable.hpp"
19#include "Teuchos_ParameterListAcceptorDefaultBase.hpp"
20#include "Teuchos_VerboseObject.hpp"
23#include "KokkosCompat_View.hpp"
24#include "Kokkos_Core.hpp"
25#include "Kokkos_TeuchosCommAdapters.hpp"
103 public Teuchos::Describable,
104 public Teuchos::ParameterListAcceptorDefaultBase {
117 explicit Distributor (
const Teuchos::RCP<
const Teuchos::Comm<int> >& comm);
130 Distributor (
const Teuchos::RCP<
const Teuchos::Comm<int> >& comm,
131 const Teuchos::RCP<Teuchos::FancyOStream>& out);
146 Distributor (
const Teuchos::RCP<
const Teuchos::Comm<int> >& comm,
147 const Teuchos::RCP<Teuchos::ParameterList>& plist);
165 Distributor (
const Teuchos::RCP<
const Teuchos::Comm<int> >& comm,
166 const Teuchos::RCP<Teuchos::FancyOStream>& out,
167 const Teuchos::RCP<Teuchos::ParameterList>& plist);
224 size_t createFromSends (
const Teuchos::ArrayView<const int>& exportProcIDs);
259 template <
class Ordinal>
262 const Teuchos::ArrayView<const int>& remoteProcIDs,
263 Teuchos::Array<Ordinal>& exportIDs,
264 Teuchos::Array<int>& exportProcIDs);
275 const Teuchos::ArrayView<const int>& remoteProcIDs);
310 Teuchos::ArrayView<const int>
getProcsTo()
const;
335 return plan_.howInitialized();
352 Teuchos::RCP<Distributor>
getReverse(
bool create=
true)
const;
394 template <
class ExpView,
class ImpView>
395 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
397 const ExpView &exports,
399 const ImpView &imports);
422 template <
class ExpView,
class ImpView>
423 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
425 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
426 const ImpView &imports,
427 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID);
453 template <
class ExpView,
class ImpView>
454 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
455 doPosts (
const ExpView &exports,
457 const ImpView &imports);
477 template <
class ExpView,
class ImpView>
478 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
479 doPosts (
const ExpView &exports,
480 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
481 const ImpView &imports,
482 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID);
488 template <
class ExpView,
class ImpView>
489 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
492 const ImpView &imports);
498 template <
class ExpView,
class ImpView>
499 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
501 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
502 const ImpView &imports,
503 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID);
509 template <
class ExpView,
class ImpView>
510 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
513 const ImpView &imports);
519 template <
class ExpView,
class ImpView>
520 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
522 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
523 const ImpView &imports,
524 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID);
555 describe (Teuchos::FancyOStream& out,
556 const Teuchos::EVerbosityLevel verbLevel =
557 Teuchos::Describable::verbLevel_default)
const;
567 Details::DistributorActor actor_;
573 static bool getVerbose();
579 std::unique_ptr<std::string>
580 createPrefix(
const char methodName[])
const;
583 bool verbose_ = getVerbose();
590 mutable Teuchos::RCP<Distributor> reverseDistributor_;
604 template <
class Ordinal>
605 void computeSends (
const Teuchos::ArrayView<const Ordinal> &remoteGIDs,
606 const Teuchos::ArrayView<const int> &remoteProcIDs,
607 Teuchos::Array<Ordinal> &exportGIDs,
608 Teuchos::Array<int> &exportProcIDs);
611 void createReverseDistributor()
const;
619 localDescribeToString (
const Teuchos::EVerbosityLevel vl)
const;
622 template <
class ExpView,
class ImpView>
623 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
627 const ImpView& imports)
629 actor_.doPostsAndWaits(plan_, exports, numPackets, imports);
632 template <
class ExpView,
class ImpView>
633 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
636 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
637 const ImpView& imports,
638 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID)
640 actor_.doPostsAndWaits(plan_, exports, numExportPacketsPerLID, imports, numImportPacketsPerLID);
644 template <
class ExpView,
class ImpView>
645 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
647 doPosts (const ExpView &exports,
649 const ImpView &imports)
651 actor_.doPosts(plan_, exports, numPackets, imports);
654 template <
class ExpView,
class ImpView>
655 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
657 doPosts (
const ExpView &exports,
658 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
659 const ImpView &imports,
660 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID)
662 actor_.doPosts(plan_, exports, numExportPacketsPerLID, imports, numImportPacketsPerLID);
665 template <
class ExpView,
class ImpView>
666 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
670 const ImpView& imports)
676 template <
class ExpView,
class ImpView>
677 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
680 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
681 const ImpView& imports,
682 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID)
685 numImportPacketsPerLID);
689 template <
class ExpView,
class ImpView>
690 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
694 const ImpView &imports)
697 TEUCHOS_TEST_FOR_EXCEPTION(
698 ! plan_.getIndicesTo().is_null(), std::runtime_error,
699 "Tpetra::Distributor::doReversePosts(3 args): Can only do "
700 "reverse communication when original data are blocked by process.");
701 if (reverseDistributor_.is_null ()) {
702 createReverseDistributor ();
704 reverseDistributor_->doPosts (exports, numPackets, imports);
707 template <
class ExpView,
class ImpView>
708 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
711 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
712 const ImpView &imports,
713 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID)
716 TEUCHOS_TEST_FOR_EXCEPTION(
717 ! plan_.getIndicesTo().is_null(), std::runtime_error,
718 "Tpetra::Distributor::doReversePosts(3 args): Can only do "
719 "reverse communication when original data are blocked by process.");
720 if (reverseDistributor_.is_null ()) {
721 createReverseDistributor ();
723 reverseDistributor_->doPosts (exports, numExportPacketsPerLID,
724 imports, numImportPacketsPerLID);
727 template <
class OrdinalType>
729 computeSends(
const Teuchos::ArrayView<const OrdinalType>& importGIDs,
730 const Teuchos::ArrayView<const int>& importProcIDs,
731 Teuchos::Array<OrdinalType>& exportGIDs,
732 Teuchos::Array<int>& exportProcIDs)
741 using Teuchos::ArrayView;
743 using size_type =
typename ArrayView<const OrdinalType>::size_type;
744 const char errPrefix[] =
"Tpetra::Distributor::computeSends: ";
745 const char suffix[] =
746 " Please report this bug to the Tpetra developers.";
748 const int myRank = plan_.getComm()->getRank ();
750 TEUCHOS_TEST_FOR_EXCEPTION
751 (importGIDs.size () != importProcIDs.size (),
752 std::invalid_argument, errPrefix <<
"On Process " << myRank
753 <<
": importProcIDs.size()=" << importProcIDs.size()
754 <<
" != importGIDs.size()=" << importGIDs.size() <<
".");
756 const size_type numImports = importProcIDs.size();
757 Kokkos::View<size_t*, Kokkos::HostSpace> importObjs(
"importObjs", 2*numImports);
759 for (size_type i = 0; i < numImports; ++i) {
760 importObjs[2*i] =
static_cast<size_t>(importGIDs[i]);
761 importObjs[2*i+1] =
static_cast<size_t>(myRank);
770 const size_t numExportsAsSizeT =
771 tempPlan.createFromSends(importProcIDs);
772 const size_type numExports =
773 static_cast<size_type
>(numExportsAsSizeT);
774 TEUCHOS_TEST_FOR_EXCEPTION
775 (numExports < 0, std::logic_error, errPrefix <<
776 "tempPlan.createFromSends() returned numExports="
777 << numExportsAsSizeT <<
" as a size_t, which overflows to "
778 << numExports <<
" when cast to " <<
779 Teuchos::TypeNameTraits<size_type>::name () <<
"." << suffix);
780 TEUCHOS_TEST_FOR_EXCEPTION
781 (size_type(tempPlan.getTotalReceiveLength()) != numExports,
782 std::logic_error, errPrefix <<
"tempPlan.getTotalReceiveLength()="
783 << tempPlan.getTotalReceiveLength () <<
" != numExports="
784 << numExports <<
"." << suffix);
786 if (numExports > 0) {
787 exportGIDs.resize(numExports);
788 exportProcIDs.resize(numExports);
799 static_assert(
sizeof(size_t) >=
sizeof(OrdinalType),
800 "Tpetra::Distributor::computeSends: "
801 "sizeof(size_t) < sizeof(OrdinalType).");
803 TEUCHOS_TEST_FOR_EXCEPTION
804 (tempPlan.getTotalReceiveLength () <
size_t(numExports),
806 errPrefix <<
"tempPlan.getTotalReceiveLength()="
807 << tempPlan.getTotalReceiveLength() <<
" < numExports="
808 << numExports <<
"." << suffix);
810 Kokkos::View<size_t*, Kokkos::HostSpace> exportObjs(
"exportObjs", tempPlan.getTotalReceiveLength() * 2);
811 tempPlan.doPostsAndWaits(importObjs, 2, exportObjs);
814 for (size_type i = 0; i < numExports; ++i) {
815 exportGIDs[i] =
static_cast<OrdinalType
> (exportObjs[2*i]);
816 exportProcIDs[i] =
static_cast<int> (exportObjs[2*i+1]);
820 template <
class OrdinalType>
822 createFromRecvs (
const Teuchos::ArrayView<const OrdinalType> &remoteGIDs,
823 const Teuchos::ArrayView<const int> &remoteProcIDs,
824 Teuchos::Array<OrdinalType> &exportGIDs,
825 Teuchos::Array<int> &exportProcIDs)
828 const char errPrefix[] =
"Tpetra::Distributor::createFromRecvs: ";
829 const int myRank = plan_.getComm()->getRank();
831 std::unique_ptr<std::string> prefix;
833 prefix = createPrefix(
"createFromRecvs");
834 std::ostringstream os;
835 os << *prefix <<
"Start" << endl;
836 std::cerr << os.str();
841 using Teuchos::outArg;
842 using Teuchos::REDUCE_MAX;
843 using Teuchos::reduceAll;
847 (remoteGIDs.size () != remoteProcIDs.size ()) ? myRank : -1;
849 reduceAll(*plan_.getComm(), REDUCE_MAX, errProc, outArg(maxErrProc));
850 TEUCHOS_TEST_FOR_EXCEPTION
851 (maxErrProc != -1, std::runtime_error, errPrefix <<
"Lists "
852 "of remote IDs and remote process IDs must have the same "
853 "size on all participating processes. Maximum process ID "
854 "with error: " << maxErrProc <<
".");
859 TEUCHOS_TEST_FOR_EXCEPTION
860 (remoteGIDs.size() != remoteProcIDs.size(), std::runtime_error,
861 errPrefix <<
"On Process " << myRank <<
": "
862 "remoteGIDs.size()=" << remoteGIDs.size() <<
863 " != remoteProcIDs.size()=" << remoteProcIDs.size() <<
".");
866 computeSends(remoteGIDs, remoteProcIDs, exportGIDs, exportProcIDs);
868 plan_.createFromRecvs(remoteProcIDs);
871 std::ostringstream os;
872 os << *prefix <<
"Done" << endl;
873 std::cerr << os.str();
Declaration of Tpetra::Details::Behavior, a class that describes Tpetra's behavior.
Stand-alone utility functions and macros.
static bool debug()
Whether Tpetra is in debug mode.
std::enable_if<(Kokkos::is_view< ExpView >::value &&Kokkos::is_view< ImpView >::value)>::type doPosts(const ExpView &exports, size_t numPackets, const ImpView &imports)
Post the data for a forward plan, but do not execute the waits yet.
std::enable_if<(Kokkos::is_view< ExpView >::value &&Kokkos::is_view< ImpView >::value)>::type doPostsAndWaits(const ExpView &exports, size_t numPackets, const ImpView &imports)
Execute the (forward) communication plan.
const Details::DistributorPlan & getPlan() const
Get this Distributor's DistributorPlan.
size_t getMaxSendLength() const
Maximum number of values this process will send to another single process.
Teuchos::RCP< Distributor > getReverse(bool create=true) const
A reverse communication plan Distributor.
void createFromRecvs(const Teuchos::ArrayView< const Ordinal > &remoteIDs, const Teuchos::ArrayView< const int > &remoteProcIDs, Teuchos::Array< Ordinal > &exportIDs, Teuchos::Array< int > &exportProcIDs)
Set up Distributor using list of process ranks from which to receive.
Teuchos::ArrayView< const int > getProcsTo() const
Ranks of the processes to which this process will send values.
size_t getNumReceives() const
The number of processes from which we will receive data.
std::enable_if<(Kokkos::is_view< ExpView >::value &&Kokkos::is_view< ImpView >::value)>::type doReversePostsAndWaits(const ExpView &exports, size_t numPackets, const ImpView &imports)
Execute the reverse communication plan.
void setParameterList(const Teuchos::RCP< Teuchos::ParameterList > &plist)
Set Distributor parameters.
size_t getTotalReceiveLength() const
Total number of values this process will receive from other processes.
virtual ~Distributor()=default
Destructor (virtual for memory safety).
bool hasSelfMessage() const
Whether the calling process will send or receive messages to itself.
void swap(Distributor &rhs)
Swap the contents of rhs with those of *this.
Teuchos::ArrayView< const size_t > getLengthsTo() const
Number of values this process will send to each process.
Teuchos::ArrayView< const int > getProcsFrom() const
Ranks of the processes sending values to this process.
Distributor(const Teuchos::RCP< const Teuchos::Comm< int > > &comm)
Construct using the specified communicator and default parameters.
std::string description() const
Return a one-line description of this object.
size_t createFromSends(const Teuchos::ArrayView< const int > &exportProcIDs)
Set up Distributor using list of process ranks to which this process will send.
void createFromSendsAndRecvs(const Teuchos::ArrayView< const int > &exportProcIDs, const Teuchos::ArrayView< const int > &remoteProcIDs)
Set up Distributor using list of process ranks to which to send, and list of process ranks from which...
Teuchos::RCP< const Teuchos::ParameterList > getValidParameters() const
List of valid Distributor parameters.
Teuchos::ArrayView< const size_t > getLengthsFrom() const
Number of values this process will receive from each process.
std::enable_if<(Kokkos::is_view< ExpView >::value &&Kokkos::is_view< ImpView >::value)>::type doReversePosts(const ExpView &exports, size_t numPackets, const ImpView &imports)
Post the data for a reverse plan, but do not execute the waits yet.
Details::EDistributorHowInitialized howInitialized() const
Return an enum indicating whether and how a Distributor was initialized.
size_t getNumSends() const
The number of processes to which we will send data.
void describe(Teuchos::FancyOStream &out, const Teuchos::EVerbosityLevel verbLevel=Teuchos::Describable::verbLevel_default) const
Describe this object in a human-readable way to the given output stream.
EDistributorHowInitialized
Enum indicating how and whether a Distributor was initialized.
Namespace Tpetra contains the class and methods constituting the Tpetra library.
Teuchos::Array< std::string > distributorSendTypes()
Valid values for Distributor's "Send type" parameter.