[Boost-users] [boost-mpi] spawning and leaked comm handles

Subject: [Boost-users] [boost-mpi] spawning and leaked comm handles
From: Brian Budge (brian.budge_at_[hidden])
Date: 2012-08-30 12:49:26

Hi all -

I wasn't entirely sure if I should post this here, or at the OpenMPI
list, but I believe my issue is coming from the way I'm using
boost.mpi.  I appear to be leaking MPI_Comm handles.  Since I intend
to spawn hundreds of times during the lifetime of my program, this is
less than ideal ;)

Here's the code

//common.h
#include <boost/mpi.hpp>

std::string get_processor_name() {
    char buf[256];
    int blen = 256;
    MPI_Get_processor_name(buf, &blen);
    return std::string(buf);
}

//master.cpp
#include "common.h"
void doWork(size_t numChildren, char *cmd) {
    namespace mpi = boost::mpi;
    MPI_Comm all;
    MPI_Comm_spawn(cmd, MPI_ARGV_NULL, numChildren, MPI_INFO_NULL, 0,
MPI_COMM_SELF, &all, MPI_ERRCODES_IGNORE);
    mpi::intercommunicator intercomm(all, mpi::comm_duplicate);
    mpi::communicator comm = intercomm.merge(false);
    std::cerr << "spawned by " << get_processor_name() << " (rank " <<
comm.rank() << ")" << std::endl;

    std::string str;
    for(int i = 1; i < comm.size(); ++i) {
        comm.recv(i, 0, str);
        std::cerr << "got string " << str << " from rank " << i << std::endl;
    }
    std::string mymsg = "hi back form the master";

    broadcast(comm, mymsg, 0);
}

int main(int argc, char **args) {
    std::cerr << "hi, it's me" << std::endl;
    //mpi::environment env;
    int provided;
    MPI_Init_thread(NULL, NULL, MPI_THREAD_MULTIPLE, &provided);

    std::ifstream inFile("hostsfile");
    std::string dest;
    size_t totalProcesses = 0;
    std::string hosts;
    while(std::getline(inFile, dest)) {
        totalProcesses += (!dest.empty() && dest[0] != '#');
    }

    char *cmd = strdup("/home/budgeb/tests/test_mpi/spawnslave");
    for(size_t j = 0; j < 2; ++j) {
        doWork(totalProcesses - 1, cmd);
    }
    free(cmd);
    MPI_Finalize();
    return 0;
}

//slave.cpp
#include "common.h"
#include <boost/lexical_cast.hpp>
#include <math.h>

void doWork() {
    namespace mpi = boost::mpi;
    MPI_Comm parent;
    MPI_Comm_get_parent(&parent);
    mpi::intercommunicator intercomm(parent, mpi::comm_duplicate);
    mpi::communicator comm = intercomm.merge(true);

    std::cerr << "slave " << get_processor_name() << " of rank " <<
comm.rank() << std::endl;

    double d = comm.rank();
    for(size_t i = 0; i < 100000000; ++i) {
        d = sin(exp(d));
    }

    comm.send(0, 0, std::string("hello world (") +
boost::lexical_cast<std::string>(d) + ")");

    std::string masterMsg;
    broadcast(comm, masterMsg, 0);

    std::cerr << "master sent this: " << masterMsg << std::endl;
}

int main(int argc, char **args) {
    int provided;
    MPI_Init_thread(NULL, NULL, MPI_THREAD_MULTIPLE, &provided);
    doWork();
    MPI_Finalize();

    return 0;
}

When I run this program, my output is fine, but upon shutdown, I get this:
[budgeb-interlagos:04473] WARNING: MPI_Comm still allocated in MPI_Finalize
[budgeb-interlagos:04473] Dumping information for comm_cid 4
[budgeb-interlagos:04473]   f2c index:4 cube_dim: 0
[budgeb-interlagos:04473]   Local group: size = 1 my_rank = 0
[budgeb-interlagos:04473]   Communicator is:
[budgeb-interlagos:04473]  inter-comm,
[budgeb-interlagos:04473]
[budgeb-interlagos:04473]   Remote group size:3
[budgeb-interlagos:04473] WARNING: MPI_Comm still allocated in MPI_Finalize
[budgeb-interlagos:04473] Dumping information for comm_cid 6
[budgeb-interlagos:04473]   f2c index:6 cube_dim: 0
[budgeb-interlagos:04473]   Local group: size = 1 my_rank = 0
[budgeb-interlagos:04473]   Communicator is:
[budgeb-interlagos:04473]  inter-comm,
[budgeb-interlagos:04473]
[budgeb-interlagos:04473]   Remote group size:3

Does anyone know why this leak might occur?

Thanks,
  Brian