[Boost-users] [Interprocess] Unable to Access More than 7 GB of Memory

Subject: [Boost-users] [Interprocess] Unable to Access More than 7 GB of Memory
From: Hering Cheng (hering.cheng_at_[hidden])
Date: 2012-06-27 14:00:25

Hi,

Using Boost.Interprocess, with either unmanaged or managed shared memory, I
am unable to access beyond 7 GB of the 20 GB of memory I allocated, despite
the fact that none of the Interprocess API calls returned any error or
threw any exception.  I run into segmentation fault when I actually try to
access the portion of the memory above the (approximately) 7 GB limit.

If I perform a similar test with shared memory constructed via manual calls
to System V IPC, then I can access all of the 20 GB of memory without any
problem.

I suspect this may be due to mmap() (using by Interprocess along with POSIX
shm_open() and other calls) but I have been unable to verify.  Does anyone
have any experience on this?

The machine is running RedHat 5 with 16 GB of RAM.  (Yes, when I go through
the 20 GB of memory a lot of paging/swapping occurred.)  From below, I do
not think that the problem was due to configuration.

$ uname -a
Linux NJ-CEL01 2.6.18-164.11.1.el5 #1 SMP Wed Jan 6 13:26:04 EST 2010
x86_64 x86_64 x86_64 GNU/Linux

$ ulimit -a
core file size          (blocks, -c) 2147483648
data seg size           (kbytes, -d) unlimited
scheduling priority             (-e) 0
file size               (blocks, -f) unlimited
pending signals                 (-i) 141312
max locked memory       (kbytes, -l) 32
max memory size         (kbytes, -m) unlimited
open files                      (-n) 1024
pipe size            (512 bytes, -p) 8
POSIX message queues     (bytes, -q) 819200
real-time priority              (-r) 0
stack size              (kbytes, -s) 10240
cpu time               (seconds, -t) unlimited
max user processes              (-u) 141312
virtual memory          (kbytes, -v) unlimited
file locks                      (-x) unlimited

$ grep . /proc/sys/kernel/shm*
/proc/sys/kernel/shmall:4294967296
/proc/sys/kernel/shmmax:68719476736
/proc/sys/kernel/shmmni:4096

$ getconf PAGE_SIZE
4096

The following is the code for the test cases.

struct entry_t {
    uint64_t f[10];
};

BOOST_FIXTURE_TEST_CASE(sysv_ipc, fixture) {
    std::cout << "System V IPC shared memory" << std::endl;

    const std::string shm_key = "/proc/sys/kernel/shmall";
    const std::size_t shm_size = 20L * 1024L * 1024L * 1024L;
    const key_t ipc_key = ftok(shm_key.c_str(), 0);
    const int rm_rc = shmctl(ipc_key, IPC_RMID, NULL);

    if (rm_rc < 0) {
        std::cerr << "Unable to remove shared memory: " << strerror(errno)
                << std::endl;
    }

    const int shm_id = shmget(ipc_key, shm_size, 0644 | IPC_CREAT |
IPC_EXCL);
    BOOST_ASSERT_MSG(shm_id >= 0, strerror(errno));
    void * const shm_addr = shmat(shm_id, NULL, 0);
    BOOST_ASSERT_MSG(shm_addr != reinterpret_cast<void *>(-1),
strerror(errno));

    const std::size_t entry_count = 90L * (shm_size / sizeof(entry_t)) /
100L;
    entry_t * array = reinterpret_cast<entry_t *>(shm_addr);
    std::cout << "Initializing " << entry_count << " entries..." <<
std::endl;

    for (std::size_t i = 0; i < entry_count; ++i) {
        entry_t & entry = array[i];
        void * addr = & entry;
        new (addr) entry_t();

        if (i % 1000000 == 0) {
            std::cout << i << std::endl;
        }
    }

    std::cout << "Initialized all entries" << std::endl;
    shmctl(ipc_key, IPC_RMID, NULL);
}

BOOST_FIXTURE_TEST_CASE(unmanaged, fixture) {
    std::cout << "Unmanaged shared memory" << std::endl;

    const std::string shm_key = "test";
    const std::size_t shm_size = 20L * 1024L * 1024L * 1024L;
    bipc::shared_memory_object::remove(shm_key.c_str());
    bipc::shared_memory_object shm_obj(bipc::create_only,
            shm_key.c_str(), bipc::mode_t::read_write,
            bipc::permissions(0666));
    shm_obj.truncate(shm_size);
    bipc::mapped_region region(shm_obj, bipc::read_write);
    const std::size_t entry_count = 90L * (shm_size / sizeof(entry_t)) /
100L;
    entry_t * array = reinterpret_cast<entry_t *>(region.get_address());

    for (std::size_t i = 0; i < entry_count; ++i) {
        entry_t & entry = array[i];
        void * addr = & entry;
        new (addr) entry_t();

        if (i % 1000000 == 0) {
            std::cout << i << std::endl;
        }
    }

    bipc::shared_memory_object::remove(shm_key.c_str());
}

BOOST_FIXTURE_TEST_CASE(managed, fixture) {
    std::cout << "Managed shared memory" << std::endl;

    const std::string shm_key = "test";
    const std::size_t shm_size = 20L * 1024L * 1024L * 1024L;
    bipc::shared_memory_object::remove(shm_key.c_str());
    bipc::managed_shared_memory shm(bipc::create_only, shm_key.c_str(),
            shm_size);
    typedef bipc::managed_shared_memory::allocator<entry_t>::type
allocator_t;
    allocator_t array_alloc(shm.get_segment_manager());
    const std::size_t entry_count = 90L * (shm_size / sizeof(entry_t)) /
100L;
    allocator_t::pointer array_ptr = array_alloc.allocate(entry_count);
    entry_t * array = array_ptr.get();

    for (std::size_t i = 0; i < entry_count; ++i) {
        entry_t & entry = array[i];
        void * addr = & entry;
        new (addr) entry_t();

        if (i % 1000000 == 0) {
            std::cout << i << std::endl;
        }
    }

    bipc::shared_memory_object::remove(shm_key.c_str());
}

Thanks.
Hering