Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

8292083: Detected container memory limit may exceed physical machine memory #9880

Closed
wants to merge 25 commits into from
Closed
Show file tree
Hide file tree
Changes from 22 commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
b0c9cfb
Bound container memory above by host memory
jerboaa Aug 9, 2022
2304a76
Linux: clamp os::physical_memory by Linux::physical_memory
jmtd Aug 11, 2022
2de864c
Add a test for 8292083
jmtd Aug 9, 2022
7a63709
Ensure trace log is enabled before trace logging
jmtd Aug 17, 2022
8d7e80c
Separate out debug logging for three invalid memory limit scenarios
jmtd Aug 17, 2022
7f5307d
restructure conditions for legibility
jmtd Aug 17, 2022
7ef0076
Rename to more descriptive testContainerMemExceedsPhysical
jmtd Aug 18, 2022
4d8a857
Don't sanity check mem limit in OSContainer::init
jmtd Aug 18, 2022
9866464
Remove set_physical_memory (unneeded)
jmtd Aug 18, 2022
fc2ae1b
fixup! Don't sanity check mem limit in OSContainer::init
jmtd Aug 18, 2022
07a1d9b
debug log physical memory (not cgroup constrained)
jmtd Aug 18, 2022
ff57cf4
Simplify testContainerMemExceedsPhysical, avoid OperatingSystemMXBean
jmtd Aug 18, 2022
7e64194
tidy up log_debug calls in os::physical_memory
jmtd Aug 19, 2022
caa7913
Remove cgroup sanity checking logic from os::Linux::available_memory
jmtd Aug 19, 2022
66bb149
Replace _unlimited_memory with calls to os::Linux
jmtd Aug 22, 2022
a88bb62
Merge remote-tracking branch 'origin/master' into 8292083-cgroups-bad…
jmtd Aug 22, 2022
9de831d
Rework os::Linux::host_swap() to calculate on call
jmtd Aug 22, 2022
4984ddd
TestMemoryAwareness: assert we can get baseline physical RAM
jmtd Aug 22, 2022
6086505
Avoid memory_usage_in_bytes when unconstrained
jmtd Aug 23, 2022
bc493c2
Move cgroup max memory sanity checking to
jmtd Aug 23, 2022
40930a8
Remove superfluous log line from os::Linux::available_memory
jmtd Aug 23, 2022
d13fae1
Improve sanity checking legibility and document in a comment
jmtd Aug 23, 2022
ac64693
avoid calling OSContainer::memory_usage_in_bytes
jmtd Aug 23, 2022
bd24938
adjust shouldMatch regex to account for cgroups 1 vs 2 differences
jmtd Aug 23, 2022
af509e1
Address style nit
jmtd Aug 24, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
23 changes: 23 additions & 0 deletions src/hotspot/os/linux/cgroupSubsystem_linux.cpp
Expand Up @@ -558,7 +558,30 @@ jlong CgroupSubsystem::memory_limit_in_bytes() {
if (!memory_limit->should_check_metric()) {
return memory_limit->value();
}
jlong phys_mem = os::Linux::physical_memory();
log_trace(os, container)("total physical memory: " JLONG_FORMAT, phys_mem);
jlong mem_limit = read_memory_limit_in_bytes();

if (mem_limit <= 0 || mem_limit >= phys_mem) {
jlong read_mem_limit = mem_limit;
const char *reason;
if (mem_limit >= phys_mem) {
// Exceeding physical memory is treated as unlimited. Cg v1's implementation
// of read_memory_limit_in_bytes() caps this at phys_mem since Cg v1 has no
// value to represent 'max'. Cg v2 may return a value >= phys_mem if e.g. the
// container engine was started with a memory flag exceeding it.
reason = "ignored";
mem_limit = -1;
} else if (OSCONTAINER_ERROR == mem_limit) {
reason = "failed";
} else {
assert(mem_limit == -1, "Expected unlimited");
reason = "unlimited";
}
log_debug(os, container)("container memory limit %s: " JLONG_FORMAT ", using host value " JLONG_FORMAT,
reason, read_mem_limit, phys_mem);
}

// Update cached metric to avoid re-reading container settings too often
memory_limit->set_value(mem_limit, OSCONTAINER_CACHE_TIMEOUT);
return mem_limit;
Expand Down
15 changes: 9 additions & 6 deletions src/hotspot/os/linux/cgroupV1Subsystem_linux.cpp
Expand Up @@ -31,6 +31,7 @@
#include "runtime/globals.hpp"
#include "runtime/os.hpp"
#include "utilities/globalDefinitions.hpp"
#include "os_linux.hpp"

/*
* Set directory to subsystem specific files based
Expand Down Expand Up @@ -91,15 +92,15 @@ jlong CgroupV1Subsystem::read_memory_limit_in_bytes() {
GET_CONTAINER_INFO(julong, _memory->controller(), "/memory.limit_in_bytes",
"Memory Limit is: " JULONG_FORMAT, JULONG_FORMAT, memlimit);

if (memlimit >= _unlimited_memory) {
if (memlimit >= os::Linux::physical_memory()) {
log_trace(os, container)("Non-Hierarchical Memory Limit is: Unlimited");
CgroupV1MemoryController* mem_controller = reinterpret_cast<CgroupV1MemoryController*>(_memory->controller());
if (mem_controller->is_hierarchical()) {
const char* matchline = "hierarchical_memory_limit";
const char* format = "%s " JULONG_FORMAT;
GET_CONTAINER_INFO_LINE(julong, _memory->controller(), "/memory.stat", matchline,
"Hierarchical Memory Limit is: " JULONG_FORMAT, format, hier_memlimit)
if (hier_memlimit >= _unlimited_memory) {
if (hier_memlimit >= os::Linux::physical_memory()) {
log_trace(os, container)("Hierarchical Memory Limit is: Unlimited");
} else {
return (jlong)hier_memlimit;
Expand All @@ -113,17 +114,19 @@ jlong CgroupV1Subsystem::read_memory_limit_in_bytes() {
}

jlong CgroupV1Subsystem::memory_and_swap_limit_in_bytes() {
julong host_total_memsw;
GET_CONTAINER_INFO(julong, _memory->controller(), "/memory.memsw.limit_in_bytes",
"Memory and Swap Limit is: " JULONG_FORMAT, JULONG_FORMAT, memswlimit);
if (memswlimit >= _unlimited_memory) {
host_total_memsw = os::Linux::host_swap() + os::Linux::physical_memory();
if (memswlimit >= host_total_memsw) {
log_trace(os, container)("Non-Hierarchical Memory and Swap Limit is: Unlimited");
CgroupV1MemoryController* mem_controller = reinterpret_cast<CgroupV1MemoryController*>(_memory->controller());
if (mem_controller->is_hierarchical()) {
const char* matchline = "hierarchical_memsw_limit";
const char* format = "%s " JULONG_FORMAT;
GET_CONTAINER_INFO_LINE(julong, _memory->controller(), "/memory.stat", matchline,
"Hierarchical Memory and Swap Limit is : " JULONG_FORMAT, format, hier_memswlimit)
if (hier_memswlimit >= _unlimited_memory) {
if (hier_memswlimit >= host_total_memsw) {
log_trace(os, container)("Hierarchical Memory and Swap Limit is: Unlimited");
} else {
jlong swappiness = read_mem_swappiness();
Expand Down Expand Up @@ -158,7 +161,7 @@ jlong CgroupV1Subsystem::read_mem_swappiness() {
jlong CgroupV1Subsystem::memory_soft_limit_in_bytes() {
GET_CONTAINER_INFO(julong, _memory->controller(), "/memory.soft_limit_in_bytes",
"Memory Soft Limit is: " JULONG_FORMAT, JULONG_FORMAT, memsoftlimit);
if (memsoftlimit >= _unlimited_memory) {
if (memsoftlimit >= os::Linux::physical_memory()) {
log_trace(os, container)("Memory Soft Limit is: Unlimited");
return (jlong)-1;
} else {
Expand Down Expand Up @@ -205,7 +208,7 @@ jlong CgroupV1Subsystem::kernel_memory_usage_in_bytes() {
jlong CgroupV1Subsystem::kernel_memory_limit_in_bytes() {
GET_CONTAINER_INFO(julong, _memory->controller(), "/memory.kmem.limit_in_bytes",
"Kernel Memory Limit is: " JULONG_FORMAT, JULONG_FORMAT, kmem_limit);
if (kmem_limit >= _unlimited_memory) {
if (kmem_limit >= os::Linux::physical_memory()) {
return (jlong)-1;
}
return (jlong)kmem_limit;
Expand Down
3 changes: 0 additions & 3 deletions src/hotspot/os/linux/cgroupV1Subsystem_linux.hpp
Expand Up @@ -104,8 +104,6 @@ class CgroupV1Subsystem: public CgroupSubsystem {
CachingCgroupController * cpu_controller() { return _cpu; }

private:
julong _unlimited_memory;

/* controllers */
CachingCgroupController* _memory = NULL;
CgroupV1Controller* _cpuset = NULL;
Expand All @@ -128,7 +126,6 @@ class CgroupV1Subsystem: public CgroupSubsystem {
_cpuacct = cpuacct;
_pids = pids;
_memory = new CachingCgroupController(memory);
_unlimited_memory = (LONG_MAX / os::vm_page_size()) * os::vm_page_size();
}
};

Expand Down
9 changes: 0 additions & 9 deletions src/hotspot/os/linux/osContainer_linux.cpp
Expand Up @@ -43,8 +43,6 @@ CgroupSubsystem* cgroup_subsystem;
* we are running under cgroup control.
*/
void OSContainer::init() {
jlong mem_limit;

assert(!_is_initialized, "Initializing OSContainer more than once");

_is_initialized = true;
Expand All @@ -60,15 +58,8 @@ void OSContainer::init() {
if (cgroup_subsystem == NULL) {
return; // Required subsystem files not found or other error
}
// We need to update the amount of physical memory now that
// cgroup subsystem files have been processed.
if ((mem_limit = cgroup_subsystem->memory_limit_in_bytes()) > 0) {
jmtd marked this conversation as resolved.
Show resolved Hide resolved
os::Linux::set_physical_memory(mem_limit);
log_info(os, container)("Memory Limit is: " JLONG_FORMAT, mem_limit);
}

_is_containerized = true;

}

const char * OSContainer::container_type() {
Expand Down
21 changes: 12 additions & 9 deletions src/hotspot/os/linux/os_linux.cpp
Expand Up @@ -194,15 +194,12 @@ julong os::Linux::available_memory() {
julong avail_mem;

if (OSContainer::is_containerized()) {
jlong mem_limit, mem_usage;
if ((mem_limit = OSContainer::memory_limit_in_bytes()) < 1) {
log_debug(os, container)("container memory limit %s: " JLONG_FORMAT ", using host value",
mem_limit == OSCONTAINER_ERROR ? "failed" : "unlimited", mem_limit);
}
if (mem_limit > 0 && (mem_usage = OSContainer::memory_usage_in_bytes()) < 1) {
jlong mem_limit = OSContainer::memory_limit_in_bytes();
jlong mem_usage = OSContainer::memory_usage_in_bytes();
if (mem_limit > 0 && mem_usage < 1) {
jmtd marked this conversation as resolved.
Show resolved Hide resolved
log_debug(os, container)("container memory usage failed: " JLONG_FORMAT ", using host value", mem_usage);
}
if (mem_limit > 0 && mem_usage > 0 ) {
if (mem_limit > 0 && mem_usage > 0) {
avail_mem = mem_limit > mem_usage ? (julong)mem_limit - (julong)mem_usage : 0;
log_trace(os)("available container memory: " JULONG_FORMAT, avail_mem);
return avail_mem;
Expand All @@ -223,8 +220,6 @@ julong os::physical_memory() {
log_trace(os)("total container memory: " JLONG_FORMAT, mem_limit);
return mem_limit;
}
log_debug(os, container)("container memory limit %s: " JLONG_FORMAT ", using host value",
mem_limit == OSCONTAINER_ERROR ? "failed" : "unlimited", mem_limit);
jmtd marked this conversation as resolved.
Show resolved Hide resolved
}

phys_mem = Linux::physical_memory();
Expand Down Expand Up @@ -340,6 +335,14 @@ pid_t os::Linux::gettid() {
return (pid_t)rslt;
}

// Returns the amount of swap currently configured, in bytes.
// This can change at any time.
julong os::Linux::host_swap() {
struct sysinfo si;
sysinfo(&si);
return (julong)si.totalswap;
}

// Most versions of linux have a bug where the number of processors are
// determined by looking at the /proc file system. In a chroot environment,
// the system call returns 1.
Expand Down
5 changes: 3 additions & 2 deletions src/hotspot/os/linux/os_linux.hpp
Expand Up @@ -57,8 +57,6 @@ class os::Linux {
static pthread_t _main_thread;

static julong available_memory();
static julong physical_memory() { return _physical_memory; }
static void set_physical_memory(julong phys_mem) { _physical_memory = phys_mem; }
static int active_processor_count();

static void initialize_system_info();
Expand Down Expand Up @@ -131,6 +129,9 @@ class os::Linux {
static address initial_thread_stack_bottom(void) { return _initial_thread_stack_bottom; }
static uintptr_t initial_thread_stack_size(void) { return _initial_thread_stack_size; }

static julong physical_memory() { return _physical_memory; }
static julong host_swap();

static intptr_t* ucontext_get_sp(const ucontext_t* uc);
static intptr_t* ucontext_get_fp(const ucontext_t* uc);

Expand Down
26 changes: 26 additions & 0 deletions test/hotspot/jtreg/containers/docker/TestMemoryAwareness.java
Expand Up @@ -24,6 +24,7 @@

/*
* @test
* @bug 8146115 8292083
* @key cgroups
* @summary Test JVM's memory resource awareness when running inside docker container
* @requires docker.support
Expand All @@ -41,6 +42,8 @@
import jdk.test.lib.containers.docker.DockerTestUtils;
import jdk.test.lib.process.OutputAnalyzer;

import static jdk.test.lib.Asserts.assertNotNull;

public class TestMemoryAwareness {
private static final String imageName = Common.imageName("memory");

Expand Down Expand Up @@ -76,6 +79,7 @@ public static void main(String[] args) throws Exception {
"1G", Integer.toString(((int) Math.pow(2, 20)) * 1024),
"1500M", Integer.toString(((int) Math.pow(2, 20)) * (1500 - 1024))
);
testContainerMemExceedsPhysical();
} finally {
if (!DockerTestUtils.RETAIN_IMAGE_AFTER_TEST) {
DockerTestUtils.removeDockerImage(imageName);
Expand All @@ -96,6 +100,28 @@ private static void testMemoryLimit(String valueToSet, String expectedTraceValue
.shouldMatch("Memory Limit is:.*" + expectedTraceValue);
}

// JDK-8292083
// Ensure that Java ignores container memory limit values above the host's physical memory.
private static void testContainerMemExceedsPhysical()
throws Exception {

Common.logNewTestCase("container memory limit exceeds physical memory");

DockerRunOptions opts = Common.newOpts(imageName);

// first run: establish physical memory in test environment and derive
// a bad value one power of ten larger
String goodMem = Common.run(opts).firstMatch("total physical memory: (\\d+)", 1);
jmtd marked this conversation as resolved.
Show resolved Hide resolved
assertNotNull(goodMem, "no match for 'total physical memory' in trace output");
String badMem = goodMem + "0";

// second run: set a container memory limit to the bad value
opts = Common.newOpts(imageName)
.addDockerOpts("--memory", badMem);
Common.run(opts)
.shouldMatch("container memory limit ignored: "+badMem+", using host value "+goodMem);
jmtd marked this conversation as resolved.
Show resolved Hide resolved
}


private static void testMemorySoftLimit(String valueToSet, String expectedTraceValue)
throws Exception {
Expand Down