Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

HPCC-32764 Periodically log information about cpu-throttling #19186

Merged
merged 1 commit into from
Oct 14, 2024
Merged

HPCC-32764 Periodically log information about cpu-throttling #19186

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
112 changes: 108 additions & 4 deletions system/jlib/jdebug.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -816,6 +816,54 @@ static struct CNtKernelInformation
} NtKernelFunctions;
#endif

//===========================================================================

#ifndef _WIN32

static std::atomic<bool> gatheredGroup{false};
static StringAttr cgroup;
static CriticalSection csgroupCs;
static const char * queryCGroup()
{
if (!gatheredGroup)
{
CriticalBlock block(csgroupCs);
if (!gatheredGroup)
{
StringBuffer contents;
if (loadBinaryFile(contents, "/proc/self/cgroup", false))
{
auto processLine = [](size_t len, const char * ln)
{
//Note ln points at the start of the line, but the line is not null terminated
switch (*ln)
{
case '0':
if (strncmp(ln, "0::/", 4) == 0)
{
//Format is 0::/<cgroup>
//If not running in a container the "cgroup" may be something like user.slice/user-1000.slice/user@1000.service/....
//If so ignore because it is not a real cgroup
if (!memchr(ln+4, '/', len-4))
cgroup.set(ln+4, len-4);
}
break;
}
//Some systems with version 1 cgroups have <n>:cpu,cpuacct:/<cgroup>
const char * match = (const char *)jmemmem(len, ln, 14, ":cpu,cpuacct:/");
if (match)
cgroup.set(match+14, (ln + len) - (match + 14));
};

processLines(contents, processLine);
}
gatheredGroup = true;
}
}
return cgroup.get();
}

#endif

//===========================================================================

Expand Down Expand Up @@ -893,6 +941,9 @@ SystemProcessInfo SystemProcessInfo::operator - (const SystemProcessInfo & rhs)
result.activeDataMemory = activeDataMemory - rhs.activeDataMemory;
result.majorFaults = majorFaults - rhs.majorFaults;
result.numThreads = numThreads - rhs.numThreads;
result.numPeriods = numPeriods - rhs.numPeriods;
result.numThrottledPeriods = numThrottledPeriods - rhs.numThrottledPeriods;
result.timeThrottledNs = timeThrottledNs - rhs.timeThrottledNs;
return result;
}

Expand Down Expand Up @@ -995,7 +1046,7 @@ bool ProcessInfo::update(unsigned flags)
if (loadBinaryFile(contents, "/proc/self/status", false))
{
contextSwitches = 0;
auto processLine = [this](const char * cur)
auto processLine = [this](size_t len, const char * cur)
{
__uint64 value;
switch (*cur)
Expand Down Expand Up @@ -1070,7 +1121,7 @@ bool SystemInfo::update(unsigned flags)
StringBuffer contents;
if (loadBinaryFile(contents, "/proc/stat", false))
{
auto processLine = [this](const char * ln)
auto processLine = [this](size_t len, const char * ln)
{
switch (*ln)
{
Expand Down Expand Up @@ -1105,6 +1156,52 @@ bool SystemInfo::update(unsigned flags)
processLines(contents, processLine);
}

auto processLine = [this](size_t len, const char * ln)
{
switch (*ln)
{
case 'n':
if (strncmp(ln, "nr_periods ", 11) == 0)
numPeriods = strtod(ln+11, nullptr);
else if (strncmp(ln, "nr_throttled ", 13) == 0)
numThrottledPeriods = strtod(ln+13, nullptr);
break;
case 't':
if (strncmp(ln, "throttled_usec ", 15) == 0)
timeThrottledNs = strtod(ln+15, nullptr) * 1000;
else if (strncmp(ln, "throttled_time ", 15) == 0)
timeThrottledNs = strtod(ln+15, nullptr);
break;
}
};

bool done = false;
const char * cgroup = queryCGroup();
if (cgroup)
{
//Version 2 of cgroups has the information in cgroup/<cgroup>
VStringBuffer filename("/sys/fs/cgroup/%s/cpu.stat", cgroup);
if (loadBinaryFile(contents.clear(), filename.str(), false))
{
processLines(contents, processLine);
done = true;
}
else
{
//Some systems with version 1 cgroups have the information in /sys/fs/cgroup/cpu/<cgroup>/cpu.stat
filename.clear().appendf("/sys/fs/cgroup/cpu/%s/cpu.stat", cgroup);
if (loadBinaryFile(contents.clear(), filename.str(), false))
{
processLines(contents, processLine);
done = true;
}
}
}

//If the version 2 file was not found look for ther version 1 information in cgroup/cpu
if (!done && loadBinaryFile(contents.clear(), "/sys/fs/cgroup/cpu/cpu.stat", false))
processLines(contents, processLine);

return true;
#endif
}
Expand Down Expand Up @@ -2502,9 +2599,16 @@ class CExtendedStats // Disk network and cpu stats
}
if (totalcpu)
{
if (out.length()&&(out.charAt(out.length()-1)!=' '))
out.append(' ');
ensureSeparator(out, ' ');
out.appendf("CPU: usr=%d sys=%d iow=%d idle=%d", deltacpu.getUserPercent(), deltacpu.getSystemPercent(), deltacpu.getIoWaitPercent(), deltacpu.getIdlePercent());

__uint64 periods = deltacpu.getNumPeriods();
if (periods)
{
unsigned throttling = deltacpu.getNumThrottledPeriods() * 100 / periods;
__uint64 timeThrottledNs = deltacpu.getTimeThrottledNs();
out.appendf(" thr=%u%% thrns=%llu", throttling, timeThrottledNs);
}
}
return true;
}
Expand Down
7 changes: 7 additions & 0 deletions system/jlib/jdebug.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -446,6 +446,10 @@ class jlib_decl SystemProcessInfo
unsigned getSystemPercent() const;
unsigned getUserPercent() const;

__uint64 getNumPeriods() const { return numPeriods; }
__uint64 getNumThrottledPeriods() const { return numThrottledPeriods; }
__uint64 getTimeThrottledNs() const { return timeThrottledNs; }

__uint64 getTotal() const { return user + system + idle + iowait; }
protected:
__uint64 user = 0; // user time in jiffies (~`1/100s)
Expand All @@ -461,6 +465,9 @@ class jlib_decl SystemProcessInfo
__uint64 activeDataMemory = 0;
__uint64 majorFaults = 0;
__uint64 numThreads = 0;
__uint64 numPeriods = 0;
__uint64 numThrottledPeriods = 0;
__uint64 timeThrottledNs = 0;
};

class jlib_decl ProcessInfo : public SystemProcessInfo
Expand Down
34 changes: 34 additions & 0 deletions system/jlib/jstring.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2902,6 +2902,12 @@ void getSnakeCase(StringBuffer & out, const char * camelValue)
}
}

void ensureSeparator(StringBuffer & out, char separator)
{
if (out.length() && (out.charAt(out.length()-1) != separator))
out.append(separator);
}

/**
* stristr - Case insensitive strstr()
* @haystack: Where we will search for our @needle
Expand Down Expand Up @@ -2949,3 +2955,31 @@ const char * stristr (const char *haystack, const char *needle)
}
return nullptr;
}


const void * jmemmem(size_t lenHaystack, const void * haystack, size_t lenNeedle, const void *needle)
{
if (lenNeedle == 0)
return haystack;

if (lenHaystack < lenNeedle)
return nullptr;

const char * search = (const char *)needle;
char first = *search;
if (lenNeedle == 1)
return memchr(haystack, first, lenHaystack);

const char * buffer = (const char *)haystack;
for (size_t i = 0; i <= lenHaystack - lenNeedle; i++)
{
//Special case the first character to avoid a function call each iteration.
if (buffer[i] == first)
{
if (memcmp(buffer + i + 1, search + 1, lenNeedle-1) == 0)
return buffer + i;
}
}

return nullptr;
}
18 changes: 15 additions & 3 deletions system/jlib/jstring.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -632,11 +632,18 @@ void processLines(const StringBuffer & content, LineProcessor process)
const char * cur = content;
while (*cur)
{
process(cur);
const char * next = strchr(cur, '\n');
if (!next)
if (next)
{
if (next != cur)
process(next-cur, cur);
cur = next+1;
}
else
{
process(strlen(cur), cur);
break;
cur = next+1;
}
}
}

Expand All @@ -646,5 +653,10 @@ extern jlib_decl void processOptionString(const char * options, optionCallback c

extern jlib_decl const char * stristr(const char *haystack, const char *needle);
extern jlib_decl void getSnakeCase(StringBuffer & out, const char * camelValue);
//If the string has any characters, ensure the last character matches the separator
extern jlib_decl void ensureSeparator(StringBuffer & out, char separator);

//Search for one block of bytes within another block of bytes - memmem is not standard, so we provide our own
extern jlib_decl const void * jmemmem(size_t lenHaystack, const void * haystack, size_t lenNeedle, const void *needle);

#endif
17 changes: 17 additions & 0 deletions testing/unittests/jlibtests.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3390,6 +3390,7 @@ class MachineInfoTimingTest : public CppUnit::TestFixture
DBGLOG(" Process: User(%u) System(%u) Total(%u) %u%% Ctx(%" I64F "u)",
(unsigned)(deltaProcess.getUserNs() / 1000000), (unsigned)(deltaProcess.getSystemNs() / 1000000), (unsigned)(deltaProcess.getTotalNs() / 1000000),
(unsigned)((deltaProcess.getUserNs() * 100) / deltaSystem.getTotalNs()), deltaProcess.getNumContextSwitches());
DBGLOG(" Throttled: Periods(%llu/%llu) TimeNs(%llu)", deltaSystem.getNumThrottledPeriods(), deltaSystem.getNumPeriods(), deltaSystem.getTimeThrottledNs());
}

for (unsigned j=0; j < i*100000000; j++)
Expand Down Expand Up @@ -4630,6 +4631,7 @@ class JLibStringTest : public CppUnit::TestFixture
public:
CPPUNIT_TEST_SUITE(JLibStringTest);
CPPUNIT_TEST(testStristr);
CPPUNIT_TEST(testMemMem);
CPPUNIT_TEST_SUITE_END();

void testStristr()
Expand All @@ -4647,6 +4649,21 @@ class JLibStringTest : public CppUnit::TestFixture
CPPUNIT_ASSERT_EQUAL_STR(stristr("", "ABC"), "");
CPPUNIT_ASSERT_EQUAL_STR(stristr("ABC", ""), "");
}

void testMemMem()
{
constexpr const char * haystack = "abcdefghijklmnopqrstuvwxyz";
CPPUNIT_ASSERT_EQUAL((const void*)(haystack), jmemmem(10, haystack, 0, nullptr));
CPPUNIT_ASSERT_EQUAL((const void*)(haystack), jmemmem(10, haystack, 3, "abc"));
CPPUNIT_ASSERT_EQUAL((const void*)(haystack), jmemmem(3, haystack, 3, "abc"));
CPPUNIT_ASSERT_EQUAL((const void*)nullptr, jmemmem(2, haystack, 3, "abc"));
CPPUNIT_ASSERT_EQUAL((const void*)(haystack+7), jmemmem(10, haystack, 3, "hij"));
CPPUNIT_ASSERT_EQUAL((const void*)nullptr, jmemmem(10, haystack, 3, "ijk"));
CPPUNIT_ASSERT_EQUAL((const void*)(haystack+8), jmemmem(10, haystack, 1, "i"));
CPPUNIT_ASSERT_EQUAL((const void*)(nullptr), jmemmem(8, haystack, 1, "i"));
CPPUNIT_ASSERT_EQUAL((const void*)(nullptr), jmemmem(9, haystack, 2, "ij"));
CPPUNIT_ASSERT_EQUAL((const void*)(haystack+8), jmemmem(10, haystack, 2, "ij"));
}
};

CPPUNIT_TEST_SUITE_REGISTRATION( JLibStringTest );
Expand Down
Loading