unittests: Adds step to remove stale SHM regions.

Some of the unit tests we run will leak shm regions. Presumably this is
because they never called `shmctl(IPC_RMID)` so the ID is laked forever.

This can be seen by querying `/proc/sysvipc/shm` to see a list of old
shm regions that eventually hit the maximum capacity of 4096 shm ids.

Once CI is done running, run a utility application that all it does is
check for SHM IDs that have zero attachments (thus unused), it was
created by the UID of the runner, and it is older than ten minutes. At
which point it will erase it.

This will fix spurious failures in our CI caused by running out of SHM
IDs, previously I had a cron job setup to restart the CI runners every
hour or so which caused its own spurious failure problems.

FINALLY this bug was triaged which has been annoying us for...years?
This commit is contained in:
Ryan Houdek 2023-05-12 18:41:38 -07:00
parent ce4e991a6e
commit fde64aedf7
5 changed files with 65 additions and 0 deletions

View File

@ -241,6 +241,12 @@ jobs:
# ASM tests get quite close to 10MB
run: truncate --size=<20M ${{runner.workspace}}/build/Testing/Temporary/LastTest_*.log || true
- name: Remove old SHM regions
if: ${{ always() }}
shell: bash
working-directory: ${{runner.workspace}}/build
run: cmake --build . --config $BUILD_TYPE --target remove_old_shm_regions
- name: Set runner name
if: ${{ always() }}
run: echo "runner_name=$(hostname)" >> $GITHUB_ENV

View File

@ -176,6 +176,12 @@ jobs:
# ASM tests get quite close to 10MB
run: truncate --size=<20M ${{runner.workspace}}/build/Testing/Temporary/LastTest_*.log || true
- name: Remove old SHM regions
if: ${{ always() }}
shell: bash
working-directory: ${{runner.workspace}}/build
run: cmake --build . --config $BUILD_TYPE --target remove_old_shm_regions
- name: Set runner name
if: ${{ always() }}
run: echo "runner_name=$(hostname)" >> $GITHUB_ENV

View File

@ -6,6 +6,7 @@ add_subdirectory(POSIX/)
add_subdirectory(gvisor-tests/)
add_subdirectory(gcc-target-tests-32/)
add_subdirectory(gcc-target-tests-64/)
add_subdirectory(Utilities/)
if (BUILD_THUNKS)
add_subdirectory(ThunkLibs)

View File

@ -0,0 +1,10 @@
add_executable(DeleteOldSHMRegions
DeleteOldSHMRegions.cpp)
set_target_properties(DeleteOldSHMRegions PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/TestUtilities")
add_custom_target(
remove_old_shm_regions
WORKING_DIRECTORY "${CMAKE_BINARY_DIR}/TestUtilities/"
USES_TERMINAL
COMMAND "DeleteOldSHMRegions")

View File

@ -0,0 +1,42 @@
#include <fstream>
#include <stdio.h>
#include <sys/shm.h>
#include <unistd.h>
static bool ctime_is_old(uint64_t ctime) {
time_t curtime;
time(&curtime);
// If it is older than ten minutes.
return (curtime - ctime) > 600;
}
int main() {
// key shmid perms size cpid lpid nattch uid gid cuid cgid atime dtime ctime rss swap
// 0 360448 777 4096 165187 165187 0 1002 1002 1002 1002 1679659857 1679659857 1679659857 4096 0
// 0 32769 777 4096 153814 153814 0 1002 1002 1002 1002 1676490841 1676490841 1676490841 0 4096
std::ifstream fs{"/proc/sysvipc/shm", std::fstream::binary};
std::string Line;
// Remove first line
std::getline(fs, Line);
const auto current_uid = getuid();
while (std::getline(fs, Line)) {
if (fs.eof()) break;
int shmid;
int uid;
int attach;
uint64_t ctime;
if (sscanf(Line.c_str(), "%*d %d %*d %*d %*d %*d %d %d %*d %*d %*d %*d %*d %ld", &shmid, &attach, &uid, &ctime) == 4) {
// If the UID matches and nothing is attached AND it is old then delete it.
if (uid == current_uid &&
attach == 0 &&
ctime_is_old(ctime)) {
shmctl(shmid, IPC_RMID, nullptr);
}
}
}
return 0;
}