mirror of
https://github.com/milvus-io/milvus.git
synced 2026-10-12 02:19:10 +00:00
zhikunyao
73e495d7d5
## What this fixes Linux builds currently enable split DWARF by default, but Milvus installation and published debug images do not include the generated `.dwo` files. The installed ELF files therefore contain dangling DWO references and GDB cannot load the expected debug information. ## Changes - make split DWARF an explicit CMake opt-in, defaulting to OFF - expose `USE_SPLIT_DWARF=ON` through `scripts/core_build.sh` for build-only workflows that preserve `.dwo` files - keep the existing release image, tag, multi-arch manifest, and strip pipeline unchanged With the default build, debug information remains self-contained in the installed ELF files. The existing release pipeline can continue publishing the unstripped build as `-debug` and stripping the same build for the regular image without copying sidecar debug files into production images. ## Release impact - regular stripped images should retain their current layout and strip behavior - debug images can grow because debug sections are embedded instead of being left behind in the build tree - Release builds still use the existing `-g1`; changing the debug level or publishing `.dwp` artifacts is intentionally out of scope - the first C++ build after this flag change may have lower cache reuse ## Validation - `bash -n scripts/core_build.sh` - parsed `DefineOptions.cmake` with CMake - verified `MILVUS_USE_SPLIT_DWARF` defaults to OFF and accepts explicit ON - `git diff --check` Supersedes #53188, which was recreated from the contributor fork to follow the repository contribution workflow. Fixes #53182 Signed-off-by: Zhikun Yao <[email protected]>
104 lines
3.7 KiB
Bash
Executable File
104 lines
3.7 KiB
Bash
Executable File
#!/bin/bash
|
|
#
|
|
# Debug a coredump produced by the stripped (production) Milvus image
|
|
# using the matching unstripped debug image.
|
|
#
|
|
# Starting from v2.6.15, each Milvus release publishes two image variants:
|
|
# milvusdb/milvus:<tag> - stripped, production (default, ~1/3 size)
|
|
# milvusdb/milvus:<tag>-debug - unstripped, self-contained debug info (for GDB)
|
|
#
|
|
# The stripped and debug images share byte-identical code sections, so GDB
|
|
# can use the debug image's symbols to resolve addresses in a coredump
|
|
# produced by the stripped image.
|
|
#
|
|
# Usage:
|
|
# ./milvus-debug.sh <coredump> <stripped-image> <debug-image>
|
|
#
|
|
# Examples:
|
|
# ./milvus-debug.sh ./core.12345 milvusdb/milvus:v2.6.15 milvusdb/milvus:v2.6.15-debug
|
|
# ./milvus-debug.sh ./core.12345 harbor.example.com/milvus:v2.6.15 harbor.example.com/milvus:v2.6.15-debug
|
|
#
|
|
# Typical workflow when a Milvus pod crashes in production:
|
|
# # 1. Copy the coredump out of the crashed pod
|
|
# kubectl cp <ns>/<pod>:/tmp/cores/core.<pid> ./core.<pid>
|
|
#
|
|
# # 2. Run this script (GDB starts automatically with backtrace)
|
|
# ./milvus-debug.sh ./core.<pid> milvusdb/milvus:v2.6.15 milvusdb/milvus:v2.6.15-debug
|
|
#
|
|
set -euo pipefail
|
|
|
|
COREDUMP="${1:?Usage: $0 <coredump> <stripped-image> <debug-image>}"
|
|
IMAGE="${2:?Missing stripped image, e.g. milvusdb/milvus:v2.6.15}"
|
|
IMAGE_NON_STRIP="${3:?Missing debug image, e.g. milvusdb/milvus:v2.6.15-debug}"
|
|
|
|
# Resolve to absolute path
|
|
COREDUMP="$(cd "$(dirname "$COREDUMP")" && pwd)/$(basename "$COREDUMP")"
|
|
|
|
if [ ! -f "$COREDUMP" ]; then
|
|
echo "ERROR: coredump file not found: $COREDUMP"
|
|
exit 1
|
|
fi
|
|
|
|
# Warn if stripped and debug images don't share the same base tag
|
|
# Expected: <image>:<tag> and <image>:<tag>-debug
|
|
STRIPPED_TAG="${IMAGE##*:}"
|
|
DEBUG_TAG="${IMAGE_NON_STRIP##*:}"
|
|
if [ "${DEBUG_TAG}" != "${STRIPPED_TAG}-debug" ]; then
|
|
echo "WARNING: debug image tag '${DEBUG_TAG}' does not match '${STRIPPED_TAG}-debug'."
|
|
echo " Symbols may not align with the coredump addresses."
|
|
echo ""
|
|
fi
|
|
|
|
echo "==> Pulling debug image: ${IMAGE_NON_STRIP}"
|
|
docker pull "${IMAGE_NON_STRIP}"
|
|
|
|
echo ""
|
|
echo "==> Launching GDB inside debug container..."
|
|
echo ""
|
|
echo " Coredump: ${COREDUMP}"
|
|
echo " Stripped image: ${IMAGE}"
|
|
echo " Debug image: ${IMAGE_NON_STRIP}"
|
|
echo ""
|
|
echo "--- GDB will start. Useful commands: ---"
|
|
echo " bt - backtrace of current thread"
|
|
echo " thread apply all bt - backtrace of all threads"
|
|
echo " info threads - list threads"
|
|
echo " thread <n> - switch to thread n"
|
|
echo " info locals - local variables"
|
|
echo " print <var> - inspect variable"
|
|
echo " list - show source context"
|
|
echo " info sharedlibrary - check .so symbol status"
|
|
echo "----------------------------------------"
|
|
echo ""
|
|
|
|
docker run -it --rm \
|
|
-v "${COREDUMP}:/tmp/core:ro" \
|
|
--entrypoint "" \
|
|
"${IMAGE_NON_STRIP}" \
|
|
bash -c '
|
|
# Locate the milvus binary
|
|
MILVUS_BIN=$(find /milvus -name milvus -type f 2>/dev/null | head -1)
|
|
if [ -z "$MILVUS_BIN" ]; then
|
|
echo "ERROR: milvus binary not found in image"
|
|
exit 1
|
|
fi
|
|
echo "Using binary: $MILVUS_BIN"
|
|
|
|
# Install gdb if not present
|
|
if ! command -v gdb &>/dev/null; then
|
|
echo "Installing gdb..."
|
|
apt-get update -qq && apt-get install -y -qq gdb >/dev/null 2>&1
|
|
fi
|
|
|
|
# Set library paths so GDB can resolve all .so symbols
|
|
export LD_LIBRARY_PATH="/milvus/lib:${LD_LIBRARY_PATH:-}"
|
|
|
|
# Launch GDB
|
|
# - solib-search-path: tells GDB where to find unstripped .so files
|
|
# - auto-loads symbols for milvus binary + all shared libraries
|
|
gdb "$MILVUS_BIN" /tmp/core \
|
|
-ex "set solib-search-path /milvus/lib" \
|
|
-ex "set print pretty on" \
|
|
-ex "bt"
|
|
'
|