debuginfod 2/2: server side

Add the server to the debuginfod/ subdirectory. This is a highly multithreaded c++11 program (still buildable on rhel7's gcc 4.8, which is only partly c++11 compliant). Includes an initial suite of tests, man pages, and a sample systemd service. Signed-off-by: Frank Ch. Eigler <fche@redhat.com> Signed-off-by: Aaron Merey <amerey@redhat.com>
2026-07-01 06:41:51 -04:00 · 2019-10-28 13:29:26 -04:00
parent 288f6b199a
commit e27e30cae0
31 changed files with 3445 additions and 26 deletions
@@ -1,3 +1,10 @@
+2019-10-28  Frank Ch. Eigler  <fche@redhat.com>
+
+	* eu.am (AM_CXXFLAGS): Clone & amend AM_CFLAGS for c++11 code.
+	* debuginfod.service, debuginfod.sysconfig: New files: systemd.
+	* Makefile.am: Install them.
+	* elfutils.spec.in: Add debuginfod and debuginfod-client subrpms.
+
 2019-08-29  Mark Wielaard  <mark@klomp.org>

 	* elfutils.spec.in (%description devel): Remove libebl text.
@@ -28,8 +28,9 @@
 ## the GNU Lesser General Public License along with this program.  If
 ## not, see <http://www.gnu.org/licenses/>.
 ##
-EXTRA_DIST = elfutils.spec.in known-dwarf.awk 10-default-yama-scope.conf
-	     libelf.pc.in libdw.pc.in libdebuginfod.pc.in
+EXTRA_DIST = elfutils.spec.in known-dwarf.awk 10-default-yama-scope.conf \
+	     libelf.pc.in libdw.pc.in libdebuginfod.pc.in \
+	     debuginfod.service debuginfod.sysconfig

 pkgconfigdir = $(libdir)/pkgconfig
 pkgconfig_DATA = libelf.pc libdw.pc libdebuginfod.pc
@@ -0,0 +1,15 @@
+[Unit]
+Description=elfutils debuginfo-over-http server
+Documentation=http://elfutils.org/
+After=network.target
+
+[Service]
+EnvironmentFile=/etc/sysconfig/debuginfod
+User=debuginfod
+Group=debuginfod
+#CacheDirectory=debuginfod
+ExecStart=/usr/bin/debuginfod -d /var/cache/debuginfod/debuginfod.sqlite -p $DEBUGINFOD_PORT $DEBUGINFOD_VERBOSE $DEBUGINFOD_PRAGMAS $DEBUGINFOD_PATHS
+TimeoutStopSec=10
+
+[Install]
+WantedBy=multi-user.target
@@ -0,0 +1,14 @@
+#
+DEBUGINFOD_PORT="8002"
+#DEBUGINFOD_VERBOSE="-v"
+
+# some common places to find trustworthy ELF/DWARF files and RPMs
+DEBUGINFOD_PATHS="-t43200 -F -R /usr/lib/debug /usr/bin /usr/libexec /usr/sbin /usr/lib /usr/lib64 /var/cache/yum /var/cache/dnf"
+
+# prefer reliability/durability over performance
+#DEBUGINFOD_PRAGMAS="-D 'pragma synchronous=full;'"
+
+# upstream debuginfods
+#DEBUGINFOD_URLS="http://secondhost:8002 http://thirdhost:8002"
+#DEBUGINFOD_TIMEOUT="5"
+#DEBUGINFOD_CACHE_DIR=""
@@ -12,6 +12,11 @@ Requires: elfutils-libelf = %{version}-%{release}
 Requires: glibc >= 2.7
 Requires: libstdc++
 Requires: default-yama-scope
+%if 0%{?rhel} >= 8 || 0%{?fedora} >= 20
+Recommends: elfutils-debuginfod-client
+%else
+Requires: elfutils-debuginfod-client
+%endif

 # ExcludeArch: xxx

@@ -23,10 +28,20 @@ BuildRequires: flex >= 2.5.4a
 BuildRequires: bzip2
 BuildRequires: m4
 BuildRequires: gettext
-BuildRequires: zlib-devel
+BuildRequires: pkgconfig(zlib)
+%if 0%{?rhel} == 7
 BuildRequires: bzip2-devel
-BuildRequires: xz-devel
+%else
+BuildRequires: pkgconfig(bzip2)
+%endif
+BuildRequires: pkgconfig(liblzma)
 BuildRequires: gcc-c++
+BuildRequires: pkgconfig(libmicrohttpd) >= 0.9.33
+BuildRequires: pkgconfig(libcurl) >= 7.29.0
+BuildRequires: pkgconfig(sqlite3) >= 3.7.17
+BuildRequires: pkgconfig(libarchive) >= 3.1.2
+# for the run-debuginfod-find.sh test case in %check for /usr/sbin/ss
+BuildRequires: iproute

 %define _gnu %{nil}
 %define _programprefix eu-
@@ -116,18 +131,53 @@ interprocess services, communication and introspection
 (like synchronisation, signaling, debugging, tracing and
 profiling) of processes.

+%package debuginfod-client
+Summary: Libraries and command-line frontend for HTTP ELF/DWARF file server addressed by build-id.
+License: GPLv3+ and (GPLv2+ or LGPLv3+)
+
+%package debuginfod-client-devel
+Summary: Libraries and headers to build debuginfod client applications.
+License: GPLv2+ or LGPLv3+
+
+%package debuginfod
+Summary: HTTP ELF/DWARF file server addressed by build-id.
+License: GPLv3+
+BuildRequires: systemd
+Requires(post):   systemd
+Requires(preun):  systemd
+Requires(postun): systemd
+Requires: shadow-utils
+Requires: /usr/bin/rpm2cpio
+
+%description debuginfod-client
+The elfutils-debuginfod-client package contains shared libraries
+dynamically loaded from -ldw, which use a debuginfod service
+to look up debuginfo and associated data. Also includes a
+command-line frontend.
+
+%description debuginfod-client-devel
+The elfutils-debuginfod-client-devel package contains the libraries
+to create applications to use the debuginfod service.
+
+%description debuginfod
+The elfutils-debuginfod package contains the debuginfod binary
+and control files for a service that can provide ELF/DWARF
+files to remote clients, based on build-id identification.
+The ELF/DWARF file searching functions in libdwfl can query
+such servers to download those files on demand.
+
 %prep
 %setup -q

 %build
-%configure --program-prefix=%{_programprefix}
+%configure --program-prefix=%{_programprefix} --enable-debuginfod
 make

 %install
 rm -rf ${RPM_BUILD_ROOT}
 mkdir -p ${RPM_BUILD_ROOT}%{_prefix}

-%makeinstall
+%make_install

 chmod +x ${RPM_BUILD_ROOT}%{_prefix}/%{_lib}/lib*.so*

@@ -140,6 +190,11 @@ chmod +x ${RPM_BUILD_ROOT}%{_prefix}/%{_lib}/lib*.so*

 install -Dm0644 config/10-default-yama-scope.conf ${RPM_BUILD_ROOT}%{_sysctldir}/10-default-yama-scope.conf

+install -Dm0644 config/debuginfod.service ${RPM_BUILD_ROOT}%{_unitdir}/debuginfod.service
+install -Dm0644 config/debuginfod.sysconfig ${RPM_BUILD_ROOT}%{_sysconfdir}/sysconfig/debuginfod
+mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/cache/debuginfod
+touch ${RPM_BUILD_ROOT}%{_localstatedir}/cache/debuginfod/debuginfod.sqlite
+
 %check
 make check

@@ -225,6 +280,43 @@ rm -rf ${RPM_BUILD_ROOT}
 %files default-yama-scope
 %{_sysctldir}/10-default-yama-scope.conf

+%files debuginfod-client
+%defattr(-,root,root)
+%{_libdir}/libdebuginfod-%{version}.so
+%{_bindir}/debuginfod-find
+%{_mandir}/man1/debuginfod-find.1*
+
+%files debuginfod-client-devel
+%defattr(-,root,root)
+%{_libdir}/pkgconfig/libdebuginfod.pc
+%{_mandir}/man3/debuginfod_*.3*
+%{_includedir}/elfutils/debuginfod.h
+%{_libdir}/libdebuginfod.so*
+
+%files debuginfod
+%defattr(-,root,root)
+%{_bindir}/debuginfod
+%config(noreplace) %verify(not md5 size mtime) %{_sysconfdir}/sysconfig/debuginfod
+%{_unitdir}/debuginfod.service
+%{_sysconfdir}/sysconfig/debuginfod
+%{_mandir}/man8/debuginfod.8*
+
+%dir %attr(0700,debuginfod,debuginfod) %{_localstatedir}/cache/debuginfod
+%verify(not md5 size mtime) %attr(0600,debuginfod,debuginfod) %{_localstatedir}/cache/debuginfod/debuginfod.sqlite
+
+%pre debuginfod
+getent group debuginfod >/dev/null || groupadd -r debuginfod
+getent passwd debuginfod >/dev/null || \
+    useradd -r -g debuginfod -d /var/cache/debuginfod -s /sbin/nologin \
+            -c "elfutils debuginfo server" debuginfod
+exit 0
+
+%post debuginfod
+%systemd_post debuginfod.service
+
+%postun debuginfod
+%systemd_postun_with_restart debuginfod.service
+
 %changelog
 * Tue Aug 13 2019 Mark Wielaard <mark@klomp.org> 0.177-1
 - elfclassify: New tool to analyze ELF objects.
@@ -79,6 +79,16 @@ AM_CFLAGS = -std=gnu99 -Wall -Wshadow -Wformat=2 \
 	    $(if $($(*F)_no_Wpacked_not_aligned),-Wno-packed-not-aligned,) \
 	    $($(*F)_CFLAGS)

+AM_CXXFLAGS = -std=c++11 -Wall -Wshadow \
+	   -Wtrampolines \
+	   $(LOGICAL_OP_WARNING) $(DUPLICATED_COND_WARNING) \
+	   $(NULL_DEREFERENCE_WARNING) $(IMPLICIT_FALLTHROUGH_WARNING) \
+	   $(if $($(*F)_no_Werror),,-Werror) \
+	   $(if $($(*F)_no_Wunused),,-Wunused -Wextra) \
+	   $(if $($(*F)_no_Wstack_usage),,$(STACK_USAGE_WARNING)) \
+	   $(if $($(*F)_no_Wpacked_not_aligned),-Wno-packed-not-aligned,) \
+	   $($(*F)_CXXFLAGS)
+
 COMPILE.os = $(filter-out -fprofile-arcs -ftest-coverage, $(COMPILE))

 DEFS.os = -DPIC -DSHARED
@@ -1,7 +1,7 @@
 dnl Process this file with autoconf to produce a configure script.
 dnl Configure input file for elfutils.                     -*-autoconf-*-
 dnl
-dnl Copyright (C) 1996-2018 Red Hat, Inc.
+dnl Copyright (C) 1996-2019 Red Hat, Inc.
 dnl
 dnl This file is part of elfutils.
 dnl
@@ -88,8 +88,6 @@ AS_IF([test "$use_locks" = yes],
 AH_TEMPLATE([USE_LOCKS], [Defined if libraries should be thread-safe.])

 AC_PROG_CC
-AC_PROG_CXX
-AX_CXX_COMPILE_STDCXX(11, noext, optional)
 AC_PROG_RANLIB
 AC_PROG_YACC
 AM_PROG_LEX
@@ -676,17 +674,25 @@ fi

 # Look for libmicrohttpd, libcurl, libarchive, sqlite for debuginfo server
 # minimum versions as per rhel7.  Single --enable-* option arranges to build
-# both client libs and server process.
+# both client and server.
+AC_ARG_ENABLE([debuginfod],AC_HELP_STRING([--enable-debuginfod], [Build debuginfod server and client]))
+AC_PROG_CXX
+AX_CXX_COMPILE_STDCXX(11, noext, optional)
+AS_IF([test "x$enable_debuginfod" != "xno"], [
+    AC_MSG_NOTICE([checking debuginfod dependencies, disable to skip])
+    enable_debuginfod=yes # presume success
+    PKG_PROG_PKG_CONFIG
+    if test "x$ac_cv_prog_ac_ct_CXX" = "x"; then enable_debuginfod=no; fi
+    PKG_CHECK_MODULES([libmicrohttpd],[libmicrohttpd >= 0.9.33],[],[enable_debuginfod=no])
+    PKG_CHECK_MODULES([libcurl],[libcurl >= 7.29.0],[],[enable_debuginfod=no])
+    PKG_CHECK_MODULES([sqlite3],[sqlite3 >= 3.7.17],[],[enable_debuginfod=no])
+    PKG_CHECK_MODULES([libarchive],[libarchive >= 3.1.2],[],[enable_debuginfod=no])
+    if test "x$enable_debuginfod" = "xno"; then
+      AC_MSG_ERROR([C++ compiler or dependencies not found, use --disable-debuginfod to disable.])
+    fi
+])

-PKG_PROG_PKG_CONFIG
-AC_ARG_ENABLE([debuginfod], AC_HELP_STRING([--enable-debuginfod], [Build debuginfo server and client solib]))
-AS_IF([test "x$enable_debuginfod" = "xyes"], [
-    AC_DEFINE([ENABLE_DEBUGINFOD],[1],[Build debuginfo-server])
-    PKG_CHECK_MODULES([libmicrohttpd],[libmicrohttpd >= 0.9.33])
-    PKG_CHECK_MODULES([libcurl],[libcurl >= 7.29.0])
-    PKG_CHECK_MODULES([sqlite3],[sqlite3 >= 3.7.17])
-    PKG_CHECK_MODULES([libarchive],[libarchive >= 3.1.2])
-], [enable_debuginfod="no"])
+AS_IF([test "x$enable_debuginfod" != "xno"],AC_DEFINE([ENABLE_DEBUGINFOD],[1],[Build debuginfod]))
 AM_CONDITIONAL([DEBUGINFOD],[test "x$enable_debuginfod" = "xyes"])


@@ -719,7 +725,7 @@ AC_MSG_NOTICE([
    Deterministic archives by default  : ${default_ar_deterministic}
    Native language support            : ${USE_NLS}
    Extra Valgrind annotations         : ${use_vg_annotations}
-    Debuginfo client/server support    : ${enable_debuginfod}
+    Debuginfod client/server support   : ${enable_debuginfod}

  EXTRA TEST FEATURES (used with make check)
    have bunzip2 installed (required)  : ${HAVE_BUNZIP2}
@@ -1,3 +1,9 @@
+2019-10-28  Frank Ch. Eigler  <fche@redhat.com>
+
+	* debuginfod.cxx: New file: debuginfod server.
+	* debuginfod.8: New file: man page.
+	* Makefile.am: Build it.
+
 2019-10-28  Aaron Merey  <amerey@redhat.com>

 	* debuginfod-client.c: New file: debuginfod client library.
@@ -31,7 +31,9 @@
 ##
 include $(top_srcdir)/config/eu.am
 AM_CPPFLAGS += -I$(srcdir) -I$(srcdir)/../libelf -I$(srcdir)/../libebl \
-	   -I$(srcdir)/../libdw -I$(srcdir)/../libdwelf
+	   -I$(srcdir)/../libdw -I$(srcdir)/../libdwelf \
+	   $(libmicrohttpd_CFLAGS) $(libcurl_CFLAGS) $(sqlite3_CFLAGS) \
+	   $(libarchive_CFLAGS)
 VERSION = 1

 # Disable eu- prefixing for artifacts (binaries & man pages) in this
@@ -55,7 +57,9 @@ libeu = ../lib/libeu.a

 AM_LDFLAGS = -Wl,-rpath-link,../libelf:../libdw:.

-bin_PROGRAMS = debuginfod-find
+bin_PROGRAMS = debuginfod debuginfod-find
+debuginfod_SOURCES = debuginfod.cxx
+debuginfod_LDADD = $(libdw) $(libelf) $(libeu) $(libdebuginfod) $(libmicrohttpd_LIBS) $(libcurl_LIBS) $(sqlite3_LIBS) $(libarchive_LIBS) -lpthread -ldl

 debuginfod_find_SOURCES = debuginfod-find.c
 debuginfod_find_LDADD = $(libeu) $(libdebuginfod)
@@ -19,10 +19,11 @@
 EXTRA_DIST = COPYING-GFDL README
 dist_man1_MANS=readelf.1 elfclassify.1
 notrans_dist_man3_MANS=elf_update.3 elf_getdata.3 elf_clone.3 elf_begin.3
-
+notrans_dist_man8_MANS=
 notrans_dist_man1_MANS=

 if DEBUGINFOD
+notrans_dist_man8_MANS += debuginfod.8
 notrans_dist_man3_MANS += debuginfod_find_debuginfo.3 debuginfod_find_source.3 debuginfod_find_executable.3
 notrans_dist_man1_MANS += debuginfod-find.1
 endif
@@ -0,0 +1,369 @@
+'\"! tbl | nroff \-man
+'\" t macro stdmacro
+
+.de SAMPLE
+.br
+.RS 0
+.nf
+.nh
+..
+.de ESAMPLE
+.hy
+.fi
+.RE
+..
+
+.TH DEBUGINFOD 8
+.SH NAME
+debuginfod \- debuginfo-related http file-server daemon
+
+.SH SYNOPSIS
+.B debuginfod
+[\fIOPTION\fP]... [\fIPATH\fP]...
+
+.SH DESCRIPTION
+\fBdebuginfod\fP serves debuginfo-related artifacts over HTTP.  It
+periodically scans a set of directories for ELF/DWARF files and their
+associated source code, as well as RPM files containing the above, to
+build an index by their buildid.  This index is used when remote
+clients use the HTTP webapi, to fetch these files by the same buildid.
+
+If a debuginfod cannot service a given buildid artifact request
+itself, and it is configured with information about upstream
+debuginfod servers, it queries them for the same information, just as
+\fBdebuginfod-find\fP would.  If successful, it locally caches then
+relays the file content to the original requester.
+
+If the \fB\-F\fP option is given, each listed PATH creates a thread to
+scan for matching ELF/DWARF/source files under the given physical
+directory.  Source files are matched with DWARF files based on the
+AT_comp_dir (compilation directory) attributes inside it.  Duplicate
+directories are ignored.  You may use a file name for a PATH, but
+source code indexing may be incomplete; prefer using a directory that
+contains the binaries.  Caution: source files listed in the DWARF may
+be a path \fIanywhere\fP in the file system, and debuginfod will
+readily serve their content on demand.  (Imagine a doctored DWARF file
+that lists \fI/etc/passwd\fP as a source file.)  If this is a concern,
+audit your binaries with tools such as:
+
+.SAMPLE
+% eu-readelf -wline BINARY | sed -n '/^Directory.table/,/^File.name.table/p'
+or
+% eu-readelf -wline BINARY | sed -n '/^Directory.table/,/^Line.number/p'
+or even use debuginfod itself:
+% debuginfod -vvv -d :memory: -F BINARY 2>&1 | grep 'recorded.*source'
+^C
+.ESAMPLE
+
+If the \fB\-R\fP option is given each listed PATH creates a thread to
+scan for ELF/DWARF/source files contained in matching RPMs under the
+given physical directory.  Duplicate directories are ignored.  You may
+use a file name for a PATH, but source code indexing may be
+incomplete; prefer using a directory that contains normal RPMs
+alongside debuginfo/debugsource RPMs.  Because of complications such
+as DWZ-compressed debuginfo, may require \fItwo\fP scan passes to
+identify all source code.  Source files for RPMs are only served
+from other RPMs, so the caution for \-F does not apply.
+
+If no PATH is listed, or neither \-F nor \-R option is given, then
+\fBdebuginfod\fP will simply serve content that it scanned into its
+index in previous runs: the data is cumulative.
+
+File names must match extended regular expressions given by the \-I
+option and not the \-X option (if any) in order to be considered.
+
+
+.SH OPTIONS
+
+.TP
+.B "\-F"
+Activate ELF/DWARF file scanning threads.  The default is off.
+
+.TP
+.B "\-R"
+Activate RPM file scanning threads.  The default is off.
+
+.TP
+.B "\-d FILE" "\-\-database=FILE"
+Set the path of the sqlite database used to store the index.  This
+file is disposable in the sense that a later rescan will repopulate
+data.  It will contain absolute file path names, so it may not be
+portable across machines.  It may be frequently read/written, so it
+should be on a fast filesytem.  It should not be shared across
+machines or users, to maximize sqlite locking performance.  The
+default database file is $HOME/.debuginfod.sqlite.
+
+.TP
+.B "\-D SQL" "\-\-ddl=SQL"
+Execute given sqlite statement after the database is opened and
+initialized as extra DDL (SQL data definition language).  This may be
+useful to tune performance-related pragmas or indexes.  May be
+repeated.  The default is nothing extra.
+
+.TP
+.B "\-p NUM" "\-\-port=NUM"
+Set the TCP port number on which debuginfod should listen, to service
+HTTP requests.  Both IPv4 and IPV6 sockets are opened, if possible.
+The webapi is documented below.  The default port number is 8002.
+
+.TP
+.B "\-I REGEX"  "\-\-include=REGEX"  "\-X REGEX"  "\-\-exclude=REGEX"
+Govern the inclusion and exclusion of file names under the search
+paths.  The regular expressions are interpreted as unanchored POSIX
+extended REs, thus may include alternation.  They are evaluated
+against the full path of each file, based on its \fBrealpath(3)\fP
+canonicalization.  By default, all files are included and none are
+excluded.  A file that matches both include and exclude REGEX is
+excluded.  (The \fIcontents\fP of RPM files are not subject to
+inclusion or exclusion filtering: they are all processed.)
+
+.TP
+.B "\-t SECONDS"  "\-\-rescan\-time=SECONDS"
+Set the rescan time for the file and RPM directories.  This is the
+amount of time the scanning threads will wait after finishing a scan,
+before doing it again.  A rescan for unchanged files is fast (because
+the index also stores the file mtimes).  A time of zero is acceptable,
+and means that only one initial scan should performed.  The default
+rescan time is 300 seconds.  Receiving a SIGUSR1 signal triggers a new
+scan, independent of the rescan time (including if it was zero).
+
+.TP
+.B "\-g SECONDS" "\-\-groom\-time=SECONDS"
+Set the groom time for the index database.  This is the amount of time
+the grooming thread will wait after finishing a grooming pass before
+doing it again.  A groom operation quickly rescans all previously
+scanned files, only to see if they are still present and current, so
+it can deindex obsolete files.  See also the \fIDATA MANAGEMENT\fP
+section.  The default groom time is 86400 seconds (1 day).  A time of
+zero is acceptable, and means that only one initial groom should be
+performed.  Receiving a SIGUSR2 signal triggers a new grooming pass,
+independent of the groom time (including if it was zero).
+
+.TP
+.B "\-G"
+Run an extraordinary maximal-grooming pass at debuginfod startup.
+This pass can take considerable time, because it tries to remove any
+debuginfo-unrelated content from the RPM-related parts of the index.
+It should not be run if any recent RPM-related indexing operations
+were aborted early.  It can take considerable space, because it
+finishes up with an sqlite "vacuum" operation, which repacks the
+database file by triplicating it temporarily.  The default is not to
+do maximal-grooming.  See also the \fIDATA MANAGEMENT\fP section.
+
+.TP
+.B "\-c NUM"  "\-\-concurrency=NUM"
+Set the concurrency limit for all the scanning threads.  While many
+threads may be spawned to cover all the given PATHs, only NUM may
+concurrently do CPU-intensive operations like parsing an ELF file
+or an RPM.  The default is the number of processors on the system;
+the minimum is 1.
+
+.TP
+.B "\-v"
+Increase verbosity of logging to the standard error file descriptor.
+May be repeated to increase details.  The default verbosity is 0.
+
+.SH WEBAPI
+
+.\" Much of the following text is duplicated with debuginfod-find.1
+
+The debuginfod's webapi resembles ordinary file service, where a GET
+request with a path containing a known buildid results in a file.
+Unknown buildid / request combinations result in HTTP error codes.
+This file service resemblance is intentional, so that an installation
+can take advantage of standard HTTP management infrastructure.
+
+There are three requests.  In each case, the buildid is encoded as a
+lowercase hexadecimal string.  For example, for a program \fI/bin/ls\fP,
+look at the ELF note GNU_BUILD_ID:
+
+.SAMPLE
+% readelf -n /bin/ls | grep -A4 build.id
+Note section [ 4] '.note.gnu.buildid' of 36 bytes at offset 0x340:
+Owner          Data size  Type
+GNU                   20  GNU_BUILD_ID
+Build ID: 8713b9c3fb8a720137a4a08b325905c7aaf8429d
+.ESAMPLE
+
+Then the hexadecimal BUILDID is simply:
+
+.SAMPLE
+8713b9c3fb8a720137a4a08b325905c7aaf8429d
+.ESAMPLE
+
+.SS /buildid/\fIBUILDID\fP/debuginfo
+
+If the given buildid is known to the server, this request will result
+in a binary object that contains the customary \fB.*debug_*\fP
+sections.  This may be a split debuginfo file as created by
+\fBstrip\fP, or it may be an original unstripped executable.
+
+.SS /buildid/\fIBUILDID\fP/executable
+
+If the given buildid is known to the server, this request will result
+in a binary object that contains the normal executable segments.  This
+may be a executable stripped by \fBstrip\fP, or it may be an original
+unstripped executable.  \fBET_DYN\fP shared libraries are considered
+to be a type of executable.
+
+.SS /buildid/\fIBUILDID\fP/source\fI/SOURCE/FILE\fP
+
+If the given buildid is known to the server, this request will result
+in a binary object that contains the source file mentioned.  The path
+should be absolute.  Relative path names commonly appear in the DWARF
+file's source directory, but these paths are relative to
+individual compilation unit AT_comp_dir paths, and yet an executable
+is made up of multiple CUs.  Therefore, to disambiguate, debuginfod
+expects source queries to prefix relative path names with the CU
+compilation-directory, followed by a mandatory "/".
+
+Note: contrary to RFC 3986, the client should not elide \fB../\fP or
+\fB/./\fP or extraneous \fB///\fP sorts of path components in the
+directory names, because if this is how those names appear in the
+DWARF files, that is what debuginfod needs to see too.
+
+For example:
+.TS
+l l.
+#include <stdio.h>	/buildid/BUILDID/source/usr/include/stdio.h
+/path/to/foo.c	/buildid/BUILDID/source/path/to/foo.c
+\../bar/foo.c AT_comp_dir=/zoo/	/buildid/BUILDID/source/zoo//../bar/foo.c
+.TE
+
+.SH DATA MANAGEMENT
+
+debuginfod stores its index in an sqlite database in a densely packed
+set of interlinked tables.  While the representation is as efficient
+as we have been able to make it, it still takes a considerable amount
+of data to record all debuginfo-related data of potentially a great
+many files.  This section offers some advice about the implications.
+
+As a general explanation for size, consider that debuginfod indexes
+ELF/DWARF files, it stores their names and referenced source file
+names, and buildids will be stored.  When indexing RPMs, it stores
+every file name \fIof or in\fP an RPM, every buildid, plus every
+source file name referenced from a DWARF file.  (Indexing RPMs takes
+more space because the source files often reside in separate
+subpackages that may not be indexed at the same pass, so extra
+metadata has to be kept.)
+
+Getting down to numbers, in the case of Fedora RPMs (essentially,
+gzip-compressed cpio files), the sqlite index database tends to be
+from 0.5% to 3% of their size.  It's larger for binaries that are
+assembled out of a great many source files, or packages that carry
+much debuginfo-unrelated content.  It may be even larger during the
+indexing phase due to temporary sqlite write-ahead-logging files;
+these are checkpointed (cleaned out and removed) at shutdown.  It may
+be helpful to apply tight \-I or \-X regular-expression constraints to
+exclude files from scanning that you know have no debuginfo-relevant
+content.
+
+As debuginfod runs, it periodically rescans its target directories,
+and any new content found is added to the database.  Old content, such
+as data for files that have disappeared or that have been replaced
+with newer versions is removed at a periodic \fIgrooming\fP pass.
+This means that the sqlite files grow fast during initial indexing,
+slowly during index rescans, and periodically shrink during grooming.
+There is also an optional one-shot \fImaximal grooming\fP pass is
+available.  It removes information debuginfo-unrelated data from the
+RPM content index such as file names found in RPMs ("rpm sdef"
+records) that are not referred to as source files from any binaries
+find in RPMs ("rpm sref" records).  This can save considerable disk
+space.  However, it is slow and temporarily requires up to twice the
+database size as free space.  Worse: it may result in missing
+source-code info if the RPM traversals were interrupted, so the not
+all source file references were known.  Use it rarely to polish a
+complete index.
+
+You should ensure that ample disk space remains available.  (The flood
+of error messages on -ENOSPC is ugly and nagging.  But, like for most
+other errors, debuginfod will resume when resources permit.)  If
+necessary, debuginfod can be stopped, the database file moved or
+removed, and debuginfod restarted.
+
+sqlite offers several performance-related options in the form of
+pragmas.  Some may be useful to fine-tune the defaults plus the
+debuginfod extras.  The \-D option may be useful to tell debuginfod to
+execute the given bits of SQL after the basic schema creation
+commands.  For example, the "synchronous", "cache_size",
+"auto_vacuum", "threads", "journal_mode" pragmas may be fun to tweak
+via \-D, if you're searching for peak performance.  The "optimize",
+"wal_checkpoint" pragmas may be useful to run periodically, outside
+debuginfod.  The default settings are performance- rather than
+reliability-oriented, so a hardware crash might corrupt the database.
+In these cases, it may be necessary to manually delete the sqlite
+database and start over.
+
+As debuginfod changes in the future, we may have no choice but to
+change the database schema in an incompatible manner.  If this
+happens, new versions of debuginfod will issue SQL statements to
+\fIdrop\fP all prior schema & data, and start over.  So, disk space
+will not be wasted for retaining a no-longer-useable dataset.
+
+In summary, if your system can bear a 0.5%-3% index-to-RPM-dataset
+size ratio, and slow growth afterwards, you should not need to
+worry about disk space.  If a system crash corrupts the database,
+or you want to force debuginfod to reset and start over, simply
+erase the sqlite file before restarting debuginfod.
+
+
+.SH SECURITY
+
+debuginfod \fBdoes not\fP include any particular security features.
+While it is robust with respect to inputs, some abuse is possible.  It
+forks a new thread for each incoming HTTP request, which could lead to
+a denial-of-service in terms of RAM, CPU, disk I/O, or network I/O.
+If this is a problem, users are advised to install debuginfod with a
+HTTPS reverse-proxy front-end that enforces site policies for
+firewalling, authentication, integrity, authorization, and load
+control.
+
+When relaying queries to upstream debuginfods, debuginfod \fBdoes not\fP
+include any particular security features.  It trusts that the binaries
+returned by the debuginfods are accurate.  Therefore, the list of
+servers should include only trustworthy ones.  If accessed across HTTP
+rather than HTTPS, the network should be trustworthy.  Authentication
+information through the internal \fIlibcurl\fP library is not currently
+enabled.
+
+
+.SH "ENVIRONMENT VARIABLES"
+
+.TP 21
+.B DEBUGINFOD_URLS
+This environment variable contains a list of URL prefixes for trusted
+debuginfod instances.  Alternate URL prefixes are separated by space.
+Avoid referential loops that cause a server to contact itself, directly
+or indirectly - the results would be hilarious.
+
+.TP 21
+.B DEBUGINFOD_TIMEOUT
+This environment variable governs the timeout for each debuginfod HTTP
+connection.  A server that fails to respond within this many seconds
+is skipped.  The default is 5.
+
+.TP 21
+.B DEBUGINFOD_CACHE_PATH
+This environment variable governs the location of the cache where
+downloaded files are kept.  It is cleaned periodically as this
+program is reexecuted.  The default is $HOME/.debuginfod_client_cache.
+.\" XXX describe cache eviction policy
+
+.SH FILES
+.LP
+.PD .1v
+.TP 20
+.B $HOME/.debuginfod.sqlite
+Default database file.
+.PD
+
+.TP 20
+.B $HOME/.debuginfod_client_cache
+Default cache directory for content from upstream debuginfods.
+.PD
+
+
+.SH "SEE ALSO"
+.I "debuginfod-find(1)"
+.I "sqlite3(1)"
+.I \%https://prometheus.io/docs/instrumenting/exporters/
@@ -1,3 +1,10 @@
+2019-10-28  Aaron Merey  <amerey@redhat.com>
+	    Frank Ch. Eigler  <fche@redhat.com>
+
+	* run-debuginfod-find.sh, debuginfod_build_id_find.c: New test.
+	* testfile-debuginfod-*.rpm.bz2: New data files for test.
+	* Makefile.am: Run it.
+
 2019-11-14  Andreas Schwab  <schwab@suse.de>

 	* run-large-elf-file.sh: Skip if available memory cannot be
@@ -1,6 +1,6 @@
 ## Process this file with automake to create Makefile.in
 ##
-## Copyright (C) 1996-2018 Red Hat, Inc.
+## Copyright (C) 1996-2019 Red Hat, Inc.
 ## This file is part of elfutils.
 ##
 ## This file is free software; you can redistribute it and/or modify
@@ -190,6 +190,11 @@ check_PROGRAMS += $(asm_TESTS)
 TESTS += $(asm_TESTS) run-disasm-bpf.sh
 endif

+if DEBUGINFOD
+check_PROGRAMS += debuginfod_build_id_find
+TESTS += run-debuginfod-find.sh
+endif
+
 EXTRA_DIST = run-arextract.sh run-arsymtest.sh run-ar.sh \
 	     run-show-die-info.sh run-get-files.sh run-get-lines.sh \
 	     run-next-files.sh run-next-lines.sh testfile-only-debug-line.bz2 \
@@ -440,7 +445,25 @@ EXTRA_DIST = run-arextract.sh run-arsymtest.sh run-ar.sh \
 	     run-dwelf_elf_e_machine_string.sh \
 	     run-elfclassify.sh run-elfclassify-self.sh \
 	     run-disasm-riscv64.sh \
-	     testfile-riscv64-dis1.o.bz2 testfile-riscv64-dis1.expect.bz2
+	     testfile-riscv64-dis1.o.bz2 testfile-riscv64-dis1.expect.bz2 \
+             run-debuginfod-find.sh \
+	     debuginfod-rpms/fedora30/hello2-1.0-2.src.rpm \
+	     debuginfod-rpms/fedora30/hello2-1.0-2.x86_64.rpm \
+	     debuginfod-rpms/fedora30/hello2-debuginfo-1.0-2.x86_64.rpm \
+	     debuginfod-rpms/fedora30/hello2-debugsource-1.0-2.x86_64.rpm \
+	     debuginfod-rpms/fedora30/hello2-two-1.0-2.x86_64.rpm \
+	     debuginfod-rpms/fedora30/hello2-two-debuginfo-1.0-2.x86_64.rpm \
+	     debuginfod-rpms/hello2.spec. \
+	     debuginfod-rpms/rhel6/hello2-1.0-2.i686.rpm \
+	     debuginfod-rpms/rhel6/hello2-1.0-2.src.rpm \
+	     debuginfod-rpms/rhel6/hello2-debuginfo-1.0-2.i686.rpm \
+	     debuginfod-rpms/rhel6/hello2-two-1.0-2.i686.rpm \
+	     debuginfod-rpms/rhel7/hello2-1.0-2.src.rpm \
+	     debuginfod-rpms/rhel7/hello2-1.0-2.x86_64.rpm \
+	     debuginfod-rpms/rhel7/hello2-debuginfo-1.0-2.x86_64.rpm \
+	     debuginfod-rpms/rhel7/hello2-two-1.0-2.x86_64.rpm
+
+

 if USE_VALGRIND
 valgrind_cmd='valgrind -q --leak-check=full --error-exitcode=1'
@@ -474,7 +497,7 @@ TESTS_ENVIRONMENT = LC_ALL=C; LANG=C; VALGRIND_CMD=$(valgrind_cmd); \
 		    export LC_ALL; export LANG; export VALGRIND_CMD; \
 		    NM=$(NM); export NM;
 LOG_COMPILER = $(abs_srcdir)/test-wrapper.sh \
-	       $(abs_top_builddir)/libdw:$(abs_top_builddir)/backends:$(abs_top_builddir)/libelf:$(abs_top_builddir)/libasm
+	       $(abs_top_builddir)/libdw:$(abs_top_builddir)/backends:$(abs_top_builddir)/libelf:$(abs_top_builddir)/libasm:$(abs_top_builddir)/debuginfod

 installcheck-local:
 	$(MAKE) $(AM_MAKEFLAGS) \
@@ -610,6 +633,7 @@ unit_info_LDADD = $(libdw)
 next_cfi_LDADD = $(libelf) $(libdw)
 elfcopy_LDADD = $(libelf)
 addsections_LDADD = $(libelf)
+debuginfod_build_id_find_LDADD = $(libelf) $(libdw)
 xlate_notes_LDADD = $(libelf)
 elfrdwrnop_LDADD = $(libelf)
 dwelf_elf_e_machine_string_LDADD = $(libelf) $(libdw)
@@ -0,0 +1,57 @@
+Summary: hello2 -- double hello, world rpm
+Name: hello2
+Version: 1.0
+Release: 2
+Group: Utilities
+License: GPL
+Distribution: RPM ^W Elfutils test suite.
+Vendor: Red Hat Software
+Packager: Red Hat Software <bugs@redhat.com>
+URL: http://www.redhat.com
+BuildRequires: gcc make
+Source0: hello-1.0.tar.gz
+
+%description
+Simple rpm demonstration with an eye to consumption by debuginfod.
+
+%package two
+Summary: hello2two
+License: GPL
+
+%description two
+Dittoish.
+
+%prep
+%setup -q -n hello-1.0
+
+%build
+gcc -g -O1 hello.c -o hello
+gcc -g -O2 -D_FORTIFY_SOURCE=2 hello.c -o hello2
+
+%install
+rm -rf $RPM_BUILD_ROOT
+mkdir -p $RPM_BUILD_ROOT/usr/local/bin
+cp hello $RPM_BUILD_ROOT/usr/local/bin/
+cp hello2 $RPM_BUILD_ROOT/usr/local/bin/
+
+%clean
+rm -rf $RPM_BUILD_ROOT
+
+%files 
+%defattr(-,root,root)
+%attr(0751,root,root)   /usr/local/bin/hello
+
+%files two
+%defattr(-,root,root)
+%attr(0751,root,root)   /usr/local/bin/hello2
+
+%changelog
+* Thu Nov 14 2019 Frank Ch. Eigler <fche@redhat.com>
+- Added source code right here to make spec file self-contained.
+- Dropped misc files not relevant to debuginfod testing.
+
+* Wed May 18 2016 Mark Wielaard <mjw@redhat.com>
+- Add hello2 for dwz testing support.
+
+* Tue Oct 20 1998 Jeff Johnson <jbj@redhat.com>
+- create.
@@ -0,0 +1,62 @@
+/* Test program for fetching debuginfo with debuginfo-server.
+   Copyright (C) 2019 Red Hat, Inc.
+   This file is part of elfutils.
+
+   This file is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   elfutils is distributed in the hope that it will be useful, but
+   WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.  */
+
+
+#ifdef HAVE_CONFIG_H
+# include <config.h>
+#endif
+#include <stdio.h>
+#include ELFUTILS_HEADER(dwfl)
+#include <elf.h>
+#include <dwarf.h>
+#include <argp.h>
+#include <assert.h>
+#include <string.h>
+
+static const char *debuginfo_path = "";
+static const Dwfl_Callbacks cb  =
+  {
+    NULL,
+    dwfl_standard_find_debuginfo,
+    NULL,
+    (char **)&debuginfo_path,
+  };
+
+int
+main (int argc __attribute__ ((unused)), char **argv)
+{
+  int expect_pass = strcmp(argv[3], "0");
+  Dwarf_Addr bias = 0;
+  Dwfl *dwfl = dwfl_begin(&cb);
+  dwfl_report_begin(dwfl);
+
+  /* Open an executable.  */
+  Dwfl_Module *mod = dwfl_report_offline(dwfl, argv[2], argv[2], -1);
+
+  /* The corresponding debuginfo will not be found in debuginfo_path
+     (since it's empty), causing the server to be queried.  */
+
+  Dwarf *res = dwfl_module_getdwarf(mod, &bias);
+  if (expect_pass)
+    assert(res);
+  else
+    assert(!res);
+
+  dwfl_end (dwfl);
+
+  return 0;
+}
@@ -0,0 +1,230 @@
+#!/bin/bash
+#
+# Copyright (C) 2019 Red Hat, Inc.
+# This file is part of elfutils.
+#
+# This file is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# elfutils is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+set -x
+. $srcdir/test-subr.sh  # includes set -e
+
+DB=${PWD}/.debuginfod_tmp.sqlite
+export DEBUGINFOD_CACHE_PATH=${PWD}/.client_cache
+
+# clean up trash if we were aborted early
+trap 'kill $PID1 $PID2 || true; sleep 5; rm -rf F R ${PWD}/.client_cache*; exit_cleanup' 0 1 2 3 5 9 15
+
+# find an unused port number
+while true; do
+    PORT1=`expr '(' $RANDOM % 1000 ')' + 9000`
+    ss -atn | fgrep ":$PORT1" || break
+done    
+
+# We want to run debuginfod in the background.  We also want to start
+# it with the same check/installcheck-sensitive LD_LIBRARY_PATH stuff
+# that the testrun alias sets.  But: we if we just use
+#    testrun .../debuginfod
+# it runs in a subshell, with different pid, so not helpful.
+#
+# So we gather the LD_LIBRARY_PATH with this cunning trick:
+ldpath=`testrun sh -c 'echo $LD_LIBRARY_PATH'`
+
+mkdir F R
+# not tempfiles F R - they are directories which we clean up manually
+env DEBUGINFOD_TEST_WEBAPI_SLEEP=3 LD_LIBRARY_PATH=$ldpath DEBUGINFOD_URLS= ${abs_builddir}/../debuginfod/debuginfod -F -R -vvvv -d $DB -p $PORT1 -t0 -g0 R F &
+PID1=$!
+sleep 3
+export DEBUGINFOD_URLS=http://localhost:$PORT1/   # or without trailing /
+
+# Be patient when run on a busy machine things might take a bit.
+# And under valgrind debuginfod-find is really, really slow.
+if [ "x$VALGRIND_CMD" = "x" ]; then
+  export DEBUGINFOD_TIMEOUT=60
+else
+  export DEBUGINFOD_TIMEOUT=300
+fi
+
+# We use -t0 and -g0 here to turn off time-based scanning & grooming.
+# For testing purposes, we just sic SIGUSR1 / SIGUSR2 at the process.
+
+########################################################################
+
+# Compile a simple program, strip its debuginfo and save the build-id.
+# Also move the debuginfo into another directory so that elfutils
+# cannot find it without debuginfod.
+echo "int main() { return 0; }" > ${PWD}/prog.c
+tempfiles prog.c
+gcc -g -o prog ${PWD}/prog.c
+ ${abs_top_builddir}/src/strip -g -f prog.debug ${PWD}/prog
+BUILDID=`env LD_LIBRARY_PATH=$ldpath ${abs_builddir}/../src/readelf \
+          -a prog | grep 'Build ID' | cut -d ' ' -f 7`
+
+mv prog F
+mv prog.debug F
+kill -USR1 $PID1
+sleep 3 # give enough time for scanning pass 
+
+########################################################################
+
+# Test whether elfutils, via the debuginfod client library dlopen hooks,
+# is able to fetch debuginfo from the local debuginfod.
+testrun ${abs_builddir}/debuginfod_build_id_find -e F/prog 1
+
+########################################################################
+
+# Test whether debuginfod-find is able to fetch those files.
+rm -rf $DEBUGINFOD_CACHE_PATH # clean it from previous tests
+filename=`testrun ${abs_top_builddir}/debuginfod/debuginfod-find debuginfo $BUILDID`
+cmp $filename F/prog.debug
+
+filename=`testrun ${abs_top_builddir}/debuginfod/debuginfod-find executable $BUILDID`
+cmp $filename F/prog
+
+filename=`testrun ${abs_top_builddir}/debuginfod/debuginfod-find source $BUILDID ${PWD}/prog.c`
+cmp $filename  ${PWD}/prog.c
+
+########################################################################
+
+# Add artifacts to the search paths and test whether debuginfod finds them while already running.
+
+# Build another, non-stripped binary
+echo "int main() { return 0; }" > ${PWD}/prog2.c
+tempfiles prog2.c
+gcc -g -o prog2 ${PWD}/prog2.c
+BUILDID2=`env LD_LIBRARY_PATH=$ldpath ${abs_builddir}/../src/readelf \
+          -a prog2 | grep 'Build ID' | cut -d ' ' -f 7`
+
+mv prog2 F
+kill -USR1 $PID1
+sleep 3
+
+# Rerun same tests for the prog2 binary
+filename=`testrun ${abs_top_builddir}/debuginfod/debuginfod-find debuginfo $BUILDID2`
+cmp $filename F/prog2
+filename=`testrun ${abs_top_builddir}/debuginfod/debuginfod-find executable $BUILDID2`
+cmp $filename F/prog2
+filename=`testrun ${abs_top_builddir}/debuginfod/debuginfod-find source $BUILDID2 ${PWD}/prog2.c`
+cmp $filename ${PWD}/prog2.c
+
+cp -rp ${abs_srcdir}/debuginfod-rpms R
+kill -USR1 $PID1
+sleep 10
+kill -USR1 $PID1  # two hits of SIGUSR1 may be needed to resolve .debug->dwz->srefs
+sleep 10
+
+
+# Run a bank of queries against the debuginfod-rpms test cases
+
+rpm_test() {
+    __BUILDID=$1
+    __SOURCEPATH=$2
+    __SOURCESHA1=$3
+    
+    filename=`testrun ${abs_top_builddir}/debuginfod/debuginfod-find executable $__BUILDID`
+    buildid=`env LD_LIBRARY_PATH=$ldpath ${abs_builddir}/../src/readelf \
+             -a $filename | grep 'Build ID' | cut -d ' ' -f 7`
+    test $__BUILDID = $buildid
+
+    filename=`testrun ${abs_top_builddir}/debuginfod/debuginfod-find debuginfo $__BUILDID`
+    buildid=`env LD_LIBRARY_PATH=$ldpath ${abs_builddir}/../src/readelf \
+             -a $filename | grep 'Build ID' | cut -d ' ' -f 7`
+    test $__BUILDID = $buildid
+    
+    filename=`testrun ${abs_top_builddir}/debuginfod/debuginfod-find source $__BUILDID $__SOURCEPATH`
+    hash=`cat $filename | sha1sum | awk '{print $1}'`
+    test $__SOURCESHA1 = $hash
+}
+
+
+# common source file sha1
+SHA=f4a1a8062be998ae93b8f1cd744a398c6de6dbb1
+# fedora30
+rpm_test c36708a78618d597dee15d0dc989f093ca5f9120 /usr/src/debug/hello2-1.0-2.x86_64/hello.c $SHA
+rpm_test 41a236eb667c362a1c4196018cc4581e09722b1b /usr/src/debug/hello2-1.0-2.x86_64/hello.c $SHA
+# rhel7
+rpm_test bc1febfd03ca05e030f0d205f7659db29f8a4b30 /usr/src/debug/hello-1.0/hello.c $SHA
+rpm_test f0aa15b8aba4f3c28cac3c2a73801fefa644a9f2 /usr/src/debug/hello-1.0/hello.c $SHA
+# rhel6
+rpm_test bbbf92ebee5228310e398609c23c2d7d53f6e2f9 /usr/src/debug/hello-1.0/hello.c $SHA
+rpm_test d44d42cbd7d915bc938c81333a21e355a6022fb7 /usr/src/debug/hello-1.0/hello.c $SHA
+
+RPM_BUILDID=d44d42cbd7d915bc938c81333a21e355a6022fb7 # in rhel6/ subdir, for a later test
+
+
+########################################################################
+
+# Drop some of the artifacts, run a groom cycle; confirm that
+# debuginfod has forgotten them, but remembers others
+
+rm -r R/debuginfod-rpms/rhel6/*
+kill -USR2 $PID1  # groom cycle
+sleep 3
+rm -rf $DEBUGINFOD_CACHE_PATH # clean it from previous tests
+
+testrun ${abs_top_builddir}/debuginfod/debuginfod-find executable $RPM_BUILDID && false || true
+
+testrun ${abs_top_builddir}/debuginfod/debuginfod-find executable $BUILDID2
+
+########################################################################
+
+# Federation mode
+
+# find another unused port
+while true; do
+    PORT2=`expr '(' $RANDOM % 1000 ')' + 9000`
+    ss -atn | fgrep ":$PORT2" || break
+done
+
+export DEBUGINFOD_CACHE_PATH=${PWD}/.client_cache2
+mkdir -p $DEBUGINFOD_CACHE_PATH
+# NB: inherits the DEBUGINFOD_URLS to the first server
+env LD_LIBRARY_PATH=$ldpath ${abs_builddir}/../debuginfod/debuginfod -F -vvvv -d ${DB}_2 -p $PORT2 &
+PID2=$!
+sleep 3
+
+# have clients contact the new server
+export DEBUGINFOD_URLS=http://localhost:$PORT2
+testrun ${abs_builddir}/debuginfod_build_id_find -e F/prog 1
+
+# test parallel queries in client
+export DEBUGINFOD_CACHE_PATH=${PWD}/.client_cache3
+mkdir -p $DEBUGINFOD_CACHE_PATH
+export DEBUGINFOD_URLS="BAD http://localhost:$PORT1 localhost:$PORT1 http://localhost:$PORT2 DNE"
+
+testrun ${abs_builddir}/debuginfod_build_id_find -e F/prog2 1
+
+
+########################################################################
+
+# Run the tests again without the servers running. The target file should
+# be found in the cache.
+
+kill -INT $PID1 $PID2
+sleep 5
+tempfiles .debuginfod_*
+
+testrun ${abs_builddir}/debuginfod_build_id_find -e F/prog2 1
+
+########################################################################
+
+# Trigger a cache clean and run the tests again. The clients should be unable to
+# find the target.
+echo 0 > $DEBUGINFOD_CACHE_PATH/cache_clean_interval_s
+echo 0 > $DEBUGINFOD_CACHE_PATH/max_unused_age_s
+
+testrun ${abs_builddir}/debuginfod_build_id_find -e F/prog 1
+
+testrun ${abs_top_builddir}/debuginfod/debuginfod-find debuginfo $BUILDID2 && false || true
+
+exit 0