From 69c4befba15ff5ace7408e0f107c38c699c294a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Tue, 24 Jan 2023 18:00:53 +0000 Subject: [PATCH 01/36] scripts/ci: update gitlab-runner playbook to use latest runner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We were using quite and old runner on our machines and running into issues with stalling jobs. Gitlab in the meantime now reliably provide the latest packaged versions of the runner under a stable URL. This update: - creates a per-arch subdir for builds - switches from binary tarballs to deb packages - re-uses the same binary for the secondary runner - updates distro check for second to 22.04 Note this script isn't fully idempotent as we end up accumulating runners especially during testing. However we also want to be able to run twice with different GitLab keys (e.g. project and personal) so I think we just have to be mindful of that during testing. Signed-off-by: Alex Bennée Acked-by: Richard Henderson Message-Id: <20230124180127.1881110-2-alex.bennee@linaro.org> --- scripts/ci/setup/gitlab-runner.yml | 60 ++++++++---------------------- scripts/ci/setup/vars.yml.template | 2 - 2 files changed, 15 insertions(+), 47 deletions(-) diff --git a/scripts/ci/setup/gitlab-runner.yml b/scripts/ci/setup/gitlab-runner.yml index 33128be85d..95d4199c03 100644 --- a/scripts/ci/setup/gitlab-runner.yml +++ b/scripts/ci/setup/gitlab-runner.yml @@ -50,17 +50,25 @@ - name: Download the matching gitlab-runner get_url: - dest: /usr/local/bin/gitlab-runner - url: "https://s3.amazonaws.com/gitlab-runner-downloads/v{{ gitlab_runner_version }}/binaries/gitlab-runner-{{ gitlab_runner_os }}-{{ gitlab_runner_arch }}" - owner: gitlab-runner - group: gitlab-runner - mode: u=rwx,g=rwx,o=rx + dest: "/root/" + url: "https://gitlab-runner-downloads.s3.amazonaws.com/latest/deb/gitlab-runner_{{ gitlab_runner_arch }}.deb" + + - name: Install gitlab-runner via package manager + apt: deb="/root/gitlab-runner_{{ gitlab_runner_arch }}.deb" - name: Register the gitlab-runner - command: "/usr/local/bin/gitlab-runner register --non-interactive --url {{ gitlab_runner_server_url }} --registration-token {{ gitlab_runner_registration_token }} --executor shell --tag-list {{ ansible_facts[\"architecture\"] }},{{ ansible_facts[\"distribution\"]|lower }}_{{ ansible_facts[\"distribution_version\"] }} --description '{{ ansible_facts[\"distribution\"] }} {{ ansible_facts[\"distribution_version\"] }} {{ ansible_facts[\"architecture\"] }} ({{ ansible_facts[\"os_family\"] }})'" + command: "/usr/bin/gitlab-runner register --non-interactive --url {{ gitlab_runner_server_url }} --registration-token {{ gitlab_runner_registration_token }} --executor shell --tag-list {{ ansible_facts[\"architecture\"] }},{{ ansible_facts[\"distribution\"]|lower }}_{{ ansible_facts[\"distribution_version\"] }} --description '{{ ansible_facts[\"distribution\"] }} {{ ansible_facts[\"distribution_version\"] }} {{ ansible_facts[\"architecture\"] }} ({{ ansible_facts[\"os_family\"] }})'" + + # The secondary runner will still run under the single gitlab-runner service + - name: Register secondary gitlab-runner + command: "/usr/bin/gitlab-runner register --non-interactive --url {{ gitlab_runner_server_url }} --registration-token {{ gitlab_runner_registration_token }} --executor shell --tag-list aarch32,{{ ansible_facts[\"distribution\"]|lower }}_{{ ansible_facts[\"distribution_version\"] }} --description '{{ ansible_facts[\"distribution\"] }} {{ ansible_facts[\"distribution_version\"] }} {{ ansible_facts[\"architecture\"] }} ({{ ansible_facts[\"os_family\"] }})'" + when: + - ansible_facts['distribution'] == 'Ubuntu' + - ansible_facts['architecture'] == 'aarch64' + - ansible_facts['distribution_version'] == '22.04' - name: Install the gitlab-runner service using its own functionality - command: /usr/local/bin/gitlab-runner install --user gitlab-runner --working-directory /home/gitlab-runner + command: "/usr/bin/gitlab-runner install --user gitlab-runner --working-directory /home/gitlab-runner" register: gitlab_runner_install_service_result failed_when: "gitlab_runner_install_service_result.rc != 0 and \"already exists\" not in gitlab_runner_install_service_result.stderr" @@ -69,41 +77,3 @@ name: gitlab-runner state: started enabled: yes - - - name: Download secondary gitlab-runner - get_url: - dest: /usr/local/bin/gitlab-runner-arm - url: "https://s3.amazonaws.com/gitlab-runner-downloads/v{{ gitlab_runner_version }}/binaries/gitlab-runner-{{ gitlab_runner_os }}-arm" - owner: gitlab-runner - group: gitlab-runner - mode: u=rwx,g=rwx,o=rx - when: - - ansible_facts['distribution'] == 'Ubuntu' - - ansible_facts['architecture'] == 'aarch64' - - ansible_facts['distribution_version'] == '20.04' - - - name: Register secondary gitlab-runner - command: "/usr/local/bin/gitlab-runner-arm register --non-interactive --url {{ gitlab_runner_server_url }} --registration-token {{ gitlab_runner_registration_token }} --executor shell --tag-list aarch32,{{ ansible_facts[\"distribution\"]|lower }}_{{ ansible_facts[\"distribution_version\"] }} --description '{{ ansible_facts[\"distribution\"] }} {{ ansible_facts[\"distribution_version\"] }} {{ ansible_facts[\"architecture\"] }} ({{ ansible_facts[\"os_family\"] }})'" - when: - - ansible_facts['distribution'] == 'Ubuntu' - - ansible_facts['architecture'] == 'aarch64' - - ansible_facts['distribution_version'] == '20.04' - - - name: Install the secondary gitlab-runner service using its own functionality - command: /usr/local/bin/gitlab-runner-arm install --user gitlab-runner --working-directory /home/gitlab-runner/arm -n gitlab-runner-arm - register: gitlab_runner_install_service_result - failed_when: "gitlab_runner_install_service_result.rc != 0 and \"already exists\" not in gitlab_runner_install_service_result.stderr" - when: - - ansible_facts['distribution'] == 'Ubuntu' - - ansible_facts['architecture'] == 'aarch64' - - ansible_facts['distribution_version'] == '20.04' - - - name: Enable the secondary gitlab-runner service - service: - name: gitlab-runner-arm - state: started - enabled: yes - when: - - ansible_facts['distribution'] == 'Ubuntu' - - ansible_facts['architecture'] == 'aarch64' - - ansible_facts['distribution_version'] == '20.04' diff --git a/scripts/ci/setup/vars.yml.template b/scripts/ci/setup/vars.yml.template index e48089761f..4b355fb80f 100644 --- a/scripts/ci/setup/vars.yml.template +++ b/scripts/ci/setup/vars.yml.template @@ -1,5 +1,3 @@ -# The version of the gitlab-runner to use -gitlab_runner_version: 13.12.0 # The URL of the gitlab server to use, usually https://gitlab.com unless you're # using a private GitLab instance gitlab_runner_server_url: https://gitlab.com From 075d909d044ef63a0b0cd1c04ad94d16f06ca923 Mon Sep 17 00:00:00 2001 From: Mark Cave-Ayland Date: Tue, 24 Jan 2023 18:00:54 +0000 Subject: [PATCH 02/36] gitlab: add FF_SCRIPT_SECTIONS for timings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Suggested-by: Mark Cave-Ayland Signed-off-by: Mark Cave-Ayland Signed-off-by: Alex Bennée Reviewed-by: Thomas Huth Reviewed-by: Philippe Mathieu-Daudé Reviewed-by: Daniel P. Berrangé Message-Id: <20230124180127.1881110-3-alex.bennee@linaro.org> --- .gitlab-ci.d/base.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitlab-ci.d/base.yml b/.gitlab-ci.d/base.yml index 69b36c148a..50fb59e147 100644 --- a/.gitlab-ci.d/base.yml +++ b/.gitlab-ci.d/base.yml @@ -6,6 +6,11 @@ # most restrictive to least restrictive # .base_job_template: + variables: + # Each script line from will be in a collapsible section in the job output + # and show the duration of each line. + FF_SCRIPT_SECTIONS: 1 + rules: ############################################################# # Stage 1: exclude scenarios where we definitely don't From c34bf19feb349360faa57147f9bbc16b1f27d6e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Tue, 24 Jan 2023 18:00:55 +0000 Subject: [PATCH 03/36] gitlab: just use plain --cc=clang for custom runner build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I think this was because older Ubuntu's didn't alias clang to whatever the latest version was. They do now so lets use that and not break. Reviewed-by: Philippe Mathieu-Daudé Reviewed-by: Richard Henderson Signed-off-by: Alex Bennée Message-Id: <20230124180127.1881110-4-alex.bennee@linaro.org> --- .gitlab-ci.d/custom-runners/ubuntu-22.04-aarch64.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.d/custom-runners/ubuntu-22.04-aarch64.yml b/.gitlab-ci.d/custom-runners/ubuntu-22.04-aarch64.yml index abeb33eaff..725ca8ffea 100644 --- a/.gitlab-ci.d/custom-runners/ubuntu-22.04-aarch64.yml +++ b/.gitlab-ci.d/custom-runners/ubuntu-22.04-aarch64.yml @@ -81,7 +81,7 @@ ubuntu-22.04-aarch64-clang: script: - mkdir build - cd build - - ../configure --disable-libssh --cc=clang-10 --cxx=clang++-10 --enable-sanitizers + - ../configure --disable-libssh --cc=clang --cxx=clang++ --enable-sanitizers || { cat config.log meson-logs/meson-log.txt; exit 1; } - make --output-sync -j`nproc --ignore=40` - make --output-sync -j`nproc --ignore=40` check From c906e6fbaa50a3d9f9a5b24987e1a9d4ad70e9a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Tue, 24 Jan 2023 18:00:56 +0000 Subject: [PATCH 04/36] tests/unit: drop hacky race avoidance in test-io-channel-command MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We don't need to play timing games to ensure one socat wins over the other, just create the fifo they both can use before spawning the processes. However in the process we need to disable two tests for Windows platforms as we don't have an abstraction for mkfifo(). Resolves: https://gitlab.com/qemu-project/qemu/-/issues/1403 Signed-off-by: Alex Bennée Reviewed-by: Thomas Huth Message-Id: <20230124180127.1881110-5-alex.bennee@linaro.org> --- tests/unit/test-io-channel-command.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/unit/test-io-channel-command.c b/tests/unit/test-io-channel-command.c index 19f72eab96..425e2f5594 100644 --- a/tests/unit/test-io-channel-command.c +++ b/tests/unit/test-io-channel-command.c @@ -20,6 +20,8 @@ #include "qemu/osdep.h" #include +#include +#include #include "io/channel-command.h" #include "io-channel-helpers.h" #include "qapi/error.h" @@ -29,6 +31,7 @@ static char *socat = NULL; +#ifndef _WIN32 static void test_io_channel_command_fifo(bool async) { g_autofree gchar *tmpdir = g_dir_make_tmp("qemu-test-io-channel.XXXXXX", NULL); @@ -40,12 +43,13 @@ static void test_io_channel_command_fifo(bool async) QIOChannel *src, *dst; QIOChannelTest *test; + if (mkfifo(fifo, 0600)) { + g_error("mkfifo: %s", strerror(errno)); + } + src = QIO_CHANNEL(qio_channel_command_new_spawn((const char **) srcargv, O_WRONLY, &error_abort)); - /* try to avoid a race to create the socket */ - g_usleep(1000); - dst = QIO_CHANNEL(qio_channel_command_new_spawn((const char **) dstargv, O_RDONLY, &error_abort)); @@ -60,7 +64,6 @@ static void test_io_channel_command_fifo(bool async) g_rmdir(tmpdir); } - static void test_io_channel_command_fifo_async(void) { if (!socat) { @@ -80,6 +83,7 @@ static void test_io_channel_command_fifo_sync(void) test_io_channel_command_fifo(false); } +#endif static void test_io_channel_command_echo(bool async) @@ -124,10 +128,12 @@ int main(int argc, char **argv) socat = g_find_program_in_path("socat"); +#ifndef _WIN32 g_test_add_func("/io/channel/command/fifo/sync", test_io_channel_command_fifo_sync); g_test_add_func("/io/channel/command/fifo/async", test_io_channel_command_fifo_async); +#endif g_test_add_func("/io/channel/command/echo/sync", test_io_channel_command_echo_sync); g_test_add_func("/io/channel/command/echo/async", From e2c4012bc35894d60e54bd077ceaaae565d43c15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Lureau?= Date: Tue, 24 Jan 2023 18:00:57 +0000 Subject: [PATCH 05/36] build-sys: fix crlf-ending C code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On msys2, the shader-to-C script produces bad C: ./ui/shader/texture-blit-vert.h:2:5: error: missing terminating " character [-Werror] Fix it by changing the line ending from crlf to lf, and convert the script to Python (qemu build seems perl-free after that). Signed-off-by: Marc-André Lureau Acked-by: Thomas Huth Message-Id: <20230110132700.833690-2-marcandre.lureau@redhat.com> Signed-off-by: Alex Bennée Message-Id: <20230124180127.1881110-6-alex.bennee@linaro.org> --- meson.build | 2 +- scripts/shaderinclude.pl | 16 ---------------- scripts/shaderinclude.py | 26 ++++++++++++++++++++++++++ 3 files changed, 27 insertions(+), 17 deletions(-) delete mode 100644 scripts/shaderinclude.pl create mode 100644 scripts/shaderinclude.py diff --git a/meson.build b/meson.build index 6d3b665629..56320ae717 100644 --- a/meson.build +++ b/meson.build @@ -2779,7 +2779,7 @@ config_host_data.set('CONFIG_SLIRP', slirp.found()) genh += configure_file(output: 'config-host.h', configuration: config_host_data) hxtool = find_program('scripts/hxtool') -shaderinclude = find_program('scripts/shaderinclude.pl') +shaderinclude = find_program('scripts/shaderinclude.py') qapi_gen = find_program('scripts/qapi-gen.py') qapi_gen_depends = [ meson.current_source_dir() / 'scripts/qapi/__init__.py', meson.current_source_dir() / 'scripts/qapi/commands.py', diff --git a/scripts/shaderinclude.pl b/scripts/shaderinclude.pl deleted file mode 100644 index cd3bb40b12..0000000000 --- a/scripts/shaderinclude.pl +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env perl -use strict; -use warnings; - -my $file = shift; -open FILE, "<", $file or die "open $file: $!"; -my $name = $file; -$name =~ s|.*/||; -$name =~ s/[-.]/_/g; -print "static GLchar ${name}_src[] =\n"; -while () { - chomp; - printf " \"%s\\n\"\n", $_; -} -print " \"\\n\";\n"; -close FILE; diff --git a/scripts/shaderinclude.py b/scripts/shaderinclude.py new file mode 100644 index 0000000000..ab2aade2cd --- /dev/null +++ b/scripts/shaderinclude.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2023 Red Hat, Inc. +# +# SPDX-License-Identifier: GPL-2.0-or-later + +import sys +import os + + +def main(args): + file_path = args[1] + basename = os.path.basename(file_path) + varname = basename.replace('-', '_').replace('.', '_') + + with os.fdopen(sys.stdout.fileno(), "wt", closefd=False, newline='\n') as stdout: + with open(file_path, "r", encoding='utf-8') as file: + print(f'static GLchar {varname}_src[] =', file=stdout) + for line in file: + line = line.rstrip() + print(f' "{line}\\n"', file=stdout) + print(' "\\n";', file=stdout) + + +if __name__ == '__main__': + sys.exit(main(sys.argv)) From 7195f30248c4e55ebd56cbe370547571fdf5c144 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Lureau?= Date: Tue, 24 Jan 2023 18:00:58 +0000 Subject: [PATCH 06/36] .gitlab-ci.d/windows: do not disable opengl MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous patch should have fixed shader compilation. Signed-off-by: Marc-André Lureau Reviewed-by: Thomas Huth Reviewed-by: Philippe Mathieu-Daudé Message-Id: <20230110132700.833690-3-marcandre.lureau@redhat.com> [AJB: tweak commit message] Signed-off-by: Alex Bennée Message-Id: <20230124180127.1881110-7-alex.bennee@linaro.org> --- .gitlab-ci.d/windows.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.d/windows.yml b/.gitlab-ci.d/windows.yml index a1d5790580..cf445b77f6 100644 --- a/.gitlab-ci.d/windows.yml +++ b/.gitlab-ci.d/windows.yml @@ -71,7 +71,7 @@ msys2-64bit: # for the msys2 64-bit job, due to the build could not complete within # the project timeout. - ..\msys64\usr\bin\bash -lc '../configure --target-list=x86_64-softmmu - --without-default-devices --disable-opengl' + --without-default-devices' - ..\msys64\usr\bin\bash -lc 'make' # qTests don't run successfully with "--without-default-devices", # so let's exclude the qtests from CI for now. @@ -113,8 +113,7 @@ msys2-32bit: - $env:MSYS = 'winsymlinks:native' # Enable native Windows symlink - mkdir output - cd output - - ..\msys64\usr\bin\bash -lc '../configure --target-list=ppc64-softmmu - --disable-opengl' + - ..\msys64\usr\bin\bash -lc '../configure --target-list=ppc64-softmmu' - ..\msys64\usr\bin\bash -lc 'make' - ..\msys64\usr\bin\bash -lc 'make check MTESTARGS=\"--no-suite qtest\" || { cat meson-logs/testlog.txt; exit 1; }' From ec91e923784df086bdb3bd39569203524aa75875 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Lureau?= Date: Tue, 24 Jan 2023 18:00:59 +0000 Subject: [PATCH 07/36] meson: replace Perl usage with Python MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Let's try to remove Perl usage during build time. Signed-off-by: Marc-André Lureau Reviewed-by: Daniel P. Berrangé Message-Id: <20230110132700.833690-5-marcandre.lureau@redhat.com> Signed-off-by: Alex Bennée Message-Id: <20230124180127.1881110-8-alex.bennee@linaro.org> --- tests/qapi-schema/meson.build | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/qapi-schema/meson.build b/tests/qapi-schema/meson.build index 9dfe98bc9a..d85b14f28c 100644 --- a/tests/qapi-schema/meson.build +++ b/tests/qapi-schema/meson.build @@ -259,22 +259,23 @@ if build_docs # Fix possible inconsistency in line endings in generated output and # in the golden reference (which could otherwise cause test failures # on Windows hosts). Unfortunately diff --strip-trailing-cr - # is GNU-diff only. The odd-looking perl is because we must avoid + # is GNU-diff only. The odd-looking python is because we must avoid # using an explicit '\' character in the command arguments to # a custom_target(), as Meson will unhelpfully replace it with a '/' # (https://github.com/mesonbuild/meson/issues/1564) + remove_cr = [python, '-c', 'import sys;[sys.stdout.write(line.replace(chr(13), "")) for line in sys.stdin]'] qapi_doc_out_nocr = custom_target('QAPI rST doc newline-sanitized', output: ['doc-good.txt.nocr'], input: qapi_doc_out[0], build_by_default: true, - command: ['perl', '-pe', '$x = chr 13; s/$x$//', '@INPUT@'], + command: [remove_cr, '@INPUT@'], capture: true) qapi_doc_ref_nocr = custom_target('QAPI rST doc reference newline-sanitized', output: ['doc-good.ref.nocr'], input: files('doc-good.txt'), build_by_default: true, - command: ['perl', '-pe', '$x = chr 13; s/$x$//', '@INPUT@'], + command: [remove_cr, '@INPUT@'], capture: true) test('QAPI rST doc', diff, args: ['-u', qapi_doc_ref_nocr[0], qapi_doc_out_nocr[0]], From d89935c95e5ba7e2685da71e4756fa63fde764af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Lureau?= Date: Tue, 24 Jan 2023 18:01:00 +0000 Subject: [PATCH 08/36] docs: drop texinfo options MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It looks like this is no longer wanted, we only build the html output. Signed-off-by: Marc-André Lureau Reviewed-by: Daniel P. Berrangé Message-Id: <20230110132700.833690-6-marcandre.lureau@redhat.com> Signed-off-by: Alex Bennée Message-Id: <20230124180127.1881110-9-alex.bennee@linaro.org> --- docs/conf.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index e33cf3d381..73a287a4f2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -297,19 +297,6 @@ man_pages = [ ] man_make_section_directory = False -# -- Options for Texinfo output ------------------------------------------- - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - (master_doc, 'QEMU', u'QEMU Documentation', - author, 'QEMU', 'One line description of project.', - 'Miscellaneous'), -] - - - # We use paths starting from qemu_docdir here so that you can run # sphinx-build from anywhere and the kerneldoc extension can still # find everything. From 3ab8bf8341ae42efbefbb99bff6d703c302b1d4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Tue, 24 Jan 2023 18:01:01 +0000 Subject: [PATCH 09/36] gitlab: add lsan suppression file to workaround tcmalloc issues MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The up-coming upgrade to Fedora 37 will bring in libtcmalloc as a dependency of libglusterfs which confuses our fuzz run. Rather than disable the build lets use LSAN's suppression mechanism to prevent the job from failing. Signed-off-by: Alex Bennée Cc: Daniel P. Berrangé Reviewed-by: Philippe Mathieu-Daudé Reviewed-by: Thomas Huth Message-Id: <20230124180127.1881110-10-alex.bennee@linaro.org> --- .gitlab-ci.d/buildtest.yml | 1 + scripts/oss-fuzz/lsan_suppressions.txt | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 scripts/oss-fuzz/lsan_suppressions.txt diff --git a/.gitlab-ci.d/buildtest.yml b/.gitlab-ci.d/buildtest.yml index f09a898c3e..9a6ba1fe3b 100644 --- a/.gitlab-ci.d/buildtest.yml +++ b/.gitlab-ci.d/buildtest.yml @@ -511,6 +511,7 @@ build-oss-fuzz: IMAGE: fedora script: - mkdir build-oss-fuzz + - export LSAN_OPTIONS=suppressions=scripts/oss-fuzz/lsan_suppressions.txt - CC="clang" CXX="clang++" CFLAGS="-fsanitize=address" ./scripts/oss-fuzz/build.sh - export ASAN_OPTIONS="fast_unwind_on_malloc=0" diff --git a/scripts/oss-fuzz/lsan_suppressions.txt b/scripts/oss-fuzz/lsan_suppressions.txt new file mode 100644 index 0000000000..02ec0a6ed5 --- /dev/null +++ b/scripts/oss-fuzz/lsan_suppressions.txt @@ -0,0 +1,2 @@ +# The tcmalloc on Fedora37 confuses things +leak:/lib64/libtcmalloc_minimal.so.4 From 0054dc8bde408d61257bacdb489bbacc4ca5cde5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Lureau?= Date: Tue, 24 Jan 2023 18:01:02 +0000 Subject: [PATCH 10/36] Update lcitool and fedora to 37 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fedora 35 is EOL. Update to upstream lcitool, that dropped f35 and added f37. Signed-off-by: Marc-André Lureau Reviewed-by: Thomas Huth Reviewed-by: Philippe Mathieu-Daudé Message-Id: <20230110132700.833690-7-marcandre.lureau@redhat.com> Signed-off-by: Alex Bennée Message-Id: <20230124180127.1881110-11-alex.bennee@linaro.org> --- tests/docker/dockerfiles/fedora-win32-cross.docker | 4 ++-- tests/docker/dockerfiles/fedora-win64-cross.docker | 4 ++-- tests/docker/dockerfiles/fedora.docker | 4 ++-- tests/lcitool/libvirt-ci | 2 +- tests/lcitool/refresh | 6 +++--- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/docker/dockerfiles/fedora-win32-cross.docker b/tests/docker/dockerfiles/fedora-win32-cross.docker index 75383ba185..cc5d1ac4be 100644 --- a/tests/docker/dockerfiles/fedora-win32-cross.docker +++ b/tests/docker/dockerfiles/fedora-win32-cross.docker @@ -1,10 +1,10 @@ # THIS FILE WAS AUTO-GENERATED # -# $ lcitool dockerfile --layers all --cross mingw32 fedora-35 qemu +# $ lcitool dockerfile --layers all --cross mingw32 fedora-37 qemu # # https://gitlab.com/libvirt/libvirt-ci -FROM registry.fedoraproject.org/fedora:35 +FROM registry.fedoraproject.org/fedora:37 RUN dnf install -y nosync && \ echo -e '#!/bin/sh\n\ diff --git a/tests/docker/dockerfiles/fedora-win64-cross.docker b/tests/docker/dockerfiles/fedora-win64-cross.docker index 98c03dc13b..cabbf4edfc 100644 --- a/tests/docker/dockerfiles/fedora-win64-cross.docker +++ b/tests/docker/dockerfiles/fedora-win64-cross.docker @@ -1,10 +1,10 @@ # THIS FILE WAS AUTO-GENERATED # -# $ lcitool dockerfile --layers all --cross mingw64 fedora-35 qemu +# $ lcitool dockerfile --layers all --cross mingw64 fedora-37 qemu # # https://gitlab.com/libvirt/libvirt-ci -FROM registry.fedoraproject.org/fedora:35 +FROM registry.fedoraproject.org/fedora:37 RUN dnf install -y nosync && \ echo -e '#!/bin/sh\n\ diff --git a/tests/docker/dockerfiles/fedora.docker b/tests/docker/dockerfiles/fedora.docker index d200c7fc10..f44b005000 100644 --- a/tests/docker/dockerfiles/fedora.docker +++ b/tests/docker/dockerfiles/fedora.docker @@ -1,10 +1,10 @@ # THIS FILE WAS AUTO-GENERATED # -# $ lcitool dockerfile --layers all fedora-35 qemu +# $ lcitool dockerfile --layers all fedora-37 qemu # # https://gitlab.com/libvirt/libvirt-ci -FROM registry.fedoraproject.org/fedora:35 +FROM registry.fedoraproject.org/fedora:37 RUN dnf install -y nosync && \ echo -e '#!/bin/sh\n\ diff --git a/tests/lcitool/libvirt-ci b/tests/lcitool/libvirt-ci index e3eb28cf2e..319a534c22 160000 --- a/tests/lcitool/libvirt-ci +++ b/tests/lcitool/libvirt-ci @@ -1 +1 @@ -Subproject commit e3eb28cf2e17fbcf7fe7e19505ee432b8ec5bbb5 +Subproject commit 319a534c220f53fc8670254cac25d6f662c82112 diff --git a/tests/lcitool/refresh b/tests/lcitool/refresh index fa966e4009..a5ea0efc3b 100755 --- a/tests/lcitool/refresh +++ b/tests/lcitool/refresh @@ -111,7 +111,7 @@ try: generate_dockerfile("centos8", "centos-stream-8") generate_dockerfile("debian-amd64", "debian-11", trailer="".join(debian11_extras)) - generate_dockerfile("fedora", "fedora-35") + generate_dockerfile("fedora", "fedora-37") generate_dockerfile("opensuse-leap", "opensuse-leap-153") generate_dockerfile("ubuntu2004", "ubuntu-2004", trailer="".join(ubuntu2004_tsanhack)) @@ -161,12 +161,12 @@ try: trailer=cross_build("s390x-linux-gnu-", "s390x-softmmu,s390x-linux-user")) - generate_dockerfile("fedora-win32-cross", "fedora-35", + generate_dockerfile("fedora-win32-cross", "fedora-37", cross="mingw32", trailer=cross_build("i686-w64-mingw32-", "i386-softmmu")) - generate_dockerfile("fedora-win64-cross", "fedora-35", + generate_dockerfile("fedora-win64-cross", "fedora-37", cross="mingw64", trailer=cross_build("x86_64-w64-mingw32-", "x86_64-softmmu")) From 7a6e869cb5eebc45d6820457e9e65f53132407d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Lureau?= Date: Tue, 24 Jan 2023 18:01:03 +0000 Subject: [PATCH 11/36] lcitool: drop perl from QEMU project/dependencies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Marc-André Lureau Reviewed-by: Daniel P. Berrangé Reviewed-by: Philippe Mathieu-Daudé Message-Id: <20230110132700.833690-8-marcandre.lureau@redhat.com> Signed-off-by: Alex Bennée Message-Id: <20230124180127.1881110-12-alex.bennee@linaro.org> --- .gitlab-ci.d/cirrus/freebsd-12.vars | 2 +- .gitlab-ci.d/cirrus/freebsd-13.vars | 2 +- .gitlab-ci.d/cirrus/macos-12.vars | 2 +- scripts/ci/setup/build-environment.yml | 1 - tests/docker/dockerfiles/alpine.docker | 1 - tests/docker/dockerfiles/centos8.docker | 1 - tests/docker/dockerfiles/debian-amd64-cross.docker | 1 - tests/docker/dockerfiles/debian-amd64.docker | 1 - tests/docker/dockerfiles/debian-arm64-cross.docker | 1 - tests/docker/dockerfiles/debian-armel-cross.docker | 1 - tests/docker/dockerfiles/debian-armhf-cross.docker | 1 - tests/docker/dockerfiles/debian-mips64el-cross.docker | 1 - tests/docker/dockerfiles/debian-mipsel-cross.docker | 1 - tests/docker/dockerfiles/debian-ppc64el-cross.docker | 1 - tests/docker/dockerfiles/debian-s390x-cross.docker | 1 - tests/docker/dockerfiles/debian-tricore-cross.docker | 1 - tests/docker/dockerfiles/fedora-win32-cross.docker | 1 - tests/docker/dockerfiles/fedora-win64-cross.docker | 1 - tests/docker/dockerfiles/fedora.docker | 1 - tests/docker/dockerfiles/opensuse-leap.docker | 1 - tests/docker/dockerfiles/ubuntu2004.docker | 1 - tests/lcitool/projects/qemu.yml | 1 - tests/vm/centos.aarch64 | 2 +- 23 files changed, 4 insertions(+), 23 deletions(-) diff --git a/.gitlab-ci.d/cirrus/freebsd-12.vars b/.gitlab-ci.d/cirrus/freebsd-12.vars index e3fc3235b9..f32f01a954 100644 --- a/.gitlab-ci.d/cirrus/freebsd-12.vars +++ b/.gitlab-ci.d/cirrus/freebsd-12.vars @@ -11,6 +11,6 @@ MAKE='/usr/local/bin/gmake' NINJA='/usr/local/bin/ninja' PACKAGING_COMMAND='pkg' PIP3='/usr/local/bin/pip-3.8' -PKGS='alsa-lib bash bison bzip2 ca_root_nss capstone4 ccache cdrkit-genisoimage cmocka ctags curl cyrus-sasl dbus diffutils dtc flex fusefs-libs3 gettext git glib gmake gnutls gsed gtk3 json-c libepoxy libffi libgcrypt libjpeg-turbo libnfs libslirp libspice-server libssh libtasn1 llvm lzo2 meson ncurses nettle ninja opencv perl5 pixman pkgconf png py39-numpy py39-pillow py39-pip py39-sphinx py39-sphinx_rtd_theme py39-yaml python3 rpm2cpio sdl2 sdl2_image snappy sndio spice-protocol tesseract texinfo usbredir virglrenderer vte3 zstd' +PKGS='alsa-lib bash bison bzip2 ca_root_nss capstone4 ccache cdrkit-genisoimage cmocka ctags curl cyrus-sasl dbus diffutils dtc flex fusefs-libs3 gettext git glib gmake gnutls gsed gtk3 json-c libepoxy libffi libgcrypt libjpeg-turbo libnfs libslirp libspice-server libssh libtasn1 llvm lzo2 meson ncurses nettle ninja opencv pixman pkgconf png py39-numpy py39-pillow py39-pip py39-sphinx py39-sphinx_rtd_theme py39-yaml python3 rpm2cpio sdl2 sdl2_image snappy sndio spice-protocol tesseract texinfo usbredir virglrenderer vte3 zstd' PYPI_PKGS='' PYTHON='/usr/local/bin/python3' diff --git a/.gitlab-ci.d/cirrus/freebsd-13.vars b/.gitlab-ci.d/cirrus/freebsd-13.vars index 9f56babd9c..813c051616 100644 --- a/.gitlab-ci.d/cirrus/freebsd-13.vars +++ b/.gitlab-ci.d/cirrus/freebsd-13.vars @@ -11,6 +11,6 @@ MAKE='/usr/local/bin/gmake' NINJA='/usr/local/bin/ninja' PACKAGING_COMMAND='pkg' PIP3='/usr/local/bin/pip-3.8' -PKGS='alsa-lib bash bison bzip2 ca_root_nss capstone4 ccache cdrkit-genisoimage cmocka ctags curl cyrus-sasl dbus diffutils dtc flex fusefs-libs3 gettext git glib gmake gnutls gsed gtk3 json-c libepoxy libffi libgcrypt libjpeg-turbo libnfs libslirp libspice-server libssh libtasn1 llvm lzo2 meson ncurses nettle ninja opencv perl5 pixman pkgconf png py39-numpy py39-pillow py39-pip py39-sphinx py39-sphinx_rtd_theme py39-yaml python3 rpm2cpio sdl2 sdl2_image snappy sndio spice-protocol tesseract texinfo usbredir virglrenderer vte3 zstd' +PKGS='alsa-lib bash bison bzip2 ca_root_nss capstone4 ccache cdrkit-genisoimage cmocka ctags curl cyrus-sasl dbus diffutils dtc flex fusefs-libs3 gettext git glib gmake gnutls gsed gtk3 json-c libepoxy libffi libgcrypt libjpeg-turbo libnfs libslirp libspice-server libssh libtasn1 llvm lzo2 meson ncurses nettle ninja opencv pixman pkgconf png py39-numpy py39-pillow py39-pip py39-sphinx py39-sphinx_rtd_theme py39-yaml python3 rpm2cpio sdl2 sdl2_image snappy sndio spice-protocol tesseract texinfo usbredir virglrenderer vte3 zstd' PYPI_PKGS='' PYTHON='/usr/local/bin/python3' diff --git a/.gitlab-ci.d/cirrus/macos-12.vars b/.gitlab-ci.d/cirrus/macos-12.vars index ef9e14b373..33bb4e1040 100644 --- a/.gitlab-ci.d/cirrus/macos-12.vars +++ b/.gitlab-ci.d/cirrus/macos-12.vars @@ -11,6 +11,6 @@ MAKE='/opt/homebrew/bin/gmake' NINJA='/opt/homebrew/bin/ninja' PACKAGING_COMMAND='brew' PIP3='/opt/homebrew/bin/pip3' -PKGS='bash bc bison bzip2 capstone ccache cmocka ctags curl dbus diffutils dtc flex gcovr gettext git glib gnu-sed gnutls gtk+3 jemalloc jpeg-turbo json-c libepoxy libffi libgcrypt libiscsi libnfs libpng libslirp libssh libtasn1 libusb llvm lzo make meson ncurses nettle ninja perl pixman pkg-config python3 rpm2cpio sdl2 sdl2_image snappy sparse spice-protocol tesseract texinfo usbredir vde vte3 zlib zstd' +PKGS='bash bc bison bzip2 capstone ccache cmocka ctags curl dbus diffutils dtc flex gcovr gettext git glib gnu-sed gnutls gtk+3 jemalloc jpeg-turbo json-c libepoxy libffi libgcrypt libiscsi libnfs libpng libslirp libssh libtasn1 libusb llvm lzo make meson ncurses nettle ninja pixman pkg-config python3 rpm2cpio sdl2 sdl2_image snappy sparse spice-protocol tesseract texinfo usbredir vde vte3 zlib zstd' PYPI_PKGS='PyYAML numpy pillow sphinx sphinx-rtd-theme' PYTHON='/opt/homebrew/bin/python3' diff --git a/scripts/ci/setup/build-environment.yml b/scripts/ci/setup/build-environment.yml index b04c2b7cee..58438008ee 100644 --- a/scripts/ci/setup/build-environment.yml +++ b/scripts/ci/setup/build-environment.yml @@ -155,7 +155,6 @@ - nettle-devel - ninja-build - nmap-ncat - - perl-Test-Harness - pixman-devel - python36 - rdma-core-devel diff --git a/tests/docker/dockerfiles/alpine.docker b/tests/docker/dockerfiles/alpine.docker index 094f66f4eb..3293c790c9 100644 --- a/tests/docker/dockerfiles/alpine.docker +++ b/tests/docker/dockerfiles/alpine.docker @@ -77,7 +77,6 @@ RUN apk update && \ numactl-dev \ openssh-client \ pcre-dev \ - perl \ pixman-dev \ pkgconf \ pulseaudio-dev \ diff --git a/tests/docker/dockerfiles/centos8.docker b/tests/docker/dockerfiles/centos8.docker index 1f70d41aeb..f13745e6cc 100644 --- a/tests/docker/dockerfiles/centos8.docker +++ b/tests/docker/dockerfiles/centos8.docker @@ -91,7 +91,6 @@ RUN dnf distro-sync -y && \ openssh-clients \ pam-devel \ pcre-static \ - perl \ pixman-devel \ pkgconfig \ pulseaudio-libs-devel \ diff --git a/tests/docker/dockerfiles/debian-amd64-cross.docker b/tests/docker/dockerfiles/debian-amd64-cross.docker index 5e57309361..d0ace6d0f7 100644 --- a/tests/docker/dockerfiles/debian-amd64-cross.docker +++ b/tests/docker/dockerfiles/debian-amd64-cross.docker @@ -40,7 +40,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ ncat \ ninja-build \ openssh-client \ - perl-base \ pkgconf \ python3 \ python3-numpy \ diff --git a/tests/docker/dockerfiles/debian-amd64.docker b/tests/docker/dockerfiles/debian-amd64.docker index bfeab01ee3..0517c4c315 100644 --- a/tests/docker/dockerfiles/debian-amd64.docker +++ b/tests/docker/dockerfiles/debian-amd64.docker @@ -108,7 +108,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ nettle-dev \ ninja-build \ openssh-client \ - perl-base \ pkgconf \ python3 \ python3-numpy \ diff --git a/tests/docker/dockerfiles/debian-arm64-cross.docker b/tests/docker/dockerfiles/debian-arm64-cross.docker index 98885bd0ee..9ac1c1ba3f 100644 --- a/tests/docker/dockerfiles/debian-arm64-cross.docker +++ b/tests/docker/dockerfiles/debian-arm64-cross.docker @@ -40,7 +40,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ ncat \ ninja-build \ openssh-client \ - perl-base \ pkgconf \ python3 \ python3-numpy \ diff --git a/tests/docker/dockerfiles/debian-armel-cross.docker b/tests/docker/dockerfiles/debian-armel-cross.docker index d5c08714e4..8be492f4ad 100644 --- a/tests/docker/dockerfiles/debian-armel-cross.docker +++ b/tests/docker/dockerfiles/debian-armel-cross.docker @@ -40,7 +40,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ ncat \ ninja-build \ openssh-client \ - perl-base \ pkgconf \ python3 \ python3-numpy \ diff --git a/tests/docker/dockerfiles/debian-armhf-cross.docker b/tests/docker/dockerfiles/debian-armhf-cross.docker index 471444fcf4..da789e04af 100644 --- a/tests/docker/dockerfiles/debian-armhf-cross.docker +++ b/tests/docker/dockerfiles/debian-armhf-cross.docker @@ -40,7 +40,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ ncat \ ninja-build \ openssh-client \ - perl-base \ pkgconf \ python3 \ python3-numpy \ diff --git a/tests/docker/dockerfiles/debian-mips64el-cross.docker b/tests/docker/dockerfiles/debian-mips64el-cross.docker index 15b0224b76..8b7c59c4f9 100644 --- a/tests/docker/dockerfiles/debian-mips64el-cross.docker +++ b/tests/docker/dockerfiles/debian-mips64el-cross.docker @@ -40,7 +40,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ ncat \ ninja-build \ openssh-client \ - perl-base \ pkgconf \ python3 \ python3-numpy \ diff --git a/tests/docker/dockerfiles/debian-mipsel-cross.docker b/tests/docker/dockerfiles/debian-mipsel-cross.docker index a5d3ca6e2f..f9f1ed5fd2 100644 --- a/tests/docker/dockerfiles/debian-mipsel-cross.docker +++ b/tests/docker/dockerfiles/debian-mipsel-cross.docker @@ -40,7 +40,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ ncat \ ninja-build \ openssh-client \ - perl-base \ pkgconf \ python3 \ python3-numpy \ diff --git a/tests/docker/dockerfiles/debian-ppc64el-cross.docker b/tests/docker/dockerfiles/debian-ppc64el-cross.docker index d2954e61f6..e423d88c2d 100644 --- a/tests/docker/dockerfiles/debian-ppc64el-cross.docker +++ b/tests/docker/dockerfiles/debian-ppc64el-cross.docker @@ -40,7 +40,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ ncat \ ninja-build \ openssh-client \ - perl-base \ pkgconf \ python3 \ python3-numpy \ diff --git a/tests/docker/dockerfiles/debian-s390x-cross.docker b/tests/docker/dockerfiles/debian-s390x-cross.docker index d43ce16317..c1134f4cec 100644 --- a/tests/docker/dockerfiles/debian-s390x-cross.docker +++ b/tests/docker/dockerfiles/debian-s390x-cross.docker @@ -40,7 +40,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ ncat \ ninja-build \ openssh-client \ - perl-base \ pkgconf \ python3 \ python3-numpy \ diff --git a/tests/docker/dockerfiles/debian-tricore-cross.docker b/tests/docker/dockerfiles/debian-tricore-cross.docker index b573b9ded2..34b2cea4e3 100644 --- a/tests/docker/dockerfiles/debian-tricore-cross.docker +++ b/tests/docker/dockerfiles/debian-tricore-cross.docker @@ -28,7 +28,6 @@ RUN apt update && \ locales \ make \ ninja-build \ - perl-base \ pkgconf \ python3-pip \ python3-setuptools \ diff --git a/tests/docker/dockerfiles/fedora-win32-cross.docker b/tests/docker/dockerfiles/fedora-win32-cross.docker index cc5d1ac4be..de811b332b 100644 --- a/tests/docker/dockerfiles/fedora-win32-cross.docker +++ b/tests/docker/dockerfiles/fedora-win32-cross.docker @@ -42,7 +42,6 @@ exec "$@"' > /usr/bin/nosync && \ nmap-ncat \ openssh-clients \ pcre-static \ - perl-base \ python3 \ python3-PyYAML \ python3-numpy \ diff --git a/tests/docker/dockerfiles/fedora-win64-cross.docker b/tests/docker/dockerfiles/fedora-win64-cross.docker index cabbf4edfc..71681d6f92 100644 --- a/tests/docker/dockerfiles/fedora-win64-cross.docker +++ b/tests/docker/dockerfiles/fedora-win64-cross.docker @@ -42,7 +42,6 @@ exec "$@"' > /usr/bin/nosync && \ nmap-ncat \ openssh-clients \ pcre-static \ - perl-base \ python3 \ python3-PyYAML \ python3-numpy \ diff --git a/tests/docker/dockerfiles/fedora.docker b/tests/docker/dockerfiles/fedora.docker index f44b005000..ca3793b04b 100644 --- a/tests/docker/dockerfiles/fedora.docker +++ b/tests/docker/dockerfiles/fedora.docker @@ -98,7 +98,6 @@ exec "$@"' > /usr/bin/nosync && \ openssh-clients \ pam-devel \ pcre-static \ - perl-base \ pixman-devel \ pkgconfig \ pulseaudio-libs-devel \ diff --git a/tests/docker/dockerfiles/opensuse-leap.docker b/tests/docker/dockerfiles/opensuse-leap.docker index 4361b01464..680f49e7dc 100644 --- a/tests/docker/dockerfiles/opensuse-leap.docker +++ b/tests/docker/dockerfiles/opensuse-leap.docker @@ -88,7 +88,6 @@ RUN zypper update -y && \ openssh \ pam-devel \ pcre-devel-static \ - perl-base \ pkgconfig \ python3-Pillow \ python3-PyYAML \ diff --git a/tests/docker/dockerfiles/ubuntu2004.docker b/tests/docker/dockerfiles/ubuntu2004.docker index 9417bca2fa..6594bba338 100644 --- a/tests/docker/dockerfiles/ubuntu2004.docker +++ b/tests/docker/dockerfiles/ubuntu2004.docker @@ -105,7 +105,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ nettle-dev \ ninja-build \ openssh-client \ - perl-base \ pkgconf \ python3 \ python3-numpy \ diff --git a/tests/lcitool/projects/qemu.yml b/tests/lcitool/projects/qemu.yml index c62dbc00f9..c2af92348a 100644 --- a/tests/lcitool/projects/qemu.yml +++ b/tests/lcitool/projects/qemu.yml @@ -83,7 +83,6 @@ packages: - ncursesw - pam - pcre-static - - perl - pixman - pkg-config - pulseaudio diff --git a/tests/vm/centos.aarch64 b/tests/vm/centos.aarch64 index 2de7ef6992..3f58de1e64 100755 --- a/tests/vm/centos.aarch64 +++ b/tests/vm/centos.aarch64 @@ -28,7 +28,7 @@ DEFAULT_CONFIG = { "dnf config-manager --set-enabled powertools, " "dnf config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo, " "dnf install -y make ninja-build git python38 gcc gcc-c++ flex bison "\ - "glib2-devel perl pixman-devel zlib-devel docker-ce.aarch64, " + "glib2-devel pixman-devel zlib-devel docker-ce.aarch64, " "systemctl enable docker, " ), # We increase beyond the default time since during boot From df07c72a74fd43bfc1e6431a19eb3252a3cfe1dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Lureau?= Date: Tue, 24 Jan 2023 18:01:04 +0000 Subject: [PATCH 12/36] lcitool: drop texinfo from QEMU project/dependencies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Marc-André Lureau Reviewed-by: Daniel P. Berrangé Message-Id: <20230110132700.833690-9-marcandre.lureau@redhat.com> Signed-off-by: Alex Bennée Message-Id: <20230124180127.1881110-13-alex.bennee@linaro.org> --- .gitlab-ci.d/cirrus/freebsd-12.vars | 2 +- .gitlab-ci.d/cirrus/freebsd-13.vars | 2 +- .gitlab-ci.d/cirrus/macos-12.vars | 2 +- tests/docker/dockerfiles/alpine.docker | 1 - tests/docker/dockerfiles/centos8.docker | 1 - tests/docker/dockerfiles/debian-amd64-cross.docker | 3 +-- tests/docker/dockerfiles/debian-amd64.docker | 1 - tests/docker/dockerfiles/debian-arm64-cross.docker | 3 +-- tests/docker/dockerfiles/debian-armel-cross.docker | 3 +-- tests/docker/dockerfiles/debian-armhf-cross.docker | 3 +-- tests/docker/dockerfiles/debian-mips64el-cross.docker | 3 +-- tests/docker/dockerfiles/debian-mipsel-cross.docker | 3 +-- tests/docker/dockerfiles/debian-ppc64el-cross.docker | 3 +-- tests/docker/dockerfiles/debian-s390x-cross.docker | 3 +-- tests/docker/dockerfiles/debian-toolchain.docker | 1 - tests/docker/dockerfiles/fedora-win32-cross.docker | 1 - tests/docker/dockerfiles/fedora-win64-cross.docker | 1 - tests/docker/dockerfiles/fedora.docker | 1 - tests/docker/dockerfiles/opensuse-leap.docker | 1 - tests/docker/dockerfiles/ubuntu2004.docker | 1 - tests/lcitool/projects/qemu.yml | 1 - 21 files changed, 11 insertions(+), 29 deletions(-) diff --git a/.gitlab-ci.d/cirrus/freebsd-12.vars b/.gitlab-ci.d/cirrus/freebsd-12.vars index f32f01a954..8934e5d57f 100644 --- a/.gitlab-ci.d/cirrus/freebsd-12.vars +++ b/.gitlab-ci.d/cirrus/freebsd-12.vars @@ -11,6 +11,6 @@ MAKE='/usr/local/bin/gmake' NINJA='/usr/local/bin/ninja' PACKAGING_COMMAND='pkg' PIP3='/usr/local/bin/pip-3.8' -PKGS='alsa-lib bash bison bzip2 ca_root_nss capstone4 ccache cdrkit-genisoimage cmocka ctags curl cyrus-sasl dbus diffutils dtc flex fusefs-libs3 gettext git glib gmake gnutls gsed gtk3 json-c libepoxy libffi libgcrypt libjpeg-turbo libnfs libslirp libspice-server libssh libtasn1 llvm lzo2 meson ncurses nettle ninja opencv pixman pkgconf png py39-numpy py39-pillow py39-pip py39-sphinx py39-sphinx_rtd_theme py39-yaml python3 rpm2cpio sdl2 sdl2_image snappy sndio spice-protocol tesseract texinfo usbredir virglrenderer vte3 zstd' +PKGS='alsa-lib bash bison bzip2 ca_root_nss capstone4 ccache cdrkit-genisoimage cmocka ctags curl cyrus-sasl dbus diffutils dtc flex fusefs-libs3 gettext git glib gmake gnutls gsed gtk3 json-c libepoxy libffi libgcrypt libjpeg-turbo libnfs libslirp libspice-server libssh libtasn1 llvm lzo2 meson ncurses nettle ninja opencv pixman pkgconf png py39-numpy py39-pillow py39-pip py39-sphinx py39-sphinx_rtd_theme py39-yaml python3 rpm2cpio sdl2 sdl2_image snappy sndio spice-protocol tesseract usbredir virglrenderer vte3 zstd' PYPI_PKGS='' PYTHON='/usr/local/bin/python3' diff --git a/.gitlab-ci.d/cirrus/freebsd-13.vars b/.gitlab-ci.d/cirrus/freebsd-13.vars index 813c051616..65ce456c48 100644 --- a/.gitlab-ci.d/cirrus/freebsd-13.vars +++ b/.gitlab-ci.d/cirrus/freebsd-13.vars @@ -11,6 +11,6 @@ MAKE='/usr/local/bin/gmake' NINJA='/usr/local/bin/ninja' PACKAGING_COMMAND='pkg' PIP3='/usr/local/bin/pip-3.8' -PKGS='alsa-lib bash bison bzip2 ca_root_nss capstone4 ccache cdrkit-genisoimage cmocka ctags curl cyrus-sasl dbus diffutils dtc flex fusefs-libs3 gettext git glib gmake gnutls gsed gtk3 json-c libepoxy libffi libgcrypt libjpeg-turbo libnfs libslirp libspice-server libssh libtasn1 llvm lzo2 meson ncurses nettle ninja opencv pixman pkgconf png py39-numpy py39-pillow py39-pip py39-sphinx py39-sphinx_rtd_theme py39-yaml python3 rpm2cpio sdl2 sdl2_image snappy sndio spice-protocol tesseract texinfo usbredir virglrenderer vte3 zstd' +PKGS='alsa-lib bash bison bzip2 ca_root_nss capstone4 ccache cdrkit-genisoimage cmocka ctags curl cyrus-sasl dbus diffutils dtc flex fusefs-libs3 gettext git glib gmake gnutls gsed gtk3 json-c libepoxy libffi libgcrypt libjpeg-turbo libnfs libslirp libspice-server libssh libtasn1 llvm lzo2 meson ncurses nettle ninja opencv pixman pkgconf png py39-numpy py39-pillow py39-pip py39-sphinx py39-sphinx_rtd_theme py39-yaml python3 rpm2cpio sdl2 sdl2_image snappy sndio spice-protocol tesseract usbredir virglrenderer vte3 zstd' PYPI_PKGS='' PYTHON='/usr/local/bin/python3' diff --git a/.gitlab-ci.d/cirrus/macos-12.vars b/.gitlab-ci.d/cirrus/macos-12.vars index 33bb4e1040..65b78fa08f 100644 --- a/.gitlab-ci.d/cirrus/macos-12.vars +++ b/.gitlab-ci.d/cirrus/macos-12.vars @@ -11,6 +11,6 @@ MAKE='/opt/homebrew/bin/gmake' NINJA='/opt/homebrew/bin/ninja' PACKAGING_COMMAND='brew' PIP3='/opt/homebrew/bin/pip3' -PKGS='bash bc bison bzip2 capstone ccache cmocka ctags curl dbus diffutils dtc flex gcovr gettext git glib gnu-sed gnutls gtk+3 jemalloc jpeg-turbo json-c libepoxy libffi libgcrypt libiscsi libnfs libpng libslirp libssh libtasn1 libusb llvm lzo make meson ncurses nettle ninja pixman pkg-config python3 rpm2cpio sdl2 sdl2_image snappy sparse spice-protocol tesseract texinfo usbredir vde vte3 zlib zstd' +PKGS='bash bc bison bzip2 capstone ccache cmocka ctags curl dbus diffutils dtc flex gcovr gettext git glib gnu-sed gnutls gtk+3 jemalloc jpeg-turbo json-c libepoxy libffi libgcrypt libiscsi libnfs libpng libslirp libssh libtasn1 libusb llvm lzo make meson ncurses nettle ninja pixman pkg-config python3 rpm2cpio sdl2 sdl2_image snappy sparse spice-protocol tesseract usbredir vde vte3 zlib zstd' PYPI_PKGS='PyYAML numpy pillow sphinx sphinx-rtd-theme' PYTHON='/opt/homebrew/bin/python3' diff --git a/tests/docker/dockerfiles/alpine.docker b/tests/docker/dockerfiles/alpine.docker index 3293c790c9..4a569d82f6 100644 --- a/tests/docker/dockerfiles/alpine.docker +++ b/tests/docker/dockerfiles/alpine.docker @@ -99,7 +99,6 @@ RUN apk update && \ spice-protocol \ tar \ tesseract-ocr \ - texinfo \ usbredir-dev \ util-linux \ vde2-dev \ diff --git a/tests/docker/dockerfiles/centos8.docker b/tests/docker/dockerfiles/centos8.docker index f13745e6cc..fbc953c6dc 100644 --- a/tests/docker/dockerfiles/centos8.docker +++ b/tests/docker/dockerfiles/centos8.docker @@ -110,7 +110,6 @@ RUN dnf distro-sync -y && \ systemd-devel \ systemtap-sdt-devel \ tar \ - texinfo \ usbredir-devel \ util-linux \ virglrenderer-devel \ diff --git a/tests/docker/dockerfiles/debian-amd64-cross.docker b/tests/docker/dockerfiles/debian-amd64-cross.docker index d0ace6d0f7..5175095a85 100644 --- a/tests/docker/dockerfiles/debian-amd64-cross.docker +++ b/tests/docker/dockerfiles/debian-amd64-cross.docker @@ -55,8 +55,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ sparse \ tar \ tesseract-ocr \ - tesseract-ocr-eng \ - texinfo && \ + tesseract-ocr-eng && \ eatmydata apt-get autoremove -y && \ eatmydata apt-get autoclean -y && \ sed -Ei 's,^# (en_US\.UTF-8 .*)$,\1,' /etc/locale.gen && \ diff --git a/tests/docker/dockerfiles/debian-amd64.docker b/tests/docker/dockerfiles/debian-amd64.docker index 0517c4c315..b61f664ea2 100644 --- a/tests/docker/dockerfiles/debian-amd64.docker +++ b/tests/docker/dockerfiles/debian-amd64.docker @@ -125,7 +125,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ tar \ tesseract-ocr \ tesseract-ocr-eng \ - texinfo \ xfslibs-dev \ zlib1g-dev && \ eatmydata apt-get autoremove -y && \ diff --git a/tests/docker/dockerfiles/debian-arm64-cross.docker b/tests/docker/dockerfiles/debian-arm64-cross.docker index 9ac1c1ba3f..b69958c69f 100644 --- a/tests/docker/dockerfiles/debian-arm64-cross.docker +++ b/tests/docker/dockerfiles/debian-arm64-cross.docker @@ -55,8 +55,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ sparse \ tar \ tesseract-ocr \ - tesseract-ocr-eng \ - texinfo && \ + tesseract-ocr-eng && \ eatmydata apt-get autoremove -y && \ eatmydata apt-get autoclean -y && \ sed -Ei 's,^# (en_US\.UTF-8 .*)$,\1,' /etc/locale.gen && \ diff --git a/tests/docker/dockerfiles/debian-armel-cross.docker b/tests/docker/dockerfiles/debian-armel-cross.docker index 8be492f4ad..96b524fab6 100644 --- a/tests/docker/dockerfiles/debian-armel-cross.docker +++ b/tests/docker/dockerfiles/debian-armel-cross.docker @@ -55,8 +55,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ sparse \ tar \ tesseract-ocr \ - tesseract-ocr-eng \ - texinfo && \ + tesseract-ocr-eng && \ eatmydata apt-get autoremove -y && \ eatmydata apt-get autoclean -y && \ sed -Ei 's,^# (en_US\.UTF-8 .*)$,\1,' /etc/locale.gen && \ diff --git a/tests/docker/dockerfiles/debian-armhf-cross.docker b/tests/docker/dockerfiles/debian-armhf-cross.docker index da789e04af..08a75cebdb 100644 --- a/tests/docker/dockerfiles/debian-armhf-cross.docker +++ b/tests/docker/dockerfiles/debian-armhf-cross.docker @@ -55,8 +55,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ sparse \ tar \ tesseract-ocr \ - tesseract-ocr-eng \ - texinfo && \ + tesseract-ocr-eng && \ eatmydata apt-get autoremove -y && \ eatmydata apt-get autoclean -y && \ sed -Ei 's,^# (en_US\.UTF-8 .*)$,\1,' /etc/locale.gen && \ diff --git a/tests/docker/dockerfiles/debian-mips64el-cross.docker b/tests/docker/dockerfiles/debian-mips64el-cross.docker index 8b7c59c4f9..5930e6fa5d 100644 --- a/tests/docker/dockerfiles/debian-mips64el-cross.docker +++ b/tests/docker/dockerfiles/debian-mips64el-cross.docker @@ -55,8 +55,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ sparse \ tar \ tesseract-ocr \ - tesseract-ocr-eng \ - texinfo && \ + tesseract-ocr-eng && \ eatmydata apt-get autoremove -y && \ eatmydata apt-get autoclean -y && \ sed -Ei 's,^# (en_US\.UTF-8 .*)$,\1,' /etc/locale.gen && \ diff --git a/tests/docker/dockerfiles/debian-mipsel-cross.docker b/tests/docker/dockerfiles/debian-mipsel-cross.docker index f9f1ed5fd2..c65d9830e7 100644 --- a/tests/docker/dockerfiles/debian-mipsel-cross.docker +++ b/tests/docker/dockerfiles/debian-mipsel-cross.docker @@ -55,8 +55,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ sparse \ tar \ tesseract-ocr \ - tesseract-ocr-eng \ - texinfo && \ + tesseract-ocr-eng && \ eatmydata apt-get autoremove -y && \ eatmydata apt-get autoclean -y && \ sed -Ei 's,^# (en_US\.UTF-8 .*)$,\1,' /etc/locale.gen && \ diff --git a/tests/docker/dockerfiles/debian-ppc64el-cross.docker b/tests/docker/dockerfiles/debian-ppc64el-cross.docker index e423d88c2d..2ae56c978e 100644 --- a/tests/docker/dockerfiles/debian-ppc64el-cross.docker +++ b/tests/docker/dockerfiles/debian-ppc64el-cross.docker @@ -55,8 +55,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ sparse \ tar \ tesseract-ocr \ - tesseract-ocr-eng \ - texinfo && \ + tesseract-ocr-eng && \ eatmydata apt-get autoremove -y && \ eatmydata apt-get autoclean -y && \ sed -Ei 's,^# (en_US\.UTF-8 .*)$,\1,' /etc/locale.gen && \ diff --git a/tests/docker/dockerfiles/debian-s390x-cross.docker b/tests/docker/dockerfiles/debian-s390x-cross.docker index c1134f4cec..0db86a0fcd 100644 --- a/tests/docker/dockerfiles/debian-s390x-cross.docker +++ b/tests/docker/dockerfiles/debian-s390x-cross.docker @@ -55,8 +55,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ sparse \ tar \ tesseract-ocr \ - tesseract-ocr-eng \ - texinfo && \ + tesseract-ocr-eng && \ eatmydata apt-get autoremove -y && \ eatmydata apt-get autoclean -y && \ sed -Ei 's,^# (en_US\.UTF-8 .*)$,\1,' /etc/locale.gen && \ diff --git a/tests/docker/dockerfiles/debian-toolchain.docker b/tests/docker/dockerfiles/debian-toolchain.docker index d3d4d3344e..6c73408b34 100644 --- a/tests/docker/dockerfiles/debian-toolchain.docker +++ b/tests/docker/dockerfiles/debian-toolchain.docker @@ -21,7 +21,6 @@ RUN apt update && \ libmpc-dev \ libmpfr-dev \ rsync \ - texinfo \ wget && \ DEBIAN_FRONTEND=noninteractive eatmydata \ apt build-dep -yy --arch-only gcc glibc diff --git a/tests/docker/dockerfiles/fedora-win32-cross.docker b/tests/docker/dockerfiles/fedora-win32-cross.docker index de811b332b..b659c0b8a8 100644 --- a/tests/docker/dockerfiles/fedora-win32-cross.docker +++ b/tests/docker/dockerfiles/fedora-win32-cross.docker @@ -57,7 +57,6 @@ exec "$@"' > /usr/bin/nosync && \ tar \ tesseract \ tesseract-langpack-eng \ - texinfo \ util-linux \ which && \ nosync dnf autoremove -y && \ diff --git a/tests/docker/dockerfiles/fedora-win64-cross.docker b/tests/docker/dockerfiles/fedora-win64-cross.docker index 71681d6f92..0a404c15bf 100644 --- a/tests/docker/dockerfiles/fedora-win64-cross.docker +++ b/tests/docker/dockerfiles/fedora-win64-cross.docker @@ -57,7 +57,6 @@ exec "$@"' > /usr/bin/nosync && \ tar \ tesseract \ tesseract-langpack-eng \ - texinfo \ util-linux \ which && \ nosync dnf autoremove -y && \ diff --git a/tests/docker/dockerfiles/fedora.docker b/tests/docker/dockerfiles/fedora.docker index ca3793b04b..5d60a96141 100644 --- a/tests/docker/dockerfiles/fedora.docker +++ b/tests/docker/dockerfiles/fedora.docker @@ -121,7 +121,6 @@ exec "$@"' > /usr/bin/nosync && \ tar \ tesseract \ tesseract-langpack-eng \ - texinfo \ usbredir-devel \ util-linux \ virglrenderer-devel \ diff --git a/tests/docker/dockerfiles/opensuse-leap.docker b/tests/docker/dockerfiles/opensuse-leap.docker index 680f49e7dc..4b2c02d6ab 100644 --- a/tests/docker/dockerfiles/opensuse-leap.docker +++ b/tests/docker/dockerfiles/opensuse-leap.docker @@ -111,7 +111,6 @@ RUN zypper update -y && \ tar \ tesseract-ocr \ tesseract-ocr-traineddata-english \ - texinfo \ usbredir-devel \ util-linux \ virglrenderer-devel \ diff --git a/tests/docker/dockerfiles/ubuntu2004.docker b/tests/docker/dockerfiles/ubuntu2004.docker index 6594bba338..13ab0b6887 100644 --- a/tests/docker/dockerfiles/ubuntu2004.docker +++ b/tests/docker/dockerfiles/ubuntu2004.docker @@ -124,7 +124,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ tar \ tesseract-ocr \ tesseract-ocr-eng \ - texinfo \ xfslibs-dev \ zlib1g-dev && \ eatmydata apt-get autoremove -y && \ diff --git a/tests/lcitool/projects/qemu.yml b/tests/lcitool/projects/qemu.yml index c2af92348a..6467bcf08a 100644 --- a/tests/lcitool/projects/qemu.yml +++ b/tests/lcitool/projects/qemu.yml @@ -109,7 +109,6 @@ packages: - tar - tesseract - tesseract-eng - - texinfo - usbredir - virglrenderer - vte From 7ac17cb8b2ab012e6289b4d77fd220a96511d88d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Tue, 24 Jan 2023 18:01:05 +0000 Subject: [PATCH 13/36] tests/docker: Install flex in debian-tricore-cross MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When flex is not available, binutils sources default to the 'missing' script, but the current script available is not in the format expected by the 'configure' script: $ ./configure ... /usr/src/binutils/missing: Unknown `--run' option Try `/usr/src/binutils/missing --help' for more information configure: WARNING: `missing' script is too old or missing ... checking for bison... bison -y checking for flex... no checking for lex... no checking for flex... /usr/src/binutils/missing flex $ make ... updating ldgram.h gcc -DHAVE_CONFIG_H -I. -I. -I. -D_GNU_SOURCE -I. -I. -I../bfd -I./../bfd -I./../include -I./../intl -I../intl -w -DLOCALEDIR="\"/usr/local/share/locale\"" -W -Wall -Wstrict-prototypes -Wmissing-prototypes -w -c `test -f 'ldgram.c' || echo './'`ldgram.c `test -f ldlex.l || echo './'`ldlex.l /bin/sh: 1: ldlex.l: not found make[3]: *** [Makefile:662: ldlex.c] Error 127 make[3]: Leaving directory '/usr/src/binutils/ld' make[2]: *** [Makefile:799: all-recursive] Error 1 By pass the 'missing' script use by directly installing 'flex' in the container. Reported-by: Peter Maydell Suggested-by: Peter Maydell Signed-off-by: Philippe Mathieu-Daudé Message-Id: <20230112155643.7408-1-philmd@linaro.org> Reviewed-by: Bastian-Koppelmann Signed-off-by: Alex Bennée Message-Id: <20230124180127.1881110-14-alex.bennee@linaro.org> --- tests/docker/dockerfiles/debian-tricore-cross.docker | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/docker/dockerfiles/debian-tricore-cross.docker b/tests/docker/dockerfiles/debian-tricore-cross.docker index 34b2cea4e3..5ae58efa09 100644 --- a/tests/docker/dockerfiles/debian-tricore-cross.docker +++ b/tests/docker/dockerfiles/debian-tricore-cross.docker @@ -20,6 +20,7 @@ RUN apt update && \ bzip2 \ ca-certificates \ ccache \ + flex \ g++ \ gcc \ git \ From 5b9636326248a736b1d14b6132a9ae41d5f92fb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Tue, 24 Jan 2023 18:01:06 +0000 Subject: [PATCH 14/36] tests/docker: drop debian-tricore-cross's partial status MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This image is perfectly capable of building QEMU, and indeed we do that on gitlab. Drop the DOCKER_PARTIAL_IMAGES setting so we can also test the gitlab build locally. Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Alex Bennée Message-Id: <20230124180127.1881110-15-alex.bennee@linaro.org> --- tests/docker/Makefile.include | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/docker/Makefile.include b/tests/docker/Makefile.include index 665ddde518..bfb0dcac21 100644 --- a/tests/docker/Makefile.include +++ b/tests/docker/Makefile.include @@ -128,7 +128,6 @@ DOCKER_PARTIAL_IMAGES += debian-mips-cross DOCKER_PARTIAL_IMAGES += debian-nios2-cross DOCKER_PARTIAL_IMAGES += debian-riscv64-test-cross DOCKER_PARTIAL_IMAGES += debian-sh4-cross debian-sparc64-cross -DOCKER_PARTIAL_IMAGES += debian-tricore-cross DOCKER_PARTIAL_IMAGES += debian-xtensa-cross DOCKER_PARTIAL_IMAGES += fedora-cris-cross From 5842de51573fdbd7299ab4b33d64b7446cc07649 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Tue, 24 Jan 2023 18:01:07 +0000 Subject: [PATCH 15/36] tests/tcg: skip the vma-pthread test on CI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We are getting a lot of failures that are not related to changes so this could be a flaky test. Reviewed-by: Richard Henderson Signed-off-by: Alex Bennée Message-Id: <20230124180127.1881110-16-alex.bennee@linaro.org> --- tests/tcg/multiarch/Makefile.target | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/tcg/multiarch/Makefile.target b/tests/tcg/multiarch/Makefile.target index e7213af492..ae8b3d7268 100644 --- a/tests/tcg/multiarch/Makefile.target +++ b/tests/tcg/multiarch/Makefile.target @@ -42,6 +42,15 @@ munmap-pthread: LDFLAGS+=-pthread vma-pthread: CFLAGS+=-pthread vma-pthread: LDFLAGS+=-pthread +# The vma-pthread seems very sensitive on gitlab and we currently +# don't know if its exposing a real bug or the test is flaky. +ifneq ($(GITLAB_CI),) +run-vma-pthread: vma-pthread + $(call skip-test, $<, "flaky on CI?") +run-plugin-vma-pthread-with-%: vma-pthread + $(call skip-test, $<, "flaky on CI?") +endif + # We define the runner for test-mmap after the individual # architectures have defined their supported pages sizes. If no # additional page sizes are defined we only run the default test. From 57dbce5a4f76c2e2201905a6211eb2a83a8bacce Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Tue, 24 Jan 2023 18:01:08 +0000 Subject: [PATCH 16/36] tests/tcg: Use SIGKILL for timeout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit linux-user blocks all signals while attempting to handle guest signals (e.g. ABRT), which means that the default TERM sent by timeout has no effect -- KILL instead. Signed-off-by: Richard Henderson Message-Id: <20230117035701.168514-2-richard.henderson@linaro.org> [AJB: expanded commit message from cover letter] Signed-off-by: Alex Bennée Reviewed-by: Philippe Mathieu-Daudé Message-Id: <20230124180127.1881110-17-alex.bennee@linaro.org> --- tests/tcg/Makefile.target | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/tcg/Makefile.target b/tests/tcg/Makefile.target index 14bc013181..a3b0aaf8af 100644 --- a/tests/tcg/Makefile.target +++ b/tests/tcg/Makefile.target @@ -54,10 +54,10 @@ cc-option = if $(call cc-test, $1); then \ # $1 = test name, $2 = cmd, $3 = desc ifeq ($(filter %-softmmu, $(TARGET)),) -run-test = $(call quiet-command, timeout --foreground $(TIMEOUT) $2 > $1.out, \ +run-test = $(call quiet-command, timeout -s KILL --foreground $(TIMEOUT) $2 > $1.out, \ TEST,$(or $3, $*, $<) on $(TARGET_NAME)) else -run-test = $(call quiet-command, timeout --foreground $(TIMEOUT) $2, \ +run-test = $(call quiet-command, timeout -s KILL --foreground $(TIMEOUT) $2, \ TEST,$(or $3, $*, $<) on $(TARGET_NAME)) endif From 6e890b0521674c74c4a34b847fa8d7c57e43aadf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Tue, 24 Jan 2023 18:01:09 +0000 Subject: [PATCH 17/36] gitlab: wrap up test results for custom runners MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of spewing the whole log to stdout lets just define them as build artefacts so we can examine them later. Where we are running check-tcg run it first as those tests are yet to be integrated into meson. To avoid confusion we don't run multiple check-tcg tests at once. Reviewed-by: Thomas Huth Signed-off-by: Alex Bennée Message-Id: <20230124180127.1881110-18-alex.bennee@linaro.org> --- .gitlab-ci.d/custom-runners.yml | 11 +++++++++++ .gitlab-ci.d/custom-runners/ubuntu-20.04-s390x.yml | 13 ++++++------- .../custom-runners/ubuntu-22.04-aarch32.yml | 2 +- .../custom-runners/ubuntu-22.04-aarch64.yml | 13 ++++++------- 4 files changed, 24 insertions(+), 15 deletions(-) diff --git a/.gitlab-ci.d/custom-runners.yml b/.gitlab-ci.d/custom-runners.yml index 97f99e29c2..9fdc476c48 100644 --- a/.gitlab-ci.d/custom-runners.yml +++ b/.gitlab-ci.d/custom-runners.yml @@ -13,6 +13,17 @@ variables: GIT_STRATEGY: clone +# All custom runners can extend this template to upload the testlog +# data as an artifact and also feed the junit report +.custom_artifacts_template: + artifacts: + name: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG" + expire_in: 7 days + paths: + - build/meson-logs/testlog.txt + reports: + junit: build/meson-logs/testlog.junit.xml + include: - local: '/.gitlab-ci.d/custom-runners/ubuntu-20.04-s390x.yml' - local: '/.gitlab-ci.d/custom-runners/ubuntu-22.04-aarch64.yml' diff --git a/.gitlab-ci.d/custom-runners/ubuntu-20.04-s390x.yml b/.gitlab-ci.d/custom-runners/ubuntu-20.04-s390x.yml index fcaef9e5ef..f512eaeaa3 100644 --- a/.gitlab-ci.d/custom-runners/ubuntu-20.04-s390x.yml +++ b/.gitlab-ci.d/custom-runners/ubuntu-20.04-s390x.yml @@ -3,6 +3,7 @@ # "Install basic packages to build QEMU on Ubuntu 20.04/20.04" ubuntu-20.04-s390x-all-linux-static: + extends: .custom_artifacts_template needs: [] stage: build tags: @@ -19,12 +20,11 @@ ubuntu-20.04-s390x-all-linux-static: - ../configure --enable-debug --static --disable-system --disable-glusterfs --disable-libssh || { cat config.log meson-logs/meson-log.txt; exit 1; } - make --output-sync -j`nproc` + - make --output-sync check-tcg - make --output-sync -j`nproc` check - || { cat meson-logs/testlog.txt; exit 1; } ; - - make --output-sync -j`nproc` check-tcg - || { cat meson-logs/testlog.txt; exit 1; } ; ubuntu-20.04-s390x-all: + extends: .custom_artifacts_template needs: [] stage: build tags: @@ -41,9 +41,9 @@ ubuntu-20.04-s390x-all: || { cat config.log meson-logs/meson-log.txt; exit 1; } - make --output-sync -j`nproc` - make --output-sync -j`nproc` check - || { cat meson-logs/testlog.txt; exit 1; } ; ubuntu-20.04-s390x-alldbg: + extends: .custom_artifacts_template needs: [] stage: build tags: @@ -64,9 +64,9 @@ ubuntu-20.04-s390x-alldbg: - make clean - make --output-sync -j`nproc` - make --output-sync -j`nproc` check - || { cat meson-logs/testlog.txt; exit 1; } ; ubuntu-20.04-s390x-clang: + extends: .custom_artifacts_template needs: [] stage: build tags: @@ -86,7 +86,6 @@ ubuntu-20.04-s390x-clang: || { cat config.log meson-logs/meson-log.txt; exit 1; } - make --output-sync -j`nproc` - make --output-sync -j`nproc` check - || { cat meson-logs/testlog.txt; exit 1; } ; ubuntu-20.04-s390x-tci: needs: [] @@ -109,6 +108,7 @@ ubuntu-20.04-s390x-tci: - make --output-sync -j`nproc` ubuntu-20.04-s390x-notcg: + extends: .custom_artifacts_template needs: [] stage: build tags: @@ -128,4 +128,3 @@ ubuntu-20.04-s390x-notcg: || { cat config.log meson-logs/meson-log.txt; exit 1; } - make --output-sync -j`nproc` - make --output-sync -j`nproc` check - || { cat meson-logs/testlog.txt; exit 1; } ; diff --git a/.gitlab-ci.d/custom-runners/ubuntu-22.04-aarch32.yml b/.gitlab-ci.d/custom-runners/ubuntu-22.04-aarch32.yml index 2c386fa3e9..42137aaf2a 100644 --- a/.gitlab-ci.d/custom-runners/ubuntu-22.04-aarch32.yml +++ b/.gitlab-ci.d/custom-runners/ubuntu-22.04-aarch32.yml @@ -3,6 +3,7 @@ # "Install basic packages to build QEMU on Ubuntu 20.04" ubuntu-22.04-aarch32-all: + extends: .custom_artifacts_template needs: [] stage: build tags: @@ -22,4 +23,3 @@ ubuntu-22.04-aarch32-all: || { cat config.log meson-logs/meson-log.txt; exit 1; } - make --output-sync -j`nproc --ignore=40` - make --output-sync -j`nproc --ignore=40` check - || { cat meson-logs/testlog.txt; exit 1; } ; diff --git a/.gitlab-ci.d/custom-runners/ubuntu-22.04-aarch64.yml b/.gitlab-ci.d/custom-runners/ubuntu-22.04-aarch64.yml index 725ca8ffea..8ba85be440 100644 --- a/.gitlab-ci.d/custom-runners/ubuntu-22.04-aarch64.yml +++ b/.gitlab-ci.d/custom-runners/ubuntu-22.04-aarch64.yml @@ -3,6 +3,7 @@ # "Install basic packages to build QEMU on Ubuntu 20.04" ubuntu-22.04-aarch64-all-linux-static: + extends: .custom_artifacts_template needs: [] stage: build tags: @@ -19,12 +20,11 @@ ubuntu-22.04-aarch64-all-linux-static: - ../configure --enable-debug --static --disable-system --disable-pie || { cat config.log meson-logs/meson-log.txt; exit 1; } - make --output-sync -j`nproc --ignore=40` + - make check-tcg - make --output-sync -j`nproc --ignore=40` check - || { cat meson-logs/testlog.txt; exit 1; } ; - - make --output-sync -j`nproc --ignore=40` check-tcg - || { cat meson-logs/testlog.txt; exit 1; } ; ubuntu-22.04-aarch64-all: + extends: .custom_artifacts_template needs: [] stage: build tags: @@ -44,9 +44,9 @@ ubuntu-22.04-aarch64-all: || { cat config.log meson-logs/meson-log.txt; exit 1; } - make --output-sync -j`nproc --ignore=40` - make --output-sync -j`nproc --ignore=40` check - || { cat meson-logs/testlog.txt; exit 1; } ; ubuntu-22.04-aarch64-alldbg: + extends: .custom_artifacts_template needs: [] stage: build tags: @@ -63,9 +63,9 @@ ubuntu-22.04-aarch64-alldbg: - make clean - make --output-sync -j`nproc --ignore=40` - make --output-sync -j`nproc --ignore=40` check - || { cat meson-logs/testlog.txt; exit 1; } ; ubuntu-22.04-aarch64-clang: + extends: .custom_artifacts_template needs: [] stage: build tags: @@ -85,7 +85,6 @@ ubuntu-22.04-aarch64-clang: || { cat config.log meson-logs/meson-log.txt; exit 1; } - make --output-sync -j`nproc --ignore=40` - make --output-sync -j`nproc --ignore=40` check - || { cat meson-logs/testlog.txt; exit 1; } ; ubuntu-22.04-aarch64-tci: needs: [] @@ -108,6 +107,7 @@ ubuntu-22.04-aarch64-tci: - make --output-sync -j`nproc --ignore=40` ubuntu-22.04-aarch64-notcg: + extends: .custom_artifacts_template needs: [] stage: build tags: @@ -127,4 +127,3 @@ ubuntu-22.04-aarch64-notcg: || { cat config.log meson-logs/meson-log.txt; exit 1; } - make --output-sync -j`nproc --ignore=40` - make --output-sync -j`nproc --ignore=40` check - || { cat meson-logs/testlog.txt; exit 1; } ; From 51d11acf3163b2f09e3e3d1ab7c102c82cacc40c Mon Sep 17 00:00:00 2001 From: Thomas Huth Date: Tue, 24 Jan 2023 18:01:10 +0000 Subject: [PATCH 18/36] MAINTAINERS: Fix the entry for tests/tcg/nios2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tests/tcg/nios2/Makefile.target has accidentally been added to the Microblaze section. Move it into the correct nios2 section instead - and while we're at it, it should also cover the whole folder, and not only the Makefile. Fixes: 67f80eb4d0 ("tests/tcg: enable debian-nios2-cross for test building") Signed-off-by: Thomas Huth Reviewed-by: Philippe Mathieu-Daudé Message-Id: <20230119130326.2030297-1-thuth@redhat.com> Signed-off-by: Alex Bennée Message-Id: <20230124180127.1881110-19-alex.bennee@linaro.org> --- MAINTAINERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MAINTAINERS b/MAINTAINERS index c581c11a64..629ab5bbb1 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -240,7 +240,6 @@ F: target/microblaze/ F: hw/microblaze/ F: disas/microblaze.c F: tests/docker/dockerfiles/debian-microblaze-cross.d/build-toolchain.sh -F: tests/tcg/nios2/Makefile.target MIPS TCG CPUs M: Philippe Mathieu-Daudé @@ -262,6 +261,7 @@ F: hw/nios2/ F: disas/nios2.c F: configs/devices/nios2-softmmu/default.mak F: tests/docker/dockerfiles/debian-nios2-cross.d/build-toolchain.sh +F: tests/tcg/nios2/ OpenRISC TCG CPUs M: Stafford Horne From 9d195efd6588100f8074a1c8daf9f403ebf39740 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Tue, 24 Jan 2023 18:01:11 +0000 Subject: [PATCH 19/36] docs: add hotlinks to about preface text MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make it easier to navigate the documentation. Reviewed-by: Peter Maydell Acked-by: Richard Henderson Signed-off-by: Alex Bennée Reviewed-by: Philippe Mathieu-Daudé Message-Id: <20230124180127.1881110-20-alex.bennee@linaro.org> --- docs/about/index.rst | 16 ++++++++-------- docs/system/index.rst | 2 ++ docs/tools/index.rst | 2 ++ docs/user/index.rst | 2 ++ 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/docs/about/index.rst b/docs/about/index.rst index 5bea653c07..bae1309cc6 100644 --- a/docs/about/index.rst +++ b/docs/about/index.rst @@ -5,19 +5,19 @@ About QEMU QEMU is a generic and open source machine emulator and virtualizer. QEMU can be used in several different ways. The most common is for -"system emulation", where it provides a virtual model of an +:ref:`System Emulation`, where it provides a virtual model of an entire machine (CPU, memory and emulated devices) to run a guest OS. -In this mode the CPU may be fully emulated, or it may work with -a hypervisor such as KVM, Xen, Hax or Hypervisor.Framework to -allow the guest to run directly on the host CPU. +In this mode the CPU may be fully emulated, or it may work with a +hypervisor such as KVM, Xen, Hax or Hypervisor.Framework to allow the +guest to run directly on the host CPU. -The second supported way to use QEMU is "user mode emulation", +The second supported way to use QEMU is :ref:`User Mode Emulation`, where QEMU can launch processes compiled for one CPU on another CPU. In this mode the CPU is always emulated. -QEMU also provides a number of standalone commandline utilities, -such as the ``qemu-img`` disk image utility that allows you to create, -convert and modify disk images. +QEMU also provides a number of standalone :ref:`command line +utilities`, such as the ``qemu-img`` disk image utility that +allows you to create, convert and modify disk images. .. toctree:: :maxdepth: 2 diff --git a/docs/system/index.rst b/docs/system/index.rst index e3695649c5..282b6ffb56 100644 --- a/docs/system/index.rst +++ b/docs/system/index.rst @@ -1,3 +1,5 @@ +.. _System Emulation: + ---------------- System Emulation ---------------- diff --git a/docs/tools/index.rst b/docs/tools/index.rst index 1edd5a8054..2151adcf78 100644 --- a/docs/tools/index.rst +++ b/docs/tools/index.rst @@ -1,3 +1,5 @@ +.. _Tools: + ----- Tools ----- diff --git a/docs/user/index.rst b/docs/user/index.rst index 2c4e29f3db..782d27cda2 100644 --- a/docs/user/index.rst +++ b/docs/user/index.rst @@ -1,3 +1,5 @@ +.. _User Mode Emulation: + ------------------- User Mode Emulation ------------------- From a0a6754bb549b255a269b9ab63674e06a1d14ddb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Tue, 24 Jan 2023 18:01:12 +0000 Subject: [PATCH 20/36] docs: add a new section to outline emulation support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This affects both system and user mode emulation so we should probably list it up front. Acked-by: Richard Henderson Signed-off-by: Alex Bennée Message-Id: <20230124180127.1881110-21-alex.bennee@linaro.org> --- docs/about/emulation.rst | 103 ++++++++++++++++++++++++++++++++++ docs/about/index.rst | 1 + docs/devel/tcg-plugins.rst | 2 + docs/system/arm/emulation.rst | 2 + 4 files changed, 108 insertions(+) create mode 100644 docs/about/emulation.rst diff --git a/docs/about/emulation.rst b/docs/about/emulation.rst new file mode 100644 index 0000000000..7ec7e09381 --- /dev/null +++ b/docs/about/emulation.rst @@ -0,0 +1,103 @@ +Emulation +========= + +QEMU's Tiny Code Generator (TCG) provides the ability to emulate a +number of CPU architectures on any supported host platform. Both +:ref:`System Emulation` and :ref:`User Mode Emulation` are supported +depending on the guest architecture. + +.. list-table:: Supported Guest Architectures for Emulation + :widths: 30 10 10 50 + :header-rows: 1 + + * - Architecture (qemu name) + - System + - User + - Notes + * - Alpha + - Yes + - Yes + - Legacy 64 bit RISC ISA developed by DEC + * - Arm (arm, aarch64) + - :ref:`Yes` + - Yes + - Wide range of features, see :ref:`Arm Emulation` for details + * - AVR + - :ref:`Yes` + - No + - 8 bit micro controller, often used in maker projects + * - Cris + - Yes + - Yes + - Embedded RISC chip developed by AXIS + * - Hexagon + - No + - Yes + - Family of DSPs by Qualcomm + * - PA-RISC (hppa) + - Yes + - Yes + - A legacy RISC system used in HP's old minicomputers + * - x86 (i386, x86_64) + - :ref:`Yes` + - Yes + - The ubiquitous desktop PC CPU architecture, 32 and 64 bit. + * - Loongarch + - Yes + - Yes + - A MIPS-like 64bit RISC architecture developed in China + * - m68k + - :ref:`Yes` + - Yes + - Motorola 68000 variants and ColdFire + * - Microblaze + - Yes + - Yes + - RISC based soft-core by Xilinx + * - MIPS (mips*) + - :ref:`Yes` + - Yes + - Venerable RISC architecture originally out of Stanford University + * - Nios2 + - Yes + - Yes + - 32 bit embedded soft-core by Altera + * - OpenRISC + - :ref:`Yes` + - Yes + - Open source RISC architecture developed by the OpenRISC community + * - Power (ppc, ppc64) + - :ref:`Yes` + - Yes + - A general purpose RISC architecture now managed by IBM + * - RISC-V + - :ref:`Yes` + - Yes + - An open standard RISC ISA maintained by RISC-V International + * - RX + - :ref:`Yes` + - No + - A 32 bit micro controller developed by Renesas + * - s390x + - :ref:`Yes` + - Yes + - A 64 bit CPU found in IBM's System Z mainframes + * - sh4 + - Yes + - Yes + - A 32 bit RISC embedded CPU developed by Hitachi + * - SPARC (sparc, sparc64) + - :ref:`Yes` + - Yes + - A RISC ISA originally developed by Sun Microsystems + * - Tricore + - Yes + - No + - A 32 bit RISC/uController/DSP developed by Infineon + * - Xtensa + - :ref:`Yes` + - Yes + - A configurable 32 bit soft core now owned by Cadence + +A number of features are are only available when running under +emulation including :ref:`Record/Replay` and :ref:`TCG Plugins`. diff --git a/docs/about/index.rst b/docs/about/index.rst index bae1309cc6..b00b584b31 100644 --- a/docs/about/index.rst +++ b/docs/about/index.rst @@ -23,6 +23,7 @@ allows you to create, convert and modify disk images. :maxdepth: 2 build-platforms + emulation deprecated removed-features license diff --git a/docs/devel/tcg-plugins.rst b/docs/devel/tcg-plugins.rst index 9740a70406..81dcd43a61 100644 --- a/docs/devel/tcg-plugins.rst +++ b/docs/devel/tcg-plugins.rst @@ -3,6 +3,8 @@ Copyright (c) 2019, Linaro Limited Written by Emilio Cota and Alex Bennée +.. _TCG Plugins: + QEMU TCG Plugins ================ diff --git a/docs/system/arm/emulation.rst b/docs/system/arm/emulation.rst index b33d7c28dc..b87e064d9d 100644 --- a/docs/system/arm/emulation.rst +++ b/docs/system/arm/emulation.rst @@ -1,3 +1,5 @@ +.. _Arm Emulation: + A-profile CPU architecture support ================================== From 2da9d21360cdabde1bb0dad7ae06e7df5dcb7835 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Tue, 24 Jan 2023 18:01:13 +0000 Subject: [PATCH 21/36] semihosting: add semihosting section to the docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The main reason to do this is to document our O_BINARY implementation decision somewhere. However I've also moved some of the implementation details out of qemu-options and added links between the two. As a bonus I've highlighted the scary warnings about host access with the appropriate RST tags. Acked-by: Richard Henderson Reviewed-by: Richard Henderson Signed-off-by: Alex Bennée Message-Id: <20230124180127.1881110-22-alex.bennee@linaro.org> --- docs/about/emulation.rst | 87 ++++++++++++++++++++++++++++++++++++++++ qemu-options.hx | 25 ++++-------- 2 files changed, 95 insertions(+), 17 deletions(-) diff --git a/docs/about/emulation.rst b/docs/about/emulation.rst index 7ec7e09381..b510a54418 100644 --- a/docs/about/emulation.rst +++ b/docs/about/emulation.rst @@ -101,3 +101,90 @@ depending on the guest architecture. A number of features are are only available when running under emulation including :ref:`Record/Replay` and :ref:`TCG Plugins`. + +.. _Semihosting: + +Semihosting +----------- + +Semihosting is a feature defined by the owner of the architecture to +allow programs to interact with a debugging host system. On real +hardware this is usually provided by an In-circuit emulator (ICE) +hooked directly to the board. QEMU's implementation allows for +semihosting calls to be passed to the host system or via the +``gdbstub``. + +Generally semihosting makes it easier to bring up low level code before a +more fully functional operating system has been enabled. On QEMU it +also allows for embedded micro-controller code which typically doesn't +have a full libc to be run as "bare-metal" code under QEMU's user-mode +emulation. It is also useful for writing test cases and indeed a +number of compiler suites as well as QEMU itself use semihosting calls +to exit test code while reporting the success state. + +Semihosting is only available using TCG emulation. This is because the +instructions to trigger a semihosting call are typically reserved +causing most hypervisors to trap and fault on them. + +.. warning:: + Semihosting inherently bypasses any isolation there may be between + the guest and the host. As a result a program using semihosting can + happily trash your host system. You should only ever run trusted + code with semihosting enabled. + +Redirection +~~~~~~~~~~~ + +Semihosting calls can be re-directed to a (potentially remote) gdb +during debugging via the :ref:`gdbstub`. Output to the +semihosting console is configured as a ``chardev`` so can be +redirected to a file, pipe or socket like any other ``chardev`` +device. + +Supported Targets +~~~~~~~~~~~~~~~~~ + +Most targets offer similar semihosting implementations with some +minor changes to define the appropriate instruction to encode the +semihosting call and which registers hold the parameters. They tend to +presents a simple POSIX-like API which allows your program to read and +write files, access the console and some other basic interactions. + +For full details of the ABI for a particular target, and the set of +calls it provides, you should consult the semihosting specification +for that architecture. + +.. note:: + QEMU makes an implementation decision to implement all file + access in ``O_BINARY`` mode. The user-visible effect of this is + regardless of the text/binary mode the program sets QEMU will + always select a binary mode ensuring no line-terminator conversion + is performed on input or output. This is because gdb semihosting + support doesn't make the distinction between the modes and + magically processing line endings can be confusing. + +.. list-table:: Guest Architectures supporting Semihosting + :widths: 10 10 80 + :header-rows: 1 + + * - Architecture + - Modes + - Specification + * - Arm + - System and User-mode + - https://github.com/ARM-software/abi-aa/blob/main/semihosting/semihosting.rst + * - m68k + - System + - https://sourceware.org/git/?p=newlib-cygwin.git;a=blob;f=libgloss/m68k/m68k-semi.txt;hb=HEAD + * - MIPS + - System + - Unified Hosting Interface (MD01069) + * - Nios II + - System + - https://sourceware.org/git/gitweb.cgi?p=newlib-cygwin.git;a=blob;f=libgloss/nios2/nios2-semi.txt;hb=HEAD + * - RISC-V + - System and User-mode + - https://github.com/riscv/riscv-semihosting-spec/blob/main/riscv-semihosting-spec.adoc + * - Xtensa + - System + - Tensilica ISS SIMCALL diff --git a/qemu-options.hx b/qemu-options.hx index d59d19704b..88e93c6103 100644 --- a/qemu-options.hx +++ b/qemu-options.hx @@ -4633,10 +4633,11 @@ DEF("semihosting", 0, QEMU_OPTION_semihosting, QEMU_ARCH_MIPS | QEMU_ARCH_NIOS2 | QEMU_ARCH_RISCV) SRST ``-semihosting`` - Enable semihosting mode (ARM, M68K, Xtensa, MIPS, Nios II, RISC-V only). + Enable :ref:`Semihosting` mode (ARM, M68K, Xtensa, MIPS, Nios II, RISC-V only). - Note that this allows guest direct access to the host filesystem, so - should only be used with a trusted guest OS. + .. warning:: + Note that this allows guest direct access to the host filesystem, so + should only be used with a trusted guest OS. See the -semihosting-config option documentation for further information about the facilities this enables. @@ -4648,22 +4649,12 @@ QEMU_ARCH_ARM | QEMU_ARCH_M68K | QEMU_ARCH_XTENSA | QEMU_ARCH_MIPS | QEMU_ARCH_NIOS2 | QEMU_ARCH_RISCV) SRST ``-semihosting-config [enable=on|off][,target=native|gdb|auto][,chardev=id][,userspace=on|off][,arg=str[,...]]`` - Enable and configure semihosting (ARM, M68K, Xtensa, MIPS, Nios II, RISC-V + Enable and configure :ref:`Semihosting` (ARM, M68K, Xtensa, MIPS, Nios II, RISC-V only). - Note that this allows guest direct access to the host filesystem, so - should only be used with a trusted guest OS. - - On Arm this implements the standard semihosting API, version 2.0. - - On M68K this implements the "ColdFire GDB" interface used by - libgloss. - - Xtensa semihosting provides basic file IO calls, such as - open/read/write/seek/select. Tensilica baremetal libc for ISS and - linux platform "sim" use this interface. - - On RISC-V this implements the standard semihosting API, version 0.2. + .. warning:: + Note that this allows guest direct access to the host filesystem, so + should only be used with a trusted guest OS. ``target=native|gdb|auto`` Defines where the semihosting calls will be addressed, to QEMU From 2c46bc240a1ac9b99974fcd4b735a6715ce99798 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Tue, 24 Jan 2023 18:01:14 +0000 Subject: [PATCH 22/36] docs: add an introduction to the system docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drop the frankly misleading quickstart section for a more rounded introduction section. This new section gives an overview of the accelerators as well as a high level introduction to some of the key features of the emulator. We also expand on a general form for a QEMU command line with a hopefully not too scary worked example of what this looks like. Acked-by: Richard Henderson Signed-off-by: Alex Bennée Reviewed-by: Kashyap Chamarthy Message-Id: <20230124180127.1881110-23-alex.bennee@linaro.org> --- docs/interop/live-block-operations.rst | 2 + docs/interop/qemu-qmp-ref.rst | 2 + docs/system/index.rst | 2 +- docs/system/introduction.rst | 220 +++++++++++++++++++++++++ docs/system/multi-process.rst | 2 + docs/system/quickstart.rst | 21 --- 6 files changed, 227 insertions(+), 22 deletions(-) create mode 100644 docs/system/introduction.rst delete mode 100644 docs/system/quickstart.rst diff --git a/docs/interop/live-block-operations.rst b/docs/interop/live-block-operations.rst index 135784ab33..691429c7af 100644 --- a/docs/interop/live-block-operations.rst +++ b/docs/interop/live-block-operations.rst @@ -4,6 +4,8 @@ This work is licensed under the terms of the GNU GPL, version 2 or later. See the COPYING file in the top-level directory. +.. _Live Block Operations: + ============================ Live Block Device Operations ============================ diff --git a/docs/interop/qemu-qmp-ref.rst b/docs/interop/qemu-qmp-ref.rst index 357effd64f..f94614a0b2 100644 --- a/docs/interop/qemu-qmp-ref.rst +++ b/docs/interop/qemu-qmp-ref.rst @@ -1,3 +1,5 @@ +.. _QMP Ref: + QEMU QMP Reference Manual ========================= diff --git a/docs/system/index.rst b/docs/system/index.rst index 282b6ffb56..3605bbe1ce 100644 --- a/docs/system/index.rst +++ b/docs/system/index.rst @@ -12,7 +12,7 @@ or Hypervisor.Framework. .. toctree:: :maxdepth: 3 - quickstart + introduction invocation device-emulation keys diff --git a/docs/system/introduction.rst b/docs/system/introduction.rst new file mode 100644 index 0000000000..c8a9fe6c1d --- /dev/null +++ b/docs/system/introduction.rst @@ -0,0 +1,220 @@ +Introduction +============ + +Virtualisation Accelerators +--------------------------- + +QEMU's system emulation provides a virtual model of a machine (CPU, +memory and emulated devices) to run a guest OS. It supports a number +of hypervisors (known as accelerators) as well as a JIT known as the +Tiny Code Generator (TCG) capable of emulating many CPUs. + +.. list-table:: Supported Accelerators + :header-rows: 1 + + * - Accelerator + - Host OS + - Host Architectures + * - KVM + - Linux + - Arm (64 bit only), MIPS, PPC, RISC-V, s390x, x86 + * - Xen + - Linux (as dom0) + - Arm, x86 + * - Intel HAXM (hax) + - Linux, Windows + - x86 + * - Hypervisor Framework (hvf) + - MacOS + - x86 (64 bit only), Arm (64 bit only) + * - Windows Hypervisor Platform (wphx) + - Windows + - x86 + * - NetBSD Virtual Machine Monitor (nvmm) + - NetBSD + - x86 + * - Tiny Code Generator (tcg) + - Linux, other POSIX, Windows, MacOS + - Arm, x86, Loongarch64, MIPS, PPC, s390x, Sparc64 + +Feature Overview +---------------- + +System emulation provides a wide range of device models to emulate +various hardware components you may want to add to your machine. This +includes a wide number of VirtIO devices which are specifically tuned +for efficient operation under virtualisation. Some of the device +emulation can be offloaded from the main QEMU process using either +vhost-user (for VirtIO) or :ref:`Multi-process QEMU`. If the platform +supports it QEMU also supports directly passing devices through to +guest VMs to eliminate the device emulation overhead. See +:ref:`device-emulation` for more details. + +There is a full :ref:`featured block layer` +which allows for construction of complex storage topology which can be +stacked across multiple layers supporting redirection, networking, +snapshots and migration support. + +The flexible ``chardev`` system allows for handling IO from character +like devices using stdio, files, unix sockets and TCP networking. + +QEMU provides a number of management interfaces including a line based +:ref:`Human Monitor Protocol (HMP)` that allows you to +dynamically add and remove devices as well as introspect the system +state. The :ref:`QEMU Monitor Protocol` (QMP) is a well +defined, versioned, machine usable API that presents a rich interface +to other tools to create, control and manage Virtual Machines. This is +the interface used by higher level tools interfaces such as `Virt +Manager `_ using the `libvirt framework +`_. + +For the common accelerators QEMU, supported debugging with its +:ref:`gdbstub` which allows users to connect GDB and debug +system software images. + +Running +------- + +QEMU provides a rich and complex API which can be overwhelming to +understand. While some architectures can boot something with just a +disk image, those examples elide a lot of details with defaults that +may not be optimal for modern systems. + +For a non-x86 system where we emulate a broad range of machine types, +the command lines are generally more explicit in defining the machine +and boot behaviour. You will find often find example command lines in +the :ref:`system-targets-ref` section of the manual. + +While the project doesn't want to discourage users from using the +command line to launch VMs, we do want to highlight that there are a +number of projects dedicated to providing a more user friendly +experience. Those built around the ``libvirt`` framework can make use +of feature probing to build modern VM images tailored to run on the +hardware you have. + +That said, the general form of a QEMU command line can be expressed +as: + +.. parsed-literal:: + + $ |qemu_system| [machine opts] \\ + [cpu opts] \\ + [accelerator opts] \\ + [device opts] \\ + [backend opts] \\ + [interface opts] \\ + [boot opts] + +Most options will generate some help information. So for example: + +.. parsed-literal:: + + $ |qemu_system| -M help + +will list the machine types supported by that QEMU binary. ``help`` +can also be passed as an argument to another option. For example: + +.. parsed-literal:: + + $ |qemu_system| -device scsi-hd,help + +will list the arguments and their default values of additional options +that can control the behaviour of the ``scsi-hd`` device. + +.. list-table:: Options Overview + :header-rows: 1 + :widths: 10, 90 + + * - Options + - + * - Machine + - Define the machine type, amount of memory etc + * - CPU + - Type and number/topology of vCPUs. Most accelerators offer + a ``host`` cpu option which simply passes through your host CPU + configuration without filtering out any features. + * - Accelerator + - This will depend on the hypervisor you run. Note that the + default is TCG, which is purely emulated, so you must specify an + accelerator type to take advantage of hardware virtualization. + * - Devices + - Additional devices that are not defined by default with the + machine type. + * - Backends + - Backends are how QEMU deals with the guest's data, for example + how a block device is stored, how network devices see the + network or how a serial device is directed to the outside world. + * - Interfaces + - How the system is displayed, how it is managed and controlled or + debugged. + * - Boot + - How the system boots, via firmware or direct kernel boot. + +In the following example we first define a ``virt`` machine which is a +general purpose platform for running Aarch64 guests. We enable +virtualisation so we can use KVM inside the emulated guest. As the +``virt`` machine comes with some built in pflash devices we give them +names so we can override the defaults later. + +.. code:: + + $ qemu-system-aarch64 \ + -machine type=virt,virtualization=on,pflash0=rom,pflash1=efivars \ + -m 4096 \ + +We then define the 4 vCPUs using the ``max`` option which gives us all +the Arm features QEMU is capable of emulating. We enable a more +emulation friendly implementation of Arm's pointer authentication +algorithm. We explicitly specify TCG acceleration even though QEMU +would default to it anyway. + +.. code:: + + -cpu max,pauth-impdef=on \ + -smp 4 \ + -accel tcg \ + +As the ``virt`` platform doesn't have any default network or storage +devices we need to define them. We give them ids so we can link them +with the backend later on. + +.. code:: + + -device virtio-net-pci,netdev=unet \ + -device virtio-scsi-pci \ + -device scsi-hd,drive=hd \ + +We connect the user-mode networking to our network device. As +user-mode networking isn't directly accessible from the outside world +we forward localhost port 2222 to the ssh port on the guest. + +.. code:: + + -netdev user,id=unet,hostfwd=tcp::2222-:22 \ + +We connect the guest visible block device to an LVM partition we have +set aside for our guest. + +.. code:: + + -blockdev driver=raw,node-name=hd,file.driver=host_device,file.filename=/dev/lvm-disk/debian-bullseye-arm64 \ + +We then tell QEMU to multiplex the :ref:`QEMU monitor` with the serial +port output (we can switch between the two using :ref:`keys in the +character backend multiplexer`). As there is no default graphical +device we disable the display as we can work entirely in the terminal. + +.. code:: + + -serial mon:stdio \ + -display none \ + +Finally we override the default firmware to ensure we have some +storage for EFI to persist its configuration. That firmware is +responsible for finding the disk, booting grub and eventually running +our system. + +.. code:: + + -blockdev node-name=rom,driver=file,filename=(pwd)/pc-bios/edk2-aarch64-code.fd,read-only=true \ + -blockdev node-name=efivars,driver=file,filename=$HOME/images/qemu-arm64-efivars diff --git a/docs/system/multi-process.rst b/docs/system/multi-process.rst index 210531ee17..16f0352416 100644 --- a/docs/system/multi-process.rst +++ b/docs/system/multi-process.rst @@ -1,3 +1,5 @@ +.. _Multi-process QEMU: + Multi-process QEMU ================== diff --git a/docs/system/quickstart.rst b/docs/system/quickstart.rst deleted file mode 100644 index 681678c86e..0000000000 --- a/docs/system/quickstart.rst +++ /dev/null @@ -1,21 +0,0 @@ -.. _pcsys_005fquickstart: - -Quick Start ------------ - -Download and uncompress a PC hard disk image with Linux installed (e.g. -``linux.img``) and type: - -.. parsed-literal:: - - |qemu_system| linux.img - -Linux should boot and give you a prompt. - -Users should be aware the above example elides a lot of the complexity -of setting up a VM with x86_64 specific defaults and assumes the -first non switch argument is a PC compatible disk image with a boot -sector. For a non-x86 system where we emulate a broad range of machine -types, the command lines are generally more explicit in defining the -machine and boot behaviour. You will find more example command lines -in the :ref:`system-targets-ref` section of the manual. From 978c2bf97b2f8b0938dc6570c46275ed8d8e96f8 Mon Sep 17 00:00:00 2001 From: Keith Packard Date: Tue, 24 Jan 2023 18:01:15 +0000 Subject: [PATCH 23/36] semihosting: Write back semihosting data before completion callback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 'lock_user' allocates a host buffer to shadow a target buffer, 'unlock_user' copies that host buffer back to the target and frees the host memory. If the completion function uses the target buffer, it must be called after unlock_user to ensure the data are present. This caused the arm-compatible TARGET_SYS_READC to fail as the completion function, common_semi_readc_cb, pulled data from the target buffer which would not have been gotten the console data. I decided to fix all instances of this pattern instead of just the console_read function to make things consistent and potentially fix bugs in other cases. Signed-off-by: Keith Packard Reviewed-by: Richard Henderson Message-Id: <20221012014822.1242170-1-keithp@keithp.com> Signed-off-by: Alex Bennée Reviewed-by: Philippe Mathieu-Daudé Message-Id: <20230124180127.1881110-24-alex.bennee@linaro.org> --- semihosting/syscalls.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/semihosting/syscalls.c b/semihosting/syscalls.c index 5893c760c5..ba28194b59 100644 --- a/semihosting/syscalls.c +++ b/semihosting/syscalls.c @@ -319,11 +319,11 @@ static void host_read(CPUState *cs, gdb_syscall_complete_cb complete, } ret = RETRY_ON_EINTR(read(gf->hostfd, ptr, len)); if (ret == -1) { - complete(cs, -1, errno); unlock_user(ptr, buf, 0); + complete(cs, -1, errno); } else { - complete(cs, ret, 0); unlock_user(ptr, buf, ret); + complete(cs, ret, 0); } } @@ -339,8 +339,8 @@ static void host_write(CPUState *cs, gdb_syscall_complete_cb complete, return; } ret = write(gf->hostfd, ptr, len); - complete(cs, ret, ret == -1 ? errno : 0); unlock_user(ptr, buf, 0); + complete(cs, ret, ret == -1 ? errno : 0); } static void host_lseek(CPUState *cs, gdb_syscall_complete_cb complete, @@ -426,8 +426,8 @@ static void host_stat(CPUState *cs, gdb_syscall_complete_cb complete, ret = -1; } } - complete(cs, ret, err); unlock_user(name, fname, 0); + complete(cs, ret, err); } static void host_remove(CPUState *cs, gdb_syscall_complete_cb complete, @@ -444,8 +444,8 @@ static void host_remove(CPUState *cs, gdb_syscall_complete_cb complete, } ret = remove(p); - complete(cs, ret, ret ? errno : 0); unlock_user(p, fname, 0); + complete(cs, ret, ret ? errno : 0); } static void host_rename(CPUState *cs, gdb_syscall_complete_cb complete, @@ -469,9 +469,9 @@ static void host_rename(CPUState *cs, gdb_syscall_complete_cb complete, } ret = rename(ostr, nstr); - complete(cs, ret, ret ? errno : 0); unlock_user(ostr, oname, 0); unlock_user(nstr, nname, 0); + complete(cs, ret, ret ? errno : 0); } static void host_system(CPUState *cs, gdb_syscall_complete_cb complete, @@ -488,8 +488,8 @@ static void host_system(CPUState *cs, gdb_syscall_complete_cb complete, } ret = system(p); - complete(cs, ret, ret == -1 ? errno : 0); unlock_user(p, cmd, 0); + complete(cs, ret, ret == -1 ? errno : 0); } static void host_gettimeofday(CPUState *cs, gdb_syscall_complete_cb complete, @@ -554,8 +554,8 @@ static void staticfile_read(CPUState *cs, gdb_syscall_complete_cb complete, } memcpy(ptr, gf->staticfile.data + gf->staticfile.off, len); gf->staticfile.off += len; - complete(cs, len, 0); unlock_user(ptr, buf, len); + complete(cs, len, 0); } static void staticfile_lseek(CPUState *cs, gdb_syscall_complete_cb complete, @@ -608,8 +608,8 @@ static void console_read(CPUState *cs, gdb_syscall_complete_cb complete, return; } ret = qemu_semihosting_console_read(cs, ptr, len); - complete(cs, ret, 0); unlock_user(ptr, buf, ret); + complete(cs, ret, 0); } static void console_write(CPUState *cs, gdb_syscall_complete_cb complete, @@ -624,8 +624,8 @@ static void console_write(CPUState *cs, gdb_syscall_complete_cb complete, return; } ret = qemu_semihosting_console_write(ptr, len); - complete(cs, ret ? ret : -1, ret ? 0 : EIO); unlock_user(ptr, buf, 0); + complete(cs, ret ? ret : -1, ret ? 0 : EIO); } static void console_fstat(CPUState *cs, gdb_syscall_complete_cb complete, From 0bccdb42df66b697e0a4681c2ad4f10eccef5330 Mon Sep 17 00:00:00 2001 From: Evgeny Iakovlev Date: Tue, 24 Jan 2023 18:01:16 +0000 Subject: [PATCH 24/36] semihosting: add O_BINARY flag in host_open for NT compatibility MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Windows open(2) implementation opens files in text mode by default and needs a Windows-only O_BINARY flag to open files as binary. QEMU already knows about that flag in osdep and it is defined to 0 on non-Windows, so we can just add it to the host_flags for better compatibility. Signed-off-by: Evgeny Iakovlev Reviewed-by: Philippe Mathieu-Daudé Reviewed-by: Bin Meng Message-Id: <20230106102018.20520-1-eiakovlev@linux.microsoft.com> Signed-off-by: Alex Bennée Message-Id: <20230124180127.1881110-25-alex.bennee@linaro.org> --- semihosting/syscalls.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/semihosting/syscalls.c b/semihosting/syscalls.c index ba28194b59..e89992cf90 100644 --- a/semihosting/syscalls.c +++ b/semihosting/syscalls.c @@ -253,7 +253,7 @@ static void host_open(CPUState *cs, gdb_syscall_complete_cb complete, { CPUArchState *env G_GNUC_UNUSED = cs->env_ptr; char *p; - int ret, host_flags; + int ret, host_flags = O_BINARY; ret = validate_lock_user_string(&p, cs, fname, fname_len); if (ret < 0) { @@ -262,11 +262,11 @@ static void host_open(CPUState *cs, gdb_syscall_complete_cb complete, } if (gdb_flags & GDB_O_WRONLY) { - host_flags = O_WRONLY; + host_flags |= O_WRONLY; } else if (gdb_flags & GDB_O_RDWR) { - host_flags = O_RDWR; + host_flags |= O_RDWR; } else { - host_flags = O_RDONLY; + host_flags |= O_RDONLY; } if (gdb_flags & GDB_O_CREAT) { host_flags |= O_CREAT; From d54c6d3b5ddc0555f8d3bb6b0ba59da1f59b18f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Tue, 24 Jan 2023 18:01:17 +0000 Subject: [PATCH 25/36] tests/tcg: add memory-sve test for aarch64 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This will be helpful in debugging problems with tracking SVE memory accesses via the TCG plugins system. Reviewed-by: Richard Henderson Signed-off-by: Alex Bennée Cc: Robert Henry Cc: Aaron Lindsay Message-Id: <20230124180127.1881110-26-alex.bennee@linaro.org> --- tests/tcg/aarch64/Makefile.softmmu-target | 7 +++++++ tests/tcg/aarch64/system/boot.S | 3 ++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/tcg/aarch64/Makefile.softmmu-target b/tests/tcg/aarch64/Makefile.softmmu-target index a1368905f5..df9747bae8 100644 --- a/tests/tcg/aarch64/Makefile.softmmu-target +++ b/tests/tcg/aarch64/Makefile.softmmu-target @@ -36,6 +36,13 @@ config-cc.mak: Makefile memory: CFLAGS+=-DCHECK_UNALIGNED=1 +memory-sve: memory.c $(LINK_SCRIPT) $(CRT_OBJS) $(MINILIB_OBJS) + $(CC) $(CFLAGS) $(EXTRA_CFLAGS) $< -o $@ $(LDFLAGS) + +memory-sve: CFLAGS+=-DCHECK_UNALIGNED=1 -march=armv8.1-a+sve -O3 -fno-tree-loop-distribute-patterns + +TESTS+=memory-sve + # Running QEMU_BASE_MACHINE=-M virt -cpu max -display none QEMU_OPTS+=$(QEMU_BASE_MACHINE) -semihosting-config enable=on,target=native,chardev=output -kernel diff --git a/tests/tcg/aarch64/system/boot.S b/tests/tcg/aarch64/system/boot.S index e190b1efa6..f136363d2a 100644 --- a/tests/tcg/aarch64/system/boot.S +++ b/tests/tcg/aarch64/system/boot.S @@ -179,12 +179,13 @@ __start: isb /* - * Enable FP registers. The standard C pre-amble will be + * Enable FP/SVE registers. The standard C pre-amble will be * saving these and A-profile compilers will use AdvSIMD * registers unless we tell it not to. */ mrs x0, cpacr_el1 orr x0, x0, #(3 << 20) + orr x0, x0, #(3 << 16) msr cpacr_el1, x0 /* Setup some stack space and enter the test code. From 4731f89b3b9690efccb6084b9fee43083669de14 Mon Sep 17 00:00:00 2001 From: Emilio Cota Date: Tue, 24 Jan 2023 18:01:18 +0000 Subject: [PATCH 26/36] cpu: free cpu->tb_jmp_cache with RCU MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes the appended use-after-free. The root cause is that during tb invalidation we use CPU_FOREACH, and therefore to safely free a vCPU we must wait for an RCU grace period to elapse. $ x86_64-linux-user/qemu-x86_64 tests/tcg/x86_64-linux-user/munmap-pthread ================================================================= ==1800604==ERROR: AddressSanitizer: heap-use-after-free on address 0x62d0005f7418 at pc 0x5593da6704eb bp 0x7f4961a7ac70 sp 0x7f4961a7ac60 READ of size 8 at 0x62d0005f7418 thread T2 #0 0x5593da6704ea in tb_jmp_cache_inval_tb ../accel/tcg/tb-maint.c:244 #1 0x5593da6704ea in do_tb_phys_invalidate ../accel/tcg/tb-maint.c:290 #2 0x5593da670631 in tb_phys_invalidate__locked ../accel/tcg/tb-maint.c:306 #3 0x5593da670631 in tb_invalidate_phys_page_range__locked ../accel/tcg/tb-maint.c:542 #4 0x5593da67106d in tb_invalidate_phys_range ../accel/tcg/tb-maint.c:614 #5 0x5593da6a64d4 in target_munmap ../linux-user/mmap.c:766 #6 0x5593da6dba05 in do_syscall1 ../linux-user/syscall.c:10105 #7 0x5593da6f564c in do_syscall ../linux-user/syscall.c:13329 #8 0x5593da49e80c in cpu_loop ../linux-user/x86_64/../i386/cpu_loop.c:233 #9 0x5593da6be28c in clone_func ../linux-user/syscall.c:6633 #10 0x7f496231cb42 in start_thread nptl/pthread_create.c:442 #11 0x7f49623ae9ff (/lib/x86_64-linux-gnu/libc.so.6+0x1269ff) 0x62d0005f7418 is located 28696 bytes inside of 32768-byte region [0x62d0005f0400,0x62d0005f8400) freed by thread T148 here: #0 0x7f49627b6460 in __interceptor_free ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:52 #1 0x5593da5ac057 in cpu_exec_unrealizefn ../cpu.c:180 #2 0x5593da81f851 (/home/cota/src/qemu/build/qemu-x86_64+0x484851) Signed-off-by: Emilio Cota Reviewed-by: Richard Henderson Message-Id: <20230111151628.320011-2-cota@braap.org> Signed-off-by: Alex Bennée Message-Id: <20230124180127.1881110-27-alex.bennee@linaro.org> --- accel/tcg/cpu-exec.c | 3 +-- accel/tcg/tb-jmp-cache.h | 1 + cpu.c | 11 ++++++++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/accel/tcg/cpu-exec.c b/accel/tcg/cpu-exec.c index 04cd1f3092..25ec73ef9a 100644 --- a/accel/tcg/cpu-exec.c +++ b/accel/tcg/cpu-exec.c @@ -1064,13 +1064,12 @@ void tcg_exec_realizefn(CPUState *cpu, Error **errp) /* undo the initializations in reverse order */ void tcg_exec_unrealizefn(CPUState *cpu) { - qemu_plugin_vcpu_exit_hook(cpu); #ifndef CONFIG_USER_ONLY tcg_iommu_free_notifier_list(cpu); #endif /* !CONFIG_USER_ONLY */ tlb_destroy(cpu); - g_free(cpu->tb_jmp_cache); + g_free_rcu(cpu->tb_jmp_cache, rcu); } #ifndef CONFIG_USER_ONLY diff --git a/accel/tcg/tb-jmp-cache.h b/accel/tcg/tb-jmp-cache.h index ff5ffc8fc2..b3f6e78835 100644 --- a/accel/tcg/tb-jmp-cache.h +++ b/accel/tcg/tb-jmp-cache.h @@ -18,6 +18,7 @@ * a load_acquire/store_release to 'tb'. */ struct CPUJumpCache { + struct rcu_head rcu; struct { TranslationBlock *tb; #if TARGET_TB_PCREL diff --git a/cpu.c b/cpu.c index 4a7d865427..21cf809614 100644 --- a/cpu.c +++ b/cpu.c @@ -176,11 +176,20 @@ void cpu_exec_unrealizefn(CPUState *cpu) vmstate_unregister(NULL, &vmstate_cpu_common, cpu); } #endif + + /* Call the plugin hook before clearing cpu->cpu_index in cpu_list_remove */ if (tcg_enabled()) { - tcg_exec_unrealizefn(cpu); + qemu_plugin_vcpu_exit_hook(cpu); } cpu_list_remove(cpu); + /* + * Now that the vCPU has been removed from the RCU list, we can call + * tcg_exec_unrealizefn, which may free fields using call_rcu. + */ + if (tcg_enabled()) { + tcg_exec_unrealizefn(cpu); + } } /* From def48dddcfb2b8d9ef64ba5e6845ace006d30d27 Mon Sep 17 00:00:00 2001 From: Emilio Cota Date: Tue, 24 Jan 2023 18:01:19 +0000 Subject: [PATCH 27/36] util/qht: add missing atomic_set(hashes[i]) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We forgot to add this one in "a890643958 util/qht: atomically set b->hashes". Detected with tsan. Reviewed-by: Richard Henderson Reviewed-by: Philippe Mathieu-Daudé Reviewed-by: Alex Bennée Signed-off-by: Emilio Cota Message-Id: <20230111151628.320011-3-cota@braap.org> Signed-off-by: Alex Bennée Message-Id: <20230124180127.1881110-28-alex.bennee@linaro.org> --- util/qht.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/qht.c b/util/qht.c index 065fc501f4..15866299e6 100644 --- a/util/qht.c +++ b/util/qht.c @@ -688,7 +688,7 @@ static inline void qht_bucket_remove_entry(struct qht_bucket *orig, int pos) int i; if (qht_entry_is_last(orig, pos)) { - orig->hashes[pos] = 0; + qatomic_set(&orig->hashes[pos], 0); qatomic_set(&orig->pointers[pos], NULL); return; } From 047e2bd3383ba488f928b335f9d99cef3a216418 Mon Sep 17 00:00:00 2001 From: Emilio Cota Date: Tue, 24 Jan 2023 18:01:20 +0000 Subject: [PATCH 28/36] thread: de-const qemu_spin_destroy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewed-by: Alex Bennée Signed-off-by: Emilio Cota Reviewed-by: Richard Henderson Message-Id: <20230111151628.320011-4-cota@braap.org> Signed-off-by: Alex Bennée Message-Id: <20230124180127.1881110-29-alex.bennee@linaro.org> --- include/qemu/thread.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/include/qemu/thread.h b/include/qemu/thread.h index 7c6703bce3..7841084199 100644 --- a/include/qemu/thread.h +++ b/include/qemu/thread.h @@ -237,11 +237,10 @@ static inline void qemu_spin_init(QemuSpin *spin) #endif } -/* const parameter because the only purpose here is the TSAN annotation */ -static inline void qemu_spin_destroy(const QemuSpin *spin) +static inline void qemu_spin_destroy(QemuSpin *spin) { #ifdef CONFIG_TSAN - __tsan_mutex_destroy((void *)spin, __tsan_mutex_not_static); + __tsan_mutex_destroy(spin, __tsan_mutex_not_static); #endif } From 68f7b2be532ad5b134cd0053173641cf52cfa247 Mon Sep 17 00:00:00 2001 From: Emilio Cota Date: Tue, 24 Jan 2023 18:01:21 +0000 Subject: [PATCH 29/36] util/qht: use striped locks under TSAN MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes this tsan crash, easy to reproduce with any large enough program: $ tests/unit/test-qht 1..2 ThreadSanitizer: CHECK failed: sanitizer_deadlock_detector.h:67 "((n_all_locks_)) < (((sizeof(all_locks_with_contexts_)/sizeof((all_locks_with_contexts_)[0]))))" (0x40, 0x40) (tid=1821568) #0 __tsan::CheckUnwind() ../../../../src/libsanitizer/tsan/tsan_rtl.cpp:353 (libtsan.so.2+0x90034) #1 __sanitizer::CheckFailed(char const*, int, char const*, unsigned long long, unsigned long long) ../../../../src/libsanitizer/sanitizer_common/sanitizer_termination.cpp:86 (libtsan.so.2+0xca555) #2 __sanitizer::DeadlockDetectorTLS<__sanitizer::TwoLevelBitVector<1ul, __sanitizer::BasicBitVector > >::addLock(unsigned long, unsigned long, unsigned int) ../../../../src/libsanitizer/sanitizer_common/sanitizer_deadlock_detector.h:67 (libtsan.so.2+0xb3616) #3 __sanitizer::DeadlockDetectorTLS<__sanitizer::TwoLevelBitVector<1ul, __sanitizer::BasicBitVector > >::addLock(unsigned long, unsigned long, unsigned int) ../../../../src/libsanitizer/sanitizer_common/sanitizer_deadlock_detector.h:59 (libtsan.so.2+0xb3616) #4 __sanitizer::DeadlockDetector<__sanitizer::TwoLevelBitVector<1ul, __sanitizer::BasicBitVector > >::onLockAfter(__sanitizer::DeadlockDetectorTLS<__sanitizer::TwoLevelBitVector<1ul, __sanitizer::BasicBitVector > >*, unsigned long, unsigned int) ../../../../src/libsanitizer/sanitizer_common/sanitizer_deadlock_detector.h:216 (libtsan.so.2+0xb3616) #5 __sanitizer::DD::MutexAfterLock(__sanitizer::DDCallback*, __sanitizer::DDMutex*, bool, bool) ../../../../src/libsanitizer/sanitizer_common/sanitizer_deadlock_detector1.cpp:169 (libtsan.so.2+0xb3616) #6 __tsan::MutexPostLock(__tsan::ThreadState*, unsigned long, unsigned long, unsigned int, int) ../../../../src/libsanitizer/tsan/tsan_rtl_mutex.cpp:200 (libtsan.so.2+0xa3382) #7 __tsan_mutex_post_lock ../../../../src/libsanitizer/tsan/tsan_interface_ann.cpp:384 (libtsan.so.2+0x76bc3) #8 qemu_spin_lock /home/cota/src/qemu/include/qemu/thread.h:259 (test-qht+0x44a97) #9 qht_map_lock_buckets ../util/qht.c:253 (test-qht+0x44a97) #10 do_qht_iter ../util/qht.c:809 (test-qht+0x45f33) #11 qht_iter ../util/qht.c:821 (test-qht+0x45f33) #12 iter_check ../tests/unit/test-qht.c:121 (test-qht+0xe473) #13 qht_do_test ../tests/unit/test-qht.c:202 (test-qht+0xe473) #14 qht_test ../tests/unit/test-qht.c:240 (test-qht+0xe7c1) #15 test_default ../tests/unit/test-qht.c:246 (test-qht+0xe828) #16 (libglib-2.0.so.0+0x7daed) #17 (libglib-2.0.so.0+0x7d80a) #18 (libglib-2.0.so.0+0x7d80a) #19 g_test_run_suite (libglib-2.0.so.0+0x7dfe9) #20 g_test_run (libglib-2.0.so.0+0x7e055) #21 main ../tests/unit/test-qht.c:259 (test-qht+0xd2c6) #22 __libc_start_call_main ../sysdeps/nptl/libc_start_call_main.h:58 (libc.so.6+0x29d8f) #23 __libc_start_main_impl ../csu/libc-start.c:392 (libc.so.6+0x29e3f) #24 _start (test-qht+0xdb44) Signed-off-by: Emilio Cota Reviewed-by: Richard Henderson Message-Id: <20230111151628.320011-5-cota@braap.org> Signed-off-by: Alex Bennée Message-Id: <20230124180127.1881110-30-alex.bennee@linaro.org> --- util/qht.c | 95 ++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 81 insertions(+), 14 deletions(-) diff --git a/util/qht.c b/util/qht.c index 15866299e6..92c6b78759 100644 --- a/util/qht.c +++ b/util/qht.c @@ -151,6 +151,22 @@ struct qht_bucket { QEMU_BUILD_BUG_ON(sizeof(struct qht_bucket) > QHT_BUCKET_ALIGN); +/* + * Under TSAN, we use striped locks instead of one lock per bucket chain. + * This avoids crashing under TSAN, since TSAN aborts the program if more than + * 64 locks are held (this is a hardcoded limit in TSAN). + * When resizing a QHT we grab all the buckets' locks, which can easily + * go over TSAN's limit. By using striped locks, we avoid this problem. + * + * Note: this number must be a power of two for easy index computation. + */ +#define QHT_TSAN_BUCKET_LOCKS_BITS 4 +#define QHT_TSAN_BUCKET_LOCKS (1 << QHT_TSAN_BUCKET_LOCKS_BITS) + +struct qht_tsan_lock { + QemuSpin lock; +} QEMU_ALIGNED(QHT_BUCKET_ALIGN); + /** * struct qht_map - structure to track an array of buckets * @rcu: used by RCU. Keep it as the top field in the struct to help valgrind @@ -160,6 +176,7 @@ QEMU_BUILD_BUG_ON(sizeof(struct qht_bucket) > QHT_BUCKET_ALIGN); * @n_added_buckets: number of added (i.e. "non-head") buckets * @n_added_buckets_threshold: threshold to trigger an upward resize once the * number of added buckets surpasses it. + * @tsan_bucket_locks: Array of striped locks to be used only under TSAN. * * Buckets are tracked in what we call a "map", i.e. this structure. */ @@ -169,6 +186,9 @@ struct qht_map { size_t n_buckets; size_t n_added_buckets; size_t n_added_buckets_threshold; +#ifdef CONFIG_TSAN + struct qht_tsan_lock tsan_bucket_locks[QHT_TSAN_BUCKET_LOCKS]; +#endif }; /* trigger a resize when n_added_buckets > n_buckets / div */ @@ -229,10 +249,56 @@ static inline size_t qht_elems_to_buckets(size_t n_elems) return pow2ceil(n_elems / QHT_BUCKET_ENTRIES); } -static inline void qht_head_init(struct qht_bucket *b) +/* + * When using striped locks (i.e. under TSAN), we have to be careful not + * to operate on the same lock twice (e.g. when iterating through all buckets). + * We achieve this by operating only on each stripe's first matching lock. + */ +static inline void qht_do_if_first_in_stripe(struct qht_map *map, + struct qht_bucket *b, + void (*func)(QemuSpin *spin)) +{ +#ifdef CONFIG_TSAN + unsigned long bucket_idx = b - map->buckets; + bool is_first_in_stripe = (bucket_idx >> QHT_TSAN_BUCKET_LOCKS_BITS) == 0; + if (is_first_in_stripe) { + unsigned long lock_idx = bucket_idx & (QHT_TSAN_BUCKET_LOCKS - 1); + func(&map->tsan_bucket_locks[lock_idx].lock); + } +#else + func(&b->lock); +#endif +} + +static inline void qht_bucket_lock_do(struct qht_map *map, + struct qht_bucket *b, + void (*func)(QemuSpin *lock)) +{ +#ifdef CONFIG_TSAN + unsigned long bucket_idx = b - map->buckets; + unsigned long lock_idx = bucket_idx & (QHT_TSAN_BUCKET_LOCKS - 1); + func(&map->tsan_bucket_locks[lock_idx].lock); +#else + func(&b->lock); +#endif +} + +static inline void qht_bucket_lock(struct qht_map *map, + struct qht_bucket *b) +{ + qht_bucket_lock_do(map, b, qemu_spin_lock); +} + +static inline void qht_bucket_unlock(struct qht_map *map, + struct qht_bucket *b) +{ + qht_bucket_lock_do(map, b, qemu_spin_unlock); +} + +static inline void qht_head_init(struct qht_map *map, struct qht_bucket *b) { memset(b, 0, sizeof(*b)); - qemu_spin_init(&b->lock); + qht_do_if_first_in_stripe(map, b, qemu_spin_init); seqlock_init(&b->sequence); } @@ -250,7 +316,7 @@ static void qht_map_lock_buckets(struct qht_map *map) for (i = 0; i < map->n_buckets; i++) { struct qht_bucket *b = &map->buckets[i]; - qemu_spin_lock(&b->lock); + qht_do_if_first_in_stripe(map, b, qemu_spin_lock); } } @@ -261,7 +327,7 @@ static void qht_map_unlock_buckets(struct qht_map *map) for (i = 0; i < map->n_buckets; i++) { struct qht_bucket *b = &map->buckets[i]; - qemu_spin_unlock(&b->lock); + qht_do_if_first_in_stripe(map, b, qemu_spin_unlock); } } @@ -308,7 +374,7 @@ void qht_map_lock_buckets__no_stale(struct qht *ht, struct qht_map **pmap) * Get a head bucket and lock it, making sure its parent map is not stale. * @pmap is filled with a pointer to the bucket's parent map. * - * Unlock with qemu_spin_unlock(&b->lock). + * Unlock with qht_bucket_unlock. * * Note: callers cannot have ht->lock held. */ @@ -322,18 +388,18 @@ struct qht_bucket *qht_bucket_lock__no_stale(struct qht *ht, uint32_t hash, map = qatomic_rcu_read(&ht->map); b = qht_map_to_bucket(map, hash); - qemu_spin_lock(&b->lock); + qht_bucket_lock(map, b); if (likely(!qht_map_is_stale__locked(ht, map))) { *pmap = map; return b; } - qemu_spin_unlock(&b->lock); + qht_bucket_unlock(map, b); /* we raced with a resize; acquire ht->lock to see the updated ht->map */ qht_lock(ht); map = ht->map; b = qht_map_to_bucket(map, hash); - qemu_spin_lock(&b->lock); + qht_bucket_lock(map, b); qht_unlock(ht); *pmap = map; return b; @@ -345,12 +411,13 @@ static inline bool qht_map_needs_resize(const struct qht_map *map) map->n_added_buckets_threshold; } -static inline void qht_chain_destroy(const struct qht_bucket *head) +static inline void qht_chain_destroy(struct qht_map *map, + struct qht_bucket *head) { struct qht_bucket *curr = head->next; struct qht_bucket *prev; - qemu_spin_destroy(&head->lock); + qht_do_if_first_in_stripe(map, head, qemu_spin_destroy); while (curr) { prev = curr; curr = curr->next; @@ -364,7 +431,7 @@ static void qht_map_destroy(struct qht_map *map) size_t i; for (i = 0; i < map->n_buckets; i++) { - qht_chain_destroy(&map->buckets[i]); + qht_chain_destroy(map, &map->buckets[i]); } qemu_vfree(map->buckets); g_free(map); @@ -390,7 +457,7 @@ static struct qht_map *qht_map_create(size_t n_buckets) map->buckets = qemu_memalign(QHT_BUCKET_ALIGN, sizeof(*map->buckets) * n_buckets); for (i = 0; i < n_buckets; i++) { - qht_head_init(&map->buckets[i]); + qht_head_init(map, &map->buckets[i]); } return map; } @@ -638,7 +705,7 @@ bool qht_insert(struct qht *ht, void *p, uint32_t hash, void **existing) b = qht_bucket_lock__no_stale(ht, hash, &map); prev = qht_insert__locked(ht, map, b, p, hash, &needs_resize); qht_bucket_debug__locked(b); - qemu_spin_unlock(&b->lock); + qht_bucket_unlock(map, b); if (unlikely(needs_resize) && ht->mode & QHT_MODE_AUTO_RESIZE) { qht_grow_maybe(ht); @@ -749,7 +816,7 @@ bool qht_remove(struct qht *ht, const void *p, uint32_t hash) b = qht_bucket_lock__no_stale(ht, hash, &map); ret = qht_remove__locked(b, p, hash); qht_bucket_debug__locked(b); - qemu_spin_unlock(&b->lock); + qht_bucket_unlock(map, b); return ret; } From 2bbbc1be8d9a21b25d0c80b9a7345074d54abd51 Mon Sep 17 00:00:00 2001 From: Emilio Cota Date: Tue, 24 Jan 2023 18:01:22 +0000 Subject: [PATCH 30/36] plugins: make qemu_plugin_user_exit's locking order consistent with fork_start's MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To fix potential deadlocks as reported by tsan. Reviewed-by: Richard Henderson Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Emilio Cota Message-Id: <20230111151628.320011-6-cota@braap.org> Signed-off-by: Alex Bennée Message-Id: <20230124180127.1881110-31-alex.bennee@linaro.org> --- plugins/core.c | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/plugins/core.c b/plugins/core.c index ccb770a485..728bacef95 100644 --- a/plugins/core.c +++ b/plugins/core.c @@ -500,10 +500,17 @@ void qemu_plugin_user_exit(void) enum qemu_plugin_event ev; CPUState *cpu; - QEMU_LOCK_GUARD(&plugin.lock); - + /* + * Locking order: we must acquire locks in an order that is consistent + * with the one in fork_start(). That is: + * - start_exclusive(), which acquires qemu_cpu_list_lock, + * must be called before acquiring plugin.lock. + * - tb_flush(), which acquires mmap_lock(), must be called + * while plugin.lock is not held. + */ start_exclusive(); + qemu_rec_mutex_lock(&plugin.lock); /* un-register all callbacks except the final AT_EXIT one */ for (ev = 0; ev < QEMU_PLUGIN_EV_MAX; ev++) { if (ev != QEMU_PLUGIN_EV_ATEXIT) { @@ -513,13 +520,12 @@ void qemu_plugin_user_exit(void) } } } - - tb_flush(current_cpu); - CPU_FOREACH(cpu) { qemu_plugin_disable_mem_helpers(cpu); } + qemu_rec_mutex_unlock(&plugin.lock); + tb_flush(current_cpu); end_exclusive(); /* now it's safe to handle the exit case */ From 3fd62e73ad8193b58a1bc15ad6f6d8b4f284f6a7 Mon Sep 17 00:00:00 2001 From: Emilio Cota Date: Tue, 24 Jan 2023 18:01:23 +0000 Subject: [PATCH 31/36] plugins: fix optimization in plugin_gen_disable_mem_helpers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We were mistakenly checking tcg_ctx->plugin_insn as a canary to know whether the TB had emitted helpers that might have accessed memory. The problem is that tcg_ctx->plugin_insn gets updated on every instruction in the TB, which results in us wrongly performing the optimization (i.e. not clearing cpu->plugin_mem_cbs) way too often, since it's not rare that the last instruction in the TB doesn't use helpers. Fix it by tracking a per-TB canary. While at it, expand documentation. Related: #1381 Signed-off-by: Emilio Cota Message-Id: <20230108164731.61469-2-cota@braap.org> Signed-off-by: Alex Bennée Message-Id: <20230124180127.1881110-32-alex.bennee@linaro.org> --- accel/tcg/plugin-gen.c | 26 ++++++++++++++++++-------- include/qemu/plugin.h | 7 +++++++ 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/accel/tcg/plugin-gen.c b/accel/tcg/plugin-gen.c index c7d6514840..17a686bd9e 100644 --- a/accel/tcg/plugin-gen.c +++ b/accel/tcg/plugin-gen.c @@ -579,7 +579,8 @@ static void inject_mem_helper(TCGOp *begin_op, GArray *arr) * is possible that the code we generate after the instruction is * dead, we also add checks before generating tb_exit etc. */ -static void inject_mem_enable_helper(struct qemu_plugin_insn *plugin_insn, +static void inject_mem_enable_helper(struct qemu_plugin_tb *ptb, + struct qemu_plugin_insn *plugin_insn, TCGOp *begin_op) { GArray *cbs[2]; @@ -599,6 +600,7 @@ static void inject_mem_enable_helper(struct qemu_plugin_insn *plugin_insn, rm_ops(begin_op); return; } + ptb->mem_helper = true; arr = g_array_sized_new(false, false, sizeof(struct qemu_plugin_dyn_cb), n_cbs); @@ -626,15 +628,22 @@ void plugin_gen_disable_mem_helpers(void) { TCGv_ptr ptr; - if (likely(tcg_ctx->plugin_insn == NULL || - !tcg_ctx->plugin_insn->mem_helper)) { + /* + * We could emit the clearing unconditionally and be done. However, this can + * be wasteful if for instance plugins don't track memory accesses, or if + * most TBs don't use helpers. Instead, emit the clearing iff the TB calls + * helpers that might access guest memory. + * + * Note: we do not reset plugin_tb->mem_helper here; a TB might have several + * exit points, and we want to emit the clearing from all of them. + */ + if (!tcg_ctx->plugin_tb->mem_helper) { return; } ptr = tcg_const_ptr(NULL); tcg_gen_st_ptr(ptr, cpu_env, offsetof(CPUState, plugin_mem_cbs) - offsetof(ArchCPU, env)); tcg_temp_free_ptr(ptr); - tcg_ctx->plugin_insn->mem_helper = false; } static void plugin_gen_tb_udata(const struct qemu_plugin_tb *ptb, @@ -682,14 +691,14 @@ static void plugin_gen_mem_inline(const struct qemu_plugin_tb *ptb, inject_inline_cb(cbs, begin_op, op_rw); } -static void plugin_gen_enable_mem_helper(const struct qemu_plugin_tb *ptb, +static void plugin_gen_enable_mem_helper(struct qemu_plugin_tb *ptb, TCGOp *begin_op, int insn_idx) { struct qemu_plugin_insn *insn = g_ptr_array_index(ptb->insns, insn_idx); - inject_mem_enable_helper(insn, begin_op); + inject_mem_enable_helper(ptb, insn, begin_op); } -static void plugin_gen_disable_mem_helper(const struct qemu_plugin_tb *ptb, +static void plugin_gen_disable_mem_helper(struct qemu_plugin_tb *ptb, TCGOp *begin_op, int insn_idx) { struct qemu_plugin_insn *insn = g_ptr_array_index(ptb->insns, insn_idx); @@ -750,7 +759,7 @@ static void pr_ops(void) #endif } -static void plugin_gen_inject(const struct qemu_plugin_tb *plugin_tb) +static void plugin_gen_inject(struct qemu_plugin_tb *plugin_tb) { TCGOp *op; int insn_idx = -1; @@ -870,6 +879,7 @@ bool plugin_gen_tb_start(CPUState *cpu, const DisasContextBase *db, ptb->haddr1 = db->host_addr[0]; ptb->haddr2 = NULL; ptb->mem_only = mem_only; + ptb->mem_helper = false; plugin_gen_empty_callback(PLUGIN_GEN_FROM_TB); } diff --git a/include/qemu/plugin.h b/include/qemu/plugin.h index a772e14193..e0ebedef84 100644 --- a/include/qemu/plugin.h +++ b/include/qemu/plugin.h @@ -118,7 +118,10 @@ struct qemu_plugin_insn { void *haddr; GArray *cbs[PLUGIN_N_CB_TYPES][PLUGIN_N_CB_SUBTYPES]; bool calls_helpers; + + /* if set, the instruction calls helpers that might access guest memory */ bool mem_helper; + bool mem_only; }; @@ -158,6 +161,10 @@ struct qemu_plugin_tb { void *haddr1; void *haddr2; bool mem_only; + + /* if set, the TB calls helpers that might access guest memory */ + bool mem_helper; + GArray *cbs[PLUGIN_N_CB_SUBTYPES]; }; From 0f92d94ae37952ed3e6771bc429de4b739eb6ca3 Mon Sep 17 00:00:00 2001 From: Emilio Cota Date: Tue, 24 Jan 2023 18:01:24 +0000 Subject: [PATCH 32/36] translator: always pair plugin_gen_insn_{start, end} calls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Related: #1381 Signed-off-by: Emilio Cota Reviewed-by: Philippe Mathieu-Daudé Reviewed-by: Richard Henderson Message-Id: <20230108164731.61469-3-cota@braap.org> Signed-off-by: Alex Bennée Message-Id: <20230124180127.1881110-33-alex.bennee@linaro.org> --- accel/tcg/translator.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/accel/tcg/translator.c b/accel/tcg/translator.c index 061519691f..ef5193c67e 100644 --- a/accel/tcg/translator.c +++ b/accel/tcg/translator.c @@ -100,19 +100,24 @@ void translator_loop(CPUState *cpu, TranslationBlock *tb, int max_insns, ops->translate_insn(db, cpu); } - /* Stop translation if translate_insn so indicated. */ - if (db->is_jmp != DISAS_NEXT) { - break; - } - /* * We can't instrument after instructions that change control * flow although this only really affects post-load operations. + * + * Calling plugin_gen_insn_end() before we possibly stop translation + * is important. Even if this ends up as dead code, plugin generation + * needs to see a matching plugin_gen_insn_{start,end}() pair in order + * to accurately track instrumented helpers that might access memory. */ if (plugin_enabled) { plugin_gen_insn_end(); } + /* Stop translation if translate_insn so indicated. */ + if (db->is_jmp != DISAS_NEXT) { + break; + } + /* Stop translation if the output buffer is full, or we have executed all of the allowed instructions. */ if (tcg_op_buf_full() || db->num_insns >= db->max_insns) { From 17083f6fa6e02326f1c1e8b67b8b282480d013d2 Mon Sep 17 00:00:00 2001 From: Emilio Cota Date: Tue, 24 Jan 2023 18:01:25 +0000 Subject: [PATCH 33/36] tcg: exclude non-memory effecting helpers from instrumentation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There are actually a whole bunch of helpers that don't affect memory that we shouldn't instrument. They are helpfully identified by the TCG_CALL_NO_SIDE_EFFECTS flag which marks out lookup_tb_ptr as well as a lot of the maths helpers. To avoid the string compare we introduce a new flag for plugin internals so we skip that too. Related: #1381 Signed-off-by: Emilio Cota Message-Id: <20230108164731.61469-4-cota@braap.org> [AJB: updated to skip all no SE plugins, add flag for plugin helper] Signed-off-by: Alex Bennée Reviewed-by: Richard Henderson Message-Id: <20230124180127.1881110-34-alex.bennee@linaro.org> --- accel/tcg/plugin-helpers.h | 4 ++-- include/tcg/tcg.h | 2 ++ tcg/tcg.c | 6 ++++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/accel/tcg/plugin-helpers.h b/accel/tcg/plugin-helpers.h index 9829abe4a9..8e685e0654 100644 --- a/accel/tcg/plugin-helpers.h +++ b/accel/tcg/plugin-helpers.h @@ -1,4 +1,4 @@ #ifdef CONFIG_PLUGIN -DEF_HELPER_FLAGS_2(plugin_vcpu_udata_cb, TCG_CALL_NO_RWG, void, i32, ptr) -DEF_HELPER_FLAGS_4(plugin_vcpu_mem_cb, TCG_CALL_NO_RWG, void, i32, i32, i64, ptr) +DEF_HELPER_FLAGS_2(plugin_vcpu_udata_cb, TCG_CALL_NO_RWG | TCG_CALL_PLUGIN, void, i32, ptr) +DEF_HELPER_FLAGS_4(plugin_vcpu_mem_cb, TCG_CALL_NO_RWG | TCG_CALL_PLUGIN, void, i32, i32, i64, ptr) #endif diff --git a/include/tcg/tcg.h b/include/tcg/tcg.h index 6f497172f8..8dc291d030 100644 --- a/include/tcg/tcg.h +++ b/include/tcg/tcg.h @@ -405,6 +405,8 @@ typedef TCGv_ptr TCGv_env; #define TCG_CALL_NO_SIDE_EFFECTS 0x0004 /* Helper is G_NORETURN. */ #define TCG_CALL_NO_RETURN 0x0008 +/* Helper is part of Plugins. */ +#define TCG_CALL_PLUGIN 0x0010 /* convenience version of most used call flags */ #define TCG_CALL_NO_RWG TCG_CALL_NO_READ_GLOBALS diff --git a/tcg/tcg.c b/tcg/tcg.c index d502327be2..fd557d55d3 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -1674,8 +1674,10 @@ void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args) op = tcg_op_alloc(INDEX_op_call, total_args); #ifdef CONFIG_PLUGIN - /* detect non-plugin helpers */ - if (tcg_ctx->plugin_insn && unlikely(strncmp(info->name, "plugin_", 7))) { + /* Flag helpers that may affect guest state */ + if (tcg_ctx->plugin_insn && + !(info->flags & TCG_CALL_PLUGIN) && + !(info->flags & TCG_CALL_NO_SIDE_EFFECTS)) { tcg_ctx->plugin_insn->calls_helpers = true; } #endif From 882f5b1b4418cfa8f65ff2be52286247db40d152 Mon Sep 17 00:00:00 2001 From: Emilio Cota Date: Tue, 24 Jan 2023 18:01:26 +0000 Subject: [PATCH 34/36] cpu-exec: assert that plugin_mem_cbs is NULL after execution MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes: #1381 Signed-off-by: Emilio Cota Message-Id: <20230108165107.62488-1-cota@braap.org> [AJB: manually applied follow-up fix] Signed-off-by: Alex Bennée Reviewed-by: Philippe Mathieu-Daudé Reviewed-by: Richard Henderson Message-Id: <20230124180127.1881110-35-alex.bennee@linaro.org> --- accel/tcg/cpu-exec.c | 2 ++ include/qemu/plugin.h | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/accel/tcg/cpu-exec.c b/accel/tcg/cpu-exec.c index 25ec73ef9a..9c857eeb07 100644 --- a/accel/tcg/cpu-exec.c +++ b/accel/tcg/cpu-exec.c @@ -504,6 +504,7 @@ static void cpu_exec_exit(CPUState *cpu) if (cc->tcg_ops->cpu_exec_exit) { cc->tcg_ops->cpu_exec_exit(cpu); } + QEMU_PLUGIN_ASSERT(cpu->plugin_mem_cbs == NULL); } void cpu_exec_step_atomic(CPUState *cpu) @@ -980,6 +981,7 @@ cpu_exec_loop(CPUState *cpu, SyncClocks *sc) cpu_loop_exec_tb(cpu, tb, pc, &last_tb, &tb_exit); + QEMU_PLUGIN_ASSERT(cpu->plugin_mem_cbs == NULL); /* Try to align the host and virtual clocks if the guest is in advance */ align_clocks(sc, cpu); diff --git a/include/qemu/plugin.h b/include/qemu/plugin.h index e0ebedef84..fb338ba576 100644 --- a/include/qemu/plugin.h +++ b/include/qemu/plugin.h @@ -59,6 +59,8 @@ get_plugin_meminfo_rw(qemu_plugin_meminfo_t i) #ifdef CONFIG_PLUGIN extern QemuOptsList qemu_plugin_opts; +#define QEMU_PLUGIN_ASSERT(cond) g_assert(cond) + static inline void qemu_plugin_add_opts(void) { qemu_add_opts(&qemu_plugin_opts); @@ -250,6 +252,8 @@ void qemu_plugin_user_postfork(bool is_child); #else /* !CONFIG_PLUGIN */ +#define QEMU_PLUGIN_ASSERT(cond) + static inline void qemu_plugin_add_opts(void) { } From f45549233f67e5fd3038d2c886ef7de876c8ff01 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Tue, 24 Jan 2023 18:01:27 +0000 Subject: [PATCH 35/36] plugins: Iterate on cb_lists in qemu_plugin_user_exit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rather than iterate over all plugins for all events, iterate over plugins that have registered a given event. Signed-off-by: Richard Henderson Message-Id: <20230117035701.168514-4-richard.henderson@linaro.org> Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Alex Bennée Message-Id: <20230124180127.1881110-36-alex.bennee@linaro.org> --- plugins/core.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/plugins/core.c b/plugins/core.c index 728bacef95..e04ffa1ba4 100644 --- a/plugins/core.c +++ b/plugins/core.c @@ -514,9 +514,10 @@ void qemu_plugin_user_exit(void) /* un-register all callbacks except the final AT_EXIT one */ for (ev = 0; ev < QEMU_PLUGIN_EV_MAX; ev++) { if (ev != QEMU_PLUGIN_EV_ATEXIT) { - struct qemu_plugin_ctx *ctx; - QTAILQ_FOREACH(ctx, &plugin.ctxs, entry) { - plugin_unregister_cb__locked(ctx, ev); + struct qemu_plugin_cb *cb, *next; + + QLIST_FOREACH_SAFE_RCU(cb, &plugin.cb_lists[ev], entry, next) { + plugin_unregister_cb__locked(cb->ctx, ev); } } } From b3ca9646b9a5c44dfd110c5db9b4a8b8497de34e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Thu, 2 Feb 2023 13:25:27 +0000 Subject: [PATCH 36/36] gitlab: cut even more from cross-win64-system build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This keeps hitting the time limits so cut even more from the list. Signed-off-by: Alex Bennée --- .gitlab-ci.d/crossbuilds.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.d/crossbuilds.yml b/.gitlab-ci.d/crossbuilds.yml index 8dbbb8f881..74d6259b90 100644 --- a/.gitlab-ci.d/crossbuilds.yml +++ b/.gitlab-ci.d/crossbuilds.yml @@ -187,7 +187,9 @@ cross-win64-system: job: win64-fedora-cross-container variables: IMAGE: fedora-win64-cross - CROSS_SKIP_TARGETS: or1k-softmmu rx-softmmu sh4eb-softmmu sparc64-softmmu + CROSS_SKIP_TARGETS: alpha-softmmu avr-softmmu hppa-softmmu + m68k-softmmu microblazeel-softmmu nios2-softmmu + or1k-softmmu rx-softmmu sh4eb-softmmu sparc64-softmmu tricore-softmmu xtensaeb-softmmu artifacts: paths: