Add support to send metrics

profiling/CMakeLists.txt

@@ -1,6 +1,6 @@
 set(PROFILING_FLAGS
     -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc
-    -fno-builtin-free -fno-omit-frame-pointer -g -O3)
+    -fno-builtin-free -fno-omit-frame-pointer -g -gdwarf-5 -O3)
 
 set(SRC_TARGETS
   elf_profiler.cpp

scripts/docker/ci/linux/x86_64/py-testing.toml

@@ -8,7 +8,7 @@ c-compiler       = "clang"
 extra-targets    = ["all"]
 extra-cmake-opt  = [
   "-DLIEF_TESTS=on",
-  "-DPython_VERSION=3.11",
+  "-DPython_VERSION={python_version}",
   "-DCMAKE_COLOR_DIAGNOSTICS=ON",
   "-DCMAKE_CXX_COMPILER=clang++",
   "-DCMAKE_C_COMPILER=clang",
@@ -35,5 +35,5 @@ debug   = false
 
 [lief.cross-compilation]
 platform  = "manylinux_2_31_x86_64"
-pyversion = "cp311"
-abi       = "cp311"
+pyversion = "cp{python_version_alt}"
+abi       = "cp{python_version_alt}"

scripts/docker/ci/osx/python.toml

@@ -8,12 +8,12 @@ cxx-compiler  = "clang++"
 c-compiler    = "clang"
 extra-targets = ["all"]
 extra-cmake-opt = [
-  "-DPython_VERSION=3.12",
+  "-DPython_VERSION={python_version}",
   "-DCMAKE_COLOR_DIAGNOSTICS=ON",
   "-DCMAKE_CXX_COMPILER=clang++",
   "-DCMAKE_C_COMPILER=clang",
   "-DCMAKE_TOOLCHAIN_FILE=/cmake/osx/toolchain.cmake",
-  "-DCMAKE_SYSTEM_PROCESSOR=arm64",
+  "-DCMAKE_SYSTEM_PROCESSOR={architecture}",
   "-DCMAKE_MODULE_PATH=/cmake/osx/",
 ]
 
@@ -36,6 +36,6 @@ enabled = true
 debug   = false
 
 [lief.cross-compilation]
-platform  = "macosx_11_0_arm64"
-pyversion = "cp312"
-abi       = "cp312"
+platform  = "macosx_11_0_{architecture}"
+pyversion = "cp{python_version_alt}"
+abi       = "cp{python_version_alt}"

scripts/docker/ci/osx/x86_64/py-3.12.toml

@@ -1,41 +0,0 @@
-[lief.build]
-type          = "Release"
-cache         = true
-ninja         = true
-strip         = true
-parallel-jobs = 0
-cxx-compiler  = "clang++"
-c-compiler    = "clang"
-extra-targets = ["all"]
-extra-cmake-opt = [
-  "-DPython_VERSION=3.12",
-  "-DCMAKE_COLOR_DIAGNOSTICS=ON",
-  "-DCMAKE_CXX_COMPILER=clang++",
-  "-DCMAKE_C_COMPILER=clang",
-  "-DCMAKE_TOOLCHAIN_FILE=/cmake/osx/toolchain.cmake",
-  "-DCMAKE_SYSTEM_PROCESSOR=x86_64",
-  "-DCMAKE_MODULE_PATH=/cmake/osx/",
-]
-
-[lief.formats]
-elf     = true
-pe      = true
-macho   = true
-android = true
-art     = true
-vdex    = true
-oat     = true
-dex     = true
-
-[lief.features]
-json    = true
-frozen  = true
-
-[lief.logging]
-enabled = true
-debug   = false
-
-[lief.cross-compilation]
-platform  = "macosx_11_0_x86_64"
-pyversion = "cp312"
-abi       = "cp312"

scripts/docker/ci/windows/x86_64/python.toml

@@ -8,7 +8,7 @@ cxx-compiler  = "clang++"
 c-compiler    = "clang"
 extra-targets = ["all"]
 extra-cmake-opt = [
-  "-DPython_VERSION=3.11",
+  "-DPython_VERSION={python_version}",
   "-DCMAKE_C_COMPILER_LAUNCHER=ccache",
   "-DCMAKE_CXX_COMPILER_LAUNCHER=ccache",
   "-DCMAKE_COLOR_DIAGNOSTICS=ON",
@@ -39,5 +39,5 @@ debug   = false
 
 [lief.cross-compilation]
 platform  = "win_amd64"
-pyversion = "cp311"
-abi       = "cp311"
+pyversion = "cp{python_version_alt}"
+abi       = "cp{python_version_alt}"

scripts/metrics/collectors/python-bindings-size

@@ -0,0 +1,43 @@
+#!/python
+import sys
+from pathlib import Path
+sys.path.insert(0, (Path(__file__).parent / "..").as_posix())
+
+import lief
+import argparse
+from models import Metrics
+from models.python import Metrics as PythonMetric
+from models.python.native_library import Metrics as NativeLibraryMetric
+
+def collect_native_library_info() -> str:
+    lief_path = Path(lief._lief.__file__)
+
+    metrics = Metrics(
+        python=PythonMetric(
+            native_library=NativeLibraryMetric(
+                size=lief_path.stat().st_size
+            )
+        )
+    )
+
+    return metrics.model_dump_json(
+        exclude_none=True, exclude_unset=True
+    )
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--output",
+        metavar="OUTPUT",
+        type=Path,
+        dest='output',
+        required=True
+    )
+
+    args = parser.parse_args()
+    output: Path = args.output
+    output.write_text(collect_native_library_info())
+    return 0
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/metrics/convert/cpp_unittest_junit.py

@@ -0,0 +1,71 @@
+#!python
+import sys
+import argparse
+from pathlib import Path
+sys.path.insert(0, (Path(__file__).parent / "..").as_posix())
+from models import Metrics
+from models.cpp import Metrics as CppMetrics
+from models.cpp.unittest import Metrics as CppUnittestMetrics
+from lxml import etree
+
+def convert(tree) -> str:
+    if tree.tag != 'testsuites' or len(tree) == 0:
+        raise RuntimeError(f"Expecting a <testsuite> tag at the beginning ({tree.tag})")
+
+    testsuite = tree[0]
+    nb_errors = testsuite.get("errors", 0)
+    nb_failures = testsuite.get("failures", 0)
+    nb_skipped = testsuite.get("skipped", 0)
+    nb_tests = testsuite.get("tests", 0)
+    time = float(testsuite.get("time"))
+    timestamp = testsuite.get("timestamp")
+
+    metrics = Metrics(
+        cpp=CppMetrics(
+            unittest=CppUnittestMetrics(
+                duration=time,
+                nb_tests=nb_tests,
+                nb_errors=nb_errors,
+                nb_failures=nb_failures,
+                nb_skipped=nb_skipped,
+            )
+        )
+    )
+    return metrics.model_dump_json()
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--input",
+        metavar="FILE",
+        type=Path,
+        dest='junit_xml_report',
+        required=True
+    )
+    parser.add_argument(
+        "--output",
+        metavar="FILE",
+        type=Path,
+        dest='json_output',
+        required=True
+    )
+
+    args = parser.parse_args()
+
+    input_file: Path = args.junit_xml_report
+    output_file: Path = args.json_output
+    if not input_file.is_file():
+        print(f"{input_file} does not exist")
+        return 1
+
+    content = input_file.read_bytes()
+    if len(content) == 0:
+        print(f"{input_file} is empty")
+        return 0
+
+    root = etree.fromstring(input_file.read_bytes())
+    output_file.write_text(convert(root))
+    return 0
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/metrics/convert/heap-track-report

@@ -0,0 +1,87 @@
+#!python
+import sys
+import re
+import argparse
+from pathlib import Path
+sys.path.insert(0, (Path(__file__).parent / "..").as_posix())
+from models import Metrics
+from models.cpp import Metrics as CppMetrics
+from models.cpp.memory import Metrics as MemoryMetrics
+
+PEAK_MEM_RE = re.compile(r"peak heap memory consumption:\s*(.*)")
+PEAK_RSS_RE = re.compile(r"peak RSS.*:\s*(.*)")
+LEAK_MEM_RE = re.compile(r"total memory leaked:\s*(.*)")
+MIN_HEAP_LINES = 10
+
+def convert_str_bytes(size: str) -> int:
+    unit = size[-1]
+    match unit:
+        case 'B':
+            return int(size[:-1])
+        case 'K':
+            return int(float(size[:-1]) * 1000)
+        case 'M':
+            return int(float(size[:-1]) * 1000 * 1000)
+        case 'G':
+            return int(float(size[:-1]) * 1000 * 1000 * 1000)
+        case 'T':
+            return int(float(size[:-1]) * 1000 * 1000 * 1000 * 1000)
+    print(f"Unknown unit: {unit} ({size})")
+    return 0
+
+
+def convert(content: str) -> str:
+    lines = content.splitlines()
+    memory_metrics = MemoryMetrics()
+    if len(lines) >= MIN_HEAP_LINES:
+        result_lines = lines[-MIN_HEAP_LINES:]
+        for line in result_lines:
+            if match := re.match(PEAK_MEM_RE, line):
+                memory_metrics.peak_heap = convert_str_bytes(match.group(1))
+            if match := re.match(PEAK_RSS_RE, line):
+                memory_metrics.peak_rss = convert_str_bytes(match.group(1))
+            if match := re.match(LEAK_MEM_RE, line):
+                memory_metrics.leak = convert_str_bytes(match.group(1))
+    print(memory_metrics.model_dump())
+    metrics = Metrics(
+        cpp=CppMetrics(
+            memory=memory_metrics
+        )
+    )
+    return metrics.model_dump_json()
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--input",
+        metavar="FILE",
+        type=Path,
+        dest='heaptrack',
+        required=True
+    )
+    parser.add_argument(
+        "--output",
+        metavar="FILE",
+        type=Path,
+        dest='json_output',
+        required=True
+    )
+
+    args = parser.parse_args()
+
+    input_file: Path = args.heaptrack
+    output_file: Path = args.json_output
+    if not input_file.is_file():
+        print(f"{input_file} does not exist")
+        return 1
+
+    content = input_file.read_text()
+    if len(content) == 0:
+        print(f"{input_file} is empty")
+        return 0
+
+    output_file.write_text(convert(content))
+    return 0
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/metrics/convert/python_unittest_junit.py

@@ -0,0 +1,71 @@
+#!python
+import sys
+import argparse
+from pathlib import Path
+sys.path.insert(0, (Path(__file__).parent / "..").as_posix())
+from models import Metrics
+from models.python import Metrics as PythonMetrics
+from models.python.unittest import Metrics as PythonUnittestMetrics
+from lxml import etree
+
+def convert(tree) -> str:
+    if tree.tag != 'testsuites' or len(tree) == 0:
+        raise RuntimeError(f"Expecting a <testsuite> tag at the beginning ({tree.tag})")
+
+    testsuite = tree[0]
+    nb_errors = testsuite.get("errors", 0)
+    nb_failures = testsuite.get("failures", 0)
+    nb_skipped = testsuite.get("skipped", 0)
+    nb_tests = testsuite.get("tests", 0)
+    time = float(testsuite.get("time"))
+    timestamp = testsuite.get("timestamp")
+
+    metrics = Metrics(
+        python=PythonMetrics(
+            unittest=PythonUnittestMetrics(
+                duration=time,
+                nb_tests=nb_tests,
+                nb_errors=nb_errors,
+                nb_failures=nb_failures,
+                nb_skipped=nb_skipped,
+            )
+        )
+    )
+    return metrics.model_dump_json()
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--input",
+        metavar="FILE",
+        type=Path,
+        dest='junit_xml_report',
+        required=True
+    )
+    parser.add_argument(
+        "--output",
+        metavar="FILE",
+        type=Path,
+        dest='json_output',
+        required=True
+    )
+
+    args = parser.parse_args()
+
+    input_file: Path = args.junit_xml_report
+    output_file: Path = args.json_output
+    if not input_file.is_file():
+        print(f"{input_file} does not exist")
+        return 1
+
+    content = input_file.read_bytes()
+    if len(content) == 0:
+        print(f"{input_file} is empty")
+        return 0
+
+    root = etree.fromstring(input_file.read_bytes())
+    output_file.write_text(convert(root))
+    return 0
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/metrics/models/__init__.py

@@ -0,0 +1,8 @@
+from pydantic import BaseModel
+from typing import Optional
+from models.python import Metrics as PythonMetrics
+from models.cpp import Metrics as CppMetrics
+
+class Metrics(BaseModel):
+    python: Optional[PythonMetrics] = None
+    cpp: Optional[CppMetrics] = None

scripts/metrics/models/cpp/__init__.py

@@ -0,0 +1,8 @@
+from pydantic import BaseModel
+from typing import Optional
+from models.cpp.unittest import Metrics as UnittestMetrics
+from models.cpp.memory import Metrics as MemoryMetrics
+
+class Metrics(BaseModel):
+    unittest: Optional[UnittestMetrics] = None
+    memory: Optional[MemoryMetrics] = None

scripts/metrics/models/cpp/memory.py

@@ -0,0 +1,7 @@
+from pydantic import BaseModel
+from typing import Optional
+
+class Metrics(BaseModel):
+    peak_heap: Optional[int] = None
+    peak_rss: Optional[int] = None
+    leak: Optional[int] = None

scripts/metrics/models/cpp/unittest.py

@@ -0,0 +1,9 @@
+from pydantic import BaseModel
+from typing import Optional
+
+class Metrics(BaseModel):
+    duration: Optional[float] = None
+    nb_errors: int = 0
+    nb_skipped: int = 0
+    nb_failures: int = 0
+    nb_tests: int = 0

scripts/metrics/models/python/__init__.py

@@ -0,0 +1,8 @@
+from pydantic import BaseModel
+from typing import Optional
+from models.python.unittest import Metrics as UnittestMetrics
+from models.python.native_library import Metrics as NativeLibraryMetric
+
+class Metrics(BaseModel):
+    unittest: Optional[UnittestMetrics] = None
+    native_library: Optional[NativeLibraryMetric] = None

scripts/metrics/models/python/native_library.py

@@ -0,0 +1,6 @@
+from pydantic import BaseModel
+from typing import Optional
+
+class Metrics(BaseModel):
+    size: int = 0
+    text_section: Optional[int] = None

scripts/metrics/models/python/unittest.py

@@ -0,0 +1,9 @@
+from pydantic import BaseModel
+from typing import Optional
+
+class Metrics(BaseModel):
+    duration: Optional[float] = None
+    nb_errors: int = 0
+    nb_skipped: int = 0
+    nb_failures: int = 0
+    nb_tests: int = 0

scripts/metrics/requirements.txt

@@ -0,0 +1,4 @@
+influxdb-client==1.48.0
+pydantic==2.8.2
+pydantic-settings==2.6.1
+lxml==5.3.0

scripts/metrics/upload.py

@@ -0,0 +1,192 @@
+#!python
+import sys
+import os
+import argparse
+sys.path.insert(0, os.path.dirname(__file__))
+
+from models import Metrics
+from models.python.unittest import Metrics as PythonUnittestMetrics
+from models.cpp.unittest import Metrics as CppUnittestMetrics
+from typing import Optional, Dict, Any, Tuple
+from functools import lru_cache
+from pathlib import Path
+from pydantic import BaseModel
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+from influxdb_client.client.write_api import SYNCHRONOUS
+from influxdb_client import InfluxDBClient, Point
+
+class InfluxSettings(BaseSettings):
+    model_config = SettingsConfigDict(env_prefix='influx_')
+    url: str
+    token: str
+    bucket: str = "LIEF"
+    org: str = "server"
+
+class Config(BaseSettings):
+    influx: InfluxSettings = InfluxSettings()
+
+class Serialize:
+    def __init__(self, metrics: Metrics,
+                 platform: str, arch: str,
+                 commit: Optional[str] = None,
+                 machine: Optional[str] = None,
+                 tags: list[Tuple[str, str]] = []):
+        self.metrics = metrics
+        self.platform = platform
+        self.arch = arch
+        self.commit = commit
+        self.machine = machine
+        self.tags = tags
+
+    def _attach_tags(self, record: Point) -> 'Serialize':
+        record.tag("arch", self.arch)
+        record.tag("platform", self.platform)
+        if self.machine is not None:
+            record.tag("machine", self.machine)
+
+        for k, v in self.tags:
+            record.tag(k, v)
+        return self
+
+    def _attach_generic_fields(self, record: Point) -> 'Serialize':
+        if self.commit is not None:
+            record.field("commit", self.commit)
+        return self
+
+    def _serialize_metrics(self, name: str, metrics: BaseModel) -> Optional[Point]:
+        fields: Dict[str, Any] = metrics.model_dump(
+            exclude_none=True, exclude_unset=True
+        )
+
+        if len(fields) == 0:
+            return None
+
+        record = Point.measurement(name)
+        self._attach_tags(record) \
+            ._attach_generic_fields(record)
+
+        for k, v in fields.items():
+            record.field(k, v)
+        return record
+
+    def serialize(self) -> list[Point]:
+        points: list[Point] = []
+        if python := self.metrics.python:
+            if unittest := python.unittest:
+                if rec := self._serialize_metrics("python_unittest", unittest):
+                    points.append(rec)
+            if native_library_info := python.native_library:
+                if rec := self._serialize_metrics("python_native_library", native_library_info):
+                    points.append(rec)
+
+        if cpp := self.metrics.cpp:
+            if unittest := cpp.unittest:
+                if rec := self._serialize_metrics("cpp_unittest", unittest):
+                    points.append(rec)
+
+            if memory := cpp.memory:
+                if rec := self._serialize_metrics("memory_consumption", memory):
+                    points.append(rec)
+        return points
+
+@lru_cache(maxsize=1)
+def config() -> Config:
+    return Config()
+
+@lru_cache()
+def client() -> InfluxDBClient:
+    conf = config()
+    return InfluxDBClient(url=conf.influx.url, token=conf.influx.token, org=conf.influx.org)
+
+def push_metrics(metrics: Metrics, platform: str, arch: str,
+                 commit: Optional[str] = None, machine: Optional[str] = None,
+                 tags: list[Tuple[str, str]] = [],
+                 ) -> int:
+    conf = config()
+    api = client().write_api(write_options=SYNCHRONOUS)
+    serializer = Serialize(metrics, platform, arch, commit, machine, tags)
+    count = 0
+    for record in serializer.serialize():
+        count += 1
+        api.write(bucket=conf.influx.bucket, record=record)
+    return count
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--file",
+        metavar="FILE",
+        type=Path,
+        dest='metrics_file',
+        required=True,
+        nargs='+'
+    )
+
+    parser.add_argument(
+        "--platform",
+        metavar="PLATFORM",
+        type=str,
+        dest='metrics_platform',
+        required=True
+    )
+
+    parser.add_argument(
+        "--architecture",
+        metavar="ARCHITECTURE",
+        type=str,
+        dest='metrics_architecture',
+        required=True
+    )
+
+    parser.add_argument(
+        "--commit",
+        metavar="COMMIT",
+        type=str,
+        dest='metrics_commit',
+        default=None
+    )
+
+    parser.add_argument(
+        "--machine",
+        metavar="MACHINE",
+        type=str,
+        dest='metrics_machine',
+        default=None
+    )
+
+    parser.add_argument(
+        "--tags",
+        metavar="TAGS",
+        type=str,
+        dest='metrics_tags',
+        required=False,
+        nargs='*',
+        default=[]
+    )
+
+    args = parser.parse_args()
+
+    files: list[Path] = args.metrics_file
+    platform: str = args.metrics_platform
+    arch: str = args.metrics_architecture
+    commit: Optional[str] = args.metrics_commit
+    machine: Optional[str] = args.metrics_machine
+    arg_tags: list[str] = args.metrics_tags
+
+    tags: list[Tuple[str, str]] = []
+    for tag in arg_tags:
+        k, v = tag.split('=')
+        tags.append((k, v))
+
+    count = 0
+    for file in files:
+        metrics = Metrics.model_validate_json(file.read_bytes())
+        count += push_metrics(metrics, platform, arch, commit, machine, tags)
+
+    files_str = ", ".join([f.as_posix() for f in files])
+    print(f"#{count} metrics pushed for {platform}-{arch} from {files_str} ({commit})")
+    return 0
+
+if __name__ == "__main__":
+    sys.exit(main())

tests/requirements.txt

@@ -1,3 +1,3 @@
-pytest
-requests
-progressbar2
+pytest==8.3.3
+requests==2.32.3
+progressbar2==4.5.0

tests/run_pytest.py

@@ -1,11 +1,15 @@
 import pytest
 import sys
+import time
+import argparse
+
+from typing import Optional
 from pathlib import Path
 
 CWD = Path(__file__).parent
 
-if __name__ == "__main__":
-    retcode = pytest.main([
+def run(junit_xml: Optional[str] = None):
+    args = [
         (CWD / "macho"),
         (CWD / "pe"),
         (CWD / "elf"),
@@ -21,6 +25,26 @@ if __name__ == "__main__":
         (CWD / "assembly"),
         (CWD / "abstract"),
         "--verbose"
-    ])
+    ]
+    if junit_xml is not None:
+        args.append(f"--junit-xml={junit_xml}")
+
+    retcode = pytest.main(args)
+
     print(f"Retcode: {retcode}")
     sys.exit(retcode)
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--junit-xml",
+        metavar="JUNIT_XML",
+        dest="junit_xml",
+        default=None,
+        type=str
+    )
+    args = parser.parse_args()
+    run(args.junit_xml)
+
+if __name__ == "__main__":
+    main()