test(plugin): measure what the plugin adds with claude plugin eval (#66)

.gitignore

@@ -71,3 +71,6 @@ Thumbs.db
 
 # Complete static website output, including generated language editions.
 website/_site/
+
+# claude plugin eval output
+evals/results/

evals/README.md

@@ -0,0 +1,31 @@
+# Plugin evals
+
+`claude plugin eval` runs each case twice: once with the ReversePort plugin and once without it. The difference
+(`Δ`) is what the plugin adds. Each case needs Claude Code 2.1.269 or later.
+
+| Case            | Question                                                  | Passes when the answer has |
+| --------------- | --------------------------------------------------------- | -------------------------- |
+| `godot-recon`   | Engine, version and start scene of a small Godot project  | Godot, 4.3 and `main.tscn` |
+| `pck-contents`  | The GDScript files inside an exported Godot `.pck`        | both `res://scripts/*.gd`  |
+| `coords-unreal` | A Godot position (1, 2, 3) in Unreal Engine 5 coordinates | -300, 100, 200             |
+
+The `used-*` graders only show whether the plugin's tool was called; they do not count towards the score.
+
+## Run
+
+Every run is a real model call on your own account. Three cases, three runs each, two arms: 18 runs.
+
+```bash
+claude plugin eval . --trust-plugin --no-publish --max-cost-usd 5
+```
+
+By default the ReversePort tools answer from `mocks/reverseport/`: the real output of ReversePort on the
+fixtures in each case folder, so a run needs no engine and never touches your files. To test the real server
+instead (it runs as you, outside the eval sandbox; ReversePort's own tools only read):
+
+```bash
+claude plugin eval . --trust-plugin --no-publish --mocks off \
+  --allow-tools "mcp__plugin_reverseport_reverseport__*"
+```
+
+Results go to `evals/results/<time>/` (`report.html`, `aggregate-result.json`); that folder is not committed.

evals/coords-unreal/case.yaml

@@ -0,0 +1,2 @@
+schema_version: "1.1"
+name: coords-unreal

evals/coords-unreal/graders/position.md

@@ -0,0 +1,4 @@
+---
+type: regex
+pattern: '[-−]300(\.0+)?\D+100(\.0+)?\D+200'
+---

evals/coords-unreal/graders/used-port-convert.md

@@ -0,0 +1,5 @@
+---
+type: tool_used
+tool: mcp__plugin_reverseport_reverseport__port_convert
+arm: with-only
+---

evals/coords-unreal/prompt.md

@@ -0,0 +1,7 @@
+---
+runs: 3
+max_turns: 8
+allowed_tools: [Read, Glob, Grep, Skill]
+---
+
+In a Godot 4 scene a node sits at position (1, 2, 3). Where is that in Unreal Engine 5 world coordinates, in Unreal units? Give X, Y and Z.

evals/godot-recon/case.yaml

@@ -0,0 +1,4 @@
+schema_version: "1.1"
+name: godot-recon
+context:
+  add_dirs: [game]

evals/godot-recon/game/actors/bullet.gd

@@ -0,0 +1,7 @@
+extends RigidBody3D
+
+func _ready() -> void:
+	apply_central_impulse(-global_transform.basis.z * 30.0)
+	var t := create_tween()
+	t.tween_property(self, "scale", Vector3.ZERO, 2.0)
+	t.tween_callback(queue_free)

evals/godot-recon/game/actors/bullet.tscn

@@ -0,0 +1,12 @@
+[gd_scene load_steps=3 format=3 uid="uid://bullet01"]
+
+[ext_resource type="Script" path="res://actors/bullet.gd" id="1_b"]
+
+[sub_resource type="SphereShape3D" id="SphereShape3D_1"]
+radius = 0.1
+
+[node name="Bullet" type="RigidBody3D"]
+script = ExtResource("1_b")
+
+[node name="Shape" type="CollisionShape3D" parent="."]
+shape = SubResource("SphereShape3D_1")

evals/godot-recon/game/actors/enemy.gd

@@ -0,0 +1,12 @@
+class_name Enemy
+extends Area3D
+
+@export var damage := 10
+
+func _ready() -> void:
+	body_entered.connect(_on_body_entered)
+
+func _on_body_entered(body: Node3D) -> void:
+	if body is Player:
+		body.hit.emit(damage)
+		print("enemy hit player")

evals/godot-recon/game/actors/enemy.tscn

@@ -0,0 +1,13 @@
+[gd_scene load_steps=3 format=3 uid="uid://enemy01"]
+
+[ext_resource type="Script" path="res://actors/enemy.gd" id="1_e"]
+
+[sub_resource type="BoxShape3D" id="BoxShape3D_1"]
+size = Vector3(2, 2, 2)
+
+[node name="Enemy" type="Area3D" groups=["enemies"]]
+script = ExtResource("1_e")
+damage = 25
+
+[node name="Shape" type="CollisionShape3D" parent="."]
+shape = SubResource("BoxShape3D_1")

evals/godot-recon/game/actors/player.gd

@@ -0,0 +1,36 @@
+class_name Player
+extends CharacterBody3D
+
+signal hit(damage: int)
+
+@export var speed: float = 6.0
+@export var jump_velocity := 4.5
+@onready var anim: AnimationPlayer = $AnimationPlayer
+@onready var muzzle := $Gun/Muzzle
+
+const BULLET = preload("res://actors/bullet.tscn")
+
+func _ready() -> void:
+	add_to_group("players")
+	hit.connect(_on_hit)
+
+func _physics_process(delta: float) -> void:
+	if not is_on_floor():
+		velocity += get_gravity() * delta
+	var dir := Input.get_axis("move_left", "move_right")
+	velocity.x = dir * speed
+	if Input.is_action_just_pressed("fire"):
+		shoot()
+	move_and_slide()
+
+func shoot() -> void:
+	var b := BULLET.instantiate()
+	get_tree().current_scene.add_child(b)
+	b.global_transform = muzzle.global_transform
+	anim.play("shoot")
+
+func _on_hit(damage: int) -> void:
+	Global.add_score(-damage)
+	if damage > 50:
+		await get_tree().create_timer(0.5).timeout
+		queue_free()

evals/godot-recon/game/actors/player.tscn

@@ -0,0 +1,25 @@
+[gd_scene load_steps=4 format=3 uid="uid://player01"]
+
+[ext_resource type="Script" path="res://actors/player.gd" id="1_p"]
+
+[sub_resource type="CapsuleShape3D" id="CapsuleShape3D_1"]
+
+[sub_resource type="CapsuleMesh" id="CapsuleMesh_1"]
+
+[node name="Player" type="CharacterBody3D" groups=["players"]]
+script = ExtResource("1_p")
+speed = 8.0
+
+[node name="Shape" type="CollisionShape3D" parent="."]
+shape = SubResource("CapsuleShape3D_1")
+
+[node name="Mesh" type="MeshInstance3D" parent="."]
+mesh = SubResource("CapsuleMesh_1")
+
+[node name="Gun" type="Node3D" parent="."]
+transform = Transform3D(-4.37114e-08, 0, 1, 0, 1, 0, -1, 0, -4.37114e-08, 0.5, 1.2, -0.3)
+
+[node name="Muzzle" type="Marker3D" parent="Gun"]
+transform = Transform3D(1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, -0.6)
+
+[node name="AnimationPlayer" type="AnimationPlayer" parent="."]

evals/godot-recon/game/global.gd

@@ -0,0 +1,10 @@
+extends Node
+## Autoload: score and game state.
+
+signal score_changed(new_score: int)
+
+var score := 0
+
+func add_score(points: int) -> void:
+	score += points
+	score_changed.emit(score)  # notify the HUD

evals/godot-recon/game/items/coin.tres

@@ -0,0 +1,8 @@
+[gd_resource type="Resource" script_class="ItemData" load_steps=2 format=3 uid="uid://coin01"]
+
+[ext_resource type="Script" path="res://items/item_data.gd" id="1_i"]
+
+[resource]
+script = ExtResource("1_i")
+name = "Coin"
+value = 5

evals/godot-recon/game/items/item_data.gd

@@ -0,0 +1,5 @@
+class_name ItemData
+extends Resource
+
+@export var name: String = ""
+@export var value: int = 1

evals/godot-recon/game/main.tscn

@@ -0,0 +1,26 @@
+[gd_scene load_steps=5 format=3 uid="uid://main01"]
+
+[ext_resource type="PackedScene" uid="uid://player01" path="res://actors/player.tscn" id="1_pl"]
+[ext_resource type="PackedScene" uid="uid://enemy01" path="res://actors/enemy.tscn" id="2_en"]
+[ext_resource type="PackedScene" uid="uid://hud01" path="res://ui/hud.tscn" id="3_hu"]
+
+[node name="Main" type="Node3D"]
+
+[node name="Sun" type="DirectionalLight3D" parent="."]
+transform = Transform3D(1, 0, 0, 0, 0.707107, 0.707107, 0, -0.707107, 0.707107, 0, 10, 0)
+
+[node name="Camera3D" type="Camera3D" parent="."]
+transform = Transform3D(1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 2, 8)
+fov = 70.0
+
+[node name="Player" parent="." instance=ExtResource("1_pl")]
+transform = Transform3D(1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0)
+
+[node name="Enemies" type="Node3D" parent="."]
+
+[node name="Enemy" parent="Enemies" instance=ExtResource("2_en")]
+transform = Transform3D(2, 0, 0, 0, 2, 0, 0, 0, 2, 5, 1, -10)
+
+[node name="HUD" parent="." instance=ExtResource("3_hu")]
+
+[connection signal="hit" from="Player" to="HUD" method="_on_player_hit"]

evals/godot-recon/game/project.godot

@@ -0,0 +1,35 @@
+; Engine configuration file.
+; It's best edited using the editor UI and not directly,
+; since the parameters that go here are not all obvious.
+;
+; Format:
+;   [section] ; section goes between []
+;   param=value ; assign values to parameters
+
+config_version=5
+
+[application]
+
+config/name="Mini Heli"
+run/main_scene="res://main.tscn"
+config/features=PackedStringArray("4.3", "Forward Plus")
+
+[autoload]
+
+Global="*res://global.gd"
+
+[input]
+
+move_left={
+"deadzone": 0.5,
+"events": [Object(InputEventKey,"resource_local_to_scene":false,"resource_name":"","device":-1,"window_id":0,"alt_pressed":false,"shift_pressed":false,"ctrl_pressed":false,"meta_pressed":false,"pressed":false,"keycode":0,"physical_keycode":65,"key_label":0,"unicode":97,"location":0,"echo":false,"script":null)
+]
+}
+fire={
+"deadzone": 0.5,
+"events": []
+}
+
+[physics]
+
+common/physics_ticks_per_second=60

evals/godot-recon/game/ui/hud.gd

@@ -0,0 +1,9 @@
+extends CanvasLayer
+
+@onready var label: Label = $Score
+
+func _ready() -> void:
+	Global.score_changed.connect(_on_score_changed)
+
+func _on_score_changed(new_score: int) -> void:
+	label.text = "Score: %d" % new_score

evals/godot-recon/game/ui/hud.tscn

@@ -0,0 +1,11 @@
+[gd_scene load_steps=2 format=3 uid="uid://hud01"]
+
+[ext_resource type="Script" path="res://ui/hud.gd" id="1_h"]
+
+[node name="HUD" type="CanvasLayer"]
+script = ExtResource("1_h")
+
+[node name="Score" type="Label" parent="."]
+offset_right = 200.0
+offset_bottom = 40.0
+text = "Score: 0"

evals/godot-recon/graders/engine.md

@@ -0,0 +1,5 @@
+---
+type: regex
+pattern: "godot"
+flags: i
+---

evals/godot-recon/graders/main-scene.md

@@ -0,0 +1,5 @@
+---
+type: regex
+pattern: 'main\.tscn'
+flags: i
+---

evals/godot-recon/graders/used-re-scan.md

@@ -0,0 +1,5 @@
+---
+type: tool_used
+tool: mcp__plugin_reverseport_reverseport__re_scan
+arm: with-only
+---

evals/godot-recon/graders/version.md

@@ -0,0 +1,4 @@
+---
+type: regex
+pattern: '4\.3'
+---

evals/godot-recon/prompt.md

@@ -0,0 +1,7 @@
+---
+runs: 3
+max_turns: 8
+allowed_tools: [Read, Glob, Grep, Skill]
+---
+
+The read-only folder `game` holds a game project. Which engine and engine version is it made with, and which scene starts the game? Answer in two sentences.

evals/mocks/reverseport/port_convert.md

@@ -0,0 +1,7 @@
+---
+expect:
+  from: string
+  to: string
+---
+
+{"from": "Godot (right-handed, Y up, -Z forward, m)", "to": "Unreal (left-handed, Z up, X forward, Y right, cm)", "axisMatrix": [[0, 0, -1], [1, 0, 0], [0, 1, 0]], "unitFactor": 100, "position": [-300, 100, 200], "notes": ["Positions and rotations stay visually identical; the axis matrix shows where each source axis goes.", "Handedness changes (mirror): meshes imported separately keep their own importer orientation \u2014 if a model looks backwards, rotate that node 180\u00b0 about the up axis.", "Distances scale \u00d7100 (speeds, forces and ranges in code too)."]}

evals/mocks/reverseport/re_archive.md

@@ -0,0 +1,6 @@
+---
+expect:
+  file: string
+---
+
+{"format": "godot-pck", "file": "{{input.file}}", "size": 6624, "packVersion": 2, "godotVersion": "4.3.0", "embedded": false, "start": 0, "encryptedDirectory": false, "totalEntries": 13, "matching": 13, "totalBytes": 5355, "types": [{"ext": ".scn", "files": 2, "bytes": 1852}, {"ext": ".gdc", "files": 2, "bytes": 1803}, {"ext": ".binary", "files": 1, "bytes": 765}, {"ext": ".remap", "files": 5, "bytes": 372}, {"ext": ".res", "files": 1, "bytes": 341}, {"ext": ".cfg", "files": 1, "bytes": 149}, {"ext": ".bin", "files": 1, "bytes": 73}], "entries": [{"path": "res://.godot/exported/133200997/export-3ad5c15c4f3250da0cc7c1af1770d85f-main.scn", "offset": 2080, "size": 1276, "md5": "9c3139498a65621969ed38936be9b6a3"}, {"path": "res://.godot/exported/133200997/export-7e19f985116542ca6f391bc3d1279cfe-data.res", "offset": 1152, "size": 341, "md5": "4aba81a99d147a9aa13d141d767d85dd"}, {"path": "res://.godot/exported/133200997/export-a930820ec0ab16db6719aff4daa500f9-enemy.scn", "offset": 1504, "size": 576, "md5": "ee5974f9dd6d4d9ef63ee13d2e7a3024"}, {"path": "res://.godot/global_script_class_cache.cfg", "offset": 5616, "size": 149, "md5": "18be9c02087936fb8fe9a2d2e7691ae9"}, {"path": "res://.godot/uid_cache.bin", "offset": 5776, "size": 73, "md5": "fcefec906fc1d1f7bc9154094800381f"}, {"path": "res://project.binary", "offset": 5856, "size": 765, "md5": "33bcee7c45fd00ab5a251aac1b12b6f3"}, {"path": "res://scenes/data.tres.remap", "offset": 5184, "size": 97, "md5": "fc0ae6b3a629487b50312bed332d0538"}, {"path": "res://scenes/enemy.tscn.remap", "offset": 5296, "size": 98, "md5": "dc4bd92be56ab2410b946be1853cdb33"}, {"path": "res://scenes/main.tscn.remap", "offset": 5408, "size": 97, "md5": "0ab3b58be7f208f78c8c53387aea9d73"}, {"path": "res://scripts/game.gd.remap", "offset": 5520, "size": 39, "md5": "d09d7499784225fd170969c0c7fc2cff"}, {"path": "res://scripts/game.gdc", "offset": 3360, "size": 244, "md5": "b674ce020a9109968f8fe6059a97e1e1"}, {"path": "res://scripts/player.gd.remap", "offset": 5568, "size": 41, "md5": "f81b27b98ee7907504a70d34a65358fd"}, {"path": "res://scripts/player.gdc", "offset": 3616, "size": 1559, "md5": "9f6ec15bb581b428da5f1cd5371b3e71"}], "truncated": false}

evals/mocks/reverseport/re_scan.md

@@ -0,0 +1,6 @@
+---
+expect:
+  path: string
+---
+
+{"root": "{{input.path}}", "files": 13, "truncated": false, "totalBytes": 5887, "skippedDirs": [], "primary": {"id": "godot", "name": "Godot 4.3", "kind": "project"}, "engines": [{"id": "godot", "name": "Godot 4.3", "kind": "project", "root": "", "version": "4.3", "details": {"name": "Mini Heli", "mainScene": "res://main.tscn", "godotMajor": 4, "version": "4.3", "renderer": "Forward Plus", "dotnet": false, "autoload": [{"name": "Global", "path": "res://global.gd", "singleton": true}], "inputActions": ["move_left", "fire"], "physicsTicks": 60}, "evidence": ["project.godot"]}], "languages": {"gdscript": {"files": 6, "lines": 79, "bytes": 1859}}, "categories": {"scene": {"files": 5, "bytes": 2930}, "code": {"files": 6, "bytes": 1859}, "config": {"files": 1, "bytes": 868}, "resource": {"files": 1, "bytes": 230}}, "entryPoints": [{"file": "main.tscn", "why": "Godot main scene (run/main_scene)"}, {"file": "global.gd", "why": "Godot autoload \"Global\""}], "biggest": [{"file": "main.tscn", "size": 1066, "category": "scene"}, {"file": "actors/player.gd", "size": 898, "category": "code"}, {"file": "project.godot", "size": 868, "category": "config"}, {"file": "actors/player.tscn", "size": 839, "category": "scene"}, {"file": "actors/enemy.tscn", "size": 383, "category": "scene"}, {"file": "actors/bullet.tscn", "size": 358, "category": "scene"}, {"file": "ui/hud.tscn", "size": 284, "category": "scene"}, {"file": "actors/enemy.gd", "size": 240, "category": "code"}, {"file": "items/coin.tres", "size": 230, "category": "resource"}, {"file": "ui/hud.gd", "size": 216, "category": "code"}, {"file": "actors/bullet.gd", "size": 208, "category": "code"}, {"file": "global.gd", "size": 202, "category": "code"}], "binaries": [], "next": "re_graph {\"path\":\"{{input.path}}\"} for the dependency graph and porting order", "nextSteps": ["re_graph {\"path\":\"{{input.path}}\"} for the dependency graph and porting order", "re_symbols {\"path\":\"{{input.path}}\",\"match\":\"<name>\"} for classes and functions", "port_plan {\"source\":\"{{input.path}}\",\"to\":\"<unity|godot|godot-csharp|unreal|threejs|language>\"} to start a port"]}

evals/pck-contents/case.yaml

@@ -0,0 +1,4 @@
+schema_version: "1.1"
+name: pck-contents
+context:
+  add_dirs: [game]

evals/pck-contents/graders/game-script.md

@@ -0,0 +1,4 @@
+---
+type: regex
+pattern: 'res://scripts/game\.gd'
+---

evals/pck-contents/graders/player-script.md

@@ -0,0 +1,4 @@
+---
+type: regex
+pattern: 'res://scripts/player\.gd'
+---

evals/pck-contents/graders/used-re-archive.md

@@ -0,0 +1,5 @@
+---
+type: tool_used
+tool: mcp__plugin_reverseport_reverseport__re_archive
+arm: with-only
+---

evals/pck-contents/prompt.md

@@ -0,0 +1,7 @@
+---
+runs: 3
+max_turns: 8
+allowed_tools: [Read, Glob, Grep, Skill]
+---
+
+The read-only folder `game` holds `game.pck`, the pack of an exported Godot 4 game. Which GDScript files are inside it? Give their res:// paths.