mirror of
https://github.com/michaelshimeles/terminal.git
synced 2026-11-09 11:51:25 -10:00
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Michael ShimelesandClaude Opus 5.5
|
6e53585b50 | ||
|
Michael ShimelesandClaude Opus 5.5
|
b5776f2abf |
No files matched your search
@@ -40,9 +40,9 @@ jobs:
|
||||
TR_PROVIDER: sample
|
||||
TR_CONGRESS_REFRESH_MIN: "0"
|
||||
TR_SMOKE_SECS: "8"
|
||||
TR_DATA_DIR: ${{ runner.temp }}/terminal
|
||||
TR_DATA_DIR: ${{ runner.temp }}/terminal-rs
|
||||
run: |
|
||||
./target/release/terminal > smoke.log 2>&1 &
|
||||
./target/release/terminal-rs > smoke.log 2>&1 &
|
||||
pid=$!
|
||||
sleep 6
|
||||
screencapture -x screenshot-${{ matrix.arch }}.png || true
|
||||
@@ -56,9 +56,9 @@ jobs:
|
||||
env:
|
||||
TR_SMOKE_SECS: "12"
|
||||
TR_CONGRESS_REFRESH_MIN: "0"
|
||||
TR_DATA_DIR: ${{ runner.temp }}/terminal-live
|
||||
TR_DATA_DIR: ${{ runner.temp }}/terminal-rs-live
|
||||
run: |
|
||||
./target/release/terminal > live.log 2>&1 &
|
||||
./target/release/terminal-rs > live.log 2>&1 &
|
||||
pid=$!
|
||||
sleep 10
|
||||
screencapture -x screenshot-live-${{ matrix.arch }}.png || true
|
||||
@@ -84,18 +84,15 @@ jobs:
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
components: clippy, rustfmt
|
||||
targets: wasm32-unknown-unknown
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- run: sudo apt-get update && sudo apt-get install -y libxkbcommon-x11-0 libxcb-xkb1 mesa-vulkan-drivers xvfb
|
||||
- run: cargo fmt --all -- --check
|
||||
- run: cargo build --release --workspace --all-targets
|
||||
- run: cargo test --release --workspace
|
||||
- run: cargo clippy --release --workspace --all-targets -- -D warnings
|
||||
- name: The protocol crate builds for the browser
|
||||
run: cargo build -p tr-proto --target wasm32-unknown-unknown
|
||||
- name: Run the app (smoke test)
|
||||
env:
|
||||
TR_PROVIDER: sample
|
||||
TR_CONGRESS_REFRESH_MIN: "0"
|
||||
TR_SMOKE_SECS: "6"
|
||||
run: xvfb-run -a ./target/release/terminal 2>&1 | tee smoke.log && grep -q "smoke:" smoke.log
|
||||
run: xvfb-run -a ./target/release/terminal-rs 2>&1 | tee smoke.log && grep -q "smoke:" smoke.log
|
||||
Generated
+14
-255
@@ -225,63 +225,6 @@ dependencies = [
|
||||
"pkg-config",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "axum"
|
||||
version = "0.8.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90"
|
||||
dependencies = [
|
||||
"axum-core",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"futures-util",
|
||||
"http",
|
||||
"http-body",
|
||||
"http-body-util",
|
||||
"hyper",
|
||||
"hyper-util",
|
||||
"itoa",
|
||||
"matchit",
|
||||
"memchr",
|
||||
"mime",
|
||||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
"serde_core",
|
||||
"serde_json",
|
||||
"serde_path_to_error",
|
||||
"sha1 0.10.7",
|
||||
"sync_wrapper",
|
||||
"tokio",
|
||||
"tokio-tungstenite 0.29.0",
|
||||
"tower",
|
||||
"tower-layer",
|
||||
"tower-service",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "axum-core"
|
||||
version = "0.5.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures-core",
|
||||
"http",
|
||||
"http-body",
|
||||
"http-body-util",
|
||||
"mime",
|
||||
"pin-project-lite",
|
||||
"sync_wrapper",
|
||||
"tower-layer",
|
||||
"tower-service",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "base64"
|
||||
version = "0.22.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
|
||||
|
||||
[[package]]
|
||||
name = "base64"
|
||||
version = "0.23.1"
|
||||
@@ -510,7 +453,7 @@ checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"cpufeatures 0.3.1",
|
||||
"rand_core 0.10.1",
|
||||
"rand_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -607,15 +550,6 @@ dependencies = [
|
||||
"cc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cobs"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1"
|
||||
dependencies = [
|
||||
"thiserror 2.0.21",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "codespan-reporting"
|
||||
version = "0.13.1"
|
||||
@@ -1186,18 +1120,6 @@ dependencies = [
|
||||
"bytemuck",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "embedded-io"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced"
|
||||
|
||||
[[package]]
|
||||
name = "embedded-io"
|
||||
version = "0.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d"
|
||||
|
||||
[[package]]
|
||||
name = "encoding_rs"
|
||||
version = "0.8.42"
|
||||
@@ -1548,7 +1470,7 @@ dependencies = [
|
||||
"js-sys",
|
||||
"libc",
|
||||
"r-efi 6.0.0",
|
||||
"rand_core 0.10.1",
|
||||
"rand_core",
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
@@ -1797,12 +1719,6 @@ version = "1.10.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87"
|
||||
|
||||
[[package]]
|
||||
name = "httpdate"
|
||||
version = "1.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
|
||||
|
||||
[[package]]
|
||||
name = "hybrid-array"
|
||||
version = "0.4.15"
|
||||
@@ -1824,7 +1740,6 @@ dependencies = [
|
||||
"http",
|
||||
"http-body",
|
||||
"httparse",
|
||||
"httpdate",
|
||||
"itoa",
|
||||
"pin-project-lite",
|
||||
"smallvec",
|
||||
@@ -1853,7 +1768,7 @@ version = "0.1.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff"
|
||||
dependencies = [
|
||||
"base64 0.23.1",
|
||||
"base64",
|
||||
"bytes",
|
||||
"futures-channel",
|
||||
"futures-util",
|
||||
@@ -2285,7 +2200,7 @@ dependencies = [
|
||||
"log",
|
||||
"md-5",
|
||||
"nom",
|
||||
"rand 0.10.3",
|
||||
"rand",
|
||||
"rangemap",
|
||||
"sha2",
|
||||
"stringprep",
|
||||
@@ -2320,12 +2235,6 @@ dependencies = [
|
||||
"regex-automata",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "matchit"
|
||||
version = "0.8.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3"
|
||||
|
||||
[[package]]
|
||||
name = "md-5"
|
||||
version = "0.10.6"
|
||||
@@ -2351,12 +2260,6 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mime"
|
||||
version = "0.3.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
|
||||
|
||||
[[package]]
|
||||
name = "miniz_oxide"
|
||||
version = "0.8.9"
|
||||
@@ -3156,18 +3059,6 @@ dependencies = [
|
||||
"portable-atomic",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "postcard"
|
||||
version = "1.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24"
|
||||
dependencies = [
|
||||
"cobs",
|
||||
"embedded-io 0.4.0",
|
||||
"embedded-io 0.6.1",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "postscript"
|
||||
version = "0.14.1"
|
||||
@@ -3189,15 +3080,6 @@ version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4a6394b9e965e73d0a289ee54f589087e2c676aedf60885baf52c76b771e4958"
|
||||
|
||||
[[package]]
|
||||
name = "ppv-lite86"
|
||||
version = "0.2.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9"
|
||||
dependencies = [
|
||||
"zerocopy",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "precomputed-hash"
|
||||
version = "0.1.1"
|
||||
@@ -3311,7 +3193,7 @@ dependencies = [
|
||||
"bytes",
|
||||
"getrandom 0.4.3",
|
||||
"lru-slab",
|
||||
"rand 0.10.3",
|
||||
"rand",
|
||||
"rand_pcg",
|
||||
"ring",
|
||||
"rustc-hash 2.1.3",
|
||||
@@ -3359,16 +3241,6 @@ version = "6.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
|
||||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.9.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41"
|
||||
dependencies = [
|
||||
"rand_chacha",
|
||||
"rand_core 0.9.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.10.3"
|
||||
@@ -3377,26 +3249,7 @@ checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af"
|
||||
dependencies = [
|
||||
"chacha20",
|
||||
"getrandom 0.4.3",
|
||||
"rand_core 0.10.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_chacha"
|
||||
version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
|
||||
dependencies = [
|
||||
"ppv-lite86",
|
||||
"rand_core 0.9.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_core"
|
||||
version = "0.9.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c"
|
||||
dependencies = [
|
||||
"getrandom 0.3.4",
|
||||
"rand_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3411,7 +3264,7 @@ version = "0.10.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a"
|
||||
dependencies = [
|
||||
"rand_core 0.10.1",
|
||||
"rand_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3543,7 +3396,7 @@ version = "0.13.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029"
|
||||
dependencies = [
|
||||
"base64 0.23.1",
|
||||
"base64",
|
||||
"bytes",
|
||||
"cookie",
|
||||
"cookie_store",
|
||||
@@ -3901,17 +3754,6 @@ dependencies = [
|
||||
"zmij",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_path_to_error"
|
||||
version = "0.1.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457"
|
||||
dependencies = [
|
||||
"itoa",
|
||||
"serde",
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_spanned"
|
||||
version = "1.1.2"
|
||||
@@ -3942,17 +3784,6 @@ dependencies = [
|
||||
"stable_deref_trait",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sha1"
|
||||
version = "0.10.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"cpufeatures 0.2.17",
|
||||
"digest 0.10.7",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sha1"
|
||||
version = "0.11.0"
|
||||
@@ -3990,16 +3821,6 @@ version = "2.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
|
||||
|
||||
[[package]]
|
||||
name = "signal-hook-registry"
|
||||
version = "1.4.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b"
|
||||
dependencies = [
|
||||
"errno",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "simd-adler32"
|
||||
version = "0.3.10"
|
||||
@@ -4276,11 +4097,10 @@ dependencies = [
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "terminal"
|
||||
name = "terminal-rs"
|
||||
version = "0.1.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
"chrono",
|
||||
"eframe",
|
||||
"egui",
|
||||
@@ -4437,7 +4257,6 @@ dependencies = [
|
||||
"libc",
|
||||
"mio",
|
||||
"pin-project-lite",
|
||||
"signal-hook-registry",
|
||||
"socket2",
|
||||
"tokio-macros",
|
||||
"windows-sys 0.61.2",
|
||||
@@ -4464,18 +4283,6 @@ dependencies = [
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-tungstenite"
|
||||
version = "0.29.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f72a05e828585856dacd553fba484c242c46e391fb0e58917c942ee9202915c"
|
||||
dependencies = [
|
||||
"futures-util",
|
||||
"log",
|
||||
"tokio",
|
||||
"tungstenite 0.29.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-tungstenite"
|
||||
version = "0.30.0"
|
||||
@@ -4488,7 +4295,7 @@ dependencies = [
|
||||
"rustls-pki-types",
|
||||
"tokio",
|
||||
"tokio-rustls",
|
||||
"tungstenite 0.30.0",
|
||||
"tungstenite",
|
||||
"webpki-roots 0.26.11",
|
||||
]
|
||||
|
||||
@@ -4620,7 +4427,7 @@ name = "tr-broker"
|
||||
version = "0.1.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"base64 0.23.1",
|
||||
"base64",
|
||||
"chrono",
|
||||
"futures",
|
||||
"hmac",
|
||||
@@ -4696,9 +4503,8 @@ dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tokio",
|
||||
"tokio-tungstenite 0.30.0",
|
||||
"tokio-tungstenite",
|
||||
"tr-core",
|
||||
"tr-proto",
|
||||
"tracing",
|
||||
"wasm-bindgen",
|
||||
"wasm-bindgen-futures",
|
||||
@@ -4706,27 +4512,6 @@ dependencies = [
|
||||
"web-time",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tr-gateway"
|
||||
version = "0.1.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
"axum",
|
||||
"futures",
|
||||
"reqwest",
|
||||
"rustls",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tokio",
|
||||
"tokio-tungstenite 0.30.0",
|
||||
"tr-core",
|
||||
"tr-data",
|
||||
"tr-proto",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tr-macos"
|
||||
version = "0.1.1"
|
||||
@@ -4736,16 +4521,6 @@ dependencies = [
|
||||
"objc2-foundation 0.3.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tr-proto"
|
||||
version = "0.1.1"
|
||||
dependencies = [
|
||||
"postcard",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tr-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tracing"
|
||||
version = "0.1.44"
|
||||
@@ -4820,22 +4595,6 @@ version = "0.25.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d2df906b07856748fa3f6e0ad0cbaa047052d4a7dd609e231c4f72cee8c36f31"
|
||||
|
||||
[[package]]
|
||||
name = "tungstenite"
|
||||
version = "0.29.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6c01152af293afb9c7c2a57e4b559c5620b421f6d133261c60dd2d0cdb38e6b8"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"data-encoding",
|
||||
"http",
|
||||
"httparse",
|
||||
"log",
|
||||
"rand 0.9.5",
|
||||
"sha1 0.10.7",
|
||||
"thiserror 2.0.21",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tungstenite"
|
||||
version = "0.30.0"
|
||||
@@ -4847,10 +4606,10 @@ dependencies = [
|
||||
"http",
|
||||
"httparse",
|
||||
"log",
|
||||
"rand 0.10.3",
|
||||
"rand",
|
||||
"rustls",
|
||||
"rustls-pki-types",
|
||||
"sha1 0.11.0",
|
||||
"sha1",
|
||||
"thiserror 2.0.21",
|
||||
]
|
||||
|
||||
|
||||
+1
-2
@@ -1,6 +1,6 @@
|
||||
[workspace]
|
||||
resolver = "2"
|
||||
members = ["crates/core", "crates/broker", "crates/congress", "crates/data", "crates/chart", "crates/macos", "crates/proto", "app", "server"]
|
||||
members = ["crates/core", "crates/broker", "crates/congress", "crates/data", "crates/chart", "crates/macos", "app"]
|
||||
default-members = ["app"]
|
||||
|
||||
[workspace.package]
|
||||
@@ -15,7 +15,6 @@ tr-congress = { path = "crates/congress" }
|
||||
tr-data = { path = "crates/data" }
|
||||
tr-chart = { path = "crates/chart" }
|
||||
tr-macos = { path = "crates/macos" }
|
||||
tr-proto = { path = "crates/proto" }
|
||||
anyhow = "1"
|
||||
thiserror = "2"
|
||||
chrono = { version = "0.4", default-features = false, features = ["std", "clock", "serde"] }
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# Terminal
|
||||
# terminal-rs
|
||||
|
||||
A fast, keyboard-driven market terminal in pure Rust: TradingView-class GPU charting with a
|
||||
Bloomberg Terminal look (dense tiled panes, monospace numbers, a command line like `AAPL <GO>`)
|
||||
@@ -9,7 +9,7 @@ systems** and **your own brokerage accounts through [SnapTrade](https://snaptrad
|
||||
to Solarized, and a fully dockable
|
||||
workspace with saved layouts.
|
||||
|
||||
|
||||
|
||||
|
||||
> **Built from reverse-engineering research.** Chart UX was studied behaviourally on
|
||||
> tradingview.com with [REA](https://github.com/morluto/rea) in website mode (passive captures of
|
||||
@@ -122,7 +122,7 @@ gets the native title bar with traffic lights, window vibrancy behind the sideba
|
||||
row in a linked watchlist or world-markets pane loads it into the group.
|
||||
- **Saved layouts**: save, name, rename, duplicate, delete and switch (`LAYOUT <name>`, the
|
||||
sidebar list, ⌥1…⌥9); the active layout autosaves and is restored on launch. Each layout is a
|
||||
JSON file in `~/Library/Application Support/terminal/layouts` (`$XDG_DATA_HOME` on Linux)
|
||||
JSON file in `~/Library/Application Support/terminal-rs/layouts` (`$XDG_DATA_HOME` on Linux)
|
||||
and can be imported or exported (`LAYOUT EXPORT`/`IMPORT`, or Settings › Layouts). Per-pane
|
||||
settings (chart type, indicators, drawings, filters, depth, link, tab title) and, optionally,
|
||||
the theme (`LAYOUT THEME ON`) save with the layout.
|
||||
@@ -181,12 +181,9 @@ TR_PROVIDER=sample cargo run --release # offline
|
||||
APCA_API_KEY_ID=… APCA_API_SECRET_KEY=… cargo run --release # Alpaca
|
||||
```
|
||||
|
||||
Data lives in `~/Library/Application Support/terminal` (macOS) or
|
||||
`$XDG_DATA_HOME/terminal` (Linux), or `TR_DATA_DIR`. Versions from before the project was renamed
|
||||
used a `terminal-rs` folder in the same place; the first launch moves it to `terminal` and leaves
|
||||
a link at the old path, so builds that still use the old name find the same files. The first
|
||||
congress ingest covers the current and previous year (a few minutes; it runs in the background
|
||||
while you work).
|
||||
Data lives in `~/Library/Application Support/terminal-rs` (macOS) or
|
||||
`$XDG_DATA_HOME/terminal-rs` (Linux), or `TR_DATA_DIR`. The first congress ingest covers the
|
||||
current and previous year (a few minutes; it runs in the background while you work).
|
||||
`cargo run -p tr-congress --example ingest -- congress.sqlite3` runs one ingest from the CLI.
|
||||
|
||||
Appearance: `TR_THEME=light|dark|<theme name>` overrides the saved theme, `TR_MSAA=1|2|4|8` sets
|
||||
@@ -229,9 +226,7 @@ What the browser build can and can't do yet:
|
||||
proxy can fix that later.
|
||||
- Congress disclosures, SnapTrade accounts and live order entry need the desktop app. The paper
|
||||
trading demo account works.
|
||||
- Layouts, settings, themes and the demo account are saved in `localStorage` under `terminal/`.
|
||||
The first load copies anything saved under the old `terminal-rs/` prefix, without overwriting
|
||||
newer entries.
|
||||
- Layouts, settings, themes and the demo account are saved in `localStorage` under `terminal-rs/`.
|
||||
- Charts need WebGPU, since they read bars from storage buffers in the vertex shader. Browsers
|
||||
without a WebGPU adapter get a second, larger build on WebGL2 where everything except the chart
|
||||
series works.
|
||||
@@ -273,9 +268,8 @@ the portal, and **Disconnect** removes a connection and its accounts from SnapTr
|
||||
explicit confirmation.
|
||||
|
||||
**Where credentials live.** On macOS the client ID, consumer key, user ID and user secret are
|
||||
stored together in the **Keychain** (generic password, service `terminal`, account
|
||||
`snaptrade`); credentials saved under the old service, `terminal-rs`, move there the first time
|
||||
the app reads them. Elsewhere they go in `<data dir>/secrets/snaptrade.json` with owner-only permissions
|
||||
stored together in the **Keychain** (generic password, service `terminal-rs`, account
|
||||
`snaptrade`). Elsewhere they go in `<data dir>/secrets/snaptrade.json` with owner-only permissions
|
||||
(0600 in a 0700 directory). They are never written to the repository, layouts, prefs or logs;
|
||||
request errors are scrubbed of the URL (which carries the user secret), and the settings screen
|
||||
masks secrets. **Forget Credentials** removes them. `SNAPTRADE_CLIENT_ID`,
|
||||
|
||||
+2
-6
@@ -1,9 +1,9 @@
|
||||
[package]
|
||||
name = "terminal"
|
||||
name = "terminal-rs"
|
||||
description = "Bloomberg-style trading terminal with GPU charts and congressional trade tracking"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
default-run = "terminal"
|
||||
default-run = "terminal-rs"
|
||||
|
||||
[features]
|
||||
# Browser build only: add wgpu's WebGL2 backend for browsers without WebGPU (about 1 MB more,
|
||||
@@ -60,10 +60,6 @@ web-sys = { version = "0.3", features = [
|
||||
"Response",
|
||||
"Storage",
|
||||
"Url",
|
||||
"UrlSearchParams",
|
||||
"Window",
|
||||
"console",
|
||||
] }
|
||||
|
||||
[dev-dependencies]
|
||||
async-trait = "0.1"
|
||||
+1
-2
@@ -957,7 +957,6 @@ impl TerminalApp {
|
||||
prepended,
|
||||
from_cache,
|
||||
exhausted,
|
||||
full,
|
||||
live,
|
||||
} => {
|
||||
let worker = &self.worker;
|
||||
@@ -965,7 +964,7 @@ impl TerminalApp {
|
||||
&& c.symbol == symbol
|
||||
&& c.tf == tf
|
||||
{
|
||||
c.on_history(generation, bars, gpu, prepended, from_cache, exhausted, full, live, worker);
|
||||
c.on_history(generation, bars, gpu, prepended, from_cache, exhausted, live, worker);
|
||||
}
|
||||
}
|
||||
Response::HistoryError { pane, generation, error } => {
|
||||
|
||||
@@ -101,11 +101,7 @@ pub struct BrokerState {
|
||||
|
||||
impl BrokerState {
|
||||
pub fn load(offline: bool) -> Self {
|
||||
Self::with_store(SecretStore::platform(&data_dir()), offline)
|
||||
}
|
||||
|
||||
/// [`Self::load`] with credentials from `store` instead of the platform's.
|
||||
pub fn with_store(store: SecretStore, offline: bool) -> Self {
|
||||
let store = SecretStore::platform(&data_dir());
|
||||
let mut message = None;
|
||||
// The browser build can't reach SnapTrade, and keys don't belong in page storage.
|
||||
let stored = if cfg!(target_arch = "wasm32") { Ok(None) } else { store.load() };
|
||||
@@ -194,7 +190,7 @@ pub fn new_user_id() -> String {
|
||||
// Browsers have no process ids.
|
||||
let pid = if cfg!(target_arch = "wasm32") { 0 } else { std::process::id() };
|
||||
let mix = nanos ^ (u128::from(pid) << 64);
|
||||
format!("terminal-{:016x}", (mix as u64) ^ ((mix >> 64) as u64))
|
||||
format!("terminal-rs-{:016x}", (mix as u64) ^ ((mix >> 64) as u64))
|
||||
}
|
||||
|
||||
impl TerminalApp {
|
||||
@@ -806,7 +802,7 @@ mod tests {
|
||||
let a = new_user_id();
|
||||
std::thread::sleep(Duration::from_millis(1));
|
||||
let b = new_user_id();
|
||||
assert!(a.starts_with("terminal-") && a.len() == "terminal-".len() + 16);
|
||||
assert!(a.starts_with("terminal-rs-") && a.len() == "terminal-rs-".len() + 16);
|
||||
assert_ne!(a, b);
|
||||
}
|
||||
|
||||
|
||||
+14
-40
@@ -108,7 +108,7 @@ impl TerminalApp {
|
||||
p.rect_filled(chip, 9.0, theme::control());
|
||||
p.circle_filled(pos2(chip.left() + 8.0, cy), 3.0, if live { theme::up() } else { amber() });
|
||||
p.galley(pos2(chip.left() + 14.0, cy - chip_text.size().y / 2.0), chip_text, muted());
|
||||
ui.interact(chip, Id::new("source-chip"), Sense::hover()).on_hover_text(provider_label(&self.provider));
|
||||
ui.interact(chip, Id::new("source-chip"), Sense::hover()).on_hover_text(self.provider_label());
|
||||
x = chip.right() + 8.0;
|
||||
|
||||
// Which account trades: DEMO (paper) or LIVE (real money, in red). Click to switch.
|
||||
@@ -294,6 +294,18 @@ impl TerminalApp {
|
||||
}
|
||||
}
|
||||
|
||||
fn provider_label(&self) -> String {
|
||||
let n = self.provider.to_lowercase();
|
||||
if n == "sample" {
|
||||
"Offline · sample data".into()
|
||||
} else if n.contains("sample") {
|
||||
// The browser build (tr_data::web::WebMarkets).
|
||||
"Live crypto from Binance and Coinbase; every other market is sample data in the browser".into()
|
||||
} else {
|
||||
format!("Live · {}", self.provider)
|
||||
}
|
||||
}
|
||||
|
||||
/// The Bloomberg command line, drawn as a rounded macOS search field.
|
||||
fn command_field(&mut self, ui: &mut Ui, field: Rect) -> bool {
|
||||
let suggestions = self.suggestions();
|
||||
@@ -549,7 +561,7 @@ impl TerminalApp {
|
||||
let head = Rect::from_min_size(rect.min, vec2(rect.width(), self.toolbar_h()));
|
||||
window_drag_region(ui, head, "sidebar");
|
||||
if !cfg!(target_os = "macos") {
|
||||
p.text(head.left_center() + vec2(14.0, 0.0), Align2::LEFT_CENTER, "TERMINAL", mono(12.5), amber());
|
||||
p.text(head.left_center() + vec2(14.0, 0.0), Align2::LEFT_CENTER, "TERMINAL-RS", mono(12.5), amber());
|
||||
}
|
||||
|
||||
let body = Rect::from_min_max(pos2(rect.left(), head.bottom()), rect.max);
|
||||
@@ -976,41 +988,3 @@ fn nav_row(ui: &mut Ui, icon: Icon, label: &str, trailing: Option<&str>, selecte
|
||||
}
|
||||
resp
|
||||
}
|
||||
|
||||
/// What the data-source chip says on hover, from the provider's name.
|
||||
fn provider_label(provider: &str) -> String {
|
||||
let n = provider.to_lowercase();
|
||||
if n == "sample" {
|
||||
"Offline · sample data".into()
|
||||
} else if let Some(rest) = provider.strip_prefix("server+") {
|
||||
// A Terminal server (tr_data::remote::RemoteMarkets), in front of the browser's sources
|
||||
// (tr_data::web::WebMarkets) or the desktop's.
|
||||
if rest.contains("sample") {
|
||||
"Live crypto from Hyperliquid perps through the Terminal server, or from Binance and Coinbase while \
|
||||
it is unreachable; every other market is sample data in the browser"
|
||||
.into()
|
||||
} else {
|
||||
format!("Live · crypto from Hyperliquid perps through the Terminal server; other markets from {rest}")
|
||||
}
|
||||
} else if n.contains("sample") {
|
||||
// The browser build (tr_data::web::WebMarkets).
|
||||
"Live crypto from Binance and Coinbase; every other market is sample data in the browser".into()
|
||||
} else {
|
||||
format!("Live · {provider}")
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::provider_label;
|
||||
|
||||
#[test]
|
||||
fn the_source_tooltip_names_the_terminal_server() {
|
||||
assert_eq!(provider_label("sample"), "Offline · sample data");
|
||||
assert!(provider_label("binance+sample").starts_with("Live crypto from Binance"));
|
||||
let web = provider_label("server+binance+sample");
|
||||
assert!(web.contains("Terminal server") && web.contains("sample data in the browser"), "{web}");
|
||||
let desktop = provider_label("server+markets");
|
||||
assert!(desktop.contains("Terminal server") && desktop.ends_with("other markets from markets"), "{desktop}");
|
||||
}
|
||||
}
|
||||
@@ -1174,9 +1174,7 @@ mod tests {
|
||||
#[test]
|
||||
fn renders_every_view_without_panicking() {
|
||||
let ctx = egui::Context::default();
|
||||
// A store that holds nothing, so the test never reads (or moves) the user's Keychain item.
|
||||
let none = std::env::temp_dir().join(format!("tr-portfolio-test-{}/snaptrade.json", std::process::id()));
|
||||
let mut b = BrokerState::with_store(tr_broker::SecretStore::file(none), true);
|
||||
let mut b = BrokerState::load(true);
|
||||
b.creds = tr_broker::Credentials::default();
|
||||
b.portfolio = Some(Box::new(tr_broker::sample::portfolio()));
|
||||
let quotes = HashMap::new();
|
||||
|
||||
+2
-19
@@ -397,7 +397,6 @@ impl ChartState {
|
||||
prepended: usize,
|
||||
from_cache: bool,
|
||||
exhausted: bool,
|
||||
full: bool,
|
||||
live: bool,
|
||||
w: &Worker,
|
||||
) {
|
||||
@@ -422,7 +421,7 @@ impl ChartState {
|
||||
self.bars = bars;
|
||||
self.gpu = Some(gpu);
|
||||
self.paging = false;
|
||||
self.exhausted = end_of_history(self.exhausted, exhausted, prepended, full);
|
||||
self.exhausted = exhausted || self.exhausted && prepended == 0;
|
||||
if !from_cache {
|
||||
self.loading = false;
|
||||
}
|
||||
@@ -440,7 +439,7 @@ impl ChartState {
|
||||
}
|
||||
self.apply_range();
|
||||
// Live ticks only move the last bar: refresh indicators at most every 1.5 s and
|
||||
// leave the compare series alone. (A live feed filling a gap arrives as a refresh.)
|
||||
// leave the compare series alone.
|
||||
let due = self.last_indicator_req.is_none_or(|t| t.elapsed().as_millis() >= 1500);
|
||||
if !live || due {
|
||||
self.request_indicators(w);
|
||||
@@ -1436,26 +1435,10 @@ fn hover_card(ui: &Ui, at: Pos2, t: &Trade) {
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether the chart has reached the start of its history after a response. A full load decides
|
||||
/// it, since it may come from another source than the bars before it. A page decides it too. Live
|
||||
/// updates and refreshes keep what the chart knew.
|
||||
fn end_of_history(was: bool, exhausted: bool, prepended: usize, full: bool) -> bool {
|
||||
if full { exhausted } else { exhausted || was && prepended == 0 }
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn a_full_load_from_a_new_source_can_page_again() {
|
||||
assert!(!end_of_history(true, false, 0, true), "a reload from a source with more history");
|
||||
assert!(end_of_history(false, true, 0, true));
|
||||
assert!(end_of_history(true, false, 0, false), "a live update keeps the end");
|
||||
assert!(!end_of_history(true, false, 5, false), "a page that added bars");
|
||||
assert!(end_of_history(false, true, 0, false), "an empty page");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn glide_pans_linearly_and_zooms_evenly() {
|
||||
let from = Viewport { right: 100.0, bar_px: 2.0, width_px: 800.0 };
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
//! Named, saved workspace layouts: the dock tree, each pane's kind and settings, link
|
||||
//! groups and popped-out windows. Stored one JSON file per layout in
|
||||
//! `<data dir>/layouts/` (`~/Library/Application Support/terminal/layouts` on macOS),
|
||||
//! `<data dir>/layouts/` (`~/Library/Application Support/terminal-rs/layouts` on macOS),
|
||||
//! so they can be shared, hand-edited, imported and exported.
|
||||
|
||||
use std::collections::HashSet;
|
||||
|
||||
+4
-4
@@ -1,4 +1,4 @@
|
||||
//! Terminal: a keyboard-driven, Bloomberg-style market terminal with GPU charts and
|
||||
//! terminal-rs: a keyboard-driven, Bloomberg-style market terminal with GPU charts and
|
||||
//! congressional (STOCK Act) trade tracking.
|
||||
|
||||
mod anim;
|
||||
@@ -31,7 +31,7 @@ fn create_app(
|
||||
msaa: u32,
|
||||
t0: Instant,
|
||||
) -> Result<TerminalApp, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let rs = cc.wgpu_render_state.as_ref().ok_or("Terminal requires the wgpu renderer")?;
|
||||
let rs = cc.wgpu_render_state.as_ref().ok_or("terminal-rs requires the wgpu renderer")?;
|
||||
// Charts read their series from storage buffers in the vertex shader. WebGL2 has none,
|
||||
// so there chart panes keep their axes and labels but draw no series.
|
||||
let charts = rs.adapter.get_downlevel_capabilities().flags.contains(eframe::wgpu::DownlevelFlags::VERTEX_STORAGE);
|
||||
@@ -81,7 +81,7 @@ fn main() -> eframe::Result {
|
||||
.unwrap_or(if cfg!(target_os = "macos") { 4 } else { 1 });
|
||||
let mut viewport = egui::ViewportBuilder::default()
|
||||
.with_title("Terminal")
|
||||
.with_app_id("terminal")
|
||||
.with_app_id("terminal-rs")
|
||||
.with_icon(eframe::icon_data::from_png_bytes(include_bytes!("../assets/icon.png")).expect("app icon"))
|
||||
.with_inner_size([1600.0, 960.0])
|
||||
.with_min_inner_size([900.0, 560.0]);
|
||||
@@ -101,7 +101,7 @@ fn main() -> eframe::Result {
|
||||
..Default::default()
|
||||
};
|
||||
eframe::run_native(
|
||||
"terminal",
|
||||
"terminal-rs",
|
||||
options,
|
||||
Box::new(move |cc| Ok(Box::new(create_app(cc, tr_data::from_env(), u32::from(msaa), t0)?))),
|
||||
)
|
||||
|
||||
+1
-297
@@ -11,114 +11,10 @@ use std::path::{Path, PathBuf};
|
||||
|
||||
pub use imp::*;
|
||||
|
||||
/// Saved documents keyed by path, as the browser keeps them.
|
||||
#[cfg(any(target_arch = "wasm32", test))]
|
||||
trait KeyStore {
|
||||
fn keys(&self) -> Vec<String>;
|
||||
fn get(&self, key: &str) -> Option<String>;
|
||||
/// False when the store refuses the write (full or blocked).
|
||||
fn set(&mut self, key: &str, value: &str) -> bool;
|
||||
fn remove(&mut self, key: &str);
|
||||
}
|
||||
|
||||
/// Copies every key under `old/` to the same path under `new/`, once. Returns whether to use
|
||||
/// `new`. Keys already under `new/` are never overwritten, and the old keys stay as they are. A
|
||||
/// marker key records that the copy finished, so a file deleted afterwards doesn't come back on
|
||||
/// the next load. If the store refuses a write, the copies made so far are removed and this
|
||||
/// returns false, so the page keeps its complete old set and the next load tries again.
|
||||
#[cfg(any(target_arch = "wasm32", test))]
|
||||
fn copy_keys(store: &mut impl KeyStore, old: &str, new: &str) -> bool {
|
||||
let marker = marker(new);
|
||||
if store.get(&marker).is_some() {
|
||||
return true;
|
||||
}
|
||||
let keys: Vec<(String, String)> = store
|
||||
.keys()
|
||||
.into_iter()
|
||||
.filter_map(|k| Some((format!("{new}/{}", k.strip_prefix(old)?.strip_prefix('/')?), k)))
|
||||
.collect();
|
||||
if keys.is_empty() {
|
||||
return true;
|
||||
}
|
||||
let mut copied = Vec::new();
|
||||
let mut ok = true;
|
||||
for (to, from) in keys {
|
||||
if store.get(&to).is_some() {
|
||||
continue;
|
||||
}
|
||||
let Some(value) = store.get(&from) else { continue };
|
||||
if !store.set(&to, &value) {
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
copied.push(to);
|
||||
}
|
||||
if ok && store.set(&marker, old) {
|
||||
tracing::info!("copied {} saved files from {old}/ to {new}/", copied.len());
|
||||
return true;
|
||||
}
|
||||
for k in &copied {
|
||||
store.remove(k);
|
||||
}
|
||||
tracing::warn!("browser storage refused copying saved files from {old}/ to {new}/; using {old}/ for now");
|
||||
false
|
||||
}
|
||||
|
||||
/// The key [`copy_keys`] sets once everything under `new/` is in place.
|
||||
#[cfg(any(target_arch = "wasm32", test))]
|
||||
fn marker(new: &str) -> String {
|
||||
format!("{new}/.migrated")
|
||||
}
|
||||
|
||||
/// Where a page that kept the old prefix (because [`copy_keys`] failed) should read and write
|
||||
/// `key`. Once another tab has finished the copy, keys under `old` map to `new`, so this page's
|
||||
/// saves land where later loads look instead of under keys they no longer read.
|
||||
#[cfg(any(target_arch = "wasm32", test))]
|
||||
fn redirect(store: &impl KeyStore, old: &str, new: &str, key: String) -> String {
|
||||
match key.strip_prefix(old) {
|
||||
Some(rest) if (rest.is_empty() || rest.starts_with('/')) && store.get(&marker(new)).is_some() => {
|
||||
format!("{new}{rest}")
|
||||
}
|
||||
_ => key,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
mod imp {
|
||||
use super::*;
|
||||
|
||||
/// Moves the data folder from `old` (its name before the app was renamed) to `new` the first
|
||||
/// time, and leaves a link at the old path so builds that still use the old name find the
|
||||
/// same files. Returns the folder to use: `new`, or `old` if it couldn't be moved. Never
|
||||
/// replaces an existing `new`.
|
||||
pub fn migrate(old: &Path, new: &Path) -> PathBuf {
|
||||
if new.exists() || !old.is_dir() {
|
||||
return new.to_path_buf();
|
||||
}
|
||||
if let Err(e) = std::fs::rename(old, new) {
|
||||
// Another copy of the app may have moved it first.
|
||||
if new.is_dir() {
|
||||
return new.to_path_buf();
|
||||
}
|
||||
tracing::warn!("could not move {} to {} ({e}); using it where it is", old.display(), new.display());
|
||||
return old.to_path_buf();
|
||||
}
|
||||
tracing::info!("moved {} to {}", old.display(), new.display());
|
||||
#[cfg(unix)]
|
||||
{
|
||||
// Name the sibling rather than repeating `new`, which may be relative (an unset HOME
|
||||
// or a relative XDG_DATA_HOME) and would then resolve from the link's folder.
|
||||
let target = match new.file_name() {
|
||||
Some(name) if old.parent() == new.parent() => PathBuf::from(name),
|
||||
_ => std::path::absolute(new).unwrap_or_else(|_| new.to_path_buf()),
|
||||
};
|
||||
if let Err(e) = std::os::unix::fs::symlink(&target, old) {
|
||||
tracing::warn!("linking {} to {}: {e}", old.display(), target.display());
|
||||
}
|
||||
}
|
||||
new.to_path_buf()
|
||||
}
|
||||
|
||||
pub fn read(path: &Path) -> io::Result<Vec<u8>> {
|
||||
std::fs::read(path)
|
||||
}
|
||||
@@ -179,47 +75,8 @@ mod imp {
|
||||
.ok_or_else(|| io::Error::new(io::ErrorKind::Unsupported, "browser storage is unavailable"))
|
||||
}
|
||||
|
||||
thread_local! {
|
||||
/// The old and new prefixes while this page keeps the old one (see [`migrate`]).
|
||||
static FALLBACK: std::cell::RefCell<Option<(String, String)>> = const { std::cell::RefCell::new(None) };
|
||||
}
|
||||
|
||||
fn key(path: &Path) -> String {
|
||||
let key = path.to_string_lossy().into_owned();
|
||||
let Some((old, new)) = FALLBACK.with_borrow(Clone::clone) else { return key };
|
||||
match store() {
|
||||
Ok(s) => redirect(&s, &old, &new, key),
|
||||
Err(_) => key,
|
||||
}
|
||||
}
|
||||
|
||||
impl KeyStore for web_sys::Storage {
|
||||
fn keys(&self) -> Vec<String> {
|
||||
(0..self.length().unwrap_or(0)).filter_map(|i| self.key(i).ok().flatten()).collect()
|
||||
}
|
||||
fn get(&self, key: &str) -> Option<String> {
|
||||
self.get_item(key).ok().flatten()
|
||||
}
|
||||
fn set(&mut self, key: &str, value: &str) -> bool {
|
||||
self.set_item(key, value).is_ok()
|
||||
}
|
||||
fn remove(&mut self, key: &str) {
|
||||
let _ = self.remove_item(key);
|
||||
}
|
||||
}
|
||||
|
||||
/// Copies what the page saved under `old/` (its prefix before the app was renamed) to
|
||||
/// `new/`, once (see [`copy_keys`]). Returns the prefix to use. If the copy fails, the page
|
||||
/// keeps `old`, and its keys follow [`redirect`] from then on, in case another tab finishes
|
||||
/// the copy while this one is open.
|
||||
pub fn migrate(old: &Path, new: &Path) -> PathBuf {
|
||||
let Ok(mut s) = store() else { return new.to_path_buf() };
|
||||
let (old, new) = (key(old), key(new));
|
||||
if copy_keys(&mut s, &old, &new) {
|
||||
return PathBuf::from(new);
|
||||
}
|
||||
FALLBACK.with_borrow_mut(|f| *f = Some((old.clone(), new)));
|
||||
PathBuf::from(old)
|
||||
path.to_string_lossy().into_owned()
|
||||
}
|
||||
|
||||
pub fn read(path: &Path) -> io::Result<Vec<u8>> {
|
||||
@@ -313,157 +170,4 @@ mod tests {
|
||||
assert!(list(&dir.join("sub")).is_empty());
|
||||
std::fs::remove_dir_all(&dir).unwrap();
|
||||
}
|
||||
|
||||
fn scratch(name: &str) -> PathBuf {
|
||||
let dir = std::env::temp_dir().join(format!("tr-migrate-{name}-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
dir
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn migrate_moves_the_old_folder_and_links_it() {
|
||||
let root = scratch("move");
|
||||
let (old, new) = (root.join("terminal-rs"), root.join("terminal"));
|
||||
write_atomic(&old.join("layouts/Desk.json"), b"{}").unwrap();
|
||||
assert_eq!(migrate(&old, &new), new);
|
||||
assert_eq!(read(&new.join("layouts/Desk.json")).unwrap(), b"{}");
|
||||
#[cfg(unix)]
|
||||
{
|
||||
assert_eq!(std::fs::read_link(&old).unwrap(), Path::new("terminal"), "relative to the link's folder");
|
||||
assert_eq!(read(&old.join("layouts/Desk.json")).unwrap(), b"{}", "old builds still find the files");
|
||||
}
|
||||
// Later launches find the new folder and leave everything as it is.
|
||||
write_atomic(&new.join("settings.json"), b"[]").unwrap();
|
||||
assert_eq!(migrate(&old, &new), new);
|
||||
assert_eq!(read(&new.join("settings.json")).unwrap(), b"[]");
|
||||
std::fs::remove_dir_all(&root).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn migrate_never_replaces_an_existing_folder() {
|
||||
let root = scratch("keep");
|
||||
let (old, new) = (root.join("terminal-rs"), root.join("terminal"));
|
||||
write_atomic(&old.join("settings.json"), b"old").unwrap();
|
||||
write_atomic(&new.join("settings.json"), b"new").unwrap();
|
||||
assert_eq!(migrate(&old, &new), new);
|
||||
assert_eq!(read(&new.join("settings.json")).unwrap(), b"new");
|
||||
assert_eq!(read(&old.join("settings.json")).unwrap(), b"old", "the old folder is left alone");
|
||||
std::fs::remove_dir_all(&root).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn migrate_without_an_old_folder_creates_nothing() {
|
||||
let root = scratch("fresh");
|
||||
let (old, new) = (root.join("terminal-rs"), root.join("terminal"));
|
||||
assert_eq!(migrate(&old, &new), new);
|
||||
assert!(!old.exists() && !new.exists());
|
||||
std::fs::remove_dir_all(&root).unwrap();
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn migrate_keeps_using_the_old_folder_when_it_cannot_move() {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
let root = scratch("stuck");
|
||||
let (old, new) = (root.join("terminal-rs"), root.join("terminal"));
|
||||
write_atomic(&old.join("settings.json"), b"{}").unwrap();
|
||||
std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o555)).unwrap();
|
||||
if std::fs::create_dir(root.join("probe")).is_ok() {
|
||||
// Running as root, where a read-only folder doesn't stop the move.
|
||||
std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o755)).unwrap();
|
||||
std::fs::remove_dir_all(&root).unwrap();
|
||||
return;
|
||||
}
|
||||
let used = migrate(&old, &new);
|
||||
std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o755)).unwrap();
|
||||
assert_eq!(used, old);
|
||||
assert_eq!(read(&old.join("settings.json")).unwrap(), b"{}");
|
||||
std::fs::remove_dir_all(&root).unwrap();
|
||||
}
|
||||
|
||||
/// `localStorage` stand-in: a map that refuses writes once it holds `room` keys.
|
||||
struct Keys {
|
||||
map: std::collections::BTreeMap<String, String>,
|
||||
room: usize,
|
||||
}
|
||||
|
||||
impl Keys {
|
||||
fn new(entries: &[(&str, &str)]) -> Self {
|
||||
Self { map: entries.iter().map(|(k, v)| (k.to_string(), v.to_string())).collect(), room: usize::MAX }
|
||||
}
|
||||
}
|
||||
|
||||
impl KeyStore for Keys {
|
||||
fn keys(&self) -> Vec<String> {
|
||||
self.map.keys().cloned().collect()
|
||||
}
|
||||
fn get(&self, key: &str) -> Option<String> {
|
||||
self.map.get(key).cloned()
|
||||
}
|
||||
fn set(&mut self, key: &str, value: &str) -> bool {
|
||||
if !self.map.contains_key(key) && self.map.len() >= self.room {
|
||||
return false;
|
||||
}
|
||||
self.map.insert(key.into(), value.into());
|
||||
true
|
||||
}
|
||||
fn remove(&mut self, key: &str) {
|
||||
self.map.remove(key);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn copy_keys_copies_once_without_overwriting() {
|
||||
let mut s = Keys::new(&[
|
||||
("terminal-rs/settings.json", "old settings"),
|
||||
("terminal-rs/layouts/Desk.json", "desk"),
|
||||
("terminal/settings.json", "newer settings"),
|
||||
("other-site-key", "untouched"),
|
||||
]);
|
||||
assert!(copy_keys(&mut s, "terminal-rs", "terminal"));
|
||||
assert_eq!(s.get("terminal/settings.json").as_deref(), Some("newer settings"), "newer data wins");
|
||||
assert_eq!(s.get("terminal/layouts/Desk.json").as_deref(), Some("desk"));
|
||||
assert_eq!(s.get("terminal-rs/layouts/Desk.json").as_deref(), Some("desk"), "the old keys stay");
|
||||
assert_eq!(s.get("other-site-key").as_deref(), Some("untouched"));
|
||||
// A layout deleted after the copy doesn't come back on the next load.
|
||||
s.remove("terminal/layouts/Desk.json");
|
||||
assert!(copy_keys(&mut s, "terminal-rs", "terminal"));
|
||||
assert_eq!(s.get("terminal/layouts/Desk.json"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn copy_keys_undoes_a_partial_copy() {
|
||||
let mut s = Keys::new(&[("terminal-rs/a.json", "a"), ("terminal-rs/b.json", "b")]);
|
||||
s.room = 3;
|
||||
assert!(!copy_keys(&mut s, "terminal-rs", "terminal"), "storage is full: keep using the old keys");
|
||||
assert_eq!(s.keys(), ["terminal-rs/a.json", "terminal-rs/b.json"]);
|
||||
s.room = usize::MAX;
|
||||
assert!(copy_keys(&mut s, "terminal-rs", "terminal"));
|
||||
assert_eq!(s.get("terminal/b.json").as_deref(), Some("b"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_page_left_on_the_old_prefix_follows_another_tabs_copy() {
|
||||
let mut s = Keys::new(&[("terminal-rs/settings.json", "a")]);
|
||||
s.room = 1;
|
||||
// Tab A can't copy, so it keeps the old prefix.
|
||||
assert!(!copy_keys(&mut s, "terminal-rs", "terminal"));
|
||||
let key = |s: &Keys, k: &str| redirect(s, "terminal-rs", "terminal", k.into());
|
||||
assert_eq!(key(&s, "terminal-rs/settings.json"), "terminal-rs/settings.json");
|
||||
// Space frees up and tab B finishes the copy; tab A's saves now go where loads read.
|
||||
s.room = usize::MAX;
|
||||
assert!(copy_keys(&mut s, "terminal-rs", "terminal"));
|
||||
assert_eq!(key(&s, "terminal-rs/settings.json"), "terminal/settings.json");
|
||||
assert_eq!(key(&s, "terminal-rs/layouts"), "terminal/layouts");
|
||||
assert_eq!(key(&s, "terminal-rs"), "terminal");
|
||||
assert_eq!(key(&s, "terminal-rsx/a.json"), "terminal-rsx/a.json", "only that prefix");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn copy_keys_with_nothing_saved_writes_nothing() {
|
||||
let mut s = Keys::new(&[]);
|
||||
assert!(copy_keys(&mut s, "terminal-rs", "terminal"));
|
||||
assert!(s.keys().is_empty());
|
||||
}
|
||||
}
|
||||
+1
-12
@@ -3,10 +3,6 @@
|
||||
//!
|
||||
//! Query flags: `?sample` serves offline sample data only (like `TR_PROVIDER=sample`), and
|
||||
//! `?fps` shows the frame-time readout and repaints continuously (like `TR_FPS`).
|
||||
//!
|
||||
//! Crypto markets come from the Terminal server that `TR_SERVER_URL` named when the page was built
|
||||
//! (scripts/build-web.sh). `?server=off` turns it off, and `?server=ws://localhost:8787/ws` uses a
|
||||
//! server on this machine instead (see `tr_data::web::server_url`).
|
||||
|
||||
use std::cell::Cell;
|
||||
|
||||
@@ -23,13 +19,6 @@ pub fn start() {
|
||||
let query = web_sys::window().and_then(|w| w.location().search().ok()).unwrap_or_default();
|
||||
let offline = has_flag(&query, "sample");
|
||||
let perf = has_flag(&query, "fps");
|
||||
let requested = web_sys::UrlSearchParams::new_with_str(&query).ok().and_then(|p| p.get("server"));
|
||||
let server = tr_data::web::server_url(option_env!("TR_SERVER_URL"), requested.as_deref());
|
||||
if let Some(url) = server.as_deref().filter(|_| !offline) {
|
||||
tracing::info!(
|
||||
"crypto markets come from the Terminal server at {url}, or Binance and Coinbase while it's down"
|
||||
);
|
||||
}
|
||||
wasm_bindgen_futures::spawn_local(async move {
|
||||
let Some(canvas) = canvas() else { return fail("the page has no #terminal canvas") };
|
||||
let started = eframe::WebRunner::new()
|
||||
@@ -40,7 +29,7 @@ pub fn start() {
|
||||
// The first frame needs fonts; set them before it (set_fonts applies from the next pass).
|
||||
cc.egui_ctx.set_fonts(crate::theme::base_fonts());
|
||||
wasm_bindgen_futures::spawn_local(load_emoji_fonts(cc.egui_ctx.clone()));
|
||||
let mut app = crate::create_app(cc, tr_data::for_browser(offline, server.as_deref()), 1, t0)?;
|
||||
let mut app = crate::create_app(cc, tr_data::for_browser(offline), 1, t0)?;
|
||||
if perf {
|
||||
app.show_perf();
|
||||
}
|
||||
|
||||
+24
-234
@@ -34,9 +34,6 @@ pub type PaneId = u64;
|
||||
/// Minutes between Congress disclosure checks (`TR_CONGRESS_REFRESH_MIN`; 0 disables).
|
||||
pub const CONGRESS_REFRESH_MIN: u64 = 30;
|
||||
|
||||
/// The least time between a chart's reloads after its provider ended the live feed.
|
||||
const RELOAD_AFTER_FEED_ENDS: Duration = Duration::from_secs(5);
|
||||
|
||||
pub enum Request {
|
||||
/// Load the brokerage portfolio through SnapTrade (`None` = the simulated sample).
|
||||
Portfolio(Option<tr_broker::Credentials>),
|
||||
@@ -241,9 +238,6 @@ pub enum Response {
|
||||
prepended: usize,
|
||||
from_cache: bool,
|
||||
exhausted: bool,
|
||||
/// A full load from the provider. Its `exhausted` replaces the chart's, which may have
|
||||
/// come from the source the chart showed before.
|
||||
full: bool,
|
||||
/// A live-feed update of the latest bar.
|
||||
live: bool,
|
||||
},
|
||||
@@ -311,9 +305,6 @@ struct Shared {
|
||||
/// Authoritative bars per chart pane (history, paging and live ticks all merge here),
|
||||
/// tagged with the pane's load generation.
|
||||
series: std::sync::Mutex<SeriesMap>,
|
||||
/// Counts each chart pane's reloads for a changed data source, up once as one starts and again
|
||||
/// when it's done (odd while it runs), so a page of older bars that spans one is dropped.
|
||||
sources: std::sync::Mutex<HashMap<PaneId, u64>>,
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
cache: std::sync::Mutex<Option<BarCache>>,
|
||||
db: Mutex<Db>,
|
||||
@@ -329,30 +320,23 @@ impl Shared {
|
||||
}
|
||||
|
||||
/// Where saved files live. In the browser this is the key prefix in `localStorage` (see
|
||||
/// [`crate::storage`]). The first call moves files saved under the app's old name, terminal-rs,
|
||||
/// to the new one ([`crate::storage::migrate`]).
|
||||
/// [`crate::storage`]).
|
||||
pub fn data_dir() -> PathBuf {
|
||||
static DIR: std::sync::OnceLock<PathBuf> = std::sync::OnceLock::new();
|
||||
DIR.get_or_init(|| {
|
||||
// Unit tests never read, write or move the real folder.
|
||||
if cfg!(test) {
|
||||
return std::env::temp_dir().join(format!("tr-test-data-{}", std::process::id()));
|
||||
}
|
||||
let parent = if cfg!(target_arch = "wasm32") {
|
||||
PathBuf::new()
|
||||
} else if let Some(d) = std::env::var_os("TR_DATA_DIR") {
|
||||
return PathBuf::from(d);
|
||||
} else {
|
||||
let home = std::env::var_os("HOME").map(PathBuf::from).unwrap_or_else(|| PathBuf::from("."));
|
||||
if cfg!(target_os = "macos") {
|
||||
home.join("Library/Application Support")
|
||||
} else {
|
||||
std::env::var_os("XDG_DATA_HOME").map(PathBuf::from).unwrap_or_else(|| home.join(".local/share"))
|
||||
}
|
||||
};
|
||||
crate::storage::migrate(&parent.join("terminal-rs"), &parent.join("terminal"))
|
||||
})
|
||||
.clone()
|
||||
if cfg!(target_arch = "wasm32") {
|
||||
return PathBuf::from("terminal-rs");
|
||||
}
|
||||
if let Some(d) = std::env::var_os("TR_DATA_DIR") {
|
||||
return PathBuf::from(d);
|
||||
}
|
||||
let home = std::env::var_os("HOME").map(PathBuf::from).unwrap_or_else(|| PathBuf::from("."));
|
||||
if cfg!(target_os = "macos") {
|
||||
home.join("Library/Application Support/terminal-rs")
|
||||
} else {
|
||||
std::env::var_os("XDG_DATA_HOME")
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|| home.join(".local/share"))
|
||||
.join("terminal-rs")
|
||||
}
|
||||
}
|
||||
|
||||
impl Worker {
|
||||
@@ -376,7 +360,6 @@ impl Worker {
|
||||
let shared = Arc::new(Shared {
|
||||
provider,
|
||||
series: Default::default(),
|
||||
sources: Default::default(),
|
||||
cache: std::sync::Mutex::new(cache),
|
||||
db: Mutex::new(db),
|
||||
out,
|
||||
@@ -417,14 +400,8 @@ impl Worker {
|
||||
pub fn spawn(ctx: egui::Context, provider: Provider) -> anyhow::Result<Self> {
|
||||
let (tx, mut req_rx) = tokio::sync::mpsc::unbounded_channel::<Request>();
|
||||
let (out, rx) = mpsc::channel();
|
||||
let shared = Arc::new(Shared {
|
||||
provider,
|
||||
series: Default::default(),
|
||||
sources: Default::default(),
|
||||
db: Mutex::new(Db::open_in_memory()?),
|
||||
out,
|
||||
ctx,
|
||||
});
|
||||
let shared =
|
||||
Arc::new(Shared { provider, series: Default::default(), db: Mutex::new(Db::open_in_memory()?), out, ctx });
|
||||
rt::spawn(async move {
|
||||
while let Some(req) = req_rx.recv().await {
|
||||
let s = shared.clone();
|
||||
@@ -587,7 +564,6 @@ async fn handle(s: &Arc<Shared>, req: Request) {
|
||||
prepended: 0,
|
||||
from_cache: true,
|
||||
exhausted: false,
|
||||
full: false,
|
||||
live: false,
|
||||
});
|
||||
}
|
||||
@@ -610,10 +586,9 @@ async fn handle(s: &Arc<Shared>, req: Request) {
|
||||
prepended: 0,
|
||||
from_cache: false,
|
||||
exhausted,
|
||||
full: true,
|
||||
live: false,
|
||||
});
|
||||
start_live(s.clone(), pane, generation, symbol, tf, limit);
|
||||
start_live(s.clone(), pane, generation, symbol, tf);
|
||||
}
|
||||
Ok(_) => s.send(Response::HistoryError { pane, generation, error: format!("no data for {symbol}") }),
|
||||
Err(e) => s.send(Response::HistoryError { pane, generation, error: format!("{e:#}") }),
|
||||
@@ -623,12 +598,6 @@ async fn handle(s: &Arc<Shared>, req: Request) {
|
||||
// Page from the worker's copy (it may hold newer live bars than the UI's).
|
||||
let base = current_series(s, pane, generation).filter(|b| !b.is_empty()).unwrap_or(existing);
|
||||
let Some(first) = base.first().copied() else { return };
|
||||
// A page that overlaps a reload for another source is dropped. The reload answers the
|
||||
// pane, and the chart pages again from the new source.
|
||||
let source = source_switches(s, pane);
|
||||
if source % 2 == 1 {
|
||||
return;
|
||||
}
|
||||
match s.provider.history(&symbol, tf, Some(first.ts), limit).await {
|
||||
Ok(older) => {
|
||||
let older: Vec<Bar> = older.into_iter().filter(|b| b.ts < first.ts).collect();
|
||||
@@ -637,9 +606,6 @@ async fn handle(s: &Arc<Shared>, req: Request) {
|
||||
let mut map = s.series.lock().expect("series lock");
|
||||
let Some((g, cur)) = map.get_mut(&pane).filter(|(g, _)| *g == generation) else { return };
|
||||
let _ = g;
|
||||
if source_switches(s, pane) != source {
|
||||
return;
|
||||
}
|
||||
let mut merged = older;
|
||||
merged.extend_from_slice(cur);
|
||||
*cur = Arc::new(merged);
|
||||
@@ -656,14 +622,10 @@ async fn handle(s: &Arc<Shared>, req: Request) {
|
||||
prepended,
|
||||
from_cache: false,
|
||||
exhausted: prepended == 0,
|
||||
full: false,
|
||||
live: false,
|
||||
});
|
||||
}
|
||||
Err(e) if source_switches(s, pane) == source => {
|
||||
s.send(Response::HistoryError { pane, generation, error: format!("{e:#}") })
|
||||
}
|
||||
Err(_) => {}
|
||||
Err(e) => s.send(Response::HistoryError { pane, generation, error: format!("{e:#}") }),
|
||||
}
|
||||
}
|
||||
Request::Indicators { pane, generation, bars, tf, specs, palette } => {
|
||||
@@ -859,53 +821,24 @@ fn store_series(s: &Shared, pane: PaneId, generation: u64, bars: Arc<Vec<Bar>>)
|
||||
}
|
||||
}
|
||||
|
||||
fn source_switches(s: &Shared, pane: PaneId) -> u64 {
|
||||
s.sources.lock().expect("sources lock").get(&pane).copied().unwrap_or(0)
|
||||
}
|
||||
|
||||
fn current_series(s: &Shared, pane: PaneId, generation: u64) -> Option<Arc<Vec<Bar>>> {
|
||||
s.series.lock().expect("series lock").get(&pane).filter(|(g, _)| *g == generation).map(|(_, b)| b.clone())
|
||||
}
|
||||
|
||||
/// Merge the provider's live feed (if any) into the pane's series, publishing at most
|
||||
/// four updates a second. Stops when the pane reloads, closes or the feed ends.
|
||||
/// What one live bar did to a series.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||
struct Applied {
|
||||
changed: bool,
|
||||
/// It went in before the latest bar (a feed filling a gap after an outage), so every
|
||||
/// series aligned to the bars by index (compare lines) has to be rebuilt.
|
||||
inserted: bool,
|
||||
}
|
||||
|
||||
fn apply_live_bar(bars: &mut Vec<Bar>, bar: Bar) -> Applied {
|
||||
let (len, last) = (bars.len(), bars.last().map(|b| b.ts));
|
||||
let changed = Series::apply_live(bars, bar);
|
||||
Applied { changed, inserted: bars.len() > len && bars.last().map(|b| b.ts) == last }
|
||||
}
|
||||
|
||||
fn start_live(s: Arc<Shared>, pane: PaneId, generation: u64, symbol: String, tf: Timeframe, limit: usize) {
|
||||
fn start_live(s: Arc<Shared>, pane: PaneId, generation: u64, symbol: String, tf: Timeframe) {
|
||||
let Some(mut feed) = s.provider.live(&symbol, tf) else { return };
|
||||
rt::spawn(async move {
|
||||
let started = web_time::Instant::now();
|
||||
let mut tick = rt::interval(Duration::from_millis(250));
|
||||
let mut dirty = false;
|
||||
let mut inserted = false;
|
||||
let mut ended = false;
|
||||
loop {
|
||||
tokio::select! {
|
||||
bar = feed.recv() => {
|
||||
let Some(bar) = bar else {
|
||||
ended = true;
|
||||
break;
|
||||
};
|
||||
let Some(bar) = bar else { break };
|
||||
let mut map = s.series.lock().expect("series lock");
|
||||
match map.get_mut(&pane) {
|
||||
Some((g, bars)) if *g == generation => {
|
||||
let a = apply_live_bar(Arc::make_mut(bars), bar);
|
||||
dirty |= a.changed;
|
||||
inserted |= a.inserted;
|
||||
}
|
||||
Some((g, bars)) if *g == generation => dirty |= Series::apply_live(Arc::make_mut(bars), bar),
|
||||
_ => break,
|
||||
}
|
||||
}
|
||||
@@ -929,26 +862,12 @@ fn start_live(s: Arc<Shared>, pane: PaneId, generation: u64, symbol: String, tf:
|
||||
prepended: 0,
|
||||
from_cache: false,
|
||||
exhausted: false,
|
||||
full: false,
|
||||
// A bar slotted into a gap shifts everything after it: send it as a
|
||||
// refresh so the chart rebuilds its compare line and indicators.
|
||||
live: !std::mem::take(&mut inserted),
|
||||
live: true,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
tracing::debug!("live feed for pane {pane} ({symbol}) stopped");
|
||||
// A provider ends a feed the pane still shows when the chart's source changes (the Terminal
|
||||
// server went away or came back): reload the chart from wherever its data comes from now.
|
||||
if ended {
|
||||
rt::sleep(RELOAD_AFTER_FEED_ENDS.saturating_sub(started.elapsed())).await;
|
||||
if current_series(&s, pane, generation).is_some() {
|
||||
let switch = || *s.sources.lock().expect("sources lock").entry(pane).or_default() += 1;
|
||||
switch();
|
||||
handle(&s, Request::History { pane, generation, symbol, tf, limit }).await;
|
||||
switch();
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1024,20 +943,6 @@ where
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn bars_filling_a_gap_are_reported_as_inserted() {
|
||||
let b = |ts: i64| Bar::new(ts, 1.0, 1.0, 1.0, 1.0, 1.0);
|
||||
let mut bars = vec![b(0), b(60), b(240)];
|
||||
assert_eq!(apply_live_bar(&mut bars, b(300)), Applied { changed: true, inserted: false });
|
||||
assert_eq!(apply_live_bar(&mut bars, b(300)), Applied { changed: false, inserted: false });
|
||||
assert_eq!(apply_live_bar(&mut bars, b(120)), Applied { changed: true, inserted: true });
|
||||
assert_eq!(
|
||||
apply_live_bar(&mut bars, Bar::new(60, 1.0, 2.0, 1.0, 2.0, 1.0)),
|
||||
Applied { changed: true, inserted: false }
|
||||
);
|
||||
assert_eq!(bars.iter().map(|x| x.ts).collect::<Vec<_>>(), [0, 60, 120, 240, 300]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compare_alignment_carries_last_close() {
|
||||
let base: Vec<Bar> = (0..5).map(|i| Bar::new(i * 10, 100.0, 100.0, 100.0, 100.0 + i as f64, 1.0)).collect();
|
||||
@@ -1074,7 +979,6 @@ mod tests {
|
||||
let s = Shared {
|
||||
provider: std::sync::Arc::new(tr_data::sample::SampleProvider::at(0, 10)),
|
||||
series: Default::default(),
|
||||
sources: Default::default(),
|
||||
cache: std::sync::Mutex::new(None),
|
||||
db: Mutex::new(Db::open_in_memory().unwrap()),
|
||||
out,
|
||||
@@ -1089,120 +993,6 @@ mod tests {
|
||||
assert!(current_series(&s, 1, 1).is_none());
|
||||
}
|
||||
|
||||
/// Bars whose close counts the loads. Reloads wait for `reload`, and older pages for `page`.
|
||||
/// The first chart's live feed ends at once, the way `RemoteMarkets` ends one when the chart's
|
||||
/// source changes.
|
||||
#[derive(Default)]
|
||||
struct SourceChanges {
|
||||
loads: std::sync::atomic::AtomicUsize,
|
||||
open: std::sync::Mutex<Vec<tokio::sync::mpsc::Sender<Bar>>>,
|
||||
reload: tokio::sync::Notify,
|
||||
page: tokio::sync::Notify,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl tr_data::MarketData for SourceChanges {
|
||||
fn name(&self) -> &str {
|
||||
"test"
|
||||
}
|
||||
|
||||
async fn history(&self, _: &str, _: Timeframe, end: Option<i64>, _: usize) -> anyhow::Result<Vec<Bar>> {
|
||||
if end.is_some() {
|
||||
self.page.notified().await;
|
||||
return Ok(vec![Bar::new(-60, 1.0, 1.0, 1.0, 1.0, 1.0)]);
|
||||
}
|
||||
let n = self.loads.fetch_add(1, std::sync::atomic::Ordering::SeqCst) + 1;
|
||||
if n > 1 {
|
||||
self.reload.notified().await;
|
||||
}
|
||||
Ok(vec![Bar::new(0, 1.0, 1.0, 1.0, n as f64, 1.0)])
|
||||
}
|
||||
|
||||
fn live(&self, _: &str, _: Timeframe) -> Option<tr_data::LiveFeed> {
|
||||
let (tx, rx) = tokio::sync::mpsc::channel(1);
|
||||
if self.loads.load(std::sync::atomic::Ordering::SeqCst) > 1 {
|
||||
self.open.lock().unwrap().push(tx);
|
||||
}
|
||||
Some(rx)
|
||||
}
|
||||
}
|
||||
|
||||
/// A worker on `SourceChanges` with pane 1 loading `BTCUSD Curncy`, and its responses.
|
||||
async fn switching_chart() -> (Arc<Shared>, mpsc::Receiver<Response>, Arc<SourceChanges>) {
|
||||
let (out, rx) = mpsc::channel();
|
||||
let provider = Arc::new(SourceChanges::default());
|
||||
let s = Arc::new(Shared {
|
||||
provider: provider.clone(),
|
||||
series: Default::default(),
|
||||
sources: Default::default(),
|
||||
cache: std::sync::Mutex::new(None),
|
||||
db: Mutex::new(Db::open_in_memory().unwrap()),
|
||||
out,
|
||||
ctx: egui::Context::default(),
|
||||
});
|
||||
let symbol = "BTCUSD Curncy".to_string();
|
||||
handle(&s, Request::History { pane: 1, generation: 1, symbol, tf: Timeframe::D1, limit: 10 }).await;
|
||||
(s, rx, provider)
|
||||
}
|
||||
|
||||
/// The last close of each chart update, until there are `n`.
|
||||
async fn closes(rx: &mpsc::Receiver<Response>, n: usize) -> Vec<f64> {
|
||||
let t0 = std::time::Instant::now();
|
||||
let mut closes = Vec::new();
|
||||
while closes.len() < n {
|
||||
assert!(t0.elapsed() < Duration::from_secs(15), "no reload: {closes:?}");
|
||||
match rx.try_recv() {
|
||||
Ok(Response::History { generation: 1, bars, .. }) => closes.push(bars.last().unwrap().close),
|
||||
Ok(_) => {}
|
||||
Err(_) => tokio::time::sleep(Duration::from_millis(50)).await,
|
||||
}
|
||||
}
|
||||
closes
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn charts_reload_when_the_provider_ends_their_live_feed() {
|
||||
let t0 = std::time::Instant::now();
|
||||
let (_s, rx, provider) = switching_chart().await;
|
||||
provider.reload.notify_one();
|
||||
assert_eq!(closes(&rx, 2).await, [1.0, 2.0], "the reload replaced the chart");
|
||||
assert!(t0.elapsed() >= RELOAD_AFTER_FEED_ENDS, "not before the pause");
|
||||
assert_eq!(provider.loads.load(std::sync::atomic::Ordering::SeqCst), 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pages_that_overlap_a_source_switch_are_dropped() {
|
||||
let (s, rx, provider) = switching_chart().await;
|
||||
let more = || Request::MoreHistory {
|
||||
pane: 1,
|
||||
generation: 1,
|
||||
symbol: "BTCUSD Curncy".into(),
|
||||
tf: Timeframe::D1,
|
||||
existing: Arc::new(Vec::new()),
|
||||
limit: 10,
|
||||
};
|
||||
// One page starts before the switch, and its bars arrive after it.
|
||||
let before = tokio::spawn({
|
||||
let (s, req) = (s.clone(), more());
|
||||
async move { handle(&s, req).await }
|
||||
});
|
||||
let t0 = std::time::Instant::now();
|
||||
while source_switches(&s, 1) != 1 {
|
||||
assert!(t0.elapsed() < Duration::from_secs(15), "no reload");
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
}
|
||||
// Another starts while the chart reloads, and gives up at once.
|
||||
let during = tokio::time::timeout(Duration::from_secs(1), handle(&s, more())).await;
|
||||
assert!(during.is_ok(), "a page started during the reload went ahead");
|
||||
provider.reload.notify_one();
|
||||
assert_eq!(closes(&rx, 2).await, [1.0, 2.0]);
|
||||
provider.page.notify_one();
|
||||
before.await.unwrap();
|
||||
let ts: Vec<i64> = current_series(&s, 1, 1).unwrap().iter().map(|b| b.ts).collect();
|
||||
assert_eq!(ts, [0], "the old source's page joined the new source's bars");
|
||||
assert!(rx.try_iter().all(|r| !matches!(r, Response::History { prepended: 1.., .. })));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn join_all_runs_everything() {
|
||||
let rt = tokio::runtime::Builder::new_current_thread().enable_time().build().unwrap();
|
||||
|
||||
@@ -46,7 +46,7 @@ pub struct Connection {
|
||||
pub disabled: bool,
|
||||
pub created: Option<DateTime<Utc>>,
|
||||
pub updated: Option<DateTime<Utc>>,
|
||||
/// Whether the connection allows trading (Terminal only trades when the user enables it).
|
||||
/// Whether the connection allows trading (terminal-rs only trades when the user enables it).
|
||||
pub trading: bool,
|
||||
}
|
||||
|
||||
|
||||
@@ -2,12 +2,9 @@
|
||||
//!
|
||||
//! The partner keys (`clientId`, `consumerKey`) and the per-user `userId`/`userSecret` are
|
||||
//! stored together as one small JSON document: in the macOS Keychain (generic password,
|
||||
//! service `terminal`, account `snaptrade`), or elsewhere in an owner-only (0600) file in
|
||||
//! service `terminal-rs`, account `snaptrade`), or elsewhere in an owner-only (0600) file in
|
||||
//! the app's data directory. `SNAPTRADE_*` environment variables override stored values
|
||||
//! field by field. Nothing is ever written inside the repository.
|
||||
//!
|
||||
//! Before the app was renamed from terminal-rs, the Keychain service was `terminal-rs`; the
|
||||
//! first load moves credentials from there.
|
||||
|
||||
use std::fmt;
|
||||
use std::path::{Path, PathBuf};
|
||||
@@ -15,9 +12,7 @@ use std::path::{Path, PathBuf};
|
||||
use anyhow::{Context, Result};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
pub const KEYCHAIN_SERVICE: &str = "terminal";
|
||||
/// The service credentials were saved under before the rename.
|
||||
pub const LEGACY_KEYCHAIN_SERVICE: &str = "terminal-rs";
|
||||
pub const KEYCHAIN_SERVICE: &str = "terminal-rs";
|
||||
pub const KEYCHAIN_ACCOUNT: &str = "snaptrade";
|
||||
|
||||
/// Environment variables that override stored credentials.
|
||||
@@ -107,8 +102,8 @@ pub fn mask(secret: &str) -> String {
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum Backend {
|
||||
/// macOS Keychain generic password under `service`, moved there from `legacy` on first load.
|
||||
Keychain { service: String, legacy: Option<String> },
|
||||
/// macOS Keychain generic password.
|
||||
Keychain,
|
||||
/// Owner-only JSON file.
|
||||
File(PathBuf),
|
||||
}
|
||||
@@ -124,19 +119,10 @@ impl SecretStore {
|
||||
pub fn platform(data_dir: &Path) -> Self {
|
||||
let file = Backend::File(data_dir.join("secrets").join("snaptrade.json"));
|
||||
let force_file = std::env::var("TR_SECRET_STORE").is_ok_and(|v| v.eq_ignore_ascii_case("file"));
|
||||
let backend = if cfg!(target_os = "macos") && !force_file {
|
||||
Backend::Keychain { service: KEYCHAIN_SERVICE.into(), legacy: Some(LEGACY_KEYCHAIN_SERVICE.into()) }
|
||||
} else {
|
||||
file
|
||||
};
|
||||
let backend = if cfg!(target_os = "macos") && !force_file { Backend::Keychain } else { file };
|
||||
Self { backend }
|
||||
}
|
||||
|
||||
/// The Keychain under `service`, taking over an item left under `legacy`.
|
||||
pub fn keychain(service: &str, legacy: Option<&str>) -> Self {
|
||||
Self { backend: Backend::Keychain { service: service.into(), legacy: legacy.map(Into::into) } }
|
||||
}
|
||||
|
||||
pub fn file(path: impl Into<PathBuf>) -> Self {
|
||||
Self { backend: Backend::File(path.into()) }
|
||||
}
|
||||
@@ -144,14 +130,14 @@ impl SecretStore {
|
||||
/// Where credentials live, for the settings screen.
|
||||
pub fn describe(&self) -> String {
|
||||
match &self.backend {
|
||||
Backend::Keychain { service, .. } => format!("macOS Keychain ({service} / {KEYCHAIN_ACCOUNT})"),
|
||||
Backend::Keychain => format!("macOS Keychain ({KEYCHAIN_SERVICE} / {KEYCHAIN_ACCOUNT})"),
|
||||
Backend::File(p) => format!("{} (owner-only)", p.display()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn load(&self) -> Result<Option<Credentials>> {
|
||||
let text = match &self.backend {
|
||||
Backend::Keychain { service, legacy } => load_keychain(service, legacy.as_deref())?,
|
||||
Backend::Keychain => keychain::get()?,
|
||||
Backend::File(p) => match std::fs::read_to_string(p) {
|
||||
Ok(t) => Some(t),
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => None,
|
||||
@@ -169,7 +155,7 @@ impl SecretStore {
|
||||
pub fn save(&self, creds: &Credentials) -> Result<()> {
|
||||
let json = serde_json::to_string(&creds.trimmed())?;
|
||||
match &self.backend {
|
||||
Backend::Keychain { service, .. } => keychain::set(service, &json),
|
||||
Backend::Keychain => keychain::set(&json),
|
||||
Backend::File(p) => write_private(p, &json),
|
||||
}
|
||||
}
|
||||
@@ -177,11 +163,7 @@ impl SecretStore {
|
||||
/// Remove stored credentials (environment variables are unaffected).
|
||||
pub fn clear(&self) -> Result<()> {
|
||||
match &self.backend {
|
||||
Backend::Keychain { service, legacy } => {
|
||||
keychain::delete(service)?;
|
||||
// The old item too, or the credentials would come back from it on the next load.
|
||||
legacy.as_deref().map_or(Ok(()), keychain::delete)
|
||||
}
|
||||
Backend::Keychain => keychain::delete(),
|
||||
Backend::File(p) => match std::fs::remove_file(p) {
|
||||
Err(e) if e.kind() != std::io::ErrorKind::NotFound => Err(e.into()),
|
||||
_ => Ok(()),
|
||||
@@ -190,24 +172,6 @@ impl SecretStore {
|
||||
}
|
||||
}
|
||||
|
||||
/// Reads the Keychain item under `service`. If there is none, the item under `legacy` moves
|
||||
/// there: it is deleted only after the copy is written, so a failed move leaves it in place.
|
||||
fn load_keychain(service: &str, legacy: Option<&str>) -> Result<Option<String>> {
|
||||
if let Some(text) = keychain::get(service)? {
|
||||
return Ok(Some(text));
|
||||
}
|
||||
let Some(legacy) = legacy else { return Ok(None) };
|
||||
let Some(text) = keychain::get(legacy)? else { return Ok(None) };
|
||||
match keychain::set(service, &text) {
|
||||
Ok(()) => match keychain::delete(legacy) {
|
||||
Ok(()) => tracing::info!("moved SnapTrade credentials from Keychain service {legacy} to {service}"),
|
||||
Err(e) => tracing::warn!("removing SnapTrade credentials from Keychain service {legacy}: {e:#}"),
|
||||
},
|
||||
Err(e) => tracing::warn!("moving SnapTrade credentials to Keychain service {service}: {e:#}"),
|
||||
}
|
||||
Ok(Some(text))
|
||||
}
|
||||
|
||||
/// Write `text` to `path` readable only by the owner (0700 directory, 0600 file), replacing it
|
||||
/// atomically.
|
||||
fn write_private(path: &Path, text: &str) -> Result<()> {
|
||||
@@ -240,26 +204,26 @@ mod keychain {
|
||||
use anyhow::{Context, Result};
|
||||
use security_framework::passwords::{delete_generic_password, get_generic_password, set_generic_password};
|
||||
|
||||
use super::KEYCHAIN_ACCOUNT;
|
||||
use super::{KEYCHAIN_ACCOUNT, KEYCHAIN_SERVICE};
|
||||
|
||||
/// `errSecItemNotFound`.
|
||||
const NOT_FOUND: i32 = -25300;
|
||||
|
||||
pub fn get(service: &str) -> Result<Option<String>> {
|
||||
match get_generic_password(service, KEYCHAIN_ACCOUNT) {
|
||||
pub fn get() -> Result<Option<String>> {
|
||||
match get_generic_password(KEYCHAIN_SERVICE, KEYCHAIN_ACCOUNT) {
|
||||
Ok(bytes) => Ok(Some(String::from_utf8(bytes).context("Keychain item is not UTF-8")?)),
|
||||
Err(e) if e.code() == NOT_FOUND => Ok(None),
|
||||
Err(e) => Err(anyhow::anyhow!("reading the Keychain: {e}")),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn set(service: &str, value: &str) -> Result<()> {
|
||||
set_generic_password(service, KEYCHAIN_ACCOUNT, value.as_bytes())
|
||||
pub fn set(value: &str) -> Result<()> {
|
||||
set_generic_password(KEYCHAIN_SERVICE, KEYCHAIN_ACCOUNT, value.as_bytes())
|
||||
.map_err(|e| anyhow::anyhow!("writing the Keychain: {e}"))
|
||||
}
|
||||
|
||||
pub fn delete(service: &str) -> Result<()> {
|
||||
match delete_generic_password(service, KEYCHAIN_ACCOUNT) {
|
||||
pub fn delete() -> Result<()> {
|
||||
match delete_generic_password(KEYCHAIN_SERVICE, KEYCHAIN_ACCOUNT) {
|
||||
Err(e) if e.code() != NOT_FOUND => Err(anyhow::anyhow!("deleting from the Keychain: {e}")),
|
||||
_ => Ok(()),
|
||||
}
|
||||
@@ -270,13 +234,13 @@ mod keychain {
|
||||
mod keychain {
|
||||
use anyhow::{Result, bail};
|
||||
|
||||
pub fn get(_: &str) -> Result<Option<String>> {
|
||||
pub fn get() -> Result<Option<String>> {
|
||||
bail!("the Keychain is only available on macOS")
|
||||
}
|
||||
pub fn set(_: &str, _: &str) -> Result<()> {
|
||||
pub fn set(_: &str) -> Result<()> {
|
||||
bail!("the Keychain is only available on macOS")
|
||||
}
|
||||
pub fn delete(_: &str) -> Result<()> {
|
||||
pub fn delete() -> Result<()> {
|
||||
bail!("the Keychain is only available on macOS")
|
||||
}
|
||||
}
|
||||
@@ -289,7 +253,7 @@ mod tests {
|
||||
Credentials {
|
||||
client_id: "PARTNER-ID".into(),
|
||||
consumer_key: "consumer-key-0123456789".into(),
|
||||
user_id: "terminal-1".into(),
|
||||
user_id: "terminal-rs-1".into(),
|
||||
user_secret: "user-secret-abcdef".into(),
|
||||
}
|
||||
}
|
||||
@@ -349,31 +313,4 @@ mod tests {
|
||||
assert_eq!(store.load().unwrap(), None);
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
/// Writes throwaway items (dummy credentials) to the login Keychain, so it runs only when
|
||||
/// asked: `cargo test -p tr-broker -- --ignored keychain`.
|
||||
#[cfg(target_os = "macos")]
|
||||
#[test]
|
||||
#[ignore = "writes to the login Keychain"]
|
||||
fn keychain_moves_credentials_from_the_old_service() {
|
||||
let pid = std::process::id();
|
||||
let (service, legacy) = (format!("terminal-test-{pid}"), format!("terminal-rs-test-{pid}"));
|
||||
let store = SecretStore::keychain(&service, Some(&legacy));
|
||||
let new_only = SecretStore::keychain(&service, None);
|
||||
let old_only = SecretStore::keychain(&legacy, None);
|
||||
old_only.save(&creds()).unwrap();
|
||||
|
||||
assert_eq!(store.load().unwrap(), Some(creds()), "found under the old service");
|
||||
assert_eq!(new_only.load().unwrap(), Some(creds()), "and moved to the new one");
|
||||
assert_eq!(old_only.load().unwrap(), None, "the old item is gone");
|
||||
|
||||
// A stale item under the old service never replaces what the new one holds.
|
||||
old_only.save(&Credentials { client_id: "STALE".into(), ..creds() }).unwrap();
|
||||
assert_eq!(store.load().unwrap(), Some(creds()));
|
||||
|
||||
// Forgetting credentials removes both, so the old item can't bring them back.
|
||||
store.clear().unwrap();
|
||||
assert_eq!(old_only.load().unwrap(), None);
|
||||
assert_eq!(store.load().unwrap(), None);
|
||||
}
|
||||
}
|
||||
@@ -262,7 +262,7 @@ impl SnapTrade {
|
||||
bail!("SnapTrade client ID and consumer key are not set");
|
||||
}
|
||||
let http = reqwest::Client::builder()
|
||||
.user_agent(concat!("terminal/", env!("CARGO_PKG_VERSION")))
|
||||
.user_agent(concat!("terminal-rs/", env!("CARGO_PKG_VERSION")))
|
||||
.timeout(Duration::from_secs(30))
|
||||
.build()?;
|
||||
let base = Url::parse(base.trim_end_matches('/')).context("invalid SnapTrade base URL")?;
|
||||
|
||||
@@ -93,7 +93,7 @@ async fn mock(seen: Arc<Mutex<Vec<Seen>>>, failures: Arc<Mutex<Vec<String>>>) ->
|
||||
|| path.starts_with("/connection/")
|
||||
|| path == "/snapTrade/login";
|
||||
if user_scoped
|
||||
&& (param("userId").as_deref() != Some("terminal-test")
|
||||
&& (param("userId").as_deref() != Some("terminal-rs-test")
|
||||
|| param("userSecret").as_deref() != Some("s3cr3t/+="))
|
||||
{
|
||||
fail("missing or wrong user credentials".into());
|
||||
@@ -156,7 +156,7 @@ fn creds() -> Credentials {
|
||||
Credentials {
|
||||
client_id: "TEST-CLIENT".into(),
|
||||
consumer_key: CONSUMER_KEY.into(),
|
||||
user_id: "terminal-test".into(),
|
||||
user_id: "terminal-rs-test".into(),
|
||||
user_secret: "s3cr3t/+=".into(),
|
||||
}
|
||||
}
|
||||
@@ -244,8 +244,8 @@ async fn registers_a_user_opens_the_portal_and_disconnects() {
|
||||
let partner =
|
||||
SnapTrade::with_base(Credentials { user_id: String::new(), user_secret: String::new(), ..creds() }, &base)
|
||||
.unwrap();
|
||||
let (id, secret) = partner.register_user("terminal-test").await.unwrap();
|
||||
assert_eq!((id.as_str(), secret.as_str()), ("terminal-test", "fresh-secret"));
|
||||
let (id, secret) = partner.register_user("terminal-rs-test").await.unwrap();
|
||||
assert_eq!((id.as_str(), secret.as_str()), ("terminal-rs-test", "fresh-secret"));
|
||||
|
||||
let client = SnapTrade::with_base(creds(), &base).unwrap();
|
||||
let url = client.connection_portal_url(true, None).await.unwrap();
|
||||
|
||||
@@ -58,7 +58,7 @@ fn client(base: &str) -> SnapTrade {
|
||||
let creds = Credentials {
|
||||
client_id: "TEST".into(),
|
||||
consumer_key: "key".into(),
|
||||
user_id: "terminal-test".into(),
|
||||
user_id: "terminal-rs-test".into(),
|
||||
user_secret: "s".into(),
|
||||
};
|
||||
SnapTrade::with_base(creds, base).unwrap()
|
||||
|
||||
@@ -80,7 +80,7 @@ async fn mock(seen: Log, failures: Log) -> String {
|
||||
let body: Option<Value> = (!body.is_empty()).then(|| serde_json::from_slice(&body).unwrap());
|
||||
let sig = headers.iter().find(|(k, _)| k == "signature").map(|(_, v)| v.clone()).unwrap_or_default();
|
||||
let ok = sig == signature(CONSUMER_KEY, path, query, body.as_ref())
|
||||
&& param(query, "userId").as_deref() == Some("terminal-test")
|
||||
&& param(query, "userId").as_deref() == Some("terminal-rs-test")
|
||||
&& param(query, "userSecret").as_deref() == Some("s3cr3t");
|
||||
let entry = Seen { method: method.clone(), path: path.to_string(), body: body.clone() };
|
||||
if !ok {
|
||||
@@ -161,7 +161,7 @@ fn client(base: &str) -> SnapTrade {
|
||||
let creds = Credentials {
|
||||
client_id: "TEST-CLIENT".into(),
|
||||
consumer_key: CONSUMER_KEY.into(),
|
||||
user_id: "terminal-test".into(),
|
||||
user_id: "terminal-rs-test".into(),
|
||||
user_secret: "s3cr3t".into(),
|
||||
};
|
||||
SnapTrade::with_base(creds, base).unwrap()
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//! GPU chart rendering for Terminal.
|
||||
//! GPU chart rendering for terminal-rs.
|
||||
//!
|
||||
//! - [`viewport`]: bar-index/price <-> pixel math, pan and zoom.
|
||||
//! - [`lod`]: level-of-detail pyramids that bound per-frame work to the plot width.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "tr-congress"
|
||||
description = "STOCK Act disclosure ingest (Senate eFD + House Clerk PTRs) for Terminal"
|
||||
description = "STOCK Act disclosure ingest (Senate eFD + House Clerk PTRs) for terminal-rs"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
|
||||
@@ -387,7 +387,7 @@ impl HttpSource {
|
||||
pub fn new() -> Result<Self> {
|
||||
let client = reqwest::Client::builder()
|
||||
.user_agent(concat!(
|
||||
"terminal/",
|
||||
"terminal-rs/",
|
||||
env!("CARGO_PKG_VERSION"),
|
||||
" (STOCK Act disclosure reader; +https://github.com/)"
|
||||
))
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "tr-core"
|
||||
description = "Core market types, timeframes, indicators and sample data for Terminal"
|
||||
description = "Core market types, timeframes, indicators and sample data for terminal-rs"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
|
||||
+10
-24
@@ -89,24 +89,18 @@ impl Series {
|
||||
i.checked_sub(1)
|
||||
}
|
||||
|
||||
/// Apply one live bar by its `ts`: absorb it into the bar of the same period, append it if
|
||||
/// newer, or insert it where a period is missing (a feed backfilling a gap after an outage).
|
||||
/// Bars older than the whole series are ignored; paging loads those. Returns true if the
|
||||
/// series changed.
|
||||
/// Apply one live bar: absorb it into the last bar if it is the same period, append it
|
||||
/// if newer, ignore it if older. Returns true if the series changed.
|
||||
pub fn apply_live(bars: &mut Vec<Bar>, live: Bar) -> bool {
|
||||
if bars.last().is_none_or(|last| last.ts < live.ts) {
|
||||
bars.push(live);
|
||||
return true;
|
||||
}
|
||||
match bars.binary_search_by_key(&live.ts, |b| b.ts) {
|
||||
Ok(i) => {
|
||||
let before = bars[i];
|
||||
bars[i].absorb(&live);
|
||||
bars[i] != before
|
||||
match bars.last_mut() {
|
||||
Some(last) if last.ts == live.ts => {
|
||||
let before = *last;
|
||||
last.absorb(&live);
|
||||
*last != before
|
||||
}
|
||||
Err(0) => false,
|
||||
Err(i) => {
|
||||
bars.insert(i, live);
|
||||
Some(last) if last.ts > live.ts => false,
|
||||
_ => {
|
||||
bars.push(live);
|
||||
true
|
||||
}
|
||||
}
|
||||
@@ -154,14 +148,6 @@ mod tests {
|
||||
assert!(!Series::apply_live(&mut v, Bar::new(0, 1.0, 1.0, 1.0, 1.0, 1.0)));
|
||||
assert!(Series::apply_live(&mut v, Bar::new(120, 11.5, 11.6, 11.4, 11.6, 5.0)));
|
||||
assert_eq!(v.len(), 2);
|
||||
// A feed backfilling after an outage: a missing period is inserted, an older bar it
|
||||
// still holds is completed, and nothing older than the series is added.
|
||||
assert!(Series::apply_live(&mut v, Bar::new(240, 12.0, 12.0, 12.0, 12.0, 1.0)));
|
||||
assert!(Series::apply_live(&mut v, Bar::new(180, 11.6, 11.9, 11.5, 11.8, 7.0)));
|
||||
assert!(Series::apply_live(&mut v, Bar::new(120, 11.5, 11.7, 11.4, 11.65, 6.0)));
|
||||
assert_eq!(v.iter().map(|b| b.ts).collect::<Vec<_>>(), [60, 120, 180, 240]);
|
||||
assert_eq!((v[1].high, v[1].close, v[1].volume), (11.7, 11.65, 6.0));
|
||||
assert!(!Series::apply_live(&mut v, Bar::new(180, 11.6, 11.9, 11.5, 11.8, 7.0)), "no change");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//! Core market types shared across Terminal: bars, timeframes, securities and
|
||||
//! Core market types shared across terminal-rs: bars, timeframes, securities and
|
||||
//! market keys, the security catalog, technical indicators and sample data.
|
||||
|
||||
pub mod bar;
|
||||
|
||||
@@ -6,7 +6,6 @@ edition.workspace = true
|
||||
|
||||
[dependencies]
|
||||
tr-core.workspace = true
|
||||
tr-proto.workspace = true
|
||||
anyhow.workspace = true
|
||||
chrono.workspace = true
|
||||
serde.workspace = true
|
||||
|
||||
@@ -196,7 +196,7 @@ impl CryptoProvider {
|
||||
let client = reqwest::Client::builder();
|
||||
// Browsers set their own User-Agent, and fetch takes its timeout per request.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
let client = client.user_agent(concat!("terminal/", env!("CARGO_PKG_VERSION"))).timeout(TIMEOUT);
|
||||
let client = client.user_agent(concat!("terminal-rs/", env!("CARGO_PKG_VERSION"))).timeout(TIMEOUT);
|
||||
Ok(Self { client: client.build()? })
|
||||
}
|
||||
|
||||
|
||||
+8
-31
@@ -12,14 +12,9 @@
|
||||
//! The router is wrapped in [`Fallback`], which serves sample data if a source is
|
||||
//! unreachable, so the terminal always has something to draw.
|
||||
//!
|
||||
//! `TR_SERVER_URL` (`ws://` or `wss://`) opts in to the Terminal data server
|
||||
//! ([`remote::RemoteMarkets`]): crypto markets then come from Hyperliquid perps through the
|
||||
//! server, and everything else, or anything the server can't serve, from the sources above.
|
||||
//!
|
||||
//! The browser build (`wasm32`) can only reach sources that allow cross-origin requests:
|
||||
//! [`for_browser`] serves live crypto through [`web::WebMarkets`] and sample data for the
|
||||
//! rest. It has no SQLite bar cache. When the page was built with a Terminal server URL
|
||||
//! ([`web::server_url`]), crypto comes through the server first, as on the desktop.
|
||||
//! rest. It has no SQLite bar cache.
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
pub mod alpaca;
|
||||
@@ -28,7 +23,6 @@ pub mod cache;
|
||||
pub mod crypto;
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
pub mod markets;
|
||||
pub mod remote;
|
||||
pub mod rss;
|
||||
pub mod rt;
|
||||
pub mod sample;
|
||||
@@ -202,9 +196,7 @@ pub struct OptionChain {
|
||||
}
|
||||
|
||||
/// Live bar updates for one symbol and timeframe: each message is the latest state of
|
||||
/// the current (possibly still forming) bar, or, after an outage, an older bar that fills the
|
||||
/// gap ([`tr_core::Series::apply_live`] slots it in). The feed stops when the receiver is
|
||||
/// dropped.
|
||||
/// the current (possibly still forming) bar. The feed stops when the receiver is dropped.
|
||||
pub type LiveFeed = mpsc::Receiver<Bar>;
|
||||
|
||||
/// Pick curve snapshots from per-date rows: latest, nearest at or before -30d and -365d.
|
||||
@@ -420,33 +412,18 @@ pub fn from_env() -> Provider {
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
let markets = match markets::Markets::new(alpaca) {
|
||||
Ok(m) => Fallback::new(m),
|
||||
Err(_) => return Arc::new(sample::SampleProvider::default().with_live_ticks()),
|
||||
};
|
||||
match var("TR_SERVER_URL") {
|
||||
Some(url) => {
|
||||
// The URL itself stays out of the logs.
|
||||
tracing::info!("TR_SERVER_URL is set: crypto markets come through the Terminal server");
|
||||
Arc::new(remote::RemoteMarkets::new(url.trim(), markets))
|
||||
}
|
||||
None => Arc::new(markets),
|
||||
match markets::Markets::new(alpaca) {
|
||||
Ok(m) => Arc::new(Fallback::new(m)),
|
||||
Err(_) => Arc::new(sample::SampleProvider::default().with_live_ticks()),
|
||||
}
|
||||
}
|
||||
|
||||
/// The browser build's provider: sample data only when `offline`, otherwise live crypto
|
||||
/// with sample data for everything else. With a Terminal `server` URL, crypto comes through
|
||||
/// that server, and from Binance and Coinbase whenever the server can't serve it.
|
||||
pub fn for_browser(offline: bool, server: Option<&str>) -> Provider {
|
||||
/// with sample data for everything else.
|
||||
pub fn for_browser(offline: bool) -> Provider {
|
||||
if !offline {
|
||||
match web::WebMarkets::new() {
|
||||
Ok(m) => {
|
||||
let m = Fallback::new(m);
|
||||
return match server {
|
||||
Some(url) => Arc::new(remote::RemoteMarkets::new(url, m)),
|
||||
None => Arc::new(m),
|
||||
};
|
||||
}
|
||||
Ok(m) => return Arc::new(Fallback::new(m)),
|
||||
Err(e) => tracing::warn!("crypto sources unavailable, using sample data: {e:#}"),
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
+8
-51
@@ -37,51 +37,16 @@ pub async fn sleep(d: Duration) {
|
||||
tokio::time::sleep(d).await;
|
||||
}
|
||||
|
||||
/// Resolves after `d`, through a `setTimeout` set on the first poll and cleared if the sleep is
|
||||
/// dropped before it fires.
|
||||
/// Resolves after `d`, through `setTimeout`.
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
pub fn sleep(d: Duration) -> impl Future<Output = ()> + Send {
|
||||
struct Timer(Option<i32>);
|
||||
impl Drop for Timer {
|
||||
fn drop(&mut self) {
|
||||
if let (Some(id), Some(w)) = (self.0, web_sys::window()) {
|
||||
w.clear_timeout_with_handle(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
pub async fn sleep(d: Duration) {
|
||||
let ms = d.as_millis().min(i32::MAX as u128) as i32;
|
||||
// A promise isn't `Send`. The web build has one thread, so wrapping it is sound, and the provider
|
||||
// traits' `Send` futures can then wait on it.
|
||||
send_wrapper::SendWrapper::new(async move {
|
||||
let mut timer = Timer(None);
|
||||
let promise = js_sys::Promise::new(&mut |resolve, _| {
|
||||
timer.0 = web_sys::window()
|
||||
.and_then(|w| w.set_timeout_with_callback_and_timeout_and_arguments_0(&resolve, ms).ok());
|
||||
});
|
||||
let _ = wasm_bindgen_futures::JsFuture::from(promise).await;
|
||||
timer.0 = None; // it fired
|
||||
})
|
||||
}
|
||||
|
||||
/// `fut` ran past its deadline.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct Elapsed;
|
||||
|
||||
impl std::fmt::Display for Elapsed {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str("deadline has elapsed")
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for Elapsed {}
|
||||
|
||||
/// Run `fut` for at most `d`.
|
||||
pub async fn timeout<F: Future>(d: Duration, fut: F) -> Result<F::Output, Elapsed> {
|
||||
let (fut, deadline) = (std::pin::pin!(fut), std::pin::pin!(sleep(d)));
|
||||
match futures::future::select(fut, deadline).await {
|
||||
futures::future::Either::Left((out, _)) => Ok(out),
|
||||
futures::future::Either::Right(_) => Err(Elapsed),
|
||||
}
|
||||
let promise = js_sys::Promise::new(&mut |resolve, _| {
|
||||
if let Some(w) = web_sys::window() {
|
||||
let _ = w.set_timeout_with_callback_and_timeout_and_arguments_0(&resolve, ms);
|
||||
}
|
||||
});
|
||||
let _ = wasm_bindgen_futures::JsFuture::from(promise).await;
|
||||
}
|
||||
|
||||
/// A periodic timer: the first tick completes at once, later ones every `period`. A late
|
||||
@@ -121,12 +86,4 @@ mod tests {
|
||||
assert!(t0.elapsed() >= Duration::from_millis(40));
|
||||
assert_eq!(blocking(|| 6 * 7).await, Some(42));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn timeout_stops_at_the_deadline() {
|
||||
assert_eq!(timeout(Duration::from_secs(5), async { 7 }).await, Ok(7));
|
||||
let t0 = Instant::now();
|
||||
assert_eq!(timeout(Duration::from_millis(20), std::future::pending::<()>()).await, Err(Elapsed));
|
||||
assert!(t0.elapsed() >= Duration::from_millis(20));
|
||||
}
|
||||
}
|
||||
@@ -113,7 +113,7 @@ pub struct TreasuryProvider {
|
||||
impl TreasuryProvider {
|
||||
pub fn new() -> Result<Self> {
|
||||
let client = reqwest::Client::builder()
|
||||
.user_agent("Mozilla/5.0 (compatible; terminal)")
|
||||
.user_agent("Mozilla/5.0 (compatible; terminal-rs)")
|
||||
.timeout(Duration::from_secs(30))
|
||||
.build()?;
|
||||
Ok(Self { client, years: Mutex::new(HashMap::new()) })
|
||||
|
||||
@@ -5,10 +5,6 @@
|
||||
//! Treasury.gov, Alpaca and CoinDesk do not, so every other asset class comes from the
|
||||
//! sample generator and there are no headlines or network search.
|
||||
//!
|
||||
//! When the page has a Terminal server ([`server_url`]), [`crate::for_browser`] puts
|
||||
//! [`crate::remote::RemoteMarkets`] in front of these, so crypto comes from the server and
|
||||
//! from Binance and Coinbase only while the server can't serve it.
|
||||
//!
|
||||
//! TODO(web proxy): route equities, FX, indices, futures, Treasuries and news through a
|
||||
//! same-origin proxy to those sources, then use [`crate::markets::Markets`] here instead.
|
||||
|
||||
@@ -31,38 +27,6 @@ impl WebMarkets {
|
||||
}
|
||||
}
|
||||
|
||||
/// The Terminal server a page uses for crypto markets: `built`, the URL set when the page was
|
||||
/// built, unless the page's `?server=` parameter (`requested`) turns it off (`off`) or names a
|
||||
/// server on the same machine, such as `ws://localhost:8787/ws`. Other servers are refused, so a
|
||||
/// link can't point someone's page at a server that isn't ours.
|
||||
pub fn server_url(built: Option<&str>, requested: Option<&str>) -> Option<String> {
|
||||
let built = built.map(str::trim).filter(|u| !u.is_empty()).map(str::to_string);
|
||||
let Some(requested) = requested.map(str::trim) else { return built };
|
||||
if requested.eq_ignore_ascii_case("off") {
|
||||
return None;
|
||||
}
|
||||
if is_local(requested) {
|
||||
return Some(requested.to_string());
|
||||
}
|
||||
tracing::warn!("?server= takes a server on this machine (ws://localhost:8787/ws) or off; ignoring it");
|
||||
built
|
||||
}
|
||||
|
||||
/// A `ws://` or `wss://` URL whose host is this machine. Anything unusual (user info, other
|
||||
/// spellings of the address) counts as not local.
|
||||
fn is_local(url: &str) -> bool {
|
||||
let Some(rest) = url.strip_prefix("ws://").or_else(|| url.strip_prefix("wss://")) else { return false };
|
||||
let authority = rest.split(['/', '?', '#']).next().unwrap_or_default();
|
||||
if authority.contains('@') {
|
||||
return false;
|
||||
}
|
||||
let host = match authority.strip_prefix('[') {
|
||||
Some(v6) => v6.split(']').next().unwrap_or_default(),
|
||||
None => authority.split(':').next().unwrap_or_default(),
|
||||
};
|
||||
host.eq_ignore_ascii_case("localhost") || host == "127.0.0.1" || host == "::1"
|
||||
}
|
||||
|
||||
fn is_crypto(symbol: &str) -> bool {
|
||||
security(symbol).class == AssetClass::Crypto
|
||||
}
|
||||
@@ -129,35 +93,4 @@ mod tests {
|
||||
assert_eq!(m.yield_curves().await.unwrap().current.points.len(), 11);
|
||||
assert!(is_crypto("BTCUSD Curncy") && !is_crypto("US10Y Govt"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn server_url_from_the_build_or_a_local_override() {
|
||||
let prod = Some("wss://terminal-gateway.style.dev/ws");
|
||||
assert_eq!(server_url(prod, None).as_deref(), prod);
|
||||
assert_eq!(server_url(Some(" "), None), None, "an empty build setting means no server");
|
||||
assert_eq!(server_url(None, None), None);
|
||||
assert_eq!(server_url(prod, Some("off")), None);
|
||||
for local in ["ws://localhost:8787/ws", "ws://127.0.0.1:9", "wss://[::1]:8787/ws", "ws://LOCALHOST/ws"] {
|
||||
assert_eq!(server_url(prod, Some(local)).as_deref(), Some(local));
|
||||
assert_eq!(server_url(None, Some(local)).as_deref(), Some(local));
|
||||
}
|
||||
for refused in [
|
||||
"wss://evil.example/ws",
|
||||
"ws://localhost.evil.example/ws",
|
||||
"ws://[email protected]/ws",
|
||||
"ws://evil.example:80@localhost/ws",
|
||||
"http://localhost:8787/ws",
|
||||
"",
|
||||
] {
|
||||
assert_eq!(server_url(prod, Some(refused)).as_deref(), prod, "{refused}");
|
||||
assert_eq!(server_url(None, Some(refused)), None, "{refused}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_server_goes_in_front_of_the_browser_sources() {
|
||||
assert_eq!(crate::for_browser(false, None).name(), "binance+sample");
|
||||
assert_eq!(crate::for_browser(false, Some("ws://127.0.0.1:9/ws")).name(), "server+binance+sample");
|
||||
assert_eq!(crate::for_browser(true, Some("ws://127.0.0.1:9/ws")).name(), "sample");
|
||||
}
|
||||
}
|
||||
+19
-75
@@ -1,45 +1,18 @@
|
||||
//! A websocket client: tokio-tungstenite on the desktop, the browser's `WebSocket` on the web.
|
||||
|
||||
use anyhow::Result;
|
||||
//! A text-message websocket client: tokio-tungstenite on the desktop, the browser's
|
||||
//! `WebSocket` on the web.
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
pub use native::Socket;
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
pub use web::Socket;
|
||||
|
||||
/// A data message. Control frames stay inside the socket.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum Message {
|
||||
Text(String),
|
||||
Binary(Vec<u8>),
|
||||
}
|
||||
|
||||
impl Socket {
|
||||
pub async fn send(&mut self, text: String) -> Result<()> {
|
||||
self.send_message(Message::Text(text)).await
|
||||
}
|
||||
|
||||
/// The next text message, skipping binary ones; `None` once the server closes the connection.
|
||||
pub async fn next(&mut self) -> Option<Result<String>> {
|
||||
loop {
|
||||
match self.next_message().await? {
|
||||
Ok(Message::Text(t)) => return Some(Ok(t)),
|
||||
Ok(Message::Binary(_)) => {}
|
||||
Err(e) => return Some(Err(e)),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
mod native {
|
||||
use anyhow::{Context, Result};
|
||||
use futures::{SinkExt, StreamExt};
|
||||
use tokio_tungstenite::tungstenite::Message as WsMessage;
|
||||
use tokio_tungstenite::tungstenite::Message;
|
||||
use tokio_tungstenite::{MaybeTlsStream, WebSocketStream};
|
||||
|
||||
use super::Message;
|
||||
|
||||
pub struct Socket(WebSocketStream<MaybeTlsStream<tokio::net::TcpStream>>);
|
||||
|
||||
impl Socket {
|
||||
@@ -48,32 +21,22 @@ mod native {
|
||||
Ok(Self(ws))
|
||||
}
|
||||
|
||||
pub async fn send_message(&mut self, m: Message) -> Result<()> {
|
||||
let m = match m {
|
||||
Message::Text(t) => WsMessage::Text(t.into()),
|
||||
Message::Binary(b) => WsMessage::Binary(b.into()),
|
||||
};
|
||||
Ok(self.0.send(m).await?)
|
||||
pub async fn send(&mut self, text: String) -> Result<()> {
|
||||
Ok(self.0.send(Message::Text(text.into())).await?)
|
||||
}
|
||||
|
||||
/// The next data message; `None` once the server closes the connection.
|
||||
pub async fn next_message(&mut self) -> Option<Result<Message>> {
|
||||
/// The next text message; `None` once the server closes the connection.
|
||||
pub async fn next(&mut self) -> Option<Result<String>> {
|
||||
while let Some(msg) = self.0.next().await {
|
||||
match msg {
|
||||
Ok(WsMessage::Text(t)) => return Some(Ok(Message::Text(t.as_str().to_owned()))),
|
||||
Ok(WsMessage::Binary(b)) => return Some(Ok(Message::Binary(b.into()))),
|
||||
Ok(WsMessage::Close(_)) => return None,
|
||||
Ok(Message::Text(t)) => return Some(Ok(t.as_str().to_owned())),
|
||||
Ok(Message::Close(_)) => return None,
|
||||
Ok(_) => {}
|
||||
Err(e) => return Some(Err(e.into())),
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Send a close frame.
|
||||
pub async fn close(&mut self) {
|
||||
let _ = self.0.close(None).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -83,13 +46,11 @@ mod web {
|
||||
use tokio::sync::mpsc;
|
||||
use wasm_bindgen::JsCast;
|
||||
use wasm_bindgen::prelude::*;
|
||||
use web_sys::{BinaryType, CloseEvent, MessageEvent, WebSocket};
|
||||
|
||||
use super::Message;
|
||||
use web_sys::{CloseEvent, MessageEvent, WebSocket};
|
||||
|
||||
enum Event {
|
||||
Open,
|
||||
Message(Message),
|
||||
Text(String),
|
||||
Error,
|
||||
Closed,
|
||||
}
|
||||
@@ -107,8 +68,6 @@ mod web {
|
||||
impl Socket {
|
||||
pub async fn connect(url: &str) -> Result<Self> {
|
||||
let ws = WebSocket::new(url).map_err(|e| anyhow!("websocket {url}: {e:?}"))?;
|
||||
// Binary frames arrive as ArrayBuffers rather than Blobs, which can only be read asynchronously.
|
||||
ws.set_binary_type(BinaryType::Arraybuffer);
|
||||
let (tx, rx) = mpsc::unbounded_channel();
|
||||
let t = tx.clone();
|
||||
let on_open = Closure::<dyn FnMut()>::new(move || {
|
||||
@@ -116,15 +75,9 @@ mod web {
|
||||
});
|
||||
let t = tx.clone();
|
||||
let on_message = Closure::<dyn FnMut(MessageEvent)>::new(move |e: MessageEvent| {
|
||||
let data = e.data();
|
||||
let msg = if let Some(text) = data.as_string() {
|
||||
Message::Text(text)
|
||||
} else if let Some(buf) = data.dyn_ref::<js_sys::ArrayBuffer>() {
|
||||
Message::Binary(js_sys::Uint8Array::new(buf).to_vec())
|
||||
} else {
|
||||
return;
|
||||
};
|
||||
let _ = t.send(Event::Message(msg));
|
||||
if let Some(text) = e.data().as_string() {
|
||||
let _ = t.send(Event::Text(text));
|
||||
}
|
||||
});
|
||||
let t = tx.clone();
|
||||
let on_error = Closure::<dyn FnMut()>::new(move || {
|
||||
@@ -145,27 +98,18 @@ mod web {
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn send_message(&mut self, m: Message) -> Result<()> {
|
||||
match m {
|
||||
Message::Text(t) => self.ws.send_with_str(&t),
|
||||
Message::Binary(b) => self.ws.send_with_u8_array(&b),
|
||||
}
|
||||
.map_err(|e| anyhow!("websocket send: {e:?}"))
|
||||
pub async fn send(&mut self, text: String) -> Result<()> {
|
||||
self.ws.send_with_str(&text).map_err(|e| anyhow!("websocket send: {e:?}"))
|
||||
}
|
||||
|
||||
/// The next data message; `None` once the server closes the connection.
|
||||
pub async fn next_message(&mut self) -> Option<Result<Message>> {
|
||||
/// The next text message; `None` once the server closes the connection.
|
||||
pub async fn next(&mut self) -> Option<Result<String>> {
|
||||
match self.rx.recv().await? {
|
||||
Event::Message(m) => Some(Ok(m)),
|
||||
Event::Text(t) => Some(Ok(t)),
|
||||
Event::Error => Some(Err(anyhow!("websocket error"))),
|
||||
Event::Open | Event::Closed => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Send a close frame.
|
||||
pub async fn close(&mut self) {
|
||||
let _ = self.ws.close();
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Socket {
|
||||
|
||||
@@ -25,7 +25,7 @@ impl YahooProvider {
|
||||
pub fn new() -> Result<Self> {
|
||||
let client = reqwest::Client::builder()
|
||||
// Yahoo rejects requests without a browser-like user agent.
|
||||
.user_agent("Mozilla/5.0 (compatible; terminal)")
|
||||
.user_agent("Mozilla/5.0 (compatible; terminal-rs)")
|
||||
.timeout(Duration::from_secs(20))
|
||||
.cookie_store(true)
|
||||
.build()?;
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "tr-macos"
|
||||
description = "macOS window chrome for Terminal: unified toolbar, vibrancy, accent colour, updates"
|
||||
description = "macOS window chrome for terminal-rs: unified toolbar, vibrancy, accent colour, updates"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
|
||||
@@ -50,7 +50,7 @@ mod imp {
|
||||
let Some(window) = view.window() else { return false };
|
||||
window.setTitleVisibility(NSWindowTitleVisibility::Hidden);
|
||||
window.setTitlebarAppearsTransparent(true);
|
||||
let toolbar = NSToolbar::initWithIdentifier(NSToolbar::alloc(mtm), &NSString::from_str("terminal.toolbar"));
|
||||
let toolbar = NSToolbar::initWithIdentifier(NSToolbar::alloc(mtm), &NSString::from_str("terminal-rs.toolbar"));
|
||||
window.setToolbar(Some(&toolbar));
|
||||
// Compact unified toolbar: one slim row shared with the traffic lights.
|
||||
window.setToolbarStyle(NSWindowToolbarStyle::UnifiedCompact);
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
//! [`check_for_updates`] reports that it couldn't check.
|
||||
|
||||
/// The project's releases on GitHub.
|
||||
pub const RELEASES: &str = "https://github.com/michaelshimeles/terminal/releases";
|
||||
pub const RELEASES: &str = "https://github.com/michaelshimeles/terminal-rs/releases";
|
||||
|
||||
/// Starts Sparkle, which checks the feed once a day and shows its update window when a newer
|
||||
/// release is out, and adds "Check for Updates…" to the app menu. Main thread; later calls do
|
||||
|
||||
@@ -1,13 +0,0 @@
|
||||
[package]
|
||||
name = "tr-proto"
|
||||
description = "Client protocol between Terminal clients and the Terminal data server"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
# Shared by the server, the desktop app and the browser build, so it must build for
|
||||
# wasm32-unknown-unknown: serde, tr-core and the two codecs only, nothing async or networked.
|
||||
[dependencies]
|
||||
tr-core.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
postcard = { version = "1.1", default-features = false, features = ["use-std"] }
|
||||
@@ -1,629 +0,0 @@
|
||||
//! The protocol between Terminal clients (desktop, web, CLI) and the Terminal data server
|
||||
//! (`tr-gateway`), as designed in `docs/architecture/server.md` §4.
|
||||
//!
|
||||
//! Each message is one websocket frame. postcard over binary frames is the default encoding;
|
||||
//! JSON over text frames is a debug encoding a client asks for in [`ClientMsg::Hello`]. The
|
||||
//! server reads binary frames as postcard and text frames as JSON, and answers in the encoding
|
||||
//! the client's `Hello` named.
|
||||
//!
|
||||
//! # Evolving the protocol
|
||||
//!
|
||||
//! postcard is not self-describing: an enum variant travels as its index and a struct as its
|
||||
//! fields in order. So:
|
||||
//! - enum variants are only ever appended, never reordered or removed;
|
||||
//! - a struct never gains or loses a field (a new field means a new message variant);
|
||||
//! - any other change bumps [`PROTOCOL_VERSION`], and the server keeps serving N and N-1.
|
||||
//!
|
||||
//! The `wire_tags_are_stable` test pins every variant's index. The design's later variants
|
||||
//! (`DocOp`, `DocAck`/`DocConflict`, and the `Ctx`, Congress, Portfolio, Docs and Alerts
|
||||
//! topics) are not here yet; each is appended after the variants below when a server serves it.
|
||||
//! [`Bar`] and [`Timeframe`] come from `tr-core`, so their field and variant order is part of
|
||||
//! the wire format too.
|
||||
|
||||
use std::fmt;
|
||||
|
||||
use serde::de::DeserializeOwned;
|
||||
use serde::{Deserialize, Serialize};
|
||||
pub use tr_core::{Bar, Timeframe};
|
||||
|
||||
/// Bumped on any change the evolution rules above don't allow.
|
||||
pub const PROTOCOL_VERSION: u16 = 1;
|
||||
|
||||
/// Client-chosen subscription id, unique per connection.
|
||||
pub type SubId = u32;
|
||||
/// Client-chosen request id, unique per connection.
|
||||
pub type ReqId = u32;
|
||||
|
||||
/// A market as a client names it: a venue-native key (`BTC`, `xyz:TSLA`, `@107`, `PURR/USDC`)
|
||||
/// or a market key the server maps to one (`BTCUSD Curncy`).
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)]
|
||||
#[serde(transparent)]
|
||||
pub struct MarketKey(pub String);
|
||||
|
||||
impl MarketKey {
|
||||
pub fn new(key: impl Into<String>) -> Self {
|
||||
Self(key.into())
|
||||
}
|
||||
|
||||
pub fn as_str(&self) -> &str {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for MarketKey {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(&self.0)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
pub enum Encoding {
|
||||
Postcard,
|
||||
Json,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct ClientInfo {
|
||||
/// For example `terminal-desktop` or `tr-cli`.
|
||||
pub name: String,
|
||||
pub version: String,
|
||||
}
|
||||
|
||||
/// Reserved for session resume (§4.3). Servers ignore it for now.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(transparent)]
|
||||
pub struct ResumeToken(pub String);
|
||||
|
||||
/// Client to server.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub enum ClientMsg {
|
||||
/// The first message on every connection. `token` is reserved for sign-in (§8) and is
|
||||
/// ignored for now; `resume` likewise.
|
||||
Hello {
|
||||
version: u16,
|
||||
encoding: Encoding,
|
||||
client: ClientInfo,
|
||||
token: Option<String>,
|
||||
resume: Option<ResumeToken>,
|
||||
},
|
||||
Subscribe {
|
||||
id: SubId,
|
||||
topic: TopicReq,
|
||||
},
|
||||
Unsubscribe {
|
||||
id: SubId,
|
||||
},
|
||||
/// The client saw a gap in `seq`; the server answers with a fresh `Snapshot`.
|
||||
Resync {
|
||||
id: SubId,
|
||||
},
|
||||
Request {
|
||||
id: ReqId,
|
||||
req: Rpc,
|
||||
},
|
||||
Ping {
|
||||
t: u64,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub enum TopicReq {
|
||||
/// Prices for many markets in one subscription, at most `max_hz` deltas a second.
|
||||
Quotes { keys: Vec<MarketKey>, max_hz: u8 },
|
||||
/// Snapshot: the last `history` bars, the forming bar last. Deltas: [`BarDelta`].
|
||||
Bars { key: MarketKey, tf: Timeframe, history: u32 },
|
||||
/// The top `depth` levels a side, bucketed to multiples of `group` when set.
|
||||
Book { key: MarketKey, depth: u8, group: Option<f64> },
|
||||
/// Snapshot: recent trades. Deltas: new trades, oldest first.
|
||||
Trades { key: MarketKey },
|
||||
}
|
||||
|
||||
/// One-shot requests.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub enum Rpc {
|
||||
/// Up to `limit` bars before `end` (Unix seconds; `None` = now), oldest first. The same
|
||||
/// contract as `MarketData::history` in `tr-data`.
|
||||
History { key: MarketKey, tf: Timeframe, end: Option<i64>, limit: u32 },
|
||||
/// Every market the server can stream right now.
|
||||
Markets,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub enum RpcResult {
|
||||
History(Vec<Bar>),
|
||||
Markets(Vec<MarketInfo>),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct MarketInfo {
|
||||
pub key: MarketKey,
|
||||
/// Where it trades: a Hyperliquid perp dex (`""` is the core dex, `xyz`, ...) or `spot`.
|
||||
pub venue: String,
|
||||
pub kind: InstrumentKind,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
pub enum InstrumentKind {
|
||||
Perp,
|
||||
Spot,
|
||||
}
|
||||
|
||||
/// Opaque per-connection id, for logs and support.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
#[serde(transparent)]
|
||||
pub struct SessionId(pub u64);
|
||||
|
||||
/// Reserved for sign-in (§8). Always `None` for now.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct UserInfo {
|
||||
pub id: String,
|
||||
pub name: String,
|
||||
}
|
||||
|
||||
/// What the server accepts on this connection.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct Limits {
|
||||
pub max_subscriptions: u32,
|
||||
pub max_quote_keys: u32,
|
||||
pub max_book_depth: u8,
|
||||
pub max_history: u32,
|
||||
}
|
||||
|
||||
/// Server to client.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub enum ServerMsg {
|
||||
Welcome {
|
||||
version: u16,
|
||||
session: SessionId,
|
||||
user: Option<UserInfo>,
|
||||
server_time_ms: i64,
|
||||
limits: Limits,
|
||||
},
|
||||
/// Full state. Resets the subscription's `seq`.
|
||||
Snapshot {
|
||||
id: SubId,
|
||||
seq: u64,
|
||||
data: TopicData,
|
||||
},
|
||||
/// `seq` is the previous message's `seq` + 1.
|
||||
Delta {
|
||||
id: SubId,
|
||||
seq: u64,
|
||||
data: TopicDelta,
|
||||
},
|
||||
/// The server dropped deltas for this subscription; a `Snapshot` follows.
|
||||
Resync {
|
||||
id: SubId,
|
||||
reason: ResyncReason,
|
||||
},
|
||||
Reply {
|
||||
id: ReqId,
|
||||
result: Result<RpcResult, ApiError>,
|
||||
},
|
||||
Status {
|
||||
id: SubId,
|
||||
status: TopicStatus,
|
||||
},
|
||||
Pong {
|
||||
t: u64,
|
||||
server_time_ms: i64,
|
||||
},
|
||||
Error {
|
||||
code: ErrorCode,
|
||||
message: String,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub enum TopicData {
|
||||
Quotes(Vec<Quote>),
|
||||
Bars(Vec<Bar>),
|
||||
Book(Book),
|
||||
Trades(Vec<Trade>),
|
||||
}
|
||||
|
||||
/// For state topics (quotes, book) a delta is the new value, so a lost one is harmless.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub enum TopicDelta {
|
||||
/// The quotes that changed since the last message.
|
||||
Quotes(Vec<Quote>),
|
||||
Bar(BarDelta),
|
||||
Book(Book),
|
||||
Trades(Vec<Trade>),
|
||||
}
|
||||
|
||||
/// Bars stream as the forming bar's latest state, and a `Closed` bar with its final values
|
||||
/// before the first `Forming` of the next bar. The server may drop a `Forming` that a later
|
||||
/// message for the same `ts` supersedes; it never drops a `Closed`. Clients apply `Closed` by
|
||||
/// replacing the bar with that `ts` (§4.3).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
|
||||
pub enum BarDelta {
|
||||
Forming(Bar),
|
||||
Closed(Bar),
|
||||
}
|
||||
|
||||
impl BarDelta {
|
||||
pub fn bar(&self) -> &Bar {
|
||||
match self {
|
||||
BarDelta::Forming(b) | BarDelta::Closed(b) => b,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct Quote {
|
||||
/// The key as the client sent it.
|
||||
pub key: MarketKey,
|
||||
/// The venue market that backs it (`BTC` for `BTCUSD Curncy`).
|
||||
pub source: MarketKey,
|
||||
pub px: f64,
|
||||
pub basis: PriceBasis,
|
||||
/// The price 24 hours earlier, when the venue publishes one.
|
||||
pub prev_day: Option<f64>,
|
||||
/// When the server received this price (Unix milliseconds).
|
||||
pub ts_ms: i64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
pub enum PriceBasis {
|
||||
Mid,
|
||||
Mark,
|
||||
Oracle,
|
||||
Last,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct Book {
|
||||
/// Best first: bids descending, asks ascending.
|
||||
pub bids: Vec<Level>,
|
||||
pub asks: Vec<Level>,
|
||||
/// The venue's time for this book (Unix milliseconds).
|
||||
pub time_ms: i64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
|
||||
pub struct Level {
|
||||
pub px: f64,
|
||||
pub sz: f64,
|
||||
/// Orders at this level (0 when grouped levels were merged and the venue gave none).
|
||||
pub n: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
|
||||
pub struct Trade {
|
||||
pub px: f64,
|
||||
pub sz: f64,
|
||||
/// The aggressor's side.
|
||||
pub side: Side,
|
||||
/// The venue's trade time (Unix milliseconds).
|
||||
pub time_ms: i64,
|
||||
/// Venue trade id; unique together with `time_ms`.
|
||||
pub tid: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
pub enum Side {
|
||||
Buy,
|
||||
Sell,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
pub enum ResyncReason {
|
||||
/// The client fell too far behind an event topic.
|
||||
Overflow,
|
||||
/// The server's own upstream reconnected, so its state may have jumped.
|
||||
Upstream,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub enum TopicStatus {
|
||||
Live,
|
||||
/// No fresh data since `since_ms` (Unix milliseconds); the last values may be old.
|
||||
Stale {
|
||||
since_ms: i64,
|
||||
},
|
||||
/// The server can't serve this subscription. It stays open but gets no data.
|
||||
Unavailable {
|
||||
reason: UnavailableReason,
|
||||
detail: String,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
pub enum UnavailableReason {
|
||||
UnknownMarket,
|
||||
/// The server is at its upstream subscription budget.
|
||||
Capacity,
|
||||
Unsupported,
|
||||
Upstream,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
pub enum ErrorCode {
|
||||
BadMessage,
|
||||
UnsupportedVersion,
|
||||
HelloRequired,
|
||||
DuplicateId,
|
||||
UnknownId,
|
||||
TooManySubscriptions,
|
||||
SlowConsumer,
|
||||
UnknownMarket,
|
||||
Upstream,
|
||||
Internal,
|
||||
/// The client sent messages faster than `Limits` allows; the server ignored this one.
|
||||
RateLimited,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct ApiError {
|
||||
pub code: ErrorCode,
|
||||
pub message: String,
|
||||
}
|
||||
|
||||
impl fmt::Display for ApiError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "{:?}: {}", self.code, self.message)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for ApiError {}
|
||||
|
||||
/// One encoded websocket frame.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum Frame {
|
||||
Binary(Vec<u8>),
|
||||
Text(String),
|
||||
}
|
||||
|
||||
impl Frame {
|
||||
pub fn len(&self) -> usize {
|
||||
match self {
|
||||
Frame::Binary(b) => b.len(),
|
||||
Frame::Text(t) => t.len(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.len() == 0
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct CodecError(pub String);
|
||||
|
||||
impl fmt::Display for CodecError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "protocol codec: {}", self.0)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for CodecError {}
|
||||
|
||||
/// Encode a message as one frame: postcard in a binary frame, or JSON in a text frame.
|
||||
pub fn encode<T: Serialize>(msg: &T, encoding: Encoding) -> Result<Frame, CodecError> {
|
||||
match encoding {
|
||||
Encoding::Postcard => postcard::to_stdvec(msg).map(Frame::Binary).map_err(|e| CodecError(e.to_string())),
|
||||
Encoding::Json => serde_json::to_string(msg).map(Frame::Text).map_err(|e| CodecError(e.to_string())),
|
||||
}
|
||||
}
|
||||
|
||||
/// Decode a binary (postcard) frame.
|
||||
pub fn decode_binary<T: DeserializeOwned>(bytes: &[u8]) -> Result<T, CodecError> {
|
||||
postcard::from_bytes(bytes).map_err(|e| CodecError(e.to_string()))
|
||||
}
|
||||
|
||||
/// Decode a text (JSON) frame.
|
||||
pub fn decode_text<T: DeserializeOwned>(text: &str) -> Result<T, CodecError> {
|
||||
serde_json::from_str(text).map_err(|e| CodecError(e.to_string()))
|
||||
}
|
||||
|
||||
/// Decode either kind of frame.
|
||||
pub fn decode<T: DeserializeOwned>(frame: &Frame) -> Result<T, CodecError> {
|
||||
match frame {
|
||||
Frame::Binary(b) => decode_binary(b),
|
||||
Frame::Text(t) => decode_text(t),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn key(s: &str) -> MarketKey {
|
||||
MarketKey::new(s)
|
||||
}
|
||||
|
||||
fn bar(ts: i64) -> Bar {
|
||||
Bar::new(ts, 76_612.0, 76_640.5, 76_443.0, 76_521.0, 74.46557)
|
||||
}
|
||||
|
||||
fn book(levels: usize) -> Book {
|
||||
let side = |sign: f64| {
|
||||
(0..levels)
|
||||
.map(|i| Level { px: 76_530.0 + sign * i as f64, sz: 1.5 + i as f64 * 0.1, n: 3 + i as u32 })
|
||||
.collect()
|
||||
};
|
||||
Book { bids: side(-1.0), asks: side(1.0), time_ms: 1_793_325_942_459 }
|
||||
}
|
||||
|
||||
fn quote(k: &str, src: &str) -> Quote {
|
||||
Quote {
|
||||
key: key(k),
|
||||
source: key(src),
|
||||
px: 76_521.5,
|
||||
basis: PriceBasis::Mid,
|
||||
prev_day: Some(75_000.0),
|
||||
ts_ms: 1_793_325_942_459,
|
||||
}
|
||||
}
|
||||
|
||||
fn trade(i: u64) -> Trade {
|
||||
Trade { px: 76_528.0, sz: 0.41745, side: Side::Buy, time_ms: 1_793_325_941_395, tid: 1_005_251_635_753_713 + i }
|
||||
}
|
||||
|
||||
fn client_msgs() -> Vec<ClientMsg> {
|
||||
vec![
|
||||
ClientMsg::Hello {
|
||||
version: PROTOCOL_VERSION,
|
||||
encoding: Encoding::Postcard,
|
||||
client: ClientInfo { name: "tr-cli".into(), version: "0.1.1".into() },
|
||||
token: None,
|
||||
resume: None,
|
||||
},
|
||||
ClientMsg::Subscribe {
|
||||
id: 1,
|
||||
topic: TopicReq::Quotes { keys: vec![key("BTC"), key("xyz:TSLA")], max_hz: 4 },
|
||||
},
|
||||
ClientMsg::Unsubscribe { id: 1 },
|
||||
ClientMsg::Resync { id: 2 },
|
||||
ClientMsg::Request {
|
||||
id: 7,
|
||||
req: Rpc::History { key: key("BTCUSD Curncy"), tf: Timeframe::M1, end: None, limit: 500 },
|
||||
},
|
||||
ClientMsg::Ping { t: 42 },
|
||||
]
|
||||
}
|
||||
|
||||
fn server_msgs() -> Vec<ServerMsg> {
|
||||
let limits = Limits { max_subscriptions: 64, max_quote_keys: 512, max_book_depth: 20, max_history: 5000 };
|
||||
vec![
|
||||
ServerMsg::Welcome {
|
||||
version: PROTOCOL_VERSION,
|
||||
session: SessionId(9),
|
||||
user: None,
|
||||
server_time_ms: 1_793_325_942_000,
|
||||
limits,
|
||||
},
|
||||
ServerMsg::Snapshot { id: 1, seq: 0, data: TopicData::Quotes(vec![quote("BTCUSD Curncy", "BTC")]) },
|
||||
ServerMsg::Delta { id: 1, seq: 1, data: TopicDelta::Bar(BarDelta::Closed(bar(60))) },
|
||||
ServerMsg::Resync { id: 1, reason: ResyncReason::Overflow },
|
||||
ServerMsg::Reply { id: 7, result: Ok(RpcResult::History(vec![bar(0), bar(60)])) },
|
||||
ServerMsg::Status {
|
||||
id: 3,
|
||||
status: TopicStatus::Unavailable { reason: UnavailableReason::Capacity, detail: "budget".into() },
|
||||
},
|
||||
ServerMsg::Pong { t: 42, server_time_ms: 1_793_325_942_000 },
|
||||
ServerMsg::Error { code: ErrorCode::BadMessage, message: "nope".into() },
|
||||
]
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn round_trips_in_both_encodings() {
|
||||
for enc in [Encoding::Postcard, Encoding::Json] {
|
||||
for m in client_msgs() {
|
||||
let f = encode(&m, enc).unwrap();
|
||||
assert_eq!(decode::<ClientMsg>(&f).unwrap(), m, "{enc:?}");
|
||||
}
|
||||
for m in server_msgs() {
|
||||
let f = encode(&m, enc).unwrap();
|
||||
assert_eq!(decode::<ServerMsg>(&f).unwrap(), m, "{enc:?}");
|
||||
}
|
||||
let extra = [
|
||||
ServerMsg::Snapshot { id: 4, seq: 10, data: TopicData::Book(book(20)) },
|
||||
ServerMsg::Snapshot { id: 5, seq: 0, data: TopicData::Trades(vec![trade(0), trade(1)]) },
|
||||
ServerMsg::Snapshot { id: 6, seq: 0, data: TopicData::Bars(vec![bar(0)]) },
|
||||
ServerMsg::Delta { id: 4, seq: 11, data: TopicDelta::Book(book(5)) },
|
||||
ServerMsg::Delta { id: 5, seq: 1, data: TopicDelta::Trades(vec![trade(2)]) },
|
||||
ServerMsg::Delta { id: 1, seq: 2, data: TopicDelta::Quotes(vec![quote("BTC", "BTC")]) },
|
||||
ServerMsg::Reply {
|
||||
id: 8,
|
||||
result: Ok(RpcResult::Markets(vec![MarketInfo {
|
||||
key: key("xyz:TSLA"),
|
||||
venue: "xyz".into(),
|
||||
kind: InstrumentKind::Perp,
|
||||
}])),
|
||||
},
|
||||
ServerMsg::Reply {
|
||||
id: 9,
|
||||
result: Err(ApiError { code: ErrorCode::UnknownMarket, message: "x".into() }),
|
||||
},
|
||||
ServerMsg::Status { id: 1, status: TopicStatus::Stale { since_ms: 5 } },
|
||||
];
|
||||
for m in extra {
|
||||
assert_eq!(decode::<ServerMsg>(&encode(&m, enc).unwrap()).unwrap(), m);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encodings_pick_the_frame_kind() {
|
||||
let m = ClientMsg::Ping { t: 1 };
|
||||
assert!(matches!(encode(&m, Encoding::Postcard).unwrap(), Frame::Binary(_)));
|
||||
assert_eq!(encode(&m, Encoding::Json).unwrap(), Frame::Text(r#"{"Ping":{"t":1}}"#.into()));
|
||||
assert!(decode_binary::<ClientMsg>(&[0xff, 0xff]).is_err());
|
||||
assert!(decode_text::<ClientMsg>("{}").is_err());
|
||||
}
|
||||
|
||||
/// postcard writes an enum variant as its index. Appending variants keeps these; reordering
|
||||
/// or removing one changes them and breaks every deployed client, so this test pins them.
|
||||
#[test]
|
||||
fn wire_tags_are_stable() {
|
||||
let tag = |f: Frame| match f {
|
||||
Frame::Binary(b) => b[0],
|
||||
Frame::Text(_) => unreachable!(),
|
||||
};
|
||||
let client: Vec<u8> = client_msgs().iter().map(|m| tag(encode(m, Encoding::Postcard).unwrap())).collect();
|
||||
assert_eq!(client, [0, 1, 2, 3, 4, 5]);
|
||||
let server: Vec<u8> = server_msgs().iter().map(|m| tag(encode(m, Encoding::Postcard).unwrap())).collect();
|
||||
assert_eq!(server, [0, 1, 2, 3, 4, 5, 6, 7]);
|
||||
let t = |v: Vec<u8>| v[0];
|
||||
let topics = [
|
||||
TopicReq::Quotes { keys: vec![], max_hz: 1 },
|
||||
TopicReq::Bars { key: key("BTC"), tf: Timeframe::M1, history: 1 },
|
||||
TopicReq::Book { key: key("BTC"), depth: 1, group: None },
|
||||
TopicReq::Trades { key: key("BTC") },
|
||||
];
|
||||
assert_eq!(topics.map(|x| t(postcard::to_stdvec(&x).unwrap())), [0, 1, 2, 3]);
|
||||
let data =
|
||||
[TopicData::Quotes(vec![]), TopicData::Bars(vec![]), TopicData::Book(book(0)), TopicData::Trades(vec![])];
|
||||
assert_eq!(data.map(|x| t(postcard::to_stdvec(&x).unwrap())), [0, 1, 2, 3]);
|
||||
let deltas = [
|
||||
TopicDelta::Quotes(vec![]),
|
||||
TopicDelta::Bar(BarDelta::Forming(bar(0))),
|
||||
TopicDelta::Book(book(0)),
|
||||
TopicDelta::Trades(vec![]),
|
||||
];
|
||||
assert_eq!(deltas.map(|x| t(postcard::to_stdvec(&x).unwrap())), [0, 1, 2, 3]);
|
||||
assert_eq!(
|
||||
[BarDelta::Forming(bar(0)), BarDelta::Closed(bar(0))].map(|x| t(postcard::to_stdvec(&x).unwrap())),
|
||||
[0, 1]
|
||||
);
|
||||
assert_eq!(
|
||||
[Rpc::History { key: key("B"), tf: Timeframe::D1, end: None, limit: 1 }, Rpc::Markets]
|
||||
.map(|x| t(postcard::to_stdvec(&x).unwrap())),
|
||||
[0, 1]
|
||||
);
|
||||
let status = [
|
||||
TopicStatus::Live,
|
||||
TopicStatus::Stale { since_ms: 0 },
|
||||
TopicStatus::Unavailable { reason: UnavailableReason::Upstream, detail: String::new() },
|
||||
];
|
||||
assert_eq!(status.map(|x| t(postcard::to_stdvec(&x).unwrap())), [0, 1, 2]);
|
||||
// tr-core types are part of the wire format: Timeframe's variant order and Bar's fields.
|
||||
let tfs: Vec<u8> = Timeframe::ALL.iter().map(|tf| t(postcard::to_stdvec(tf).unwrap())).collect();
|
||||
assert_eq!(tfs, [0, 1, 2, 3, 4, 5, 6, 7, 8]);
|
||||
let b = postcard::to_stdvec(&Bar::new(1, 2.0, 3.0, 4.0, 5.0, 6.0)).unwrap();
|
||||
assert_eq!(b.len(), 1 + 5 * 8, "ts as a varint, then five f64 in field order");
|
||||
assert_eq!(&b[1..9], &2.0f64.to_le_bytes());
|
||||
}
|
||||
|
||||
/// Frame sizes for the messages that dominate traffic (§11 estimates ~30 bytes a quote).
|
||||
#[test]
|
||||
fn frame_sizes() {
|
||||
let size = |m: &ServerMsg, e| encode(m, e).unwrap().len();
|
||||
let one_quote = ServerMsg::Delta { id: 1, seq: 1234, data: TopicDelta::Quotes(vec![quote("BTC", "BTC")]) };
|
||||
let book20 = ServerMsg::Delta { id: 2, seq: 1234, data: TopicDelta::Book(book(20)) };
|
||||
let one_trade = ServerMsg::Delta { id: 3, seq: 1234, data: TopicDelta::Trades(vec![trade(0)]) };
|
||||
let forming = ServerMsg::Delta { id: 4, seq: 1234, data: TopicDelta::Bar(BarDelta::Forming(bar(60))) };
|
||||
let rows = [("quote", &one_quote), ("book 20x2", &book20), ("trade", &one_trade), ("bar", &forming)];
|
||||
for (name, m) in rows {
|
||||
let (p, j) = (size(m, Encoding::Postcard), size(m, Encoding::Json));
|
||||
println!("{name:>9}: postcard {p:>4} B, json {j:>5} B ({:.1}x)", j as f64 / p as f64);
|
||||
assert!(p < j, "{name}: postcard should be smaller");
|
||||
}
|
||||
assert!(size(&one_quote, Encoding::Postcard) <= 40);
|
||||
assert!(size(&one_trade, Encoding::Postcard) <= 40);
|
||||
assert!(size(&forming, Encoding::Postcard) <= 50);
|
||||
assert!(size(&book20, Encoding::Postcard) <= 40 * 18 + 16);
|
||||
}
|
||||
}
|
||||
@@ -1,51 +0,0 @@
|
||||
# tr-gateway on Freestyle
|
||||
|
||||
The Terminal data server runs on a Freestyle VM at **https://terminal-gateway.style.dev**. Clients connect to `wss://terminal-gateway.style.dev/ws`, for example the desktop app:
|
||||
|
||||
```sh
|
||||
TR_SERVER_URL=wss://terminal-gateway.style.dev cargo run --release
|
||||
```
|
||||
|
||||
## What runs where
|
||||
|
||||
- One VM, slug `terminal-gateway`, from the `freestyle/ubuntu-sm` snapshot: 2 vCPU, 4 GiB of memory, 16 GB of disk.
|
||||
- `tr-gateway` runs under systemd (`tr-gateway.service`) as the unprivileged user `trgw`, bound to `0.0.0.0:8080`. systemd restarts it whenever it exits.
|
||||
- Freestyle's edge terminates TLS on 443 and forwards to port 8080, so the gateway itself serves plain HTTP and websockets.
|
||||
- The source lives in `/opt/terminal/src`. `/opt/terminal/DEPLOYED_COMMIT` holds the commit that's running, and the CLI example is installed as `tr-cli`.
|
||||
|
||||
## Why this size
|
||||
|
||||
The VM builds the gateway from source on every deploy, so the 16 GB disk holds the Rust toolchain and the build cache. On 2 vCPU a first build takes about 4 minutes, and later ones are incremental. The gateway itself is one process with a few upstream websockets: on a Mac against the live feed, with one client watching BTC's quotes, book, trades and 1m bars, its resident memory was about 15 MiB. A bigger VM buys nothing until there are many more clients.
|
||||
|
||||
## Deploy or update
|
||||
|
||||
From a machine with Node (for `npx`), `jq`, `curl` and a Freestyle API key:
|
||||
|
||||
```sh
|
||||
export FREESTYLE_API_KEY=... # never commit or paste it
|
||||
deploy/freestyle/deploy.sh master # or a branch, tag or full commit
|
||||
```
|
||||
|
||||
`deploy.sh` creates the VM if it's missing and publishes the domain with a TLS rule if that's missing. It then copies `vm-setup.sh` to the VM and runs it at the ref you give. The VM builds that ref from GitHub, so push it first. `vm-setup.sh` installs the build tools and Rust (keeping Rust on the latest stable, like CI), builds `tr-gateway` and the CLI with `--locked`, installs them, writes the systemd unit and restarts the service. It waits for `/readyz` on the VM, and `deploy.sh` then waits for `/readyz` through the public URL.
|
||||
|
||||
A deploy restarts the gateway, which drops every client connection. Clients reconnect and resubscribe on their own. A subscription made while the gateway is still starting gets `Stale`, then `Live` and its snapshot once the gateway has its instrument list and mids (`server/README.md`).
|
||||
|
||||
## Check health
|
||||
|
||||
```sh
|
||||
curl -fsS https://terminal-gateway.style.dev/healthz # the process is up
|
||||
curl -fsS https://terminal-gateway.style.dev/readyz # upstream connected, every venue's mids in
|
||||
cli() { cargo run -q --release -p tr-gateway --example cli -- --url wss://terminal-gateway.style.dev "$@"; }
|
||||
cli stats
|
||||
cli watch --quotes BTC --book BTC --trades BTC --bars BTC:1m --secs 30
|
||||
```
|
||||
|
||||
On the VM, `systemctl status tr-gateway` and `journalctl -u tr-gateway` show the service and its logs, and `cat /opt/terminal/DEPLOYED_COMMIT` shows what's deployed. The gateway's logs never carry addresses.
|
||||
|
||||
## Cost
|
||||
|
||||
2 vCPU and 4 GiB running around the clock come to about $80 a month of Freestyle usage above the free allowance.
|
||||
|
||||
## `/stats` and `/metrics` are public
|
||||
|
||||
Anyone can read `/stats` (JSON) and `/metrics` (Prometheus text) through the public URL right now. They show counts, budgets, latency and the upstream subscriptions, which reveal the markets clients are watching. They carry no client addresses or user data. Micky hasn't decided whether to restrict them, so they stay as they are for now. Restricting them later means serving them on a second port the edge doesn't publish, or checking a token.
|
||||
@@ -1,49 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Deploy or update tr-gateway on a Freestyle VM, published at https://$DOMAIN (wss://$DOMAIN/ws).
|
||||
# The VM builds the ref from GitHub, so push it first.
|
||||
# Needs the Freestyle CLI auth in the environment (FREESTYLE_API_KEY); nothing is printed from it.
|
||||
# Usage: deploy/freestyle/deploy.sh [git-ref] (default: master)
|
||||
# Env: TR_VM_SLUG (default terminal-gateway), TR_DOMAIN (default terminal-gateway.style.dev),
|
||||
# TR_VM_SNAPSHOT (default freestyle/ubuntu-sm: 2 vCPU / 4 GiB / 16 GB),
|
||||
# TR_FREESTYLE_CLI (default [email protected], the npm package that runs the CLI)
|
||||
set -euo pipefail
|
||||
REF="${1:-master}"
|
||||
SLUG="${TR_VM_SLUG:-terminal-gateway}"
|
||||
DOMAIN="${TR_DOMAIN:-terminal-gateway.style.dev}"
|
||||
SNAP="${TR_VM_SNAPSHOT:-freestyle/ubuntu-sm}"
|
||||
CLI="${TR_FREESTYLE_CLI:-freestyle@0.2.21}"
|
||||
HERE="$(cd "$(dirname "$0")" && pwd)"
|
||||
|
||||
[ -n "${FREESTYLE_API_KEY:-}" ] || { echo "set FREESTYLE_API_KEY" >&2; exit 1; }
|
||||
for tool in npx jq curl; do
|
||||
command -v "$tool" >/dev/null || { echo "$tool is required" >&2; exit 1; }
|
||||
done
|
||||
fs() { npx -y "$CLI" "$@"; }
|
||||
|
||||
if ! fs --output json vm get "$SLUG" >/dev/null 2>&1; then
|
||||
echo "creating VM $SLUG from $SNAP"
|
||||
fs --output json vm create --snapshot-id "$SNAP" --slug "$SLUG" --display-name "Terminal data server (tr-gateway)" \
|
||||
--metadata app=terminal --metadata role=tr-gateway --no-ssh >/dev/null
|
||||
fi
|
||||
VM_ID="$(fs --output json vm get "$SLUG" | jq -r .id)"
|
||||
|
||||
if ! fs --output json tls list | jq -e --arg d "$DOMAIN" '.rules[]? | select(.domain==$d)' >/dev/null; then
|
||||
echo "publishing https://$DOMAIN -> $SLUG:8080"
|
||||
fs --output json tls create --domain "$DOMAIN" --from public --to "vm=$VM_ID,port=8080" >/dev/null
|
||||
fi
|
||||
|
||||
fs vm scp "$HERE/vm-setup.sh" "$SLUG:/root/vm-setup.sh" >/dev/null
|
||||
# The first build takes about 4 minutes on 2 vCPU; later ones are incremental.
|
||||
fs vm exec "$SLUG" --linux-user root --timeout-ms 1800000 -- bash /root/vm-setup.sh "$REF"
|
||||
|
||||
# The VM said it's ready; check the public path through Freestyle's edge too.
|
||||
deadline=$((SECONDS + 60))
|
||||
while [ "$SECONDS" -lt "$deadline" ]; do
|
||||
if curl -fsS --connect-timeout 5 --max-time 10 "https://$DOMAIN/readyz" >/dev/null 2>&1; then
|
||||
echo "ready: wss://$DOMAIN/ws"
|
||||
exit 0
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
echo "deployed, but https://$DOMAIN/readyz is not answering 200 yet" >&2
|
||||
exit 1
|
||||
@@ -1,81 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Runs as root inside a Freestyle Ubuntu VM. Builds tr-gateway from a git ref and
|
||||
# runs it under systemd on 0.0.0.0:8080; Freestyle's edge terminates TLS on 443.
|
||||
# Usage: vm-setup.sh [git-ref] (a branch, tag or full commit; default: master)
|
||||
set -euo pipefail
|
||||
REF="${1:-master}"
|
||||
REPO="https://github.com/michaelshimeles/terminal.git"
|
||||
SRC=/opt/terminal/src
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
apt-get update -qq
|
||||
apt-get install -y -qq build-essential pkg-config libssl-dev git curl ca-certificates >/dev/null
|
||||
|
||||
if ! id trgw >/dev/null 2>&1; then useradd --system --create-home --home-dir /var/lib/trgw --shell /usr/sbin/nologin trgw; fi
|
||||
|
||||
# CI builds with the latest stable Rust, so the VM keeps up with it.
|
||||
export RUSTUP_HOME=/opt/rust/rustup CARGO_HOME=/opt/rust/cargo PATH=/opt/rust/cargo/bin:$PATH
|
||||
if command -v cargo >/dev/null; then
|
||||
rustup update --no-self-update stable >/dev/null
|
||||
else
|
||||
curl -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain stable --no-modify-path
|
||||
fi
|
||||
|
||||
mkdir -p /opt/terminal
|
||||
if [ ! -d "$SRC/.git" ]; then
|
||||
git clone --quiet "$REPO" "$SRC"
|
||||
fi
|
||||
git -C "$SRC" fetch --quiet origin "$REF"
|
||||
git -C "$SRC" checkout --quiet --detach FETCH_HEAD
|
||||
cd "$SRC"
|
||||
echo "building $(git rev-parse HEAD)"
|
||||
cargo build --release --locked -p tr-gateway --bin tr-gateway --example cli
|
||||
install -m 0755 target/release/tr-gateway /usr/local/bin/tr-gateway.new
|
||||
mv /usr/local/bin/tr-gateway.new /usr/local/bin/tr-gateway
|
||||
install -m 0755 target/release/examples/cli /usr/local/bin/tr-cli
|
||||
git rev-parse HEAD > /opt/terminal/DEPLOYED_COMMIT
|
||||
|
||||
cat > /etc/systemd/system/tr-gateway.service <<'UNIT'
|
||||
[Unit]
|
||||
Description=Terminal data server (tr-gateway)
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
# Keep restarting however often it fails; systemd would otherwise give up after 5 quick tries.
|
||||
StartLimitIntervalSec=0
|
||||
|
||||
[Service]
|
||||
User=trgw
|
||||
Environment=TR_GATEWAY_BIND=0.0.0.0:8080
|
||||
Environment=RUST_LOG=info
|
||||
ExecStart=/usr/local/bin/tr-gateway
|
||||
Restart=always
|
||||
RestartSec=2
|
||||
LimitNOFILE=65536
|
||||
NoNewPrivileges=true
|
||||
ProtectSystem=strict
|
||||
ProtectHome=true
|
||||
PrivateTmp=true
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
UNIT
|
||||
systemctl daemon-reload
|
||||
systemctl enable tr-gateway >/dev/null 2>&1
|
||||
systemctl restart tr-gateway
|
||||
|
||||
# Wait until the process is up (/healthz), then until it has the instrument list and
|
||||
# every venue's mids (/readyz).
|
||||
wait_for() {
|
||||
local deadline=$((SECONDS + $2))
|
||||
while [ "$SECONDS" -lt "$deadline" ]; do
|
||||
if curl -fsS --connect-timeout 2 --max-time 5 "http://127.0.0.1:8080/$1" >/dev/null 2>&1; then return 0; fi
|
||||
sleep 1
|
||||
done
|
||||
echo "/$1 did not answer 200 within $2 s" >&2
|
||||
systemctl --no-pager status tr-gateway >&2 || true
|
||||
journalctl --no-pager -u tr-gateway -n 30 >&2 || true
|
||||
return 1
|
||||
}
|
||||
wait_for healthz 30
|
||||
wait_for readyz 90
|
||||
echo "deployed $(cat /opt/terminal/DEPLOYED_COMMIT); ready"
|
||||
@@ -1,785 +0,0 @@
|
||||
# Terminal data server: design
|
||||
|
||||
Status: **draft for review**. The first server code runs on an interim public feed (see [Interim: public feed](#interim-public-feed)).
|
||||
Author: Grok Bot, for Micky. Research date: 2026-10-29. Times are UTC unless marked.
|
||||
|
||||
Terminal today fetches all of its own data: Yahoo, Binance and Coinbase websockets, Treasury.gov, Senate eFD and the House Clerk, and SnapTrade. Each desktop process opens its own connections. The browser build serves sample data wherever a source blocks cross-origin requests (`TODO(web proxy)` in `crates/data/src/web.rs` and `crates/congress/src/db_web.rs`). This doc proposes the replacement Micky asked for: **one Rust server that every client (desktop, web, later mobile) connects to, so market data and user state sync in real time**. Hyperliquid is the first market-data source, and the design leaves a clean provider interface for licensed equities and options later.
|
||||
|
||||
**Decision (Micky, 2026-10-29): Terminal runs on our own Hyperliquid non-validator node(s) and our own systems from day one.** Live data comes only from the node's output files and its local info server: books, trades, candles, mids, marks, oracle prices and funding. The public Hyperliquid API is used for nothing live. At most, it serves a one-time history backfill (§5.6).
|
||||
|
||||
## Interim: public feed
|
||||
|
||||
On 2026-10-29 Micky approved starting on Hyperliquid's public API, because a node box (about $735 a month, §6.3) costs too much for now. Until the node is up, this is what runs:
|
||||
|
||||
- `tr-gateway` (`server/`) with `HlPublicProvider`, which implements the `Provider` trait (§3.1) over `wss://api.hyperliquid.xyz/ws` and the public `info` endpoint. Clients speak `tr-proto` (§4) and can't tell providers apart, so `HlNodeProvider` replaces it later with no client or protocol change.
|
||||
- One set of upstream connections serves every client. Mids for the core dex and for every HIP-3 dex with listed markets stay subscribed. The gateway subscribes a market's book, trades and candles only while a client watches it, and drops them 20 s after the last one leaves.
|
||||
- The gateway keeps its own budgets under Hyperliquid's per-IP limits [doc: RL]: at most 8 of 10 connections, 950 of 1000 subscriptions, 1800 of 2000 sent messages a minute and 1100 of 1200 REST weight a minute (defaults 3, 900, 1000 and 800). It refuses a client subscription past the budget, and it never rotates IPs or uses proxies [doc: TOU §3.1.8].
|
||||
- Hyperliquid closes the whole websocket when asked for a coin it doesn't list, with no error message [observed 2026-10-29]. The gateway only subscribes coins that have a live mid, and keeps the permanent mids on a connection of their own.
|
||||
- History comes from `candleSnapshot`, so it stops at the most recent 5000 bars per interval [doc: INFO]. For 1m bars that is about 3.5 days. Node-built candles (§5.3) lift the cap.
|
||||
- It makes read-only public calls only. No `exchange` endpoint, no user subscriptions, no orders.
|
||||
- The desktop opts in with `TR_SERVER_URL` (§4.6). Crypto keys such as `BTCUSD Curncy` then come from Hyperliquid core perps, and the order book pane names the source. Everything else is unchanged.
|
||||
- The web build uses the server for the same crypto markets. `scripts/build-web.sh` builds in `wss://terminal-gateway.style.dev/ws` unless `TR_SERVER_URL` says otherwise. While the server is unreachable, the page falls back to Binance and Coinbase as before.
|
||||
- It runs on one Freestyle VM (2 vCPU, 4 GiB) at `wss://terminal-gateway.style.dev/ws`, behind Freestyle's TLS edge. `deploy/freestyle/README.md` covers deploys, health checks and cost, which is about $80 a month above the free allowance.
|
||||
|
||||
How facts are marked:
|
||||
|
||||
- **[doc: …]** is a fact from official documentation. Sources are listed in [References](#references).
|
||||
- **[observed]** is a fact from read-only calls I made against `api.hyperliquid.xyz` on 2026-10-29 around 17:35 UTC, or from reading the public `order_book_server` source. These are snapshots, not guarantees.
|
||||
- **[unverified]** is an assumption or estimate that nobody has checked yet. It needs confirming before we build on it.
|
||||
- **[target]** is a goal we set for ourselves. Nobody has measured it yet.
|
||||
|
||||
---
|
||||
|
||||
## 1. Goals and non-goals
|
||||
|
||||
### Goals
|
||||
|
||||
1. **Our own node, our own systems.** We run Hyperliquid non-validator nodes in Tokyo (§6). Our `tr-ingest` builds every order book, trade tape, candle and price from the node's outputs (§5). There are no live dependencies on `api.hyperliquid.xyz`, so its rate limits don't apply to live data.
|
||||
2. **One source, many clients.** Ingest computes everything once. The gateway fans it out, so cost scales with instruments, not with clients (§5.9).
|
||||
3. **The same data everywhere.** Desktop, web (WASM) and mobile see the same quotes, bars, books and user documents, with live updates.
|
||||
4. **Hyperliquid first:** core perps, spot, and HIP-3 builder dexes (trade.xyz `xyz:` stocks, indices, FX and commodities, and the other dexes).
|
||||
5. **A provider interface clean enough** that a licensed equities and options vendor (for example Intrinio, Twelve Data or Massive) slots in later with no protocol change. Each provider also carries a display-rights policy (§3.1).
|
||||
6. **User sync:** accounts, layouts, watchlists, drawings, preferences and the demo (paper) portfolio, with explicit conflict rules.
|
||||
7. **The web build gets real data.** This removes the `TODO(web proxy)` stubs.
|
||||
8. **Candles at every interval with no history cap,** built from trades on our node (§5.3).
|
||||
9. **The terminal stays snappy.** The protocol is binary and conflated, and the UI thread never waits on the network. This keeps the existing worker model in `app/src/worker.rs`.
|
||||
|
||||
### Non-goals (for this design)
|
||||
|
||||
- **No trading through the server.** The server is data only. It never signs, routes or places orders on Hyperliquid or anywhere else. The SnapTrade order paths in `crates/broker/src/trading.rs` stay client-side on the desktop, with the user's local credentials. They are unchanged and out of scope (§9). The reasons are the legal flag in §13 (Hyperliquid's Terms of Use) and Micky's steer. The demo account is paper trading against our own quotes and never touches a venue.
|
||||
- No licensed stock or options data yet. Only the interface for it is in scope (§3.1).
|
||||
- No public API for third parties. The protocol is for our own clients.
|
||||
- No running of a Hyperliquid **validator**. We run non-validators only (§6).
|
||||
- No live use of the public Hyperliquid API (the one-time backfill in §5.6 is the only exception).
|
||||
- No multi-region active-active for v1.
|
||||
|
||||
---
|
||||
|
||||
## 2. Current data layer (what we map onto)
|
||||
|
||||
| Area | Today | Code |
|
||||
|---|---|---|
|
||||
| Provider trait | `MarketData` (async): `history`, `quotes`, `live`, `search`, `news`, `options`, `order_book`, `yield_curves`. Wrapped in `Fallback`, which serves sample data on error. | `crates/data/src/lib.rs` |
|
||||
| Keys | Bloomberg-style market keys from `Security { ticker, class }`: `AAPL US Equity`, `BTCUSD Curncy`, `SPX Index`, `CL1 Comdty`, `US10Y Govt`. `AssetClass` has Equity, Etf, Index, Fx, Crypto, Govt and Cmdty. The yellow keys are Equity, Index, Curncy, Govt and Comdty. | `crates/core/src/security.rs`, `catalog.rs` |
|
||||
| Data types | `Bar {ts, open, high, low, close, volume}` (ts is the open time in Unix seconds), `Series`, `Timeframe` (M1 to MN1), `Quote {symbol,last,prev_close,ts,spark}`, `OrderBook {bids,asks,source,simulated}`, `OptionChain`, `YieldCurves`, `NewsItem`, `SearchHit`. All of them derive serde. | `crates/core/src/bar.rs`, `timeframe.rs`, `crates/data/src/lib.rs` |
|
||||
| Live data | `LiveFeed = mpsc::Receiver<Bar>` holds the latest state of the forming bar. Crypto comes from Binance kline websockets, with Coinbase as the fallback. | `crates/data/src/crypto.rs`, `ws.rs` |
|
||||
| Runtime | tokio on the desktop. On wasm32, `rt::spawn` runs tasks on the browser event loop. | `crates/data/src/rt.rs` |
|
||||
| Worker | The UI sends `Request` (History, MoreHistory, Quotes, OrderBook, Options, News, Search, Congress queries, Portfolio, Broker*) and drains `Response`, so it never blocks. | `app/src/worker.rs` |
|
||||
| Bar cache | SQLite `BarCache` (desktop only). | `crates/data/src/cache.rs` |
|
||||
| Congress | Senate eFD and House Clerk ingest into SQLite, refreshed every 30 minutes (`CONGRESS_REFRESH_MIN`). The web build gets an empty stub. | `crates/congress/` |
|
||||
| Brokerage | SnapTrade, read-only. The partner keys and the per-user secret live in the Keychain or a 0600 file. | `crates/broker/src/snaptrade.rs`, `secrets.rs` |
|
||||
| Demo | `DemoBook` starts with $100k (`STARTING_CASH`) and is saved as local JSON. Orders fill at the terminal's live quote. | `crates/broker/src/demo.rs` |
|
||||
| User docs | Layouts (`LayoutDoc` JSON, which holds the panes, their drawings and the watchlist panes), prefs, themes and palette history. They are files on the desktop and `localStorage` in the browser. | `app/src/storage.rs`, `layouts.rs`, `prefs.rs` |
|
||||
|
||||
The plan keeps `MarketData` as the *client-side* trait. A new `RemoteMarkets` implementation forwards each call to the server (§4.6), so the panes, the worker and the chart code don't change. On the server, a new, richer `Provider` trait (§3.1) sits behind the ingest adapters.
|
||||
|
||||
---
|
||||
|
||||
## 3. Components
|
||||
|
||||
```
|
||||
Tokyo (node hosts, x2 for failover) Gateway host(s)
|
||||
+----------------------------------------------+ +----------------------------------------------+
|
||||
| hl-visor -> hl-node (non-validator) | | tr-gateway |
|
||||
| ~/hl/data/node_fills_by_block/... | | hot store (watch/broadcast), refcounts |
|
||||
| ~/hl/data/node_order_statuses_by_block/... | internal | auth, subs, conflation, backpressure |--> desktop
|
||||
| ~/hl/data/node_raw_book_diffs_by_block/... | stream | user sync (docs, drawings, demo book) |--> web (WASM)
|
||||
| ~/hl/data/hip3_oracle_updates/..., misc_.. |--------->| Congress, SnapTrade (read-only), others |--> mobile
|
||||
| localhost:3001/info (subset, fileSnapshot) | tr-proto | persistence: Postgres/SQLite |
|
||||
| tr-ingest (same host) | over +----------------------------------------------+
|
||||
| file tailers -> book builder (L4->L2/BBO) | WG/mTLS
|
||||
| trade tape -> candle builder (all TFs) |
|
||||
| prices: mid, last, oracle, mark, funding |
|
||||
| candle store (no cap), checkpoints |
|
||||
+----------------------------------------------+
|
||||
```
|
||||
|
||||
`tr-ingest` has to run **on the node host**, because it reads the node's local files and its `localhost` info server. The reference order-book server is also meant to run on the same machine as the node [doc: OBS]. `tr-gateway` can share the host or sit on a separate small machine (§11).
|
||||
|
||||
### 3.1 Ingest adapters behind a `Provider` trait
|
||||
|
||||
```rust
|
||||
/// Server-side source of market data. One instance per upstream (Hyperliquid, a licensed
|
||||
/// equities vendor, ...). The registry asks each provider what it can serve.
|
||||
#[async_trait]
|
||||
pub trait Provider: Send + Sync + 'static {
|
||||
fn id(&self) -> ProviderId; // "hyperliquid", "intrinio", ...
|
||||
fn capabilities(&self) -> Capabilities; // channels × asset classes it serves
|
||||
/// Display rights for data from this provider. Enforced by the gateway, not the client.
|
||||
fn display_policy(&self) -> DisplayPolicy;
|
||||
/// Instruments this provider lists right now (refreshed periodically; delistings appear here).
|
||||
async fn instruments(&self) -> Result<Vec<InstrumentDef>>;
|
||||
/// Historical bars, newest `limit` before `end` (same contract as MarketData::history).
|
||||
async fn history(&self, inst: &InstrumentId, tf: Timeframe, end: Option<i64>, limit: usize)
|
||||
-> Result<Vec<Bar>>;
|
||||
/// Start an upstream stream; events go to `sink` until the handle is dropped.
|
||||
async fn subscribe(&self, ch: Channel, inst: &InstrumentId, sink: UpstreamSink)
|
||||
-> Result<UpstreamHandle>;
|
||||
}
|
||||
|
||||
pub struct DisplayPolicy {
|
||||
/// May we show this data to the general public (any signed-in or anonymous user)?
|
||||
pub public_display: bool,
|
||||
/// If not public: which entitlement unlocks it (e.g. a licensed seat, a BYO key, staff).
|
||||
pub entitlement: Option<Entitlement>,
|
||||
/// Required delay before display to non-entitled users (licensed vendors often have one).
|
||||
pub delay: Option<Duration>,
|
||||
/// Attribution string the UI must show ("Data: Hyperliquid", "trade.xyz oracle", ...).
|
||||
pub attribution: &'static str,
|
||||
}
|
||||
```
|
||||
|
||||
Rules:
|
||||
|
||||
- **Secondary crypto exchanges are off by default for public display.** Coinbase and OKX ban public display without written consent, and Kraken requires permission (provider research from the parent agent, which I have not re-checked here). Binance's terms are not reviewed [unverified]. So the existing `CryptoProvider` (Binance and Coinbase) moves server-side with `public_display: false`. It is only served to entitled users (staff, for example) until each venue's rights are confirmed in writing.
|
||||
- Yahoo Finance and the RSS news sources get `public_display: false` until reviewed [unverified]. Treasury.gov is US government data and is *likely* fine [unverified].
|
||||
- The gateway checks `DisplayPolicy` on every subscribe (§4). A client can't opt itself in.
|
||||
- Licensed stocks and options (Intrinio, Twelve Data, Massive or another vendor) arrive later as one more `Provider`. Their entitlement and delay map straight onto `DisplayPolicy`.
|
||||
|
||||
Adapters at launch:
|
||||
|
||||
| Adapter | Source | Notes |
|
||||
|---|---|---|
|
||||
| `HlNodeProvider` | Our node's output files plus its local info server (`localhost:3001/info`) | §5 and §6. The only live Hyperliquid source. |
|
||||
| `HlBackfill` (one-time job) | S3 node archives and/or public `candleSnapshot` | §5.6. Not a live provider. |
|
||||
| `CongressProvider` | Senate eFD and House Clerk | Reuses `tr-congress` unchanged (§9) |
|
||||
| `SnapTradeProvider` | SnapTrade, read-only server mode | Per-user and private, so it is not a market-data topic. Desktop local mode is unchanged (§9). |
|
||||
| `CryptoProvider` (existing) | Binance and Coinbase | Off for public display |
|
||||
| `TreasuryProvider` (existing) | Treasury.gov, with FRED as fallback | Yield curves |
|
||||
|
||||
### 3.2 Normalizer
|
||||
|
||||
This converts provider payloads into the shared types in `tr-core` and `tr-data`, plus a few new fields:
|
||||
|
||||
- **Numbers.** Node outputs carry prices and sizes as decimal **strings** (for example `"px": "51.367"`) [doc: L1]. The public API does the same, including websocket candles, which the docs type as numbers [observed]. Parsers accept either. The normalizer parses to `f64` for the existing types. It keeps the original string only where exact decimals matter: tick or lot rounding in the demo book.
|
||||
- **Time.** Node events carry `time` as an ISO-like string (for example `"2024-07-26T08:26:25.899"`) [doc: L1]. Batched output adds `block_time` and `block_number` [doc: NODE]. We treat these as UTC [unverified: no zone suffix in the examples]. `Bar.ts` is the bar's open time in seconds, as today.
|
||||
- **Intervals.** We build every app `Timeframe` (M1, M5, M15, M30, H1, H4, D1, W1, MN1) ourselves, plus any extra interval we choose (§5.3). Hyperliquid's own intervals (`1m` to `1M` [doc: INFO]) only matter for the one-time backfill.
|
||||
- **Books.** Our L2 aggregation (§5.2) becomes `OrderBook { bids, asks, source: "Hyperliquid (own node)", simulated: false }`. L4 (per-order) views get a new type.
|
||||
- **Price basis.** This is new and required for the perp-as-proxy mapping (§7). Every quote carries `basis: Mid | Mark | Oracle | Last`, `venue`, `instrument_kind: Spot | Perp | Equity | …`, plus `session: Regular | Extended | Closed | Internal24x7` when the provider knows it.
|
||||
|
||||
### 3.3 In-memory hot store with fan-out
|
||||
|
||||
- The store is a **topic table** keyed by `Topic` (for example `Quote(MarketKey)`, `Bars(MarketKey, Timeframe)`, `Book(MarketKey, depth, group)`, `Trades(MarketKey)`, `Ctx(MarketKey)`, `Congress(Latest)`, `Doc(UserId, DocId)`).
|
||||
- **State topics** (quote, book, ctx) use `tokio::sync::watch` style "latest value wins". A slow reader skips intermediate values, which conflates naturally.
|
||||
- **Bars are a hybrid, not a pure state topic.** Updates to the *forming* bar may be conflated, but only within the same bar `ts`. When the bar `ts` advances, the server first emits a non-conflatable `BarClosed { bar }` event with the previous bar's final OHLCV, then the new forming bar. `BarClosed` goes through the event ring like trades, so a slow client can never skip a closed bar's final values. If it overflows, the client gets `Resync` and a fresh history snapshot (§4.3).
|
||||
- **Event topics** (trades, congress alerts, document ops) use a bounded `broadcast` ring with a per-topic sequence number. A reader that falls behind the ring gets `Resync` (§4.3).
|
||||
- Each topic keeps a **refcount** of client subscribers. Ingest maintains *every* book, tape and candle regardless of demand, because it has to apply every diff to stay correct. The gateway's refcount only decides which per-coin topics it materialises and serialises for clients. Topics are dropped after a **linger** period (start at 60 s [target]) once the count returns to 0.
|
||||
- Snapshots are served from memory, so a new client gets the current book or quote immediately.
|
||||
|
||||
### 3.4 Persistence (candles and history)
|
||||
|
||||
- **Candles:** built by ingest from node trades (§5.3) and stored with **no cap**. We keep 1m bars forever, plus materialised higher intervals. Each bar records its provenance: `node:<host>`, `backfill:s3`, `backfill:api` or `repaired`.
|
||||
- **Storage:** v1 uses SQLite (WAL) on the node host's NVMe for the candle store, reusing the `BarCache` schema ideas, and replicates closed bars to the gateway's database. Postgres or Timescale can come once there are several gateway instances. Size: ~1,000 active instruments × 1,440 1m bars/day ≈ 1.4M rows/day, roughly **70–150 MB/day or 25–55 GB/year** at 50–100 bytes per row [unverified, rough; this is an upper bound because quiet minutes write no row].
|
||||
- **Trade tape:** we keep the deduplicated trades (§5.3) for 30 days [target] for the tape and for rebuilding bars. Raw node files are rotated away (§6.4).
|
||||
- **Also persisted** (gateway side): congress trades, user documents, demo books and audit logs (§8).
|
||||
|
||||
### 3.5 Client gateway
|
||||
|
||||
- **Transport:** WebSocket over TLS (axum with tokio-tungstenite on the server). The web client uses the browser `WebSocket` that `crates/data/src/ws.rs` already wraps.
|
||||
- **Per-connection task:** auth handshake, a subscription set, a bounded outbound queue, a conflation timer and heartbeats.
|
||||
- **Enforcement:** auth, per-user limits (subscriptions per connection, messages per second), and `DisplayPolicy`.
|
||||
- **HTTP endpoints:** `/healthz`, `/readyz` and `/metrics` (internal), plus OAuth callback routes (§8).
|
||||
|
||||
---
|
||||
|
||||
## 4. Client protocol
|
||||
|
||||
### 4.1 Shared crate: `crates/proto` (`tr-proto`)
|
||||
|
||||
The protocol is one crate, used by the server, the desktop app and the WASM build. It depends only on `serde` and `tr-core`, so it compiles for `wasm32-unknown-unknown` (CI checks this). Every message type lives here, and the server and clients cannot drift apart because they compile the same definitions.
|
||||
|
||||
### 4.2 Message schema
|
||||
|
||||
```rust
|
||||
pub const PROTOCOL_VERSION: u16 = 1;
|
||||
|
||||
/// Client → server
|
||||
pub enum ClientMsg {
|
||||
Hello { version: u16, encoding: Encoding, client: ClientInfo, token: Option<String>, resume: Option<ResumeToken> },
|
||||
Subscribe { id: SubId, topic: TopicReq }, // client-chosen id, unique per connection
|
||||
Unsubscribe { id: SubId },
|
||||
Resync { id: SubId }, // client detected a gap; ask for a fresh snapshot
|
||||
Request { id: ReqId, req: Rpc }, // one-shot: History{key,tf,end,limit}, Search, Options…
|
||||
DocOp { id: ReqId, op_id: OpId, doc: DocId, base_rev: u64, op: DocOperation }, // user-sync writes (§8); op_id is a client UUID, stable across retries
|
||||
Ping { t: u64 },
|
||||
}
|
||||
|
||||
/// Server → client
|
||||
pub enum ServerMsg {
|
||||
Welcome { version: u16, session: SessionId, user: Option<UserInfo>, server_time_ms: i64, limits: Limits },
|
||||
Snapshot { id: SubId, seq: u64, data: TopicData }, // full state; resets the subscription's seq
|
||||
Delta { id: SubId, seq: u64, data: TopicDelta }, // seq = previous + 1
|
||||
Resync { id: SubId, reason: ResyncReason }, // server dropped deltas; a Snapshot follows
|
||||
Reply { id: ReqId, result: Result<RpcResult, ApiError> },
|
||||
DocAck { id: ReqId, doc: DocId, rev: u64 } | DocConflict { id: ReqId, doc: DocId, current: DocSnapshot },
|
||||
Status { id: SubId, status: TopicStatus }, // Stale{since}, Delisted, Unavailable{reason}, DisplayRestricted
|
||||
Pong { t: u64, server_time_ms: i64 },
|
||||
Error { code: ErrorCode, message: String },
|
||||
}
|
||||
|
||||
pub enum TopicReq {
|
||||
Quotes { keys: Vec<MarketKey>, max_hz: u8 }, // watchlists / tickers: one sub, many keys
|
||||
Bars { key: MarketKey, tf: Timeframe, history: usize },// snapshot = last `history` bars, deltas = forming bar / new bar
|
||||
Book { key: MarketKey, depth: u8, group: Option<f64> },
|
||||
Trades { key: MarketKey },
|
||||
Ctx { key: MarketKey }, // funding, OI, mark/oracle (perps), supply (spot)
|
||||
Congress(CongressTopic), Portfolio, Docs { kinds: Vec<DocKind> }, Alerts,
|
||||
}
|
||||
```
|
||||
|
||||
### 4.3 Snapshot + delta, sequencing and resync
|
||||
|
||||
- Every subscription has its own `seq`. A `Snapshot` sets it, and each `Delta` increments it by 1.
|
||||
- **The client checks continuity.** On a gap, or on a `Delta` before its `Snapshot`, the client sends `Resync`. The server answers with a new `Snapshot`.
|
||||
- **The server sends `Resync` itself** when an event topic's ring overflowed for that client (§5.8), or after ingest recovers from a gap or a node failover (§5.5, §6.6), because the server's own state may have jumped.
|
||||
- **For state topics a delta is just the new value** (a quote or a book), so a lost delta is harmless and those topics never need a resync on overflow. Bars deltas are `Forming(bar)` (conflatable within one `ts`) or `Closed(bar)` (never conflated, always delivered before the first `Forming` of the next `ts`). The client applies `Closed` by *replacing* the bar with that `ts`, which repairs any partial bar it had. Today's `Series::apply_live` only appends a newer bar, so `RemoteMarkets` must apply `Closed` with a replace before forwarding to the existing `LiveFeed` consumer. A test in phase 5 covers a slow reader across a boundary.
|
||||
- **Reconnect:** `Hello.resume` holds the previous session's resume token. The server re-attaches the subscriptions and sends a fresh `Snapshot` for each one. Clients never replay old deltas.
|
||||
|
||||
### 4.4 Encoding
|
||||
|
||||
| Option | Pros | Cons |
|
||||
|---|---|---|
|
||||
| **postcard** (serde, compact varint) | Smallest frames. no_std-friendly, so it builds fine for WASM. Same serde derives as today. | Not self-describing: fields can't be added to an existing struct without a version bump. |
|
||||
| bincode 2 | Fast and mature | Larger than postcard for small numbers. Same evolution caveat. |
|
||||
| JSON (serde_json) | Debuggable, browser devtools-friendly | 3–5× larger, and parse cost on the WASM main thread [unverified ratio] |
|
||||
| MessagePack (rmp-serde) | Self-describing and compact-ish | Middle ground. Another dependency. |
|
||||
|
||||
**Recommendation: postcard over binary WebSocket frames for every client, with JSON as a negotiated debug encoding** (`Hello.encoding`). The web client is Rust compiled to WASM, so there is *no JavaScript consumer*: the usual "JSON for the web" argument doesn't apply here. Evolution rules: append-only enum variants, a new message variant (never a changed struct) for new fields, and `PROTOCOL_VERSION` checked in `Hello`/`Welcome`. The server supports N and N−1.
|
||||
|
||||
Compression: the browser and tungstenite support for permessage-deflate needs checking [unverified]. postcard plus conflation may make it unnecessary. We measure in phase 6.
|
||||
|
||||
### 4.5 Latency targets
|
||||
|
||||
| Path | Target |
|
||||
|---|---|
|
||||
| Node writes a block's events → ingest has applied them (book, tape, bar) | **< 5 ms / < 20 ms** p50/p99 [target], using `--disable-output-file-buffering` [doc: LAT] |
|
||||
| Ingest event → written to client sockets by the gateway (same region, p50 / p99) | **< 5 ms / < 25 ms** [target] |
|
||||
| Subscribe → Snapshot for a hot topic (already in memory) | **< 50 ms** plus RTT [target] |
|
||||
| Subscribe → Snapshot for a cold topic | **< 100 ms** plus RTT [target]. Ingest already holds every book. |
|
||||
| Quote fan-out cadence to UI | Default 4 Hz per watchlist. The visible chart's bar and book stream at block speed. [target] |
|
||||
|
||||
For comparison, the public `l2Book` feed is pushed "on each block that is at least 0.5 since last push" [doc: WS-SUB]. Our own builder emits on every block. The docs say node outputs are "faster and more granular data than the API" [doc: LAT], but give no numbers [unverified until we measure]. For North American users, the Tokyo-to-client network hop dominates end to end [unverified RTT; to measure].
|
||||
|
||||
### 4.6 Client integration
|
||||
|
||||
- **`tr-data::remote::RemoteMarkets`** implements `MarketData` over one shared connection. `history` maps to an RPC, `quotes` to a short-lived or shared `Quotes` subscription, and `live` to a `Bars` subscription bridged into the existing `mpsc::Receiver<Bar>`.
|
||||
- **Selection:** `TR_SERVER_URL` (or a settings toggle) picks `RemoteMarkets`. The direct providers stay as the offline and dev fallback (`TR_PROVIDER=sample` keeps working).
|
||||
- **Web build:** `for_browser()` puts `RemoteMarkets` in front of the browser's own sources when the page was built with a server URL (`TR_SERVER_URL`, read by `scripts/build-web.sh`; production uses `wss://terminal-gateway.style.dev/ws`). The client runs on the page's event loop over the browser's `WebSocket`, with postcard in binary frames like the desktop. `?server=ws://localhost:8787/ws` points a page at a server on the same machine, and `?server=off` turns the server off. Crypto is the only asset class the server serves so far, so the `TODO(web proxy)` stubs in `crates/data/src/web.rs` and `crates/congress/src/db_web.rs` stay until it serves the rest.
|
||||
|
||||
---
|
||||
|
||||
## 5. Ingest from our node
|
||||
|
||||
Everything live comes from our own non-validator node (§6) through `tr-ingest`, which runs on the node host.
|
||||
|
||||
### 5.1 What the node writes, and the flags we run
|
||||
|
||||
`hl-visor run-non-validator` writes to `~/hl/data`. The output we use is opt-in via flags [doc: NODE]:
|
||||
|
||||
| Flag | Output | We use it for |
|
||||
|---|---|---|
|
||||
| `--write-fills` | `node_fills` (API fills format). It "overrides `--write-trades` if both are set" and also writes TWAP statuses. HIP-3 fills include `deployerFee`. | The trade tape and candles (§5.3). Fees and user fields are dropped before anything leaves ingest. |
|
||||
| `--write-order-statuses` | `node_order_statuses`, every order status update, "a substantial amount of data" | L4 order metadata (tif, cloid, timestamp) and order lifecycle (§5.2) |
|
||||
| `--write-raw-book-diffs` | `node_raw_book_diffs`: per-order `new {sz}`, `update {origSz, newSz}` or `remove` with `user, oid, coin, side, px` [doc: L1] | Applying changes to the L4 book (§5.2) |
|
||||
| `--write-hip3-oracle-updates` | `hip3_oracle_updates`: every HIP-3 deployer `setOracle` | HIP-3 oracle and mark inputs (§5.4) |
|
||||
| `--write-misc-events` | `misc_events`, including a `Funding` event `{coin, usdc, szi, fundingRate, nSamples}` and ledger updates [doc: L1] | Realised funding (§5.4) |
|
||||
| `--batch-by-block` | Writes `*_by_block` files, one line per block: `{local_time, block_time, block_number, events}` | Block-aligned processing, gap detection, deduplication across two nodes |
|
||||
| `--disable-output-file-buffering` | Flushes each line as soon as it's written | Latency [doc: LAT] |
|
||||
| `--serve-info` | Local info server at `localhost:3001/info` with a subset of info requests, plus `fileSnapshot` | The registry, L4 snapshots, freshness checks, marks (§5.4) |
|
||||
| `--replica-cmds-style actions-and-responses` (optional) | `replica_cmds`: every L1 transaction | Core oracle derivation, if needed (§5.4). Off unless phase 1 shows we need it, because it is large [unverified size]. |
|
||||
|
||||
Every output is hourly-partitioned under `{dir}/hourly/{date}/{hour}` [doc: NODE]. Ingest tails the current file with a checkpoint `(file, byte offset, last block_number)`, written after the derived state has been persisted. So a restart resumes exactly where it stopped. The docs say batching by block costs "a few milliseconds" against the streaming mode [doc: OBS, LAT]. v1 keeps `--batch-by-block` for correctness, and we revisit it once we measure.
|
||||
|
||||
### 5.2 Order books: can we rebuild full L2 and L4? Verdict: yes, with caveats
|
||||
|
||||
**Yes for core perps.** The node's own docs describe this exact use:
|
||||
- L4 snapshots hold "the entire order book with full information about the orders for each level. This can be used as a checkpoint upon which the order statuses stream may be applied, allowing users to stream an L4 book in realtime" [doc: L1].
|
||||
- The latency guide says node outputs let us "construct the book" locally, with "faster and more granular data than the API" [doc: LAT].
|
||||
- The reference [`order_book_server`](https://github.com/hyperliquid-dex/order_book_server) does it in Rust. It serves `l2Book` (up to 100 levels), `trades` and a new `l4Book` over a websocket, built from the `*_by_block` files for fills, order statuses and raw book diffs [doc: OBS].
|
||||
|
||||
**How the rebuild works** (the reference server's approach, which we copy) [doc: OBS; observed in source]:
|
||||
1. Checkpoint: POST `{"type":"fileSnapshot","request":{"type":"l4Snapshots","includeUsers":true,"includeTriggerOrders":false},"outPath":…,"includeHeightInOutput":true}` to the local info server. It writes the current L4 book for every coin, at a known height, and "uses the latest state instead of an older snapshot file" [doc: NODE]. As an offline alternative, `hl-node compute-l4-snapshots` can build one from a `periodic_abci_states` file (written every 10,000 blocks) [doc: NODE].
|
||||
2. Apply every raw book diff and order status from the `*_by_block` files with `block_number` above the snapshot height, in order. `new` inserts an order (`oid`, side, `px`, size) at the back of its price level. `update` changes its size. `remove` deletes it. Order statuses give the metadata the diffs lack (timestamp, tif, cloid, order type).
|
||||
3. L2 is an aggregation of L4: Σ size and order count `n` per price. We do any number of levels and our own price grouping, without the API's 20-level cap [doc: INFO]. BBO and mid come out of L2.
|
||||
4. Self-check: the reference server periodically takes a fresh snapshot, compares it with its in-memory book, and exits on a mismatch. It also exits if no events arrive for 5 s [observed in source]. We do the same, but rebuild that coin instead of exiting.
|
||||
|
||||
**Caveats and gaps (stated plainly):**
|
||||
- **HIP-3 coverage is likely but not documented.** Nothing in the node docs says explicitly whether HIP-3 dex books appear in `node_raw_book_diffs`, `node_order_statuses` or `l4Snapshots`. The evidence points to yes:
|
||||
- the fills docs mention `deployerFee` "for HIP-3 fills" [doc: NODE];
|
||||
- HIP-3 markets trade on HyperCore's matching engine [doc: HIP3];
|
||||
- the reference server takes its coin universe from the snapshot and has no dex filter [observed in source].
|
||||
|
||||
**[unverified]**. It's the phase 1 acceptance test (§12): grep a day of diffs and a snapshot for `xyz:`, `flx:` and so on, and compare one `xyz:` book against the public `l2Book` once. If HIP-3 books are missing, that is a real gap. We'd have no source for HIP-3 books except the public API, which goes against the decision. We'd come back to Micky before going further.
|
||||
- **Spot:** the reference README says it "does not support spot order books", but its code has `is_spot` handling (`@…` and `PURR/USDC`) and an `ignore_spot` switch [doc: OBS; observed in source]. Spot diffs should be in the same files [unverified]. Our builder treats spot like perps and verifies it the same way.
|
||||
- **Trigger orders** (stop and take-profit) aren't resting liquidity. The reference server leaves them out [doc: OBS]. Snapshots can include them with `isTrigger`, at the end of each level [doc: L1]. We leave them out of L2 too.
|
||||
- **Queue position:** "ALO priority fees or other edge cases" can break naive time priority within a level [doc: LAT]. L2 is unaffected. L4 order position within a level is best-effort [unverified]. The snapshot self-check catches drift.
|
||||
- **The reference code is unmaintained.** It is "not actively maintained" [doc: LAT], "not written by the core team" and provided as-is [doc: OBS]; its last commit was 2026-07-15 [observed]. So we write our own `tr-book` crate, using it as a reference, and we test it against recorded node files.
|
||||
|
||||
### 5.3 Trades and candles at every interval, no cap
|
||||
|
||||
- **Tape:** with `--write-fills`, each match shows up as two fills, one per side. Ingest dedupes them into one public trade, keyed by `(coin, tid)` [unverified that node fills carry `tid`; the trades schema carries `hash` plus buyer and seller `oid`s, which works as a fallback key, doc: L1]. The aggressor side comes from the taker fill.
|
||||
- **1m bars** are built from the deduplicated trades by UTC minute: OHLC from trade prices, volume Σ sz (base), notional Σ px·sz, and trade count `n`. A bar closes when the first block with a `block_time` at or after the minute boundary has been applied. That makes closes deterministic and identical on both nodes.
|
||||
- **Higher intervals** are aggregated from 1m. OHLCV aggregation is exact, so 5m, 15m, 1h, 4h, 1d, 1w and 1M (and any custom interval) are consistent by construction. Weeks start Monday 00:00 UTC and months are calendar months [target; Hyperliquid's own boundary convention is undocumented, unverified].
|
||||
- **Minutes with no trades** produce no bar row. Clients render the gap, which matches how sparse HIP-3 markets look today.
|
||||
- **No cap:** every bar since our node started is kept (§3.4). This replaces the public API's "most recent 5000 candles" limit [doc: INFO].
|
||||
- Live bars reach clients as `Forming` and `Closed` (§4.3).
|
||||
|
||||
### 5.4 Mids, marks, oracle prices and funding
|
||||
|
||||
| Value | Where it comes from | Confidence |
|
||||
|---|---|---|
|
||||
| **Mid** | Our book: `(best bid + best ask) / 2`. One-sided or empty books publish no mid, and clients fall back to mark with a label. | Derived. Exact for two-sided books. |
|
||||
| **Last trade** | Our tape (§5.3) | Exact |
|
||||
| **HIP-3 oracle and mark inputs** | `--write-hip3-oracle-updates` records each deployer `setOracle`, which carries `oraclePxs`, `markPxs` and `externalPerpPxs` per coin [doc: NODE, HIP3-ACT]. The oracle price is used as-is. | Exact for the oracle |
|
||||
| **HIP-3 mark** | The protocol forms the final mark from the deployer's `markPxs` with documented rules: the mark is clamped to move at most 1% per update, and stale marks fall back to the local mark after 10 s [doc: HIP3-ACT]. We implement those rules and reconcile the result against `activeAssetData` (below). | The final combination is [unverified]. To validate in phase 1. |
|
||||
| **Core perp oracle** | Validators publish oracle prices about every 3 s. Each is a weighted median of CEX spot prices (Binance 3, OKX 2, Bybit 2, Kraken, Kucoin, Gate, MEXC and Hyperliquid spot 1 each), and the final price is a stake-weighted median of the validators' submissions [doc: ORACLE]. On the node, these validator actions should be in `replica_cmds` [unverified; the action schema is undocumented]. | Derivable [unverified]. Fallback: the abci state (below). |
|
||||
| **Core perp mark** | The median of (a) oracle + a 150 s EMA of (Hyperliquid mid − oracle), (b) the median of best bid, best ask and last trade, and (c) a weighted median of Binance, OKX, Bybit, Gate and MEXC perp mids [doc: MARK]. We can compute (a) and (b), but **not (c)** from node data. So we **read** mark rather than derive it. The local info server supports `activeAssetData`, which returns `markPx` per coin [doc: NODE, PERP]. Ingest polls it locally (no rate limit) every 1 s for watched coins and every 3 s for the rest [target]. Mark only updates with the oracle, about every 3 s [doc: MARK]. | [unverified that the local server returns a live `markPx`]. Fallback: a computed approximation from (a) and (b), labelled "est.". |
|
||||
| **Funding (realised)** | Hourly `Funding` events in `misc_events` carry `coin` and `fundingRate` [doc: L1] | Exact |
|
||||
| **Funding (predicted)** | Computed with the documented formula: the hourly rate is ⅛ of the 8 h rate, F = average premium + clamp(interest − premium, ±0.0005), interest is 0.01% per 8 h, the premium is sampled every 5 s from impact prices against the oracle, and funding is capped at 4% per hour [doc: FUNDING]. Impact prices come from our own book. HIP-3 dexes can set their own multipliers and interest rates [doc: HIP3-ACT] [unverified where these show up in node output]. | Derived |
|
||||
| **Open interest, 24h volume, prev-day price** | Volume and the previous day's price come from our candles. Open interest isn't in the documented outputs. We'd derive it from the `periodic_abci_states` state, translated with `translate-abci-state` [doc: NODE], at that file's cadence, or track it live from fills (`start_pos`) [unverified; later]. | Partial |
|
||||
|
||||
Reconciliation: every 10,000 blocks, a job translates the `periodic_abci_states` file and checks our oracle, mark, funding and open interest against it [unverified that every field is present]. Mismatches raise an alert.
|
||||
|
||||
### 5.5 Catch-up, gaps and failover inside ingest
|
||||
|
||||
- **Restart:** ingest resumes from its checkpoint (§5.1). Books are never trusted across a restart: it takes a fresh L4 snapshot (§5.2) and replays diffs after that height.
|
||||
- **Gaps:** `block_number` must increase by exactly 1. A gap or a stall (no new block for 5 s [target], the same threshold the reference server uses) marks every topic `Stale`. Ingest re-snapshots the books, and asks the peer node's ingest for the missing blocks' trades so bars are complete (§6.6). If both nodes miss a range, a repair job fills the trades later from the S3 archive (§5.6), and the affected bars are flagged `repaired`.
|
||||
- **Node restart:** see §6.5.
|
||||
|
||||
### 5.6 The one-time history backfill (the only public data use)
|
||||
|
||||
Node-built bars cover everything from the first full minute after our node is live. History before that is a one-time job, run once and then never again:
|
||||
|
||||
1. **Primary: S3 node archives, rebuilt into trades and then candles with the same code as live.** The bucket `hl-mainnet-node-data` holds `node_fills_by_block` (written with `--write-fills --batch-by-block`), older `node_fills` (API format) and `node_trades` (a different, older format), plus `misc_events_by_block` (funding and more), `explorer_blocks` and `replica_cmds` [doc: HIST]. The archives are **requester-pays**: we pay the S3 transfer, which needs our own AWS account [doc: HIST]. The **start date and total size are not documented** [unverified]. Listing the bucket needs AWS credentials, which I don't have, so phase 1 measures them first. This route gives true 1m (and finer) history and is consistent with live bars.
|
||||
2. **Secondary: one public `candleSnapshot` sweep**, for instruments or ranges the archive lacks. It returns only the most recent 5000 candles per interval [doc: INFO]: 1m ≈ 3.5 days, 1h ≈ 208 days, 1d back to listing (BTC 1d starts 2020-08-19) [observed]. The weight math for a one-off sweep: ~900 instruments (552 perp entries including delisted, plus 347 spot pairs [observed, §5.7]) × 5 intervals (1m, 15m, 1h, 4h, 1d) × ~104 weight (20 + 5000/60) ≈ 468k weight. That is ≈ 7.8 hours at a 1000/min self-limit, under the 1200/min IP limit [doc: RL]. It runs once, from one IP, with no evasion of limits.
|
||||
3. **Optional: `hyperliquid-archive/asset_ctxs/[date].csv.lz4`** for daily historical funding, open interest and mark context. It's uploaded about monthly, "with no guarantee", and has no candles or spot asset data [doc: HIST].
|
||||
|
||||
**Stitching:** for each `(instrument, interval)`, node-built bars win for every bar at or after node start. Backfill bars fill the earlier range, and each bar keeps its provenance. Where S3-rebuilt and `candleSnapshot` bars overlap, we compare them as a test of our aggregation. Any differences are logged, and the S3 bars win.
|
||||
|
||||
### 5.7 Addressing (HIP-3, spot and outcomes)
|
||||
|
||||
- `perpDexs` returns `[null, {name, fullName, deployer, oracleUpdater, feeRecipient, assetToStreamingOiCap, …}, …]`. `null` is the first (core) perp dex [doc: PERP]. I observed 11 entries: `null`, `xyz` (XYZ), `flx` (Felix Exchange), `vntl` (Ventuals), `hyna` (HyENA), `km` (Markets by Kinetiq), `abcd` (ABCDEx), `cash` (dreamcash), `para` (Paragon), `mkts` (Markets By Kinetiq) and `io` (EntropyIO). The objects also carried `subDeployers`, `assetToFundingMultiplier`, `assetToFundingInterestRate` and `assetToFundingClamp`, which are not in the doc example [observed].
|
||||
- `meta`, `metaAndAssetCtxs`, `allMids`, `clearinghouseState` and others take `"dex": "<name>"`. The empty string means the first perp dex [doc: PERP, INFO]. `allPerpMetas` returns every dex at once [doc: PERP].
|
||||
- HIP-3 coin names are always `{dex}:{coin}`, for example `xyz:TSLA` [doc: ASSET, INFO]. `candleSnapshot` needs the prefix: "For HIP-3, you need to prefix with the dex name, e.g. "xyz:XYZ100"" [doc: INFO]. The WS `allMids` with `dex:"xyz"` returned only `xyz:`-prefixed keys plus a `dex` field [observed].
|
||||
- Spot coins are `PURR/USDC` or `@{index}` (for example, HYPE is `@107`). "Some assets may be remapped on user interfaces", for example BTC/USDC in the app is UBTC/USDC on HyperCore [doc: INFO].
|
||||
- Outcome (HIP-4) markets show up as `#<encoding>` keys [doc: ASSET]. I observed 762 `#` keys and 401 `@` keys among the 1399 keys of the default `allMids` [observed]. **Outcomes are out of scope** and are filtered at ingest. They also carry separate legal restrictions [doc: TOU §1.7].
|
||||
- Snapshot on 2026-10-29 [observed]: the core `meta` had 235 entries, 56 flagged `isDelisted`. The `xyz` `meta` had 141 entries, 21 flagged `isDelisted`. `spotMeta` had 347 pairs and 511 tokens. `allPerpMetas` listed per-dex universe sizes of core 235, `xyz` 141, `para` 39, `hyna` 25, `mkts` 24, `km` 23, `cash` 17, `flx` 16, `io` 16, `vntl` 15 and `abcd` 1, for 552 perp entries in total.
|
||||
- **In our system** the registry comes from the node's local info server, which supports `perpDexs`, `meta` and `spotMeta` [doc: NODE]. The public-API observations above are from research and are only for scale.
|
||||
|
||||
### 5.8 Backpressure
|
||||
|
||||
- Each client has a **bounded outbound queue** (start at 256 frames or 1 MB [target]).
|
||||
- **State topics** are conflated per client. On each flush tick (or immediately, for the active chart), the server writes the latest value of every dirty topic, so a slow client gets fewer updates but never stale ones.
|
||||
- **Event topics:** if a client's cursor falls behind the ring, the server drops that subscription's backlog and sends `Resync` plus a `Snapshot`. For trades, the snapshot is the last N trades.
|
||||
- If the queue stays full for more than 10 s [target], the server closes the connection with a "slow consumer" code, and the client reconnects with `resume`.
|
||||
- **Ingest is never blocked by a client.** The node-file tailers only write into the hot store. Fan-out runs in per-connection tasks.
|
||||
|
||||
### 5.9 Scale without Hyperliquid rate limits
|
||||
|
||||
- Ingest work is proportional to market activity (all coins, every block), not to clients. The gateway's work is proportional to client subscriptions. Neither touches a Hyperliquid rate limit.
|
||||
- The public API limits (1200 weight/min, 1000 WS subscriptions, 10 connections per IP [doc: RL]) apply only to the one-time backfill (§5.6). That is the sanctioned way to get unlimited data: the ToU forbids bots that "bypass rate limits" [doc: TOU §3.1.8], and a node avoids the question entirely.
|
||||
- The internal ingest-to-gateway stream carries per-block batches of book changes, trades, bars and prices for every coin. Its bandwidth is unmeasured [unverified; phase 2 measures it]. Gateways can subscribe only to the topics their clients use.
|
||||
|
||||
---
|
||||
|
||||
## 6. Node operations
|
||||
|
||||
### 6.1 Hosting choice and region
|
||||
|
||||
- **Region: Tokyo.** The README says "for lowest latency, run the node in Tokyo, Japan" [doc: NODE], and the Foundation's own node is in AWS `apne1-az1` [doc: FND].
|
||||
- **We are likely ineligible for the Foundation node** (it requires 10k staked HYPE and Tier 1 maker volume, and excludes "certain activities involving the U.S. or Ontario") [doc: FND]. So we run our own.
|
||||
- **Provider:** single-tenant bare metal in Tokyo from two different providers, so a single provider outage doesn't take out both (§6.6). The comparison, prices and recommendation are in §6.3: Vultr for node A and Cherry Servers for node B.
|
||||
- **Ruled out:**
|
||||
- Freestyle (freestyle.sh): VMs too small (64 GiB / 256 GB max), and no raw inbound TCP for gossip [doc: FREESTYLE].
|
||||
- OVHcloud: no Japan region [doc: OVH-REGIONS].
|
||||
- Equinix Metal: discontinued [doc: EQX-EOL].
|
||||
- **Peers:** mainnet requires at least one root peer in `~/override_gossip_config.json`. Community root peers are mostly in Japan, and a `gossipRootIps` info request lists recent ones [doc: NODE]. The Foundation also lists paid infra providers that offer reliable peering [doc: INFRA-PEERS]. Running "against a reliable peer" is the first latency recommendation [doc: LAT]. Peer addresses live in the host's config, never in this repo or its docs.
|
||||
|
||||
### 6.2 Machine spec and OS
|
||||
|
||||
- **Minimum (non-validator):** 16 vCPU, 128 GB RAM, 500 GB SSD [doc: NODE]. "Currently only Ubuntu 24.04 is supported" [doc: NODE].
|
||||
- **For low latency:** at least 32 logical cores, 128 GB RAM and 500 MB/s disk [doc: LAT].
|
||||
- **Our target per node:** bare metal with 16+ physical cores (32+ threads), 128 GB+, **≥ 2 TB local NVMe**. Disk is sized for ~1–2 days of raw output (§6.4) plus abci states and the candle store. Ubuntu 24.04 LTS. Ingest, and in v1 the gateway, share the box (§6.3, "Colocating our Rust server") [unverified until measured].
|
||||
- **Network:** gossip ports 4001 and 4002 open publicly [doc: NODE]. Everything else, including the info server and ingest's internal stream, is firewalled, with the internal stream over WireGuard or mTLS. Node bandwidth is undocumented [unverified; measure in week 1].
|
||||
- **Install:** `hl-visor` binaries are signed. Verify them with the published GPG key, and run `hl-visor run-non-validator` with `~/visor.json` set to `{"chain": "Mainnet"}` [doc: NODE]. Run both visor and ingest under systemd. `hl-visor` auto-updates `hl-node` [doc: NODE], so we alert on version changes and re-verify ingest parsers after each one.
|
||||
|
||||
### 6.3 Where to host it: providers checked, with published prices
|
||||
|
||||
Requirements (§6.2):
|
||||
- 16+ physical cores (32+ logical) and 128 GB+ RAM;
|
||||
- 500 GB+ local NVMe, ideally 2 TB;
|
||||
- Ubuntu 24.04;
|
||||
- **raw public TCP on 4001–4002**;
|
||||
- high bandwidth;
|
||||
- in or near Tokyo.
|
||||
|
||||
Prices are list prices, checked 2026-10-29, excluding tax. "Stock" is what the provider's inventory showed that day. None of these are quotes.
|
||||
|
||||
| Provider and plan (Tokyo unless noted) | Hardware | Monthly price | Bandwidth | Stock | Source |
|
||||
|---|---|---|---|---|---|
|
||||
| **Vultr bare metal `vbm-24c-256gb-amd`** | EPYC 7443P, 24 cores / 48 threads, 256 GB. The product page also lists 2 × 1.92 TB NVMe; the public plans API lists only the 2 × 480 GB SSDs [confirm at order]. 25 Gbps. | **$725** (hourly $0.993) | 10 TB included, then $0.01/GB | Offered in Tokyo (`nrt`) and Osaka (`itm`) | [doc: VULTR-API, VULTR-BM, VULTR-PRICE] |
|
||||
| Vultr bare metal `vbm-24c-384gb-amd5` | EPYC 9255, 24c/48t, 384 GB, NVMe | $995 | 10 TB | Tokyo and Osaka | [doc: VULTR-API] |
|
||||
| Vultr bare metal `vbm-8c-132gb` | E-2288G, **8 cores**, 128 GB, 2 × 1.92 TB NVMe | $350 | 10 TB | Tokyo | **Fails** the 16-core minimum [doc: VULTR-API] |
|
||||
| **Cherry Servers AMD EPYC 9275F** | 24c/48t at 4.1 GHz, 384 GB, 2 × 1 TB + 2 × 2 TB NVMe, 10 Gbps, Equinix TY15 | **$874.17** | 100 TB free egress | Listed as deployable in Tokyo | [doc: CHERRY-TYO] |
|
||||
| Cherry Servers AMD Ryzen 9950X | 16c/32t, 128 GB (non-ECC), 2 × 1 TB NVMe, 10 Gbps | €439 (€373 on a 12-month term) | 100 TB | **Sold out in Tokyo** | [doc: CHERRY-9950X] (third-party tracker) |
|
||||
| Latitude.sh `m4.metal.medium` | EPYC 9124, 16c/32t, 128 GB, 2 × 480 GB + 2 × 1.9 TB NVMe, 2 × 10 Gbps | $522 Tokyo ($365 on a 12-month term); $456 US list | 20 TB | **Sold out** in TYO3 and TYO4 | [doc: LATITUDE, LAT-M4] (Tokyo price from a third-party tracker) |
|
||||
| Latitude.sh `f4.metal.medium` | EPYC 4564P, 16c/32t at 4.5 GHz, 128 GB, same disks | $715 Tokyo ($501 on term); $555 US list | 20 TB | **Sold out** in Tokyo | [doc: LATITUDE, LAT-F4] |
|
||||
| AWS `i4i.4xlarge` (on demand) | 16 vCPU (8 cores), 128 GiB, 1 × 3,750 GB local NVMe | $1.61/h ≈ **$1,175** | Egress billed separately | `ap-northeast-1` | [doc: AWS-PRICE] |
|
||||
| AWS `r6i.4xlarge` + 2 TB gp3 | 16 vCPU, 128 GiB, EBS only | $1.216/h ≈ $888, plus ≈ $215 EBS ≈ **$1,100** | Egress billed separately | `ap-northeast-1` | [doc: AWS-PRICE]; EBS from earlier third-party summaries |
|
||||
| OVHcloud | — | — | — | **No Japan region.** Its APAC dedicated servers are in Singapore, Sydney and Mumbai only, with metered traffic that is throttled at the quota. | [doc: OVH-REGIONS, OVH-APAC] |
|
||||
| Equinix Metal | — | — | — | **Discontinued.** Sunset on 2026-06-30. | [doc: EQX-EOL] |
|
||||
| Freestyle (freestyle.sh) VMs | Pro tops out at 32 vCPU / **64 GiB** / **256 GB** per VM | — | — | **Ruled out:** below the 128 GB RAM and 500 GB disk minimums, and inbound traffic only reaches a VM through the TLS/SNI edge on 443, so there's no raw TCP on 4001–4002 for gossip | [doc: FREESTYLE] |
|
||||
|
||||
All three bare-metal providers above are single-tenant with full root, so we open 4001–4002 and firewall the rest ourselves with nftables. Ubuntu 24.04 is in Vultr's OS list [doc: VULTR-API] and Cherry's Tokyo FAQ [doc: CHERRY-TYO]. Node bandwidth needs are undocumented [unverified]. Vultr's 10 TB is the tightest allowance, and its overage ($0.01/GB) is cheap.
|
||||
|
||||
**Recommendation:**
|
||||
- **Node A: Vultr `vbm-24c-256gb-amd` in Tokyo, $725/month.**
|
||||
- 24 cores / 48 threads meets the 32-logical-core latency advice [doc: LAT].
|
||||
- 256 GB of RAM leaves headroom for ingest and the gateway on the same box.
|
||||
- It's orderable today.
|
||||
- Confirm the NVMe drives at order time.
|
||||
- **Node B: Cherry Servers EPYC 9275F in Tokyo (Equinix TY15), $874.17/month.**
|
||||
- A different provider and building, so one provider outage doesn't take out both (§6.6).
|
||||
- 100 TB egress, and faster cores.
|
||||
- Alternatively, a second Vultr box in Osaka (`itm`): a different city, but the same provider.
|
||||
- **Two-node total ≈ $1,600/month.** Budget path: one node at ≈ $725, or the 16-core Latitude and Cherry boxes ($365–522) once Tokyo stock returns.
|
||||
- **AWS works but costs more for less:** 16 vCPU is 8 physical cores, and egress is extra. It mainly helps the S3 backfill (§5.6), which can run from a short-lived AWS instance instead.
|
||||
|
||||
**Colocating our Rust server:**
|
||||
- **`tr-ingest` runs on the node box** (required: it tails local files and calls `localhost:3001`).
|
||||
- **v1: `tr-gateway` on the same box as well.** The node gets most of the cores via systemd `CPUAffinity`/cgroup limits, and ingest plus the gateway get a reserved slice (for example 8 of 48 threads) [target; tune after measuring]. That makes the internal stream loopback, with no extra machine.
|
||||
- **When client load grows,** move `tr-gateway` to a small box on the **same provider's private LAN** in the same city (Vultr VPC or Cherry private VLAN [doc: CHERRY-TYO]), so the ingest-to-gateway hop stays sub-millisecond and off the public internet [unverified latency; measure]. Never put the gateway across the Pacific from ingest. The data starts in Tokyo either way (§11).
|
||||
|
||||
**Cost summary (per month, list):**
|
||||
- **One node:** ≈ **$725** (Vultr bare metal). A 16-core Latitude or Cherry box would be ≈ $365–522 when Tokyo stock returns; AWS is ≈ $1,100–1,175.
|
||||
- **Two nodes (recommended, §6.6):** ≈ **$1,600** (Vultr + Cherry). The range is ≈ $1,450–2,050, depending on which pair we use.
|
||||
- These are list prices, not quotes. Third-party tracker figures are labelled as such in the table.
|
||||
|
||||
### 6.4 Disk and log rotation (~100 GB/day)
|
||||
|
||||
- The node writes "around 100 GB of logs per day", which should be archived or deleted [doc: NODE]. Order statuses and raw book diffs add "a substantial amount" on top [doc: NODE]. We size for **100–200 GB/day** [unverified; measure in week 1].
|
||||
- **Pruner** (a systemd timer, hourly):
|
||||
- Delete hourly output directories older than **24 h** [target], *only* when ingest's committed checkpoint is past them. The pruner never deletes a file that ingest hasn't committed.
|
||||
- Keep the last 3 `periodic_abci_states` files.
|
||||
- Keep `replica_cmds` only if we enable it, with `--replica-cmds-style recent-actions` where possible [doc: NODE].
|
||||
- Optionally compress `node_fills_by_block` to object storage (zstd), as our own archive for repairs.
|
||||
- **Alerts:** disk use above 70% (warn) and 85% (page). If the disk fills, the node stops writing and ingest goes stale.
|
||||
|
||||
### 6.5 Monitoring and restart catch-up
|
||||
|
||||
- **Liveness:**
|
||||
- `applied block X` log lines show the node is live [doc: NODE]. We export the block height and blocks per second.
|
||||
- Ping the local `exchangeStatus` to compare the L1 timestamp with the local timestamp [doc: NODE]. We alert when it's behind by more than 2 s [target].
|
||||
- Compare the heights of the two nodes.
|
||||
- **Crashes:** `visor_child_stderr` and `node_logs/status` [doc: NODE] are shipped to logs, with an alert on new stderr content and on restarts.
|
||||
- **Restart catch-up:** after a restart, the node "may take a while as the node navigates the network to find an appropriate peer" [doc: NODE]. How long catch-up takes, and whether output files are written for the blocks it replays, isn't documented [unverified; we test it with a planned restart in phase 1]. Ingest treats the node as unusable until `exchangeStatus` is fresh, then resumes from its checkpoint (§5.5). Any block range without output is filled from the other node.
|
||||
|
||||
### 6.6 Failover with a second node
|
||||
|
||||
- **Two nodes, A and B,** in Tokyo on different providers or availability zones. Each runs its own ingest and builds its own books and bars.
|
||||
- **Primary and standby:** the gateway consumes both streams, tagged with `block_number`. One is primary. The gateway switches when the primary is behind by more than 20 blocks or 2 s [target], or stops sending. On a switch, it sends `Resync` for the book topics (books are per node), and uses `block_number` to drop duplicate or out-of-order trades and bar updates.
|
||||
- **Bars stay identical** on both nodes because bar closes are block-time based (§5.3). The standby's candle store doubles as a hot replica for repairing gaps on the primary.
|
||||
- **Upgrades:** `hl-visor` auto-upgrades [doc: NODE]. Both nodes upgrading at once is a correlated failure we can't fully prevent [unverified whether upgrades can be staggered]. We alert on it.
|
||||
- **Single node (budget fallback):** one node plus the replay and repair path (S3, §5.6). Outages then mean stale data until the node catches up, shown as `Stale` in the UI.
|
||||
|
||||
## 7. Symbol mapping to Bloomberg-style keys
|
||||
|
||||
### 7.1 Principles
|
||||
|
||||
1. **A perp is never silently shown as a stock.** If `AAPL US Equity` is backed by `xyz:AAPL`, every surface shows a **`PERP·xyz`** badge, the attribution "trade.xyz perp on Hyperliquid", and the price basis (`mid` or `mark`). Tooltips explain that this is a perpetual futures price, not the exchange-listed share price.
|
||||
2. **Venue-native keys always work.** `xyz:AAPL <GO>`, `BTC-PERP`, `@107` and `HYPE/USDC` resolve directly.
|
||||
3. **The server owns the mapping table** (a versioned config, not code), so we can fix it without a client release.
|
||||
|
||||
### 7.2 Model changes (in `tr-core`)
|
||||
|
||||
```rust
|
||||
pub enum InstrumentKind { Spot, Perp, Equity, Etf, Index, Fx, Future, Govt }
|
||||
pub struct VenueRef { provider: ProviderId, venue: String /* "hyperliquid", "xyz" */, symbol: String /* "xyz:AAPL" */ }
|
||||
|
||||
pub struct Resolution {
|
||||
pub key: MarketKey, // what the user typed / the pane is linked to: "AAPL US Equity"
|
||||
pub source: VenueRef, // what actually backs it: hyperliquid / xyz / "xyz:AAPL"
|
||||
pub kind: InstrumentKind, // Perp
|
||||
pub proxy: bool, // true: the source is a different instrument than the key
|
||||
pub basis: PriceBasis, // Mid | Mark | Oracle | Last
|
||||
pub session: Option<SessionCalendar>, // underlying's hours (for "underlying closed" labels)
|
||||
}
|
||||
```
|
||||
|
||||
`Security { ticker, class }` stays as it is for back-compat. `Resolution` travels alongside it in `Quote`, `Snapshot` and `SearchHit`.
|
||||
|
||||
### 7.3 Initial mapping table (sketch; every row needs checking against the trade.xyz spec index)
|
||||
|
||||
| App key | Backed by | Kind | Notes |
|
||||
|---|---|---|---|
|
||||
| `AAPL US Equity`, `TSLA`, `NVDA`, `MSFT`, … | `xyz:<TICKER>` | Perp (proxy) | Only for tickers present and not `isDelisted` in `meta{dex:"xyz"}` |
|
||||
| `SPX Index` | `xyz:SP500` | Perp (proxy) | Whether it references the S&P 500 index exactly is [unverified] |
|
||||
| `NDX Index` | `xyz:XYZ100` | Perp (proxy) | XYZ100's exact reference is [unverified] |
|
||||
| `NKY Index`, `KOSPI2 Index` | `xyz:JP225`, `xyz:KR200` | Perp (proxy) | [unverified] references |
|
||||
| `GC1 Comdty`, `SI1`, `HG1`, `CL1`, `CO1`, `NG1` | `xyz:GOLD`, `SILVER`, `COPPER`, `CL`, `BRENTOIL`, `NATGAS` | Perp (proxy) | The futures roll vs spot reference is [unverified] |
|
||||
| `EURUSD Curncy`, `GBPUSD`, `USDJPY` | `xyz:EUR`, `xyz:GBP`, `xyz:JPY` | Perp (proxy) | **The quote direction for JPY is [unverified]**. An inversion could be needed. |
|
||||
| `BTCUSD Curncy`, `ETHUSD`, … | core perp `BTC`, `ETH` (mid) | Perp | An alternative is HL spot (`UBTC/USDC`, remapped) [doc: INFO]. Default open question (§15). |
|
||||
| Other dexes (`flx:`, `km:`, `para:`, …) | venue keys only | Perp | Not mapped to Bloomberg keys in v1. They overlap with xyz (for example `flx:NVDA`) [observed]. |
|
||||
|
||||
`perpConciseAnnotations` returns categories such as `stocks` and `commodities` for HIP-3 coins [doc: PERP; observed]. We use it to suggest new rows, never to auto-map them.
|
||||
|
||||
### 7.4 Off-hours labelling
|
||||
|
||||
trade.xyz US stock perps take external prices 24/5, Sunday 8:05 PM ET to Friday 8:00 PM ET. That spans pre-market, regular, post-market and an overnight session from Blue Ocean ATS [doc: XYZ-US]. Outside external coverage, the oracle advances through an internal EMA of order-book impact prices, with τ = 30 min [doc: XYZ-ORACLE; changelog XYZ-TAU]. Mark price is bounded to ±(1/max leverage) around a reference price, which is the last external price over weekends [doc: XYZ-BOUNDS]. So the UI shows the session state (`Regular`, `Extended`, `Overnight`, `Closed: perp-only price discovery`) next to proxy quotes. A snapshot of the gap, from `xyz:AAPL` during US hours: `markPx 349.72` vs `oraclePx 349.8` [observed].
|
||||
|
||||
### 7.5 Delistings and churn
|
||||
|
||||
The registry refreshes `perpDexs`, `meta` (per dex) and `spotMeta` from the node's local info server every minute [target]. These calls are local, so there's no rate limit [doc: NODE]. If an instrument turns `isDelisted`, or disappears, the server marks the topic `Delisted`, stops live streams, keeps its history, and notifies users who have it in watchlists or layouts. HIP-3 deployers can also halt and settle a market with `haltTrading`, which can later resume and "recycle" the asset [doc: HIP3]. That means the *same name may come back as a different contract*. History is therefore keyed by `(dex, coin, listing_epoch)`, and a relisting starts a new epoch [target].
|
||||
|
||||
---
|
||||
|
||||
## 8. User sync: accounts, documents and conflicts
|
||||
|
||||
### 8.1 Accounts and auth
|
||||
|
||||
- **Sign-in:** OAuth (Apple, Google, GitHub) plus email magic link. Desktop uses the system browser with a loopback redirect (RFC 8252 style). The web uses a same-site HTTP-only session cookie. Mobile uses the platform's auth session.
|
||||
- **Tokens:** short-lived access tokens (JWT, 15 min [target]) plus refresh tokens, rotated and revocable. The WS `Hello` carries the access token. When it expires, the client re-authenticates in place without dropping subscriptions.
|
||||
- **Anonymous mode:** public market data with no sync, which matches today's experience.
|
||||
- **Build or buy:** a hosted identity provider (Clerk, WorkOS, Auth0 or Supabase Auth) vs a small self-hosted OAuth layer in Rust. See the open question in §15.
|
||||
- **Secrets:** SnapTrade per-user secrets are encrypted at rest (envelope encryption with a KMS or age key). They are never logged and never sent back to clients.
|
||||
|
||||
### 8.2 Documents
|
||||
|
||||
Each file in `storage.rs` becomes a server document `(user_id, kind, name)` with a server-assigned, monotonically increasing `rev`:
|
||||
|
||||
| Kind | Today | Server model | Conflict rule |
|
||||
|---|---|---|---|
|
||||
| Layouts | `LayoutDoc` JSON | Whole-document, optimistic concurrency (`base_rev`) | If `base_rev` doesn't match, the server keeps both and saves the losing write as "Name (conflict, Device, time)". This mirrors how people expect files to behave and never loses work. |
|
||||
| Watchlists | Watchlist panes or prefs | Ordered set of items with stable ids | Per-item ops (`Add`, `Remove`, `Move{after}`), last-writer-wins per item, with tombstones. Concurrent adds from two devices both survive. |
|
||||
| Drawings | `Vec<Drawing>` inside chart panes | Stored separately per `(user, market_key)`, each drawing with a **new stable `id`** (the `Drawing` struct needs one) | Last-writer-wins per drawing (`rev` and `updated_at`). Delete wins over a concurrent edit. Drawings follow the symbol across layouts. |
|
||||
| Prefs, themes, hotkeys, palette history | TOML/JSON files | Key-level merge | Last-writer-wins per key |
|
||||
| Demo portfolio | `DemoBook` JSON | **Server-authoritative.** Clients send commands, and the server fills against its own hot quotes and broadcasts the book. | No merge needed: one writer. Paper only; it never routes to a venue. |
|
||||
|
||||
Offline behaviour: clients keep today's local files as a cache and an outbox. Each queued `DocOp` carries an `op_id` (a UUID generated once, when the edit is made, and persisted with the outbox entry). The server records `(user_id, op_id) → result (rev or conflict outcome)` **in the same transaction as the write**, and keeps it for at least 30 days [target]. On reconnect, clients replay queued `DocOp`s with their original `op_id` and `base_rev`. If the server has already applied that `op_id`, it returns the original `DocAck` or `DocConflict` instead of re-evaluating `base_rev`. So a lost acknowledgement never creates a spurious conflict copy. Genuine conflicts resolve as above. The first sign-in uploads the local documents. If the server already has documents, the user chooses "merge" or "keep server".
|
||||
|
||||
The demo book fills at the server's quotes, which come from Hyperliquid perp prices (§7). So the demo account must show the same `PERP` labelling, and its fills can happen off-hours, a behaviour change from today [unverified UX decision].
|
||||
|
||||
---
|
||||
|
||||
## 9. Moving the existing feeds server-side
|
||||
|
||||
| Feed | Today | Server |
|
||||
|---|---|---|
|
||||
| Congress (Senate eFD, House Clerk) | Each desktop ingests every 30 min. The web gets nothing. | One `CongressProvider` runs `tr_congress::refresh` on schedule into the server DB. Topics: `Congress(Latest)`, `Congress(Ticker)`, `Congress(Politician)`. Tracking alerts are per-user pushes. The web gets the full feature. **Legal note:** the reports carry statutory limits on commercial use (Ethics in Government Act) [unverified for our use; needs legal review]. |
|
||||
| SnapTrade | Partner keys (`clientId`, `consumerKey`) and the user secret sit on each machine. The desktop uses them for reads *and* for order preview, place and cancel (`crates/broker/src/trading.rs`). | **Two modes.** (a) *Server mode* (web, mobile, and opt-in desktop): partner keys live only on the server. Each Terminal user maps to a SnapTrade `userId` and an encrypted `userSecret`, and the server proxies only the read-only endpoints (accounts, positions, orders, activities) plus the Connection Portal link. No trading in this mode. (b) *Local mode* (desktop, today's behaviour, unchanged): the local credentials stay in the Keychain or the 0600 file, and the desktop keeps calling SnapTrade directly for reads and for its existing order paths. This design does not migrate or remove local credentials, so existing desktop trading keeps working. Moving trading server-side is out of scope (§1) and would need its own design. |
|
||||
| Yahoo, Treasury, FRED, Alpaca, RSS | Client-side; the web gets sample data | Server-side adapters with `DisplayPolicy` (§3.1). Yahoo and RSS are off for public display until reviewed [unverified]. |
|
||||
| Binance and Coinbase crypto | Client-side websockets | Server-side, `public_display: false` (§3.1). Hyperliquid core perps and spot become the default crypto source. |
|
||||
|
||||
---
|
||||
|
||||
## 10. Observability
|
||||
|
||||
- **Tracing:** `tracing` with JSON logs and OpenTelemetry export. Logs carry no secrets, tokens or raw IPs (hash client IPs), and no user document contents.
|
||||
- **Prometheus metrics:**
|
||||
- **Node (per host):** block height, blocks per second, the `exchangeStatus` lag between L1 time and local time, the `hl-node` version, restarts, and new `visor_child_stderr` lines (§6.5). Also disk use and free space, output bytes per hour, network in/out, and the gap in heights between node A and node B.
|
||||
- **Ingest:** file-tail lag (block written → applied), last applied `block_number`, block gaps, snapshot self-check mismatches per coin, book rebuilds, trade dedupe counts, bars closed and repaired, and the age of the latest oracle and mark per coin.
|
||||
- **Hot store:** topics, refcounts, lingering topics, evictions, and staleness per instrument (p50/p99 and the worst 10).
|
||||
- **Gateway:** connections, subscriptions per connection, outbound queue depth, conflation drops, resyncs (by reason), slow-consumer disconnects, the server-added latency histogram (ingest event to socket write), and the active node (A or B) with failover count.
|
||||
- **Sync:** doc ops per second, conflicts by kind, and outbox replays.
|
||||
- **Jobs:** backfill progress (one-time), pruner runs and bytes freed, gap repairs, and congress refresh duration and result.
|
||||
- **Health:** `/healthz` (process up) and `/readyz` (a fresh node stream on at least one node, registry loaded, DB writable).
|
||||
- **Alerts:**
|
||||
- Node: `exchangeStatus` lag above 2 s, no new block for 5 s, disk above 70% (warn) or 85% (page), crash loops, or a version change.
|
||||
- Ingest: a snapshot mismatch, a block gap, or a failover.
|
||||
- General: staleness p99 above 10 s during the underlying's open hours, and the error rate.
|
||||
- **Dashboards:** Grafana, or a hosted equivalent.
|
||||
|
||||
---
|
||||
|
||||
## 11. Deployment and hosting cost
|
||||
|
||||
**v1 topology (colocated, §6.3):**
|
||||
- **Node host A** (Vultr bare metal, Tokyo) runs `hl-visor`, `hl-node`, `tr-ingest` **and the primary `tr-gateway` plus its database**. Ingest-to-gateway traffic stays on loopback.
|
||||
- **Node host B** (Cherry Servers, Tokyo, Equinix TY15) runs `hl-visor`, `hl-node`, `tr-ingest` **and a warm-standby `tr-gateway`**:
|
||||
- B's ingest streams to A's gateway over WireGuard/mTLS across providers (a short in-city hop [unverified; measure]).
|
||||
- **Market data fails over automatically.** B builds its own books and bars (§6.6). If A's health check fails, DNS points clients at B, and they resync the market topics.
|
||||
- **User writes have exactly one writer, with synchronous replication and an epoch fence:**
|
||||
- **One writer.** Document ops, their `(user_id, op_id)` results (§8) and demo-book commands are accepted by the **writer** gateway only. The writer is A in v1, holding writer epoch `e`. B's gateway rejects writes while it's a standby; clients keep them in their outbox (§8).
|
||||
- **Synchronous ack.** The writer sends `DocAck`/`DocConflict` only after the change **and its `op_id` result** are committed locally **and** confirmed durable on the other host, tagged with epoch `e`. So an acknowledged edit always exists on both hosts, and a client never drops from its outbox an edit the survivor lacks.
|
||||
- **If the other host can't confirm, the writer stops acking.** Clients queue edits offline (§8) until replication returns. Market data is unaffected. We trade write availability for never losing acknowledged edits [target; revisit after measuring].
|
||||
- **Promotion is operator-initiated in v1** (a runbook step, not a health check):
|
||||
1. The operator promotes B to epoch `e+1`.
|
||||
2. From then on, B refuses replication with any epoch `≤ e`.
|
||||
3. A can no longer get confirmations, so it **cannot ack any further write** even if some clients can still reach it. When A reconnects, it sees the higher epoch and demotes itself to standby. That prevents split-brain divergence of documents and the demo book.
|
||||
- Clients replay their outbox against the new writer, and `op_id` dedupe returns the original result for edits that had been acked (§8).
|
||||
- **Bars and other derived data** are rebuilt per host from node output, so they need no write fence (§3.4, §6.6).
|
||||
- **The gateway runs on the node boxes, not a separate cloud VM.** Node A has 24 cores / 48 threads and 256 GB, enough headroom for both [unverified until measured]. The node gets most of the cores via cgroup/`CPUAffinity` limits, so client load can't starve block processing.
|
||||
- If measurement shows contention, the gateway moves to a small box on the **same provider's private LAN** (Vultr VPC, Cherry private VLAN) in Tokyo. It never goes to another region.
|
||||
- **Market data starts in Tokyo,** so the hop to North American users is the same whether the gateway is in Tokyo or North America. The cost is a longer RTT for document sync from North America, ~150 ms [unverified; to measure], which is fine for that traffic. North American edge gateways can come later, if measurement shows it matters.
|
||||
|
||||
| Item | Monthly cost (list, §6.3) |
|
||||
|---|---|
|
||||
| Node A: Vultr `vbm-24c-256gb-amd`, Tokyo (node + ingest + gateway) | **$725** |
|
||||
| Node B: Cherry Servers EPYC 9275F, Tokyo (node + ingest + standby gateway) | **$874.17** |
|
||||
| Extra bandwidth beyond Vultr's 10 TB, at $0.01/GB | ≈ $0–50 [unverified; depends on §5.9 measurements] |
|
||||
| Object storage for our own fills archive (optional, §6.4) | ≈ $10–30 [unverified] |
|
||||
| One-time S3 backfill, requester-pays transfer (§5.6) | One-off; unknown until the bucket size is measured [unverified] |
|
||||
| **Total (two nodes, gateway colocated)** | **≈ $1,610–1,680** |
|
||||
| Total (single-node budget fallback, §6.6) | ≈ $735–805 (Vultr only); ≈ $375–600 on a 16-core Latitude or Cherry box once Tokyo stock returns |
|
||||
|
||||
Egress estimate (per always-on client): a 50-symbol watchlist at 4 Hz × ~30 bytes per postcard update ≈ 6 KB/s ≈ **15 GB/month** worst case [unverified, rough]. Most clients are idle or backgrounded most of the time. We measure real numbers in phase 6.
|
||||
|
||||
**Recommendation:**
|
||||
- Two bare-metal nodes on two providers in Tokyo (Vultr + Cherry Servers), with `tr-ingest` and `tr-gateway` colocated on the node boxes. Primary on A, warm standby on B.
|
||||
- Containerise ingest and the gateway (one Dockerfile each). The node itself runs on the host under systemd, as the README expects.
|
||||
- CI deploys from `master` tags, alongside the existing `deploy-site.yml`.
|
||||
|
||||
---
|
||||
|
||||
## 12. Phased plan (node first; small PRs, each proving one thing)
|
||||
|
||||
| # | PR / step | Proves |
|
||||
|---|---|---|
|
||||
| 0 | **This doc** | Agreement on scope, protocol, node plan and risks |
|
||||
| 1 | **Node A live in Tokyo** (ops, not app code): an Ubuntu 24.04 host per §6.2, GPG-verified `hl-visor`, the flags in §5.1, the pruner, and node metrics and alerts. A runbook goes in `docs/ops/node.md` (no addresses or secrets). | The node syncs and `exchangeStatus` stays fresh. We measure GB/day, bandwidth and catch-up after a planned restart. **Acceptance tests for the §5.2 verdict:** `xyz:` and other HIP-3 coins present in raw book diffs and `l4Snapshots`, spot present, one book compared once against the public `l2Book`, local `activeAssetData.markPx` working, and the oracle actions located in `replica_cmds`. |
|
||||
| 2 | `crates/hl-node`: parsers for node files (fills, order statuses, raw book diffs, HIP-3 oracle updates, misc events, batched blocks), from **recorded node fixtures**, plus the checkpointed file tailer | Our parsers match the real shapes. Resume-from-checkpoint is exact. |
|
||||
| 3 | `crates/book` (`tr-book`): L4 from snapshot plus diffs, L2 and BBO aggregation, and a self-check against `fileSnapshot` | Books match fresh snapshots for 24 h with zero unrepaired mismatches, across core, HIP-3 and spot. |
|
||||
| 4 | `tr-ingest`: tape dedupe, the candle builder (all intervals, no cap), the candle store, and prices (mid, last, oracle, mark, funding, §5.4) | Bars are deterministic and identical when built twice from the same files. Prices reconcile against the abci state. |
|
||||
| 5 | `crates/proto` (`tr-proto`): message types, postcard and JSON codecs, round-trip and size tests, a `wasm32` build in CI | The schema compiles everywhere. Frame sizes are known. |
|
||||
| 6 | `server/` (`tr-gateway`) skeleton: the ingest → gateway stream, axum WS, `Hello`/`Welcome`, ping, and the `Quotes` topic. A CLI client and a 1k-client load test. | **One ingest fans out to N clients.** Server-added latency, internal-stream bandwidth and egress are measured. |
|
||||
| 7 | Symbol registry (from the local info server) and mapping (§7), `Resolution` in quotes, and `RemoteMarkets` in `tr-data` behind `TR_SERVER_URL`. The web build uses it. | The web build shows live `xyz:` and core prices with `PERP` labelling. The `TODO(web proxy)` stubs are gone for quotes. |
|
||||
| 8 | The `history` RPC and live `Bars` with `BarClosed`, plus `Book` (L2 and L4), `Trades` and `Ctx` topics, linger and eviction, and slow-client backpressure tests | Charts and books work through the server. Slow clients don't hurt others. |
|
||||
| 9 | **One-time backfill** (§5.6): measure the S3 bucket, rebuild from `node_fills_by_block`, run the `candleSnapshot` sweep for what's left, and stitch with provenance | History before node start exists. Overlap comparisons pass. |
|
||||
| 10 | **Node B and failover** (§6.6) | Killing node A mid-session causes one `Resync` and no lost closed bars. |
|
||||
| 11 | Auth (sign-in) and accounts. Document sync for layouts, prefs and watchlists, with the conflict rules. | Two devices stay in sync, and conflicts never lose work. |
|
||||
| 12 | Drawings (stable ids) and a server-authoritative demo book | Real-time cross-device drawings. One paper book. |
|
||||
| 13 | `CongressProvider` server-side. The web gets the Congress panes. | Congress data reaches the web and the stub is deleted. |
|
||||
| 14 | `SnapTradeProvider` server-side (read-only server mode). Desktop local mode is left untouched. | Web and mobile get portfolios with no partner secrets in those clients. Desktop trading still works. |
|
||||
| 15 | Staging and prod deploys for the gateway, plus dashboards and the full alert set | It runs unattended and we can see when it doesn't. |
|
||||
| 16 | (Later) a licensed equities and options `Provider` (Intrinio, Twelve Data or Massive) | The provider interface holds up against a second real source. |
|
||||
|
||||
---
|
||||
|
||||
## 13. Risks
|
||||
|
||||
1. **Legal: Hyperliquid's Terms of Use.** The Terms (last updated June 15, 2026) define "Restricted Persons" to include persons and entities that "reside in, are located in, are incorporated in, or have a registered office in the United States of America or Ontario, Canada", and forbid them from using the Interface. They also prohibit circumventing restrictions, for example with VPNs [doc: TOU §1.6, §3.1.5]. The Terms cover the website *Interface*. They contain **no explicit license or prohibition for redistributing or displaying API data** [doc: TOU; I found no data-licensing clause]. Goshen Labs is in Toronto, and many of Terminal's users will be in the US. **This needs legal review before launch**, and it is the main reason the server is data-only. The Foundation node also excludes "certain activities involving the U.S. or Ontario" [doc: FND]. Running our own node doesn't change this analysis. It's a question about displaying Hyperliquid data, not about how we fetch it.
|
||||
2. **Perp ≠ stock.** HIP-3 equity perps follow the underlying through an oracle during external sessions. Off-hours (weekends, holidays, and anything outside 24/5 coverage) the oracle follows trade.xyz's own order flow through an EMA, and mark is bounded around the last external price [doc: XYZ-ORACLE, XYZ-BOUNDS]. Prices *will* diverge from the listed stock: at the open, around earnings, and on corporate actions (splits and dividends handling [unverified]). Funding also moves the perp relative to spot. Mitigations: mandatory labelling (§7.1), session badges (§7.4), and never computing "stock" analytics (for example the Congress performance estimates) from perp prices without labelling them.
|
||||
3. **Builder delistings and settlements.** HIP-3 deployers can halt and settle markets at any time, and can recycle names [doc: HIP3]. Validators can delist validator-operated perps [doc: DELIST]. Snapshot counts: 21 of 141 `xyz` markets and 56 of 235 core markets flagged delisted [observed]. Mitigations: epochs (§7.5), notifications, and keeping history.
|
||||
4. **Dependence on one venue and one software stack.** Everything in v1 comes from one chain, and from `hl-node` binaries we can't audit. Output formats can change without notice. Mitigations: the `Provider` trait, staleness UI, recorded fixtures with parser tests in CI, and alerts on version changes.
|
||||
5. **The node is a single point of failure** if we run only one. Mitigation: two nodes on two providers (§6.6). With one node, outages show as stale data until catch-up (§6.5).
|
||||
6. **HIP-3 (and spot) book coverage is undocumented** (§5.2). If node book outputs leave out HIP-3 dexes, our core use case, HIP-3 equity perps, has no book source except the public API. Mitigation: the phase 1 acceptance test, run before any app code is built on top of it.
|
||||
7. **The reference book code is unmaintained** [doc: LAT, OBS], and queue-priority edge cases exist [doc: LAT]. Mitigations: our own `tr-book` with snapshot self-checks, per-coin rebuilds and recorded-day replay tests.
|
||||
8. **Core mark can't be fully derived from node data** (it uses external CEX perp mids [doc: MARK]). We rely on the local `activeAssetData` [unverified], with a labelled estimate as the fallback (§5.4).
|
||||
9. **Disk and I/O.** Writing ~100–200 GB/day [doc: NODE; unverified total] can fill a disk within days if the pruner fails. Mitigations: checkpoint-gated pruning, 70%/85% alerts, and NVMe sized for headroom.
|
||||
10. **Peers and auto-upgrades.** Startup depends on finding a good peer [doc: NODE]. `hl-visor` auto-upgrades `hl-node`, which can change output schemas or restart both nodes at once. Mitigations: reliable peers (§6.1), version-change alerts, and parser tests against fresh fixtures after every upgrade.
|
||||
11. **Unknown backfill size and cost.** The S3 archive range and size are undocumented, and transfer is requester-pays [doc: HIST]. Mitigation: measure first. The `candleSnapshot` sweep is the cheap fallback.
|
||||
12. **Display rights for other feeds.** Coinbase, OKX and Kraken (per the parent's research), Yahoo, and Congress data commercial-use limits. Mitigation: `DisplayPolicy` defaults to off.
|
||||
13. **Thin HIP-3 liquidity.** Some markets are small (`abcd` had 1 asset [observed]). Mid prices can be noisy or missing when a book is empty [unverified for specific markets]. We fall back to mark price, with the basis label.
|
||||
14. **A single stateful gateway.** Restarting it drops the hot store. Clients reconnect and resync, but it is visible. Mitigation: fast startup from the registry and SQLite, plus a warm-standby gateway on node host B (§11). Market data fails over automatically. User writes need synchronous replication with an epoch fence, and promotion is operator-initiated, so acknowledged edits are never lost and the two hosts never diverge. The cost is that edits queue in client outboxes while replication or promotion is pending. Later, a two-instance setup with sticky sessions. Colocation means a node-host outage also takes out that host's gateway; that's why the standby sits on the other provider.
|
||||
15. **Protocol evolution with postcard.** A breaking change needs a version bump. Mitigations: the rules in §4.4 and support for N/N−1.
|
||||
|
||||
---
|
||||
|
||||
## 14. Hyperliquid facts referenced (summary)
|
||||
|
||||
| Fact | Value | Source |
|
||||
|---|---|---|
|
||||
| Node spec | Non-validator: 16 vCPU, 128 GB RAM, 500 GB SSD. Ubuntu 24.04 only. Ports 4001–4002 public. Tokyo for lowest latency. | [doc: NODE] |
|
||||
| Latency spec | ≥ 32 logical cores, 128 GB RAM, 500 MB/s disk, a reliable peer, `--disable-output-file-buffering` | [doc: LAT] |
|
||||
| Output volume | "Around 100 GB of logs per day" (more with order statuses and book diffs) | [doc: NODE] |
|
||||
| Book outputs | `--write-raw-book-diffs`, `--write-order-statuses`, `--write-fills`, `--batch-by-block` | [doc: NODE, L1] |
|
||||
| L4 checkpoint | `fileSnapshot` → `l4Snapshots` on the local info server (latest state), or `compute-l4-snapshots` from `periodic_abci_states` (every 10,000 blocks) | [doc: NODE] |
|
||||
| L4 rebuild | Snapshot + apply order statuses = realtime L4 book | [doc: L1] |
|
||||
| Reference builder | `order_book_server`: l2Book (≤ 100 levels), trades, l4Book. Unmaintained, no spot (per README), no trigger orders. | [doc: OBS, LAT] |
|
||||
| Local info server | Subset only (`meta`, `spotMeta`, `perpDexs`, `exchangeStatus`, `activeAssetData`, …). No time series, no WS, no `l2Book`. | [doc: NODE] |
|
||||
| Oracle | Validators publish about every 3 s. Weighted CEX spot median, then a stake-weighted median across validators. | [doc: ORACLE] |
|
||||
| Mark | Median of oracle+EMA, the book/last-trade median, and CEX perp mids | [doc: MARK] |
|
||||
| Funding | Hourly. Premium sampled every 5 s. Interest 0.01%/8h. Cap 4%/h. | [doc: FUNDING] |
|
||||
| HIP-3 oracle | `setOracle` about every 3 s with `oraclePxs`, `markPxs` and `externalPerpPxs`. Node flag `--write-hip3-oracle-updates`. | [doc: HIP3-ACT, NODE] |
|
||||
| S3 archives | Requester-pays. `hl-mainnet-node-data` (fills, trades, misc events, blocks). `hyperliquid-archive` (l2Book snapshots, asset_ctxs). No candles. | [doc: HIST] |
|
||||
| Public candle depth (backfill only) | Most recent 5000 candles per interval | [doc: INFO] |
|
||||
| Public REST weight (backfill only) | 1200/min per IP. `candleSnapshot` costs 20 plus 1 per 60 items. | [doc: RL] |
|
||||
| HIP-3 naming | `{dex}:{coin}`. Asset id `100000 + dex_index*10000 + index` | [doc: ASSET, PERP] |
|
||||
|
||||
---
|
||||
|
||||
## 15. Open questions for Micky
|
||||
|
||||
1. **Legal:** who reviews the Hyperliquid ToS (US/Ontario "Restricted Persons") for a Toronto company displaying Hyperliquid data to US and Canadian users? Is it OK to bring the node and server up before that review, given that it's data-only?
|
||||
2. **Node budget:** two nodes (≈ $1,600/month, list) or one node to start (≈ $725)? Is there a monthly ceiling?
|
||||
3. **Hosting providers:** Vultr bare metal + Cherry Servers in Tokyo as the two-provider pair (§6.3), or wait for cheaper 16-core Latitude or Cherry stock in Tokyo? Freestyle, OVHcloud and Equinix Metal are ruled out.
|
||||
4. **AWS account for the backfill:** OK to open or use one for requester-pays S3 transfer, and what's the cap on the one-time backfill spend?
|
||||
5. **Gateway placement:** colocated on the node boxes in Tokyo (recommended for v1, §11), or on a separate same-LAN box from the start?
|
||||
6. **Mark price:** OK to show the node-read mark (`activeAssetData`) and fall back to a labelled estimate if that doesn't work?
|
||||
7. **If HIP-3 books are missing from node output** (the phase 1 test): pause, or allow the public `l2Book` for HIP-3 only, as a stopgap?
|
||||
8. **Default crypto source:** core perp mid (`BTC`) or HL spot (`UBTC/USDC`), and how prominently do we label "perp" for crypto?
|
||||
9. **Proxy mapping:** should `AAPL US Equity` resolve to `xyz:AAPL` by default (with the `PERP` badge), or only when the user types `xyz:AAPL`? And should other dexes (`flx:`, `km:`, …) be mapped when xyz lacks a ticker?
|
||||
10. **Off-hours:** show the perp price over weekends as the "last" price, or freeze the equity quote at the Friday close and show the perp separately?
|
||||
11. **Auth:** a hosted identity provider (Clerk, WorkOS, Supabase) or self-hosted OAuth in Rust? Which sign-in methods first?
|
||||
12. **Anonymous use:** can signed-out users get live data (today's behaviour), or is sign-in required for the web build?
|
||||
13. **Demo account:** OK to make it server-authoritative and let it fill off-hours at perp prices?
|
||||
14. **Encoding:** OK with postcard only (JSON for debugging), given that every client is Rust?
|
||||
15. **Secondary feeds:** keep Binance, Coinbase and Yahoo server-side but staff-only, or drop them from the server entirely?
|
||||
16. **Congress data:** get legal sign-off on the commercial-use limits before serving it publicly from our server?
|
||||
17. **Licensed vendor:** is there a preferred vendor (Intrinio, Twelve Data, Massive) and a timeline, so the `DisplayPolicy` and entitlement model can be shaped around its contract?
|
||||
|
||||
---
|
||||
|
||||
## References
|
||||
|
||||
- **RL**: Rate limits and user limits. https://hyperliquid.gitbook.io/hyperliquid-docs/for-developers/api/rate-limits-and-user-limits
|
||||
- **INFO**: Info endpoint. https://hyperliquid.gitbook.io/hyperliquid-docs/for-developers/api/info-endpoint
|
||||
- **PERP**: Info endpoint, Perpetuals. https://hyperliquid.gitbook.io/hyperliquid-docs/for-developers/api/info-endpoint/perpetuals
|
||||
- **WS**: Websocket. https://hyperliquid.gitbook.io/hyperliquid-docs/for-developers/api/websocket
|
||||
- **WS-SUB**: Websocket subscriptions. https://hyperliquid.gitbook.io/hyperliquid-docs/for-developers/api/websocket/subscriptions
|
||||
- **HB**: Timeouts and heartbeats. https://hyperliquid.gitbook.io/hyperliquid-docs/for-developers/api/websocket/timeouts-and-heartbeats
|
||||
- **ASSET**: Asset IDs. https://hyperliquid.gitbook.io/hyperliquid-docs/for-developers/api/asset-ids
|
||||
- **HIP3**: HIP-3, builder-deployed perpetuals. https://hyperliquid.gitbook.io/hyperliquid-docs/hyperliquid-improvement-proposals-hips/hip-3-builder-deployed-perpetuals
|
||||
- **DELIST**: Delisting. https://hyperliquid.gitbook.io/hyperliquid-docs/trading/delisting
|
||||
- **HIST**: Historical data. https://hyperliquid.gitbook.io/hyperliquid-docs/historical-data
|
||||
- **LAT**: Optimizing latency. https://hyperliquid.gitbook.io/hyperliquid-docs/for-developers/api/optimizing-latency
|
||||
- **L1**: L1 data schemas. https://hyperliquid.gitbook.io/hyperliquid-docs/for-developers/nodes/l1-data-schemas
|
||||
- **FND**: Foundation non-validating node. https://hyperliquid.gitbook.io/hyperliquid-docs/for-developers/nodes/foundation-non-validating-node
|
||||
- **NODE**: hyperliquid-dex/node README. https://github.com/hyperliquid-dex/node
|
||||
- **OBS**: hyperliquid-dex/order_book_server README. https://github.com/hyperliquid-dex/order_book_server
|
||||
- **TOU**: Hyperliquid Terms of Use (last updated June 15, 2026). https://app.hyperliquid.xyz/terms
|
||||
- **XYZ-US**: trade[XYZ] US stocks. https://docs.trade.xyz/perpetuals/markets/stocks/us
|
||||
- **XYZ-ORACLE**: trade[XYZ] oracle price. https://docs.trade.xyz/perpetuals/mechanics/oracle-price
|
||||
- **XYZ-BOUNDS**: trade[XYZ] discovery bounds. https://docs.trade.xyz/perpetuals/mechanics/discovery-bounds
|
||||
- **XYZ-TAU**: trade[XYZ] oracle time constant updates. https://docs.trade.xyz/perpetuals/changelog/oracle-time-constant-updates
|
||||
- **ORACLE**: Oracle. https://hyperliquid.gitbook.io/hyperliquid-docs/hypercore/oracle
|
||||
- **MARK**: Robust price indices. https://hyperliquid.gitbook.io/hyperliquid-docs/trading/robust-price-indices
|
||||
- **FUNDING**: Funding. https://hyperliquid.gitbook.io/hyperliquid-docs/trading/funding
|
||||
- **HIP3-ACT**: HIP-3 deployer actions. https://hyperliquid.gitbook.io/hyperliquid-docs/for-developers/api/hip-3-deployer-actions
|
||||
- **INFRA-PEERS**: Foundation non-validating node for infra providers. https://hyperliquid.gitbook.io/hyperliquid-docs/for-developers/nodes/foundation-non-validating-node-for-infra-providers
|
||||
- **LATITUDE**: Latitude.sh pricing. https://www.latitude.sh/pricing
|
||||
- **LAT-M4**: Latitude.sh m4.metal.medium (official page; $456 US list) https://www.latitude.sh/pricing/m4-metal-medium. Tokyo price and stock come from a third-party tracker: https://servermarket.dev/providers/latitude/m4-metal-medium
|
||||
- **LAT-F4**: Latitude.sh f4.metal.medium, Tokyo price and stock (third-party tracker). https://servermarket.dev/providers/latitude/f4-metal-medium
|
||||
- **VULTR-API**: Vultr public plans API (plans, monthly price and locations; `nrt` is Tokyo, `itm` is Osaka) and OS list, read 2026-10-29. https://api.vultr.com/v2/plans-metal, https://api.vultr.com/v2/plans, https://api.vultr.com/v2/os
|
||||
- **VULTR-BM**: Vultr bare metal product page. https://www.vultr.com/products/bare-metal/
|
||||
- **VULTR-PRICE**: Vultr pricing (bandwidth overage). https://www.vultr.com/pricing/
|
||||
- **CHERRY-TYO**: Cherry Servers Tokyo dedicated servers (prices, Equinix TY15, 100 TB egress, Ubuntu 24.04, private VLANs). https://www.cherryservers.com/bare-metal-dedicated-servers/tokyo. Launch post: https://www.cherryservers.com/blog/cherry-servers-tokyo-location-live
|
||||
- **CHERRY-9950X**: Cherry Servers Ryzen 9950X price and stock by location (third-party tracker). https://servermarket.dev/providers/cherry/amd-ryzen-9950x. Official page: https://www.cherryservers.com/pricing/dedicated-servers/amd-ryzen-9950x
|
||||
- **AWS-PRICE**: AWS EC2 on-demand Linux prices for Asia Pacific (Tokyo), from the price data behind the official pricing page, read 2026-10-29. https://aws.amazon.com/ec2/pricing/on-demand/
|
||||
- **OVH-REGIONS**: OVHcloud dedicated-server regions (APAC: Singapore, Mumbai, Sydney). https://www.ovhcloud.com/en/bare-metal/regions-availability/
|
||||
- **OVH-APAC**: OVHcloud APAC traffic quotas and throttling. https://support.us.ovhcloud.com/hc/en-us/articles/4404330636819-All-About-APAC-Region-Servers
|
||||
- **EQX-EOL**: Equinix Metal sunset (June 30, 2026). https://docs.equinix.com/goodbye-metal/
|
||||
- **FREESTYLE**: Freestyle VM pricing and limits, and inbound TLS. https://www.freestyle.sh/docs/vms/pricing-and-limits, https://www.freestyle.sh/docs/vms/network/tls-inbound
|
||||
- **AWS prices** (third-party trackers, accessed 2026-10-29, not quotes): r6i.4xlarge Tokyo https://vmprices.com and https://www.usage.ai, r7a.4xlarge https://cloudcostdb.com, EBS gp3 Tokyo summaries. Official structure: https://aws.amazon.com/ec2/pricing/on-demand/ and https://aws.amazon.com/ebs/pricing/
|
||||
|
||||
### Observations log (read-only calls, 2026-10-29 ~17:35 UTC)
|
||||
|
||||
- `POST /info {"type":"perpDexs"}` returned 11 entries (null plus 10 named dexes).
|
||||
- `{"type":"meta"}` returned 235 universe entries (56 `isDelisted`). `{"type":"meta","dex":"xyz"}` returned 141 entries (21 `isDelisted`), all names `xyz:`-prefixed.
|
||||
- `{"type":"spotMeta"}` returned 347 pairs and 511 tokens.
|
||||
- `{"type":"allMids"}` returned 1399 keys (401 `@…` spot, 762 `#…` outcomes, no `dex:` keys). `{"type":"allMids","dex":"xyz"}` returned `xyz:` keys only.
|
||||
- `candleSnapshot` (startTime 0) returned:
|
||||
- BTC 1m: 5104 bars (~3.5 days)
|
||||
- BTC 1h: 5002 bars (from 2026-04-04)
|
||||
- BTC 1d: 2263 bars (from 2020-08-19)
|
||||
- `xyz:AAPL` 1d: 343 bars (from 2025-11-21)
|
||||
- Numbers came back as strings.
|
||||
- WS `wss://api.hyperliquid.xyz/ws`, 12 s window, subscriptions to `allMids{dex:xyz}`, `l2Book BTC`, `trades BTC`, `candle xyz:AAPL 1m`, `bbo xyz:AAPL`, `activeAssetCtx xyz:AAPL` and `activeAssetCtx @107`:
|
||||
- Each got a `subscriptionResponse`.
|
||||
- Message counts: l2Book 3, trades 14, activeAssetCtx 13, activeSpotAssetCtx 13, bbo 9, allMids 2, candle 2.
|
||||
- `ping` → `{"channel":"pong"}`.
|
||||
- Candle OHLCV came as strings.
|
||||
- WS `allDexsAssetCtxs` returned ~122 KB messages (2 in 8 s). WS `fastAssetCtxs` returned 10 messages in 8 s (1.9–28 KB), with a first snapshot of 2240 coins after base64 and raw-DEFLATE decoding.
|
||||
- No `exchange` endpoint calls, no orders, no user-specific subscriptions.
|
||||
- `POST /info {"type":"allPerpMetas"}` returned per-dex universe sizes as listed in §5.7.
|
||||
- Hosting research (2026-10-29): read-only GETs of the Vultr public plans and OS APIs (no account), plus AWS's public Tokyo on-demand price data. Nothing was ordered or provisioned.
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 170 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 173 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 103 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 99 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 99 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 103 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 106 KiB |
+2
-17
@@ -12,11 +12,6 @@
|
||||
# cache everything under immutable/ for good (site/vercel.json), and a page can't pair one
|
||||
# build's JS with another build's wasm.
|
||||
#
|
||||
# The app gets crypto markets from the Terminal data server at TR_SERVER_URL, which is built
|
||||
# in. It defaults to the production server; set it to another ws:// or wss:// URL, or to
|
||||
# nothing (TR_SERVER_URL=) for no server. A page can also ask for a server on the same machine
|
||||
# with ?server=ws://localhost:8787/ws, or for none with ?server=off.
|
||||
#
|
||||
# Needs the wasm32-unknown-unknown target (rustup target add wasm32-unknown-unknown),
|
||||
# wasm-bindgen-cli at the version pinned in Cargo.lock, and wasm-opt from binaryen.
|
||||
set -euo pipefail
|
||||
@@ -24,17 +19,7 @@ cd "$(dirname "$0")/.."
|
||||
|
||||
out="${1:-site/static/app}"
|
||||
target=wasm32-unknown-unknown
|
||||
wasm="${CARGO_TARGET_DIR:-target}/$target/web/terminal.wasm"
|
||||
|
||||
export TR_SERVER_URL="${TR_SERVER_URL-wss://terminal-gateway.style.dev/ws}"
|
||||
case "$TR_SERVER_URL" in
|
||||
"") echo "no Terminal server: crypto markets come from Binance and Coinbase" ;;
|
||||
ws://* | wss://*) echo "Terminal server: $TR_SERVER_URL" ;;
|
||||
*)
|
||||
echo "TR_SERVER_URL must be a ws:// or wss:// URL, or empty for no server" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
wasm="target/$target/web/terminal-rs.wasm"
|
||||
|
||||
want=$(awk '/^name = "wasm-bindgen"$/ { getline; gsub(/version = |"/, ""); print; exit }' Cargo.lock)
|
||||
have=$(wasm-bindgen --version 2>/dev/null | awk '{ print $2 }' || true)
|
||||
@@ -62,7 +47,7 @@ sha256() { if command -v sha256sum >/dev/null; then sha256sum; else shasum -a 25
|
||||
build() {
|
||||
local name=$1 tmp="$out/immutable/.$1" hash
|
||||
shift
|
||||
cargo build --profile web --target "$target" -p terminal "$@"
|
||||
cargo build --profile web --target "$target" -p terminal-rs "$@"
|
||||
wasm-bindgen --target web --no-typescript --out-dir "$tmp" --out-name "$name" "$wasm"
|
||||
wasm-opt -Os "${features[@]}" --strip-debug --strip-producers "$tmp/${name}_bg.wasm" -o "$tmp/${name}_bg.wasm"
|
||||
hash=$(cd "$tmp" && find . -type f | LC_ALL=C sort | xargs cat | sha256 | cut -c 1-16)
|
||||
|
||||
@@ -1,22 +1,18 @@
|
||||
#!/usr/bin/env bash
|
||||
# Wrap a release binary in a minimal macOS .app bundle (retina-capable, dark-mode aware), written
|
||||
# next to the binary as Terminal.app: target/release/terminal unless another path is given.
|
||||
# next to the binary as Terminal.app: target/release/terminal-rs unless another path is given.
|
||||
# The executable inside is named Terminal too, since the app menu's About, Hide and Quit items
|
||||
# take the process name. The bundle embeds Sparkle, which updates installed copies; it is left
|
||||
# unsigned, and scripts/release-macos.sh signs it.
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
version=$(sed -n 's/^version = "\(.*\)"/\1/p' Cargo.toml | head -1)
|
||||
bin=${1:-target/release/terminal}
|
||||
bin=${1:-target/release/terminal-rs}
|
||||
app=$(dirname "$bin")/Terminal.app
|
||||
# The bundle ID keeps the project's old name. Released copies have it, and macOS ties the app's
|
||||
# Keychain access, privacy permissions and preferences (Sparkle's among them) to it, so changing
|
||||
# it would make the next update a different app as far as macOS is concerned. Nothing shows it.
|
||||
bundle_id=dev.terminal-rs.app
|
||||
# Sparkle checks this feed and installs only updates signed with the private half of this key,
|
||||
# which `generate_keys --account terminal-rs` keeps in the release Mac's login keychain. Every
|
||||
# release must be signed with the same key.
|
||||
feed=https://github.com/michaelshimeles/terminal/releases/latest/download/appcast.xml
|
||||
feed=https://github.com/michaelshimeles/terminal-rs/releases/latest/download/appcast.xml
|
||||
sparkle_key=Kq1jkRyRLHyfeNeAv3Fl7we7ARPnyuPh+MI/XSZV77U=
|
||||
sparkle=$(scripts/fetch-sparkle.sh)
|
||||
rm -rf "$app"
|
||||
@@ -37,7 +33,7 @@ cat > "$app/Contents/Info.plist" <<PLIST
|
||||
<dict>
|
||||
<key>CFBundleName</key><string>Terminal</string>
|
||||
<key>CFBundleDisplayName</key><string>Terminal</string>
|
||||
<key>CFBundleIdentifier</key><string>${bundle_id}</string>
|
||||
<key>CFBundleIdentifier</key><string>dev.terminal-rs.app</string>
|
||||
<key>CFBundleExecutable</key><string>Terminal</string>
|
||||
<key>CFBundleIconFile</key><string>AppIcon</string>
|
||||
<key>CFBundlePackageType</key><string>APPL</string>
|
||||
|
||||
@@ -20,9 +20,6 @@
|
||||
# target/sparkle/<version>/bin/generate_keys --account terminal-rs -x terminal-rs-sparkle.key
|
||||
# target/sparkle/<version>/bin/generate_keys --account terminal-rs -f terminal-rs-sparkle.key # on a new Mac
|
||||
#
|
||||
# The notary profile and the signing key's account keep the project's old name, terminal-rs,
|
||||
# because they name items that already exist in the release Mac's keychain.
|
||||
#
|
||||
# SIGN_IDENTITY signing identity (default: the first "Developer ID Application" in the keychain)
|
||||
# NOTARY_PROFILE notarytool keychain profile (default: terminal-rs-notary)
|
||||
# NOTARY_KEYCHAIN keychain holding that profile (default: the login keychain)
|
||||
@@ -42,7 +39,7 @@ if [ "$notarize" = 1 ] && ! xcrun notarytool history "${notary[@]}" >/dev/null;
|
||||
fi
|
||||
|
||||
version=$(sed -n 's/^version = "\(.*\)"/\1/p' Cargo.toml | head -1)
|
||||
repo=https://github.com/michaelshimeles/terminal
|
||||
repo=https://github.com/michaelshimeles/terminal-rs
|
||||
sparkle=$(scripts/fetch-sparkle.sh)
|
||||
sparkle_account=${SPARKLE_ACCOUNT:-terminal-rs}
|
||||
sparkle_key=$(sed -n 's/^sparkle_key=//p' scripts/bundle-macos.sh)
|
||||
@@ -81,14 +78,14 @@ notarize() {
|
||||
}
|
||||
|
||||
for target in aarch64-apple-darwin x86_64-apple-darwin; do
|
||||
cargo build --release --locked --target "$target" -p terminal
|
||||
cargo build --release --locked --target "$target" -p terminal-rs
|
||||
done
|
||||
out=target/universal-apple-darwin/release
|
||||
mkdir -p "$out"
|
||||
rm -f "$out"/Terminal-*.zip "$out/appcast.xml"
|
||||
lipo -create -output "$out/terminal" \
|
||||
target/aarch64-apple-darwin/release/terminal target/x86_64-apple-darwin/release/terminal
|
||||
scripts/bundle-macos.sh "$out/terminal"
|
||||
lipo -create -output "$out/terminal-rs" \
|
||||
target/aarch64-apple-darwin/release/terminal-rs target/x86_64-apple-darwin/release/terminal-rs
|
||||
scripts/bundle-macos.sh "$out/terminal-rs"
|
||||
app=$out/Terminal.app
|
||||
framework=$app/Contents/Frameworks/Sparkle.framework
|
||||
|
||||
|
||||
@@ -1,26 +0,0 @@
|
||||
[package]
|
||||
name = "tr-gateway"
|
||||
description = "Terminal data server: one upstream feed fanned out to every Terminal client"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
[dependencies]
|
||||
tr-core.workspace = true
|
||||
tr-proto.workspace = true
|
||||
anyhow.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json = { workspace = true, features = ["raw_value"] }
|
||||
reqwest.workspace = true
|
||||
tracing.workspace = true
|
||||
tracing-subscriber.workspace = true
|
||||
tokio = { workspace = true, features = ["net", "signal"] }
|
||||
async-trait = "0.1"
|
||||
futures = "0.3"
|
||||
axum = { version = "0.8", default-features = false, features = ["http1", "json", "tokio", "ws"] }
|
||||
tokio-tungstenite = { version = "0.30", default-features = false, features = ["connect", "rustls-tls-webpki-roots"] }
|
||||
rustls = { version = "0.23", default-features = false, features = ["aws_lc_rs", "std"] }
|
||||
|
||||
[dev-dependencies]
|
||||
# The CLI example and the integration tests talk to the gateway through the desktop's client.
|
||||
tr-data.workspace = true
|
||||
tokio = { workspace = true, features = ["net", "signal", "test-util"] }
|
||||
@@ -1,62 +0,0 @@
|
||||
# tr-gateway
|
||||
|
||||
The Terminal data server. For now it carries Hyperliquid's public API (see "Interim: public feed" in `docs/architecture/server.md`). One set of upstream websocket connections serves every client, fanned out over the protocol in `crates/proto`.
|
||||
|
||||
## Run it
|
||||
|
||||
```sh
|
||||
cargo run --release -p tr-gateway
|
||||
```
|
||||
|
||||
It listens on `localhost:8787`. Clients connect to `ws://localhost:8787/ws`. `/healthz` says the process is up, `/readyz` says the upstream is connected and every venue's mids have arrived, `/metrics` is Prometheus text and `/stats` is JSON.
|
||||
|
||||
The desktop app uses it for crypto markets when `TR_SERVER_URL` is set:
|
||||
|
||||
```sh
|
||||
TR_SERVER_URL=ws://localhost:8787 cargo run --release
|
||||
```
|
||||
|
||||
The browser build uses it too. `scripts/build-web.sh` builds in `wss://terminal-gateway.style.dev/ws`, or `TR_SERVER_URL` when it is set (empty for none). To try a local gateway with a built page, open the page with `?server=ws://localhost:8787/ws`.
|
||||
|
||||
The gateway serves plain HTTP and websockets. For `wss://` on 443, put a TLS-terminating edge in front of it and bind to an address the edge can reach, for example `TR_GATEWAY_BIND=0.0.0.0:8080`. Clients accept `wss://` URLs. The gateway pings each client every 20 s so idle connections survive proxies. The hosted one runs on Freestyle; see `deploy/freestyle/README.md`.
|
||||
|
||||
## Restarts
|
||||
|
||||
A restarted gateway listens before it knows which markets exist. It learns that from the instrument list and the first mids from every venue, which usually takes a few seconds. Clients that reconnect in that window keep their subscriptions. Each one gets `Stale`, then `Live` and its snapshot once the gateway is ready. Requests wait up to 10 s and then fail with `Upstream` rather than `UnknownMarket`, so clients can retry them. If one venue is slower than the rest, the gateway gets ready without it after 10 s. Subscriptions to that venue's markets keep waiting for it, and requests for them fail with `Upstream` until it reports. The client in `crates/data/src/remote.rs` subscribes again, with backoff from 1 s to 30 s, to anything a server ended with `Unavailable { reason: Upstream }`, so it also recovers from older gateways that gave up on such subscriptions.
|
||||
|
||||
## Try it
|
||||
|
||||
```sh
|
||||
cli() { cargo run -q --release -p tr-gateway --example cli -- "$@"; }
|
||||
cli markets xyz:
|
||||
cli watch --quotes BTC,xyz:TSLA --book BTC --trades BTC --bars BTC:1m --secs 30
|
||||
cli load --clients 5 --book BTC --trades BTC --secs 30
|
||||
cli stats
|
||||
```
|
||||
|
||||
`watch` prints about one line a second per topic, then latency. Quote latency counts from the moment the server received the price, book and trade latency from Hyperliquid's own timestamps. `/stats` reports the part the server adds, from an upstream frame arriving to the client frame it causes being written.
|
||||
|
||||
## Settings
|
||||
|
||||
Environment variables, also listed by `tr-gateway --help`:
|
||||
|
||||
| Variable | Default | |
|
||||
|---|---|---|
|
||||
| `TR_GATEWAY_BIND` | `localhost:8787` | Listen address, `host:port` |
|
||||
| `TR_HL_DEXES` | `all` | HIP-3 dexes besides the core dex, for example `xyz,flx` |
|
||||
| `TR_UPSTREAM_CONNECTIONS` | 3 | Upstream websocket connections, 2 to 8 |
|
||||
| `TR_UPSTREAM_SUBSCRIPTIONS` | 900 | Upstream subscriptions, at most 950 |
|
||||
| `TR_UPSTREAM_MESSAGES_PER_MIN` | 1000 | Messages sent upstream a minute, at most 1800 |
|
||||
| `TR_REST_WEIGHT_PER_MIN` | 800 | `info` weight a minute, at most 1100 |
|
||||
| `TR_LINGER_SECS` | 20 | How long an unwatched market keeps its upstream streams |
|
||||
| `TR_MAX_CLIENTS` | 1000 | Client connections |
|
||||
|
||||
## Hyperliquid's limits
|
||||
|
||||
Per IP, Hyperliquid allows 10 websocket connections, 30 new connections a minute, 1000 subscriptions, 2000 sent messages a minute and 1200 REST weight a minute. The gateway keeps its own budget below each one (the caps in the table, and 20 new connections a minute). A subscription past the budget gets `Unavailable { reason: Capacity }`. The gateway never rotates IPs, uses proxies or otherwise works around a limit (Hyperliquid's terms, §3.1.8).
|
||||
|
||||
- The gateway subscribes every dex's mids for good, on a connection of their own.
|
||||
- It subscribes a market's book, trades and candles only while some client watches it, and drops them `TR_LINGER_SECS` after the last one leaves.
|
||||
- Hyperliquid closes the whole connection when asked for a coin it doesn't list, so the gateway only subscribes coins that have a live mid.
|
||||
- History comes from `candleSnapshot`, which serves only the most recent 5000 bars per interval.
|
||||
- It makes read-only public calls only: `info` requests and public websocket channels. No `exchange` endpoint, no user data, no orders.
|
||||
@@ -1,353 +0,0 @@
|
||||
//! A small client for trying and measuring a running gateway.
|
||||
//!
|
||||
//! ```text
|
||||
//! cargo run --release -p tr-gateway --example cli -- [--url ws://localhost:8787] [--json] COMMAND
|
||||
//!
|
||||
//! markets [FILTER] list the markets the server streams
|
||||
//! watch [TOPICS] [--secs N]
|
||||
//! stream topics, print about one line a second per topic, then latency stats
|
||||
//! load --clients N [TOPICS] [--secs N]
|
||||
//! N connections with the same subscriptions; prints totals and latency
|
||||
//! stats print the server's /stats
|
||||
//!
|
||||
//! TOPICS: --quotes K,K,.. --book K --trades K --bars K:TF (TF: 1m 5m 15m 30m 1h 4h 1d 1w 1M)
|
||||
//! ```
|
||||
//!
|
||||
//! Latency, measured at the client on the same clock as the server when both run on one host:
|
||||
//! quotes from the server's upstream receive time (`Quote::ts_ms`), books and trades from
|
||||
//! Hyperliquid's own timestamps.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
use tr_data::remote::{Client, Event, Subscription, normalize_url};
|
||||
use tr_proto::{BarDelta, Encoding, MarketKey, Rpc, RpcResult, Timeframe, TopicData, TopicDelta, TopicReq};
|
||||
|
||||
#[derive(Default)]
|
||||
struct Args {
|
||||
url: String,
|
||||
json: bool,
|
||||
command: String,
|
||||
filter: Option<String>,
|
||||
topics: Vec<TopicReq>,
|
||||
secs: u64,
|
||||
clients: usize,
|
||||
}
|
||||
|
||||
const USAGE: &str = "\
|
||||
usage: cli [--url ws://localhost:8787] [--json] COMMAND
|
||||
markets [FILTER] list the markets the server streams
|
||||
watch [TOPICS] [--secs N] stream topics, then print latency stats
|
||||
load --clients N [TOPICS] [--secs N]
|
||||
N connections with the same subscriptions
|
||||
stats print the server's /stats
|
||||
TOPICS: --quotes K,K,.. --book K --trades K --bars K:TF (1m 5m 15m 30m 1h 4h 1d 1w 1M)";
|
||||
|
||||
fn usage() -> ! {
|
||||
eprintln!("{USAGE}");
|
||||
std::process::exit(2)
|
||||
}
|
||||
|
||||
fn parse_tf(s: &str) -> Result<Timeframe> {
|
||||
Ok(match s {
|
||||
"1m" => Timeframe::M1,
|
||||
"5m" => Timeframe::M5,
|
||||
"15m" => Timeframe::M15,
|
||||
"30m" => Timeframe::M30,
|
||||
"1h" => Timeframe::H1,
|
||||
"4h" => Timeframe::H4,
|
||||
"1d" => Timeframe::D1,
|
||||
"1w" => Timeframe::W1,
|
||||
"1M" => Timeframe::MN1,
|
||||
_ => bail!("unknown timeframe {s}"),
|
||||
})
|
||||
}
|
||||
|
||||
fn parse() -> Result<Args> {
|
||||
let mut a = Args { url: "ws://localhost:8787".into(), secs: 30, clients: 1, ..Default::default() };
|
||||
let mut it = std::env::args().skip(1);
|
||||
while let Some(arg) = it.next() {
|
||||
let mut val = || it.next().with_context(|| format!("{arg} needs a value"));
|
||||
match arg.as_str() {
|
||||
"--url" => a.url = val()?,
|
||||
"--json" => a.json = true,
|
||||
"--secs" => a.secs = val()?.parse()?,
|
||||
"--clients" => a.clients = val()?.parse()?,
|
||||
"--quotes" => {
|
||||
let keys = val()?.split(',').map(|k| MarketKey::new(k.trim())).collect();
|
||||
a.topics.push(TopicReq::Quotes { keys, max_hz: 10 });
|
||||
}
|
||||
"--book" => a.topics.push(TopicReq::Book { key: MarketKey::new(val()?), depth: 20, group: None }),
|
||||
"--trades" => a.topics.push(TopicReq::Trades { key: MarketKey::new(val()?) }),
|
||||
"--bars" => {
|
||||
let v = val()?;
|
||||
let (k, tf) = v.rsplit_once(':').context("--bars wants KEY:TF")?;
|
||||
a.topics.push(TopicReq::Bars { key: MarketKey::new(k), tf: parse_tf(tf)?, history: 5 });
|
||||
}
|
||||
"-h" | "--help" => usage(),
|
||||
c if a.command.is_empty() => a.command = c.into(),
|
||||
f if a.command == "markets" && a.filter.is_none() => a.filter = Some(f.into()),
|
||||
other => bail!("unexpected argument {other}"),
|
||||
}
|
||||
}
|
||||
if a.command.is_empty() {
|
||||
usage();
|
||||
}
|
||||
Ok(a)
|
||||
}
|
||||
|
||||
fn now_ms(t: SystemTime) -> i64 {
|
||||
t.duration_since(UNIX_EPOCH).map(|d| d.as_millis() as i64).unwrap_or(0)
|
||||
}
|
||||
|
||||
fn topic_name(t: &TopicReq) -> String {
|
||||
match t {
|
||||
TopicReq::Quotes { keys, .. } => {
|
||||
format!("quotes[{}]", keys.iter().map(|k| k.as_str()).collect::<Vec<_>>().join(","))
|
||||
}
|
||||
TopicReq::Book { key, .. } => format!("book {key}"),
|
||||
TopicReq::Trades { key } => format!("trades {key}"),
|
||||
TopicReq::Bars { key, tf, .. } => format!("bars {key} {tf}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Latency samples (ms) by measure, plus message counts by topic.
|
||||
#[derive(Default)]
|
||||
struct Tally {
|
||||
lat: BTreeMap<&'static str, Vec<i64>>,
|
||||
msgs: BTreeMap<String, u64>,
|
||||
resyncs: u64,
|
||||
statuses: Vec<String>,
|
||||
}
|
||||
|
||||
impl Tally {
|
||||
fn report(&self) {
|
||||
for (name, n) in &self.msgs {
|
||||
println!(" {name}: {n} messages");
|
||||
}
|
||||
for (what, v) in &self.lat {
|
||||
let mut v = v.clone();
|
||||
v.sort_unstable();
|
||||
let q = |p: f64| v[((v.len() - 1) as f64 * p).round() as usize];
|
||||
if !v.is_empty() {
|
||||
println!(
|
||||
" latency {what}: n={} p50={}ms p90={}ms p99={}ms max={}ms",
|
||||
v.len(),
|
||||
q(0.5),
|
||||
q(0.9),
|
||||
q(0.99),
|
||||
v[v.len() - 1]
|
||||
);
|
||||
}
|
||||
}
|
||||
println!(" resyncs: {} statuses: {:?}", self.resyncs, self.statuses);
|
||||
}
|
||||
}
|
||||
|
||||
const QUOTE_LAT: &str = "quotes (server upstream receive -> client)";
|
||||
const BOOK_LAT: &str = "book (Hyperliquid time -> client)";
|
||||
const TRADE_LAT: &str = "trades (Hyperliquid time -> client)";
|
||||
|
||||
/// Read one subscription until `until`, tallying and (when `print`) describing what arrives.
|
||||
async fn drain(mut sub: Subscription, name: String, tally: Arc<Mutex<Tally>>, until: Instant, print: bool) {
|
||||
let mut last_print = Instant::now() - Duration::from_secs(5);
|
||||
// Hyperliquid opens a trades subscription with recent history; only later trades are live.
|
||||
let since = now_ms(SystemTime::now());
|
||||
loop {
|
||||
let Ok(Some(u)) = tokio::time::timeout_at(until.into(), sub.next()).await else { return };
|
||||
let at = now_ms(u.received);
|
||||
let mut line = None;
|
||||
{
|
||||
let mut t = tally.lock().unwrap();
|
||||
*t.msgs.entry(name.clone()).or_default() += 1;
|
||||
match &u.event {
|
||||
Event::Snapshot(TopicData::Quotes(q)) | Event::Delta(TopicDelta::Quotes(q)) => {
|
||||
// A snapshot's `ts_ms` is the price's age; a delta's is when it changed.
|
||||
for x in q.iter().filter(|_| matches!(u.event, Event::Delta(_))) {
|
||||
t.lat.entry(QUOTE_LAT).or_default().push(at - x.ts_ms);
|
||||
}
|
||||
let f = |x: &tr_proto::Quote| {
|
||||
format!("{}={} (from {}, prev day {:?})", x.key, x.px, x.source, x.prev_day)
|
||||
};
|
||||
line = Some(q.iter().map(f).collect::<Vec<_>>().join(" "));
|
||||
}
|
||||
Event::Snapshot(TopicData::Book(b)) | Event::Delta(TopicDelta::Book(b)) => {
|
||||
t.lat.entry(BOOK_LAT).or_default().push(at - b.time_ms);
|
||||
if let (Some(bid), Some(ask)) = (b.bids.first(), b.asks.first()) {
|
||||
line = Some(format!(
|
||||
"{}x{} levels, best {} x {} / {} x {}",
|
||||
b.bids.len(),
|
||||
b.asks.len(),
|
||||
bid.px,
|
||||
bid.sz,
|
||||
ask.px,
|
||||
ask.sz
|
||||
));
|
||||
}
|
||||
}
|
||||
Event::Snapshot(TopicData::Trades(tr)) | Event::Delta(TopicDelta::Trades(tr)) => {
|
||||
for x in tr.iter().filter(|x| x.time_ms >= since) {
|
||||
t.lat.entry(TRADE_LAT).or_default().push(at - x.time_ms);
|
||||
}
|
||||
if let Some(x) = tr.last() {
|
||||
line = Some(format!("{} trades, last {:?} {} @ {}", tr.len(), x.side, x.sz, x.px));
|
||||
}
|
||||
}
|
||||
Event::Snapshot(TopicData::Bars(b)) => {
|
||||
line = b.last().map(|x| format!("{} bars, last ts {} close {}", b.len(), x.ts, x.close));
|
||||
}
|
||||
Event::Delta(TopicDelta::Bar(d)) => {
|
||||
let (kind, b) = match d {
|
||||
BarDelta::Forming(b) => ("forming", b),
|
||||
BarDelta::Closed(b) => ("CLOSED", b),
|
||||
};
|
||||
line = Some(format!(
|
||||
"{kind} ts {} o {} h {} l {} c {} v {:.4}",
|
||||
b.ts, b.open, b.high, b.low, b.close, b.volume
|
||||
));
|
||||
}
|
||||
Event::Resync(r) => {
|
||||
t.resyncs += 1;
|
||||
line = Some(format!("resync ({r:?})"));
|
||||
}
|
||||
Event::Status(s) => {
|
||||
t.statuses.push(format!("{name}: {s:?}"));
|
||||
line = Some(format!("status {s:?}"));
|
||||
}
|
||||
Event::Reconnecting => line = Some("reconnecting".into()),
|
||||
Event::Resubscribing => line = Some("resubscribing".into()),
|
||||
}
|
||||
}
|
||||
let kind = match &u.event {
|
||||
Event::Snapshot(_) => "snapshot",
|
||||
Event::Delta(_) => "delta",
|
||||
_ => "event",
|
||||
};
|
||||
let important =
|
||||
!matches!(u.event, Event::Delta(TopicDelta::Book(_) | TopicDelta::Quotes(_) | TopicDelta::Trades(_)))
|
||||
|| last_print.elapsed() >= Duration::from_secs(1);
|
||||
if print
|
||||
&& important
|
||||
&& let Some(l) = line
|
||||
{
|
||||
println!("{name} {kind}: {l}");
|
||||
last_print = Instant::now();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn http_base(ws: &str) -> String {
|
||||
let u = normalize_url(ws);
|
||||
let u = u.replacen("wss://", "https://", 1).replacen("ws://", "http://", 1);
|
||||
u.strip_suffix("/ws").unwrap_or(&u).to_string()
|
||||
}
|
||||
|
||||
async fn print_stats(url: &str) -> Result<()> {
|
||||
let body = reqwest::get(format!("{}/stats", http_base(url))).await?.text().await?;
|
||||
let v: serde_json::Value = serde_json::from_str(&body)?;
|
||||
let u = &v["upstream"];
|
||||
println!(
|
||||
"server: upstream {}/{} connections up, {} of {} subscriptions {} by kind {}; clients {}, client subscriptions {}; topics {}",
|
||||
u["connections_up"],
|
||||
u["max_connections"],
|
||||
u["subscriptions"],
|
||||
u["max_subscriptions"],
|
||||
u["per_connection"],
|
||||
u["by_kind"],
|
||||
v["clients"],
|
||||
v["client_subscriptions"],
|
||||
v["topics"],
|
||||
);
|
||||
let keys: Vec<&str> = u["keys"]
|
||||
.as_array()
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.filter_map(|k| k.as_str())
|
||||
.filter(|k| !k.starts_with("allMids"))
|
||||
.collect();
|
||||
println!("server: on-demand upstream subscriptions: {keys:?}");
|
||||
println!(
|
||||
"server: sent upstream {} messages in the last minute (budget {}), REST weight {} (budget {})",
|
||||
u["messages_sent_last_min"], u["messages_per_min"], u["rest_weight_last_min"], u["rest_weight_per_min"]
|
||||
);
|
||||
for (topic, q) in v["latency_us"].as_object().into_iter().flatten() {
|
||||
if q["window"].as_u64().unwrap_or(0) > 0 {
|
||||
println!(
|
||||
"server: added latency {topic}: n={} p50={}us p90={}us p99={}us max={}us",
|
||||
q["window"], q["p50_us"], q["p90_us"], q["p99_us"], q["max_us"]
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<()> {
|
||||
let a = parse()?;
|
||||
let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
|
||||
let enc = if a.json { Encoding::Json } else { Encoding::Postcard };
|
||||
match a.command.as_str() {
|
||||
"stats" => print_stats(&a.url).await,
|
||||
"markets" => {
|
||||
let c = Client::connect(&a.url, enc, "tr-cli");
|
||||
let Ok(RpcResult::Markets(m)) = c.request(Rpc::Markets).await.map_err(anyhow::Error::from) else {
|
||||
bail!("no market list")
|
||||
};
|
||||
let f = a.filter.unwrap_or_default().to_lowercase();
|
||||
let shown: Vec<_> = m.iter().filter(|x| x.key.as_str().to_lowercase().contains(&f)).collect();
|
||||
for x in &shown {
|
||||
println!("{:<8} {:?} {}", if x.venue.is_empty() { "core" } else { &x.venue }, x.kind, x.key);
|
||||
}
|
||||
println!("{} of {} markets", shown.len(), m.len());
|
||||
Ok(())
|
||||
}
|
||||
"watch" | "load" => {
|
||||
if a.topics.is_empty() {
|
||||
bail!("name at least one topic (--quotes, --book, --trades, --bars)");
|
||||
}
|
||||
let n = if a.command == "watch" { 1 } else { a.clients.max(1) };
|
||||
let tally = Arc::new(Mutex::new(Tally::default()));
|
||||
let mut clients = Vec::new();
|
||||
let t0 = Instant::now();
|
||||
for _ in 0..n {
|
||||
let c = Client::connect(&a.url, enc, "tr-cli");
|
||||
if !c.wait_up(Duration::from_secs(10)).await {
|
||||
bail!("could not reach the server: {:?}", c.state());
|
||||
}
|
||||
clients.push(c);
|
||||
}
|
||||
println!(
|
||||
"{n} client connection(s) up in {:?}; subscribing to {}",
|
||||
t0.elapsed(),
|
||||
a.topics.iter().map(topic_name).collect::<Vec<_>>().join(", ")
|
||||
);
|
||||
let until = Instant::now() + Duration::from_secs(a.secs);
|
||||
let mut tasks = Vec::new();
|
||||
for (i, c) in clients.iter().enumerate() {
|
||||
for t in &a.topics {
|
||||
let name = if n == 1 { topic_name(t) } else { format!("{} (all clients)", topic_name(t)) };
|
||||
tasks.push(tokio::spawn(drain(
|
||||
c.subscribe(t.clone()),
|
||||
name,
|
||||
tally.clone(),
|
||||
until,
|
||||
i == 0 && n == 1,
|
||||
)));
|
||||
}
|
||||
}
|
||||
if a.command == "load" {
|
||||
tokio::time::sleep(Duration::from_secs(a.secs.min(5))).await;
|
||||
let _ = print_stats(&a.url).await;
|
||||
}
|
||||
for t in tasks {
|
||||
let _ = t.await;
|
||||
}
|
||||
println!("after {}s across {n} client(s):", a.secs);
|
||||
tally.lock().unwrap().report();
|
||||
Ok(())
|
||||
}
|
||||
other => bail!("unknown command {other}"),
|
||||
}
|
||||
}
|
||||
@@ -1,103 +0,0 @@
|
||||
//! Gateway settings from environment variables. See `server/README.md` for the list.
|
||||
|
||||
use std::net::{Ipv4Addr, SocketAddr, ToSocketAddrs};
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
|
||||
use crate::hl::HlConfig;
|
||||
use crate::session::SessionConfig;
|
||||
use crate::store::HubConfig;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Config {
|
||||
/// Plain HTTP and websocket. TLS ends at the edge in front of the gateway.
|
||||
pub bind: SocketAddr,
|
||||
pub hl: HlConfig,
|
||||
pub hub: HubConfig,
|
||||
pub session: SessionConfig,
|
||||
pub max_clients: usize,
|
||||
}
|
||||
|
||||
impl Default for Config {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
bind: SocketAddr::from((Ipv4Addr::LOCALHOST, 8787)),
|
||||
hl: HlConfig::default(),
|
||||
hub: HubConfig::default(),
|
||||
session: SessionConfig::default(),
|
||||
max_clients: 1000,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn var(k: &str) -> Option<String> {
|
||||
std::env::var(k).ok().map(|v| v.trim().to_string()).filter(|v| !v.is_empty())
|
||||
}
|
||||
|
||||
fn num<T: std::str::FromStr>(k: &str) -> Result<Option<T>>
|
||||
where
|
||||
T::Err: std::error::Error + Send + Sync + 'static,
|
||||
{
|
||||
var(k).map(|v| v.parse::<T>().with_context(|| format!("{k}={v}"))).transpose()
|
||||
}
|
||||
|
||||
impl Config {
|
||||
pub fn from_env() -> Result<Self> {
|
||||
let mut c = Config::default();
|
||||
if let Some(b) = var("TR_GATEWAY_BIND") {
|
||||
c.bind = b
|
||||
.to_socket_addrs()
|
||||
.ok()
|
||||
.and_then(|mut a| a.next())
|
||||
.with_context(|| format!("TR_GATEWAY_BIND={b} (want host:port)"))?;
|
||||
}
|
||||
if let Some(u) = var("TR_HL_WS_URL") {
|
||||
c.hl.pool.ws_url = u;
|
||||
}
|
||||
if let Some(u) = var("TR_HL_INFO_URL") {
|
||||
c.hl.info_url = u;
|
||||
}
|
||||
if let Some(d) = var("TR_HL_DEXES").filter(|d| !d.eq_ignore_ascii_case("all")) {
|
||||
c.hl.dexes = Some(d.split(',').map(|s| s.trim().to_string()).filter(|s| !s.is_empty()).collect());
|
||||
}
|
||||
if let Some(n) = num("TR_UPSTREAM_CONNECTIONS")? {
|
||||
c.hl.pool.max_connections = n;
|
||||
}
|
||||
if let Some(n) = num("TR_UPSTREAM_SUBSCRIPTIONS")? {
|
||||
c.hl.pool.max_subscriptions = n;
|
||||
}
|
||||
if let Some(n) = num("TR_UPSTREAM_MESSAGES_PER_MIN")? {
|
||||
c.hl.pool.messages_per_min = n;
|
||||
}
|
||||
if let Some(n) = num("TR_REST_WEIGHT_PER_MIN")? {
|
||||
c.hl.rest_weight_per_min = n;
|
||||
}
|
||||
if let Some(s) = num::<f64>("TR_LINGER_SECS")? {
|
||||
c.hub.linger = Duration::from_secs_f64(s.max(0.0));
|
||||
}
|
||||
if let Some(n) = num("TR_MAX_CLIENTS")? {
|
||||
c.max_clients = n;
|
||||
}
|
||||
Ok(c)
|
||||
}
|
||||
}
|
||||
|
||||
pub const HELP: &str = "\
|
||||
tr-gateway: the Terminal data server (interim: Hyperliquid's public feed)
|
||||
|
||||
Serves websocket clients at /ws, plus /healthz, /readyz, /metrics and /stats.
|
||||
Plain HTTP only: put a TLS-terminating edge in front of it for wss:// on 443.
|
||||
|
||||
Environment:
|
||||
TR_GATEWAY_BIND listen address, host:port (default localhost:8787)
|
||||
TR_HL_DEXES HIP-3 dexes besides the core dex: all (default) or a list like xyz,flx
|
||||
TR_UPSTREAM_CONNECTIONS upstream websocket connections, 2 to 8 (default 3)
|
||||
TR_UPSTREAM_SUBSCRIPTIONS upstream subscriptions, at most 950 (default 900)
|
||||
TR_UPSTREAM_MESSAGES_PER_MIN messages sent upstream a minute, at most 1800 (default 1000)
|
||||
TR_REST_WEIGHT_PER_MIN info endpoint weight a minute, at most 1100 (default 800)
|
||||
TR_LINGER_SECS keep an unwatched market's streams this long (default 20)
|
||||
TR_MAX_CLIENTS client connections (default 1000)
|
||||
TR_HL_WS_URL, TR_HL_INFO_URL upstream endpoints (default Hyperliquid mainnet)
|
||||
RUST_LOG log filter (default info)
|
||||
";
|
||||
@@ -1,144 +0,0 @@
|
||||
//! `HlPublicProvider`: Hyperliquid through its public API, the interim source until our own node
|
||||
//! is up (docs/architecture/server.md, "Interim: public feed"). Read-only public calls only: no
|
||||
//! `exchange` endpoint, no user-specific subscriptions, no orders. A node-backed provider
|
||||
//! replaces it behind the same [`Provider`] trait.
|
||||
|
||||
pub mod pool;
|
||||
pub mod rest;
|
||||
pub mod wire;
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use anyhow::Result;
|
||||
use async_trait::async_trait;
|
||||
use tr_core::{Bar, Timeframe};
|
||||
use tr_proto::InstrumentKind;
|
||||
|
||||
use crate::metrics::Metrics;
|
||||
use crate::provider::{
|
||||
Capabilities, Channel, ChannelKind, DisplayPolicy, InstrumentDef, InstrumentId, Provider, ProviderId,
|
||||
SubscribeError, UpstreamHandle, UpstreamSink, UpstreamStats,
|
||||
};
|
||||
use pool::{Pool, PoolConfig};
|
||||
use rest::{CANDLE_CAP, Info};
|
||||
use wire::SubKey;
|
||||
|
||||
pub const MAINNET_WS: &str = "wss://api.hyperliquid.xyz/ws";
|
||||
pub const MAINNET_INFO: &str = "https://api.hyperliquid.xyz/info";
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct HlConfig {
|
||||
pub info_url: String,
|
||||
pub pool: PoolConfig,
|
||||
/// HIP-3 dexes to carry besides the core dex; `None` means every dex `perpDexs` lists.
|
||||
pub dexes: Option<Vec<String>>,
|
||||
/// REST weight per minute. Capped at 1100, below Hyperliquid's 1200 per IP.
|
||||
pub rest_weight_per_min: u32,
|
||||
}
|
||||
|
||||
impl Default for HlConfig {
|
||||
fn default() -> Self {
|
||||
Self { info_url: MAINNET_INFO.into(), pool: PoolConfig::new(MAINNET_WS), dexes: None, rest_weight_per_min: 800 }
|
||||
}
|
||||
}
|
||||
|
||||
pub struct HlPublicProvider {
|
||||
pool: Pool,
|
||||
info: Info,
|
||||
dexes: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
impl HlPublicProvider {
|
||||
pub fn new(cfg: HlConfig, metrics: Arc<Metrics>) -> Result<Self> {
|
||||
Ok(Self {
|
||||
pool: Pool::new(cfg.pool, metrics.clone()),
|
||||
info: Info::new(&cfg.info_url, cfg.rest_weight_per_min.clamp(20, 1100), metrics)?,
|
||||
dexes: cfg.dexes,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn now_s() -> i64 {
|
||||
SystemTime::now().duration_since(UNIX_EPOCH).map(|d| d.as_secs() as i64).unwrap_or(0)
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Provider for HlPublicProvider {
|
||||
fn id(&self) -> ProviderId {
|
||||
ProviderId("hyperliquid-public")
|
||||
}
|
||||
|
||||
fn capabilities(&self) -> Capabilities {
|
||||
Capabilities {
|
||||
channels: vec![ChannelKind::Mids, ChannelKind::Book, ChannelKind::Trades, ChannelKind::Bars],
|
||||
kinds: vec![InstrumentKind::Perp, InstrumentKind::Spot],
|
||||
}
|
||||
}
|
||||
|
||||
fn display_policy(&self) -> DisplayPolicy {
|
||||
DisplayPolicy { public_display: true, entitlement: None, delay: None, attribution: "Data: Hyperliquid" }
|
||||
}
|
||||
|
||||
fn close(&self) {
|
||||
self.pool.close();
|
||||
}
|
||||
|
||||
fn stats(&self) -> UpstreamStats {
|
||||
let (used, budget) = self.info.weight();
|
||||
UpstreamStats { rest_weight_last_min: used, rest_weight_per_min: budget, ..self.pool.stats() }
|
||||
}
|
||||
|
||||
/// Perps on the core dex and the selected HIP-3 dexes, with their previous-day prices. Spot
|
||||
/// markets aren't listed here, but they stream: the core dex's mids carry them.
|
||||
async fn instruments(&self) -> Result<Vec<InstrumentDef>> {
|
||||
let mut dexes = self.info.perp_dexs().await?;
|
||||
if let Some(want) = &self.dexes {
|
||||
dexes.retain(|d| d.is_empty() || want.contains(d));
|
||||
}
|
||||
let mut out = Vec::new();
|
||||
for dex in dexes {
|
||||
match self.info.perps(&dex).await {
|
||||
Ok(perps) => out.extend(perps.into_iter().filter(|p| !p.delisted).map(|p| InstrumentDef {
|
||||
id: InstrumentId(p.name),
|
||||
venue: dex.clone(),
|
||||
kind: InstrumentKind::Perp,
|
||||
prev_day: p.prev_day,
|
||||
})),
|
||||
// One dex failing shouldn't hide the others.
|
||||
Err(e) => tracing::warn!("dex {dex:?}: {e:#}"),
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// From `candleSnapshot`, which only serves the most recent 5000 bars per interval: paging
|
||||
/// further back returns nothing (the interim feed's history cap, until node-built candles).
|
||||
async fn history(&self, inst: &InstrumentId, tf: Timeframe, end: Option<i64>, limit: usize) -> Result<Vec<Bar>> {
|
||||
let limit = limit.clamp(1, CANDLE_CAP);
|
||||
let end_s = end.unwrap_or_else(|| now_s() + 1);
|
||||
// Months are counted as 31 days so the window always covers `limit` bars.
|
||||
let span = if tf == Timeframe::MN1 { 31 * 86_400 } else { tf.seconds() };
|
||||
let start_s = end_s - span * (limit as i64 + 1);
|
||||
let bars =
|
||||
self.info.candles(inst.as_str(), wire::interval(tf), start_s * 1000, end_s * 1000 - 1, limit).await?;
|
||||
Ok(crate::clip(bars, end, limit))
|
||||
}
|
||||
|
||||
async fn subscribe(
|
||||
&self,
|
||||
ch: Channel,
|
||||
inst: &InstrumentId,
|
||||
sink: UpstreamSink,
|
||||
) -> Result<UpstreamHandle, SubscribeError> {
|
||||
let coin = inst.0.clone();
|
||||
let key = match ch {
|
||||
Channel::Mids => SubKey::AllMids { dex: coin },
|
||||
Channel::Book => SubKey::L2Book { coin },
|
||||
Channel::Trades => SubKey::Trades { coin },
|
||||
Channel::Bars(tf) => SubKey::Candle { coin, interval: wire::interval(tf) },
|
||||
};
|
||||
let guard = self.pool.subscribe(key, sink, ch == Channel::Mids)?;
|
||||
Ok(UpstreamHandle::new(guard))
|
||||
}
|
||||
}
|
||||
@@ -1,540 +0,0 @@
|
||||
//! The server's upstream connections to Hyperliquid's public websocket, shared by every client.
|
||||
//!
|
||||
//! Hyperliquid allows 10 websocket connections, 30 new connections a minute, 1000 subscriptions
|
||||
//! and 2000 sent messages a minute per IP (RL in docs/architecture/server.md). The pool keeps a
|
||||
//! budget below each and refuses a subscription past it; it never works around a limit.
|
||||
//!
|
||||
//! Permanent subscriptions (every dex's mids) get a connection of their own. Hyperliquid closes
|
||||
//! a connection outright when it is asked for a coin it doesn't list, and that must not take
|
||||
//! the mids feed down with it.
|
||||
|
||||
use std::collections::{BTreeMap, HashSet, VecDeque};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
||||
|
||||
use futures::{SinkExt, StreamExt};
|
||||
use tokio::sync::mpsc;
|
||||
use tokio_tungstenite::tungstenite::Message;
|
||||
|
||||
use super::wire::{self, Incoming, SubKey};
|
||||
use crate::limit::WindowLimiter;
|
||||
use crate::metrics::Metrics;
|
||||
use crate::provider::{SubscribeError, Upstream, UpstreamEvent, UpstreamSink, UpstreamStats};
|
||||
|
||||
/// Hyperliquid's own per-IP limits. Budgets are capped below these.
|
||||
pub const HL_MAX_CONNECTIONS: usize = 10;
|
||||
pub const HL_MAX_SUBSCRIPTIONS: usize = 1000;
|
||||
pub const HL_MAX_MESSAGES_PER_MIN: u32 = 2000;
|
||||
pub const HL_MAX_CONNECTS_PER_MIN: u32 = 30;
|
||||
|
||||
/// Open another on-demand connection once each one carries this many subscriptions.
|
||||
const SPREAD_AT: usize = 200;
|
||||
/// A subscription sent this recently and never acknowledged is blamed for a closed connection.
|
||||
const REJECT_WINDOW: Duration = Duration::from_secs(3);
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct PoolConfig {
|
||||
pub ws_url: String,
|
||||
/// Connections, the permanent one included. At least 2, at most 8.
|
||||
pub max_connections: usize,
|
||||
/// Subscriptions across all connections. At most 950.
|
||||
pub max_subscriptions: usize,
|
||||
/// Messages sent (subscribe, unsubscribe, ping) per minute across all connections.
|
||||
pub messages_per_min: u32,
|
||||
pub connects_per_min: u32,
|
||||
/// Hyperliquid closes a connection it hasn't sent anything to in 60 s.
|
||||
pub ping_every: Duration,
|
||||
/// Reconnect when nothing has arrived for this long.
|
||||
pub idle_timeout: Duration,
|
||||
}
|
||||
|
||||
impl PoolConfig {
|
||||
pub fn new(ws_url: impl Into<String>) -> Self {
|
||||
Self {
|
||||
ws_url: ws_url.into(),
|
||||
max_connections: 3,
|
||||
max_subscriptions: 900,
|
||||
messages_per_min: 1000,
|
||||
connects_per_min: 10,
|
||||
ping_every: Duration::from_secs(30),
|
||||
idle_timeout: Duration::from_secs(75),
|
||||
}
|
||||
}
|
||||
|
||||
/// Keep every budget inside Hyperliquid's limits, with room for reconnect overlap (a
|
||||
/// dropped socket may still count against the IP for a moment) and other tools on the IP.
|
||||
pub fn clamped(mut self) -> Self {
|
||||
self.max_connections = self.max_connections.clamp(2, HL_MAX_CONNECTIONS - 2);
|
||||
self.max_subscriptions = self.max_subscriptions.clamp(1, HL_MAX_SUBSCRIPTIONS - 50);
|
||||
self.messages_per_min = self.messages_per_min.clamp(10, HL_MAX_MESSAGES_PER_MIN - 200);
|
||||
self.connects_per_min = self.connects_per_min.clamp(1, HL_MAX_CONNECTS_PER_MIN - 10);
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
/// What every connection task shares. It holds no connection handles, so the tasks end once
|
||||
/// the pool and every guard are gone.
|
||||
struct Shared {
|
||||
cfg: PoolConfig,
|
||||
sends: WindowLimiter,
|
||||
connects: WindowLimiter,
|
||||
metrics: Arc<Metrics>,
|
||||
}
|
||||
|
||||
struct Conn {
|
||||
tx: mpsc::UnboundedSender<Cmd>,
|
||||
permanent: bool,
|
||||
subs: usize,
|
||||
up: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
struct State {
|
||||
conns: Vec<Conn>,
|
||||
subs: BTreeMap<SubKey, usize>,
|
||||
closed: bool,
|
||||
}
|
||||
|
||||
struct Inner {
|
||||
shared: Arc<Shared>,
|
||||
state: Mutex<State>,
|
||||
}
|
||||
|
||||
enum Cmd {
|
||||
Subscribe(SubKey, UpstreamSink),
|
||||
Unsubscribe(SubKey),
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Pool(Arc<Inner>);
|
||||
|
||||
/// Holds one upstream subscription; dropping it unsubscribes.
|
||||
pub struct SubGuard {
|
||||
pool: Arc<Inner>,
|
||||
key: SubKey,
|
||||
}
|
||||
|
||||
impl Drop for SubGuard {
|
||||
fn drop(&mut self) {
|
||||
let mut st = self.pool.state.lock().expect("pool lock");
|
||||
if let Some(i) = st.subs.remove(&self.key)
|
||||
&& let Some(c) = st.conns.get_mut(i)
|
||||
{
|
||||
c.subs -= 1;
|
||||
let _ = c.tx.send(Cmd::Unsubscribe(self.key.clone()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Pool {
|
||||
pub fn new(cfg: PoolConfig, metrics: Arc<Metrics>) -> Self {
|
||||
let cfg = cfg.clamped();
|
||||
let shared = Arc::new(Shared {
|
||||
sends: WindowLimiter::per_minute(cfg.messages_per_min),
|
||||
connects: WindowLimiter::per_minute(cfg.connects_per_min),
|
||||
cfg,
|
||||
metrics,
|
||||
});
|
||||
Self(Arc::new(Inner {
|
||||
shared,
|
||||
state: Mutex::new(State { conns: Vec::new(), subs: BTreeMap::new(), closed: false }),
|
||||
}))
|
||||
}
|
||||
|
||||
/// Subscribe upstream, within budget. Must run inside a tokio runtime (it may open a
|
||||
/// connection).
|
||||
pub fn subscribe(&self, key: SubKey, sink: UpstreamSink, permanent: bool) -> Result<SubGuard, SubscribeError> {
|
||||
let cfg = &self.0.shared.cfg;
|
||||
let mut st = self.0.state.lock().expect("pool lock");
|
||||
if st.closed {
|
||||
return Err(SubscribeError::Unsupported("the upstream pool is closed".into()));
|
||||
}
|
||||
if st.subs.contains_key(&key) {
|
||||
return Err(SubscribeError::Unsupported(format!("{key} is already subscribed")));
|
||||
}
|
||||
if st.subs.len() >= cfg.max_subscriptions {
|
||||
return Err(SubscribeError::Capacity(format!(
|
||||
"{} of {} upstream subscriptions in use",
|
||||
st.subs.len(),
|
||||
cfg.max_subscriptions
|
||||
)));
|
||||
}
|
||||
let least = st
|
||||
.conns
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(_, c)| c.permanent == permanent)
|
||||
.min_by_key(|(_, c)| c.subs)
|
||||
.map(|(i, c)| (i, c.subs));
|
||||
let i = match least {
|
||||
Some((i, _)) if permanent => i,
|
||||
Some((i, n)) if n < SPREAD_AT || st.conns.len() >= cfg.max_connections => i,
|
||||
_ if st.conns.len() < cfg.max_connections => {
|
||||
let (tx, rx) = mpsc::unbounded_channel();
|
||||
let up = Arc::new(AtomicBool::new(false));
|
||||
let id = st.conns.len();
|
||||
tokio::spawn(run_conn(self.0.shared.clone(), id, up.clone(), rx));
|
||||
st.conns.push(Conn { tx, permanent, subs: 0, up });
|
||||
id
|
||||
}
|
||||
_ => {
|
||||
return Err(SubscribeError::Capacity(format!(
|
||||
"all {} upstream connections are in use",
|
||||
cfg.max_connections
|
||||
)));
|
||||
}
|
||||
};
|
||||
st.conns[i].subs += 1;
|
||||
st.subs.insert(key.clone(), i);
|
||||
let _ = st.conns[i].tx.send(Cmd::Subscribe(key.clone(), sink));
|
||||
Ok(SubGuard { pool: self.0.clone(), key })
|
||||
}
|
||||
|
||||
/// Close every connection for good. Dropping a connection's sender ends its task, which
|
||||
/// closes the socket and never reconnects.
|
||||
pub fn close(&self) {
|
||||
let mut st = self.0.state.lock().expect("pool lock");
|
||||
st.closed = true;
|
||||
st.subs.clear();
|
||||
st.conns.clear();
|
||||
}
|
||||
|
||||
pub fn stats(&self) -> UpstreamStats {
|
||||
let sh = &self.0.shared;
|
||||
let st = self.0.state.lock().expect("pool lock");
|
||||
let mut by_kind = BTreeMap::new();
|
||||
for k in st.subs.keys() {
|
||||
*by_kind.entry(k.kind()).or_insert(0) += 1;
|
||||
}
|
||||
UpstreamStats {
|
||||
connections_open: st.conns.len(),
|
||||
connections_up: st.conns.iter().filter(|c| c.up.load(Ordering::Relaxed)).count(),
|
||||
max_connections: sh.cfg.max_connections,
|
||||
subscriptions: st.subs.len(),
|
||||
max_subscriptions: sh.cfg.max_subscriptions,
|
||||
by_kind,
|
||||
per_connection: st.conns.iter().map(|c| c.subs).collect(),
|
||||
keys: st.subs.keys().map(ToString::to_string).collect(),
|
||||
messages_sent_last_min: sh.sends.used(),
|
||||
messages_per_min: sh.sends.budget(),
|
||||
connects_last_min: sh.connects.used(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn now_ms() -> i64 {
|
||||
SystemTime::now().duration_since(UNIX_EPOCH).map(|d| d.as_millis() as i64).unwrap_or(0)
|
||||
}
|
||||
|
||||
struct Sub {
|
||||
sink: UpstreamSink,
|
||||
/// Hyperliquid acknowledged it on the current connection.
|
||||
acked: bool,
|
||||
/// When the subscribe went out on the current connection.
|
||||
sent: Option<Instant>,
|
||||
/// Sinks were told `Disconnected` and haven't heard `Reconnected` yet.
|
||||
stale: bool,
|
||||
}
|
||||
|
||||
impl Sub {
|
||||
fn event(&self, event: UpstreamEvent) {
|
||||
self.sink.send(Upstream { recv: Instant::now(), recv_ms: now_ms(), event });
|
||||
}
|
||||
}
|
||||
|
||||
enum Out {
|
||||
Subscribe(SubKey),
|
||||
Unsubscribe(SubKey),
|
||||
Ping,
|
||||
}
|
||||
|
||||
#[derive(PartialEq)]
|
||||
enum End {
|
||||
/// The connection dropped; reconnect.
|
||||
Dropped,
|
||||
/// The pool is gone; stop.
|
||||
Shutdown,
|
||||
}
|
||||
|
||||
/// Apply a command whether or not the connection is up. Returns false once the pool is gone.
|
||||
fn apply(cmd: Option<Cmd>, subs: &mut BTreeMap<SubKey, Sub>, out: Option<&mut VecDeque<Out>>) -> bool {
|
||||
match cmd {
|
||||
None => false,
|
||||
Some(Cmd::Subscribe(key, sink)) => {
|
||||
subs.insert(key.clone(), Sub { sink, acked: false, sent: None, stale: false });
|
||||
if let Some(out) = out {
|
||||
out.push_back(Out::Subscribe(key));
|
||||
}
|
||||
true
|
||||
}
|
||||
Some(Cmd::Unsubscribe(key)) => {
|
||||
if subs.remove(&key).is_some()
|
||||
&& let Some(out) = out
|
||||
{
|
||||
out.push_back(Out::Unsubscribe(key));
|
||||
}
|
||||
true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Wait out `fut` (a backoff or the connect limiter) while still taking commands.
|
||||
async fn wait<F: Future<Output = ()>>(
|
||||
fut: F,
|
||||
rx: &mut mpsc::UnboundedReceiver<Cmd>,
|
||||
subs: &mut BTreeMap<SubKey, Sub>,
|
||||
) -> bool {
|
||||
tokio::pin!(fut);
|
||||
loop {
|
||||
tokio::select! {
|
||||
_ = &mut fut => return true,
|
||||
cmd = rx.recv() => if !apply(cmd, subs, None) { return false },
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn run_conn(sh: Arc<Shared>, id: usize, up: Arc<AtomicBool>, mut rx: mpsc::UnboundedReceiver<Cmd>) {
|
||||
let mut subs: BTreeMap<SubKey, Sub> = BTreeMap::new();
|
||||
let mut backoff = Duration::from_secs(1);
|
||||
let mut connected_before = false;
|
||||
loop {
|
||||
if !wait(sh.connects.acquire(1), &mut rx, &mut subs).await {
|
||||
return;
|
||||
}
|
||||
match tokio_tungstenite::connect_async(sh.cfg.ws_url.as_str()).await {
|
||||
Ok((ws, _)) => {
|
||||
up.store(true, Ordering::Relaxed);
|
||||
if connected_before {
|
||||
Metrics::add(&sh.metrics.upstream_reconnects, 1);
|
||||
}
|
||||
connected_before = true;
|
||||
tracing::info!("upstream connection {id} up ({} subscriptions)", subs.len());
|
||||
let started = Instant::now();
|
||||
let end = session(&sh, ws, &mut subs, &mut rx).await;
|
||||
up.store(false, Ordering::Relaxed);
|
||||
blame_unacked(&sh, &mut subs);
|
||||
for s in subs.values_mut().filter(|s| !s.stale) {
|
||||
s.event(UpstreamEvent::Disconnected);
|
||||
s.stale = true;
|
||||
}
|
||||
if end == End::Shutdown {
|
||||
return;
|
||||
}
|
||||
tracing::warn!("upstream connection {id} dropped after {:?}", started.elapsed());
|
||||
if started.elapsed() > Duration::from_secs(60) {
|
||||
backoff = Duration::from_secs(1);
|
||||
}
|
||||
}
|
||||
Err(e) => tracing::warn!("upstream connection {id}: {e}"),
|
||||
}
|
||||
// Up to 25% jitter so connections don't reconnect in lockstep.
|
||||
let jitter = backoff.mul_f64(
|
||||
f64::from(SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().subsec_nanos() % 250) / 1000.0,
|
||||
);
|
||||
if !wait(tokio::time::sleep(backoff + jitter), &mut rx, &mut subs).await {
|
||||
return;
|
||||
}
|
||||
backoff = (backoff * 2).min(Duration::from_secs(30));
|
||||
}
|
||||
}
|
||||
|
||||
/// Hyperliquid answers a subscription to a coin it doesn't list by closing the connection,
|
||||
/// with no error message. A subscription sent just before the close and never acknowledged
|
||||
/// is the likely cause: drop it rather than resubscribe it into a reconnect loop.
|
||||
fn blame_unacked(sh: &Shared, subs: &mut BTreeMap<SubKey, Sub>) {
|
||||
let suspects: Vec<SubKey> = subs
|
||||
.iter()
|
||||
.filter(|(_, s)| !s.acked && s.sent.is_some_and(|t| t.elapsed() < REJECT_WINDOW))
|
||||
.map(|(k, _)| k.clone())
|
||||
.collect();
|
||||
for key in suspects {
|
||||
tracing::warn!("upstream closed the connection right after `{key}`; not resubscribing it");
|
||||
Metrics::add(&sh.metrics.upstream_rejected, 1);
|
||||
if let Some(s) = subs.remove(&key) {
|
||||
s.event(UpstreamEvent::Rejected(format!("the upstream refused {key}")));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type Ws = tokio_tungstenite::WebSocketStream<tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>>;
|
||||
|
||||
async fn session(sh: &Shared, ws: Ws, subs: &mut BTreeMap<SubKey, Sub>, rx: &mut mpsc::UnboundedReceiver<Cmd>) -> End {
|
||||
let (mut tx, mut stream) = ws.split();
|
||||
let mut out: VecDeque<Out> = subs.keys().cloned().map(Out::Subscribe).collect();
|
||||
for s in subs.values_mut() {
|
||||
s.acked = false;
|
||||
s.sent = None;
|
||||
}
|
||||
// Keys whose subscribe went out on this connection, so an unsubscribe is only ever sent
|
||||
// for something actually subscribed.
|
||||
let mut active: HashSet<SubKey> = HashSet::new();
|
||||
let mut ping = tokio::time::interval(sh.cfg.ping_every);
|
||||
ping.tick().await;
|
||||
let mut last_rx = Instant::now();
|
||||
loop {
|
||||
// Drop queued messages that no longer apply before spending send budget on them.
|
||||
while let Some(o) = out.front() {
|
||||
let stale = match o {
|
||||
Out::Subscribe(k) => !subs.contains_key(k) || active.contains(k),
|
||||
Out::Unsubscribe(k) => !active.contains(k) || subs.contains_key(k),
|
||||
Out::Ping => false,
|
||||
};
|
||||
if !stale {
|
||||
break;
|
||||
}
|
||||
out.pop_front();
|
||||
}
|
||||
tokio::select! {
|
||||
biased;
|
||||
msg = stream.next() => match msg {
|
||||
Some(Ok(Message::Text(text))) => {
|
||||
let recv = Instant::now();
|
||||
last_rx = recv;
|
||||
Metrics::add(&sh.metrics.upstream_messages, 1);
|
||||
Metrics::add(&sh.metrics.upstream_bytes, text.len() as u64);
|
||||
handle(text.as_str(), recv, subs);
|
||||
}
|
||||
Some(Ok(Message::Close(_))) | None => return End::Dropped,
|
||||
Some(Ok(_)) => last_rx = Instant::now(),
|
||||
Some(Err(e)) => {
|
||||
tracing::debug!("upstream read: {e}");
|
||||
return End::Dropped;
|
||||
}
|
||||
},
|
||||
cmd = rx.recv() => if !apply(cmd, subs, Some(&mut out)) {
|
||||
let _ = tx.close().await;
|
||||
return End::Shutdown;
|
||||
},
|
||||
_ = sh.sends.acquire(1), if !out.is_empty() => {
|
||||
let text = match out.pop_front().expect("checked non-empty") {
|
||||
Out::Subscribe(k) => {
|
||||
if let Some(s) = subs.get_mut(&k) {
|
||||
s.sent = Some(Instant::now());
|
||||
}
|
||||
let m = k.message("subscribe");
|
||||
active.insert(k);
|
||||
m
|
||||
}
|
||||
Out::Unsubscribe(k) => {
|
||||
active.remove(&k);
|
||||
k.message("unsubscribe")
|
||||
}
|
||||
Out::Ping => wire::PING.to_string(),
|
||||
};
|
||||
if let Err(e) = tx.send(Message::Text(text.into())).await {
|
||||
tracing::debug!("upstream write: {e}");
|
||||
return End::Dropped;
|
||||
}
|
||||
}
|
||||
_ = ping.tick() => {
|
||||
if last_rx.elapsed() > sh.cfg.idle_timeout {
|
||||
tracing::warn!("upstream silent for {:?}; reconnecting", last_rx.elapsed());
|
||||
return End::Dropped;
|
||||
}
|
||||
out.push_back(Out::Ping);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn handle(text: &str, recv: Instant, subs: &mut BTreeMap<SubKey, Sub>) {
|
||||
let recv_ms = now_ms();
|
||||
match wire::parse(text) {
|
||||
Ok(Incoming::Data(key, event)) => {
|
||||
if let Some(s) = subs.get_mut(&key) {
|
||||
if s.stale {
|
||||
s.stale = false;
|
||||
s.sink.send(Upstream { recv, recv_ms, event: UpstreamEvent::Reconnected });
|
||||
}
|
||||
s.acked = true;
|
||||
s.sink.send(Upstream { recv, recv_ms, event });
|
||||
}
|
||||
}
|
||||
Ok(Incoming::Subscribed(key)) => {
|
||||
if let Some(s) = subs.get_mut(&key) {
|
||||
s.acked = true;
|
||||
if s.stale {
|
||||
s.stale = false;
|
||||
s.sink.send(Upstream { recv, recv_ms, event: UpstreamEvent::Reconnected });
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(Incoming::Error(e)) => tracing::warn!("upstream error: {e}"),
|
||||
Ok(Incoming::Unsubscribed(_) | Incoming::Pong | Incoming::Ignored) => {}
|
||||
Err(e) => tracing::debug!("upstream message skipped: {e}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn sink() -> UpstreamSink {
|
||||
UpstreamSink::new(|_| {})
|
||||
}
|
||||
|
||||
fn book(c: &str) -> SubKey {
|
||||
SubKey::L2Book { coin: c.into() }
|
||||
}
|
||||
|
||||
fn pool(max_connections: usize, max_subscriptions: usize) -> Pool {
|
||||
// Nothing listens here: the connection tasks retry in the background, which these
|
||||
// tests don't need. They only check the pool's accounting.
|
||||
let mut cfg = PoolConfig::new("ws://localhost:9/ws");
|
||||
cfg.max_connections = max_connections;
|
||||
cfg.max_subscriptions = max_subscriptions;
|
||||
Pool::new(cfg, Arc::new(Metrics::default()))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn budgets_are_capped_below_hyperliquid_limits() {
|
||||
let mut cfg = PoolConfig::new("x");
|
||||
cfg.max_connections = 50;
|
||||
cfg.max_subscriptions = 5000;
|
||||
cfg.messages_per_min = 9999;
|
||||
cfg.connects_per_min = 100;
|
||||
let c = cfg.clamped();
|
||||
assert!(c.max_connections < HL_MAX_CONNECTIONS && c.max_subscriptions < HL_MAX_SUBSCRIPTIONS);
|
||||
assert!(c.messages_per_min < HL_MAX_MESSAGES_PER_MIN && c.connects_per_min < HL_MAX_CONNECTS_PER_MIN);
|
||||
assert_eq!(PoolConfig { max_connections: 1, ..PoolConfig::new("x") }.clamped().max_connections, 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn refuses_subscriptions_past_the_budget() {
|
||||
let p = pool(3, 4);
|
||||
let mids = p.subscribe(SubKey::AllMids { dex: String::new() }, sink(), true).unwrap();
|
||||
let a = p.subscribe(book("BTC"), sink(), false).unwrap();
|
||||
let _b = p.subscribe(book("ETH"), sink(), false).unwrap();
|
||||
let _c = p.subscribe(SubKey::Trades { coin: "BTC".into() }, sink(), false).unwrap();
|
||||
assert!(matches!(p.subscribe(book("SOL"), sink(), false), Err(SubscribeError::Capacity(_))));
|
||||
assert!(matches!(p.subscribe(book("ETH"), sink(), false), Err(SubscribeError::Unsupported(_))));
|
||||
let s = p.stats();
|
||||
assert_eq!((s.subscriptions, s.connections_open), (4, 2), "mids alone on one, the rest on another");
|
||||
assert_eq!(s.per_connection, vec![1, 3]);
|
||||
assert_eq!(s.by_kind.get("l2Book"), Some(&2));
|
||||
// Releasing one frees a slot.
|
||||
drop(a);
|
||||
assert_eq!(p.stats().subscriptions, 3);
|
||||
let _d = p.subscribe(book("SOL"), sink(), false).unwrap();
|
||||
drop(mids);
|
||||
assert_eq!(p.stats().per_connection, vec![0, 3]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn never_opens_more_connections_than_the_budget() {
|
||||
let p = pool(2, 900);
|
||||
let _m = p.subscribe(SubKey::AllMids { dex: "xyz".into() }, sink(), true).unwrap();
|
||||
let guards: Vec<_> =
|
||||
(0..SPREAD_AT + 20).map(|i| p.subscribe(book(&format!("C{i}")), sink(), false).unwrap()).collect();
|
||||
let s = p.stats();
|
||||
assert_eq!(s.connections_open, 2);
|
||||
assert_eq!(s.per_connection, vec![1, SPREAD_AT + 20], "past SPREAD_AT the last connection keeps filling");
|
||||
drop(guards);
|
||||
let p = pool(3, 900);
|
||||
let _m = p.subscribe(SubKey::AllMids { dex: String::new() }, sink(), true).unwrap();
|
||||
let _g: Vec<_> =
|
||||
(0..SPREAD_AT + 1).map(|i| p.subscribe(book(&format!("C{i}")), sink(), false).unwrap()).collect();
|
||||
assert_eq!(p.stats().per_connection, vec![1, SPREAD_AT, 1], "a third connection opens once the second is full");
|
||||
}
|
||||
}
|
||||
@@ -1,172 +0,0 @@
|
||||
//! Read-only calls to Hyperliquid's public `info` endpoint, under a weight self-limit that stays
|
||||
//! below Hyperliquid's 1200 per minute per IP (RL in docs/architecture/server.md).
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
use serde::Deserialize;
|
||||
use serde::de::DeserializeOwned;
|
||||
use serde_json::{Value, json};
|
||||
use tr_core::Bar;
|
||||
|
||||
use crate::limit::WindowLimiter;
|
||||
use crate::metrics::Metrics;
|
||||
|
||||
/// `candleSnapshot` serves at most this many of the most recent bars per interval.
|
||||
pub const CANDLE_CAP: usize = 5000;
|
||||
|
||||
/// Weight Hyperliquid charges for a `candleSnapshot` returning `n` bars: 20, plus 1 per 60.
|
||||
pub fn candle_weight(n: usize) -> u32 {
|
||||
20 + n.div_ceil(60) as u32
|
||||
}
|
||||
|
||||
pub struct Info {
|
||||
http: reqwest::Client,
|
||||
url: String,
|
||||
limiter: WindowLimiter,
|
||||
metrics: Arc<Metrics>,
|
||||
}
|
||||
|
||||
/// One perp from `metaAndAssetCtxs`.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct PerpCtx {
|
||||
pub name: String,
|
||||
pub prev_day: Option<f64>,
|
||||
pub delisted: bool,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct Universe {
|
||||
universe: Vec<UniverseEntry>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct UniverseEntry {
|
||||
name: String,
|
||||
#[serde(default, rename = "isDelisted")]
|
||||
delisted: bool,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct Ctx {
|
||||
#[serde(rename = "prevDayPx")]
|
||||
prev_day: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct Candle {
|
||||
t: i64,
|
||||
o: String,
|
||||
h: String,
|
||||
l: String,
|
||||
c: String,
|
||||
v: String,
|
||||
}
|
||||
|
||||
impl Info {
|
||||
pub fn new(url: &str, weight_per_min: u32, metrics: Arc<Metrics>) -> Result<Self> {
|
||||
let http = reqwest::Client::builder()
|
||||
.user_agent(concat!("terminal-gateway/", env!("CARGO_PKG_VERSION")))
|
||||
.timeout(Duration::from_secs(20))
|
||||
.build()?;
|
||||
Ok(Self { http, url: url.into(), limiter: WindowLimiter::per_minute(weight_per_min), metrics })
|
||||
}
|
||||
|
||||
/// Weight spent in the last minute, and the self-limit.
|
||||
pub fn weight(&self) -> (u32, u32) {
|
||||
(self.limiter.used(), self.limiter.budget())
|
||||
}
|
||||
|
||||
async fn post<T: DeserializeOwned>(&self, body: Value, weight: u32) -> Result<T> {
|
||||
self.limiter.acquire(weight).await;
|
||||
Metrics::add(&self.metrics.rest_requests, 1);
|
||||
let kind = body["type"].as_str().unwrap_or("?").to_string();
|
||||
let result = async {
|
||||
let resp = self.http.post(&self.url).json(&body).send().await?;
|
||||
let status = resp.status();
|
||||
if status.as_u16() == 429 {
|
||||
// Keep every caller off the endpoint for a full window.
|
||||
self.limiter.charge(self.limiter.budget());
|
||||
bail!("rate limited (HTTP 429)");
|
||||
}
|
||||
if !status.is_success() {
|
||||
bail!("HTTP {status}");
|
||||
}
|
||||
Ok(resp.json::<T>().await?)
|
||||
}
|
||||
.await;
|
||||
if result.is_err() {
|
||||
Metrics::add(&self.metrics.rest_errors, 1);
|
||||
}
|
||||
result.with_context(|| format!("info {kind}"))
|
||||
}
|
||||
|
||||
/// Perp dex names; `""` is the core dex.
|
||||
pub async fn perp_dexs(&self) -> Result<Vec<String>> {
|
||||
let v: Vec<Option<Value>> = self.post(json!({ "type": "perpDexs" }), 20).await?;
|
||||
Ok(v.into_iter().map(|d| d.and_then(|d| d["name"].as_str().map(str::to_string)).unwrap_or_default()).collect())
|
||||
}
|
||||
|
||||
pub async fn perps(&self, dex: &str) -> Result<Vec<PerpCtx>> {
|
||||
let body = if dex.is_empty() {
|
||||
json!({ "type": "metaAndAssetCtxs" })
|
||||
} else {
|
||||
json!({ "type": "metaAndAssetCtxs", "dex": dex })
|
||||
};
|
||||
let (meta, ctxs): (Universe, Vec<Ctx>) = self.post(body, 20).await?;
|
||||
Ok(meta
|
||||
.universe
|
||||
.into_iter()
|
||||
.zip(ctxs)
|
||||
.map(|(u, c)| PerpCtx {
|
||||
name: u.name,
|
||||
prev_day: c.prev_day.and_then(|p| p.parse().ok()),
|
||||
delisted: u.delisted,
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// Bars opening in `[start_ms, end_ms]`, oldest first. `expect` sizes the weight up front;
|
||||
/// any extra weight the response turns out to cost is charged afterwards.
|
||||
pub async fn candles(
|
||||
&self,
|
||||
coin: &str,
|
||||
interval: &str,
|
||||
start_ms: i64,
|
||||
end_ms: i64,
|
||||
expect: usize,
|
||||
) -> Result<Vec<Bar>> {
|
||||
let body = json!({
|
||||
"type": "candleSnapshot",
|
||||
"req": { "coin": coin, "interval": interval, "startTime": start_ms, "endTime": end_ms },
|
||||
});
|
||||
let charged = candle_weight(expect.min(CANDLE_CAP));
|
||||
let rows: Vec<Candle> = self.post(body, charged).await?;
|
||||
let actual = candle_weight(rows.len());
|
||||
if actual > charged {
|
||||
self.limiter.charge(actual - charged);
|
||||
}
|
||||
let p = |s: &str| s.parse::<f64>().with_context(|| format!("candle number {s:?}"));
|
||||
let mut bars = rows
|
||||
.iter()
|
||||
.map(|r| Ok(Bar::new(r.t.div_euclid(1000), p(&r.o)?, p(&r.h)?, p(&r.l)?, p(&r.c)?, p(&r.v)?)))
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
bars.sort_by_key(|b| b.ts);
|
||||
bars.dedup_by_key(|b| b.ts);
|
||||
Ok(bars)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn candle_weights() {
|
||||
assert_eq!(candle_weight(0), 20);
|
||||
assert_eq!(candle_weight(60), 21);
|
||||
assert_eq!(candle_weight(61), 22);
|
||||
assert_eq!(candle_weight(CANDLE_CAP), 104);
|
||||
}
|
||||
}
|
||||
@@ -1,379 +0,0 @@
|
||||
//! Hyperliquid's public websocket messages: what we send and how we read what comes back.
|
||||
//! Shapes follow the docs (WS-SUB in docs/architecture/server.md) and recorded live messages
|
||||
//! in `tests/fixtures/hl`. Prices and sizes arrive as decimal strings (candles too, although
|
||||
//! the docs type them as numbers), so every number accepts either.
|
||||
|
||||
use std::borrow::Cow;
|
||||
use std::collections::HashMap;
|
||||
use std::fmt;
|
||||
|
||||
use serde::Deserialize;
|
||||
use serde::de::{self, Deserializer, Visitor};
|
||||
use serde_json::json;
|
||||
use serde_json::value::RawValue;
|
||||
use tr_core::{Bar, Timeframe};
|
||||
use tr_proto::{Level, Side, Trade};
|
||||
|
||||
use crate::provider::UpstreamEvent;
|
||||
|
||||
/// One upstream subscription, as Hyperliquid identifies it.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
|
||||
pub enum SubKey {
|
||||
/// `dex` is `""` for the core perp dex (whose mids also carry spot).
|
||||
AllMids {
|
||||
dex: String,
|
||||
},
|
||||
L2Book {
|
||||
coin: String,
|
||||
},
|
||||
Trades {
|
||||
coin: String,
|
||||
},
|
||||
Candle {
|
||||
coin: String,
|
||||
interval: &'static str,
|
||||
},
|
||||
}
|
||||
|
||||
impl SubKey {
|
||||
pub fn subscription(&self) -> serde_json::Value {
|
||||
match self {
|
||||
SubKey::AllMids { dex } if dex.is_empty() => json!({ "type": "allMids" }),
|
||||
SubKey::AllMids { dex } => json!({ "type": "allMids", "dex": dex }),
|
||||
SubKey::L2Book { coin } => json!({ "type": "l2Book", "coin": coin }),
|
||||
SubKey::Trades { coin } => json!({ "type": "trades", "coin": coin }),
|
||||
SubKey::Candle { coin, interval } => json!({ "type": "candle", "coin": coin, "interval": interval }),
|
||||
}
|
||||
}
|
||||
|
||||
/// The `subscribe` or `unsubscribe` message for this subscription.
|
||||
pub fn message(&self, method: &str) -> String {
|
||||
json!({ "method": method, "subscription": self.subscription() }).to_string()
|
||||
}
|
||||
|
||||
pub fn kind(&self) -> &'static str {
|
||||
match self {
|
||||
SubKey::AllMids { .. } => "allMids",
|
||||
SubKey::L2Book { .. } => "l2Book",
|
||||
SubKey::Trades { .. } => "trades",
|
||||
SubKey::Candle { .. } => "candle",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for SubKey {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
SubKey::AllMids { dex } if dex.is_empty() => f.write_str("allMids"),
|
||||
SubKey::AllMids { dex } => write!(f, "allMids:{dex}"),
|
||||
SubKey::L2Book { coin } => write!(f, "l2Book:{coin}"),
|
||||
SubKey::Trades { coin } => write!(f, "trades:{coin}"),
|
||||
SubKey::Candle { coin, interval } => write!(f, "candle:{coin}:{interval}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub const PING: &str = r#"{"method":"ping"}"#;
|
||||
|
||||
/// Hyperliquid's name for a timeframe.
|
||||
pub fn interval(tf: Timeframe) -> &'static str {
|
||||
match tf {
|
||||
Timeframe::M1 => "1m",
|
||||
Timeframe::M5 => "5m",
|
||||
Timeframe::M15 => "15m",
|
||||
Timeframe::M30 => "30m",
|
||||
Timeframe::H1 => "1h",
|
||||
Timeframe::H4 => "4h",
|
||||
Timeframe::D1 => "1d",
|
||||
Timeframe::W1 => "1w",
|
||||
Timeframe::MN1 => "1M",
|
||||
}
|
||||
}
|
||||
|
||||
fn static_interval(s: &str) -> Option<&'static str> {
|
||||
Timeframe::ALL.into_iter().map(interval).find(|i| *i == s)
|
||||
}
|
||||
|
||||
/// What one upstream text frame means to us.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum Incoming {
|
||||
Data(SubKey, UpstreamEvent),
|
||||
Subscribed(SubKey),
|
||||
Unsubscribed(SubKey),
|
||||
Pong,
|
||||
Error(String),
|
||||
/// A channel we don't use, or an empty batch.
|
||||
Ignored,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct Envelope<'a> {
|
||||
#[serde(borrow)]
|
||||
channel: Cow<'a, str>,
|
||||
#[serde(borrow)]
|
||||
data: Option<&'a RawValue>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct AllMids {
|
||||
mids: HashMap<String, String>,
|
||||
#[serde(default)]
|
||||
dex: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct WsLevel {
|
||||
#[serde(deserialize_with = "num")]
|
||||
px: f64,
|
||||
#[serde(deserialize_with = "num")]
|
||||
sz: f64,
|
||||
n: u32,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct WsBook {
|
||||
coin: String,
|
||||
time: i64,
|
||||
levels: (Vec<WsLevel>, Vec<WsLevel>),
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct WsTrade {
|
||||
coin: String,
|
||||
side: String,
|
||||
#[serde(deserialize_with = "num")]
|
||||
px: f64,
|
||||
#[serde(deserialize_with = "num")]
|
||||
sz: f64,
|
||||
time: i64,
|
||||
tid: u64,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct WsCandle {
|
||||
t: i64,
|
||||
s: String,
|
||||
i: String,
|
||||
#[serde(deserialize_with = "num")]
|
||||
o: f64,
|
||||
#[serde(deserialize_with = "num")]
|
||||
h: f64,
|
||||
#[serde(deserialize_with = "num")]
|
||||
l: f64,
|
||||
#[serde(deserialize_with = "num")]
|
||||
c: f64,
|
||||
#[serde(deserialize_with = "num")]
|
||||
v: f64,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct SubAck {
|
||||
method: String,
|
||||
subscription: SubObj,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct SubObj {
|
||||
#[serde(rename = "type")]
|
||||
ty: String,
|
||||
coin: Option<String>,
|
||||
dex: Option<String>,
|
||||
interval: Option<String>,
|
||||
}
|
||||
|
||||
impl SubObj {
|
||||
fn key(self) -> Option<SubKey> {
|
||||
Some(match self.ty.as_str() {
|
||||
"allMids" => SubKey::AllMids { dex: self.dex.unwrap_or_default() },
|
||||
"l2Book" => SubKey::L2Book { coin: self.coin? },
|
||||
"trades" => SubKey::Trades { coin: self.coin? },
|
||||
"candle" => SubKey::Candle { coin: self.coin?, interval: static_interval(self.interval.as_deref()?)? },
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// A price or size that may be a JSON string or number.
|
||||
fn num<'de, D: Deserializer<'de>>(d: D) -> Result<f64, D::Error> {
|
||||
struct V;
|
||||
impl Visitor<'_> for V {
|
||||
type Value = f64;
|
||||
fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str("a number or a decimal string")
|
||||
}
|
||||
fn visit_f64<E: de::Error>(self, v: f64) -> Result<f64, E> {
|
||||
Ok(v)
|
||||
}
|
||||
fn visit_u64<E: de::Error>(self, v: u64) -> Result<f64, E> {
|
||||
Ok(v as f64)
|
||||
}
|
||||
fn visit_i64<E: de::Error>(self, v: i64) -> Result<f64, E> {
|
||||
Ok(v as f64)
|
||||
}
|
||||
fn visit_str<E: de::Error>(self, v: &str) -> Result<f64, E> {
|
||||
v.parse().map_err(|_| E::custom(format!("bad number {v:?}")))
|
||||
}
|
||||
}
|
||||
d.deserialize_any(V)
|
||||
}
|
||||
|
||||
fn levels(v: Vec<WsLevel>) -> Vec<Level> {
|
||||
v.into_iter().map(|l| Level { px: l.px, sz: l.sz, n: l.n }).collect()
|
||||
}
|
||||
|
||||
/// Parse one text frame from the upstream websocket.
|
||||
pub fn parse(text: &str) -> Result<Incoming, String> {
|
||||
let env: Envelope = serde_json::from_str(text).map_err(|e| format!("envelope: {e}"))?;
|
||||
let data = || env.data.map(RawValue::get).ok_or_else(|| format!("{} without data", env.channel));
|
||||
let err = |e: serde_json::Error| format!("{}: {e}", env.channel);
|
||||
Ok(match env.channel.as_ref() {
|
||||
"allMids" => {
|
||||
let m: AllMids = serde_json::from_str(data()?).map_err(err)?;
|
||||
// `#…` keys are outcome (HIP-4) markets, which are out of scope (§5.7).
|
||||
let mids = m
|
||||
.mids
|
||||
.into_iter()
|
||||
.filter(|(k, _)| !k.starts_with('#'))
|
||||
.filter_map(|(k, v)| Some((k, v.parse::<f64>().ok()?)))
|
||||
.collect();
|
||||
Incoming::Data(SubKey::AllMids { dex: m.dex.unwrap_or_default() }, UpstreamEvent::Mids(mids))
|
||||
}
|
||||
"l2Book" => {
|
||||
let b: WsBook = serde_json::from_str(data()?).map_err(err)?;
|
||||
let ev = UpstreamEvent::Book { bids: levels(b.levels.0), asks: levels(b.levels.1), time_ms: b.time };
|
||||
Incoming::Data(SubKey::L2Book { coin: b.coin }, ev)
|
||||
}
|
||||
"trades" => {
|
||||
let ts: Vec<WsTrade> = serde_json::from_str(data()?).map_err(err)?;
|
||||
let Some(coin) = ts.first().map(|t| t.coin.clone()) else { return Ok(Incoming::Ignored) };
|
||||
let trades = ts
|
||||
.into_iter()
|
||||
.map(|t| Trade {
|
||||
px: t.px,
|
||||
sz: t.sz,
|
||||
side: if t.side == "B" { Side::Buy } else { Side::Sell },
|
||||
time_ms: t.time,
|
||||
tid: t.tid,
|
||||
})
|
||||
.collect();
|
||||
Incoming::Data(SubKey::Trades { coin }, UpstreamEvent::Trades(trades))
|
||||
}
|
||||
"candle" => {
|
||||
let c: WsCandle = serde_json::from_str(data()?).map_err(err)?;
|
||||
let Some(interval) = static_interval(&c.i) else { return Ok(Incoming::Ignored) };
|
||||
let bar = Bar::new(c.t.div_euclid(1000), c.o, c.h, c.l, c.c, c.v);
|
||||
Incoming::Data(SubKey::Candle { coin: c.s, interval }, UpstreamEvent::Bar(bar))
|
||||
}
|
||||
"subscriptionResponse" => {
|
||||
let ack: SubAck = serde_json::from_str(data()?).map_err(err)?;
|
||||
match (ack.subscription.key(), ack.method.as_str()) {
|
||||
(Some(k), "subscribe") => Incoming::Subscribed(k),
|
||||
(Some(k), "unsubscribe") => Incoming::Unsubscribed(k),
|
||||
_ => Incoming::Ignored,
|
||||
}
|
||||
}
|
||||
"pong" => Incoming::Pong,
|
||||
"error" => Incoming::Error(match env.data {
|
||||
Some(d) => serde_json::from_str::<String>(d.get()).unwrap_or_else(|_| d.get().to_string()),
|
||||
None => String::new(),
|
||||
}),
|
||||
_ => Incoming::Ignored,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn fixture(name: &str) -> String {
|
||||
std::fs::read_to_string(format!("{}/tests/fixtures/hl/{name}", env!("CARGO_MANIFEST_DIR"))).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn subscription_messages() {
|
||||
let k = SubKey::Candle { coin: "BTC".into(), interval: interval(Timeframe::M1) };
|
||||
assert_eq!(
|
||||
k.message("subscribe"),
|
||||
r#"{"method":"subscribe","subscription":{"coin":"BTC","interval":"1m","type":"candle"}}"#
|
||||
);
|
||||
assert_eq!(SubKey::AllMids { dex: String::new() }.subscription(), json!({"type": "allMids"}));
|
||||
assert_eq!(SubKey::AllMids { dex: "xyz".into() }.to_string(), "allMids:xyz");
|
||||
assert_eq!(k.to_string(), "candle:BTC:1m");
|
||||
assert_eq!(Timeframe::ALL.map(interval), ["1m", "5m", "15m", "30m", "1h", "4h", "1d", "1w", "1M"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recorded_messages() {
|
||||
let Incoming::Data(SubKey::AllMids { dex }, UpstreamEvent::Mids(m)) =
|
||||
parse(&fixture("all_mids_xyz.json")).unwrap()
|
||||
else {
|
||||
panic!("allMids")
|
||||
};
|
||||
assert_eq!(dex, "xyz");
|
||||
assert!(m.iter().any(|(k, px)| k == "xyz:TSLA" && *px > 0.0));
|
||||
let Incoming::Data(SubKey::AllMids { dex }, UpstreamEvent::Mids(m)) =
|
||||
parse(&fixture("all_mids_core.json")).unwrap()
|
||||
else {
|
||||
panic!("core allMids")
|
||||
};
|
||||
assert_eq!(dex, "");
|
||||
assert!(m.iter().any(|(k, _)| k == "BTC") && m.iter().any(|(k, _)| k.starts_with('@')));
|
||||
assert!(!m.iter().any(|(k, _)| k.starts_with('#')), "outcome markets are dropped");
|
||||
|
||||
let Incoming::Data(SubKey::L2Book { coin }, UpstreamEvent::Book { bids, asks, time_ms }) =
|
||||
parse(&fixture("l2book_btc.json")).unwrap()
|
||||
else {
|
||||
panic!("l2Book")
|
||||
};
|
||||
assert_eq!((coin.as_str(), bids.len(), asks.len()), ("BTC", 20, 20));
|
||||
assert!(bids[0].px < asks[0].px && bids.windows(2).all(|w| w[0].px > w[1].px) && time_ms > 1_700_000_000_000);
|
||||
|
||||
let Incoming::Data(SubKey::Trades { coin }, UpstreamEvent::Trades(t)) =
|
||||
parse(&fixture("trades_btc.json")).unwrap()
|
||||
else {
|
||||
panic!("trades")
|
||||
};
|
||||
assert_eq!(coin, "BTC");
|
||||
assert!(!t.is_empty() && t.iter().all(|x| x.px > 0.0 && x.sz > 0.0 && x.tid > 0));
|
||||
|
||||
let Incoming::Data(SubKey::Candle { coin, interval }, UpstreamEvent::Bar(b)) =
|
||||
parse(&fixture("candle_btc_1m.json")).unwrap()
|
||||
else {
|
||||
panic!("candle")
|
||||
};
|
||||
assert_eq!((coin.as_str(), interval), ("BTC", "1m"));
|
||||
assert_eq!(b.ts % 60, 0);
|
||||
assert!(b.low <= b.open && b.open <= b.high && b.volume > 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn control_messages() {
|
||||
let ack = r#"{"channel":"subscriptionResponse","data":{"method":"subscribe","subscription":{"type":"l2Book","coin":"BTC","nSigFigs":null,"mantissa":null,"fast":false}}}"#;
|
||||
assert_eq!(parse(ack).unwrap(), Incoming::Subscribed(SubKey::L2Book { coin: "BTC".into() }));
|
||||
let ack = r#"{"channel":"subscriptionResponse","data":{"method":"unsubscribe","subscription":{"type":"candle","interval":"1h","coin":"xyz:TSLA"}}}"#;
|
||||
assert_eq!(
|
||||
parse(ack).unwrap(),
|
||||
Incoming::Unsubscribed(SubKey::Candle { coin: "xyz:TSLA".into(), interval: "1h" })
|
||||
);
|
||||
let ack =
|
||||
r#"{"channel":"subscriptionResponse","data":{"method":"subscribe","subscription":{"type":"allMids"}}}"#;
|
||||
assert_eq!(parse(ack).unwrap(), Incoming::Subscribed(SubKey::AllMids { dex: String::new() }));
|
||||
assert_eq!(parse(r#"{"channel":"pong"}"#).unwrap(), Incoming::Pong);
|
||||
assert_eq!(
|
||||
parse(r#"{"channel":"error","data":"Invalid subscription"}"#).unwrap(),
|
||||
Incoming::Error("Invalid subscription".into())
|
||||
);
|
||||
assert_eq!(parse(r#"{"channel":"trades","data":[]}"#).unwrap(), Incoming::Ignored);
|
||||
assert_eq!(parse(r#"{"channel":"webData3","data":{}}"#).unwrap(), Incoming::Ignored);
|
||||
assert!(parse("not json").is_err());
|
||||
// Numbers may come as JSON numbers, as the docs type candles.
|
||||
let c = r#"{"channel":"candle","data":{"t":60000,"T":119999,"s":"ETH","i":"5m","o":1,"c":2.5,"h":3,"l":0.5,"v":10,"n":4}}"#;
|
||||
assert_eq!(
|
||||
parse(c).unwrap(),
|
||||
Incoming::Data(
|
||||
SubKey::Candle { coin: "ETH".into(), interval: "5m" },
|
||||
UpstreamEvent::Bar(Bar::new(60, 1.0, 3.0, 0.5, 2.5, 10.0))
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,209 +0,0 @@
|
||||
//! The gateway's HTTP side: the client websocket at `/ws`, and `/healthz`, `/readyz`,
|
||||
//! `/metrics` (Prometheus text) and `/stats` (JSON) for operators.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::fmt::Write as _;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::time::Instant;
|
||||
|
||||
use axum::Router;
|
||||
use axum::extract::{State, WebSocketUpgrade};
|
||||
use axum::http::{StatusCode, header};
|
||||
use axum::response::{IntoResponse, Response};
|
||||
use axum::routing::get;
|
||||
use serde::Serialize;
|
||||
use tokio::sync::watch;
|
||||
|
||||
use crate::metrics::{Metrics, Quantiles, TopicKind};
|
||||
use crate::provider::UpstreamStats;
|
||||
use crate::session::{self, SessionConfig};
|
||||
use crate::store::{Hub, TopicCounts};
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct AppState {
|
||||
pub hub: Arc<Hub>,
|
||||
pub sessions: Arc<SessionConfig>,
|
||||
pub max_clients: usize,
|
||||
/// Becomes true when the gateway shuts down; every session closes.
|
||||
closing: watch::Receiver<bool>,
|
||||
next_session: Arc<AtomicU64>,
|
||||
started: Instant,
|
||||
}
|
||||
|
||||
impl AppState {
|
||||
pub fn new(hub: Arc<Hub>, sessions: SessionConfig, max_clients: usize, closing: watch::Receiver<bool>) -> Self {
|
||||
Self {
|
||||
hub,
|
||||
sessions: Arc::new(sessions),
|
||||
max_clients,
|
||||
closing,
|
||||
next_session: Arc::new(AtomicU64::new(1)),
|
||||
started: Instant::now(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn router(state: AppState) -> Router {
|
||||
Router::new()
|
||||
.route("/ws", get(ws))
|
||||
.route("/healthz", get(|| async { "ok\n" }))
|
||||
.route("/readyz", get(readyz))
|
||||
.route("/metrics", get(metrics))
|
||||
.route("/stats", get(stats_json))
|
||||
.with_state(state)
|
||||
}
|
||||
|
||||
async fn ws(State(st): State<AppState>, upgrade: WebSocketUpgrade) -> Response {
|
||||
if st.hub.metrics.clients.load(Ordering::Relaxed) >= st.max_clients as i64 {
|
||||
return (StatusCode::SERVICE_UNAVAILABLE, "too many clients\n").into_response();
|
||||
}
|
||||
let id = st.next_session.fetch_add(1, Ordering::Relaxed);
|
||||
upgrade
|
||||
.max_message_size(64 * 1024)
|
||||
.on_upgrade(move |socket| session::run(socket, st.hub, st.sessions, id, st.closing))
|
||||
}
|
||||
|
||||
async fn readyz(State(st): State<AppState>) -> Response {
|
||||
let up = st.hub.provider().stats().connections_up;
|
||||
if up > 0 && st.hub.is_ready() {
|
||||
(StatusCode::OK, "ready\n").into_response()
|
||||
} else {
|
||||
(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
format!("not ready: {up} upstream connections up, {} mids\n", st.hub.mids.len()),
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct Stats {
|
||||
pub uptime_s: u64,
|
||||
pub provider: &'static str,
|
||||
pub attribution: &'static str,
|
||||
pub upstream: UpstreamStats,
|
||||
pub upstream_messages: u64,
|
||||
pub upstream_bytes: u64,
|
||||
pub upstream_reconnects: u64,
|
||||
pub upstream_rejected: u64,
|
||||
pub rest_requests: u64,
|
||||
pub rest_errors: u64,
|
||||
pub topics: TopicCounts,
|
||||
pub clients: i64,
|
||||
pub clients_total: u64,
|
||||
pub client_subscriptions: i64,
|
||||
pub frames_sent: u64,
|
||||
pub bytes_sent: u64,
|
||||
pub resyncs: u64,
|
||||
pub slow_consumer_closes: u64,
|
||||
/// Upstream frame received → client frame written, per topic, in microseconds.
|
||||
pub latency_us: BTreeMap<&'static str, Quantiles>,
|
||||
}
|
||||
|
||||
pub fn stats(st: &AppState) -> Stats {
|
||||
let m = &st.hub.metrics;
|
||||
let p = st.hub.provider();
|
||||
Stats {
|
||||
uptime_s: st.started.elapsed().as_secs(),
|
||||
provider: p.id().0,
|
||||
attribution: p.display_policy().attribution,
|
||||
upstream: p.stats(),
|
||||
upstream_messages: Metrics::get(&m.upstream_messages),
|
||||
upstream_bytes: Metrics::get(&m.upstream_bytes),
|
||||
upstream_reconnects: Metrics::get(&m.upstream_reconnects),
|
||||
upstream_rejected: Metrics::get(&m.upstream_rejected),
|
||||
rest_requests: Metrics::get(&m.rest_requests),
|
||||
rest_errors: Metrics::get(&m.rest_errors),
|
||||
topics: st.hub.topic_counts(),
|
||||
clients: m.clients.load(Ordering::Relaxed),
|
||||
clients_total: Metrics::get(&m.clients_total),
|
||||
client_subscriptions: m.client_subscriptions.load(Ordering::Relaxed),
|
||||
frames_sent: Metrics::get(&m.frames_sent),
|
||||
bytes_sent: Metrics::get(&m.bytes_sent),
|
||||
resyncs: Metrics::get(&m.resyncs),
|
||||
slow_consumer_closes: Metrics::get(&m.slow_consumer_closes),
|
||||
latency_us: TopicKind::ALL.iter().map(|k| (k.label(), m.latency_for(*k).quantiles())).collect(),
|
||||
}
|
||||
}
|
||||
|
||||
async fn stats_json(State(st): State<AppState>) -> Response {
|
||||
axum::Json(stats(&st)).into_response()
|
||||
}
|
||||
|
||||
async fn metrics(State(st): State<AppState>) -> Response {
|
||||
let s = stats(&st);
|
||||
let mut o = String::new();
|
||||
let mut put = |name: &str, kind: &str, help: &str, rows: &[(String, f64)]| {
|
||||
let _ = writeln!(o, "# HELP tr_{name} {help}\n# TYPE tr_{name} {kind}");
|
||||
for (labels, v) in rows {
|
||||
let _ = writeln!(o, "tr_{name}{labels} {v}");
|
||||
}
|
||||
};
|
||||
let one = |v: f64| vec![(String::new(), v)];
|
||||
let u = &s.upstream;
|
||||
put(
|
||||
"upstream_connections",
|
||||
"gauge",
|
||||
"Upstream websocket connections",
|
||||
&[
|
||||
("{state=\"open\"}".into(), u.connections_open as f64),
|
||||
("{state=\"up\"}".into(), u.connections_up as f64),
|
||||
("{state=\"budget\"}".into(), u.max_connections as f64),
|
||||
],
|
||||
);
|
||||
let mut subs: Vec<(String, f64)> =
|
||||
u.by_kind.iter().map(|(k, n)| (format!("{{kind=\"{k}\"}}"), *n as f64)).collect();
|
||||
subs.push(("{kind=\"budget\"}".into(), u.max_subscriptions as f64));
|
||||
put("upstream_subscriptions", "gauge", "Upstream subscriptions by kind", &subs);
|
||||
put(
|
||||
"upstream_messages_sent_last_min",
|
||||
"gauge",
|
||||
"Messages sent upstream in the last minute",
|
||||
&one(u.messages_sent_last_min as f64),
|
||||
);
|
||||
put(
|
||||
"rest_weight_last_min",
|
||||
"gauge",
|
||||
"Info endpoint weight spent in the last minute",
|
||||
&one(u.rest_weight_last_min as f64),
|
||||
);
|
||||
put("upstream_messages_total", "counter", "Upstream messages received", &one(s.upstream_messages as f64));
|
||||
put("upstream_bytes_total", "counter", "Upstream bytes received", &one(s.upstream_bytes as f64));
|
||||
put("upstream_reconnects_total", "counter", "Upstream reconnects", &one(s.upstream_reconnects as f64));
|
||||
put("upstream_rejected_total", "counter", "Upstream subscriptions refused", &one(s.upstream_rejected as f64));
|
||||
put("rest_requests_total", "counter", "Info endpoint requests", &one(s.rest_requests as f64));
|
||||
put("rest_errors_total", "counter", "Info endpoint errors", &one(s.rest_errors as f64));
|
||||
let t = &s.topics;
|
||||
put(
|
||||
"topics",
|
||||
"gauge",
|
||||
"Hot-store topics",
|
||||
&[
|
||||
("{kind=\"book\"}".into(), t.book as f64),
|
||||
("{kind=\"trades\"}".into(), t.trades as f64),
|
||||
("{kind=\"bars\"}".into(), t.bars as f64),
|
||||
("{kind=\"lingering\"}".into(), t.lingering as f64),
|
||||
("{kind=\"mids\"}".into(), t.mids as f64),
|
||||
],
|
||||
);
|
||||
put("clients", "gauge", "Connected clients", &one(s.clients as f64));
|
||||
put("client_subscriptions", "gauge", "Client subscriptions", &one(s.client_subscriptions as f64));
|
||||
put("frames_sent_total", "counter", "Frames sent to clients", &one(s.frames_sent as f64));
|
||||
put("bytes_sent_total", "counter", "Bytes sent to clients", &one(s.bytes_sent as f64));
|
||||
put("resyncs_total", "counter", "Resyncs sent to clients", &one(s.resyncs as f64));
|
||||
put(
|
||||
"slow_consumer_closes_total",
|
||||
"counter",
|
||||
"Clients closed for falling behind",
|
||||
&one(s.slow_consumer_closes as f64),
|
||||
);
|
||||
let mut lat = Vec::new();
|
||||
for (topic, q) in &s.latency_us {
|
||||
for (quantile, v) in [("0.5", q.p50_us), ("0.9", q.p90_us), ("0.99", q.p99_us)] {
|
||||
lat.push((format!("{{topic=\"{topic}\",quantile=\"{quantile}\"}}"), f64::from(v)));
|
||||
}
|
||||
}
|
||||
put("server_latency_us", "summary", "Upstream frame received to client frame written, microseconds", &lat);
|
||||
([(header::CONTENT_TYPE, "text/plain; version=0.0.4")], o).into_response()
|
||||
}
|
||||
@@ -1,92 +0,0 @@
|
||||
//! `tr-gateway`, the Terminal data server (`docs/architecture/server.md`).
|
||||
//!
|
||||
//! For now it carries Hyperliquid's public feed ([`hl::HlPublicProvider`]): one shared set of
|
||||
//! upstream connections, fanned out to every client over the protocol in `tr-proto`.
|
||||
|
||||
pub mod config;
|
||||
pub mod hl;
|
||||
pub mod http;
|
||||
pub mod limit;
|
||||
pub mod metrics;
|
||||
pub mod provider;
|
||||
pub mod session;
|
||||
pub mod store;
|
||||
|
||||
use std::net::SocketAddr;
|
||||
use std::sync::Arc;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use tokio::sync::{oneshot, watch};
|
||||
use tokio::task::JoinHandle;
|
||||
use tr_core::Bar;
|
||||
|
||||
pub use config::Config;
|
||||
|
||||
use crate::hl::HlPublicProvider;
|
||||
use crate::metrics::Metrics;
|
||||
use crate::store::Hub;
|
||||
|
||||
/// Keep at most `limit` bars opening before `end`.
|
||||
pub fn clip(mut bars: Vec<Bar>, end: Option<i64>, limit: usize) -> Vec<Bar> {
|
||||
if let Some(end) = end {
|
||||
bars.retain(|b| b.ts < end);
|
||||
}
|
||||
if bars.len() > limit {
|
||||
bars.drain(..bars.len() - limit);
|
||||
}
|
||||
bars
|
||||
}
|
||||
|
||||
/// A running gateway.
|
||||
pub struct Gateway {
|
||||
pub addr: SocketAddr,
|
||||
pub state: http::AppState,
|
||||
stop: Option<oneshot::Sender<()>>,
|
||||
closing: watch::Sender<bool>,
|
||||
task: JoinHandle<()>,
|
||||
}
|
||||
|
||||
impl Gateway {
|
||||
pub fn hub(&self) -> &Arc<Hub> {
|
||||
&self.state.hub
|
||||
}
|
||||
|
||||
/// Stop accepting, close every client session and upstream connection, and wait for the
|
||||
/// server to finish. Nothing keeps running afterwards.
|
||||
pub async fn shutdown(mut self) {
|
||||
self.closing.send_replace(true);
|
||||
if let Some(s) = self.stop.take() {
|
||||
let _ = s.send(());
|
||||
}
|
||||
self.state.hub.stop();
|
||||
let _ = (&mut self.task).await;
|
||||
}
|
||||
|
||||
/// Run until the process is asked to stop.
|
||||
pub async fn wait(self) {
|
||||
let _ = self.task.await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Bind, connect upstream and serve in the background.
|
||||
pub async fn start(cfg: Config) -> Result<Gateway> {
|
||||
let metrics = Arc::new(Metrics::default());
|
||||
let provider = Arc::new(HlPublicProvider::new(cfg.hl.clone(), metrics.clone())?);
|
||||
let hub = Hub::new(provider, cfg.hub.clone(), metrics);
|
||||
hub.start();
|
||||
let (closing, closing_rx) = watch::channel(false);
|
||||
let state = http::AppState::new(hub, cfg.session.clone(), cfg.max_clients, closing_rx);
|
||||
let listener = tokio::net::TcpListener::bind(cfg.bind).await.context("binding the listen address")?;
|
||||
let addr = listener.local_addr()?;
|
||||
let (stop, stopped) = oneshot::channel::<()>();
|
||||
let app = http::router(state.clone());
|
||||
let task = tokio::spawn(async move {
|
||||
let shutdown = async {
|
||||
let _ = stopped.await;
|
||||
};
|
||||
if let Err(e) = axum::serve(listener, app).with_graceful_shutdown(shutdown).await {
|
||||
tracing::error!("server: {e}");
|
||||
}
|
||||
});
|
||||
Ok(Gateway { addr, state, stop: Some(stop), closing, task })
|
||||
}
|
||||
@@ -1,112 +0,0 @@
|
||||
//! A sliding-window limiter: at most `budget` units in any `window`. The server uses one for
|
||||
//! each Hyperliquid per-IP limit it has to respect (REST weight, websocket messages sent, new
|
||||
//! websocket connections), always with a budget below Hyperliquid's own.
|
||||
|
||||
use std::collections::VecDeque;
|
||||
use std::sync::Mutex;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
pub struct WindowLimiter {
|
||||
window: Duration,
|
||||
budget: u32,
|
||||
log: Mutex<VecDeque<(Instant, u32)>>,
|
||||
}
|
||||
|
||||
impl WindowLimiter {
|
||||
pub fn new(budget: u32, window: Duration) -> Self {
|
||||
Self { window, budget: budget.max(1), log: Mutex::new(VecDeque::new()) }
|
||||
}
|
||||
|
||||
pub fn per_minute(budget: u32) -> Self {
|
||||
Self::new(budget, Duration::from_secs(60))
|
||||
}
|
||||
|
||||
pub fn budget(&self) -> u32 {
|
||||
self.budget
|
||||
}
|
||||
|
||||
/// Units spent in the current window.
|
||||
pub fn used(&self) -> u32 {
|
||||
let mut log = self.log.lock().expect("limiter lock");
|
||||
Self::expire(&mut log, Instant::now(), self.window);
|
||||
log.iter().map(|(_, n)| n).sum()
|
||||
}
|
||||
|
||||
fn expire(log: &mut VecDeque<(Instant, u32)>, now: Instant, window: Duration) {
|
||||
while log.front().is_some_and(|(t, _)| now.duration_since(*t) >= window) {
|
||||
log.pop_front();
|
||||
}
|
||||
}
|
||||
|
||||
/// Spend `n` now if the window has room, or say how long until it will. A request larger
|
||||
/// than the whole budget waits for an empty window, then goes through alone.
|
||||
pub fn try_acquire(&self, n: u32) -> Result<(), Duration> {
|
||||
let now = Instant::now();
|
||||
let mut log = self.log.lock().expect("limiter lock");
|
||||
Self::expire(&mut log, now, self.window);
|
||||
let mut used: u32 = log.iter().map(|(_, k)| k).sum();
|
||||
if used + n <= self.budget || log.is_empty() {
|
||||
log.push_back((now, n));
|
||||
return Ok(());
|
||||
}
|
||||
// Wait until enough of the oldest entries expire.
|
||||
for (t, k) in log.iter() {
|
||||
used -= k;
|
||||
if used + n <= self.budget {
|
||||
return Err((*t + self.window).saturating_duration_since(now) + Duration::from_millis(1));
|
||||
}
|
||||
}
|
||||
Err(self.window)
|
||||
}
|
||||
|
||||
/// Wait for room, then spend `n`. Dropping the future before it completes spends nothing.
|
||||
pub async fn acquire(&self, n: u32) {
|
||||
while let Err(wait) = self.try_acquire(n) {
|
||||
tokio::time::sleep(wait).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Record units already spent (a response that cost more than estimated, or a 429 that
|
||||
/// should keep everyone off the endpoint for a window).
|
||||
pub fn charge(&self, n: u32) {
|
||||
self.log.lock().expect("limiter lock").push_back((Instant::now(), n));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn budget_holds_within_a_window() {
|
||||
let l = WindowLimiter::new(100, Duration::from_millis(200));
|
||||
assert!(l.try_acquire(60).is_ok());
|
||||
assert!(l.try_acquire(40).is_ok());
|
||||
let wait = l.try_acquire(1).unwrap_err();
|
||||
assert!(wait > Duration::from_millis(150) && wait <= Duration::from_millis(201), "{wait:?}");
|
||||
assert_eq!(l.used(), 100);
|
||||
std::thread::sleep(Duration::from_millis(210));
|
||||
assert_eq!(l.used(), 0);
|
||||
assert!(l.try_acquire(100).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oversized_requests_go_alone() {
|
||||
let l = WindowLimiter::new(10, Duration::from_millis(100));
|
||||
assert!(l.try_acquire(25).is_ok(), "an empty window admits one oversized request");
|
||||
assert!(l.try_acquire(1).is_err());
|
||||
l.charge(5);
|
||||
assert_eq!(l.used(), 30);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn acquire_waits_for_room() {
|
||||
let l = WindowLimiter::new(2, Duration::from_millis(150));
|
||||
let t0 = Instant::now();
|
||||
l.acquire(1).await;
|
||||
l.acquire(1).await;
|
||||
assert!(t0.elapsed() < Duration::from_millis(50));
|
||||
l.acquire(1).await;
|
||||
assert!(t0.elapsed() >= Duration::from_millis(150), "the third waited for the window to roll");
|
||||
}
|
||||
}
|
||||
@@ -1,30 +0,0 @@
|
||||
//! `tr-gateway`: see `server/README.md`.
|
||||
|
||||
use anyhow::Result;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<()> {
|
||||
if std::env::args().skip(1).any(|a| a == "-h" || a == "--help") {
|
||||
print!("{}", tr_gateway::config::HELP);
|
||||
return Ok(());
|
||||
}
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter(tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| "info".into()))
|
||||
.init();
|
||||
// reqwest and tungstenite both use rustls; pin one crypto provider, like the desktop app.
|
||||
let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
|
||||
|
||||
let cfg = tr_gateway::Config::from_env()?;
|
||||
let gw = tr_gateway::start(cfg).await?;
|
||||
// The port only: logs may end up on screen, so they never carry addresses.
|
||||
tracing::info!("listening on port {} (websocket at /ws)", gw.addr.port());
|
||||
// Ctrl-C, or SIGTERM from a service manager.
|
||||
let mut term = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())?;
|
||||
tokio::select! {
|
||||
_ = tokio::signal::ctrl_c() => {}
|
||||
_ = term.recv() => {}
|
||||
}
|
||||
tracing::info!("shutting down");
|
||||
gw.shutdown().await;
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,141 +0,0 @@
|
||||
//! Counters and the server-added latency recorder behind `/metrics` and `/stats`.
|
||||
|
||||
use std::collections::VecDeque;
|
||||
use std::sync::Mutex;
|
||||
use std::sync::atomic::{AtomicI64, AtomicU64, Ordering};
|
||||
use std::time::Duration;
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct Metrics {
|
||||
pub upstream_messages: AtomicU64,
|
||||
pub upstream_bytes: AtomicU64,
|
||||
pub upstream_reconnects: AtomicU64,
|
||||
pub upstream_rejected: AtomicU64,
|
||||
pub clients: AtomicI64,
|
||||
pub clients_total: AtomicU64,
|
||||
pub client_subscriptions: AtomicI64,
|
||||
pub frames_sent: AtomicU64,
|
||||
pub bytes_sent: AtomicU64,
|
||||
pub resyncs: AtomicU64,
|
||||
pub slow_consumer_closes: AtomicU64,
|
||||
pub rest_requests: AtomicU64,
|
||||
pub rest_errors: AtomicU64,
|
||||
/// Upstream frame received → the client frame it caused written to the socket, per topic.
|
||||
pub latency_quotes: LatencyRecorder,
|
||||
pub latency_book: LatencyRecorder,
|
||||
pub latency_trades: LatencyRecorder,
|
||||
pub latency_bars: LatencyRecorder,
|
||||
}
|
||||
|
||||
/// Which latency histogram a client frame counts towards.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum TopicKind {
|
||||
Quotes,
|
||||
Book,
|
||||
Trades,
|
||||
Bars,
|
||||
}
|
||||
|
||||
impl TopicKind {
|
||||
pub const ALL: [TopicKind; 4] = [TopicKind::Quotes, TopicKind::Book, TopicKind::Trades, TopicKind::Bars];
|
||||
|
||||
pub fn label(self) -> &'static str {
|
||||
match self {
|
||||
TopicKind::Quotes => "quotes",
|
||||
TopicKind::Book => "book",
|
||||
TopicKind::Trades => "trades",
|
||||
TopicKind::Bars => "bars",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Metrics {
|
||||
pub fn latency_for(&self, kind: TopicKind) -> &LatencyRecorder {
|
||||
match kind {
|
||||
TopicKind::Quotes => &self.latency_quotes,
|
||||
TopicKind::Book => &self.latency_book,
|
||||
TopicKind::Trades => &self.latency_trades,
|
||||
TopicKind::Bars => &self.latency_bars,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn add(counter: &AtomicU64, n: u64) {
|
||||
counter.fetch_add(n, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn get(counter: &AtomicU64) -> u64 {
|
||||
counter.load(Ordering::Relaxed)
|
||||
}
|
||||
}
|
||||
|
||||
/// Keeps the most recent samples so quantiles reflect current behaviour.
|
||||
pub struct LatencyRecorder {
|
||||
samples: Mutex<VecDeque<u32>>,
|
||||
count: AtomicU64,
|
||||
cap: usize,
|
||||
}
|
||||
|
||||
impl Default for LatencyRecorder {
|
||||
fn default() -> Self {
|
||||
Self::with_capacity(50_000)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, serde::Serialize)]
|
||||
pub struct Quantiles {
|
||||
/// Samples in the window the quantiles come from.
|
||||
pub window: usize,
|
||||
/// Samples since start.
|
||||
pub total: u64,
|
||||
pub p50_us: u32,
|
||||
pub p90_us: u32,
|
||||
pub p99_us: u32,
|
||||
pub max_us: u32,
|
||||
}
|
||||
|
||||
impl LatencyRecorder {
|
||||
pub fn with_capacity(cap: usize) -> Self {
|
||||
Self { samples: Mutex::new(VecDeque::with_capacity(cap.min(4096))), count: AtomicU64::new(0), cap }
|
||||
}
|
||||
|
||||
pub fn record(&self, d: Duration) {
|
||||
let us = d.as_micros().min(u32::MAX as u128) as u32;
|
||||
let mut s = self.samples.lock().expect("latency lock");
|
||||
if s.len() == self.cap {
|
||||
s.pop_front();
|
||||
}
|
||||
s.push_back(us);
|
||||
self.count.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn quantiles(&self) -> Quantiles {
|
||||
let mut v: Vec<u32> = self.samples.lock().expect("latency lock").iter().copied().collect();
|
||||
v.sort_unstable();
|
||||
let q = |p: f64| if v.is_empty() { 0 } else { v[((v.len() - 1) as f64 * p).round() as usize] };
|
||||
Quantiles {
|
||||
window: v.len(),
|
||||
total: self.count.load(Ordering::Relaxed),
|
||||
p50_us: q(0.50),
|
||||
p90_us: q(0.90),
|
||||
p99_us: q(0.99),
|
||||
max_us: v.last().copied().unwrap_or(0),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn quantiles_over_recent_samples() {
|
||||
let r = LatencyRecorder::with_capacity(100);
|
||||
for us in 1..=200u64 {
|
||||
r.record(Duration::from_micros(us));
|
||||
}
|
||||
let q = r.quantiles();
|
||||
assert_eq!((q.window, q.total), (100, 200));
|
||||
assert_eq!((q.p50_us, q.p99_us, q.max_us), (151, 199, 200));
|
||||
assert_eq!(LatencyRecorder::default().quantiles().p99_us, 0);
|
||||
}
|
||||
}
|
||||
@@ -1,211 +0,0 @@
|
||||
//! The server-side source of market data (`docs/architecture/server.md` §3.1). One provider
|
||||
//! per upstream. The gateway only talks to this trait, so a provider can be replaced (the
|
||||
//! public-feed `HlPublicProvider` today, a node-backed `HlNodeProvider` later) without any
|
||||
//! client or protocol change.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::fmt;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use anyhow::Result;
|
||||
use async_trait::async_trait;
|
||||
use tr_core::{Bar, Timeframe};
|
||||
use tr_proto::{InstrumentKind, Level, Trade};
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub struct ProviderId(pub &'static str);
|
||||
|
||||
/// A provider's own name for an instrument. For Hyperliquid, the coin: `BTC`, `xyz:TSLA`,
|
||||
/// `@107`. For [`Channel::Mids`] it names a venue instead (a perp dex; `""` is the core dex).
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
|
||||
pub struct InstrumentId(pub String);
|
||||
|
||||
impl InstrumentId {
|
||||
pub fn new(id: impl Into<String>) -> Self {
|
||||
Self(id.into())
|
||||
}
|
||||
|
||||
pub fn as_str(&self) -> &str {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for InstrumentId {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(&self.0)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum Channel {
|
||||
/// Every mid price on one venue.
|
||||
Mids,
|
||||
Book,
|
||||
Trades,
|
||||
Bars(Timeframe),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum ChannelKind {
|
||||
Mids,
|
||||
Book,
|
||||
Trades,
|
||||
Bars,
|
||||
}
|
||||
|
||||
/// What a provider can serve: channels × instrument kinds.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Capabilities {
|
||||
pub channels: Vec<ChannelKind>,
|
||||
pub kinds: Vec<InstrumentKind>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Entitlement {
|
||||
Staff,
|
||||
}
|
||||
|
||||
/// Display rights for a provider's data. The gateway enforces these, never the client.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct DisplayPolicy {
|
||||
/// May the general public (any signed-in or anonymous user) see it?
|
||||
pub public_display: bool,
|
||||
/// If not public: the entitlement that unlocks it.
|
||||
pub entitlement: Option<Entitlement>,
|
||||
/// Required delay before showing it to users without the entitlement.
|
||||
pub delay: Option<Duration>,
|
||||
/// Attribution the UI must show.
|
||||
pub attribution: &'static str,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct InstrumentDef {
|
||||
pub id: InstrumentId,
|
||||
/// A perp dex (`""` is the core dex) or `spot`.
|
||||
pub venue: String,
|
||||
pub kind: InstrumentKind,
|
||||
/// The price 24 hours earlier, for change figures, when the provider has one.
|
||||
pub prev_day: Option<f64>,
|
||||
}
|
||||
|
||||
/// One upstream event and when it arrived.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Upstream {
|
||||
/// When the frame carrying it arrived, for the server-added latency histogram.
|
||||
pub recv: Instant,
|
||||
/// The same moment as Unix milliseconds, sent to clients as the time the server saw it.
|
||||
pub recv_ms: i64,
|
||||
pub event: UpstreamEvent,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum UpstreamEvent {
|
||||
/// `(instrument, mid)` pairs that changed. The first message after (re)subscribing has all.
|
||||
Mids(Vec<(String, f64)>),
|
||||
Book {
|
||||
bids: Vec<Level>,
|
||||
asks: Vec<Level>,
|
||||
time_ms: i64,
|
||||
},
|
||||
/// New trades, oldest first. Right after (re)subscribing this holds recent history.
|
||||
Trades(Vec<Trade>),
|
||||
/// The latest state of the bar that `Bar::ts` opens.
|
||||
Bar(Bar),
|
||||
/// The upstream connection dropped. Data is stale until [`UpstreamEvent::Reconnected`].
|
||||
Disconnected,
|
||||
/// The upstream reconnected and resubscribed. The next data starts fresh, and anything sent
|
||||
/// in between was missed.
|
||||
Reconnected,
|
||||
/// The upstream refused this subscription; it gets no more data.
|
||||
Rejected(String),
|
||||
}
|
||||
|
||||
/// Where a subscription's events go. Called on the provider's I/O task, so it must not block.
|
||||
#[derive(Clone)]
|
||||
pub struct UpstreamSink(Arc<dyn Fn(Upstream) + Send + Sync>);
|
||||
|
||||
impl UpstreamSink {
|
||||
pub fn new(f: impl Fn(Upstream) + Send + Sync + 'static) -> Self {
|
||||
Self(Arc::new(f))
|
||||
}
|
||||
|
||||
pub fn send(&self, ev: Upstream) {
|
||||
(self.0)(ev)
|
||||
}
|
||||
}
|
||||
|
||||
/// Keeps an upstream subscription alive; dropping it unsubscribes.
|
||||
pub struct UpstreamHandle(#[allow(dead_code)] Box<dyn Send + Sync>);
|
||||
|
||||
impl UpstreamHandle {
|
||||
pub fn new(guard: impl Send + Sync + 'static) -> Self {
|
||||
Self(Box::new(guard))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum SubscribeError {
|
||||
/// At the provider's upstream subscription or connection budget.
|
||||
Capacity(String),
|
||||
UnknownInstrument(String),
|
||||
Unsupported(String),
|
||||
}
|
||||
|
||||
impl fmt::Display for SubscribeError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
SubscribeError::Capacity(m) => write!(f, "upstream budget: {m}"),
|
||||
SubscribeError::UnknownInstrument(m) => write!(f, "unknown instrument {m}"),
|
||||
SubscribeError::Unsupported(m) => write!(f, "unsupported: {m}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for SubscribeError {}
|
||||
|
||||
/// A provider's upstream usage against its budgets, for `/stats` and `/metrics`.
|
||||
#[derive(Debug, Clone, Default, PartialEq, serde::Serialize)]
|
||||
pub struct UpstreamStats {
|
||||
pub connections_open: usize,
|
||||
pub connections_up: usize,
|
||||
pub max_connections: usize,
|
||||
pub subscriptions: usize,
|
||||
pub max_subscriptions: usize,
|
||||
pub by_kind: BTreeMap<&'static str, usize>,
|
||||
pub per_connection: Vec<usize>,
|
||||
/// Every upstream subscription, for example `l2Book:BTC` or `candle:xyz:TSLA:1m`.
|
||||
pub keys: Vec<String>,
|
||||
pub messages_sent_last_min: u32,
|
||||
pub messages_per_min: u32,
|
||||
pub connects_last_min: u32,
|
||||
pub rest_weight_last_min: u32,
|
||||
pub rest_weight_per_min: u32,
|
||||
}
|
||||
|
||||
/// Server-side source of market data. The gateway asks it what it serves, fetches history from
|
||||
/// it and opens streams on it; everything else (fan-out, sequencing, backpressure) is the
|
||||
/// gateway's job.
|
||||
#[async_trait]
|
||||
pub trait Provider: Send + Sync + 'static {
|
||||
fn id(&self) -> ProviderId;
|
||||
fn capabilities(&self) -> Capabilities;
|
||||
fn display_policy(&self) -> DisplayPolicy;
|
||||
fn stats(&self) -> UpstreamStats {
|
||||
UpstreamStats::default()
|
||||
}
|
||||
/// Close every upstream connection for good (gateway shutdown).
|
||||
fn close(&self) {}
|
||||
/// Instruments listed right now. Called periodically; delistings show up here.
|
||||
async fn instruments(&self) -> Result<Vec<InstrumentDef>>;
|
||||
/// Up to `limit` bars before `end` (Unix seconds; `None` = now), oldest first: the same
|
||||
/// contract as `MarketData::history`.
|
||||
async fn history(&self, inst: &InstrumentId, tf: Timeframe, end: Option<i64>, limit: usize) -> Result<Vec<Bar>>;
|
||||
/// Start an upstream stream. Events go to `sink` until the handle is dropped.
|
||||
async fn subscribe(
|
||||
&self,
|
||||
ch: Channel,
|
||||
inst: &InstrumentId,
|
||||
sink: UpstreamSink,
|
||||
) -> Result<UpstreamHandle, SubscribeError>;
|
||||
}
|
||||
@@ -1,970 +0,0 @@
|
||||
//! One client connection (§3.5, §5.8): `Hello`/`Welcome`, then one forwarding task per
|
||||
//! subscription feeding a bounded outbound queue that a writer task drains to the socket.
|
||||
//!
|
||||
//! - Every subscription has its own `seq`: a `Snapshot` sets it and each `Delta` adds one.
|
||||
//! - State topics (quotes, book) conflate: a slow reader gets the latest value, never a backlog.
|
||||
//! - Event topics (trades, bars) read a bounded ring. A reader that falls behind it gets
|
||||
//! `Resync` and a fresh `Snapshot`; a `Forming` bar may be dropped for a newer one with the
|
||||
//! same `ts`, a `Closed` bar never is.
|
||||
//! - If the queue stays full for `slow_consumer`, the connection closes.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::Ordering;
|
||||
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
||||
|
||||
use axum::extract::ws::{CloseFrame, Message, WebSocket};
|
||||
use futures::{SinkExt, StreamExt};
|
||||
use tokio::sync::{Notify, broadcast, mpsc, watch};
|
||||
use tokio::task::JoinHandle;
|
||||
use tr_proto::{
|
||||
ApiError, BarDelta, Book, ClientMsg, Encoding, ErrorCode, Frame, Level, Limits, MarketKey, PROTOCOL_VERSION,
|
||||
PriceBasis, Quote, ResyncReason, Rpc, RpcResult, ServerMsg, SessionId, SubId, TopicData, TopicDelta, TopicReq,
|
||||
TopicStatus, UnavailableReason,
|
||||
};
|
||||
|
||||
use crate::metrics::{Metrics, TopicKind as Kind};
|
||||
use crate::store::{
|
||||
BarsTopic, BookTopic, Event, EventKind, Hub, Mid, Topic, TopicKey, TradesTopic, VenueState, merge_bars,
|
||||
};
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct SessionConfig {
|
||||
pub max_subscriptions: usize,
|
||||
pub max_quote_keys: usize,
|
||||
pub max_book_depth: u8,
|
||||
pub max_history: u32,
|
||||
pub max_quote_hz: u8,
|
||||
/// One-shot requests a connection may have running at once.
|
||||
pub max_requests_in_flight: usize,
|
||||
/// A request still running after this gets an error reply.
|
||||
pub request_timeout: Duration,
|
||||
/// Client messages per 10 s. Subscribes past this would churn the shared upstream.
|
||||
pub max_messages_per_10s: u32,
|
||||
/// Outbound frames a client may have queued.
|
||||
pub queue: usize,
|
||||
pub hello_timeout: Duration,
|
||||
pub slow_consumer: Duration,
|
||||
/// Websocket pings keep idle connections open through proxies and TLS edges.
|
||||
pub ping_every: Duration,
|
||||
/// Close a client we've heard nothing from (not even a pong) for this long.
|
||||
pub idle_timeout: Duration,
|
||||
/// How long a request waits for the upstream after a restart before it fails. Subscriptions
|
||||
/// wait as long as it takes.
|
||||
pub ready_timeout: Duration,
|
||||
}
|
||||
|
||||
impl Default for SessionConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
max_subscriptions: 64,
|
||||
max_quote_keys: 512,
|
||||
max_book_depth: 20,
|
||||
max_history: 5000,
|
||||
max_quote_hz: 10,
|
||||
max_requests_in_flight: 8,
|
||||
request_timeout: Duration::from_secs(30),
|
||||
max_messages_per_10s: 200,
|
||||
queue: 256,
|
||||
hello_timeout: Duration::from_secs(10),
|
||||
slow_consumer: Duration::from_secs(10),
|
||||
ping_every: Duration::from_secs(20),
|
||||
idle_timeout: Duration::from_secs(90),
|
||||
ready_timeout: Duration::from_secs(10),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl SessionConfig {
|
||||
pub fn limits(&self) -> Limits {
|
||||
Limits {
|
||||
max_subscriptions: self.max_subscriptions as u32,
|
||||
max_quote_keys: self.max_quote_keys as u32,
|
||||
max_book_depth: self.max_book_depth,
|
||||
max_history: self.max_history,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn now_ms() -> i64 {
|
||||
SystemTime::now().duration_since(UNIX_EPOCH).map(|d| d.as_millis() as i64).unwrap_or(0)
|
||||
}
|
||||
|
||||
struct Out {
|
||||
msg: Message,
|
||||
/// When the upstream data in this frame arrived, if it carries any.
|
||||
recv: Option<(Instant, Kind)>,
|
||||
}
|
||||
|
||||
fn to_message(f: Frame) -> Message {
|
||||
match f {
|
||||
Frame::Binary(b) => Message::Binary(b.into()),
|
||||
Frame::Text(t) => Message::Text(t.into()),
|
||||
}
|
||||
}
|
||||
|
||||
/// What every forwarding task of one session shares.
|
||||
#[derive(Clone)]
|
||||
struct Link {
|
||||
enc: Encoding,
|
||||
out: mpsc::Sender<Out>,
|
||||
kill: Arc<watch::Sender<bool>>,
|
||||
slow: Duration,
|
||||
metrics: Arc<Metrics>,
|
||||
}
|
||||
|
||||
impl Link {
|
||||
/// Queue a message. False once the session is gone or too slow to keep.
|
||||
async fn send(&self, msg: &ServerMsg, recv: Option<(Instant, Kind)>) -> bool {
|
||||
let frame = match tr_proto::encode(msg, self.enc) {
|
||||
Ok(f) => to_message(f),
|
||||
Err(e) => {
|
||||
tracing::error!("encode: {e}");
|
||||
return true;
|
||||
}
|
||||
};
|
||||
match tokio::time::timeout(self.slow, self.out.send(Out { msg: frame, recv })).await {
|
||||
Ok(Ok(())) => true,
|
||||
Ok(Err(_)) => false,
|
||||
Err(_) => {
|
||||
self.kill.send_replace(true);
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One subscription's forwarding state.
|
||||
struct Fwd {
|
||||
id: SubId,
|
||||
kind: Kind,
|
||||
link: Link,
|
||||
seq: u64,
|
||||
resync: Arc<Notify>,
|
||||
}
|
||||
|
||||
impl Fwd {
|
||||
/// Snapshots carry state of any age, so they don't count towards the latency histograms.
|
||||
async fn snapshot(&mut self, data: TopicData) -> bool {
|
||||
self.seq += 1;
|
||||
let m = ServerMsg::Snapshot { id: self.id, seq: self.seq, data };
|
||||
self.link.send(&m, None).await
|
||||
}
|
||||
|
||||
async fn delta(&mut self, data: TopicDelta, recv: Instant) -> bool {
|
||||
self.seq += 1;
|
||||
let m = ServerMsg::Delta { id: self.id, seq: self.seq, data };
|
||||
self.link.send(&m, Some((recv, self.kind))).await
|
||||
}
|
||||
|
||||
async fn status(&self, status: TopicStatus) -> bool {
|
||||
self.link.send(&ServerMsg::Status { id: self.id, status }, None).await
|
||||
}
|
||||
|
||||
async fn resync(&self, reason: ResyncReason) -> bool {
|
||||
Metrics::add(&self.link.metrics.resyncs, 1);
|
||||
self.link.send(&ServerMsg::Resync { id: self.id, reason }, None).await
|
||||
}
|
||||
|
||||
async fn unavailable(&self, reason: UnavailableReason, detail: String) -> bool {
|
||||
self.status(TopicStatus::Unavailable { reason, detail }).await
|
||||
}
|
||||
}
|
||||
|
||||
struct SubCtl {
|
||||
task: JoinHandle<()>,
|
||||
resync: Arc<Notify>,
|
||||
}
|
||||
|
||||
/// Serve one client until it leaves, misbehaves or can't keep up.
|
||||
pub async fn run(
|
||||
socket: WebSocket,
|
||||
hub: Arc<Hub>,
|
||||
cfg: Arc<SessionConfig>,
|
||||
session: u64,
|
||||
closing: watch::Receiver<bool>,
|
||||
) {
|
||||
let metrics = hub.metrics.clone();
|
||||
metrics.clients.fetch_add(1, Ordering::Relaxed);
|
||||
Metrics::add(&metrics.clients_total, 1);
|
||||
serve(socket, hub, cfg, session, closing).await;
|
||||
metrics.clients.fetch_sub(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
fn decode(msg: &Message) -> Option<Result<ClientMsg, tr_proto::CodecError>> {
|
||||
match msg {
|
||||
Message::Binary(b) => Some(tr_proto::decode_binary(b)),
|
||||
Message::Text(t) => Some(tr_proto::decode_text(t.as_str())),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
async fn reject(mut socket: WebSocket, enc: Encoding, code: ErrorCode, message: String) {
|
||||
if let Ok(f) = tr_proto::encode(&ServerMsg::Error { code, message: message.clone() }, enc) {
|
||||
let _ = socket.send(to_message(f)).await;
|
||||
}
|
||||
let reason = message.chars().take(100).collect::<String>();
|
||||
let _ = socket.send(Message::Close(Some(CloseFrame { code: 1008, reason: reason.into() }))).await;
|
||||
}
|
||||
|
||||
async fn serve(
|
||||
mut socket: WebSocket,
|
||||
hub: Arc<Hub>,
|
||||
cfg: Arc<SessionConfig>,
|
||||
session: u64,
|
||||
mut closing: watch::Receiver<bool>,
|
||||
) {
|
||||
if *closing.borrow_and_update() {
|
||||
return;
|
||||
}
|
||||
// Hello first, in either encoding.
|
||||
let first = tokio::time::timeout(cfg.hello_timeout, async {
|
||||
while let Some(Ok(m)) = socket.recv().await {
|
||||
if let Some(r) = decode(&m) {
|
||||
return Some((r, matches!(m, Message::Text(_))));
|
||||
}
|
||||
}
|
||||
None
|
||||
})
|
||||
.await;
|
||||
let (enc, client) = match first {
|
||||
Ok(Some((Ok(ClientMsg::Hello { version, encoding, client, .. }), _))) if version == PROTOCOL_VERSION => {
|
||||
(encoding, client)
|
||||
}
|
||||
Ok(Some((Ok(ClientMsg::Hello { version, encoding, .. }), _))) => {
|
||||
let msg = format!("protocol {version} is not supported; this server speaks {PROTOCOL_VERSION}");
|
||||
return reject(socket, encoding, ErrorCode::UnsupportedVersion, msg).await;
|
||||
}
|
||||
Ok(Some((_, text))) => {
|
||||
let enc = if text { Encoding::Json } else { Encoding::Postcard };
|
||||
return reject(socket, enc, ErrorCode::HelloRequired, "send Hello first".into()).await;
|
||||
}
|
||||
_ => return,
|
||||
};
|
||||
tracing::debug!("session {session}: {} {} ({enc:?})", client.name, client.version);
|
||||
let welcome = ServerMsg::Welcome {
|
||||
version: PROTOCOL_VERSION,
|
||||
session: SessionId(session),
|
||||
user: None,
|
||||
server_time_ms: now_ms(),
|
||||
limits: cfg.limits(),
|
||||
};
|
||||
let Ok(f) = tr_proto::encode(&welcome, enc) else { return };
|
||||
if socket.send(to_message(f)).await.is_err() {
|
||||
return;
|
||||
}
|
||||
|
||||
let (ws_tx, mut ws_rx) = socket.split();
|
||||
let (out_tx, out_rx) = mpsc::channel::<Out>(cfg.queue.max(1));
|
||||
let (kill, mut killed) = watch::channel(false);
|
||||
let link = Link { enc, out: out_tx, kill: Arc::new(kill), slow: cfg.slow_consumer, metrics: hub.metrics.clone() };
|
||||
let mut writer = tokio::spawn(write(ws_tx, out_rx, hub.metrics.clone(), cfg.ping_every));
|
||||
let mut subs: HashMap<SubId, SubCtl> = HashMap::new();
|
||||
let mut idle = tokio::time::interval(cfg.idle_timeout / 3);
|
||||
let mut last_heard = Instant::now();
|
||||
let mut writer_done = false;
|
||||
let mut rpcs = tokio::task::JoinSet::new();
|
||||
let rate = crate::limit::WindowLimiter::new(cfg.max_messages_per_10s, Duration::from_secs(10));
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
msg = ws_rx.next() => {
|
||||
let Some(Ok(msg)) = msg else { break };
|
||||
last_heard = Instant::now();
|
||||
let Some(decoded) = decode(&msg) else {
|
||||
if matches!(msg, Message::Close(_)) {
|
||||
break;
|
||||
}
|
||||
continue;
|
||||
};
|
||||
if rate.try_acquire(1).is_err() {
|
||||
let m = format!("more than {} messages in 10 s; this one was ignored", cfg.max_messages_per_10s);
|
||||
link.send(&ServerMsg::Error { code: ErrorCode::RateLimited, message: m }, None).await;
|
||||
continue;
|
||||
}
|
||||
match decoded {
|
||||
Ok(m) => handle(m, &hub, &cfg, &link, &mut subs, &mut rpcs).await,
|
||||
Err(e) => {
|
||||
let m = ServerMsg::Error { code: ErrorCode::BadMessage, message: e.to_string() };
|
||||
link.send(&m, None).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
_ = killed.changed() => {
|
||||
Metrics::add(&hub.metrics.slow_consumer_closes, 1);
|
||||
let m = ServerMsg::Error { code: ErrorCode::SlowConsumer, message: "too slow to keep up; reconnect".into() };
|
||||
// The queue is full; the close goes out on a fresh channel slot or not at all.
|
||||
let _ = link.out.try_send(Out { msg: to_message(tr_proto::encode(&m, enc).expect("encodes")), recv: None });
|
||||
break;
|
||||
}
|
||||
_ = idle.tick() => if last_heard.elapsed() > cfg.idle_timeout {
|
||||
tracing::debug!("session {session}: idle; closing");
|
||||
break;
|
||||
},
|
||||
_ = &mut writer => {
|
||||
writer_done = true;
|
||||
break;
|
||||
}
|
||||
// Only ever set to true, or dropped with the gateway.
|
||||
_ = closing.changed() => break,
|
||||
}
|
||||
}
|
||||
for (_, s) in subs.drain() {
|
||||
s.task.abort();
|
||||
}
|
||||
rpcs.abort_all();
|
||||
drop(link);
|
||||
// Let the writer flush what's queued (a final error, the close frame), briefly.
|
||||
if !writer_done && tokio::time::timeout(Duration::from_secs(1), &mut writer).await.is_err() {
|
||||
writer.abort();
|
||||
}
|
||||
}
|
||||
|
||||
async fn write(
|
||||
mut ws: futures::stream::SplitSink<WebSocket, Message>,
|
||||
mut rx: mpsc::Receiver<Out>,
|
||||
metrics: Arc<Metrics>,
|
||||
ping_every: Duration,
|
||||
) {
|
||||
let mut ping = tokio::time::interval(ping_every);
|
||||
ping.tick().await;
|
||||
loop {
|
||||
tokio::select! {
|
||||
out = rx.recv() => {
|
||||
let Some(out) = out else { break };
|
||||
let len = match &out.msg {
|
||||
Message::Binary(b) => b.len(),
|
||||
Message::Text(t) => t.len(),
|
||||
_ => 0,
|
||||
};
|
||||
if ws.send(out.msg).await.is_err() {
|
||||
break;
|
||||
}
|
||||
Metrics::add(&metrics.frames_sent, 1);
|
||||
Metrics::add(&metrics.bytes_sent, len as u64);
|
||||
if let Some((recv, kind)) = out.recv {
|
||||
metrics.latency_for(kind).record(recv.elapsed());
|
||||
}
|
||||
}
|
||||
_ = ping.tick() => if ws.send(Message::Ping(Default::default())).await.is_err() {
|
||||
break;
|
||||
},
|
||||
}
|
||||
}
|
||||
let _ = ws.close().await;
|
||||
}
|
||||
|
||||
async fn handle(
|
||||
msg: ClientMsg,
|
||||
hub: &Arc<Hub>,
|
||||
cfg: &Arc<SessionConfig>,
|
||||
link: &Link,
|
||||
subs: &mut HashMap<SubId, SubCtl>,
|
||||
rpcs: &mut tokio::task::JoinSet<()>,
|
||||
) {
|
||||
let error = |code, message: String| ServerMsg::Error { code, message };
|
||||
match msg {
|
||||
ClientMsg::Hello { .. } => {
|
||||
link.send(&error(ErrorCode::BadMessage, "Hello was already sent".into()), None).await;
|
||||
}
|
||||
ClientMsg::Subscribe { id, topic } => {
|
||||
if subs.contains_key(&id) {
|
||||
link.send(&error(ErrorCode::DuplicateId, format!("subscription {id} exists")), None).await;
|
||||
return;
|
||||
}
|
||||
if subs.len() >= cfg.max_subscriptions {
|
||||
let m = format!("at most {} subscriptions per connection", cfg.max_subscriptions);
|
||||
link.send(&error(ErrorCode::TooManySubscriptions, m), None).await;
|
||||
return;
|
||||
}
|
||||
let resync = Arc::new(Notify::new());
|
||||
let kind = match topic {
|
||||
TopicReq::Quotes { .. } => Kind::Quotes,
|
||||
TopicReq::Book { .. } => Kind::Book,
|
||||
TopicReq::Trades { .. } => Kind::Trades,
|
||||
TopicReq::Bars { .. } => Kind::Bars,
|
||||
};
|
||||
let fwd = Fwd { id, kind, link: link.clone(), seq: 0, resync: resync.clone() };
|
||||
let (hub, cfg) = (hub.clone(), cfg.clone());
|
||||
let metrics = hub.metrics.clone();
|
||||
let task = tokio::spawn(async move {
|
||||
metrics.client_subscriptions.fetch_add(1, Ordering::Relaxed);
|
||||
let _count = Decrement(metrics);
|
||||
forward(fwd, hub, cfg, topic).await;
|
||||
});
|
||||
subs.insert(id, SubCtl { task, resync });
|
||||
}
|
||||
ClientMsg::Unsubscribe { id } => match subs.remove(&id) {
|
||||
Some(s) => s.task.abort(),
|
||||
None => {
|
||||
link.send(&error(ErrorCode::UnknownId, format!("no subscription {id}")), None).await;
|
||||
}
|
||||
},
|
||||
ClientMsg::Resync { id } => match subs.get(&id) {
|
||||
Some(s) => s.resync.notify_one(),
|
||||
None => {
|
||||
link.send(&error(ErrorCode::UnknownId, format!("no subscription {id}")), None).await;
|
||||
}
|
||||
},
|
||||
ClientMsg::Request { id, req } => {
|
||||
while rpcs.try_join_next().is_some() {}
|
||||
if rpcs.len() >= cfg.max_requests_in_flight {
|
||||
let message = format!("at most {} requests in flight", cfg.max_requests_in_flight);
|
||||
let result = Err(ApiError { code: ErrorCode::RateLimited, message });
|
||||
link.send(&ServerMsg::Reply { id, result }, None).await;
|
||||
return;
|
||||
}
|
||||
let (hub, cfg, link) = (hub.clone(), cfg.clone(), link.clone());
|
||||
// The set aborts whatever is still running when the session ends.
|
||||
rpcs.spawn(async move {
|
||||
let result = match tokio::time::timeout(cfg.request_timeout, rpc(&hub, &cfg, req)).await {
|
||||
Ok(r) => r,
|
||||
Err(_) => Err(ApiError { code: ErrorCode::Upstream, message: "timed out".into() }),
|
||||
};
|
||||
link.send(&ServerMsg::Reply { id, result }, None).await;
|
||||
});
|
||||
}
|
||||
ClientMsg::Ping { t } => {
|
||||
link.send(&ServerMsg::Pong { t, server_time_ms: now_ms() }, None).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Counts a subscription for as long as its task lives, however it ends.
|
||||
struct Decrement(Arc<Metrics>);
|
||||
|
||||
impl Drop for Decrement {
|
||||
fn drop(&mut self) {
|
||||
self.0.client_subscriptions.fetch_sub(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
async fn rpc(hub: &Arc<Hub>, cfg: &SessionConfig, req: Rpc) -> Result<RpcResult, ApiError> {
|
||||
let err = |code, message: String| ApiError { code, message };
|
||||
// Not `UnknownMarket`, since clients stop asking about a market the server called unknown.
|
||||
if !hub.ready(cfg.ready_timeout).await {
|
||||
return Err(err(ErrorCode::Upstream, "the server is starting; no upstream data yet".into()));
|
||||
}
|
||||
match req {
|
||||
Rpc::History { key, tf, end, limit } => {
|
||||
let inst = hub.resolve(&key).ok_or_else(|| match hub.unknown(&key) {
|
||||
(UnavailableReason::Upstream, m) => err(ErrorCode::Upstream, m),
|
||||
(_, m) => err(ErrorCode::UnknownMarket, m),
|
||||
})?;
|
||||
let limit = (limit.clamp(1, cfg.max_history)) as usize;
|
||||
// With a live stream for this market, cached history must be newer than the stream's
|
||||
// last (re)start, or bars that closed while it was down could be missing or partial.
|
||||
let live = hub.bars_topic(&inst, tf).filter(|_| end.is_none());
|
||||
let fresh_after = live.as_ref().map(|t| t.overlay().live_since.unwrap_or_else(Instant::now));
|
||||
let bars = hub
|
||||
.history(&inst, tf, end, limit, fresh_after)
|
||||
.await
|
||||
.map_err(|e| err(ErrorCode::Upstream, format!("{e:#}")))?;
|
||||
// Recent history also gets whatever the live stream has seen since.
|
||||
let bars = match live {
|
||||
Some(t) => merge_bars(bars.to_vec(), &t.overlay().bars, limit),
|
||||
None => bars.to_vec(),
|
||||
};
|
||||
Ok(RpcResult::History(bars))
|
||||
}
|
||||
Rpc::Markets => Ok(RpcResult::Markets(hub.markets())),
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether the hub knows the market (for quotes, any of them).
|
||||
fn known(hub: &Hub, topic: &TopicReq) -> bool {
|
||||
match topic {
|
||||
TopicReq::Quotes { keys, .. } => keys.iter().any(|k| hub.resolve(k).is_some()),
|
||||
TopicReq::Book { key, .. } | TopicReq::Trades { key } | TopicReq::Bars { key, .. } => {
|
||||
hub.resolve(key).is_some()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Right after a restart the gateway doesn't know yet which markets exist, and one venue can
|
||||
/// be later than the rest. A subscription waits for its market, marked stale, rather than end.
|
||||
/// False once the session is gone.
|
||||
async fn wait_until_known(f: &Fwd, hub: &Hub, topic: &TopicReq) -> bool {
|
||||
let mut version = hub.mids.subscribe();
|
||||
let mut waited = false;
|
||||
loop {
|
||||
version.borrow_and_update();
|
||||
if hub.is_ready() && (!hub.loading() || known(hub, topic)) {
|
||||
return !waited || f.status(TopicStatus::Live).await;
|
||||
}
|
||||
if !waited && !f.status(TopicStatus::Stale { since_ms: now_ms() }).await {
|
||||
return false;
|
||||
}
|
||||
waited = true;
|
||||
// Mids, a venue's first report and readiness all bump the version.
|
||||
if version.changed().await.is_err() {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn forward(f: Fwd, hub: Arc<Hub>, cfg: Arc<SessionConfig>, topic: TopicReq) {
|
||||
if let TopicReq::Quotes { keys, .. } = &topic
|
||||
&& keys.len() > cfg.max_quote_keys
|
||||
{
|
||||
let m = format!("at most {} keys per quotes subscription", cfg.max_quote_keys);
|
||||
f.unavailable(UnavailableReason::Unsupported, m).await;
|
||||
return;
|
||||
}
|
||||
if !wait_until_known(&f, &hub, &topic).await {
|
||||
return;
|
||||
}
|
||||
let unknown = |k: &MarketKey| hub.unknown(k);
|
||||
let (tkey, extra) = match &topic {
|
||||
TopicReq::Quotes { keys, max_hz } => {
|
||||
let hz = (*max_hz).clamp(1, cfg.max_quote_hz);
|
||||
return quotes(f, hub, keys.clone(), hz).await;
|
||||
}
|
||||
TopicReq::Book { key, .. } => (hub.resolve(key).map(TopicKey::Book).ok_or_else(|| unknown(key)), 0),
|
||||
TopicReq::Trades { key } => (hub.resolve(key).map(TopicKey::Trades).ok_or_else(|| unknown(key)), 0),
|
||||
TopicReq::Bars { key, tf, history } => (
|
||||
hub.resolve(key).map(|i| TopicKey::Bars(i, *tf)).ok_or_else(|| unknown(key)),
|
||||
(*history).min(cfg.max_history),
|
||||
),
|
||||
};
|
||||
let held = match tkey {
|
||||
Ok(k) => hub.acquire(k).await,
|
||||
Err(e) => Err(e),
|
||||
};
|
||||
let held = match held {
|
||||
Ok(h) => h,
|
||||
Err((reason, detail)) => {
|
||||
f.unavailable(reason, detail).await;
|
||||
return;
|
||||
}
|
||||
};
|
||||
match (&held.topic, topic) {
|
||||
(Topic::Book(t), TopicReq::Book { depth, group, .. }) => {
|
||||
book(f, t.clone(), depth.clamp(1, cfg.max_book_depth) as usize, group).await
|
||||
}
|
||||
(Topic::Trades(t), TopicReq::Trades { .. }) => trades(f, t.clone()).await,
|
||||
(Topic::Bars(t), TopicReq::Bars { .. }) => bars(f, &hub, t.clone(), extra as usize).await,
|
||||
_ => unreachable!("the topic key matches the request"),
|
||||
}
|
||||
drop(held);
|
||||
}
|
||||
|
||||
fn quote_for(hub: &Hub, key: &MarketKey, coin: &str, m: &Mid) -> Quote {
|
||||
Quote {
|
||||
key: key.clone(),
|
||||
source: MarketKey::new(coin),
|
||||
px: m.px,
|
||||
basis: PriceBasis::Mid,
|
||||
prev_day: hub.prev_day(coin),
|
||||
ts_ms: m.ts_ms,
|
||||
}
|
||||
}
|
||||
|
||||
/// The worst state among a subscription's venues: unavailable, else stale (since the earliest
|
||||
/// outage), else live.
|
||||
fn combined(states: &[VenueState]) -> TopicStatus {
|
||||
let mut out = TopicStatus::Live;
|
||||
for st in states {
|
||||
match (&st.status, &out) {
|
||||
(TopicStatus::Unavailable { .. }, _) => return st.status.clone(),
|
||||
(TopicStatus::Stale { since_ms }, TopicStatus::Stale { since_ms: s }) if since_ms >= s => {}
|
||||
(TopicStatus::Stale { .. }, _) => out = st.status.clone(),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// The feeds these quotes come from, to report their outages.
|
||||
fn venues_of(hub: &Hub, resolved: &[(MarketKey, String)]) -> Vec<String> {
|
||||
let mut venues: Vec<String> =
|
||||
resolved.iter().filter_map(|(_, c)| hub.mids.get(c).map(|m| m.venue.to_string())).collect();
|
||||
venues.sort();
|
||||
venues.dedup();
|
||||
venues
|
||||
}
|
||||
|
||||
async fn quotes(mut f: Fwd, hub: Arc<Hub>, keys: Vec<MarketKey>, hz: u8) {
|
||||
let (mut resolved, mut late) = (Vec::new(), Vec::new());
|
||||
for k in keys {
|
||||
match hub.resolve(&k) {
|
||||
Some(i) => resolved.push((k, i.0)),
|
||||
None => late.push(k),
|
||||
}
|
||||
}
|
||||
if resolved.is_empty() {
|
||||
f.unavailable(UnavailableReason::UnknownMarket, "none of these markets are known".into()).await;
|
||||
return;
|
||||
}
|
||||
// Keys on a venue that is still loading join with their first delta once it's in. After
|
||||
// that, whatever still doesn't resolve is unknown.
|
||||
let mut look_again = hub.loading();
|
||||
let mut venues = venues_of(&hub, &resolved);
|
||||
let venue_states = |venues: &[String]| venues.iter().map(|v| hub.mids.venue(v)).collect::<Vec<_>>();
|
||||
let mut epochs: Vec<u64> = venue_states(&venues).iter().map(|v| v.epoch).collect();
|
||||
let mut status = TopicStatus::Live;
|
||||
let mut version = hub.mids.subscribe();
|
||||
// What each requested key last carried: two keys can share a coin (`BTC`, `BTCUSD Curncy`).
|
||||
let mut stamps: Vec<u64> = vec![0; resolved.len()];
|
||||
let mut send_snapshot = true;
|
||||
let period = Duration::from_secs(1) / u32::from(hz);
|
||||
loop {
|
||||
version.borrow_and_update();
|
||||
if look_again && !late.is_empty() {
|
||||
look_again = hub.loading();
|
||||
let before = resolved.len();
|
||||
late.retain(|k| match hub.resolve(k) {
|
||||
Some(i) => {
|
||||
resolved.push((k.clone(), i.0));
|
||||
false
|
||||
}
|
||||
None => true,
|
||||
});
|
||||
if resolved.len() > before {
|
||||
stamps.resize(resolved.len(), 0);
|
||||
venues = venues_of(&hub, &resolved);
|
||||
epochs = venue_states(&venues).iter().map(|v| v.epoch).collect();
|
||||
}
|
||||
}
|
||||
let states = venue_states(&venues);
|
||||
let now = combined(&states);
|
||||
if now != status {
|
||||
status = now;
|
||||
if !f.status(status.clone()).await {
|
||||
return;
|
||||
}
|
||||
}
|
||||
// A feed that reconnected may have skipped prices: once it's live again, start over.
|
||||
let now_epochs: Vec<u64> = states.iter().map(|v| v.epoch).collect();
|
||||
if status == TopicStatus::Live && now_epochs != epochs {
|
||||
epochs = now_epochs;
|
||||
if !send_snapshot && !f.resync(ResyncReason::Upstream).await {
|
||||
return;
|
||||
}
|
||||
send_snapshot = true;
|
||||
}
|
||||
if send_snapshot {
|
||||
send_snapshot = false;
|
||||
let snap: Vec<Quote> = resolved
|
||||
.iter()
|
||||
.zip(stamps.iter_mut())
|
||||
.filter_map(|((k, c), stamp)| {
|
||||
let m = hub.mids.get(c)?;
|
||||
*stamp = m.stamp;
|
||||
Some(quote_for(&hub, k, c, &m))
|
||||
})
|
||||
.collect();
|
||||
if !f.snapshot(TopicData::Quotes(snap)).await {
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
let mut newest: Option<Instant> = None;
|
||||
let changed: Vec<Quote> = resolved
|
||||
.iter()
|
||||
.zip(stamps.iter_mut())
|
||||
.filter_map(|((k, c), stamp)| {
|
||||
let m = hub.mids.get(c).filter(|m| m.stamp > *stamp)?;
|
||||
*stamp = m.stamp;
|
||||
newest = newest.max(Some(m.recv));
|
||||
Some(quote_for(&hub, k, c, &m))
|
||||
})
|
||||
.collect();
|
||||
if let Some(recv) = newest
|
||||
&& !f.delta(TopicDelta::Quotes(changed), recv).await
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
let sent = Instant::now();
|
||||
tokio::select! {
|
||||
r = version.changed() => if r.is_err() { return },
|
||||
_ = f.resync.notified() => send_snapshot = true,
|
||||
}
|
||||
// Conflate: at most `hz` messages a second.
|
||||
let next = sent + period;
|
||||
if next > Instant::now() {
|
||||
tokio::time::sleep_until(next.into()).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Group levels into `tick` buckets (bids down, asks up), then keep `depth` a side.
|
||||
fn shape(levels: &[Level], depth: usize, group: Option<f64>, up: bool) -> Vec<Level> {
|
||||
let Some(tick) = group.filter(|t| *t > 0.0 && t.is_finite()) else {
|
||||
return levels.iter().take(depth).copied().collect();
|
||||
};
|
||||
let mut out: Vec<Level> = Vec::with_capacity(depth);
|
||||
for l in levels {
|
||||
let b = if up { (l.px / tick).ceil() } else { (l.px / tick).floor() } * tick;
|
||||
if let Some(last) = out.last_mut().filter(|last| (last.px - b).abs() < tick * 1e-6) {
|
||||
last.sz += l.sz;
|
||||
last.n += l.n;
|
||||
} else if out.len() == depth {
|
||||
break;
|
||||
} else {
|
||||
out.push(Level { px: b, sz: l.sz, n: l.n });
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
async fn book(mut f: Fwd, topic: Arc<BookTopic>, depth: usize, group: Option<f64>) {
|
||||
let mut rx = topic.state.subscribe();
|
||||
let mut epoch: Option<u64> = None;
|
||||
let mut status = TopicStatus::Live;
|
||||
let mut last: Option<Arc<crate::store::BookSnap>> = None;
|
||||
let mut need_snapshot = true;
|
||||
loop {
|
||||
let st = rx.borrow_and_update().clone();
|
||||
if st.status != status {
|
||||
status = st.status.clone();
|
||||
if !f.status(status.clone()).await {
|
||||
return;
|
||||
}
|
||||
}
|
||||
if epoch.is_some_and(|e| e != st.epoch) && !need_snapshot {
|
||||
if !f.resync(ResyncReason::Upstream).await {
|
||||
return;
|
||||
}
|
||||
need_snapshot = true;
|
||||
}
|
||||
epoch = Some(st.epoch);
|
||||
if let Some(b) = st.book.filter(|b| last.as_ref().is_none_or(|l| !Arc::ptr_eq(l, b))) {
|
||||
let data = Book {
|
||||
bids: shape(&b.bids, depth, group, false),
|
||||
asks: shape(&b.asks, depth, group, true),
|
||||
time_ms: b.time_ms,
|
||||
};
|
||||
let ok = if need_snapshot {
|
||||
need_snapshot = false;
|
||||
f.snapshot(TopicData::Book(data)).await
|
||||
} else {
|
||||
f.delta(TopicDelta::Book(data), b.recv).await
|
||||
};
|
||||
if !ok {
|
||||
return;
|
||||
}
|
||||
last = Some(b);
|
||||
}
|
||||
tokio::select! {
|
||||
r = rx.changed() => if r.is_err() { return },
|
||||
_ = f.resync.notified() => {
|
||||
need_snapshot = true;
|
||||
last = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn trades(mut f: Fwd, topic: Arc<TradesTopic>) {
|
||||
let mut rx = topic.subscribe();
|
||||
let mut seen = 0;
|
||||
let mut snapshot = true;
|
||||
loop {
|
||||
if std::mem::take(&mut snapshot) {
|
||||
let (list, seq, status) = topic.snapshot();
|
||||
seen = seq;
|
||||
if !f.snapshot(TopicData::Trades(list)).await {
|
||||
return;
|
||||
}
|
||||
if status != TopicStatus::Live && !f.status(status).await {
|
||||
return;
|
||||
}
|
||||
}
|
||||
let ev = tokio::select! {
|
||||
ev = rx.recv() => ev,
|
||||
_ = f.resync.notified() => {
|
||||
snapshot = true;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
match ev {
|
||||
Ok(ev) if ev.seq <= seen => {}
|
||||
Ok(ev) => {
|
||||
seen = ev.seq;
|
||||
let ok = match &ev.kind {
|
||||
EventKind::Data(t) => f.delta(TopicDelta::Trades(t.clone()), ev.recv).await,
|
||||
EventKind::Status(s) => f.status(s.clone()).await,
|
||||
EventKind::Resync => {
|
||||
snapshot = true;
|
||||
f.resync(ResyncReason::Upstream).await
|
||||
}
|
||||
};
|
||||
if !ok {
|
||||
return;
|
||||
}
|
||||
}
|
||||
Err(broadcast::error::RecvError::Lagged(_)) => {
|
||||
snapshot = true;
|
||||
if !f.resync(ResyncReason::Overflow).await {
|
||||
return;
|
||||
}
|
||||
}
|
||||
Err(broadcast::error::RecvError::Closed) => return,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Drop each `Forming` bar that a later event in the batch supersedes (same `ts`, or a resync).
|
||||
/// `Closed` bars and statuses always stay, in order.
|
||||
pub fn coalesce(batch: Vec<Arc<Event<BarDelta>>>) -> Vec<Arc<Event<BarDelta>>> {
|
||||
let keep = |i: usize, e: &Event<BarDelta>| match &e.kind {
|
||||
EventKind::Data(BarDelta::Forming(b)) => !batch[i + 1..].iter().any(|later| match &later.kind {
|
||||
EventKind::Data(d) => d.bar().ts == b.ts,
|
||||
EventKind::Resync => true,
|
||||
EventKind::Status(_) => false,
|
||||
}),
|
||||
_ => true,
|
||||
};
|
||||
batch.iter().enumerate().filter(|(i, e)| keep(*i, e)).map(|(_, e)| e.clone()).collect()
|
||||
}
|
||||
|
||||
async fn bars_snapshot(f: &mut Fwd, hub: &Hub, topic: &BarsTopic, history: usize) -> Option<u64> {
|
||||
let since = topic.overlay().live_since;
|
||||
let hist = if history == 0 {
|
||||
Arc::new(Vec::new())
|
||||
} else {
|
||||
match hub.history(&topic.coin, topic.tf, None, history, since.or(Some(Instant::now()))).await {
|
||||
Ok(h) => h,
|
||||
Err(e) => {
|
||||
tracing::warn!("history {} {}: {e:#}", topic.coin, topic.tf);
|
||||
Arc::new(Vec::new())
|
||||
}
|
||||
}
|
||||
};
|
||||
let ov = topic.overlay();
|
||||
let bars = merge_bars(hist.to_vec(), &ov.bars, history.max(ov.bars.len()));
|
||||
if !f.snapshot(TopicData::Bars(bars)).await {
|
||||
return None;
|
||||
}
|
||||
if ov.status != TopicStatus::Live && !f.status(ov.status).await {
|
||||
return None;
|
||||
}
|
||||
Some(ov.seq)
|
||||
}
|
||||
|
||||
async fn bars(mut f: Fwd, hub: &Hub, topic: Arc<BarsTopic>, history: usize) {
|
||||
let mut rx = topic.subscribe();
|
||||
let Some(mut seen) = bars_snapshot(&mut f, hub, &topic, history).await else { return };
|
||||
loop {
|
||||
let first = tokio::select! {
|
||||
ev = rx.recv() => ev,
|
||||
_ = f.resync.notified() => {
|
||||
match bars_snapshot(&mut f, hub, &topic, history).await {
|
||||
Some(s) => seen = s,
|
||||
None => return,
|
||||
}
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let mut batch = Vec::new();
|
||||
let mut lagged = false;
|
||||
match first {
|
||||
Ok(e) => batch.push(e),
|
||||
Err(broadcast::error::RecvError::Lagged(_)) => lagged = true,
|
||||
Err(broadcast::error::RecvError::Closed) => return,
|
||||
}
|
||||
// Whatever else is already waiting goes out together, superseded bars dropped.
|
||||
loop {
|
||||
match rx.try_recv() {
|
||||
Ok(e) => batch.push(e),
|
||||
Err(broadcast::error::TryRecvError::Lagged(_)) => lagged = true,
|
||||
Err(_) => break,
|
||||
}
|
||||
}
|
||||
if lagged {
|
||||
if !f.resync(ResyncReason::Overflow).await {
|
||||
return;
|
||||
}
|
||||
match bars_snapshot(&mut f, hub, &topic, history).await {
|
||||
Some(s) => seen = s,
|
||||
None => return,
|
||||
}
|
||||
continue;
|
||||
}
|
||||
batch.retain(|e| e.seq > seen);
|
||||
let Some(last) = batch.last() else { continue };
|
||||
seen = last.seq;
|
||||
for e in coalesce(batch) {
|
||||
let ok = match &e.kind {
|
||||
EventKind::Data(d) => f.delta(TopicDelta::Bar(*d), e.recv).await,
|
||||
EventKind::Status(s) => f.status(s.clone()).await,
|
||||
EventKind::Resync => {
|
||||
if !f.resync(ResyncReason::Upstream).await {
|
||||
return;
|
||||
}
|
||||
// The fresh snapshot covers the rest of this batch.
|
||||
match bars_snapshot(&mut f, hub, &topic, history).await {
|
||||
Some(s) => seen = seen.max(s),
|
||||
None => return,
|
||||
}
|
||||
break;
|
||||
}
|
||||
};
|
||||
if !ok {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tr_core::Bar;
|
||||
|
||||
fn ev(seq: u64, kind: EventKind<BarDelta>) -> Arc<Event<BarDelta>> {
|
||||
Arc::new(Event { seq, recv: Instant::now(), kind })
|
||||
}
|
||||
|
||||
fn b(ts: i64, c: f64) -> Bar {
|
||||
Bar::new(ts, c, c, c, c, 1.0)
|
||||
}
|
||||
|
||||
fn kinds(v: &[Arc<Event<BarDelta>>]) -> Vec<String> {
|
||||
v.iter()
|
||||
.map(|e| match &e.kind {
|
||||
EventKind::Data(BarDelta::Forming(x)) => format!("F{}:{}", x.ts, x.close),
|
||||
EventKind::Data(BarDelta::Closed(x)) => format!("C{}:{}", x.ts, x.close),
|
||||
EventKind::Resync => "R".into(),
|
||||
EventKind::Status(_) => "S".into(),
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// A slow reader that wakes up across a bar boundary still gets the closed bar's final
|
||||
/// values before the next bar, and only the latest state of each forming bar.
|
||||
#[test]
|
||||
fn slow_reader_across_a_bar_boundary() {
|
||||
let batch = vec![
|
||||
ev(1, EventKind::Data(BarDelta::Forming(b(0, 1.0)))),
|
||||
ev(2, EventKind::Data(BarDelta::Forming(b(0, 2.0)))),
|
||||
ev(3, EventKind::Data(BarDelta::Closed(b(0, 2.5)))),
|
||||
ev(4, EventKind::Data(BarDelta::Forming(b(60, 3.0)))),
|
||||
ev(5, EventKind::Status(TopicStatus::Live)),
|
||||
ev(6, EventKind::Data(BarDelta::Forming(b(60, 4.0)))),
|
||||
];
|
||||
assert_eq!(kinds(&coalesce(batch)), ["C0:2.5", "S", "F60:4"]);
|
||||
let batch = vec![ev(1, EventKind::Data(BarDelta::Forming(b(0, 1.0)))), ev(2, EventKind::Resync)];
|
||||
assert_eq!(kinds(&coalesce(batch)), ["R"]);
|
||||
let batch = vec![ev(1, EventKind::Data(BarDelta::Forming(b(0, 1.0))))];
|
||||
assert_eq!(kinds(&coalesce(batch)), ["F0:1"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn book_grouping_and_depth() {
|
||||
let l = |px: f64, sz: f64| Level { px, sz, n: 1 };
|
||||
let bids = [l(100.4, 1.0), l(100.1, 2.0), l(99.7, 1.0), l(99.2, 1.0)];
|
||||
let asks = [l(100.6, 1.0), l(100.9, 3.0), l(101.2, 1.0)];
|
||||
assert_eq!(shape(&bids, 2, None, false), bids[..2]);
|
||||
let g = shape(&bids, 10, Some(0.5), false);
|
||||
assert_eq!(
|
||||
g.iter().map(|x| (x.px, x.sz, x.n)).collect::<Vec<_>>(),
|
||||
[(100.0, 3.0, 2), (99.5, 1.0, 1), (99.0, 1.0, 1)]
|
||||
);
|
||||
assert_eq!(shape(&bids, 1, Some(0.5), false).len(), 1);
|
||||
let g = shape(&asks, 10, Some(0.5), true);
|
||||
assert_eq!(g.iter().map(|x| (x.px, x.sz)).collect::<Vec<_>>(), [(101.0, 4.0), (101.5, 1.0)]);
|
||||
assert_eq!(shape(&asks, 5, Some(f64::NAN), true).len(), 3, "bad ticks mean no grouping");
|
||||
}
|
||||
}
|
||||
-1026
File diff suppressed because it is too large.
Load diff
-1
@@ -1 +0,0 @@
|
||||
{"channel":"allMids","data":{"mids":{"#114910":"0.5","#114911":"0.5","#114920":"0.17488","@1":"12.372","@107":"89.972","BTC":"77074.5","ETH":"2212.45","HYPE":"90.161","PURR/USDC":"0.11949","SOL":"94.6065"}}}
|
||||
-1
@@ -1 +0,0 @@
|
||||
{"channel":"allMids","data":{"dex":"xyz","mids":{"xyz:AAOI":"73.2525","xyz:AAPL":"349.85","xyz:ACN":"250.07","xyz:ALUMINIUM":"3080.0","xyz:AMAT":"401.52","xyz:AMD":"526.575","xyz:GOLD":"4013.15","xyz:TSLA":"394.6","xyz:XYZ100":"28743.5"}}}
|
||||
-1
@@ -1 +0,0 @@
|
||||
{"channel":"candle","data":{"t":1793326680000,"T":1793326739999,"s":"BTC","i":"1m","o":"77032.0","c":"77074.0","h":"77093.0","l":"76991.0","v":"15.07216","n":220}}
|
||||
-1
@@ -1 +0,0 @@
|
||||
{"channel":"l2Book","data":{"coin":"BTC","time":1793326724865,"levels":[[{"px":"77074.0","sz":"4.32001","n":9},{"px":"77073.0","sz":"0.01622","n":1},{"px":"77072.0","sz":"1.51628","n":1},{"px":"77071.0","sz":"0.84713","n":2},{"px":"77070.0","sz":"2.92355","n":2},{"px":"77069.0","sz":"0.26275","n":3},{"px":"77068.0","sz":"0.99095","n":4},{"px":"77067.0","sz":"0.1098","n":2},{"px":"77066.0","sz":"0.106","n":2},{"px":"77065.0","sz":"1.27403","n":3},{"px":"77064.0","sz":"1.29282","n":3},{"px":"77063.0","sz":"8.97664","n":4},{"px":"77062.0","sz":"1.84681","n":4},{"px":"77061.0","sz":"6.08341","n":6},{"px":"77060.0","sz":"1.49667","n":5},{"px":"77059.0","sz":"3.77298","n":3},{"px":"77058.0","sz":"1.27048","n":5},{"px":"77057.0","sz":"1.34478","n":3},{"px":"77056.0","sz":"1.36837","n":5},{"px":"77055.0","sz":"8.21667","n":5}],[{"px":"77075.0","sz":"3.92943","n":10},{"px":"77076.0","sz":"0.03119","n":1},{"px":"77078.0","sz":"1.11631","n":1},{"px":"77079.0","sz":"0.22003","n":1},{"px":"77080.0","sz":"1.12873","n":3},{"px":"77081.0","sz":"1.83735","n":3},{"px":"77082.0","sz":"1.22913","n":2},{"px":"77083.0","sz":"3.64255","n":4},{"px":"77084.0","sz":"1.62902","n":2},{"px":"77085.0","sz":"7.96303","n":12},{"px":"77086.0","sz":"10.386","n":6},{"px":"77087.0","sz":"18.13093","n":8},{"px":"77088.0","sz":"4.2013","n":5},{"px":"77089.0","sz":"2.12388","n":7},{"px":"77090.0","sz":"0.2849","n":4},{"px":"77091.0","sz":"5.41136","n":6},{"px":"77092.0","sz":"1.05595","n":7},{"px":"77093.0","sz":"2.30917","n":10},{"px":"77094.0","sz":"4.87672","n":10},{"px":"77095.0","sz":"3.61234","n":5}]]}}
|
||||
-1
@@ -1 +0,0 @@
|
||||
{"channel":"trades","data":[{"coin":"BTC","side":"B","px":"77090.0","sz":"0.00064","time":1793326719367,"hash":"0x0000000000000000000000000000000000000000000000000000000000000000","tid":494268480360624,"users":["0x0000000000000000000000000000000000000000","0x0000000000000000000000000000000000000000"]},{"coin":"BTC","side":"B","px":"77090.0","sz":"0.0022","time":1793326719367,"hash":"0x0000000000000000000000000000000000000000000000000000000000000000","tid":530242516244559,"users":["0x0000000000000000000000000000000000000000","0x0000000000000000000000000000000000000000"]},{"coin":"BTC","side":"B","px":"77093.0","sz":"0.42668","time":1793326719367,"hash":"0x0000000000000000000000000000000000000000000000000000000000000000","tid":599896480530140,"users":["0x0000000000000000000000000000000000000000","0x0000000000000000000000000000000000000000"]},{"coin":"BTC","side":"A","px":"77092.0","sz":"0.00397","time":1793326719427,"hash":"0x0000000000000000000000000000000000000000000000000000000000000000","tid":659301716729239,"users":["0x0000000000000000000000000000000000000000","0x0000000000000000000000000000000000000000"]},{"coin":"BTC","side":"B","px":"77093.0","sz":"0.01627","time":1793326719856,"hash":"0x0000000000000000000000000000000000000000000000000000000000000000","tid":1030399580240574,"users":["0x0000000000000000000000000000000000000000","0x0000000000000000000000000000000000000000"]}]}
|
||||
@@ -1,870 +0,0 @@
|
||||
//! The gateway end to end, against a mock of Hyperliquid's public API (websocket and `info` on
|
||||
//! one local port) and through the desktop's client (`tr_data::remote`).
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::net::{Ipv4Addr, SocketAddr};
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
|
||||
use axum::Router;
|
||||
use axum::extract::ws::{Message, WebSocket};
|
||||
use axum::extract::{State, WebSocketUpgrade};
|
||||
use axum::routing::{get, post};
|
||||
use futures::{SinkExt, StreamExt};
|
||||
use serde_json::{Value, json};
|
||||
use tokio::sync::mpsc;
|
||||
use tr_data::MarketData;
|
||||
use tr_data::remote::{Client, Event, RemoteMarkets, Subscription};
|
||||
use tr_gateway::Config;
|
||||
use tr_proto::{
|
||||
BarDelta, Encoding, ErrorCode, InstrumentKind, MarketKey, ResyncReason, Rpc, RpcResult, Timeframe, TopicData,
|
||||
TopicDelta, TopicReq, TopicStatus, UnavailableReason,
|
||||
};
|
||||
|
||||
const WAIT: Duration = Duration::from_secs(10);
|
||||
|
||||
// ---- the mock upstream ----
|
||||
|
||||
/// A mock connection: where to send it frames (`None` closes it), and what it subscribed to.
|
||||
type MockConn = (mpsc::UnboundedSender<Option<String>>, Arc<Mutex<HashSet<String>>>);
|
||||
|
||||
#[derive(Default)]
|
||||
struct Mock {
|
||||
/// Every subscribe and unsubscribe received, as `subscribe l2Book:BTC`.
|
||||
log: Mutex<Vec<String>>,
|
||||
conns: Mutex<Vec<MockConn>>,
|
||||
/// Connections open right now.
|
||||
open: AtomicUsize,
|
||||
/// Answer this many `candleSnapshot` requests with HTTP 500...
|
||||
fail_candles: Mutex<usize>,
|
||||
/// ...after holding each this long.
|
||||
fail_hold: Mutex<Duration>,
|
||||
/// `candleSnapshot` requests being answered now, and the most there ever were at once.
|
||||
candles_inflight: AtomicUsize,
|
||||
candles_peak: AtomicUsize,
|
||||
}
|
||||
|
||||
/// Counts a `candleSnapshot` request as in flight until it's answered.
|
||||
struct InFlight(Arc<Mock>);
|
||||
|
||||
impl InFlight {
|
||||
fn new(m: &Arc<Mock>) -> Self {
|
||||
let n = m.candles_inflight.fetch_add(1, Ordering::SeqCst) + 1;
|
||||
m.candles_peak.fetch_max(n, Ordering::SeqCst);
|
||||
Self(m.clone())
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for InFlight {
|
||||
fn drop(&mut self) {
|
||||
self.0.candles_inflight.fetch_sub(1, Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
|
||||
/// Counts a mock connection as open until its task ends.
|
||||
struct Open(Arc<Mock>);
|
||||
|
||||
impl Drop for Open {
|
||||
fn drop(&mut self) {
|
||||
self.0.open.fetch_sub(1, Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
|
||||
fn label(sub: &Value) -> String {
|
||||
let s = |k: &str| sub[k].as_str().unwrap_or_default().to_string();
|
||||
match s("type").as_str() {
|
||||
"allMids" if s("dex").is_empty() => "allMids".into(),
|
||||
"allMids" => format!("allMids:{}", s("dex")),
|
||||
"candle" => format!("candle:{}:{}", s("coin"), s("interval")),
|
||||
t => format!("{t}:{}", s("coin")),
|
||||
}
|
||||
}
|
||||
|
||||
impl Mock {
|
||||
fn subs(&self) -> HashSet<String> {
|
||||
self.conns.lock().unwrap().iter().flat_map(|(_, s)| s.lock().unwrap().clone()).collect()
|
||||
}
|
||||
|
||||
fn count(&self, entry: &str) -> usize {
|
||||
self.log.lock().unwrap().iter().filter(|e| *e == entry).count()
|
||||
}
|
||||
|
||||
/// Send `data` on `channel` to every connection subscribed to `sub`.
|
||||
fn push(&self, sub: &str, channel: &str, data: Value) {
|
||||
let text = json!({ "channel": channel, "data": data }).to_string();
|
||||
for (tx, subs) in self.conns.lock().unwrap().iter() {
|
||||
if subs.lock().unwrap().contains(sub) {
|
||||
let _ = tx.send(Some(text.clone()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Close every upstream connection, as Hyperliquid may at any time.
|
||||
fn drop_all(&self) {
|
||||
for (tx, _) in self.conns.lock().unwrap().drain(..) {
|
||||
let _ = tx.send(None);
|
||||
}
|
||||
}
|
||||
|
||||
async fn until(&self, what: &str, f: impl Fn(&Mock) -> bool) {
|
||||
let t0 = tokio::time::Instant::now();
|
||||
while !f(self) {
|
||||
assert!(t0.elapsed() < WAIT, "timed out waiting for {what}; upstream has {:?}", self.subs());
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn mock_ws(State(m): State<Arc<Mock>>, up: WebSocketUpgrade) -> axum::response::Response {
|
||||
up.on_upgrade(move |ws| mock_conn(ws, m))
|
||||
}
|
||||
|
||||
async fn mock_conn(ws: WebSocket, m: Arc<Mock>) {
|
||||
m.open.fetch_add(1, Ordering::SeqCst);
|
||||
let _open = Open(m.clone());
|
||||
let (tx, mut rx) = mpsc::unbounded_channel::<Option<String>>();
|
||||
let subs = Arc::new(Mutex::new(HashSet::new()));
|
||||
m.conns.lock().unwrap().push((tx.clone(), subs.clone()));
|
||||
let (mut sink, mut stream) = ws.split();
|
||||
loop {
|
||||
tokio::select! {
|
||||
out = rx.recv() => match out {
|
||||
Some(Some(text)) => if sink.send(Message::Text(text.into())).await.is_err() { return },
|
||||
_ => {
|
||||
let _ = sink.close().await;
|
||||
return;
|
||||
}
|
||||
},
|
||||
msg = stream.next() => {
|
||||
let Some(Ok(Message::Text(t))) = msg else {
|
||||
if matches!(msg, None | Some(Err(_)) | Some(Ok(Message::Close(_)))) { return }
|
||||
continue;
|
||||
};
|
||||
let v: Value = serde_json::from_str(t.as_str()).unwrap();
|
||||
match v["method"].as_str() {
|
||||
Some("ping") => { let _ = tx.send(Some(json!({"channel": "pong"}).to_string())); }
|
||||
Some(method @ ("subscribe" | "unsubscribe")) => {
|
||||
let l = label(&v["subscription"]);
|
||||
m.log.lock().unwrap().push(format!("{method} {l}"));
|
||||
if method == "subscribe" { subs.lock().unwrap().insert(l); } else { subs.lock().unwrap().remove(&l); }
|
||||
let ack = json!({"channel": "subscriptionResponse", "data": {"method": method, "subscription": v["subscription"]}});
|
||||
let _ = tx.send(Some(ack.to_string()));
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn interval_s(i: &str) -> i64 {
|
||||
match i {
|
||||
"1m" => 60,
|
||||
"5m" => 300,
|
||||
"1h" => 3600,
|
||||
"1d" => 86_400,
|
||||
_ => 60,
|
||||
}
|
||||
}
|
||||
|
||||
async fn mock_info(State(m): State<Arc<Mock>>, axum::Json(req): axum::Json<Value>) -> axum::response::Response {
|
||||
use axum::response::IntoResponse;
|
||||
let _inflight = (req["type"] == "candleSnapshot").then(|| InFlight::new(&m));
|
||||
let fail = req["type"] == "candleSnapshot" && {
|
||||
let mut n = m.fail_candles.lock().unwrap();
|
||||
let fail = *n > 0;
|
||||
*n = n.saturating_sub(1);
|
||||
fail
|
||||
};
|
||||
if fail {
|
||||
let hold = *m.fail_hold.lock().unwrap();
|
||||
tokio::time::sleep(hold).await;
|
||||
return axum::http::StatusCode::INTERNAL_SERVER_ERROR.into_response();
|
||||
}
|
||||
axum::Json(match req["type"].as_str().unwrap() {
|
||||
"perpDexs" => json!([null, {"name": "xyz"}]),
|
||||
"metaAndAssetCtxs" if req["dex"] == "xyz" => {
|
||||
json!([{"universe": [{"name": "xyz:TSLA"}]}, [{"prevDayPx": "400.0"}]])
|
||||
}
|
||||
"metaAndAssetCtxs" => json!([
|
||||
{"universe": [{"name": "BTC"}, {"name": "ETH"}, {"name": "OLD", "isDelisted": true}]},
|
||||
[{"prevDayPx": "100.0"}, {"prevDayPx": "10.0"}, {"prevDayPx": "1.0"}]
|
||||
]),
|
||||
"candleSnapshot" => {
|
||||
let r = &req["req"];
|
||||
if r["coin"] == "SLOW" {
|
||||
tokio::time::sleep(Duration::from_secs(3)).await;
|
||||
}
|
||||
let iv = interval_s(r["interval"].as_str().unwrap()) * 1000;
|
||||
let (start, end) = (r["startTime"].as_i64().unwrap(), r["endTime"].as_i64().unwrap());
|
||||
let first = (start + iv - 1) / iv;
|
||||
let bars: Vec<Value> = (first..=end / iv)
|
||||
.map(|k| {
|
||||
let px = format!("{}.0", 100 + k % 10);
|
||||
json!({"t": k * iv, "T": k * iv + iv - 1, "s": r["coin"], "i": r["interval"], "o": px, "h": px, "l": px, "c": px, "v": "1.0", "n": 1})
|
||||
})
|
||||
.collect();
|
||||
Value::Array(bars[bars.len().saturating_sub(5000)..].to_vec())
|
||||
}
|
||||
other => panic!("unexpected info request {other}"),
|
||||
})
|
||||
.into_response()
|
||||
}
|
||||
|
||||
async fn start_mock() -> (Arc<Mock>, SocketAddr) {
|
||||
let m = Arc::new(Mock::default());
|
||||
let app = Router::new().route("/ws", get(mock_ws)).route("/info", post(mock_info)).with_state(m.clone());
|
||||
let l = tokio::net::TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap();
|
||||
let addr = l.local_addr().unwrap();
|
||||
tokio::spawn(async move { axum::serve(l, app).await.unwrap() });
|
||||
(m, addr)
|
||||
}
|
||||
|
||||
// ---- helpers ----
|
||||
|
||||
struct Setup {
|
||||
mock: Arc<Mock>,
|
||||
gw: tr_gateway::Gateway,
|
||||
}
|
||||
|
||||
impl Setup {
|
||||
async fn new(tweak: impl FnOnce(&mut Config)) -> Setup {
|
||||
let s = Setup::starting(tweak).await;
|
||||
s.core_mids();
|
||||
s.xyz_mids();
|
||||
let t0 = tokio::time::Instant::now();
|
||||
while s.gw.hub().mids.len() < 4 {
|
||||
assert!(t0.elapsed() < WAIT, "mids never arrived");
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
assert!(s.gw.hub().ready(WAIT).await, "the hub never got ready");
|
||||
s
|
||||
}
|
||||
|
||||
/// A gateway that has subscribed every venue's mids upstream but heard none yet, as right
|
||||
/// after a restart.
|
||||
async fn starting(tweak: impl FnOnce(&mut Config)) -> Setup {
|
||||
let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
|
||||
let (mock, addr) = start_mock().await;
|
||||
let mut cfg = Config { bind: SocketAddr::from((Ipv4Addr::LOCALHOST, 0)), ..Default::default() };
|
||||
cfg.hl.info_url = format!("http://{addr}/info");
|
||||
cfg.hl.pool.ws_url = format!("ws://{addr}/ws");
|
||||
cfg.hub.linger = Duration::from_millis(300);
|
||||
tweak(&mut cfg);
|
||||
let gw = tr_gateway::start(cfg).await.unwrap();
|
||||
mock.until("both venues' mids", |m| m.subs().is_superset(&set(&["allMids", "allMids:xyz"]))).await;
|
||||
Setup { mock, gw }
|
||||
}
|
||||
|
||||
fn core_mids(&self) {
|
||||
let mids = json!({"mids": {"BTC": "100.5", "ETH": "10.25", "@107": "30.0", "#5": "0.5"}});
|
||||
self.mock.push("allMids", "allMids", mids);
|
||||
}
|
||||
|
||||
fn xyz_mids(&self) {
|
||||
self.mock.push("allMids:xyz", "allMids", json!({"dex": "xyz", "mids": {"xyz:TSLA": "401.5"}}));
|
||||
}
|
||||
|
||||
fn client(&self, enc: Encoding) -> Client {
|
||||
Client::connect(&format!("ws://{}", self.gw.addr), enc, "gateway-test")
|
||||
}
|
||||
|
||||
fn upstream_keys(&self) -> Vec<String> {
|
||||
self.gw.hub().provider().stats().keys
|
||||
}
|
||||
}
|
||||
|
||||
fn set(v: &[&str]) -> HashSet<String> {
|
||||
v.iter().map(|s| s.to_string()).collect()
|
||||
}
|
||||
|
||||
async fn recv_bar(feed: &mut tr_data::LiveFeed) -> tr_core::Bar {
|
||||
tokio::time::timeout(WAIT, feed.recv()).await.expect("a bar in time").expect("feed open")
|
||||
}
|
||||
|
||||
async fn next(sub: &mut Subscription) -> Event {
|
||||
tokio::time::timeout(WAIT, sub.next()).await.expect("an update in time").expect("client alive").event
|
||||
}
|
||||
|
||||
/// The next event that isn't a status or reconnect notice.
|
||||
async fn next_data(sub: &mut Subscription) -> Event {
|
||||
loop {
|
||||
match next(sub).await {
|
||||
Event::Status(_) | Event::Reconnecting | Event::Resubscribing => {}
|
||||
e => return e,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// What a subscription sees up to its first snapshot, and the snapshot. Fails if the server
|
||||
/// gives up on it on the way.
|
||||
async fn until_snapshot(sub: &mut Subscription) -> (Vec<String>, TopicData) {
|
||||
let mut seen = Vec::new();
|
||||
loop {
|
||||
match next(sub).await {
|
||||
Event::Snapshot(data) => return (seen, data),
|
||||
Event::Status(s @ TopicStatus::Unavailable { .. }) => panic!("the server gave up: {s:?} after {seen:?}"),
|
||||
Event::Status(TopicStatus::Live) => seen.push("live".into()),
|
||||
Event::Status(TopicStatus::Stale { .. }) => seen.push("stale".into()),
|
||||
e => seen.push(format!("{e:?}")),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn book_msg(coin: &str, bid: &str) -> Value {
|
||||
json!({"coin": coin, "time": 1_793_325_942_459i64, "levels": [
|
||||
[{"px": bid, "sz": "1.5", "n": 3}, {"px": "99.0", "sz": "2", "n": 1}],
|
||||
[{"px": "101.0", "sz": "1", "n": 2}]
|
||||
]})
|
||||
}
|
||||
|
||||
fn trade(tid: u64) -> Value {
|
||||
json!({"coin": "BTC", "side": "B", "px": "100.5", "sz": "0.1", "time": tid as i64, "hash": "0x0", "tid": tid, "users": ["0x0", "0x0"]})
|
||||
}
|
||||
|
||||
// ---- tests ----
|
||||
|
||||
#[tokio::test]
|
||||
async fn clients_share_one_upstream_subscription_until_the_last_leaves() {
|
||||
let s = Setup::new(|_| {}).await;
|
||||
let (a, b) = (s.client(Encoding::Postcard), s.client(Encoding::Postcard));
|
||||
let topic = || TopicReq::Book { key: MarketKey::new("BTC"), depth: 10, group: None };
|
||||
let mut sa = a.subscribe(topic());
|
||||
let mut sb = b.subscribe(topic());
|
||||
s.mock.until("l2Book:BTC upstream", |m| m.subs().contains("l2Book:BTC")).await;
|
||||
s.mock.push("l2Book:BTC", "l2Book", book_msg("BTC", "100.0"));
|
||||
for sub in [&mut sa, &mut sb] {
|
||||
let Event::Snapshot(TopicData::Book(book)) = next_data(sub).await else { panic!("book snapshot") };
|
||||
assert_eq!((book.bids[0].px, book.bids[0].sz, book.bids[0].n, book.asks.len()), (100.0, 1.5, 3, 1));
|
||||
}
|
||||
// A third client on the same market adds nothing upstream.
|
||||
let c = s.client(Encoding::Postcard);
|
||||
let mut sc = c.subscribe(topic());
|
||||
let Event::Snapshot(TopicData::Book(_)) = next_data(&mut sc).await else { panic!("hot snapshot") };
|
||||
assert_eq!(s.mock.count("subscribe l2Book:BTC"), 1, "one upstream subscription for three clients");
|
||||
assert_eq!(s.upstream_keys(), ["allMids", "allMids:xyz", "l2Book:BTC"]);
|
||||
// Deltas reach every client, in sequence.
|
||||
s.mock.push("l2Book:BTC", "l2Book", book_msg("BTC", "100.25"));
|
||||
for sub in [&mut sa, &mut sb, &mut sc] {
|
||||
let Event::Delta(TopicDelta::Book(book)) = next_data(sub).await else { panic!("book delta") };
|
||||
assert_eq!(book.bids[0].px, 100.25);
|
||||
}
|
||||
drop((sa, sb));
|
||||
tokio::time::sleep(Duration::from_millis(700)).await;
|
||||
assert!(s.mock.subs().contains("l2Book:BTC"), "one client still watches");
|
||||
drop(sc);
|
||||
s.mock.until("the upstream unsubscribe after the linger", |m| m.count("unsubscribe l2Book:BTC") == 1).await;
|
||||
assert_eq!(s.upstream_keys(), ["allMids", "allMids:xyz"]);
|
||||
assert_eq!(s.gw.hub().topic_counts().book, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn subscriptions_past_the_upstream_budget_are_refused() {
|
||||
// Two permanent mids subscriptions plus one more.
|
||||
let s = Setup::new(|c| c.hl.pool.max_subscriptions = 3).await;
|
||||
let client = s.client(Encoding::Postcard);
|
||||
let book = client.subscribe(TopicReq::Book { key: MarketKey::new("BTC"), depth: 5, group: None });
|
||||
s.mock.until("l2Book:BTC upstream", |m| m.subs().contains("l2Book:BTC")).await;
|
||||
let mut trades = client.subscribe(TopicReq::Trades { key: MarketKey::new("ETH") });
|
||||
match next(&mut trades).await {
|
||||
Event::Status(TopicStatus::Unavailable { reason: UnavailableReason::Capacity, detail }) => {
|
||||
assert!(detail.contains("3 of 3"), "{detail}")
|
||||
}
|
||||
e => panic!("expected a capacity refusal, got {e:?}"),
|
||||
}
|
||||
assert_eq!(s.mock.count("subscribe trades:ETH"), 0, "nothing past the budget reaches upstream");
|
||||
// Freeing the slot lets the next subscription through.
|
||||
drop(book);
|
||||
s.mock.until("the book's unsubscribe", |m| m.count("unsubscribe l2Book:BTC") == 1).await;
|
||||
let _trades = client.subscribe(TopicReq::Trades { key: MarketKey::new("ETH") });
|
||||
s.mock.until("trades:ETH upstream", |m| m.subs().contains("trades:ETH")).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unknown_markets_never_reach_upstream_and_keys_resolve() {
|
||||
let s = Setup::new(|_| {}).await;
|
||||
let client = s.client(Encoding::Postcard);
|
||||
let mut nope = client.subscribe(TopicReq::Book { key: MarketKey::new("NOTACOIN"), depth: 5, group: None });
|
||||
assert!(matches!(
|
||||
next(&mut nope).await,
|
||||
Event::Status(TopicStatus::Unavailable { reason: UnavailableReason::UnknownMarket, .. })
|
||||
));
|
||||
let keys =
|
||||
["BTCUSD Curncy", "BTC", "ETH-PERP", "xyz:TSLA", "@107", "#5", "AAPL US Equity"].map(MarketKey::new).to_vec();
|
||||
let mut q = client.subscribe(TopicReq::Quotes { keys, max_hz: 4 });
|
||||
let Event::Snapshot(TopicData::Quotes(quotes)) = next(&mut q).await else { panic!("quotes") };
|
||||
let got: HashMap<&str, (&str, f64, Option<f64>)> =
|
||||
quotes.iter().map(|q| (q.key.as_str(), (q.source.as_str(), q.px, q.prev_day))).collect();
|
||||
assert_eq!(got["BTCUSD Curncy"], ("BTC", 100.5, Some(100.0)));
|
||||
assert_eq!(got["ETH-PERP"], ("ETH", 10.25, Some(10.0)));
|
||||
assert_eq!(got["xyz:TSLA"], ("xyz:TSLA", 401.5, Some(400.0)));
|
||||
assert_eq!(got["@107"], ("@107", 30.0, None));
|
||||
assert!(!got.contains_key("#5"), "outcome markets are out of scope");
|
||||
assert!(!got.contains_key("AAPL US Equity"), "equities don't map to HIP-3 perps yet");
|
||||
// A changed mid arrives as a delta with just that market, under every key that asked for it.
|
||||
s.mock.push("allMids", "allMids", json!({"mids": {"BTC": "100.75", "ETH": "10.25", "@107": "30.0"}}));
|
||||
let Event::Delta(TopicDelta::Quotes(d)) = next(&mut q).await else { panic!("quote delta") };
|
||||
assert_eq!(
|
||||
d.iter().map(|q| (q.key.as_str(), q.px)).collect::<Vec<_>>(),
|
||||
[("BTCUSD Curncy", 100.75), ("BTC", 100.75)]
|
||||
);
|
||||
s.mock.push("allMids", "allMids", json!({"mids": {"BTC": "100.5", "ETH": "10.25", "@107": "30.0"}}));
|
||||
let Event::Delta(TopicDelta::Quotes(d)) = next(&mut q).await else { panic!("second quote delta") };
|
||||
assert_eq!(d.len(), 2, "neither key freezes: {d:?}");
|
||||
assert!(s.mock.log.lock().unwrap().iter().all(|e| !e.contains("NOTACOIN")));
|
||||
// The market list: perps by venue, spot, no outcomes.
|
||||
let Ok(RpcResult::Markets(m)) = client.request(Rpc::Markets).await else { panic!("markets") };
|
||||
let find = |k: &str| m.iter().find(|x| x.key.as_str() == k).map(|x| (x.venue.as_str(), x.kind));
|
||||
assert_eq!(find("BTC"), Some(("", InstrumentKind::Perp)));
|
||||
assert_eq!(find("xyz:TSLA"), Some(("xyz", InstrumentKind::Perp)));
|
||||
assert_eq!(find("@107"), Some(("spot", InstrumentKind::Spot)));
|
||||
assert_eq!(find("#5"), None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn upstream_reconnects_resubscribe_and_clients_resync() {
|
||||
let s = Setup::new(|_| {}).await;
|
||||
let client = s.client(Encoding::Postcard);
|
||||
let mut sub = client.subscribe(TopicReq::Trades { key: MarketKey::new("BTC") });
|
||||
s.mock.until("trades:BTC upstream", |m| m.subs().contains("trades:BTC")).await;
|
||||
let Event::Snapshot(TopicData::Trades(t)) = next(&mut sub).await else { panic!("trades snapshot") };
|
||||
assert!(t.is_empty());
|
||||
s.mock.push("trades:BTC", "trades", json!([trade(1), trade(2)]));
|
||||
let Event::Delta(TopicDelta::Trades(t)) = next(&mut sub).await else { panic!("trades delta") };
|
||||
assert_eq!(t.iter().map(|t| t.tid).collect::<Vec<_>>(), [1, 2]);
|
||||
|
||||
s.mock.drop_all();
|
||||
assert!(matches!(next(&mut sub).await, Event::Status(TopicStatus::Stale { .. })));
|
||||
s.mock
|
||||
.until("resubscribed after reconnecting", |m| {
|
||||
m.count("subscribe trades:BTC") == 2 && m.count("subscribe allMids") == 2
|
||||
})
|
||||
.await;
|
||||
// Hyperliquid starts a (re)subscription with recent trades, overlapping what we have.
|
||||
s.mock.push("trades:BTC", "trades", json!([trade(2), trade(3)]));
|
||||
let mut seen = Vec::new();
|
||||
loop {
|
||||
match next(&mut sub).await {
|
||||
Event::Snapshot(TopicData::Trades(t)) => {
|
||||
assert_eq!(t.iter().map(|t| t.tid).collect::<Vec<_>>(), [1, 2, 3], "deduplicated");
|
||||
break;
|
||||
}
|
||||
Event::Status(TopicStatus::Live) => seen.push("live"),
|
||||
Event::Resync(ResyncReason::Upstream) => seen.push("resync"),
|
||||
e => panic!("unexpected {e:?}"),
|
||||
}
|
||||
}
|
||||
assert_eq!(seen, ["live", "resync"]);
|
||||
assert_eq!(s.gw.hub().provider().stats().connections_up, 2);
|
||||
let stats = tr_gateway::http::stats(&s.gw.state);
|
||||
assert_eq!(stats.upstream_reconnects, 2, "both connections came back");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bars_stream_closed_before_the_next_bar_and_history_merges() {
|
||||
let s = Setup::new(|_| {}).await;
|
||||
let client = s.client(Encoding::Postcard);
|
||||
let Ok(RpcResult::History(h)) = client
|
||||
.request(Rpc::History { key: MarketKey::new("BTCUSD Curncy"), tf: Timeframe::H1, end: None, limit: 10 })
|
||||
.await
|
||||
else {
|
||||
panic!("history")
|
||||
};
|
||||
assert_eq!(h.len(), 10);
|
||||
assert!(h.windows(2).all(|w| w[1].ts - w[0].ts == 3600));
|
||||
let mut sub = client.subscribe(TopicReq::Bars { key: MarketKey::new("BTC"), tf: Timeframe::M1, history: 3 });
|
||||
s.mock.until("candle:BTC:1m upstream", |m| m.subs().contains("candle:BTC:1m")).await;
|
||||
let Event::Snapshot(TopicData::Bars(snap)) = next(&mut sub).await else { panic!("bars snapshot") };
|
||||
assert_eq!(snap.len(), 3);
|
||||
let t0 = snap.last().unwrap().ts + 60;
|
||||
let candle = |t: i64, c: &str| json!({"t": t * 1000, "T": t * 1000 + 59_999, "s": "BTC", "i": "1m", "o": "100", "h": "200", "l": "50", "c": c, "v": "3", "n": 9});
|
||||
for (t, c) in [(t0, "101"), (t0, "102"), (t0 + 60, "103")] {
|
||||
s.mock.push("candle:BTC:1m", "candle", candle(t, c));
|
||||
}
|
||||
let mut got = Vec::new();
|
||||
while got.last().is_none_or(|l| l != "F60:103") {
|
||||
if let Event::Delta(TopicDelta::Bar(d)) = next_data(&mut sub).await {
|
||||
got.push(match d {
|
||||
BarDelta::Forming(b) => format!("F{}:{}", b.ts - t0, b.close),
|
||||
BarDelta::Closed(b) => format!("C{}:{}", b.ts - t0, b.close),
|
||||
});
|
||||
}
|
||||
}
|
||||
// A reader may skip a forming bar a later message supersedes, but the closed bar's final
|
||||
// values always arrive, before the next bar.
|
||||
let closed = got.iter().position(|g| g == "C0:102").expect("the closed bar");
|
||||
assert_eq!(&got[closed..], ["C0:102", "F60:103"], "{got:?}");
|
||||
assert!(got[..closed].iter().all(|g| g.starts_with("F0:")), "{got:?}");
|
||||
// History now includes what the live stream has seen.
|
||||
let Ok(RpcResult::History(h)) =
|
||||
client.request(Rpc::History { key: MarketKey::new("BTC"), tf: Timeframe::M1, end: None, limit: 5 }).await
|
||||
else {
|
||||
panic!("history")
|
||||
};
|
||||
assert_eq!(h.last().map(|b| (b.ts, b.close)), Some((t0 + 60, 103.0)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn json_is_a_debug_encoding() {
|
||||
let s = Setup::new(|_| {}).await;
|
||||
let client = s.client(Encoding::Json);
|
||||
let mut q = client.subscribe(TopicReq::Quotes { keys: vec![MarketKey::new("xyz:TSLA")], max_hz: 1 });
|
||||
let Event::Snapshot(TopicData::Quotes(quotes)) = next(&mut q).await else { panic!("quotes over JSON") };
|
||||
assert_eq!(quotes[0].px, 401.5);
|
||||
let stats = tr_gateway::http::stats(&s.gw.state);
|
||||
assert_eq!((stats.clients, stats.upstream.subscriptions), (1, 2));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn clients_that_send_too_fast_are_throttled() {
|
||||
let s = Setup::new(|c| c.session.max_messages_per_10s = 3).await;
|
||||
let client = s.client(Encoding::Postcard);
|
||||
assert!(client.wait_up(WAIT).await);
|
||||
// The client's first ping and these five subscribes share a budget of three messages.
|
||||
let mut subs: Vec<_> =
|
||||
(0..5).map(|_| client.subscribe(TopicReq::Quotes { keys: vec![MarketKey::new("BTC")], max_hz: 1 })).collect();
|
||||
let mut answered = 0;
|
||||
for sub in &mut subs {
|
||||
if let Ok(Some(_)) = tokio::time::timeout(Duration::from_millis(500), sub.next()).await {
|
||||
answered += 1;
|
||||
}
|
||||
}
|
||||
assert!((2..=3).contains(&answered), "{answered} of 5 subscriptions were served");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn quote_subscribers_see_feed_outages_then_resync() {
|
||||
let s = Setup::new(|_| {}).await;
|
||||
let client = s.client(Encoding::Postcard);
|
||||
let keys = ["BTC", "xyz:TSLA"].map(MarketKey::new).to_vec();
|
||||
let mut q = client.subscribe(TopicReq::Quotes { keys, max_hz: 10 });
|
||||
assert!(matches!(next(&mut q).await, Event::Snapshot(TopicData::Quotes(_))));
|
||||
s.mock.drop_all();
|
||||
assert!(matches!(next(&mut q).await, Event::Status(TopicStatus::Stale { .. })), "the outage is visible");
|
||||
s.mock
|
||||
.until("mids resubscribed", |m| m.count("subscribe allMids") == 2 && m.count("subscribe allMids:xyz") == 2)
|
||||
.await;
|
||||
s.mock.push("allMids", "allMids", json!({"mids": {"BTC": "101.0", "ETH": "10.25", "@107": "30.0"}}));
|
||||
s.mock.push("allMids:xyz", "allMids", json!({"dex": "xyz", "mids": {"xyz:TSLA": "401.5"}}));
|
||||
let mut seen = Vec::new();
|
||||
let quotes = loop {
|
||||
match next(&mut q).await {
|
||||
Event::Snapshot(TopicData::Quotes(q)) => break q,
|
||||
Event::Status(TopicStatus::Live) => seen.push("live"),
|
||||
Event::Status(TopicStatus::Stale { .. }) => seen.push("stale"),
|
||||
Event::Resync(ResyncReason::Upstream) => seen.push("resync"),
|
||||
e => panic!("unexpected {e:?}"),
|
||||
}
|
||||
};
|
||||
assert_eq!(seen.last_chunk::<2>(), Some(&["live", "resync"]), "{seen:?}");
|
||||
let btc = quotes.iter().find(|q| q.key.as_str() == "BTC").unwrap();
|
||||
assert_eq!(btc.px, 101.0, "the fresh snapshot carries prices from after the outage");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn requests_are_bounded_and_time_out() {
|
||||
let s = Setup::new(|c| {
|
||||
c.session.max_requests_in_flight = 2;
|
||||
c.session.request_timeout = Duration::from_secs(1);
|
||||
})
|
||||
.await;
|
||||
// A market whose history the mock serves slowly.
|
||||
s.mock.push("allMids", "allMids", json!({"mids": {"BTC": "100.5", "ETH": "10.25", "@107": "30.0", "SLOW": "1.0"}}));
|
||||
let t0 = tokio::time::Instant::now();
|
||||
while !s.gw.hub().mids.contains("SLOW") {
|
||||
assert!(t0.elapsed() < WAIT);
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
let client = s.client(Encoding::Postcard);
|
||||
let req = || client.request(Rpc::History { key: MarketKey::new("SLOW"), tf: Timeframe::M1, end: None, limit: 5 });
|
||||
let t0 = tokio::time::Instant::now();
|
||||
let results = futures::future::join_all([req(), req(), req()]).await;
|
||||
let mut codes: Vec<String> = results.iter().map(|r| format!("{:?}", r.as_ref().unwrap_err().code)).collect();
|
||||
codes.sort();
|
||||
assert_eq!(codes, ["RateLimited", "Upstream", "Upstream"], "two time out, the third is refused: {results:?}");
|
||||
assert!(t0.elapsed() < Duration::from_secs(3), "nobody waited for the slow upstream");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn shutdown_closes_upstream_for_good() {
|
||||
let s = Setup::new(|_| {}).await;
|
||||
let client = s.client(Encoding::Postcard);
|
||||
let mut book = client.subscribe(TopicReq::Book { key: MarketKey::new("BTC"), depth: 5, group: None });
|
||||
s.mock.until("l2Book:BTC upstream", |m| m.subs().contains("l2Book:BTC")).await;
|
||||
assert_eq!(s.mock.open.load(Ordering::SeqCst), 2);
|
||||
let Setup { mock, gw } = s;
|
||||
tokio::time::timeout(WAIT, gw.shutdown()).await.expect("shutdown finishes");
|
||||
mock.until("every upstream connection closed", |m| m.open.load(Ordering::SeqCst) == 0).await;
|
||||
tokio::time::sleep(Duration::from_millis(1500)).await;
|
||||
assert_eq!(mock.open.load(Ordering::SeqCst), 0, "nothing reconnects after shutdown");
|
||||
// The client's session was closed too.
|
||||
while !matches!(next(&mut book).await, Event::Reconnecting) {}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn order_books_fall_back_while_the_server_is_away() {
|
||||
let s = Setup::new(|_| {}).await;
|
||||
let m = RemoteMarkets::new(&format!("ws://{}", s.gw.addr), tr_data::sample::SampleProvider::default());
|
||||
let push = async {
|
||||
s.mock.until("l2Book:BTC upstream", |mk| mk.subs().contains("l2Book:BTC")).await;
|
||||
s.mock.push("l2Book:BTC", "l2Book", book_msg("BTC", "100.0"));
|
||||
};
|
||||
let (book, ()) = tokio::join!(m.order_book("BTCUSD Curncy", 10), push);
|
||||
let book = book.unwrap();
|
||||
assert_eq!((book.source.as_str(), book.bids[0].0), (tr_data::remote::BOOK_SOURCE, 100.0));
|
||||
// The server goes away: the pane gets the fallback's book, not the last one the server sent.
|
||||
let Setup { mock: _mock, gw } = s;
|
||||
tokio::time::timeout(WAIT, gw.shutdown()).await.expect("shutdown finishes");
|
||||
let t0 = tokio::time::Instant::now();
|
||||
loop {
|
||||
let book = m.order_book("BTCUSD Curncy", 10).await.unwrap();
|
||||
if book.source != tr_data::remote::BOOK_SOURCE {
|
||||
assert!(book.simulated, "the sample provider's book");
|
||||
break;
|
||||
}
|
||||
assert!(t0.elapsed() < WAIT, "still the server's last book {:?} after it went away", book.bids[0]);
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn charts_hand_off_to_the_fallback_and_back_when_the_server_goes_and_returns() {
|
||||
let s = Setup::new(|_| {}).await;
|
||||
let addr = s.gw.addr;
|
||||
let m = RemoteMarkets::new(&format!("ws://{addr}"), tr_data::sample::SampleProvider::default().with_live_ticks());
|
||||
assert_eq!(m.history("BTCUSD Curncy", Timeframe::M1, None, 10).await.unwrap().len(), 10, "from the server");
|
||||
let mut feed = m.live("BTCUSD Curncy", Timeframe::M1).expect("the server's feed");
|
||||
s.mock.until("candle:BTC:1m upstream", |mk| mk.subs().contains("candle:BTC:1m")).await;
|
||||
// The server goes away and stays away: the feed ends, so the worker reloads the chart...
|
||||
let Setup { mock: _mock, gw } = s;
|
||||
tokio::time::timeout(WAIT, gw.shutdown()).await.expect("shutdown finishes");
|
||||
let gone = tokio::time::Instant::now();
|
||||
while tokio::time::timeout(Duration::from_secs(20), feed.recv()).await.expect("the feed ends").is_some() {}
|
||||
assert!(gone.elapsed() >= Duration::from_secs(9), "it gave the server a while to come back first");
|
||||
// ...from the fallback.
|
||||
let bars = m.history("BTCUSD Curncy", Timeframe::M1, None, 10).await.unwrap();
|
||||
assert_eq!(bars.len(), 10, "sample bars");
|
||||
let mut feed = m.live("BTCUSD Curncy", Timeframe::M1).expect("the fallback's feed");
|
||||
assert!(recv_bar(&mut feed).await.close > 0.0, "the fallback's live bars come through");
|
||||
// The server comes back on the same address: the fallback's feed ends, so the worker reloads
|
||||
// the chart from the server.
|
||||
let back = Setup::new(|c| c.bind = addr).await;
|
||||
while tokio::time::timeout(Duration::from_secs(30), feed.recv()).await.expect("the fallback feed ends").is_some() {}
|
||||
assert_eq!(m.history("BTCUSD Curncy", Timeframe::M1, None, 10).await.unwrap().len(), 10);
|
||||
let _feed = m.live("BTCUSD Curncy", Timeframe::M1).expect("the server's feed again");
|
||||
back.mock.until("candle:BTC:1m upstream again", |mk| mk.subs().contains("candle:BTC:1m")).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn charts_on_the_fallback_move_back_on_a_newer_connection_only() {
|
||||
// Nothing listens there yet, so the chart loads from the fallback.
|
||||
let addr = std::net::TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).unwrap().local_addr().unwrap();
|
||||
let m = RemoteMarkets::new(&format!("ws://{addr}"), tr_data::sample::SampleProvider::default().with_live_ticks());
|
||||
assert_eq!(m.history("BTCUSD Curncy", Timeframe::M1, None, 10).await.unwrap().len(), 10, "sample bars");
|
||||
// The server comes up before the chart starts its live feed...
|
||||
let s = Setup::new(|c| c.bind = addr).await;
|
||||
let t0 = tokio::time::Instant::now();
|
||||
while m.quotes(&["BTCUSD Curncy".into()]).await.unwrap()[0].last != 100.5 {
|
||||
assert!(t0.elapsed() < Duration::from_secs(20), "the client never connected");
|
||||
}
|
||||
// ...so the fallback's feed ends at once, and the chart reloads from the server.
|
||||
let mut feed = m.live("BTCUSD Curncy", Timeframe::M1).expect("the fallback's feed");
|
||||
let ended = tokio::time::timeout(Duration::from_secs(2), async { while feed.recv().await.is_some() {} }).await;
|
||||
assert!(ended.is_ok(), "the fallback's feed kept going with the server up");
|
||||
// A chart that falls back while the server is up (it failed the request) stays on the fallback:
|
||||
// reloading would only fail again.
|
||||
*s.mock.fail_candles.lock().unwrap() = 1000;
|
||||
assert_eq!(m.history("BTCUSD Curncy", Timeframe::M1, None, 10).await.unwrap().len(), 10, "sample bars");
|
||||
assert!(*s.mock.fail_candles.lock().unwrap() < 1000, "the server was asked and failed");
|
||||
let mut feed = m.live("BTCUSD Curncy", Timeframe::M1).expect("the fallback's feed");
|
||||
let ended = tokio::time::timeout(Duration::from_secs(2), async { while feed.recv().await.is_some() {} }).await;
|
||||
assert!(ended.is_err(), "the fallback's feed ended with no new connection");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn desktop_charts_backfill_bars_missed_in_an_outage() {
|
||||
let s = Setup::new(|_| {}).await;
|
||||
let m = RemoteMarkets::new(&format!("ws://{}", s.gw.addr), tr_data::sample::SampleProvider::default());
|
||||
let hist = m.history("BTCUSD Curncy", Timeframe::M1, None, 10).await.unwrap();
|
||||
assert_eq!(hist.len(), 10, "history from the server");
|
||||
let now = hist.last().unwrap().ts;
|
||||
let mut feed = m.live("BTCUSD Curncy", Timeframe::M1).expect("a live feed");
|
||||
s.mock.until("candle:BTC:1m upstream", |mk| mk.subs().contains("candle:BTC:1m")).await;
|
||||
let candle = |t: i64| json!({"t": t * 1000, "T": t * 1000 + 59_999, "s": "BTC", "i": "1m", "o": "1", "h": "1", "l": "1", "c": "1", "v": "1", "n": 1});
|
||||
// The stream is on a bar three minutes back when the upstream drops...
|
||||
let before = now - 180;
|
||||
s.mock.push("candle:BTC:1m", "candle", candle(before));
|
||||
let got = tokio::time::timeout(WAIT, feed.recv()).await.unwrap().unwrap();
|
||||
assert_eq!(got.ts, before);
|
||||
s.mock.drop_all();
|
||||
s.mock.until("candle resubscribed", |mk| mk.count("subscribe candle:BTC:1m") == 2).await;
|
||||
// ...and back on the current bar after. The first history request for the gap fails: the
|
||||
// live bar goes on at once, and the two bars in between follow when the retry works.
|
||||
*s.mock.fail_candles.lock().unwrap() = 1;
|
||||
s.mock.push("candle:BTC:1m", "candle", candle(now));
|
||||
let mut chart = vec![got];
|
||||
let mut ts = Vec::new();
|
||||
while !(ts.contains(&(before + 60)) && ts.contains(&(before + 120))) {
|
||||
let b = tokio::time::timeout(WAIT, feed.recv()).await.expect("the gap fills").unwrap();
|
||||
ts.push(b.ts);
|
||||
tr_core::Series::apply_live(&mut chart, b);
|
||||
}
|
||||
assert_eq!(ts.first(), Some(&now), "the live bar didn't wait for history: {ts:?}");
|
||||
assert_eq!(*s.mock.fail_candles.lock().unwrap(), 0, "the first attempt failed");
|
||||
assert_eq!(chart.iter().map(|b| b.ts).collect::<Vec<_>>(), [before, before + 60, before + 120, now], "no hole");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn desktop_backfill_keeps_one_request_in_flight_across_outages() {
|
||||
let s = Setup::new(|_| {}).await;
|
||||
let m = RemoteMarkets::new(&format!("ws://{}", s.gw.addr), tr_data::sample::SampleProvider::default());
|
||||
let now = m.history("BTCUSD Curncy", Timeframe::M1, None, 10).await.unwrap().last().unwrap().ts;
|
||||
let mut feed = m.live("BTCUSD Curncy", Timeframe::M1).expect("a live feed");
|
||||
s.mock.until("candle:BTC:1m upstream", |mk| mk.subs().contains("candle:BTC:1m")).await;
|
||||
let candle = |t: i64| json!({"t": t * 1000, "T": t * 1000 + 59_999, "s": "BTC", "i": "1m", "o": "1", "h": "1", "l": "1", "c": "1", "v": "1", "n": 1});
|
||||
s.mock.push("candle:BTC:1m", "candle", candle(now - 300));
|
||||
let mut chart = vec![recv_bar(&mut feed).await];
|
||||
// History is down, and each failed request hangs for a while: two outages in a row must
|
||||
// not stack up two retry loops.
|
||||
*s.mock.fail_candles.lock().unwrap() = 1000;
|
||||
*s.mock.fail_hold.lock().unwrap() = Duration::from_secs(4);
|
||||
s.mock.candles_peak.store(0, Ordering::SeqCst);
|
||||
for (k, t) in [(2, now - 180), (3, now - 60)] {
|
||||
s.mock.drop_all();
|
||||
s.mock.until("candle resubscribed", |mk| mk.count("subscribe candle:BTC:1m") == k).await;
|
||||
s.mock.push("candle:BTC:1m", "candle", candle(t));
|
||||
let b = recv_bar(&mut feed).await;
|
||||
assert_eq!(b.ts, t, "the live bar doesn't wait for history");
|
||||
tr_core::Series::apply_live(&mut chart, b);
|
||||
}
|
||||
assert!(s.mock.candles_inflight.load(Ordering::SeqCst) >= 1, "a backfill request is in flight");
|
||||
assert_eq!(s.mock.candles_peak.load(Ordering::SeqCst), 1, "one history request at a time per feed");
|
||||
// History comes back: one request fills both gaps.
|
||||
*s.mock.fail_candles.lock().unwrap() = 0;
|
||||
let want: Vec<i64> = (0..5).map(|i| now - 300 + i * 60).collect();
|
||||
while chart.iter().map(|b| b.ts).collect::<Vec<_>>() != want {
|
||||
tr_core::Series::apply_live(&mut chart, recv_bar(&mut feed).await);
|
||||
}
|
||||
assert_eq!(s.mock.candles_peak.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
/// A client that reconnects while the gateway is still starting (no instrument list or mids yet)
|
||||
/// keeps its subscriptions: each is marked stale, then goes live with a snapshot once the
|
||||
/// upstream is in. The gateway used to wait a few seconds, then end them with `Unavailable`.
|
||||
#[tokio::test]
|
||||
async fn subscriptions_made_while_the_gateway_starts_wait_for_the_upstream() {
|
||||
let s = Setup::starting(|c| c.session.ready_timeout = Duration::from_millis(200)).await;
|
||||
let client = s.client(Encoding::Postcard);
|
||||
let keys = ["BTC", "xyz:TSLA"].map(MarketKey::new).to_vec();
|
||||
let mut quotes = client.subscribe(TopicReq::Quotes { keys, max_hz: 10 });
|
||||
let mut book = client.subscribe(TopicReq::Book { key: MarketKey::new("BTC"), depth: 5, group: None });
|
||||
let mut trades = client.subscribe(TopicReq::Trades { key: MarketKey::new("xyz:TSLA") });
|
||||
// Well past the wait after which the gateway used to give up.
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
s.core_mids();
|
||||
s.xyz_mids();
|
||||
let (seen, data) = until_snapshot(&mut quotes).await;
|
||||
assert_eq!(seen, ["stale", "live"], "told it's waiting, then that it's live");
|
||||
let TopicData::Quotes(q) = data else { panic!("quotes") };
|
||||
let mut got: Vec<(&str, f64)> = q.iter().map(|q| (q.key.as_str(), q.px)).collect();
|
||||
got.sort_by(|a, b| a.0.cmp(b.0));
|
||||
assert_eq!(got, [("BTC", 100.5), ("xyz:TSLA", 401.5)]);
|
||||
s.mock.until("l2Book:BTC upstream", |m| m.subs().contains("l2Book:BTC")).await;
|
||||
s.mock.push("l2Book:BTC", "l2Book", book_msg("BTC", "100.0"));
|
||||
let (seen, data) = until_snapshot(&mut book).await;
|
||||
assert_eq!(seen, ["stale", "live"]);
|
||||
assert!(matches!(data, TopicData::Book(b) if b.bids[0].px == 100.0));
|
||||
let (_, data) = until_snapshot(&mut trades).await;
|
||||
assert_eq!(data, TopicData::Trades(vec![]));
|
||||
assert!(s.mock.subs().contains("trades:xyz:TSLA"));
|
||||
}
|
||||
|
||||
/// The gateway is ready once every venue's mids are in, not just the first venue's: a HIP-3
|
||||
/// market asked for while only the core dex has reported waits for its own dex, where it used
|
||||
/// to come back as an unknown market.
|
||||
#[tokio::test]
|
||||
async fn markets_on_a_dex_still_loading_are_not_unknown() {
|
||||
let s = Setup::starting(|_| {}).await;
|
||||
s.core_mids();
|
||||
let t0 = tokio::time::Instant::now();
|
||||
while !s.gw.hub().mids.contains("BTC") {
|
||||
assert!(t0.elapsed() < WAIT, "core mids never arrived");
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
let client = s.client(Encoding::Postcard);
|
||||
let mut q = client.subscribe(TopicReq::Quotes { keys: vec![MarketKey::new("xyz:TSLA")], max_hz: 10 });
|
||||
tokio::time::sleep(Duration::from_millis(300)).await;
|
||||
s.xyz_mids();
|
||||
let (_, data) = until_snapshot(&mut q).await;
|
||||
let TopicData::Quotes(q) = data else { panic!("quotes") };
|
||||
assert_eq!(q.iter().map(|q| (q.key.as_str(), q.px)).collect::<Vec<_>>(), [("xyz:TSLA", 401.5)]);
|
||||
}
|
||||
|
||||
/// A request made while the gateway starts fails as an upstream error, which clients retry, not
|
||||
/// as an unknown market: the desktop stops asking the server about a market it called unknown.
|
||||
#[tokio::test]
|
||||
async fn requests_made_while_the_gateway_starts_fail_as_retryable() {
|
||||
let s = Setup::starting(|c| c.session.ready_timeout = Duration::from_millis(200)).await;
|
||||
let client = s.client(Encoding::Postcard);
|
||||
let history = || {
|
||||
client.request(Rpc::History { key: MarketKey::new("BTCUSD Curncy"), tf: Timeframe::H1, end: None, limit: 10 })
|
||||
};
|
||||
let r = history().await;
|
||||
assert_eq!(r.as_ref().map_err(|e| e.code), Err(ErrorCode::Upstream), "{r:?}");
|
||||
s.core_mids();
|
||||
s.xyz_mids();
|
||||
assert!(s.gw.hub().ready(WAIT).await);
|
||||
let Ok(RpcResult::History(h)) = history().await else { panic!("history once ready") };
|
||||
assert_eq!(h.len(), 10);
|
||||
}
|
||||
|
||||
/// A venue whose mids never come holds readiness back only for `venue_grace`: after that, the
|
||||
/// next mids from any venue make the gateway ready without it.
|
||||
#[tokio::test]
|
||||
async fn a_silent_venue_delays_readiness_only_for_the_grace() {
|
||||
let s = Setup::starting(|c| c.hub.venue_grace = Duration::from_secs(1)).await;
|
||||
let readyz = || async {
|
||||
reqwest::get(format!("http://{}/readyz", s.gw.addr)).await.expect("readyz answers").status().as_u16()
|
||||
};
|
||||
s.core_mids();
|
||||
assert!(!s.gw.hub().ready(Duration::from_millis(300)).await, "still waiting for the xyz dex");
|
||||
assert_eq!(readyz().await, 503);
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
s.mock.push("allMids", "allMids", json!({"mids": {"BTC": "100.75", "ETH": "10.25", "@107": "30.0"}}));
|
||||
assert!(s.gw.hub().ready(WAIT).await);
|
||||
assert_eq!(readyz().await, 200);
|
||||
assert!(!s.gw.hub().mids.contains("xyz:TSLA"));
|
||||
}
|
||||
|
||||
/// A venue later than `venue_grace` doesn't make its markets unknown. Until it reports, a
|
||||
/// request for one fails as `Upstream`, which clients retry, a subscription waits for it like
|
||||
/// one made during startup, and a quotes subscription takes it in when its mids arrive.
|
||||
#[tokio::test]
|
||||
async fn markets_on_a_venue_later_than_the_grace_come_back() {
|
||||
let s = Setup::starting(|c| c.hub.venue_grace = Duration::from_millis(300)).await;
|
||||
s.core_mids();
|
||||
tokio::time::sleep(Duration::from_millis(400)).await;
|
||||
s.mock.push("allMids", "allMids", json!({"mids": {"BTC": "100.75", "ETH": "10.25", "@107": "30.0"}}));
|
||||
assert!(s.gw.hub().ready(WAIT).await, "ready without the xyz dex");
|
||||
let client = s.client(Encoding::Postcard);
|
||||
let tsla = || MarketKey::new("xyz:TSLA");
|
||||
let r = client.request(Rpc::History { key: tsla(), tf: Timeframe::H1, end: None, limit: 5 }).await;
|
||||
assert_eq!(r.as_ref().map_err(|e| e.code), Err(ErrorCode::Upstream), "{r:?}");
|
||||
let mut book = client.subscribe(TopicReq::Book { key: tsla(), depth: 5, group: None });
|
||||
let mut quotes = client.subscribe(TopicReq::Quotes { keys: vec![MarketKey::new("BTC"), tsla()], max_hz: 10 });
|
||||
let Event::Snapshot(TopicData::Quotes(q)) = next(&mut quotes).await else { panic!("quotes") };
|
||||
assert_eq!(q.iter().map(|q| q.key.as_str()).collect::<Vec<_>>(), ["BTC"]);
|
||||
let e = next(&mut book).await;
|
||||
assert!(matches!(e, Event::Status(TopicStatus::Stale { .. })), "the book waits for its venue: {e:?}");
|
||||
s.xyz_mids();
|
||||
// The quotes subscription takes the late market in...
|
||||
let Event::Delta(TopicDelta::Quotes(d)) = next(&mut quotes).await else { panic!("a quotes delta") };
|
||||
assert_eq!(d.iter().map(|q| (q.key.as_str(), q.px)).collect::<Vec<_>>(), [("xyz:TSLA", 401.5)]);
|
||||
// ...and the book subscription goes live, with no retry from the client.
|
||||
s.mock.until("l2Book:xyz:TSLA upstream", |m| m.subs().contains("l2Book:xyz:TSLA")).await;
|
||||
s.mock.push("l2Book:xyz:TSLA", "l2Book", book_msg("xyz:TSLA", "400.0"));
|
||||
let (seen, data) = until_snapshot(&mut book).await;
|
||||
assert_eq!(seen, ["live"]);
|
||||
assert!(matches!(data, TopicData::Book(b) if b.bids[0].px == 400.0));
|
||||
}
|
||||
+1
-1
@@ -37,7 +37,7 @@ npm run check
|
||||
plus the name, title and description used by the page's metadata. Update it with each release.
|
||||
`static/robots.txt` and `static/sitemap.xml` repeat the site URL.
|
||||
- `src/routes/download/+server.ts` answers `GET /download` with a 302 to
|
||||
`https://github.com/michaelshimeles/terminal/releases/latest/download/Terminal.dmg`.
|
||||
`https://github.com/michaelshimeles/terminal-rs/releases/latest/download/Terminal.dmg`.
|
||||
That URL works only once a GitHub release includes an asset named exactly `Terminal.dmg`.
|
||||
- The download buttons open `src/lib/DownloadDialog.svelte`, which asks for an email, sends it to
|
||||
`POST /download`, then starts the download with `GET /download`. The browser remembers that it
|
||||
|
||||
Generated
+2
-2
@@ -1,11 +1,11 @@
|
||||
{
|
||||
"name": "terminal-site",
|
||||
"name": "terminal-rs-site",
|
||||
"version": "0.0.1",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "terminal-site",
|
||||
"name": "terminal-rs-site",
|
||||
"version": "0.0.1",
|
||||
"dependencies": {
|
||||
"@upstash/redis": "^1.39.0",
|
||||
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"name": "terminal-site",
|
||||
"name": "terminal-rs-site",
|
||||
"private": true,
|
||||
"version": "0.0.1",
|
||||
"type": "module",
|
||||
|
||||
@@ -7,7 +7,7 @@ export const MIN_MACOS = '11';
|
||||
export const NAME = 'Terminal';
|
||||
/** `static/robots.txt` and `static/sitemap.xml` repeat this origin. */
|
||||
export const SITE_URL = 'https://terminal.rasmic.xyz';
|
||||
export const REPO_URL = 'https://github.com/michaelshimeles/terminal';
|
||||
export const REPO_URL = 'https://github.com/michaelshimeles/terminal-rs';
|
||||
export const DOWNLOAD_URL = `${REPO_URL}/releases/latest/download/Terminal.dmg`;
|
||||
|
||||
export const TITLE = `${NAME}: a fast market terminal for the Mac`;
|
||||
|
||||
Reference in new issue
Block a user