docs: mark fixed leak findings

# Conflicts:
#	src/Simplex/Messaging/Client.hs
#	tests/RSLVTests.hs

.github/workflows/build.yml

@@ -9,6 +9,10 @@ on:
       - "v*"
   pull_request:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ !startsWith(github.ref, 'refs/tags/v') }}
+
 jobs:
 
 # =============================
@@ -301,3 +305,30 @@ jobs:
             echo "All "$attempts" attempts failed."
             exit 1
           fi
+
+# =============================
+#      Resolver test job
+# =============================
+
+# The SNRC resolver is Python, so this job needs none of the Haskell toolchain
+# above and runs independently of it.
+
+  resolver-test:
+    name: "resolver (python)"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Clone project
+        uses: actions/checkout@v3
+
+      - name: Set up Python
+        # Matches the runtime stage of scripts/resolver/service/Dockerfile.
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+
+      - name: Install resolver dependencies
+        # Must match scripts/resolver/service/pyproject.toml.
+        run: python -m pip install "eth-hash[pycryptodome]>=0.7"
+
+      - name: Test
+        run: python -m unittest discover -s scripts/resolver/service -v

.gitignore

@@ -12,3 +12,4 @@ cabal.project.local~
 *.tix
 .coverage
 
+__pycache__/

apps/xftp-server/XFTPWeb.hs

@@ -58,7 +58,7 @@ xftpSubsts XFTPServerConfig {fileExpiration, logStatsInterval, allowNewFiles, ne
   [("smpConfig", Nothing), ("xftpConfig", Just "y")] <> substConfig <> serverInfoSubsts simplexmqSource information <> [("onionHost", strEncode <$> onionHost), ("iniFileName", Just "file-server.ini")]
   where
     substConfig =
-      [ ("fileExpiration", Just $ maybe "Never" (fromString . timedTTLText . ttl) fileExpiration),
+      [ ("fileExpiration", Just . fromString . timedTTLText . ttl $ fileExpiration),
         ("statsEnabled", Just . yesNo $ isJust logStatsInterval),
         ("newUploadsAllowed", Just . yesNo $ allowNewFiles),
         ("basicAuthEnabled", Just . yesNo $ isJust newFileBasicAuth)

bench/NetLag.hs

@@ -0,0 +1,111 @@
+{-# LANGUAGE DataKinds #-}
+{-# LANGUAGE FlexibleInstances #-}
+{-# LANGUAGE InstanceSigs #-}
+{-# LANGUAGE KindSignatures #-}
+{-# LANGUAGE ScopedTypeVariables #-}
+{-# LANGUAGE TypeApplications #-}
+
+-- | A latency-injecting Transport for the memory-leak bench.
+--
+-- 'LagTLS' is a newtype over 'TLS' that delegates every 'Transport' method, adding a
+-- configurable delay before each read and write and optionally swallowing writes. It is
+-- wire-identical to 'TLS' - 'transportName' is only used for thread labels and logging, so a
+-- peer speaking plain TLS interoperates with it unchanged.
+--
+-- Used as the destination relay's listener transport so that proxy->relay traffic can be
+-- delayed without touching the proxy, the clients, or any production code:
+--
+-- > withSmpServerConfigOn (transport @TLS) proxyCfg testPort $ \_ ->
+-- >   withSmpServerConfigOn (transport @LagTLS) cfgJ2 testPort2 $ \_ -> ...
+--
+-- 'setDropSnd' keeps the TLS session healthy while responses vanish, which is what the
+-- proxy sentCommands leak needs: the relay must stay up and stop answering, so the proxy's
+-- RFWD requests time out without the client being torn down.
+--
+-- Delays apply to the SMP handshake as well as to post-handshake traffic (both go through
+-- cGet/cPut), so phases that need an established session must connect first and arm the lag
+-- afterwards.
+module NetLag
+  ( LagTLS,
+    setLag,
+    setDropSnd,
+    setDropEvery,
+    clearLag,
+  )
+where
+
+import Control.Concurrent (threadDelay)
+import Control.Concurrent.STM
+import Control.Monad (unless, when)
+import Data.ByteString.Char8 (ByteString)
+import Simplex.Messaging.Transport
+import System.IO.Unsafe (unsafePerformIO)
+
+newtype LagTLS (p :: TransportPeer) = LagTLS (TLS p)
+
+data LagCtl = LagCtl
+  { rcvDelayUs :: TVar Int,
+    sndDelayUs :: TVar Int,
+    dropSnd :: TVar Bool,
+    -- drop every nth write, 0 disables. Distinct from dropSnd: dropping everything makes the
+    -- peer's monitor eventually tear the session down, while dropping a fraction keeps the
+    -- session healthy indefinitely because any reply resets its counters.
+    dropEvery :: TVar Int,
+    sndSeq :: TVar Int
+  }
+
+-- A single process-wide control: getTransportConnection has nowhere to thread per-listener
+-- state through, and the bench runs one lagged relay at a time.
+lagCtl :: LagCtl
+lagCtl =
+  unsafePerformIO $
+    LagCtl <$> newTVarIO 0 <*> newTVarIO 0 <*> newTVarIO False <*> newTVarIO 0 <*> newTVarIO 0
+{-# NOINLINE lagCtl #-}
+
+-- | One-way delays in microseconds: inbound (peer -> this transport) and outbound.
+setLag :: Int -> Int -> IO ()
+setLag rcv snd' = atomically $ do
+  writeTVar (rcvDelayUs lagCtl) rcv
+  writeTVar (sndDelayUs lagCtl) snd'
+
+-- | Silently discard everything written. The session stays open and the peer keeps waiting.
+setDropSnd :: Bool -> IO ()
+setDropSnd b = atomically $ writeTVar (dropSnd lagCtl) b
+
+-- | Silently discard every nth write, passing the rest. 0 disables.
+setDropEvery :: Int -> IO ()
+setDropEvery n = atomically $ writeTVar (dropEvery lagCtl) n
+
+clearLag :: IO ()
+clearLag = setLag 0 0 >> setDropSnd False >> setDropEvery 0
+
+delayBy :: TVar Int -> IO ()
+delayBy v = do
+  d <- readTVarIO v
+  when (d > 0) $ threadDelay d
+
+instance Transport LagTLS where
+  transportName _ = "LagTLS"
+  transportConfig (LagTLS t) = transportConfig t
+  getTransportConnection cfg sent chain ctx = LagTLS <$> getTransportConnection cfg sent chain ctx
+  certificateSent (LagTLS t) = certificateSent t
+  getPeerCertChain (LagTLS t) = getPeerCertChain t
+  getSessionALPN (LagTLS t) = getSessionALPN t
+  tlsUnique (LagTLS t) = tlsUnique t
+  closeConnection (LagTLS t) = closeConnection t
+
+  cGet :: LagTLS p -> Int -> IO ByteString
+  cGet (LagTLS t) n = delayBy (rcvDelayUs lagCtl) >> cGet t n
+
+  cPut :: LagTLS p -> ByteString -> IO ()
+  cPut (LagTLS t) s = do
+    delayBy (sndDelayUs lagCtl)
+    drop' <- atomically $ do
+      always <- readTVar (dropSnd lagCtl)
+      every <- readTVar (dropEvery lagCtl)
+      i <- stateTVar (sndSeq lagCtl) $ \n -> let n' = n + 1 in (n', n')
+      pure $ always || (every > 0 && i `mod` every == 0)
+    unless drop' $ cPut t s
+
+  getLn :: LagTLS p -> IO ByteString
+  getLn (LagTLS t) = delayBy (rcvDelayUs lagCtl) >> getLn t

docs/leak-findings.md

@@ -0,0 +1,517 @@
+# SMP server leak findings
+
+Found with `bench/MemBench.hs` using a proxy-plus-relay topology and a transport that adds latency
+and drops replies. Journal store and PostgreSQL gave the same numbers except where a row says
+otherwise.
+
+Twelve findings: three proxy-path memory leaks (Leak 1-3), two `forkClient` bugs (Bug 3-4), an
+unauthenticated resolver fan-out (Bug 5), five PostgreSQL-backend costs (Bug 6-10), and a stuck proxy
+session (Bug 11). The TLS/TCP stack is clean (last two sections).
+
+Memory at production scale, with measurements after these fixes: `smp-server-memory.md`.
+
+## Overview
+
+| # | Issue | Reachable | Backend | Evidence | Status |
+| --- | --- | --- | --- | --- | --- |
+| Leak 1 | Forwarded commands not removed on timeout | Client, pre-auth (PRXY) | all | measured | fixed, `sh/fix-proxy-leak` |
+| Leak 2 | Failed relay connects not cleared | Client, pre-auth (PRXY) | all | measured | present |
+| Leak 3 | Proxy relay queue has no reader | Client, pre-auth (PRXY) | all | measured | fixed, #1839 |
+| Bug 3 | Proxy concurrency limit does nothing | Client, pre-auth (PRXY) | all | measured | fixed, `sh/fix-rslv-fanout` |
+| Bug 4 | Stale `endThreads` entry on fast finish | Client, limited window | all | isolation only | present |
+| Bug 5 | RSLV resolver fan-out | Client, if `[NAMES]` on | all | measured | fixed, `sh/fix-rslv-fanout` |
+| Bug 6 | SEND is three DB transactions | authenticated SEND | postgres | code review | present |
+| Bug 7 | Per-transmission verify and DB lookup | Client, pre-auth | postgres | code review | present |
+| Bug 8 | Service handshake grows `services` table | Client, pre-auth | postgres | code review | present |
+| Bug 9 | Prometheus scrape scans everything | internal, periodic | postgres | code review | present |
+| Bug 10 | Subscription churn serialized | authenticated SUB | all | code review | present |
+| Bug 11 | Proxy never drops a stuck relay session | Client, pre-auth (PRXY) | all | reproduced | fixed, `sh/fix-proxy-leak` |
+
+Leak 1, Leak 2, Leak 3, Bug 3, and Bug 11 share one entry point. `PRXY` is unauthenticated unless
+`newQueueBasicAuth` is set (`Server.hs:1534`) and it names an arbitrary destination, so a client can
+point the proxy at a relay it controls.
+
+---
+
+## Leak 1: forwarded commands are never removed on timeout
+
+| | |
+| --- | --- |
+| Reachable | any client, pre-auth via PRXY |
+| Trigger | relay replies to some forwards, drops others |
+| Cost | ~20 KiB per stuck forward, unbounded |
+| Measured | yes, 64 to 1280 entries over 20 min on one session |
+| Status | present |
+
+### How it works
+
+When the agent forwards a command it stores the in-flight command in `sentCommands`, keyed by
+correlation id, so the reply can be matched later. The entry is added in `mkTransmission_`
+(`Client.hs:1414`) and removed in `processMsg` (`Client.hs:706`) when the reply arrives. Each `RFWD`
+entry holds the whole forwarded command, 16226 bytes (`Protocol.hs:319`).
+
+### The bug
+
+The timeout path, `getResponse` (`Client.hs:1378`), does not hold the map, so a forward that times
+out is never removed. The session does not die either: `monitor` (`Client.hs:668`) closes the client
+only after `timeoutErrorCount >= smpPingCount` with nothing received for 900s, and `receive`
+(`Client.hs:663`) resets both counters on any inbound transmission.
+
+### Impact
+
+About 20 KiB per unanswered forward. How long it is held depends on the relay.
+
+| Relay behaviour | Result |
+| --- | --- |
+| slow but still replies | not a leak here, the late reply deletes the entry (but see Leak 3) |
+| goes fully silent | bounded, `monitor` closes the client at ~20 min |
+| replies to some, drops others | unbounded |
+
+The third case is the leak. Any arriving reply resets `lastReceived` and `timeoutErrorCount`, so the
+drop condition is never met. Measured with 1 in 3 relay writes dropped: `proxy_sentCommands` climbed
+64 to 1280 over 20 minutes, linear at 64/min, zero disconnects. That is ~1.3 MiB/min on one session.
+Traffic has to be ongoing; stop the flood and it is reclaimed after 20 minutes.
+
+The ntf server runs the same client code and is exposed the same way through unanswered `NSUB`.
+Measured with `subtmo 200`: 200 timed-out subscriptions, `sentCommands` 0 to 200, ~1.76 KiB each. At
+the ntf batch size of 1360 that is ~2.3 MiB per unanswered batch.
+
+Pings do not help. Subscribe paths call `enablePings` (`Client.hs:953`) and the proxy send path does
+not, but in the unbounded case replies are arriving anyway and reset the counters.
+
+### Fix
+
+Deleting on timeout is wrong: the agent still needs late replies. `processMsg` forwards them as
+`STResponse` (`Client.hs:713`) and the agent acts on them, so a late `OK`/`SOK` to a `SUB` brings a
+connection back up and a late `MSG` is processed. Deleting would turn both into `STUnexpectedError`.
+
+Delete by age instead. Record the insert time on `Request` and remove `pending == False` entries once
+a late reply is no longer useful. The cutoff needs the agent's recovery behaviour measured, which is
+not done here. Two paths can be deleted immediately because no reply is coming: `sendRecv` returns
+early on transport error (`Client.hs:1364`) and on an oversized block (`Client.hs:1366`) without
+sending.
+
+---
+
+## Leak 2: failed relay connects are never cleared
+
+| | |
+| --- | --- |
+| Reachable | any client, pre-auth via PRXY |
+| Trigger | forward to distinct dead addresses |
+| Cost | ~19 KiB per address, held for the process lifetime |
+| Measured | yes, 300 entries from 300 dead addresses |
+| Status | present |
+
+### How it works
+
+Relay clients are cached in `smpClients`. A successful connect caches the client; a failed connect
+caches the error as `Left (error, expiry)` (`Client/Agent.hs:277`) so repeat lookups fail fast until
+the expiry passes.
+
+### The bug
+
+Nothing removes a failed entry on a timer. It is dropped only when the same server is looked up again
+and found expired (`:253`, `:414`). The other removals are `clientDisconnected` (`:314`, connected
+clients only) and shutdown (`:430`). Conditional on `persistErrorInterval > 0`; at 0 the entry is
+removed at once, but production sets 30 (`Server/Main.hs:608`).
+
+### Impact
+
+Host, port and key hash come from the client, so distinct addresses are unlimited. Measured:
+`proxy_smpClients = 300` after 300 dead addresses, ~19 KiB each, never freed while the process runs.
+1000 entries created in about 1s, so ~19 MiB/s when the address refuses immediately. An address that
+accepts nothing and never answers waits out the 45s connect timeout, which slows it down.
+
+### Fix
+
+Scan the map on a timer and remove entries past their expiry. The timestamp is already stored.
+
+---
+
+## Leak 3: the proxy's relay message queue has no reader
+
+> [!TIP]
+> Fixed in master by #1839 (`7d0820dd`), which took the approach proposed below. `msgQ` is now
+> `Maybe (TBQueue ...)` (`Client/Agent.hs:143`), the proxy agent is built with `msgQSize = Nothing`
+> (`Server/Main.hs:607`), and `sendMsg` logs late replies instead of enqueuing them (`Client.hs:722`).
+> The analysis below is kept for context.
+
+| | |
+| --- | --- |
+| Reachable | any client, pre-auth via PRXY |
+| Trigger | relay replies land after the 30s RFWD timeout |
+| Cost | permanent proxy stall once the queue fills |
+| Measured | yes, `msgqfill` phase |
+| Status | fixed, #1839 |
+
+### How it works
+
+`newSMPClientAgent` created one `msgQ` and gave that same queue to every relay client. The ntf server
+reads its copy (`Notifications/Server.hs`), but the SMP server never read its own: `receiveFromProxyAgent`
+reads `agentQ` only.
+
+### The bug
+
+The queue filled from late replies. `processMsg` routes a response to `msgQ` when the request is
+still in `sentCommands` but `pending` is already `False` (`Client.hs:713`), so every reply arriving
+after the proxy's 30s RFWD timeout left an entry that nothing removed. When full, `processMsgs`
+blocked in `writeTBQueue` on the `process` thread, the only reader of `rcvQ`, so the proxy stopped
+handling responses entirely.
+
+### Impact
+
+Measured with the `msgqfill` phase: 4 forwards at 40s each way so replies land after the timeout, then
+lag cleared and 3 more attempted. Only `msgQSize` differs.
+
+| `msgQSize` | `proxy_msgQ` at end | `sentCommands` at end | recovery forwards |
+| --- | --- | --- | --- |
+| 2 | 2 (at cap) | 4 and climbing | 0 of 3 |
+| 2048 (production) | 4 | 0 | 3 of 3 |
+
+The queue was never emptied, and once full the stall was permanent: the recovery forwards ran with no
+latency and got nothing back. One `msgQ` per agent and one `ProxyAgent` per server, so one slow relay
+stalled the proxy for every relay it talked to. Someone controlling the destination relay needed only
+2048 late replies. This also corrects Leak 1's "slow relay is not a leak" row: those late replies are
+what piled up here.
+
+---
+
+## Bug 3: proxy concurrency limit does nothing
+
+| | |
+| --- | --- |
+| Reachable | any client, pre-auth via PRXY |
+| Trigger | concurrent PFWDs on one connection |
+| Cost | none directly, uncaps Leak 1's growth rate |
+| Measured | yes, `conclimit` phase |
+| Status | present |
+
+### How it works
+
+`forkCmd` (`Server.hs:1591`) is meant to cap in-flight forked commands at `serverClientConcurrency`.
+`wait` blocks until a slot is free and `signal` releases it.
+
+### The bug
+
+The code is `bracket_ wait signal . forkClient clnt label $ action` (`Server.hs:1592`). `.` binds
+tighter than `$`, so it parses as `bracket_ wait signal (forkClient ... action)`. `forkClient`
+returns as soon as the thread is spawned, so `signal` runs at spawn time, not at completion. Only
+forking is serialized. Measured with `conclimit 8` and `serverClientConcurrency = 1`, all eight PFWDs
+ran at once (`first=20.0s last=20.0s spread=0.0s`); enforced, each would hold the slot for the 30s
+RFWD timeout. `procThreads` also reads near zero at any load. The counter is per-connection, so even
+when fixed the cap is per-client, not global.
+
+### Fix
+
+```haskell
+wait >> forkClient clnt label (action `finally` signal)
+```
+
+This enables the limit for the first time. Default is 32 and `wait` blocks the client's whole command
+loop when hit, so check that value first.
+
+---
+
+## Bug 4: stale endThreads entry when a command finishes fast
+
+| | |
+| --- | --- |
+| Reachable | client, narrow timing window |
+| Trigger | forked child finishes before it is registered |
+| Cost | ~320 bytes per stale entry, misleading counter |
+| Measured | isolation only, not against a running server |
+| Status | present |
+
+### How it works
+
+`forkClient` (`Server.hs:1482`) registers the child in `endThreads` for shutdown tracking. It forks
+first (`:1485`), the child deletes its own key on exit (`:1487`), and the parent inserts the weak
+thread id afterward (`:1488`).
+
+### The bug
+
+If the child finishes before the parent's insert, the delete finds no key and the later insert is
+never undone, leaving a permanent entry. Reproduced over 100k forks: 20% stale at `-N1`, 13% at `-N4`,
+~320 bytes each. `deRefWeak` returns `Nothing` for all, so no thread is retained.
+
+What decides the race is whether the child yields the CPU back, not how long it runs:
+
+| Child does | -N1 | -N4 |
+| --- | --- | --- |
+| nothing | 17.5% | 10.7% |
+| spins 1us | 19.2% | 9.7% |
+| spins 100us | 17.8% | 9.8% |
+| one failing `connect()` | 0% | 0.1% |
+
+A child that spins keeps the CPU; the parent wins only when the child hits a syscall, safe FFI call,
+or STM retry. Of the three call sites, `forkCmd` (`PFWD`/`PRXY`/`RSLV`) and `deliverServiceMessages`
+all do IO and yield, and the service path runs at most once per connection. Only
+`sendPendingEvtsThread.queueEvts` can finish without yielding, and it forks at most twice per client
+per 15s (`pendingENDInterval`) and only when the client's `sndQ` was full then emptied.
+`clientDisconnected` clears the map, so nothing outlives the session. The real cost is that the
+`endThreads` counter mixes stale entries with commands still running.
+
+### Fix
+
+```haskell
+atomically $ modifyTVar' endThreads $ IM.insert tId Nothing        -- before forkIO
+atomically $ modifyTVar' endThreads $ IM.adjust (const (Just w)) tId
+```
+
+`adjust` is a no-op if the action already removed the key.
+
+---
+
+## Bug 5: unauthenticated RSLV resolver fan-out
+
+| | |
+| --- | --- |
+| Reachable | any client, pre-auth, when `[NAMES]` is enabled |
+| Trigger | a block full of RSLV commands |
+| Cost | up to 1000 concurrent outbound TLS handshakes per connection |
+| Measured | yes, `testRslvFanOut` (64 in flight, asserts <= 8) |
+| Status | present |
+
+### How it works
+
+`RSLV` resolves a name over an outbound HTTP/TLS request. It is verified as unauthenticated
+(`Server.hs:1393`, `vc SResolver (RSLV _) = VRVerified Nothing`) and each command forks one request
+(`Server.hs:1638`). One 16 KB block carries ~255 RSLVs. Off by default, on only with `[NAMES]`.
+
+### The bug
+
+The only bound is `serverResolverConcurrency` (default 1000, `Env/STM.hs:255`), applied through the
+per-client `procThreads` counter via the same broken `forkCmd` path as Bug 3, so it is neither global
+nor effective. `managerConnCount = 10` (`HttpResolver.hs:88`) sizes the keep-alive pool, not
+concurrency, so excess requests open extra connections rather than blocking. `resolveName` has no
+cache (`Server/Names.hs:61`). One connection drives up to `serverResolverConcurrency` concurrent
+outbound handshakes, sockets and FDs; more connections multiply it with no global bound.
+
+### Fix
+
+Add a global resolver-concurrency limit, a shared semaphore in `NamesEnv` acquired around the
+outbound call, separate from the per-client counter, and lower the default. A result cache would also
+cut repeat lookups.
+
+---
+
+The findings below are from code review of the PostgreSQL backend (`store_messages: database`), not
+bench-measured.
+
+## Bug 6: every SEND is three DB transactions behind a pool of 10
+
+| | |
+| --- | --- |
+| Reachable | authenticated SEND |
+| Trigger | normal message sending |
+| Cost | SEND rate capped at ~`poolSize / 3` per transaction latency |
+| Evidence | code review |
+| Status | present |
+
+### How it works
+
+With `store_messages: database` the queue store runs `useCache = False` (`MsgStore/Postgres.hs:100`),
+so each command hits the DB.
+
+### The bug
+
+A SEND does three separate transactions: load the queue (`QueueStore/Postgres.hs:230`),
+`delete_expired_msgs` (`MsgStore/Postgres.hs:289`), then `write_message` (`MsgStore/Postgres.hs:193`).
+`expireMessagesOnSend` defaults to `True` (`Main.hs:563`, `Server.hs:2118`), so the middle one runs on
+every SEND to a non-empty queue. All draw from one pool of `poolSize` (default 10, `Main/Init.hs:44`).
+A second pool, `dbPriorityPool`, is opened (`Agent/Store/Postgres.hs:59`) but the SMP server never
+uses it. Max SEND rate is about `poolSize / (3 x per-transaction latency)` regardless of client count;
+past that all clients block on the pool, and half the opened backends sit idle.
+
+### Fix
+
+Fewer transactions per SEND; make `expireMessagesOnSend` cheaper or default off; drop or use the
+priority pool.
+
+---
+
+## Bug 7: unauthenticated batched crypto and per-transmission DB lookup
+
+| | |
+| --- | --- |
+| Reachable | any handshake-completing peer, pre-auth |
+| Trigger | one 16 KB block of SENDs to unknown queues |
+| Cost | ~130-250 asymmetric verifications and DB SELECTs per block |
+| Evidence | code review |
+| Status | present |
+
+### How it works
+
+Recipient and Notifier commands are batch parties (`Protocol.hs:420`), so their queue lookups are
+batched. SEND is not, so it takes the per-transmission path
+`mapM (\t -> verifyTransmission ...)` (`Server.hs:1281`).
+
+### The bug
+
+Each transmission runs one crypto op and, with `useCache = False`, one `getQueueRec` SELECT
+(`Server.hs:1361`). For an unknown queue, `dummyVerifyCmd` (`Server.hs:1459`) still runs a real
+Ed25519/Ed448 verify or an X25519 DH. A 16 KB block holds up to 254 transmissions (`Protocol.hs:2292`,
+one-byte count), so one write by any peer that completed the handshake costs ~130-250 asymmetric
+verifications and ~130-250 SELECTs. The attacker chooses the auth type and that the queue is absent.
+
+### Fix
+
+Batch the sender lookups as Recipient and Notifier already are; cap unknown-queue verifications per
+block.
+
+---
+
+## Bug 8: unauthenticated service handshake grows the services table
+
+| | |
+| --- | --- |
+| Reachable | any client, pre-auth |
+| Trigger | a fresh self-signed service cert per handshake |
+| Cost | one persistent `services` row and one X509 verify per connection |
+| Evidence | code review |
+| Status | present |
+
+### How it works
+
+The SMP handshake accepts a self-signed service chain (`CCSelf`, `Transport.hs:759`) and verifies the
+supplied cert (`Transport.hs:763`). `getClientService` (`Server.hs:864`) then calls `getCreateService`.
+
+### The bug
+
+`getCreateService` inserts a `services` row on any new fingerprint (`QueueStore/Postgres.hs:478`), and
+a fresh self-signed cert is a new fingerprint every time. The only guard is a role check on an
+existing fingerprint, so each connection can create one unbounded, persistent row plus one X509 verify,
+all before authentication.
+
+### Fix
+
+Require the service to be pre-registered, or authenticate and rate-limit service creation.
+
+---
+
+## Bug 9: Prometheus scrape scans all queues and folds all subscriptions
+
+| | |
+| --- | --- |
+| Reachable | internal, fires every `prometheus_interval` |
+| Trigger | Prometheus enabled |
+| Cost | scales with queue count and live subscriptions, every scrape |
+| Evidence | code review |
+| Status | present |
+
+### How it works
+
+On each scrape the server refreshes its metrics from live state rather than incremental counters.
+
+### The bug
+
+`getEntityCounts` runs six `COUNT(1)` scans over `msg_queues` and `services`
+(`QueueStore/Postgres.hs:154`, called at `Server.hs:813`), and `getDeliveredMetrics` folds over all
+clients times all their subscriptions in memory (`Server.hs:837`, called at `Server.hs:825`). Cost
+scales with the largest dimensions and recurs every interval while Prometheus is on.
+
+### Fix
+
+Maintain the counts incrementally; drop the per-scrape full fold.
+
+---
+
+## Bug 10: subscription changes serialize through one thread and one map
+
+| | |
+| --- | --- |
+| Reachable | authenticated SUB |
+| Trigger | subscription churn, batched SUB of many queues |
+| Cost | throughput bounded by one thread and one contended TVar |
+| Evidence | code review |
+| Status | present |
+
+### How it works
+
+Subscription state lives in `queueSubscribers`, one `Map` in one TVar (`Env/STM.hs:377`). Changes are
+enqueued to `subQ` and applied by a single `serverThread` (`Server.hs:284`).
+
+### The bug
+
+All churn passes through that one thread and one TVar, and a batched SUB of N queues is N separate
+writes to `subQ` (`Server.hs:1845`), so subscription throughput does not scale with cores or clients.
+
+### Fix
+
+Shard the subscriber map, or batch `subQ` events per client.
+
+---
+
+## Bug 11: proxy never drops a stuck relay session
+
+| | |
+| --- | --- |
+| Reachable | any client, pre-auth via PRXY |
+| Trigger | relay stops answering forwards once the session is open |
+| Cost | forwards keep failing, no recovery until the ~20 min monitor drop |
+| Evidence | `testProxyForwardTimeoutStuckSession` (pending spec) |
+| Status | present |
+
+### How it works
+
+Shares Leak 1's mechanism. The forward path never calls `enablePings`, so a dead relay session is
+dropped only by `monitor` (`Client.hs:668`), which closes the client when `timeoutErrorCount >=
+smpPingCount` (3, `Client.hs:439`) and the last receive is older than `recoverWindow` (900s,
+`Client.hs:685`). `receive` (`Client.hs:663`) resets both counters on any inbound byte.
+
+### The bug
+
+A broken session is not dropped, so the proxy keeps forwarding to it and every forward fails. This is
+the reported symptom: `Error forwarding to relay ... PCEResponseTimeout` continued after the relay was
+restarted, because the proxy held the old session instead of reconnecting. Reproduced by
+`testProxyForwardTimeoutStuckSession` (`SMPProxyTests.hs:499`): 10 forwards on one timed-out session
+expecting a `PROXY NO_SESSION` drop, but all 10 return `BROKER TIMEOUT`. Unlike Leak 1 this is
+availability, not memory; the same persistent session is what makes Leak 1 unbounded.
+
+### Fix
+
+Drop the relay session after a bounded number of forward timeouts so the next forward returns
+`NO_SESSION` and the agent reconnects, instead of waiting out the 900s receive window. Enabling pings
+on the forward path would also let `monitor` detect the dead session.
+
+---
+
+## Clean: TLS/TCP stack
+
+200 connections opened at once, closed, then measured again.
+
+| Test | Peak per conn | After 25s |
+| --- | --- | --- |
+| TCP connect, never start TLS | 48.2 KiB | 0.31 KiB |
+| TLS done, no SMP handshake | 203.1 KiB | 0.71 KiB |
+| handshake done, one byte, then quiet | 264.6 KiB | 0.87 KiB |
+
+All recovered. Also clean: 400 connect/disconnect rounds, and steady forwarding at 50ms each way.
+
+Measure well after closing. At +5s the middle two still read ~120 KiB per connection, which looks like
+a 24 MiB leak but is just connections still closing. A number that keeps falling is being freed; one
+that stops above where it started is leaked.
+
+The peaks still matter. 200 abandoned half-open connections hold ~40 MiB for ~25s with no
+authentication. A client that finishes the handshake then sends one byte holds ~265 KiB for as long as
+it stays connected, because there is no read timeout: `transportTimeout` is hardcoded `Nothing`
+(`Transport/Server.hs:104`).
+
+## Clean: connectivity and sockets under latency
+
+Latency set with `BENCHLAG_MS` on `proxyfwd`, one way. Sockets counted from `/proc/<pid>/fd`.
+
+| Lag each way | Delivered | Sockets | Relay connects | Reconnects | Timeouts |
+| --- | --- | --- | --- | --- | --- |
+| 0ms | 12/12 | 8 | 1 | 0 | 0 |
+| 500ms | 10/10 | 8 | 1 | 0 | 0 |
+| 5s | 6/6 | 8 | 1 | 0 | 0 |
+| 16s | 4/4 | 8 | 1 | 0 | 0 |
+| 40s | 0/2 | 8 | 1 | 0 | 1 |
+
+Nothing builds up. The socket count is the same whether forwards succeed or time out, the session is
+opened once and reused, and there are no reconnects at any latency. This is why Leak 1 has no upper
+bound: the session holding the stuck entries never closes. Forwards work to 16s each way and fail at
+40s because of the 30s RFWD timeout; the exact cutoff is not measured, since the test transport adds
+its delay per read/write rather than per message.

docs/smp-server-db-load.md

@@ -0,0 +1,200 @@
+# SMP server database load: root causes and fixes
+
+## Summary
+
+The recurring `pMsgFwdsOwn_pErrorsOther` spikes are a symptom of chronic disk saturation of the
+`smp_server` PostgreSQL database, not a proxy or forwarding bug. The baseline read load is the
+Prometheus scrape running whole-table `COUNT` scans of `msg_queues` (48.5M rows) every 30 seconds,
+compounded by the message-expiration sweep. The write load comes from indexing `updated_at`:
+`updateQueueTime` stamps it on the first send to each active queue per UTC day, and these non-HOT
+updates bloat the indexes to 104 GB and amplify WAL to about 425 GB per day (37% full-page images,
+~31% from `updateQueueTime`). Because the stamp is day-granular, this write burst clusters at 00:00 UTC,
+tipping the saturated disk into the observed spike and loading the pgBackRest backup repository.
+
+Code fixes replace the largest metric scan with an estimate and index the sweep and notifier counts.
+Reclaiming the index bloat, which also clears the remaining metric scan, and keeping it down are
+operator actions, because PostgreSQL never shrinks indexes automatically.
+
+## Environment
+
+The affected deployment runs the pure PostgreSQL backend (`store_queues: database`,
+`store_messages: database`), schema `smp_server`, `db_pool_size 20`, `prometheus_interval 30`,
+`[NAMES]` enabled, own-server proxying enabled, and forwards routed over a SOCKS proxy.
+
+`msg_queues`: 48.5M live rows, 12.3% dead tuples, 14 GB heap, 104 GB indexes, 118 GB total.
+
+## Evidence
+
+`pg_stat_statements` over a 5 minute window, ordered by disk reads (`track_io_timing` was off, so
+`shared_blks_read` is the disk proxy; the two `EXPLAIN ANALYZE` rows are manual and excluded):
+
+| query | calls | mean ms | reads | share of all reads |
+| --- | --- | --- | --- | --- |
+| `getEntityCounts` (the six-`COUNT` metric query) | 30 | 111,456 | 990 GB | 70.7% |
+| queue-record load (`SELECT recipient_id, …`) | 47,027 | 121 | 89 GB | 6.3% |
+| expiration sweep batch (`array_agg` in `expire_old_messages`) | 13 | 233,143 | 62 GB | 4.5% |
+| message peek (`DISTINCT ON (recipient_id)`) | 46,932 | 76 | 43 GB | 3.0% |
+| `updateQueueTime` (`UPDATE msg_queues SET updated_at`) | 172,885 | 4 | 7.7 GB | 0.5% |
+| `write_message` | 110,911 | 5 | 7.0 GB | 0.5% |
+
+`EXPLAIN (ANALYZE, BUFFERS)` of each `msg_queues` count inside `getEntityCounts`:
+
+| count | plan | time |
+| --- | --- | --- |
+| `queue_count` (`deleted_at IS NULL`) | Parallel Seq Scan | 30 s |
+| `notifier_count` (`deleted_at IS NULL AND notifier_id IS NOT NULL`) | Parallel Seq Scan | 32.6 s |
+| `ntf_service_queues_count` (`ntf_service_id IS NOT NULL AND deleted_at IS NULL`) | Parallel Seq Scan | 30.2 s |
+| `rcv_service_queues_count` (`rcv_service_id IS NOT NULL AND deleted_at IS NULL`) | Index Only Scan | 1.4 ms |
+
+The expiration sweep batch: 229 s for one 10,000-row batch, walking `msg_queues_pkey` and discarding
+809,729 rows by filter to find 10,000 expirable ones.
+
+`pg_stat_wal` over the measurement window: 27 TB of WAL accumulated, 37.2% of records full-page
+images, averaging about 425 GB per day; recent days reach ~470 GB (from the `archive-push` log) as
+the queue count grows.
+
+Per-statement WAL from `pg_stat_statements` (`wal_bytes`), leaf statements only (this instance runs
+`pg_stat_statements.track = all`, so the `write_message` wrapper double-counts the `INSERT` and
+`UPDATE` it runs, and is excluded):
+
+| category | share of WAL | main statements |
+| --- | --- | --- |
+| `msg_queues` updates | ~42% | `updateQueueTime` ~31%, `msg_can_write` flags ~9%, insert/delete ~2% |
+| `messages` insert and delete | ~34% | message `INSERT`, `DELETE` by `message_id` and `recipient_id` |
+| read path (hint bits, FPIs on SELECT) | ~24% | message peek ~15%, `msg_queue_size`, `getEntityCounts` |
+
+## Root causes
+
+1. Prometheus scrape. `getEntityCounts` (`QueueStore/Postgres.hs:154`) runs six `COUNT(1)` subqueries
+   every `prometheus_interval` (30 s), called from the metrics path (`Server.hs:698`). Four count
+   `msg_queues`; three of those seq-scan the 48.5M-row heap. At 70.7% of all reads it is the dominant
+   disk consumer, and it runs continuously (mean 111 s per scrape exceeds the 30 s interval).
+
+2. Message-expiration sweep. `expireMessagesThread` (`Server.hs:477`) calls the `expire_old_messages`
+   procedure (`server_schema.sql`), whose inner batch query selects expirable queues ordered by
+   `recipient_id`. With `oldQueue = 0` the `updated_at` filter matches everything, so the planner
+   walks the primary key and filters `msg_queue_expire`, discarding ~99% of rows per batch.
+
+3. Index bloat. 104 GB of indexes on a 14 GB table. `updated_at` is part of
+   `idx_msg_queues_updated_at_recipient_id` (`server_schema.sql:526`) and `updateQueueTime`
+   (`QueueStore/Postgres.hs:443`) updates it on the day's first send per queue (172,885 times in the
+   window above). Updating an indexed column prevents HOT updates, so every update writes a new row
+   version plus new entries in every index on the table, and the old index entries accumulate as bloat.
+
+4. WAL amplification. Only 15.6% of `msg_queues` updates are HOT, because its composite index covers
+   `updated_at` and `msg_queue_expire`, both changed on the hot path (cause 3); the rest rewrite every
+   index on the table. With the 104 GB of bloated indexes and frequent checkpoints (`max_wal_size 4GB`,
+   `checkpoint_timeout 5min`, `wal_compression off`), 37% of WAL records are full-page images, and the
+   server generates about 425 GB of WAL per day (27 TB accumulated; recent days near 470 GB).
+   `updateQueueTime`, stamping `updated_at` on the ~5.3M daily-active queues, is the single largest
+   statement at ~31% of WAL (see Evidence). pgBackRest `archive-push` ships all of it, loading the
+   primary disk and the backup repository. The load peaks
+   at 00:00 UTC because `getSystemDate` rounds `updated_at` to the UTC day, so the first send to each
+   active queue after midnight runs `updateQueueTime`, clustering these writes and the `archive-push`
+   volume in the 00h hour.
+
+## Index bloat and autovacuum
+
+Autovacuum is running (60 runs on this table) and is not misconfigured, but two limits apply.
+
+Autovacuum reclaims heap dead tuples and refreshes planner statistics; it does not shrink indexes.
+Only `REINDEX` or `pg_repack` rebuilds a bloated B-tree. Index bloat therefore has no automatic
+remedy.
+
+Default autovacuum pacing assumes moderate churn. `autovacuum_vacuum_scale_factor = 0.2` waits until
+20% of the table is dead (about 9.7M rows here) before vacuuming, and cost-based throttling caps its
+I/O, so a hot table stays around 12% dead. Per-table tuning makes it run sooner and faster.
+
+The amplifier is indexing `updated_at`, a column updated on every send, which the fixes and operator
+actions below address.
+
+## Implemented fixes (batch 1)
+
+Branch `sh/leaks-batch-1`, two commits.
+
+| commit | change | effect |
+| --- | --- | --- |
+| `88673eee` | migration `20260916_prometheus_indexes` adds partial indexes `idx_msg_queues_expire (recipient_id) WHERE deleted_at IS NULL AND msg_queue_expire` and `idx_msg_queues_notifier_active (notifier_id) WHERE deleted_at IS NULL AND notifier_id IS NOT NULL` | the sweep batch and `notifier_count` become index scans |
+| `2912be1e` | `getEntityCounts.queue_count` uses `pg_class.reltuples` (`QueueStore/Postgres.hs:154`) | removes the 30 s, 990 GB whole-table `COUNT` on every scrape |
+
+Verified: both indexes are created and used as index-only scans, the schema-dump test passes, and
+`smp-server` builds. `queueCount` is consumed only by metrics, logs, and display
+(`Server.hs:528,698,815,2369`), so an estimate is safe.
+
+Not fixed by new indexes:
+
+- `ntf_service_queues_count` seq-scans only because of index bloat; `idx_msg_queues_ntf_service_id`
+  already covers it. `REINDEX` restores the index-only scan, as proven by `rcv_service_queues_count`.
+- `rcv_service_queues_count` and the two `services` counts are already cheap.
+
+## Implemented fixes (batch 2)
+
+Migration `20260917_msg_queues_hot` and `MsgStore/Postgres.hs`.
+
+| change | effect |
+| --- | --- |
+| drop `idx_msg_queues_updated_at_recipient_id`, set `msg_queues` `fillfactor = 80` | `updateQueueTime` updates become HOT, so they rewrite no index entries; removes its index bloat and cuts its WAL |
+| remove the dead `updated_at > p_old_queue` filter and its parameter from `expire_old_messages`, matching the `CALL` (`MsgStore/Postgres.hs:113`) | the sweep query runs as an index-only scan on `idx_msg_queues_expire`; `updated_at` is no longer read on the hot path |
+| autovacuum reloptions on `msg_queues`, `messages` (and its TOAST), and `services` | keeps dead tuples and bloat low per table (see the table below) |
+
+Verified on PostgreSQL 16 with the daily-active pattern (~11% of queues updated per day): `updateQueueTime`
+goes from 0% HOT (indexed `updated_at`) to 100% HOT at `fillfactor = 80` (96.7% at 90), and per-update
+WAL drops about 4x. The HOT ratio depends on how many rows per page change before autovacuum reclaims
+space, so it is workload-dependent; `fillfactor = 80` reached 100% for this fraction, and updating a much
+larger share of queues at once would need a lower value. `fillfactor` applies to pages rewritten after
+the change, so the ratio ramps up as the heap turns over, or immediately after `pg_repack`.
+
+Dropping the index regresses the deprecated journal message store's expiration (`foldRecentQueueRecs`,
+`Journal.hs:434`) to a sequential scan; this is accepted because that store is being retired. The
+Postgres message store does not read `updated_at`.
+
+Autovacuum reloptions applied by the migration:
+
+| table | reloptions | reason |
+| --- | --- | --- |
+| `msg_queues` | `fillfactor 80`, `autovacuum_vacuum_scale_factor 0.02`, `autovacuum_analyze_scale_factor 0.01`, `autovacuum_vacuum_cost_limit 1000` | HOT headroom; vacuum/analyze at ~2%/1% churn instead of 20%/10%; faster vacuum |
+| `messages` (and TOAST) | `autovacuum_vacuum_scale_factor 0.02`, `autovacuum_analyze_scale_factor 0.01`, `toast.autovacuum_vacuum_scale_factor 0.02` | high insert and delete churn; 152 GB, mostly TOASTed message bodies |
+| `services` | `fillfactor 70`, `autovacuum_vacuum_threshold 1000`, `autovacuum_vacuum_scale_factor 0` | tiny, very hot table was autovacuumed tens of thousands of times; make updates HOT and vacuum it far less |
+
+## Operator actions
+
+1. Reclaim index bloat (online, no lock; needs free disk near the index size; run largest first):
+   ```
+   REINDEX INDEX CONCURRENTLY smp_server.idx_msg_queues_updated_at_recipient_id;
+   REINDEX INDEX CONCURRENTLY smp_server.idx_msg_queues_notifier_id;
+   REINDEX INDEX CONCURRENTLY smp_server.idx_msg_queues_ntf_service_id;
+   REINDEX INDEX CONCURRENTLY smp_server.idx_msg_queues_rcv_service_id;
+   REINDEX INDEX CONCURRENTLY smp_server.idx_msg_queues_sender_id;
+   REINDEX INDEX CONCURRENTLY smp_server.idx_msg_queues_link_id;
+   ANALYZE smp_server.msg_queues;
+   ```
+   Expect `pg_indexes_size('smp_server.msg_queues')` to drop from 104 GB to roughly 12 GB (measured
+   after reindexing). `ANALYZE` restores the index-only scan for `ntf_service_queues_count`. With the
+   batch-2 `fillfactor` the index bloat no longer rebuilds, so this is a one-time reclaim.
+
+2. Autovacuum reloptions are applied by the batch-2 migration (`msg_queues`, `messages` and its TOAST,
+   `services`), so no manual `ALTER TABLE` is needed. Reclaim the current `messages` bloat once with
+   `pg_repack smp_server.messages` (152 GB heap and TOAST, roughly 30-50 GB reclaimable); running
+   `pg_repack` on `msg_queues` also makes `fillfactor = 80` effective immediately rather than as pages
+   turn over. With `fillfactor = 80` the `msg_queues` index bloat no longer rebuilds, so scheduled
+   reindexing is rarely needed. To automate it anyway, run `reindexdb --concurrently` or `pg_repack`
+   from a scheduler; `pg_cron` runs each job inside a transaction and cannot execute
+   `REINDEX ... CONCURRENTLY`, so use `pg_cron` only for `VACUUM`/`ANALYZE` or a non-concurrent
+   maintenance-window `REINDEX`.
+
+3. Cut WAL and checkpoint pressure (all reloadable, no restart):
+   ```
+   ALTER SYSTEM SET wal_compression = 'lz4';
+   ALTER SYSTEM SET max_wal_size = '32GB';
+   ALTER SYSTEM SET checkpoint_timeout = '30min';
+   SELECT pg_reload_conf();
+   ```
+   Fewer checkpoints produce fewer full-page images; `wal_compression` shrinks those that remain.
+   Together these reduce the WAL written on the primary and shipped by `archive-push`. The message
+   insert and delete volume itself is fixed by traffic, but most of its WAL cost is amplification
+   (full-page images and hint-bit writes on bloated pages), which these settings and `REINDEX` reduce.
+   This is separate from pgBackRest repo-side compression, which does not reduce WAL on the primary.
+   Verify with `pg_stat_reset_shared('wal')`, wait, then re-check `wal_fpi` and `wal_bytes` in
+   `pg_stat_wal`.
+
+4. Enable `track_io_timing = on` so disk wait time is measurable in `pg_stat_statements`.

docs/smp-server-memory.md

@@ -0,0 +1,243 @@
+# SMP server memory at production scale
+
+Production: ~40k client connections per server, 32 GB RAM shared with PostgreSQL, pure PostgreSQL
+store, 14 servers proxying to each other, GHC 9.6.3, `+RTS -N -F1.2 -A16m -I0.01 -Iw15 -s -RTS`.
+With `-F1.2` the copying GC needs about 2.2x the live heap plus the nursery, so the server's live
+heap has to stay well under ~10 GB.
+
+A busy connection (20 batch-subscribed queues, 2 created with NEW, message traffic) cost 237 KiB of
+server live heap, 9.0 GiB at 40k connections. With the changes below it costs 43 KiB, 1.6 GiB.
+
+Earlier findings on leaks and PostgreSQL load are in `leak-findings.md` and `smp-server-db-load.md`.
+
+## Method
+
+All numbers are server-only: the bench client runs in a child process, so the measuring process holds
+only the server. PostgreSQL store, production RTS flags, 16-core machine. Live heap is measured after
+two major GCs with finalizers allowed to run in between (`liveBytesMiB` in `bench/MemBench.hs`).
+
+| Phase | Measures |
+| --- | --- |
+| `prodmix` | 2000 connections, each SUBs 20 queues in one block and creates 2 with NEW, then 30 s of SEND/MSG/ACK traffic |
+| `conns N` | N idle connections after the SMP handshake |
+| `subslice N` | per subscription: `SUBMODE=sub` (`SUBBATCH`, `SUBKEEP`), `new`, `newthensub` |
+| `load` | server CPU per operation under a fixed mixed workload |
+| `ntfloop`, `ntfdeliver` | notification store and its delivery loop |
+| `mesh` | proxy and relay in separate processes: relay sessions, forwards in flight, forward rate (`MESH=`) |
+| `pfwdbig` | oversized forwards left in the proxy's `sentCommands` |
+
+```sh
+cabal build -fserver_postgres exe:smp-mem-bench
+BENCHID=1 PROD_CONNS=2000 PROD_QUEUES=20 PROD_NEW=2 PROD_SEC=30 \
+  $(cabal list-bin -fserver_postgres exe:smp-mem-bench) prodmix 0 \
+  +RTS -N -F1.2 -A16m -I0.01 -Iw15 -T -ki2k -RTS
+```
+
+`BENCHID` offsets ports and the database, so runs do not collide with each other or with tests.
+
+## Results
+
+`prodmix`, live heap per connection after subscribing and peak memory in use under traffic. Each row
+adds to the previous one.
+
+| Change | KiB/conn | Peak in use | 40k conns |
+| --- | --- | --- | --- |
+| base (`sh/fix-leak`) | 236.6 | 1075 MiB | 9.0 GiB |
+| `-ki2k` | 111.5 | 719 MiB | 4.3 GiB |
+| connection commits (`sh/fix-conn-mem`) | 89.8 | 635 MiB | 3.4 GiB |
+| unpinned subscription keys, proxy commits (`sh/mem-combined`) | 80.0 | 604-622 MiB | 3.1 GiB |
+| patched tls 1.9 | 42.9 | 469 MiB | 1.6 GiB |
+
+Idle connections (`conns 2000`): 153 KiB at default flags, 68.4 KiB on `sh/mem-combined` with
+`-ki2k`, 34.8 KiB with the patched tls.
+
+Message throughput did not drop in any row. `-ki2k` costs some GC time (below).
+
+## Findings
+
+### 1. Thread stacks: 125 KiB per busy connection
+
+Each connection has several threads that block in shallow loops. A thread starts with a 1 KiB stack
+(`-ki1k`). On first overflow the RTS copies up to `-kb` (1 KiB) of frames into a new 32 KiB chunk
+(`-kc32k`), which with a 1 KiB initial stack moves the loop frame itself, so the thread never returns
+to its first chunk and keeps the 32 KiB chunk while the connection lives.
+
+| Flags | idle KiB/conn | prodmix KiB/conn |
+| --- | --- | --- |
+| default | 153 | 236.6 |
+| `-kc8k` | 100 | 135.0 |
+| `-ki2k` | 91 | 111.5 |
+
+`-ki2k` leaves the loop frame in the first chunk, so the overflow chunk is freed on return. Cost, 3
+alternating 60 s `load` runs: GC CPU +15% (4.8-5.2 s to 5.7-5.8 s per run), major GCs +43% (the live
+heap is smaller, so `-F1.2` triggers major GCs sooner), CPU per operation +0-5% (2.74-3.02 ms to
+2.91-3.35 ms, one noisy run). One `load` run with `-kc2k` grew to 6.8 GB RSS; not investigated, so
+`-kc` is left at its default.
+
+Branch `sh/rts-ki2k` sets `-with-rtsopts=-ki2k` for smp-server. Command-line `+RTS` flags still apply.
+
+### 2. Pinned blocks kept alive by small long-lived objects
+
+`ByteString`s and crypton's `ScrubbedBytes` are pinned and share 4 KiB blocks. GHC treats a pinned
+block as live if any object in it is live, and treats every object in a live pinned block as alive,
+including dead `ScrubbedBytes`, whose weak pointers and finalizers then stay allocated (GHC 9.6.3
+`Evac.c:443`, `GCAux.c:73`). A 24-byte value created during request processing, among crypto
+temporaries, therefore keeps ~4 KiB and several dead keys alive for as long as it lives. A standalone
+program reproduces it (pinned key 2742 B live per key, unpinned 80 B).
+
+Subscriptions. The subscription key is such a value.
+
+| `subslice 5000` | before | unpinned keys |
+| --- | --- | --- |
+| NEW-created | 5.25 KiB | 0.46 KiB |
+| one SUB per block | 1.67-2.42 KiB | 0.47 KiB |
+| 100 SUBs per block | 0.49 KiB | 0.46 KiB |
+| NEW0, then SUB on the same connection | 4.98 KiB | 0.47 KiB |
+
+Branch `sh/fix-sub-mem` stores the keys of `subscriptions`, `ntfSubscriptions` and `queueSubscribers`
+as `ShortByteString` and shares one key between the maps. CPU unchanged.
+
+TLS session state. tls 1.9 keeps record keys, IVs, TLS 1.3 traffic secrets, verify data and
+handshake state for the connection's lifetime, all allocated during the handshake. A heap census per
+connection before and after re-allocating them together at the end of the handshake:
+
+| | tls 1.9 | patched |
+| --- | --- | --- |
+| weak pointers | ~138 | ~45 |
+| memory in pinned blocks, outside the census | ~39 KiB | ~13 KiB |
+| idle KiB/conn | 68.4 | 34.8 |
+| prodmix KiB/conn | 80.0 | 42.9 |
+
+The patch (`tls-1.9-compact-state.patch`: `Network.TLS.Handshake.Compact`, called at the end of
+`handshake` under the context locks) re-derives the TLS 1.3 record keys from the kept traffic
+secrets, copies the IVs, secrets, session ID, verify data, randoms, main and resumption secrets and
+the transcript hash, and reseeds the context RNG. It needs a tls fork. Not compacted: the TLS 1.2 bulk
+key, keys installed later by KeyUpdate. The transcript hash copy coerces crypton's hidden `Context`
+newtype, so it must be checked on crypton upgrades.
+
+Full test suite with the patched tls: 1431 examples, 0 failures. The same run on unpatched tls had 1
+failure, the XFTP agent's "should resume sending file after restart".
+
+SMP handshake state (session secret, chain keys) has the same pattern and is not done yet; the
+remaining ~13 KiB outside the census is likely there.
+
+### 3. IDs sliced from the received block: 16 KiB per subscription
+
+Parsed `corrId` and `entityId` are slices of the ~16 KB decrypted block, and the entity ID was stored
+as the subscription key, so one live key kept the block. One SUB per block cost 16.43 KiB per
+subscription, one survivor of 100 cost 18.04 KiB. `sh/fix-sub-keys` copies both IDs in
+`tDecodeServer` (16.43 to 1.67-2.42 KiB). Unpinned keys (finding 2) also remove it for the maps.
+
+### 4. Connection structure: 21 KiB per busy connection
+
+- tls 1.9 stores the receive record state lazily, so each connection kept its last received 16 KB
+  record. `recvTLS` forces the state after each read. Not needed with tls 2.x.
+- The send and message-send loops are one thread, and one server-wide thread expires inactive clients
+  instead of one per client: 6 threads per connection become 4.
+
+`sh/fix-conn-mem`: `prodmix` 111.5 to 89.8 KiB with `-ki2k`, 236.6 to 210.5 KiB without.
+
+### 5. Notification store and its delivery loop
+
+Keys of notifiers whose notifications were delivered stayed in the store, and the loop in
+`deliverNtfsThread` sends every key to `getQueueNtfServices` (one `notifier_id IN ?` query read in
+full) every 1.5 s.
+
+| `ntfloop` (keys, no traffic) | allocation | CPU | peak in use |
+| --- | --- | --- | --- |
+| 0 | 0 MiB/s | 0 | 280 MiB |
+| 100k | 168 MiB/s | 0.42 cores | 530 MiB |
+| 300k | 263 MiB/s | 0.76 cores | 913 MiB |
+
+`ntfdeliver` with 20k delivered notifications: 20000 keys kept, idle server allocating 36 MiB/s at
+0.18 cores; with `sh/fix-ntf-store` 0 keys, 0.2 MiB/s, 0.01 cores.
+
+### 6. Proxy mesh
+
+The relay processed forwarded commands from a proxy one at a time in the connection's command loop,
+with a PostgreSQL round trip each; all users of a proxy share its connection. Forwards the relay has
+not reached wait on the proxy, each with a thread and its ~16 KB block. `mesh`, 1000 forwards per
+second for 20 s, proxy and relay in separate processes, 2 runs each:
+
+| | serial | concurrent |
+| --- | --- | --- |
+| forwards ok / failed | 19,968 / 0 | 19,968 / 0 |
+| proxy live heap at end of sending | 192-200 MiB | 13 MiB |
+| proxy memory in use | 653-672 MiB | 325 MiB (idle baseline) |
+| proxy threads | 4,026-4,137 | 420 |
+
+Here the backlog drained within the 30 s forward timeout. With a slower relay (a busy database, a
+remote relay over SOCKS) the queueing delay exceeds it and forwards fail; not reproduced.
+
+`sh/fix-proxy-mesh` forks each forwarded command under the per-connection concurrency limit, stops
+keeping the forwarded block on the proxy while waiting (53.0 to 36.8 KiB per in-flight forward,
+relay not answering) and caps forwards in flight per relay at 512 (`[PROXY] relay_concurrency`).
+Forwarded commands use per-command nonces and secrets and are matched by correlation ID, so concurrent
+processing is safe; the SimpleX agent keeps one message in flight per queue. The spec requires
+responses in order per queue within a connection (`simplex-messaging.md`, "same order within each
+queue ID"); serializing forwarded commands per queue ID is not done yet.
+
+Also in the proxy path (`sh/fix-proxy-leak`): a PFWD of 16260-16266 bytes from a client that declares
+`proxyServer = True` failed with `TELargeMsg` before sending and left its 20.2 KiB request in
+`sentCommands` for the life of the relay session (2001 PFWDs, 2001 entries); timed-out requests were
+kept too; and a relay session that stopped answering was never dropped (10 of 10 forwards timed out).
+
+### 7. Concurrency limit and name resolution
+
+`forkCmd` released its slot when the thread was forked, not when the command finished, so
+`serverClientConcurrency` and `serverResolverConcurrency` limited nothing: 16 RSLVs with a limit of 4
+all reached the resolver, 64 RSLVs from one connection were 64 concurrent requests, and 5 lookups
+answered with 502 opened 5 connections because error bodies were not read. `sh/fix-rslv-fanout` holds
+the slot until completion, adds a global limit (`[NAMES] resolver_global_concurrency`, 32) and reads
+error bodies. With the limit working, a client that hits it blocks its own command loop.
+
+### 8. Smaller findings
+
+- Active-queue statistics are `IntSet`s at 64 B per element (15M elements, 916 MiB for one set), six
+  of them, reset only when `log_stats` is on.
+- Control port `save` closes the PostgreSQL pool while the server keeps running; every later DB
+  operation blocks (`cpsave` phase: NEW after `save` gets no response).
+
+### 9. tls 2.x
+
+tls 2.1.6 (with tls-session-manager 0.0.6, crypton-connection 0.4.3, http-client-tls 0.3.6.4, an
+index-state bump and version pins) saves 2-5 KiB per connection: `prodmix` 75.4 KiB against 80.0.
+Problems found:
+
+- After a failed handshake a tls 2 client and a tls 2 server wait for each other in `bye`; this hung
+  `testServerMultipleIdentities`. Fixed on `sh/tls2` by closing the context without `bye`.
+- A tls 2 client that receives a CertificateRequest runs a timed read inside `handshake`; when the
+  timeout fires in the middle of a record the stream fails ("bad record mac") or hangs. SMP servers
+  always request client certificates. 1-2 failures per 2000 concurrent connections; present in tls
+  2.4.3. A `conns 2000` run with tls 2 on both sides stopped at 53 connections.
+
+tls 2.x is not recommended; `sh/tls2` is kept for reference.
+
+## Open
+
+- Serialize forwarded commands per queue ID on the relay (spec response order).
+- A client that sends PFWDs and never reads the responses costs the proxy 6.3 MiB live (11.4 MiB in
+  use) until the inactive-client expiry (up to 6 hours), 6 GiB per 1000 such clients.
+- PRXY to many aliases of one relay opens an unbounded number of relay sessions, each 207 KiB on the
+  proxy and 149 KiB on the relay: 2.0 GiB and 1.4 GiB per 10k.
+- Compact the SMP handshake state after the handshake, as in finding 2.
+- Publish the tls fork and reference it from `cabal.project`.
+- A SEND right after NEW or SUB can find no subscriber yet (`queueSubscribers` is updated through
+  `subQ`), so the message waits for the next SUB or ACK. `deliverIfSame` may leave a subscription
+  without a delivery thread. Both seen as occasional stuck steps in `load`; not memory.
+- `[PROXY] relay_concurrency` and `[NAMES] resolver_global_concurrency` are new INI keys.
+
+## Branches
+
+| Branch | Base | Content |
+| --- | --- | --- |
+| `sh/fix-ntf-store` | master | finding 5 |
+| `sh/fix-sub-keys` | master | finding 3 |
+| `sh/fix-proxy-leak` | master | finding 6, request leak and stuck session |
+| `sh/fix-rslv-fanout` | master | finding 7 |
+| `sh/fix-conn-mem` | master | finding 4 |
+| `sh/fix-sub-mem` | master | finding 2, subscription keys |
+| `sh/fix-proxy-mesh` | `sh/fix-proxy-leak` | finding 6, relay concurrency, per-relay limit |
+| `sh/rts-ki2k` | master | finding 1 |
+| `sh/mem-combined` | `sh/fix-leak` | findings 1-6 together, as measured in Results (local) |
+| `sh/tls2` | `sh/mem-combined` | finding 9, not for merge (local) |

docs/tls-1.9-compact-state.patch

@@ -0,0 +1,184 @@
+diff -ruN -x 'dist*' tls-1.9.0-orig/Network/TLS/Crypto.hs tls-fork/Network/TLS/Crypto.hs
+--- tls-1.9.0-orig/Network/TLS/Crypto.hs	2026-09-30 18:44:40.083085887 +0000
++++ tls-fork/Network/TLS/Crypto.hs	2026-09-30 18:44:40.106519923 +0000
+@@ -8,6 +8,7 @@
+     , hashUpdate
+     , hashUpdateSSL
+     , hashFinal
++    , hashCopy
+ 
+     , module Network.TLS.Crypto.DH
+     , module Network.TLS.Crypto.IES
+@@ -44,6 +45,8 @@
+ import qualified Crypto.Hash as H
+ import qualified Data.ByteString as B
+ import qualified Data.ByteArray as B (convert)
++import qualified Data.ByteArray as BA
++import Unsafe.Coerce (unsafeCoerce)
+ import Crypto.Error
+ import Crypto.Number.Basic (numBits)
+ import Crypto.Random
+@@ -135,6 +138,17 @@
+ hashUpdate (HashContextSSL sha1Ctx md5Ctx) b =
+     HashContextSSL (H.hashUpdate sha1Ctx b) (H.hashUpdate md5Ctx b)
+ 
++-- | Copies the context into a new allocation.
++hashCopy :: HashContext -> IO HashContext
++hashCopy (HashContext (ContextSimple h)) = HashContext . ContextSimple <$> copyContext h
++hashCopy (HashContextSSL sha1Ctx md5Ctx) = HashContextSSL <$> copyContext sha1Ctx <*> copyContext md5Ctx
++
++-- H.Context is a newtype over Bytes, with the constructor in a hidden crypton module
++copyContext :: H.Context alg -> IO (H.Context alg)
++copyContext h = do
++    b <- BA.copy h (\_ -> return ()) :: IO BA.Bytes
++    return $! unsafeCoerce b
++
+ hashUpdateSSL :: HashCtx
+               -> (B.ByteString,B.ByteString) -- ^ (for the md5 context, for the sha1 context)
+               -> HashCtx
+diff -ruN -x 'dist*' tls-1.9.0-orig/Network/TLS/Handshake/Compact.hs tls-fork/Network/TLS/Handshake/Compact.hs
+--- tls-1.9.0-orig/Network/TLS/Handshake/Compact.hs	1970-01-01 00:00:00.000000000 +0000
++++ tls-fork/Network/TLS/Handshake/Compact.hs	2026-09-30 18:45:10.137984909 +0000
+@@ -0,0 +1,111 @@
++{-# LANGUAGE OverloadedStrings #-}
++{-# OPTIONS_HADDOCK hide #-}
++module Network.TLS.Handshake.Compact
++    ( compactContext
++    ) where
++
++import Control.Concurrent.MVar
++import Control.Exception (evaluate)
++import Crypto.Random (drgNew)
++import qualified Data.ByteString as B
++import Data.IORef
++
++import Network.TLS.Cipher
++import Network.TLS.Context.Internal
++import Network.TLS.Crypto
++import Network.TLS.Extension (Cookie (..))
++import Network.TLS.Handshake.State
++import Network.TLS.Imports
++import Network.TLS.KeySchedule (hkdfExpandLabel)
++import Network.TLS.Record.State
++import Network.TLS.RNG
++import Network.TLS.State
++import Network.TLS.Struct
++import Network.TLS.Types
++
++-- | Re-allocates the state a context keeps after the handshake, next to each other.
++-- Each small pinned value created during the handshake shares a 4K block with short-lived
++-- crypto temporaries, and keeps the whole block alive, with every dead ScrubbedBytes in it.
++compactContext :: Context -> IO ()
++compactContext ctx = do
++    tx <- readMVar $ ctxTxState ctx
++    rx <- readMVar $ ctxRxState ctx
++    st <- readMVar $ ctxState ctx
++    hst <- readMVar $ ctxHandshake ctx
++    finished <- readIORef $ ctxFinished ctx
++    peerFinished <- readIORef $ ctxPeerFinished ctx
++    -- key derivation and reseeding allocate temporaries, so they precede the copies
++    txKey <- trafficKey tx
++    rxKey <- trafficKey rx
++    rng <- evaluate $ StateRNG $ fst $ withTLSRNG (stRandomGen st) drgNew
++    tx' <- copyRecordState BulkEncrypt txKey tx
++    rx' <- copyRecordState BulkDecrypt rxKey rx
++    st' <- copyTLSState rng st
++    hst' <- traverse copyHandshakeState hst
++    finished' <- traverse copyBytes finished
++    peerFinished' <- traverse copyBytes peerFinished
++    modifyMVar_ (ctxTxState ctx) $ \_ -> return tx'
++    modifyMVar_ (ctxRxState ctx) $ \_ -> return rx'
++    modifyMVar_ (ctxState ctx) $ \_ -> return st'
++    modifyMVar_ (ctxHandshake ctx) $ \_ -> return hst'
++    writeIORef (ctxFinished ctx) finished'
++    writeIORef (ctxPeerFinished ctx) peerFinished'
++
++copyBytes :: ByteString -> IO ByteString
++copyBytes = evaluate . B.copy
++
++-- TLS 1.3 record keys are derived from the traffic secret kept in the record state;
++-- earlier versions keep the bulk key only inside the cipher state.
++trafficKey :: RecordState -> IO (Maybe ByteString)
++trafficKey RecordState{stCipher = Just cipher, stCryptLevel = CryptApplicationSecret, stCryptState = cst} =
++    Just <$> evaluate (hkdfExpandLabel (cipherHash cipher) (cstMacSecret cst) "key" "" (bulkKeySize $ cipherBulk cipher))
++trafficKey _ = return Nothing
++
++copyRecordState :: BulkDirection -> Maybe ByteString -> RecordState -> IO RecordState
++copyRecordState dir key_ rs@RecordState{stCryptState = cst} = do
++    iv <- copyBytes $ cstIV cst
++    secret <- copyBytes $ cstMacSecret cst
++    bulkState <- case (key_, stCipher rs) of
++        (Just key, Just cipher) -> bulkInit (cipherBulk cipher) dir <$> copyBytes key
++        _ -> return $ cstKey cst
++    evaluate rs{stCryptState = CryptState{cstKey = bulkState, cstIV = iv, cstMacSecret = secret}}
++
++copyTLSState :: StateRNG -> TLSState -> IO TLSState
++copyTLSState rng st = do
++    session <- case stSession st of
++        Session sid -> Session <$> traverse copyBytes sid
++    clientVerified <- copyBytes $ stClientVerifiedData st
++    serverVerified <- copyBytes $ stServerVerifiedData st
++    proto <- traverse copyBytes $ stNegotiatedProtocol st
++    alpn <- traverse (traverse copyBytes) $ stClientALPNSuggest st
++    cookie <- traverse (\(Cookie c) -> Cookie <$> copyBytes c) $ stTLS13Cookie st
++    exporter <- traverse copyBytes $ stExporterMasterSecret st
++    evaluate
++        st
++            { stSession = session
++            , stClientVerifiedData = clientVerified
++            , stServerVerifiedData = serverVerified
++            , stNegotiatedProtocol = proto
++            , stClientALPNSuggest = alpn
++            , stTLS13Cookie = cookie
++            , stExporterMasterSecret = exporter
++            , stRandomGen = rng
++            }
++
++copyHandshakeState :: HandshakeState -> IO HandshakeState
++copyHandshakeState hst = do
++    clientRandom <- ClientRandom <$> copyBytes (unClientRandom $ hstClientRandom hst)
++    serverRandom <- traverse (fmap ServerRandom . copyBytes . unServerRandom) $ hstServerRandom hst
++    mainSecret <- traverse copyBytes $ hstMasterSecret hst
++    digest <- case hstHandshakeDigest hst of
++        HandshakeMessages msgs -> HandshakeMessages <$> traverse copyBytes msgs
++        HandshakeDigestContext h -> HandshakeDigestContext <$> hashCopy h
++    resumption <- traverse (\(BaseSecret s) -> BaseSecret <$> copyBytes s) $ hstTLS13ResumptionSecret hst
++    evaluate
++        hst
++            { hstClientRandom = clientRandom
++            , hstServerRandom = serverRandom
++            , hstMasterSecret = mainSecret
++            , hstHandshakeDigest = digest
++            , hstTLS13ResumptionSecret = resumption
++            }
+diff -ruN -x 'dist*' tls-1.9.0-orig/Network/TLS/Handshake.hs tls-fork/Network/TLS/Handshake.hs
+--- tls-1.9.0-orig/Network/TLS/Handshake.hs	2026-09-30 18:44:40.083878116 +0000
++++ tls-fork/Network/TLS/Handshake.hs	2026-09-30 18:45:14.390780619 +0000
+@@ -18,6 +18,7 @@
+ import Network.TLS.Struct
+ 
+ import Network.TLS.Handshake.Common
++import Network.TLS.Handshake.Compact
+ import Network.TLS.Handshake.Client
+ import Network.TLS.Handshake.Server
+ 
+@@ -27,7 +28,7 @@
+ -- This is to be called at the beginning of a connection, and during renegotiation
+ handshake :: MonadIO m => Context -> m ()
+ handshake ctx =
+-    liftIO $ withRWLock ctx $ handleException ctx (ctxDoHandshake ctx ctx)
++    liftIO $ withRWLock ctx $ handleException ctx (ctxDoHandshake ctx ctx) >> compactContext ctx
+ 
+ -- Handshake when requested by the remote end
+ -- This is called automatically by 'recvData', in a context where the read lock
+diff -ruN -x 'dist*' tls-1.9.0-orig/tls.cabal tls-fork/tls.cabal
+--- tls-1.9.0-orig/tls.cabal	2026-09-30 18:44:40.090442445 +0000
++++ tls-fork/tls.cabal	2026-09-30 19:12:16.099341583 +0000
+@@ -85,6 +85,7 @@
+         Network.TLS.Handshake.Signature
+         Network.TLS.Handshake.State
+         Network.TLS.Handshake.State13
++        Network.TLS.Handshake.Compact
+         Network.TLS.Hooks
+         Network.TLS.IO
+         Network.TLS.Imports

plans/2026-07-11-service-rpc-implementation.md

@@ -0,0 +1,135 @@
+# Service RPC implementation plan
+
+RFC: [../rfcs/2026-07-11-service-rpc.md](../rfcs/2026-07-11-service-rpc.md)
+
+Depends on: [2026-07-12-address-dr-implementation.md](2026-07-12-address-dr-implementation.md). RPC establishes the double ratchet from the address exactly as address-DR does; this is the RPC layer on top of it.
+
+**Status: implemented and tested in this repo.** Service-side idempotency (single execution by request hash) is deferred. The `simplex-chat` integration is a separate repo.
+
+**Scope.** One request, one response — no continuation, no streaming.
+
+One DR-advertising contact address serves both flows: the owner branches per incoming request on the decrypted inner message — `AgentConnInfoReply` opens a connection (`REQ`), `AgentServiceRequest` answers an RPC (`SREQ`). A request gets exactly one reply, a response or a rejection; both are the single confirming message on the requester's reply queue Q_A, after which the ephemeral reply connection is torn down. Response and rejection are the same operation parameterized by the inner message (`AgentServiceResponse` payload vs `AgentRejection` reason) and outcome (the call returns the payload vs throws an agent error).
+
+## RPC messages
+
+No new outer envelope: the request reuses `AgentContactRequest` (tag `'A'`); the reply reuses `AgentConfirmation` (the only message on Q_A).
+
+Inner `AgentMessage` (ratchet-encrypted, parsed by `parseMessage`), siblings of `AgentConnInfoReply`:
+
+```haskell
+  | AgentServiceRequest (NonEmpty SMPQueueInfo) MsgBody  -- 'A': reply queue Q_A + opaque payload
+  | AgentServiceResponse MsgBody                         -- 'P': response payload (single, terminal)
+  | AgentRejection ByteString                            -- 'J': refusal reason (single, terminal)
+```
+
+`AgentServiceRequest` carries Q_A (as `AgentConnInfoReply` does); its constructor is the only thing that distinguishes `REQ` from `SREQ`. Delivery `msgType`: `AM_SRV_RESP` routes to `sendConfirmation` (the reply is the confirming first message on Q_A). `AM_SRV_REQ` is never stored — the request is sent synchronously inside `joinConnSrv'` via `sendInvitation`, so its arms in the delivery worker are unreachable and assert (`logError`).
+
+## Ratchet establishment — reuse of the address-DR flow
+
+`joinConnSrv'` takes `mkInner :: SMPQueueInfo -> AgentMessage`; `joinConnSrv` is the one-line wrapper passing `AgentConnInfoReply`. `sendServiceRequest'` passes `AgentServiceRequest`.
+
+**Request (client).** `serviceRequest_` fails fast with `A_SERVICE ASENotDRAddress` if the address carries no ratchet keys, then creates the client connection via `newConnToJoin` with `serviceRequestExpiresAt = Just (now + reqTimeout)` (the persisted per-request deadline), registers a one-shot `TMVar` in `serviceRequests`, sends the request, and blocks on the `TMVar` up to `reqTimeout`. The connection is `RcvConnection` (Q_A) with the send ratchet.
+
+**Request (service).** `smpContactRequest` decrypts `encConnInfo` and branches on the inner message; both branches call the same `storeInvitation` → `conn_invitations`, differing only in the kind column and event:
+
+- `AgentConnInfoReply` → `REQ` (`service_request = 0`).
+- `AgentServiceRequest _ payload` → `SREQ invId payload` (`service_request = 1`).
+
+Before storing, it **deduplicates** by the sender's ratchet-key hash (`checkRatchetKeyHashExists`/`addProcessedRatchetKeyHash`, the mechanism `newRatchetKey` uses): a redelivered/retried request reuses the same Q_A and the same `e2eSndParams`, so the hash matches and the duplicate is dropped — one invitation and one `REQ`/`SREQ` per request. Receive-time establishment on unauthenticated input — the address-DR abuse bound applies.
+
+**The reply (service).** `prepareReply` fetches the invitation, enforces the kind (`CMD PROHIBITED` on the wrong one), and rejects a stale request (`A_SERVICE ASETimeout` + delete) older than `serviceResponseTimeout`; then `newConnToAccept` + `startJoinInvitationDR` build the one-directional `SndQueue` to Q_A (no reply queue back), and `storeConfirmation` queues the inner message. `sendReplySync` secures Q_A, submits the message, and deletes the connection with wait-for-delivery — **deleting the connection on failure too** (`catchAllErrors`), so a failed secure/submit does not orphan it. `sendServiceReplyAsync` defers secure+deliver+delete to the `ICReplyDel` command (retried, survives a down server). `sendServiceReply`/`Async` and `replyRequest_` return the reply `ConnId` so the caller can correlate the `SENT` event on that throwaway connection.
+
+**The response (client).** The single `AgentConfirmation` on Q_A reaches `processConnInfo` (the `RcvConnection … New` branch). Dispatch is gated on `serviceRequestExpiresAt` and the kinds are mutually exclusive:
+
+- `AgentConnInfoReply` **only when `isNothing serviceRequestExpiresAt`** (a contact connection) → `processConf`.
+- `AgentServiceResponse` only when `isJust` → the request `TMVar` gets `Right payload`.
+- `AgentRejection` when `isJust` → `Left (A_SERVICE (ASERejected reason))`; when `isNothing` → contact `RJCT`.
+- anything else → `prohibited`.
+
+The `isNothing` guard on `AgentConnInfoReply` is a security boundary: without it a malicious service could send `AgentConnInfoReply` on an RPC reply queue and drive it into the contact-`CONF` path. `dispatchServiceReply` puts the result into the `serviceRequests` `TMVar`; a reply with no pending request (e.g. post-restart) is `ERR (A_SERVICE ASENoPendingRequest)`.
+
+## Rejection
+
+A rejection is `AgentRejection reason` — the same single confirming message on Q_A as a response.
+
+- **Kind guard.** `rejectContact` only on a contact invitation, `rejectServiceRequest`/`sendServiceReply` only on a request; wrong kind is `CMD PROHIBITED`. `rejectRequest_` enforces the kind **even on a `Nothing` (silent-drop) reject** — it fetches the invitation and checks before deleting, so `rejectContact … Nothing` cannot delete a service request (or vice versa).
+- `reject*` take `Maybe ByteString`: `Nothing` → delete the invitation, send nothing (the requester times out); `Just reason` → the reply path with `AgentRejection`.
+- Requester side: `AgentRejection` on a contact reply queue → `RJCT`; on an RPC reply queue → a thrown `A_SERVICE (ASERejected reason)`.
+
+## Reply connections and cleanup
+
+No reply-queue table and no new connection type.
+
+- **Requester reply queue** (`RcvConnection` on Q_A): `connections.service_request_expires_at` is non-null only here; it is the persisted request deadline, used both to gate CONF dispatch and to reap the connection. In-memory routing is `serviceRequests :: TMap ConnId (TMVar (Either AgentErrorType MsgBody))`.
+- **Timeout race.** The async `JOIN` worker holds `withConnLock c connId` around `joinConnSrv'`, and `serviceRequest_`'s cleanup holds the same lock around `TM.delete` + `deleteConnectionAsync'`. This serializes the send with the timeout teardown, so a timing-out call cannot delete the connection mid-send; after cleanup the worker's re-check of `serviceRequests` finds nothing and skips.
+- **Service reply connection** (`SndConnection` to Q_A): ephemeral — created, sends the one reply, deleted with wait-for-delivery in the same operation.
+- **Cleanup** (`cleanupManager`, `deleteExpiredServiceReqs`): `deleteExpiredServiceRequests` reaps unanswered `conn_invitations` (service side) older than `serviceResponseTimeout`; `getExpiredServiceConns` (`service_request_expires_at < now`) → `deleteConnectionsAsync'` reaps orphaned requester reply queues.
+
+## Database schema
+
+`M20260712_address_dr_rpc` (SQLite + PostgreSQL) creates `address_ratchet_keys` (address-DR) and adds:
+
+```sql
+ALTER TABLE conn_invitations ADD COLUMN service_request INTEGER NOT NULL DEFAULT 0; -- service side: 1 = RPC request
+ALTER TABLE connections ADD COLUMN service_request_expires_at TEXT;                 -- client side: request deadline; gating + cleanup (nullable)
+```
+
+The down migration drops the columns then the table/index. Schema dump tests pass (up, down, STRICT).
+
+## Agent API — `Simplex.Messaging.Agent`
+
+```haskell
+-- service: send the one response, return the reply ConnId, then tear the reply connection down.
+sendServiceReply      :: AgentClient -> NetworkRequestMode -> UserId -> InvitationId -> MsgBody -> AE ConnId
+sendServiceReplyAsync :: AgentClient -> ACorrId -> UserId -> InvitationId -> MsgBody -> AE ConnId
+
+-- refuse a request (Just reason = AgentRejection; Nothing = silent drop). PROHIBITED on wrong kind.
+rejectServiceRequest      :: AgentClient -> NetworkRequestMode -> UserId -> InvitationId -> Maybe ByteString -> AE ()
+rejectServiceRequestAsync :: AgentClient -> ACorrId -> UserId -> InvitationId -> Maybe ByteString -> AE ()
+rejectContact      :: AgentClient -> NetworkRequestMode -> UserId -> ConfirmationId -> Maybe ByteString -> AE ()
+rejectContactAsync :: AgentClient -> ACorrId -> UserId -> ConfirmationId -> Maybe ByteString -> AE ()
+
+-- client: establish the ratchet from the address, send the request, block on the reply TMVar up to the timeout
+-- (Nothing = serviceRequestTimeout; Just t overrides per request), returning the payload. Sync fails fast if the
+-- server is down; async enqueues a retried JOIN command that survives an outage.
+sendServiceRequest      :: AgentClient -> NetworkRequestMode -> UserId -> ConnectionRequestUri 'CMContact -> Maybe NominalDiffTime -> MsgBody -> AE MsgBody
+sendServiceRequestAsync :: AgentClient -> UserId -> ConnectionRequestUri 'CMContact -> Maybe NominalDiffTime -> MsgBody -> AE MsgBody
+```
+
+Both client calls share `serviceRequest_`; the async `JOIN` worker branches on the `JRServiceReq` command to send `AgentServiceRequest`. The call blocks on the `TMVar` and returns synchronously — no events, no correlation for the app.
+
+Events (`AEvent`, entity is the address connection):
+
+```haskell
+SREQ :: InvitationId -> MsgBody -> AEvent AEConn  -- payload = the request; mirrors REQ.
+RJCT :: ConnInfo -> AEvent AEConn                 -- contact-request rejection reason.
+```
+
+Errors (`SMPAgentError`):
+
+```haskell
+| A_SERVICE {serviceError :: AgentServiceError}
+
+data AgentServiceError
+  = ASERejected {rejectReason :: Text}  -- service refused (Text: JSON-serializable, UTF-8-decoded from the reason bytes)
+  | ASETimeout                          -- no reply within the timeout
+  | ASENoPendingRequest                 -- a reply arrived with no pending request (e.g. post-restart)
+  | ASENotDRAddress                     -- the target address advertises no ratchet keys (fail fast, no send)
+```
+
+Config (`AgentConfig`): `serviceRequestTimeout` (30 s, client wait, overridable per request) and `serviceResponseTimeout` (180 s, service reply window and cleanup TTL; must exceed `serviceRequestTimeout`).
+
+## Idempotency (deferred)
+
+Not built. When built, the service will key a request by hash and cache the one response for a retention period, answering a repeat from storage without reaching the bot — single execution over at-least-once delivery, with its own tables.
+
+## Tests
+
+In `FunctionalAPITests` (plus the encoding roundtrip in `ConnectionRequestTests`), passing with `-O0`:
+
+- Request → one response, sync (`sendServiceReply`) and async (`sendServiceReplyAsync`).
+- Request → rejection (`rejectServiceRequest (Just reason)` → thrown `A_SERVICE (ASERejected …)`).
+- Resilience: `server down → send → up → receive → down → reply → up → receive response`.
+- No regression: the contact rejection and DR-join suites still pass.
+
+The `simplex-chat` end-to-end tests (happy path, drop-when-off, non-DR fail-fast) live in that repo.

plans/2026-07-12-address-dr-implementation.md

@@ -0,0 +1,279 @@
+# Establishing the double ratchet from address data - implementation plan
+
+RFC: [../rfcs/2026-07-12-address-pqdr-keys.md](../rfcs/2026-07-12-address-pqdr-keys.md)
+
+All references are to the current tree. Names of new constructors, fields, tables and functions are provisional.
+
+Goal: a contact address advertises the owner's X3DH parameters in link data; a requester establishes the double ratchet in its first message, so that message and the profile in it are under the ratchet with post-quantum protection. The change reuses the invitation/confirmation machinery, with the requester in the joiner role and the owner in the initiator role - opposite to today's contact flow, but every message and code path below is reused.
+
+Version: `addressDRVersion = VersionSMPA 8`, a plain agent-layer bump; `currentSMPAgentVersion` goes 7 → 8 (Agent/Protocol.hs:317-324). It gates the `AgentConfirmation.ratchetKeyId` field and the DR-from-address behavior. The receive-at-address path relies on ratchet-on-confirmation, already present since `ratchetOnConfSMPAgentVersion = 7` (Agent/Protocol.hs:317), so there is no cross-layer version dependency; the SMP and e2e-encryption versions are unchanged.
+
+Scope of this change: the **synchronous** DR handshake in join, gated on the address advertising `ratchetKeys`. `joinConnection`/`joinConn`/`joinConnSrv` gain an optional `Maybe AddressRatchetKeys` (the advertised `RcvE2ERatchetParamsUri` + `ratchetKeyId`), passed in from the link data the caller fetched at plan time (`LGET`); present → DR path (R2'/R3'), absent → the classic `AgentInvitation`. Chat wires that argument later (a chat change); the agent supports it now and tests pass it directly. Making the send **async** (worker retry, a "connecting" UX, the `CreatedConnLink` LGET-gate) is **deferred** - kept below under "Deferred" as future work, not part of this change.
+
+### Implementation status (as built; `lib:simplexmq` compiles)
+
+**Done** (compiles): version bump; `RatchetKeyId`/`AddressRatchetKeys` types + `Encoding`, `UserContactData.ratchetKeys` (appended, backward-compatible); `AgentConfirmation.ratchetKeyId` (version-gated encode/decode); `ContactRequest`/`DRRequest` sum with tagged `Encoding` + `cr_invitation` `ToField`/`FromField` (legacy-URI fallback); `address_ratchet_keys` table + `createAddressRatchetKeys`/`getAddressRatchetKeys` (SQLite + Postgres migrations `M20260712_address_dr`); join threading (`Maybe AddressRatchetKeys`); requester R2'/R3' (`joinAddressDR` + `sendConfirmationToAddress`); owner O1' dispatch, O2' `smpAddressConfirmation`, O3' (`acceptContact'` continue-ratchet branch), all three `connReq` readers (`acceptContact'`, `acceptContactAsync'` → `CMD PROHIBITED` for DR, `newConnToAccept` → shell from `drAgentVersion`/`drPQSupport`); requester R5' (`smpConfirmation` `RcvConnection … Nothing` branch, guarded on a ratchet existing); address-creation bundle generation (`mkAddressRatchetKeys`) wired into `createConnectionForLink'` (`IKUsePQ`-for-`SCMContact` prohibition lifted there).
+
+**Deltas from the plan discovered while building:**
+- **R5' emits `CONF` and reuses the allow step** (not auto-complete). The DR requester is a `RcvConnection` receiving the owner's reply - the same position as the classic contact requester, which goes `CONF` → `allowConnection'` → `connectReplyQueues` (msg 3). R5' mirrors that (differing only in that the ratchet already exists, so it `getRatchet` + `rcDecrypt` instead of building it), so the app supplies `ownConnInfo` for msg 3 at allow, exactly as today. No new storage.
+- **`DRRequest` carries `drAgentVersion` + `drPQSupport`** (Part 3): the sync accept creates the connection shell via `newConnToAccept`→`newConnToJoin` before O3', and there is no URI to derive the version/PQ from.
+- `cr_invitation` serialization is downgrade-safe: `CRInvitation` keeps the legacy URI (`strEncode`, byte-identical to before), so an older agent still reads classic invitations; `CRConfirmation` is JSON (`DRRequest` has manual `ToJSON`/`FromJSON`), told apart on read by the leading `{` (a URI never starts with it). JSON keeps `DRRequest` extensible. `SMPQueueInfo` gained a base64 `StrEncoding` + JSON (it only had `Encoding`) so it can sit in the JSON.
+- **DR is opt-in per address**: `createConnectionForLink'`/`createConnectionForLink` gain a `Maybe InitialKeys` DR parameter (separate from the existing connection-PQ `InitialKeys`) - `Nothing` = no DR (old behavior, existing callers), `Just ik` = advertise the bundle with `ik`. The `IKUsePQ`-for-`SCMContact` prohibition stays on the connection-PQ parameter and is lifted only for the DR bundle.
+
+**Test-matrix consequence of the version bump:** `currentSMPAgentVersion` 7 → 8 moves the version-matrix "prev" (`current − 1`) from v6 to v7. v7 ≥ `ratchetOnConfSMPAgentVersion (7)`, so a joiner/acceptor at "prev" now secures the send queue on confirmation - the `sqSecured` expectation for the prev variants in `testMatrix2`/`testMatrix2Stress`/`testBasicMatrix2` flips `False → True`. (Standard version-bump maintenance; the pre-`ratchetOnConf` unsecured path is now two versions back and no longer exercised by these matrices.)
+
+**Not yet done:** rotation (`rotateRatchetKeys`, Part 4), cleanup step (Part 4), the app-driven `LSET` upgrade API (Part 5), wiring the DR parameter into the non-prepared-link `newRcvConnSrv` path, DR-specific tests (Part 6), regenerating `agent_schema.sql` if a schema-consistency test requires it, and chat wiring (deferred by design).
+
+## Part 1 - the current contact-address handshake, step by step
+
+Requester Alice connects to owner Bob's contact address. Q_A is Alice's receive queue (Bob to Alice), Q_B is Bob's receive queue (Alice to Bob).
+
+Requester side, in `joinConnSrv … CRContactUri` (Agent.hs:1398-1428):
+
+- R1. `compatibleContactUri` (Agent.hs:1370) - version check, yields the address queue `SMPQueueInfo`.
+- R2. `mkJoinInvitation` (Agent.hs:1411): creates or reuses the receive queue Q_A; `getRatchetX3dhKeys` or `generateRcvE2EParams` produces Alice's Rcv X3DH parameters, stored by `createRatchetX3dhKeys` (Agent.hs:1424); builds `cReq = CRInvitationUri crData aliceRcvParams` (Agent.hs:1426).
+- R3. `sendInvitation` (Agent.hs:1408; Agent/Client.hs:1924-1934): sends `AgentInvitation {connReq = cReq, connInfo = aliceProfile}` to the address queue, per-queue encrypted with a fresh ephemeral key by `agentCbEncryptOnce` (Agent/Client.hs:1929-1934), unauthenticated. **`connInfo` (Alice's profile) is under the per-queue X25519 layer only - the gap this plan closes.**
+
+Owner side, receiving on the contact address:
+
+- O1. `processClientMsg` dispatch (Agent.hs:3185): state `(Nothing, Just e2ePubKey)`, `(PHEmpty, AgentInvitation {connReq, connInfo})` -> `smpInvitation` (Agent.hs:3186).
+- O2. `smpInvitation` (Agent.hs:3610): stores an `Invitation`, emits `REQ` with Alice's `connInfo`.
+- O3. `acceptContact'` (Agent.hs:1477): `getInvitation`, then `joinConn` with Alice's `connReq` (Agent.hs:1480).
+- O4. `joinConnSrv … CRInvitationUri` (Agent.hs:1383) -> `startJoinInvitation` (Agent.hs:1395).
+- O5. `startJoinInvitation` (Agent.hs:1310-1350): creates Bob's send queue to Q_A (`newSndQueue`, Agent.hs:1335); `createRatchet_` (Agent.hs:1343-1350) runs `generateSndE2EParams`, `pqX3dhSnd` against Alice's Rcv parameters, `initSndRatchet`, `createSndRatchet`.
+- O6. `secureConfirmQueue` (Agent.hs:1396, 3747-3765): `agentSecureSndQueue` secures Q_A with `SKEY` (Agent.hs:3749); `mkAgentConfirmation` (Agent.hs:3780-3785) calls `createReplyQueue` to create Bob's receive queue Q_B and returns `AgentConnInfoReply (Q_B :| []) bobInfo`; `mkConfirmation` ratchet-encrypts it and wraps `AgentConfirmation {e2eEncryption_ = Just bobSndParams, encConnInfo}`; `sendConfirmation` sends it to Q_A. This is confirmation #1.
+
+Requester side, receiving confirmation #1 on Q_A:
+
+- R4. dispatch (Agent.hs:3181-3183): state `(Nothing, Just e2ePubKey)`, `AgentConfirmation` -> `smpConfirmation`.
+- R5. `smpConfirmation`, initiating-party branch `RcvConnection … Just e2eEncryption` (Agent.hs:3405-3444): `getRatchetX3dhKeys`, `pqX3dhRcv` (Agent.hs:3408), `initRcvRatchet` (Agent.hs:3411), `createRatchet` (Agent.hs:3436), `setRcvQueueConfirmedE2E` (Agent.hs:3440); decrypts `AgentConnInfoReply` (Agent.hs:3420); `processConf` emits `CONF` (Agent.hs:3444).
+- R6. `allowConnection'` (Agent.hs:1467-1474): `acceptConfirmation`, then `ICAllowSecure` secures Q_A with Bob's sender key.
+- R7. `connectReplyQueues` (Agent.hs:3724-3737): `upgradeConn` creates Alice's send queue to Q_B; `agentSecureSndQueue` secures Q_B; `enqueueConfirmation … Nothing` (Agent.hs:3733) stores `AgentConnInfo aliceInfo` and sends `AgentConfirmation {e2eEncryption_ = Nothing, encConnInfo}` to Q_B. This is confirmation #2.
+
+Owner side, receiving confirmation #2 on Q_B:
+
+- O7. dispatch (Agent.hs:3182): `AgentConfirmation` -> `smpConfirmation`.
+- O8. `smpConfirmation`, accepting-party branch `DuplexConnection … Nothing` (Agent.hs:3447-3462): `agentRatchetDecrypt` with the established ratchet; `AgentConnInfo` -> `INFO` (Agent.hs:3452); `ICDuplexSecure` or `CON`.
+
+Completion is direct `CON` on `senderCanSecure` (SKEY) messaging-mode queues (the sender on `AgentConnInfo`, Agent.hs:2252; the receiver with no `senderKey`, Agent.hs:3459-3461); the separate `HELLO` via `helloMsg` (Agent.hs:3466) is the older non-`senderCanSecure` (duplexHandshake v2, in-band-securing) path.
+
+## Part 2 - the DR-from-address handshake, mapped to Part 1
+
+The address advertises Bob's Rcv X3DH parameters in link data (Part 3). Alice, when the address advertises them and versions are compatible, takes the joiner role; Bob takes the initiator role.
+
+Requester side - a new branch in `joinConnSrv … CRContactUri`, taken when the passed `Maybe AddressRatchetKeys` is present (the caller's plan-time `LGET`):
+
+- R2'. Replaces R2/R3. Read the passed bundle - `ratchetKeyId` and `e2eParams :: RcvE2ERatchetParamsUri 'C.X448` - and negotiate the concrete version with `compatibleVersion` against the client e2e range, as `compatibleInvitationUri` does (Agent.hs:1362-1368). Create the receive queue Q_A subscribed (`newRcvQueue` with `subMode`), messaging mode so Bob can secure it. Choose the requester's KEM with `replyKEM_ v ownerKem_ pqSup` (Ratchet.hs:839): if the bundle advertises a KEM (owner `IKUsePQ`) the requester `AcceptKEM` - a **double KEM**: it both encapsulates to the address KEM (ciphertext) and includes its own new KEM public key (`generateSndE2EParams` → `sntrup761Enc` + a fresh keypair, Ratchet.hs:433-435), so PQ is bidirectional from message 1; if the bundle has no KEM and the requester wants PQ, it `ProposeKEM` (its own key only, PQ from message 2 if the owner supports it). Run `generateSndE2EParams g v (replyKEM_ …)`, `pqX3dhSnd` against the negotiated parameters, `initSndRatchet`, `createSndRatchet` - the body of `createRatchet_` (Agent.hs:1343-1350), with parameters from the passed bundle rather than a received invitation.
+- R3'. Build `AgentConfirmation {e2eEncryption_ = Just aliceSndParams, ratchetKeyId = Just ratchetKeyId, encConnInfo = ratchetEncrypt(AgentConnInfoReply (Q_A :| []) aliceProfile)}` - the `mkAgentConfirmation`/`mkConfirmation` bodies (Agent.hs:3780-3765) with the reply queue being Alice's own Q_A. Send it to the address queue unauthenticated with `agentCbEncryptOnce`, one-shot (as `sendInvitation` sends, Agent/Client.hs:1929-1934) - **synchronous**, with the same send-failure UX as today's classic contact join. Nothing is stored: a retry (chat re-invokes the join → `mkJoinInvitation` reuses Q_A + keys, 1418) re-builds the confirmation, advancing the send ratchet, and the owner absorbs the advance - a **failed send** is skipped when the owner establishes the ratchet (`maxSkip = 512`, Ratchet.hs:988), and a **lost reply** carries the current content and updates the owner's request by `XContactId` (ContactRequest.hs:99-101, 269); both testable. The requester does **not** SKEY the address (`QMContact`, not `senderCanSecure`); rotation is handled because the passed params are the current advertised keys. **Alice's profile is now inside `encConnInfo`, under the ratchet.** Alice's connection is `RcvConnection` (Q_A) with a send ratchet, until she receives Q_B. This "New `RcvConnection` + `ratchets` row" is a new state (today a New `RcvConnection` holds x3dh keys but no ratchet - the classic initiator builds the ratchet only at R5, `createRatchet` Agent.hs:3436), and it composes: connection type is derived from queue rows alone while the `ratchets` table is keyed independently by `conn_id`, so subscription (Agent.hs:1551), `connectionStats` (2658), and `allowConnectionAsync'` (888) never read the ratchet for a `RcvConnection`; the only handshake reader on it is `smpConfirmation` (R5').
+
+### Deferred (future work): async delivery + connect UX
+
+The synchronous send above fails in the user's face on a lost reply (the same wart as today's classic contact join), even though the request may have been delivered. Making it async is a separate, later change, not part of this DR work:
+
+- Delivery cannot use the message-delivery worker: a `SndQueue` is unique per `(host, port, snd_id)` and belongs to one connection (schema PK), while a contact address is one queue that many connections send to, so no per-connection SndQueue to it can exist. It would go through the **async command worker**, keyed by `(connId, server)` (`getAsyncCmdWorker`, Agent.hs:1856-1858), which already retries the `JOIN` command (`tryMoveableCommand` → `retrySndOp`, 2016-2024); each retry re-runs `joinConnSrv` (re-build + ratchet advance, which the owner absorbs - above), so nothing is stored. (`joinConnSrvAsync` for `CRContactUri` is `CMD PROHIBITED` today, Agent.hs:1452, and the `JOIN` handler falls back to sync `joinConnSrv`, 1899-1902; the `TBC` at Agent.hs:1897 is about async *receive*-queue creation - Q_A - and is orthogonal.)
+- The async join returns "connecting" early and completes via the events chat already handles (`joinContact` sets `ConnJoined`; the DR requester emits `CONF` in R5' and the chat allows it, exactly as the classic contact requester, driving msg 3 → `CON`; a permanent send failure still surfaces as `ERR → ConnFailed`).
+- This needs a chat change: the join API takes a `CreatedConnLink` (full + short link), not the bare `ConnectionRequestUri` it takes today, so the agent can LGET-gate on the owner's server (a real reachability check) and verify the fetched `linkConnReq` equals the passed full link before reporting success. Used only for DR addresses (link data advertises `ratchetKeys`); old / non-DR addresses stay on the current sync path.
+
+Owner side - a new dispatch branch and a new receive handler:
+
+- O1'. In `processClientMsg` (Agent.hs:3176-3187), add a branch in state `(Nothing, Just e2ePubKey)`: an `AgentConfirmation` with `ratchetKeyId = Just _` **and** `e2eEncryption_ = Just _` on a `ContactConnection` -> `smpAddressConfirmation` (new). A `ratchetKeyId` without `e2eEncryption_` is ignored (it does not match this branch and falls through as a non-DR confirmation). It must be placed **before** the existing `(PHEmpty, AgentConfirmation) | senderCanSecure queueMode` case (Agent.hs:3182-3184), because a contact-address queue is `QMContact` (not `senderCanSecure`) and would otherwise fall into `prohibited "handshake: missing sender key"` (Agent.hs:3184). The address queue's `e2eDhSecret` stays `Nothing` (it is never set for a contact address - `smpInvitation` does not set it, Agent.hs:3609-3622), so every request is decrypted with its own ephemeral key via this `(Nothing, Just e2ePubKey)` path.
+- O2'. `smpAddressConfirmation` (new, modeled on `smpConfirmation` initiating branch, Agent.hs:3405-3444): select the private triple `(pk1, pk2, pKem)` by `ratchetKeyId` from `address_ratchet_keys`; `pqX3dhRcv pk1 pk2 pKem aliceSndParams`; `initRcvRatchet` with the address connection's stored `PQSupport` (`connPQEncryption` of the address `InitialKeys` - `On` for `IKUsePQ` and `IKPQOn`, `Off` for `IKPQOff`; this is what lets `IKPQOn` accept the requester's proposed KEM), combined with version compatibility as `smpConfirmation` derives `pqSupport'` (Agent.hs:3410); `rcDecrypt` of `encConnInfo` performs the first ratchet step, giving the ratchet its send side too (as it does for the initiator today), so the owner can later reply. Parse `AgentConnInfoReply (Q_A :| []) aliceProfile`. Store the request with `createInvitation` on the address connection (`contact_conn_id`), exactly as a classic invitation - except the request value is the `CRConfirmation` variant (Part 3) carrying the post-decrypt ratchet state and Q_A, and `recipient_conn_info` is `aliceProfile` - so **no connection or `ratchets` row is created at receive**, as with a classic invitation. Emit `REQ` with the `invitation_id`. A resend is not deduplicated: like a resent classic invitation it produces another `REQ` (the connect-UX fix for that is separate chat work). An unknown or expired `ratchetKeyId`, or a decryption failure: discard and acknowledge, as an undecryptable message is dropped today. This establishes ratchet state on unauthenticated input before the user accepts - see "Receive-time establishment, state, and abuse".
+- O3'. `acceptContact'` for a DR request - a new branch that continues the ratchet instead of `joinConn`. `getInvitation` returns the request; its `CRConfirmation` variant gives the stored ratchet state and Q_A. Create the connection now (as `joinConn` does for a classic invitation) and `createRatchet` (AgentStore.hs:1419) from the stored ratchet state. Reuse `mkAgentConfirmation` (Agent.hs:3780-3785) to create Bob's receive queue Q_B and return `AgentConnInfoReply (Q_B :| []) bobInfo`; create Bob's send queue to Q_A (`newSndQueue`, generating Bob's own sender key) and secure Q_A with `SKEY` using that key (`agentSecureSndQueue`, valid because Q_A is messaging mode) - the securing key is Bob's own, not taken from Alice's message; send the response to Q_A as `AgentConfirmation {e2eEncryption_ = Nothing, ratchetKeyId = Nothing, encConnInfo = ratchetEncrypt(AgentConnInfoReply (Q_B :| []) bobInfo)}` via `sendConfirmation` (`agentCbEncrypt` over Bob's send queue to Q_A, `PHEmpty` because Q_A is `senderCanSecure`) - exactly the current contact msg 2 path (Client.hs:1916), not `agentCbEncryptOnce`. The reply content is `AgentConnInfoReply`, not `AgentConnInfo`: it takes the `mkAgentConfirmation` path with `e2eEncryption_ = Nothing`, not the `enqueueConfirmation` path (which produces `AgentConnInfo`, Agent.hs:3789). `rejectContact'` deletes the `conn_invitations` row (the current behaviour), discarding the inline ratchet; no connection was created, so there is nothing else to clean up.
+
+Requester side, receiving the response on Q_A:
+
+- R5'. `smpConfirmation` needs a new branch `RcvConnection … Nothing` (today only `RcvConnection … Just` and `DuplexConnection … Nothing` exist, Agent.hs:3403-3447). It looks up the ratchet first (`getRatchet`) and, if there is none, falls through to `prohibited "conf: incorrect state"` - so a classic initiator (a New `RcvConnection` with x3dh keys but no ratchet) that receives a stray `Nothing`-confirmation keeps today's exact outcome; only a DR requester, which holds a send ratchet, takes the new path. Alice already holds the send ratchet, so `rcDecrypt` advances it and creates the receive side; parse `AgentConnInfoReply (Q_B :| []) bobInfo`. **This mirrors the classic contact requester exactly**: `setRcvQueueConfirmedE2E` on Q_A, `createRatchet` the advanced ratchet, store the reply as a `NewConfirmation`, and emit **`CONF`** - the app then calls `allowConnection'` (supplying `ownConnInfo` for msg 3), which drives `connectReplyQueues` (create Alice's send queue to Q_B, `SKEY`, upgrade to `DuplexConnection`, `enqueueConfirmation` the `AgentConnInfo` msg 3). Because Q_B is sender-securable, sending `AgentConnInfo` completes Alice with `CON` (Agent.hs:2252) - no `HELLO`. The only difference from the classic requester is that the ratchet is pre-built (from R2') rather than built from Bob's Snd params here, so there is no `CONF`-less auto-completion and no separate storage of Alice's own info.
+- R6'/completion. Unchanged from the current contact handshake, and modern (no `HELLO`). The exchange is three agent↔agent wire messages - Alice → address queue (msg 1), Bob → Q_A (msg 2, an `AgentConfirmation` carrying `AgentConnInfoReply` with Q_B), Alice → Q_B (msg 3, an `AgentConfirmation` carrying `AgentConnInfo`) - the same shape as the current contact flow, where msg 1 was `AgentInvitation`; here it is the ratchet-establishing `AgentConfirmation`. (`CON` is not a wire message - it is the agent→app event; `HELLO` and `AgentConnInfo` are the wire messages.) `HELLO` belongs to the older non-`senderCanSecure` path (duplexHandshake v2, before SKEY): there the confirmation secures the queue in-band (`PHConfirmation` carries the sender key, Client.hs:1918) and the receiver replies with `HELLO` (`ICDuplexSecure` → `enqueueDuplexHello`, Agent.hs:3457-3458). Both Q_A and Q_B here are messaging-mode - Q_A by R2', Q_B via `createReplyQueue` → `SCMInvitation` → `QMMessaging` (Agent.hs:1233,1458,3783) - so the sender secures with SKEY and sends `PHEmpty` (Client.hs:1918), the dispatch takes the `senderCanSecure` branch (Agent.hs:3182-3184), and each agent raises the `CON` app event locally off msg 3 - Bob on receiving it (`senderKey = Nothing`, Agent.hs:3459-3461), Alice on sending it (Agent.hs:2252) - with no separate `HELLO` wire message. (msg 2's `AgentConnInfoReply` only sets Q_A `Confirmed`, Agent.hs:2254.) Invitations are two messages because the initiator's queue is already in the link; a contact address needs three because Bob's receive queue Q_B is only delivered in msg 2. The third message no longer has a ratchet role: Bob's X3DH params are pre-published, so the agreement is complete once Bob receives msg 1 (in the current flow Bob's Snd params instead arrive in msg 2). msg 2 and msg 3 are queue setup - msg 2 delivers Q_B, msg 3 secures Q_B so Alice can send to Bob and signals Bob's `CON`; neither negotiates the ratchet. A one-directional exchange (the RPC) needs no Q_B and is two messages.
+
+Net code touch points: `joinConnSrv` (new requester branch), `processClientMsg` (new owner dispatch), `smpConfirmation` (new `RcvConnection … Nothing` branch and `AgentConnInfoReply` acceptance), `acceptContact'` (new continue-ratchet branch), a new `smpAddressConfirmation` reusing `createInvitation`/`getInvitation` with the sum request value, and the link data and storage of Part 3-4. `rejectContact'` is unchanged (it deletes the `conn_invitations` row either way).
+
+### Receive-time establishment, state, and abuse
+
+This is the substantive departure from the current flow. Today `smpInvitation` creates only a lightweight `NewInvitation` and emits `REQ` (Agent.hs:3618-3621); no connection or ratchet exists until the user accepts. For DR the request is under the ratchet, so to show the requester's profile in `REQ` the owner must decrypt it, which means establishing the ratchet at **receive**, before accept.
+
+Design decision (Q1): decrypt at receive. Both use cases need the request content at `REQ` - a person decides to accept from the profile, and a service bot needs the request payload to act. Deferring decryption to accept would make `REQ` contentless and does not fit the service case, so it is not done.
+
+Consequences:
+
+- No connection is created at receive, exactly as for a classic invitation. O2' stores the request with `createInvitation` on the address connection; the post-decrypt ratchet state and Q_A live inline in the `CRConfirmation` request value (`cr_invitation`). O3' (accept) creates the connection, `createRatchet` from the stored state, and adds Bob's queues, becoming a `DuplexConnection`; `rejectContact'` deletes the `conn_invitations` row.
+- Per incoming `AgentConfirmation` the owner does one `pqX3dhRcv` (three DH plus, with PQ, one `sntrup761` decapsulation) and one `rcDecrypt`, on unauthenticated input, and writes one `conn_invitations` row - more CPU than the current `NewInvitation`, the same order of state (no connection, no `ratchets` row until accept).
+
+Abuse (Q2): a contact address already accepts and processes unauthenticated invitations today, so this is a degree-worse version of an existing surface, not a new class. It is bounded by the address queue quota (an attacker fills it, the owner drains and acknowledges) and, optionally, by basic auth on the address (already supported for contact addresses, `optBasicAuth`). The per-request state is a single `conn_invitations` row - the same class as a classic contact request - so it is subject to the same limits and lifecycle, with no DR-specific dedup or TTL. Proof-of-work or a stricter gate can be added later; it is out of scope here and noted as a follow-up.
+
+`acceptContact'`/`rejectContact'` keep taking the `invitation_id` from `REQ` unchanged; the only difference is that `getInvitation` returns a request that is either a `CRInvitation` URI (current `joinConn` path, O3-O6) or a `CRConfirmation` (continue-ratchet path, O3'). Nothing in the `REQ`/accept/reject flow or the chat client changes - the change is contained in the agent.
+
+### The four communication layers, per message (verified against code)
+
+Layers, outermost (server-visible) first:
+
+- **L1 `ClientMsgEnvelope`** (Protocol.hs:1089), `PubHeader {phVersion, phE2ePubDhKey :: Maybe PublicKeyX25519}` (1096) - **this is where per-queue encryption is agreed** (not L2). `phE2ePubDhKey` is the sender's e2e DH public key; the recipient combines it with the queue's e2e private key: `(e2eDhSecret, e2ePubKey_) -> (Nothing, Just e2ePubKey) -> e2eDh = dh' e2ePubKey e2ePrivKey` (Agent.hs:3172-3178). `agentCbEncryptOnce` (Client.hs:2214) puts a **fresh ephemeral** pubkey (generated 2217, set 2223) - used when the sender has no send queue (the address queue), whose `e2eDhSecret` stays `Nothing`, so it decrypts every message with the per-message ephemeral. `agentCbEncrypt` (Client.hs:2203) puts the **send queue's persistent** e2e pubkey (`Just` on a confirmation, 2210); the recipient stores the secret via `setRcvQueueConfirmedE2E`, and *later* messages send `phE2ePubDhKey = Nothing` (`sendAgentMessage`, 2080).
+- **L2 `ClientMessage PrivHeader`** (Protocol.hs:1113), `PrivHeader = PHConfirmation APublicAuthKey | PHEmpty` (1115) - **queue securing / authorization, not encryption**. `PHConfirmation` carries the sender's AUTH key for in-band securing (v2, non-`senderCanSecure`); `PHEmpty` when the sender secured the queue with SKEY out-of-band. `PHEmpty` on every message here is about securing, and says nothing about encryption (that is L1). Set in `sendConfirmation` (Client.hs:1918), `sendInvitation` (1934), `sendAgentMessage` (2079).
+- **L3 `AgentMsgEnvelope`** (Agent/Protocol.hs:829, encoding 851) - outside the ratchet. `AgentConfirmation` ('C') carries `e2eEncryption_` (Snd X3DH params, agrees DR) + `encConnInfo`; `AgentInvitation` ('I') carries `connReq` (Rcv X3DH params) + plaintext `connInfo` (no DR); `AgentMsgEnvelope` ('M') carries `encAgentMessage`.
+- **L4 `AgentMessage`** (Agent/Protocol.hs:883, encoding 893) - inside the ratchet. `AgentConnInfo` ('I'), `AgentConnInfoReply` ('D', reply queues + info), `AgentMessage APrivHeader AMessage` ('M'; `AMessage` includes `HELLO`, Agent/Protocol.hs:1018-1020). **Absent when L3 is `AgentInvitation`** (that profile is per-queue-only - the gap this plan closes).
+
+Send routing: msg 1 (to address) → `sendInvitation` today / a new `agentCbEncryptOnce` confirmation send for DR; msg 2 → `secureConfirmQueue` → `sendConfirmation` (Agent.hs:3747); msg 3 → `connectReplyQueues` → `enqueueConfirmation` → delivery worker `AM_CONN_INFO` → `sendConfirmation` (Agent.hs:3733,3789,2183). `AM_CONN_INFO`/`AM_CONN_INFO_REPLY` both go through `sendConfirmation` (2183-2184); other `AMessage`s go through `sendAgentMessage` wrapping `AgentMsgEnvelope` 'M' (2192-2193).
+
+Current contact handshake (address does **not** advertise DR):
+
+| msg | L1 `PubHeader.phE2ePubDhKey` (per-queue enc) | L2 `PrivHeader` (securing) | L3 `AgentMsgEnvelope` | L4 `AgentMessage` |
+|---|---|---|---|---|
+| 1 Alice→addr | `Just` fresh ephemeral, `agentCbEncryptOnce` (Client.hs:1933,2223) | `PHEmpty` (1934) | `AgentInvitation` {connReq = Alice Rcv params, connInfo = profile} (Client.hs:1932) | — none (profile per-queue only) |
+| 2 Bob→Q_A | `Just` Bob's send-queue e2e pubkey, `agentCbEncrypt` (1920,2210) | `PHEmpty` [`senderCanSecure`] (1918) | `AgentConfirmation` {e2eEncryption_ = **Just Bob Snd params**, encConnInfo} (Agent.hs:3765) | `AgentConnInfoReply` (Q_B) bobInfo, DR-enc (Agent.hs:3785) |
+| 3 Alice→Q_B | `Just` Alice's send-queue e2e pubkey, `agentCbEncrypt` (1920,2210) | `PHEmpty` [`senderCanSecure`] (1918) | `AgentConfirmation` {e2eEncryption_ = **Nothing**, encConnInfo} (Agent.hs:3802) | `AgentConnInfo` aliceInfo, DR-enc (Agent.hs:3789) |
+
+New DR handshake (address advertises DR):
+
+| msg | L1 `PubHeader.phE2ePubDhKey` (per-queue enc) | L2 `PrivHeader` (securing) | L3 `AgentMsgEnvelope` | L4 `AgentMessage` |
+|---|---|---|---|---|
+| 1 Alice→addr | `Just` fresh ephemeral, `agentCbEncryptOnce` [same] | `PHEmpty` [same] | **`AgentConfirmation`** {e2eEncryption_ = **Just Alice Snd params**, **ratchetKeyId = Just**, encConnInfo} [was `AgentInvitation`] | **`AgentConnInfoReply`** (Q_A) aliceProfile, **DR-enc** [was plaintext connInfo] |
+| 2 Bob→Q_A | `Just` Bob's send-queue e2e pubkey, `agentCbEncrypt` [same] | `PHEmpty` [same] | `AgentConfirmation` {e2eEncryption_ = **Nothing**, ratchetKeyId = Nothing, encConnInfo} [was Just Bob Snd params] | `AgentConnInfoReply` (Q_B) bobInfo, DR-enc [same] |
+| 3 Alice→Q_B | `Just` Alice's send-queue e2e pubkey, `agentCbEncrypt` [same] | `PHEmpty` [same] | `AgentConfirmation` {e2eEncryption_ = Nothing, encConnInfo} [same] | `AgentConnInfo` aliceInfo, DR-enc [same] |
+
+Net difference: **only msg 1 and msg 2's L3/L4 change.** msg 1's L3 becomes `AgentConfirmation` (was `AgentInvitation`) carrying Alice's Snd params + `ratchetKeyId`, and the profile moves from plaintext L3 to DR-encrypted L4 (`AgentConnInfoReply`) - the whole point of the change. msg 2 drops `e2eEncryption_` (Bob no longer sends Snd params - the ratchet is agreed from msg 1). msg 3 is unchanged. L1 (per-queue encryption - each queue agrees its own secret via the sender's e2e pubkey in the `PubHeader` on the first message to it) and L2 (securing, `PHEmpty` because SKEY is used) are unchanged throughout; the DR change is entirely at L3/L4. The only new send code is msg 1 (an `AgentConfirmation` fired to the address with `agentCbEncryptOnce`, like `sendInvitation` but with a confirmation envelope).
+
+## Part 3 - types and link data
+
+### Fixed data - unchanged
+
+`FixedLinkData` (Protocol.hs:1824) is not touched. The double-ratchet keys go entirely in mutable data, so an existing address advertises them without a new link (the fixed data is hash-committed and cannot change). Fixed data keeps only `agentVRange`, `rootKey`, `linkConnReq`, `linkEntityId`.
+
+### Mutable data - ratchet keys bundle
+
+Appended to `UserContactData` (Protocol.hs:1840); the encoding stops at a trailing tail (Protocol.hs:1981), so earlier versions ignore it:
+
+```haskell
+newtype RatchetKeyId = RatchetKeyId ByteString -- opaque short id; one Encoding instance, shared below
+
+data AddressRatchetKeys = AddressRatchetKeys
+  { ratchetKeyId :: RatchetKeyId, -- identifies this bundle; changes on rotation, echoed in the request
+    e2eParams :: CR.RcvE2ERatchetParamsUri 'C.X448 -- version range + both X3DH keys + optional KEM
+  }
+instance Encoding AddressRatchetKeys where ... -- the key-bundle instance; both fields required
+
+data UserContactData = UserContactData
+  { direct :: Bool, owners :: [OwnerAuth], relays :: [ConnShortLink 'CMContact],
+    userData :: UserLinkData,
+    ratchetKeys :: Maybe AddressRatchetKeys -- whole bundle optional, one Encoding instance
+  }
+```
+
+`e2eParams` is the existing `RcvE2ERatchetParamsUri 'C.X448` (`E2ERatchetParamsUri VersionRangeE2E k1 k2 (Maybe (RKEMParams s))`, Ratchet.hs:282-286) - the same type a `CRInvitationUri` advertises - with `StrEncoding`/`Encoding` already defined (Ratchet.hs:302-374). There is no bespoke key type and no reconstruction: the requester negotiates the concrete version with `compatibleVersion` against its own e2e range, exactly as `compatibleInvitationUri` does for an invitation (Agent.hs:1362-1368), giving `RcvE2ERatchetParams` for `pqX3dhSnd`. The KEM is optional: `Nothing` gives an X448-only ratchet (as when `PQSupport` is off), `Just` a hybrid one, matching `generateRcvE2EParams`'s `PQSupport` gate (Ratchet.hs:439-445).
+
+The address-creation parameter is `InitialKeys` (Ratchet.hs:864) - the same 3-way choice as invitations, not a bare `PQSupport`. Currently `IKUsePQ` is prohibited for `SCMContact` (Agent.hs:990,1198) because a contact address carries no owner keys; this change lifts that prohibition. The bundle plays the published-contact-request role, so its KEM follows `initialPQEncryption False pqInitKeys` (Ratchet.hs:882) - exactly as the requester's contact request does today (Agent.hs:1422):
+
+- `IKUsePQ` - the bundle advertises the KEM; the requester encapsulates to it, so PQ from message 1.
+- `IKPQOn` (`IKLinkPQ PQSupportOn`) - the bundle is X448-only (no KEM advertised), but the owner's ratchet supports PQ (`connPQEncryption` = On, Ratchet.hs:888); the requester proposes its own KEM (R2'), so PQ from message 2.
+- `IKPQOff` (`IKLinkPQ PQSupportOff`) - X448-only, and the owner's ratchet does not support PQ even if the requester proposes it.
+
+Advertising the KEM adds ~1158 B to the rotated, widely-fetched link data, which is why `IKPQOn` exists (PQ one round later, without the size cost). The owner generates the bundle with `generateRcvE2EParams g v (initialPQEncryption False pqInitKeys)` (Ratchet.hs:439), stores the private triple `(pk1, pk2, pKem)` (Part 4), and advertises `e2eParams` by wrapping the public `E2ERatchetParams` in the address's e2e version range (`toVersionRangeT`; or `mkRcvE2ERatchetParams` from the stored privates, Ratchet.hs:412) - the same private-key shape `createRatchetX3dhKeys`/`getRatchetX3dhKeys` already store (AgentStore.hs:1362-1367). `ratchetKeys` is set by the agent when it signs mutable link data (`Crypto.ShortLink.encodeSignUserData`), not by the application.
+
+### Authentication of the advertised keys
+
+No signature is added on the keys: the mutable link data already signs them. `decryptLinkData` (Crypto/ShortLink.hs:106-114) verifies `sig2` over the mutable `UserContactData` by `rootKey`, so `ratchetKeys` is root-signed. This is the X3DH anti-substitution property: an SMP server cannot substitute the keys without forging the root signature. The signer is the root Ed25519 key (the address's signing identity); the X3DH keys are separate DH keys (X448, which cannot sign). A single owner signs address data ("we don't use multiple owners"), so the root signature alone is sufficient - no per-key signature. A malicious server can still serve an older but validly-signed `UserContactData` (rollback to a retired bundle); this is bounded by the retention window and by the ratchet advancing after the first message, and a signature does not prevent it. Inline ratchet params in a `CRInvitationUri` contact request are not in signed link data and remain unsigned - a separate change, out of scope here.
+
+### Request envelope
+
+`AgentConfirmation` (Protocol.hs:830-834) gains an optional `ratchetKeyId` - the `ratchetKeyId` of the `AddressRatchetKeys` bundle the requester used, so the owner selects the matching private keys:
+
+```haskell
+AgentConfirmation
+  { agentVersion :: VersionSMPA,
+    e2eEncryption_ :: Maybe (SndE2ERatchetParams 'C.X448), -- reused: Alice's Snd params in DR msg 1
+    ratchetKeyId :: Maybe RatchetKeyId,                    -- selects the owner's key generation
+    encConnInfo :: ByteString
+  }
+```
+
+`ratchetKeyId` is a separate optional selector (the shared `RatchetKeyId` newtype), not the bundle - the owner already holds the published public bundle and looks up its private keys by this id. It reuses the existing `e2eEncryption_` for Alice's Snd params rather than a new combined bundle; the minor cost is two correlated `Maybe`s (`ratchetKeyId = Just` is only meaningful with `e2eEncryption_ = Just`). **A `ratchetKeyId` with `e2eEncryption_ = Nothing` is ignored** - O2' requires both (there are no Snd params to run `pqX3dhRcv`), so such a message falls through to the current dispatch as if it had no `ratchetKeyId`.
+
+Encoding (extends Protocol.hs:853-866): from `addressDRVersion`, `smpEncode (agentVersion, 'C', e2eEncryption_, ratchetKeyId, Tail encConnInfo)`; `ratchetKeyId` is `Just` for an address-DR confirmation and `Nothing` for the current joiner-to-initiator and initiator-to-joiner confirmations; earlier versions omit the field entirely and use `smpEncode (agentVersion, 'C', e2eEncryption_, Tail encConnInfo)`. Parsing gates the field on `agentVersion`. `CRInvitationUri` is unchanged - a connection request URI holds Rcv parameters and must not hold Snd parameters.
+
+### Stored request - the invitation record
+
+The `conn_invitations` record stays; only the type of the stored request widens. From chat's point of view a DR request is still an invitation - it "contains a confirmation" instead of an invitation URI - so `REQ`, `acceptContact'`/`rejectContact'`, and the chat side are unchanged; the change is contained in the agent. The `NewInvitation`/`Invitation` request field (`cr_invitation`, stays `NOT NULL`) becomes a sum:
+
+```haskell
+data ContactRequest
+  = CRInvitation (ConnectionRequestUri 'CMInvitation) -- classic: joinConn on accept (O3-O6)
+  | CRConfirmation DRRequest                          -- DR: continue the ratchet on accept (O3')
+
+data DRRequest = DRRequest
+  { drRatchet :: RatchetX448,        -- post-decrypt receiving ratchet (with send side), stored inline
+    drReplyQueue :: SMPQueueInfo,    -- Q_A, where the owner replies
+    drAgentVersion :: VersionSMPA,   -- negotiated at receive; needed to build the connection shell at accept
+    drPQSupport :: PQSupport         -- the address's PQ setting for this connection
+  }
+```
+
+`recipient_conn_info` holds the profile in both cases. `getInvitation`/`createInvitation` carry `ContactRequest`; `acceptContact'` branches on the constructor. There is no dedup column: a resent request produces another `REQ`, exactly as a resent classic invitation does.
+
+`drAgentVersion`/`drPQSupport` are stored because the accept flow creates the connection **shell** through `newConnToAccept` → `newConnToJoin` (via `prepareConnectionToAccept`, called by chat's sync accept before `acceptContact'`, Internal.hs:914,925) and `newConnToJoin` today derives `connAgentVersion`/`pqSupport` from the `ConnectionRequestUri` (Agent.hs:1277-1293); a `CRConfirmation` has no URI, so the values negotiated at receive (O2') are stored and used to build the shell.
+
+Three readers of the widened `connReq` field (all via `getInvitation`) branch on the constructor:
+- `acceptContact'` (Agent.hs:1479, sync): `CRInvitation cr` → `joinConn … cr` (classic, unchanged); `CRConfirmation dr` → the O3' continue-ratchet path.
+- `newConnToAccept` (Agent.hs:1296, via `prepareConnectionToAccept`): `CRInvitation cr` → `newConnToJoin … cr` (unchanged); `CRConfirmation dr` → create the `NewConnection` shell from `drAgentVersion`/`drPQSupport` (`createNewConn`, generating the connId).
+- `acceptContactAsync'` (Agent.hs:900): `CRInvitation cr` → `joinConnAsync … cr` (unchanged); `CRConfirmation _` → `throwE $ CMD PROHIBITED` (async DR accept is deferred; DR requests accept synchronously). Chat's REQ/accept is unaffected either way - it only ever passes `invId`, never the `ContactRequest`, which stays internal to the agent.
+
+Storage: `cr_invitation`'s `ToField`/`FromField` encode `CRInvitation` as the legacy `strEncode` URI (unchanged from before, so the format is downgrade-safe) and `CRConfirmation` as JSON (`J.encode` of `DRRequest`). `FromField` peeks the first byte: `{` → JSON `CRConfirmation`, else `strDecode` → `CRInvitation` (a URI never starts with `{`). `DRRequest` uses manual `ToJSON`/`FromJSON` (extensible), and `SMPQueueInfo` gets a base64 `StrEncoding` + JSON to sit inside it. `smpInvitation` (Agent.hs:3618) wraps its `connReq` in `CRInvitation`; `smpAddressConfirmation` (O2') writes `CRConfirmation`.
+
+## Part 4 - key rotation (client-driven)
+
+Rotation is independent of the handshake above and is driven by the client app, not the agent. The agent never rotates on its own - it lacks the app's intent and the mutable link data (profile/badge and other short-link data). The app rotates by calling `setConnShortLink` with the rotate flag; the whole ratchet-keys bundle - both X448 keys and the KEM - is generated fresh each time.
+
+### Schema
+
+```sql
+-- one row per ratchet-keys generation for an address; the current generation plus the most recent retained ones.
+-- private side of the advertised RcvE2ERatchetParamsUri - same shape as the ratchets x3dh
+-- columns and createRatchetX3dhKeys (AgentStore.hs).
+CREATE TABLE address_ratchet_keys(
+  address_ratchet_key_id INTEGER PRIMARY KEY AUTOINCREMENT,
+  conn_id BLOB NOT NULL REFERENCES connections ON DELETE CASCADE,
+  ratchet_key_id BLOB NOT NULL,     -- the published id echoed by requests
+  x3dh_priv_key_1 BLOB NOT NULL,    -- X448
+  x3dh_priv_key_2 BLOB NOT NULL,    -- X448
+  pq_priv_kem BLOB,                 -- RcvPrivRKEMParams (sntrup761 keypair); NULL when PQ is off for this address
+  created_at TEXT NOT NULL DEFAULT(datetime('now'))
+);
+CREATE UNIQUE INDEX idx_address_ratchet_keys ON address_ratchet_keys(conn_id, ratchet_key_id);
+
+-- a DR request stays in conn_invitations with NO schema change: cr_invitation now holds a ContactRequest
+-- sum (an invitation URI or a confirmation carrying the post-decrypt ratchet + reply queue), so it stays
+-- NOT NULL - no nullable change, no new column on conn_invitations, no new table for the request.
+```
+
+`cr_invitation` stays `NOT NULL` - only its decoded value gains a variant (Part 3), so the invitations flow, `REQ`, and chat are unchanged; the only new storage is the `address_ratchet_keys` table. The link signing key is already on the address queue (`rcv_queues.link_priv_sig_key`, M20250322), so nothing is added there - rotation and retrofit re-sign mutable data with it. PostgreSQL mirrors this. Migration `M20260712_address_dr`.
+
+### Rotation logic
+
+The app rotates by calling `setConnShortLink` with the rotate flag; there is no automatic, agent-driven rotation. On rotation:
+
+1. `generateRcvE2EParams` for a fresh generation - two X448 keys, and an sntrup761 keypair only if PQ is on for this address - with a fresh `ratchetKeyId`.
+2. Recompute mutable link data with the new `AddressRatchetKeys` (the public `e2eParams`), re-sign with the root key (`encodeSignUserData`, key from `rcv_queues.link_priv_sig_key`), and `LSET` it to the address queue (`setConnShortLink` path).
+3. Insert the new `address_ratchet_keys` row (`x3dh_priv_key_1`, `x3dh_priv_key_2`, `pq_priv_kem`).
+
+### Retention
+
+Retention is count-based, not time-based: on each rotation `deleteOldAddressRatchetKeys` keeps the newest `keepAddressKeys` generations per address (default 3, ordered by `address_ratchet_key_id`) and deletes older ones. There is no `retired_at` column, no time window, and no `cleanupManager` step. A request that used a recently-retired bundle still decrypts while that generation is retained; how long a recorded first message stays decryptable after a compromise of the current private keys is therefore bounded by the retained-generation count and the app's rotation cadence (both app-controlled). Unaccepted DR request rows in `conn_invitations` are handled exactly like unaccepted classic invitation requests - no DR-specific cleanup (a DR request is one `conn_invitations` row, the same class of state as a classic contact request).
+
+## Part 5 - backward compatibility
+
+- A requester older than `addressDRVersion`, or an address without `ratchetKeys`, uses R2/R3 (`AgentInvitation`); the owner uses O1-O8. Unchanged.
+- The owner dispatches on the envelope: `AgentInvitation` -> `smpInvitation` (current); `AgentConfirmation` with `ratchetKeyId` on a `ContactConnection` -> `smpAddressConfirmation` (new). Both coexist.
+- `AgentConfirmation` without `ratchetKeyId` remains the current confirmation on established connections.
+- An existing address gains `ratchetKeys` via a new agent API (e.g. `updateContactAddressLink`) that the app calls with the mutable link data (profile/badge and any other short-link data): the agent generates the DR bundle if absent (the first `address_ratchet_keys` row and its stored private keys), adds `ratchetKeys` to `UserContactData`, re-signs with `rcv_queues.link_priv_sig_key`, and `LSET`s it. **Only mutable data changes - the address (link) is unchanged**, because the keys are in mutable, not fixed, data. Requesters that fetch the updated data use DR; older ones still use `AgentInvitation`. The agent does not do this on its own (it lacks the profile and the user's intent); the app drives it, combined with the full→short address migration.
+
+## Part 6 - tests
+
+- Encoding roundtrips: `UserContactData` with and without `ratchetKeys`, and with the KEM present and absent; `AgentConfirmation` with and without `ratchetKeyId`, across versions.
+- Address creation advertises `ratchetKeys` (the `RcvE2ERatchetParamsUri`); `decryptLinkData` (Crypto/ShortLink.hs:100) verifies signatures and the requester negotiates the advertised params to a concrete version, with and without the KEM.
+- Both PQ modes: an address whose bundle carries a KEM gives a hybrid ratchet (`pqEncryption` on); one without gives an X448-only ratchet.
+- End to end: a DR-advertising address; a new requester establishes the ratchet, sends its profile under it, owner emits `REQ`, accepts, both reach `CON`; assert the profile never travels under per-queue-only encryption; assert `pqEncryption` on.
+- Rotation and retrofit: request against the current bundle; against a just-retired bundle within the window still decrypts; against a bundle past the window is discarded and the requester times out; an address that adds `ratchetKeys` via `LSET` is then reached by DR while an old requester still uses `AgentInvitation`.
+- Backward compatibility: old requester against a DR address connects via `AgentInvitation`; new requester against a non-DR address falls back to `AgentInvitation`.
+
+## Part 7 - phases
+
+1. Link data: `AddressRatchetKeys` in `UserContactData` (reusing `RcvE2ERatchetParamsUri`), encoding, `encodeSignUserData`; `AgentConfirmation.ratchetKeyId`; address creation taking `InitialKeys` (lifting the `IKUsePQ`-for-`SCMContact` prohibition), generating (`generateRcvE2EParams`, KEM per `initialPQEncryption False`) and storing the first `address_ratchet_keys` row.
+2. Handshake: thread the optional `Maybe AddressRatchetKeys` through `joinConnection`/`joinConn`/`joinConnSrv` (present → DR branch, absent → classic); requester R2'/R3' (synchronous one-shot send); owner O1'/O2'/O3' storing the DR request as a `conn_invitations` row whose request value is the `CRConfirmation` variant; `smpConfirmation` `RcvConnection … Nothing` branch and `AgentConnInfoReply` acceptance; end-to-end connection with the profile under the ratchet. Tests pass `AddressRatchetKeys` directly (chat wiring is later work).
+3. Rotation and retrofit: schema migration, `rotateRatchetKeys`, retention window, cleanup step, app-driven `LSET` retrofit (with the full→short address migration), rotation/retrofit tests.

plans/2026-07-24-signed-service-requests.md

@@ -0,0 +1,59 @@
+# Signed service requests
+
+Optional Ed25519 signature on service RPC requests, constructed and verified in the agent (not the bot), bound to the request's double ratchet. Requests only — responses stay authenticated by the address/ratchet. Signing is optional; a bot decides whether to require it. The agent is stateless: the meaning of a signer key (identity, resource) is the bot's concern.
+
+## Wire — `Simplex.Messaging.Agent.Protocol`
+
+Extend the existing `'A'` inner message; `Maybe` absent = unsigned, so the unsigned path is unchanged:
+
+```haskell
+AgentServiceRequest (NonEmpty SMPQueueInfo) (Maybe RequestSignature) MsgBody
+
+data RequestSignature = RequestSignature C.PublicKeyEd25519 (C.Signature 'C.Ed25519)
+```
+
+## Binding
+
+```
+binding = sha3-256("SimpleXService" <> rcAD)
+sig     = Ed25519.sign(sk, binding <> payload)
+```
+The service recomputes `binding` from its own `rcAD` and verifies.
+
+- `rcAD` = the ratchet associated data (`Ratchet.rcAD`) — the shared connection security code: identical on both ratchets by construction (`pubKey(requester ephemeral) <> pubKey(service key)`), stable, and unique per request (fresh requester ephemeral). Already on the ratchet; nothing derived or stored.
+- sha3-256 here is not for uniformity or secrecy (both moot: the value is signed, not keyed, and only a ratchet holder can craft a valid request). It gives a canonical fixed-length, domain-tagged binding; the 32-byte fixed prefix also makes `binding <> payload` unambiguous.
+- Domain string `"SimpleXService"`: separates this signature from other uses of the signing key.
+- Not covered: reply queues (the AEAD protects them in transit; addresses may use redundant queues).
+- Anti-relay: a signature bound to one session's rcAD does not verify under another's (both parties' keys differ). Replay of the encrypted blob is handled separately by transport dedup.
+
+## Sign (requester) — `Simplex.Messaging.Agent`
+
+- `sendServiceRequest` / `sendServiceRequestAsync` gain a `Maybe` Ed25519 signing key.
+- `joinConnSrv'` DR path takes the ratchet straight from the `createRatchet_`/`getSndRatchet` line (both now yield `(RatchetX448, params)`) and computes `serviceReqBinding` from its `rcAD`; the `mkInner :: SMPQueueInfo -> ByteString -> AgentMessage` closure calls `signServiceReq signKey_ binding payload` — `RequestSignature pub (sign' pk (binding <> payload))` when a key is given, `Nothing` otherwise.
+- Async carries the key in `JRServiceReq {requestKey :: Maybe C.PrivateKeyEd25519}` (enabled `StrEncoding (PrivateKey Ed25519)`); the JOIN worker deserializes it and signs after building the ratchet.
+
+**Status:** implemented and tested in simplexmq-3 (sync + async); invalid signature → `A_SERVICE ASEBadSignature` (logs + `ERR` event), no invitation.
+
+## Verify (service) — `smpContactRequest`
+
+After `initRcvRatchet_` + decrypt, on `AgentServiceRequest (replyQueue :| _) sig_ payload`, one helper does the check:
+
+`verifyServiceReq rc payload sig_ :: Either String (Maybe C.PublicKeyEd25519)`
+- `Nothing` → `Right Nothing` (unsigned).
+- `Just (RequestSignature key sig)` → recompute `serviceReqBinding rc` and `C.verify' key sig (binding <> payload)`; `Right (Just key)` if valid, else `Left err`.
+
+Then:
+- `Right key_` → `storeInvitation … True` + `notify $ SREQ invId key_ payload`.
+- `Left err` → `logError` + `notify (ERR (AGENT (A_SERVICE ASEBadSignature)))`, no invitation.
+
+Dedup unchanged.
+
+## Event / API
+
+- `SREQ :: InvitationId -> Maybe C.PublicKeyEd25519 -> MsgBody -> AEvent AEConn`.
+- `StrEncoding (PrivateKey Ed25519)` enabled so a caller (e.g. via `JRServiceReq`) can carry the signing key.
+
+## Tests
+
+- `testSignedServiceRequest` (sync) + `testSignedServiceRequestAsync` — signed round-trip delivers the exact signer key on `SREQ` (`sigKey_ == Just signPub`).
+- Unsigned path unchanged (existing service tests carry `Nothing` for the new field).

plans/2026-08-05-proxy-relay-error-attribution.md

@@ -0,0 +1,81 @@
+## Root cause: PRXY errors are attributed to the forwarding server instead of the destination relay
+
+When private routing is enabled and the destination relay is unreachable, the client reports
+**"Error connecting to forwarding server smp5.simplex.im"** — naming a preset server that the client
+connected to successfully. Retrying rotates to the next proxy (`getNextServer`, `Agent/Client.hs:689`)
+and produces the same message with a different preset server, so the destination server is never named
+and the failure looks like an outage of our own infrastructure.
+
+### Reproduction
+
+Connecting to a contact address on an unresolvable host (`simplex.server.home`, no DNS record):
+
+```
+-- private routing off (correct)
+BROKER {brokerAddress = "smp://VvXX…@simplex.server.home:5223",
+        brokerErr = NETWORK {networkError = NEConnectError {connectError = "…does not exist (Name or service not known)"}}}
+
+-- private routing on (misattributed)
+SMP {serverAddress = "smp://…@smp5.simplex.im,…onion",
+     smpErr = PROXY {proxyErr = BROKER {brokerErr = NETWORK {networkError = NEFailedError}}}}
+```
+
+### The asymmetry between the two proxied paths
+
+A server returns `PROXY (BROKER …)` only from `smpProxyError` (`Client.hs:804-815`), which is called
+exclusively where the proxy failed to reach the relay — `PRXY` (`Server.hs:1444`) and `PFWD`
+(`Server.hs:1466`). The error therefore *always* describes the proxy→relay hop. The two paths then
+diverge in how the agent wraps it:
+
+**PFWD — keeps both addresses** (`Agent/Client.hs:1183-1189`): the proxy's error arrives as
+`Left ProxyClientError` and is thrown as `PROXY {proxyServer, relayServer, proxyErr}`.
+
+**PRXY — drops the relay** (`Agent/Client.hs:713`): `connectSMPProxiedRelay` has no `Either` layer, so
+the error arrives as `PCEProtocolError` and `liftClient SMP` maps it to `SMP <proxyAddr> (PROXY …)`
+(`Agent/Client.hs:1244`). The destination address is discarded.
+
+Both clients read the second shape as a client→proxy failure and word it accordingly
+(`SimpleXAPI.kt:2692`, `ErrorAlert.swift:117`), which is never what it means.
+
+### Fix
+
+In `newProxiedRelay`, map proxy-reported `PROXY (BROKER …)` errors to the same shape `PFWD` already
+produces:
+
+```haskell
+proxyRelayError :: HostName -> ErrorType -> AgentErrorType
+proxyRelayError proxyHost = \case
+  e@(SMP.PROXY (SMP.BROKER _)) -> PROXY {proxyServer = protocolClientServer smp, relayServer = …destSrv, proxyErr = ProxyProtocolError e}
+  e -> SMP proxyHost e
+```
+
+`liftClient` applies this only to `PCEProtocolError`, so genuine client↔proxy failures (response
+timeout, network error, proxy transport version) still map to `BROKER <proxy> …` and remain attributed
+to the proxy. Both apps already render the resulting shape correctly, with no client change:
+*"Forwarding server smp5.simplex.im failed to connect to destination server simplex.server.home."*
+
+The guard is `BROKER` rather than every `ProxyError`, so the remap covers exactly the misattributed
+class and nothing else. `BASIC_AUTH` is deliberately excluded — the proxy returns it when proxying is
+disabled or the basic auth does not match (`Server.hs:1416-1420`), which is a client↔proxy fact and is
+correctly attributed today. `NO_SESSION` is returned only for `PFWD`. `PROTOCOL` describes the relay
+but is not rendered as a proxy-connection error by either client, so leaving it unchanged keeps the
+diff to the errors that actually produce a wrong message.
+
+### Blast radius
+
+- `temporaryAgentError` (`Agent/Client.hs:1572-1580`) and `serverHostError` (`:1594-1596`) already match
+  both shapes with the same helpers — retry and proxy-fallback behaviour is unchanged.
+- `clientServiceError` (`:1268-1273`) has no `PROXY`-shape twin for `BROKER NO_SERVICE`, but both ends
+  document that case as unreachable (`Client.hs:812`); left as is.
+- simplex-chat `Subscriber.hs:1819-1820` handles both shapes; send failures move from `SndErrProxy` to
+  `SndErrProxyRelay`, i.e. "Destination server error" rather than "Error" — also more accurate.
+- `SMP _ (PROXY _)` becomes unreachable, making `smpProxyErrorAlert` in both clients dead code. Removing
+  it is a follow-up in simplex-chat, not required by this change.
+
+### Verification
+
+- Reproduced before/after with a CLI built against this branch: the error now carries
+  `relayServer = "smp://VvXX…@simplex.server.home:5223"`, and the direct (non-proxied) path is
+  byte-identical to before.
+- `SMPProxyTests`: 45 examples, 0 failures — including `fails when fallback is prohibited` and both
+  retry tests, which exercise `newProxiedRelay` and the error classification.

plans/2026-08-09-fast-queue-rotation-plan.md

@@ -0,0 +1,187 @@
+# Fast queue rotation — implementation plan
+
+Branch: ep/drop-agent-versions. RFC: ../rfcs/2026-08-09-fast-queue-rotation.md.
+
+Model: redundant delivery, no flip. `QADD` adds the new receive queue R'. From `QADD` until R' is
+secured the sender writes every message to both old and R' (double delivery, not a move); once R' is
+secured the sender writes new messages to R' only, while old delivers its already-scheduled tail and
+the `QEND` appended to it. `QEND` removes a named queue. The recipient drops duplicates (double
+ratchet), so the order and which queue delivers do not matter, as long as every message arrives on at
+least one queue. Rotation away from a dead server works because every message up to securing is
+scheduled on R' too.
+
+Roles: A initiates (its receive queue rotates; A receives on R'). B sends to A and secures R'.
+
+## Why redundant delivery removes the hard parts
+
+- No boundary, no drain, no last-message id. A never decides how much of old to read.
+- A dead old server loses nothing B still holds: every undelivered message is scheduled on R' as well.
+- A dead new server does not suspend delivery: old keeps delivering until R' is secured.
+- The double ratchet already drops duplicates (`AGENT A_DUPLICATE`) and tolerates bounded reordering,
+  and the delivery schema already writes one message to several send queues (`enqueueMessageB` +
+  `enqueueSavedMessageB`).
+
+## The one ordering constraint
+
+A must hold R''s secret before it reads any R' data message. A data message reaching R' before the
+confirmation is dropped as "no keys" (`processClientMsg`, `(Nothing, Nothing)` arm, line 3611), which
+loses it when old is dead. So the confirmation is the first message B sends on R'. R''s delivery
+worker does not start while R' is securing, so its accumulated rows cannot outrun the confirmation.
+`ICQSndSecure` sends the confirmation and only then starts the worker. This holds on restart too (see
+Worker gate).
+
+## New definitions
+
+Agent/Protocol.hs
+- Condition fast rotation on the existing `rpcAddressSMPAgentVersion` (v8, `Protocol.hs:322`).
+- New `AMessage` constructor `QEND SndQAddr` (tag `QE`), the address of the queue to remove. v8-only,
+  and only sent during fast rotation, so peers below v8 never parse it.
+- `SndSwitchStatus` constructors `SSSecuringQueue` (old, while R' secures) and `SSSendingQEND` (old,
+  after R' is secured — it drains its tail and `QEND` but takes no new messages).
+- `InternalCommand` constructor `ICQSndSecure SMP.SenderId`.
+
+No receive-side switch status, no boundary, no drain state.
+
+## Schema
+
+None. `SSSecuringQueue` uses `snd_queues.switch_status`; R' is secured into `rcv_queues.e2e_dh_secret`.
+No new columns, no migration.
+
+## Sender B
+
+### Dual scheduling from QADD
+
+`enqueueMessageB` writes a delivery row for the head send queue and for each `filter isActiveSndQ`
+tail queue (`Agent.hs:2345`). Adjust the selection two ways: additionally include a securing
+replacement queue on a v8 connection (`connAgentVersion cData >= rpcAddressSMPAgentVersion && status == New && isJust dbReplaceQueueId`),
+and exclude a terminating queue (`sndSwchStatus == Just SSSendingQEND`). The version guard keeps the
+slow path unchanged — there R' is also `New` with a replace reference during `QKEY`/`QUSE`, but it must
+not be dual-scheduled. `SSSendingQEND` is a fast-path-only status, so the exclusion never affects the
+slow path. The gate below is inert for the slow path anyway, since it never starts R''s worker while
+`New`.
+
+- `QADD` until R' secured: old is the head (active) and R' is the securing replacement, so every `SEND`
+  writes both rows. old delivers at once; R''s rows accumulate behind its gate.
+- R' secured: R' is the head (primary) and old is `SSSendingQEND` (excluded), so a `SEND` writes R'
+  only. old keeps its worker and delivers whatever was already scheduled on it, plus `QEND`.
+
+### Worker gate
+
+`submitPendingMsg` (`Agent.hs:2437`) and `resumeMsgDelivery` (`2421`) — the two `getDeliveryWorker`
+callers that start delivery — skip a queue with `status == New && isJust dbReplaceQueueId`, so neither
+a `SEND` nor startup starts R''s worker while it secures. Startup resumes delivery through
+`resumeMsgDelivery` (`resumeDelivery` line 1848, and `getAllSndQueuesForDelivery` line 1943), so R' is
+skipped there; `resumeAllCommands` (1883) resumes R''s `ICQSndSecure`, which secures R' and only then
+starts its worker.
+
+### Steps
+
+`qAddMsg` (fast branch, under the connection lock, `Agent.hs:3855`):
+- Add R' as the slow path does (line 3870): `addConnSndQueue (sq_) {primary = True, dbReplaceQueueId = Just old}`, `New`.
+- Duplicate **every** undelivered message on old to R': for each pending row on old
+  (`SELECT internal_id FROM snd_message_deliveries WHERE conn_id = ? AND snd_queue_id = old AND failed = 0`),
+  `createSndMsgDelivery db R' internalId`. This is the loss-prevention step: old's not-yet-sent
+  messages are duplicated onto R', so if old later fails they are already on R'. If old is already
+  down, nothing was sent and the whole backlog is duplicated.
+- `enqueueCommand (Just newSrv) (ICQSndSecure sndId)`; `setSndSwitchStatus SSSecuringQueue` on old
+  (where the slow path sets `SSSendingQKEY`, line 3874); notify `SWITCH QDSnd SPStarted`. old keeps
+  delivering; R''s worker is gated.
+
+`ICQSndSecure sId` (retryable, under `tryWithLock`):
+1. If old is already gone, a prior attempt finished; return. Otherwise find R' by `sId`.
+2. `secureSndQueue` (SKEY) R' — idempotent, since `sndPrivateKey` was persisted by `qAddMsg`
+   (`QueueStore/STM.hs:213`: same key → `Right ()`, different key → `AUTH`).
+3. Send the confirmation on R' (`sendConfirmation`; empty body, both peers already know each other;
+   `e2eEncryption_ = Nothing`, no ratchet step). It is the first message on R'.
+4. On success, in one transaction: `setSndQueueStatus R' Active` (the gate lifts), `setSndQueuePrimary R'`
+   (R' becomes the head; its replace reference is cleared), and `setSndSwitchStatus old (Just SSSendingQEND)`
+   (old takes no new messages but keeps its worker). Then `submitPendingMsg c R'` (the worker starts and
+   flushes the accumulated rows after the confirmation), `enqueueMessages [old, R'] (QEND oldAddr)`
+   (appended after old's tail, delivered on both), and notify `SWITCH QDSnd SPSecured`. From here a
+   `SEND` goes to R' only; old delivers its tail then `QEND` and is removed when `QEND` is sent.
+
+A temporary error retries from step 1; nothing is torn down. A permanent `AUTH` (should not occur, R'
+was secured with B's own key) leaves both queues and surfaces `A_QUEUE`.
+
+`QEND` sent — new `AM_QEND_` arm in `runSmpQueueMsgDelivery`, modelled on `AM_QTEST_` (`Agent.hs:2567`):
+on a successful send of `QEND addr`, remove the named send queue (`TM.delete` its worker,
+`deleteConnSndQueue addr`), make the remaining queue the sole primary (`setSndQueuePrimary`, which
+clears its `replace_snd_queue_id`), and notify `SWITCH QDSnd SPCompleted` (as `AM_QTEST_` does, line
+2591). This re-primary is a no-op on the send side — step 4 already made R' primary before `QEND` was
+enqueued. The handler is idempotent — a second `QEND` send finds the named queue already gone and does
+nothing. `QEND` is sent on both queues; removing old's send queue also drops any `QEND` still pending
+on old. The R' copy reliably removes old
+and reaches A even when old is dead; the old copy is best effort. Once old's send queue is gone, `SEND`
+schedules to R' only.
+
+## Recipient A
+
+A is subscribed to old (primary, `RSSendingQADD`) and R' (created at rotation start,
+`dbReplaceQueueId = old`).
+
+- **Confirmation on R'.** In `processClientMsg`, `(Nothing, Just e2ePubKey)` case, add an arm before the
+  `senderCanSecure` arm (`Agent.hs:3476`), guarded by `isJust (dbReplaceQueueId rq)`. In one
+  transaction: `setRcvQueueConfirmedE2E rq (C.dh' e2ePubKey e2ePrivKey) (min v phVer)` (secures R') and
+  `setRcvQueuePrimary R'` (clears R''s replace reference). Then `ack`, and notify `SWITCH QDRcv SPConfirmed`.
+  No conn-info processing, no ratchet step, no deferral. Redelivery is idempotent: R' now has `e2e_dh_secret`, so a re-sent
+  confirmation reaches the `(Just e2eDh, Just _)` arm (line 3608) and is acked — correct here, since
+  there is no backlog to hold.
+- **Data on R'.** With R''s replace reference cleared, a data message on R' takes the ordinary path
+  (`(_, dbReplaceQueueId=Nothing)`, line 3503) — no old-deletion, no `RSSendingQUSE` check. A copy
+  already read on old is dropped as `A_DUPLICATE`; a copy read first on R' advances the ratchet and
+  old's copy is then the duplicate.
+- **`QEND oldAddr` on either queue.** New `AMessage` handler (`qEndMsg`, a `qDuplex` handler like
+  `qAddMsg`): `findRQ oldAddr` the receive queue to remove. Mark it deleted (`setRcvQueueDeleted`, so
+  `getRcvQueuesByConnId_`'s `deleted = 0` filter excludes it at once and a restart does not resurrect
+  it) and `enqueueCommand (Just oldServer) (ICDeleteRcvQueue oldRcvId)` for the server `DEL` and record
+  removal — the async, crash-safe path `abortConnectionSwitch` uses, which resumes on restart and does
+  not block `QEND`, **not** the synchronous `deleteQueue` of `finalizeSwitch`, which would stall if old
+  is unreachable. `ICDeleteRcvQueue` (`Agent.hs:2224`) currently retries a temporary error forever;
+  bound it with the same persisted `rcv_queues.delete_errors`/`deleteErrorCount` mechanism `deleteQueueRec`
+  uses (2884): on a temporary error `incRcvDeleteErrors`, and at the limit `deleteConnRcvQueue` and
+  stop. The count is in the database, so the bound survives restarts, and its only other caller
+  (`abortConnectionSwitch'`, 2739) deletes an alive queue that succeeds well before the limit. `qEndMsg`
+  does **not** re-primary R' — the confirmation arm (above) owns R''s primary flag and replace
+  reference. `QEND` on old and the confirmation on R' travel on different queues with no order between
+  them, so `QEND` on old can be processed first (it sits only behind old's tail); re-primarying then
+  would clear R''s `dbReplaceQueueId` and the later confirmation would miss the rotation arm and never
+  secure R'. So `qEndMsg` only removes the named queue. Re-create the notification subscription
+  (`when enableNtfs $ sendNtfSubCommand ns (NSCCreate, [connId])`); notify
+  `SWITCH QDRcv SPCompleted`; `ackDel` the `QEND`. Received on both queues, the second finds it already
+  marked deleted and is a no-op.
+
+No drain, no boundary, no finalize command. Old is removed when `QEND` arrives, not by counting.
+
+## Abort / version
+
+- Fast rotation runs only when `connAgentVersion >= v8`; otherwise the `QKEY`/`QUSE` slow path runs
+  unchanged.
+- `canAbortRcvSwitch` (`Agent/Store.hs:210`) returns false for `RSSendingQADD` when `connAgentVersion >= v8`
+  (at v8 B always chooses fast, so A treats a sent `QADD` as committed). Its signature gains
+  `connAgentVersion`; both callers pass it from `cData` — `abortConnectionSwitch'` (2730) and
+  `rcvQueueInfo` in `connectionStats` (2976).
+
+## Losses and duplication
+
+- No boundary loss: the `QADD` step **duplicates** old's entire undelivered backlog onto R', and every
+  later message up to securing is scheduled on both queues, so if old fails it loses nothing B still
+  holds. The only messages old can strand are those its server already accepted but had not handed to
+  A — the ordinary store-and-forward risk, present whenever a server fails with unread messages, and
+  empty if old was already down (a down server accepted nothing).
+- Duplicates: the double ratchet drops them (`A_DUPLICATE`); `checkMsgIntegrity`'s `MsgDuplicate` is
+  only a flag, not the mechanism.
+- The 512 skip bound (`Crypto/Ratchet.hs:953`) does not bite on the rotation: each queue delivers in
+  order and every message B still holds is on R', so A reads a contiguous stream with only small
+  cross-queue reordering. A store-and-forward residual (above) is an ordinary loss, not introduced here.
+
+## Tests
+
+- new/new, old stopped right after `QADD`: rotation completes; all messages delivered on R'; old
+  removed by `QEND` on R'.
+- new/new, both alive: messages delivered on both, deduped; old removed by `QEND`.
+- new/old and old/new: fall back to the `QKEY`/`QUSE` path.
+- crash during securing: restart does not start R''s worker; `ICQSndSecure` resumes, secures R',
+  starts the worker, sends `QEND`.
+- `QEND` received on both queues: old removed once, the second receipt is a no-op.
+- `QEND` on old processed before the confirmation on R': R' still secures, because `qEndMsg` does not
+  clear R''s replace reference; rotation completes.

plans/2026-08-22-xftp-file-storage-time.md

@@ -0,0 +1,168 @@
+# Implementation plan: XFTP variable file storage time
+
+Proposal: `../rfcs/2026-08-22-xftp-file-storage-time.md`.
+
+## simplexmq: entitlement crypto
+
+New module `Simplex.Messaging.Crypto.Entitlement`, over `Simplex.Messaging.Crypto.BBS`:
+
+Types:
+
+```
+newtype MasterKey = MasterKey ByteString
+
+data Entitlement = Entitlement
+  { expiresAt :: UTCTime,
+    entitlementName :: Text,
+    extraInfo :: Text
+  }
+
+data EntitlementCredential = EntitlementCredential
+  { issuerKeyIdx :: Word16,
+    masterKey :: MasterKey,
+    entitlement :: Entitlement,
+    issuerSignature :: BBSSignature
+  }
+
+data EntitlementProof = EntitlementProof
+  { issuerKeyIdx :: Word16,
+    entitlement :: Entitlement,
+    entProof :: BBSProof
+  }
+```
+
+Functions and constants:
+
+- the disclosed-message encoding: the master key is message 0 and stays undisclosed; `expiresAt`, `entitlementName`, and `extraInfo` are messages 1 to 3 and are disclosed. The protocol encoding of `Entitlement` and its field order follow the same order
+- the BBS header string `"SimpleX badges v1"` (shared with chat's badges, which sign under it), the message count, and the disclosed indexes
+- `generateEntitlementProof :: Map Word16 BBSPublicKey -> EntitlementCredential -> BBSPresHeader -> IO (Either String EntitlementProof)` (the issuer key is looked up by the credential's index; an absent index is `Left`)
+- `verifyEntitlement :: Map Word16 BBSPublicKey -> BBSPresHeader -> EntitlementProof -> IO EntitlementVerification`, where `data EntitlementVerification = EVValid | EVInvalid | EVUnknownIssuer` (the caller supplies the presentation header; the server reconstructs it, the proof never includes it)
+- the issuer public keys constant `Map Word16 BBSPublicKey`
+
+## simplexmq: protocol, new XFTP version
+
+In `Simplex.FileTransfer.Transport`:
+
+- add the next `VersionXFTP` and set `currentXFTPVersion` to 4
+- add `entitlementProof :: Maybe EntitlementProof` to `XFTPClientHandshake`, encoded before the `Tail` and only from this version
+- the presentation header is the session id alone
+
+In `Simplex.FileTransfer.Protocol`:
+
+- add `GrantedStorageTime` and its encoding; retain the one-character sum prefix for future variants:
+
+```
+data GrantedStorageTime = GSTExpires {epochSeconds :: Int64}
+```
+
+- add the storage time (`Maybe Word32`: `Nothing` and `Just 0` request the server maximum, a value above zero requests that number of hours) to `FNEW`
+- add the granted storage to `FRSndIds` as `Maybe GrantedStorageTime` (`Nothing` when decoding a response from a server below this version)
+
+## simplexmq: server configuration
+
+In `Simplex.FileTransfer.Server.Env` and `Simplex.FileTransfer.Server.Main`:
+
+- make `fileExpiration` non-optional (`ExpirationConfig`, no longer `Maybe`); the server always expires files, so the server maximum is always a concrete number of seconds
+- read a maximum storage time (a number of hours) for each entitlement name from the `[STORE_LOG]` INI section, from the keys `expire_files_hours_for_supporter` and `expire_files_hours_for_legend`, into `fileStorageEntitlements :: Map Text EntitlementConfig`, where `newtype EntitlementConfig = EntitlementConfig {storageTime :: Int64}` holds seconds; an absent key is skipped (that name gets the default), a present but malformed value fails startup
+- exit at startup if any name's maximum is below the default file expiration
+- add `entitlementKeys :: Map Word16 BBSPublicKey` to the server config (default = the shared constant, set from `Main`); the handshake verifies the proof against it, so the trusted keys never come from the sender
+
+## simplexmq: server session
+
+In `Simplex.FileTransfer.Server`:
+
+- `processClientHandshake` verifies the proof from the handshake, once per session, and resolves the maximum storage time for the entitlement name. `HandshakeSent` carries `EntitlementChecked`, so the first handshake verifies and every later one on that connection reuses the result, including after `processHello` returns the session to `HandshakeSent`
+- verify only when the answer can change: the name is configured (startup rejects a maximum below the default), and the entitlement expired less than 24 hours ago. A proof that fails these checks gets no verification; a proof that fails to verify is logged. In both cases the session gets the default maximum
+- a verified proof becomes `peerEntitlement :: Maybe SessionEntitlement` in `THAuthServer`, where `data SessionEntitlement = SessionEntitlement {expiresAt :: SystemSeconds, entConfig :: EntitlementConfig}`; `processXFTPRequest` takes it from there, so no proof is verified while a command is processed
+- `createFile` caps the requested storage time by the session maximum, which is the entitlement's storage time when the entitlement is still valid, and the default otherwise
+
+## simplexmq: server store and expiration
+
+The `files` table gets a nullable `expires_at`. Every new file stores a concrete `expires_at`. It is NULL only for pre-feature rows, which the migration must not re-date (it has no access to the operator's configured TTL); those are expired at query time as `created_at + ttl`.
+
+Common to both stores, in `Simplex.FileTransfer.Server.Store`:
+
+- add `expiresAt :: Maybe RoundedFileTime` to `FileRec`
+- in `createFile`, cap the requested hours at the session maximum, round the expiry up to the hour, store it, and return that same value as the granted storage
+- `expiredFiles` receives `now` and `old` (= `now - ttl`). A stored expiry is deleted when `expires_at < now` (no grace — it is already rounded up); a legacy row (no `expires_at`) is deleted when `created_at + fileTimePrecision < old` (the grace covers `created_at` being floored to the hour)
+- retain `created_at` for statistics, export, and the legacy fallback
+
+STM store:
+
+- in `expiredFiles`, expire a new file when `roundedSeconds expiresAt < now`, and a legacy file (no `expiresAt`) when `created_at + fileTimePrecision < old`
+
+PostgreSQL store, in `Simplex.FileTransfer.Server.Store.Postgres` and its migrations:
+
+- add the nullable column `expires_at BIGINT` (no backfill)
+- add one composite index `idx_files_expiry ON files (expires_at, created_at)`
+- `expiredFiles` query: `(SELECT ... WHERE expires_at < ? LIMIT ?) UNION ALL (SELECT ... WHERE expires_at IS NULL AND created_at < ? LIMIT ?)` with `(now, limit, old - fileTimePrecision, limit)`. The first arm deletes stored (already rounded-up) expiries; the second drains legacy rows, with the grace folded into `old - fileTimePrecision` so the columns stay bare and sargable. Each arm is one range over the composite index and stops at its own limit; a single `OR` predicate builds a bitmap of every match before the limit applies. A `COALESCE(expires_at, created_at + ttl)` predicate is avoided (not sargable, would force a sequential scan). No `ORDER BY` — the batch loop deletes all expired rows regardless of order. New files always store an expiry, so the second arm drains permanently once the legacy rows expire, and is then removed.
+
+Store log, in `Simplex.FileTransfer.Server.StoreLog`:
+
+- add the optional expiration to the `AddFile` record; a record without it parses to `Nothing` (the configured default), never a hardcoded value
+
+## simplexmq: agent
+
+The credential belongs to the user, so the agent holds it the way it holds the user's servers: in memory, supplied when the agent is created and replaced through an API. It is not stored by the agent.
+
+Per-user state in `Simplex.Messaging.Agent.Env.SQLite` and `Simplex.Messaging.Agent.Client`:
+
+- add `entitlements :: Map UserId EntitlementCredential` to `InitialAgentServers`, beside the servers
+- add `userEntitlements :: TMap UserId EntitlementCredential` to `AgentClient`, filled from it by `newAgentClient`
+- add `entitlementKeys :: Map Word16 BBSPublicKey` to `AgentConfig` (default = the shared constant), for the issuer key that proof generation needs
+
+Public API in `Simplex.Messaging.Agent`:
+
+- add storage time (`Maybe Word32` hours) to `xftpSendFile`
+- add `setUserEntitlement :: AgentClient -> UserId -> Maybe EntitlementCredential -> IO ()`, in the shape of `setProtocolServers`: it replaces the entry, and when the credential changed it closes that user's XFTP clients, so the next upload presents the new credential
+
+Store, in both the SQLite and PostgreSQL agent stores:
+
+- add a nullable storage time column (integer hours; NULL means the server maximum) to `snd_files`
+- add the migration to both stores
+- in `createSndFile`, store the storage time
+
+Upload, in `Simplex.Messaging.Agent.Client` and `Simplex.FileTransfer.Client`:
+
+- `getXFTPClient` takes a proof for the session as a parameter, `SessionId -> IO (Maybe EntitlementProof)`, beside the callback it already takes for a closed client. The client config holds no credential and no keys
+- `getXFTPServerClient` passes a function that generates the proof over the session id from the configured issuer keys, and only for a server of this user, matched by the key hash that TLS pins, so a file description of the sender cannot direct the entitlement to another server. A server that is not the user's, a missing credential, or a failure to generate gives `Nothing`, with the failure logged
+- `xftpClientHandshakeV1` calls it with the session id from the connection, and sends the result in the handshake
+- `agentXFTPNewChunk` reads the storage time from the send record and sends FNEW with it
+- `createXFTPChunk` returns the granted expiry (epoch seconds); `agentXFTPNewChunk` stores it on `NewSndChunkReplica`
+
+Completion:
+
+- `createXFTPChunk` returns the granted expiry as `Maybe GrantedStorageTime`; `SndFileChunkReplica` and `NewSndChunkReplica` carry `expiresAt :: Maybe GrantedStorageTime`
+- persist it in a nullable `replica_expires_at` column on `snd_file_chunk_replicas` (added to the entitlement migration): `createSndFileReplica` stores `epochSeconds`, `getSndFile` and `getNextSndChunkToUpload` read it back into `GSTExpires`
+- on `SFDONE`, report the file expiry: a chunk expires when its last replica expires (`max` over replicas, absent replicas ignored, `Nothing` only if none report); the file expires when its first chunk expires (`min` over chunks, `Nothing` if any chunk is unknown). `GrantedStorageTime` derives `Ord`
+- `SFDONE` gains a trailing `Maybe GrantedStorageTime` (not str-encoded); chat consumes it (wired later)
+
+Testing:
+
+- e2e test in `tests/XFTPAgent.hs`: generate a BBS keypair, sign a supporter credential (issuer key index 1), run the server with `entitlementKeys = {1: testPk}` and a supporter maximum above the default, run the sender agent with the same `entitlementKeys` and the credential for the user, send a file requesting a number of hours above the default and below that maximum, and assert `SFDONE`'s granted expiry rounds up `now + requested` (proof of the entitlement raising the max above the default)
+- the same upload without the credential is capped at the default maximum
+- the expiry is written before the status, because `BlockingInfo`'s notice parser is terminal (`A.takeByteString`), so any field after the blocking info makes the record unparseable and the file is dropped on restart. Every field added to `AddFile` must go before the status
+- store log round trip in `tests/CoreTests/StoreLogTests.hs`, in the shape of the SMP store log test: a file record survives a write, a read into the store, and compaction, with and without the expiry, including a file blocked with a notice in both cases
+
+## simplex-chat
+
+- remove lifetime badges: make `badgeExpiry` a `UTCTime`, drop the `"lifetime"` encoding, and remove the lifetime option from the UI and the CLI
+- map `BadgeInfo` to `Entitlement` (`entitlementName = textEncode badgeType`, `expiresAt = badgeExpiry`, `extraInfo = badgeExtra`) when calling the agent
+- pass the user's credential to the agent when it is created, and through `setUserEntitlement` when the badge changes, in the same places that pass and update the user's servers
+- pass the storage time to `xftpSendFile`
+- retain the `maxXFTPFileSize` size limit
+- reuse `verifyEntitlement` for peer-badge verification
+- import the issuer public keys from the shared simplexmq constant
+
+## State
+
+Steps 2 to 5 are implemented in simplexmq. Step 1 and step 6 belong to the chat branch that carries badges.
+
+## Order
+
+1. Add the entitlement crypto module; move chat's badge verification onto it and remove lifetime badges.
+2. Add the new XFTP version, the FNEW storage time, and the response.
+3. Change the server configuration, store, expiration, and store log.
+4. Move the proof to the handshake: the handshake field, the session state on the server, and the proof for the session on the client.
+5. Hold the credential per user in the agent, and add the API to replace it.
+6. Wire chat to pass the credential and the storage time.

protocol/diagrams/duplex-messaging/queue-rotation-fast.mmd

@@ -5,13 +5,18 @@ sequenceDiagram
   participant S as Server<br>that has A's send queue<br>(B's receive queue)
   participant B as Bob
 
-  A ->> R': NEW: create new queue<br>(allow SKEY)
-  A ->> S: SEND: QADD (R'): send address<br>of the new queue(s)
+  A ->> R': NEW: create new queue (SKEY allowed)
+  A ->> S: SEND: QADD (R')
   S ->> B: MSG: QADD (R')
-  B ->> R': SKEY: secure new queue
-  B ->> R': SEND: QTEST
-  R' ->> A: MSG: QTEST
-  A ->> R: DEL: delete the old queue
-  B ->> R': SEND: send messages to the new queue
-  R' ->> A: MSG: receive messages from the new queue
-  
+  B ->> R: SEND: messages (also scheduled on R')
+  R ->> A: MSG: messages
+  B ->> R': SKEY: authorize B as sender
+  B ->> R': SEND: confirmation (establishes R' secret)
+  R' ->> A: MSG: confirmation (A secures R')
+  B ->> R': SEND: held copies (deduped), then new messages
+  R' ->> A: MSG: messages
+  B ->> R: SEND: remaining tail, then QEND
+  R ->> A: MSG: QEND
+  B ->> R': SEND: QEND
+  R' ->> A: MSG: QEND
+  A ->> R: DEL: delete the current queue

protocol/diagrams/xrcp/session.mmd

@@ -17,11 +17,11 @@ note over CC, HC: TLS handshake
 par
     note over CC: validate client X509 credentials
     CC->>CI: session code from tlsUnique
-    CI-->>CC: user confirmation
+    CI-->>CC: user confirmation (optional if known)
 and
     note over HC: validate server X509 credentials
     HC->>HI: session code from tlsUnique
-    HI-->>HC: user confirmation
+    HI-->>HC: user confirmation (optional if known)
 end
 
 note over CI, HI: 3. Session verification and protocol negotiation

protocol/security.md

@@ -168,7 +168,7 @@ This document describes the cryptographic primitives and threat model for the Si
 
 *cannot:*
 
-- impersonate a sender and send messages to the user whose database was stolen. Doing so requires also compromising the router (to place the message in the queue, that is possible until the Double-Ratchet advances forward) or the user's device at a subsequent time (to place the message in the database).
+- impersonate a sender and send messages to the user whose database was stolen, unless the database contains a new rotation queue that is not secured yet. Doing so requires also compromising the router (to place the message in the queue, that is possible until the Double-Ratchet advances forward) or the user's device at a subsequent time (to place the message in the database).
 
 - undetectably communicate at the same time as Alice with her contacts. Doing so would result in the contact getting different messages with repeated IDs.
 

protocol/simplex-messaging.md

@@ -1,4 +1,4 @@
-Version 20, 2026-05-25
+Version 21, 2026-07-05
 
 # Simplex Messaging Protocol (SMP)
 
@@ -86,7 +86,7 @@ It's designed with the focus on communication security and integrity, under the
 
 It is designed as a low level protocol for other application protocols to solve the problem of secure and private message transmission, making [MITM attack][1] very difficult at any part of the message transmission system.
 
-This document describes SMP protocol version 20. Versions 1-5 are discontinued. The version history:
+This document describes SMP protocol version 22. Versions 1-5 are discontinued. The version history:
 
 - v1: binary protocol encoding
 - v2: message flags (used to control notifications)
@@ -107,6 +107,8 @@ This document describes SMP protocol version 20. Versions 1-5 are discontinued.
 - v18: support client notices in BLOCKED error
 - v19: service subscriptions to messages (SUBS, NSUBS, SOKS, ENDS, ALLS commands)
 - v20: public namespaces resolver (RSLV command, RNAME response) — direct or forwarded via PFWD
+- v21: server public information in handshake
+- v22: `RNAME` says whether a name can be registered, not only what it resolves to
 
 ## Introduction
 
@@ -1118,6 +1120,8 @@ When the client receives PKEY response it MUST validate that:
 - the fingerprint of the received certificate matches fingerprint in the router address - it mitigates MITM attack by proxy.
 - the router session key is correctly signed with the received certificate.
 
+`smpVersionRange` in `PKEY` is not signed.
+
 The proxy router may respond with error response in case the destination router is not available or in case it has an earlier version that does not support proxied commands.
 
 #### Send command via proxy
@@ -1450,53 +1454,150 @@ reads `NameRecord` from. The reference implementation forwards each RSLV to a
 companion REST resolver process (`scripts/resolver/snrc-resolve.py`) that
 queries the SNRC contract on Ethereum; alternative backings (different chains,
 DHT, etc.) are valid as long as they expose the documented HTTP shape (`GET
-/resolve/<name>` returning a `NameRecord` on 200, 404 / 400 for unknown names
-or TLDs, 502 for upstream RPC failures) or substitute a different transport
-while still returning a `NameRecord` matching the encoding below.
+/v2/resolve/<query>` returning a `NameRegistration` on 200 for every
+registration shape, 400 for unknown TLDs, 502 for upstream failures) or
+substitute a different transport returning the same JSON. The resolver API is
+versioned separately from this protocol: `/v1/resolve/<name>` returns a bare
+`NameRecord` and is what relays before v22 call as `/resolve/<name>`.
 
 #### Resolve name command
 
-The `RSLV` command carries the canonical fully-qualified name directly as the
-payload (not JSON):
+The `RSLV` command carries the query as text, not JSON. A client sends the
+hashed form only from v22, and the name itself below it:
 
 ```abnf
-rslv = %s"RSLV" SP domain   ; domain = canonical name as non-space bytes, consuming the remainder of the transmission
+rslv   = %s"RSLV" SP query
+query  = domain / hashed        ; hashed only from v22
+domain = 1*253 OCTET            ; the name as text
+hashed = "[" 64HEXDIG "]" tld   ; keccak-256 of the second-level label
+tld    = %s".simplex" / %s".testing"
 ```
 
 `domain` is the UTF-8 canonical fully-qualified name with the TLD always
 explicit (e.g. `privacy.simplex`, `test.testing`, `example.com`), bounded to
 253 bytes.
 
+**Hashed labels.** The query is a name, or the keccak-256 of a second-level
+label in ENS's bracketed form. A label can only be letters, digits and hyphens,
+so `[` tells the two apart and no tag is needed.
+
+Only a second-level name may be hashed. A name with subnames is sent as text: it
+is resolved rather than priced, and its record names it anyway. A web TLD has no
+registry to key a hash on.
+
+From v22 a client MUST send the hash. Older routers can only read the name, so a
+client on an older session sends the name. A router answering a hashed query
+does not know the label's length, so it cannot check a minimum-length policy
+either: the client does that, from the pricing it is sent.
+
+The same form reaches the backing resolver, which is what its HTTP API takes, so
+the query is one string end to end.
+
+A hashed query still answers with the name. The registrar records the plaintext
+label when a name is registered, keyed by the hash of that label, so a router can
+look up what the hash stands for without ever being told. The router is not
+trusted for it: a client MUST check that the record names the name it asked
+about, and reject the answer otherwise. A registry that does not record the
+label cannot answer a hashed query at all, and the router answers `ERR NAME
+RESOLVER` rather than a record it knows the client will reject. An unregistered
+name has no recorded label, so it cannot be looked up either.
+
 **Server-side validation.** The names router parses `domain` as a
 fully-qualified name (TLD required — bare labels are rejected) and forwards it
 to the configured backing resolver, which is the source of truth for which
 on-chain registry maps to each TLD.
 
-The names router responds with either an `RNAME` response carrying the resolved
-record, or an `ERR NAME` error whose subcode a client iterating across several
-configured servers can act on distinctly:
+The names router responds with either an `RNAME` response saying what it knows
+about the name, or an `ERR NAME` error whose subcode a client iterating across
+several configured servers can act on distinctly:
 
 | Response | Condition | Client action |
 |---|---|---|
-| `RNAME` | record resolved | use it |
-| `ERR NAME NOT_FOUND` | name not registered, unknown TLD, or malformed name | authoritative "no such name" — stop |
+| `RNAME` | the router read the registry | use it |
+| `ERR NAME NOT_FOUND` | below v22 only: every name that does not resolve. From v22 a router never sends it | stop, and do not read it as registrable |
 | `ERR NAME NO_RESOLVER` | this router has no resolver (names role not enabled) | skip this server, try the next |
-| `ERR NAME RESOLVER <detail>` | transient failure: backing resolver error (upstream 5xx, transport, timeout, decode) | transient — retry or surface, do not treat as "not found" |
+| `ERR NAME RESOLVER <detail>` | the router cannot state an answer completely: no registrar or price oracle for the TLD, an unreachable chain, a transport failure, a timeout, a registration it could not date or resolve | surface `<detail>`; retry only if it reads as transient |
 
 A client SHOULD NOT broadcast a `name` to further servers after a name-capable
 router has answered (`NOT_FOUND` or `RESOLVER`), since that router has already
 seen the lookup key; `NO_RESOLVER` discloses nothing about the name beyond the
 fact that this router cannot resolve, so iterating past it is safe.
 
-#### Name record response
+#### Name response
 
-The `RNAME` response carries a JSON-encoded record as the payload:
+`RNAME` answers both what a name resolves to and whether it can be registered.
 
 ```abnf
-rname = %s"RNAME" SP json-bytes   ; json-bytes consumes the remainder of the transmission
+rname = %s"RNAME" SP nameResponse
 ```
 
-`json-bytes` MUST be a UTF-8 JSON object with the following schema:
+`nameResponse` is a UTF-8 JSON object consuming the remainder of the
+transmission: the `registration` below, and `lastBlockTs`, the block timestamp the
+registry was read at. Money is US cents, times are seconds since the Unix epoch, and lengths are
+characters.
+
+| Field | JSON type | Constraints |
+|---|---|---|
+| `lastBlockTs` | number | block timestamp the answer was read at. Absent only from a v20/v21 router, which sent the record alone |
+| `registration` | object | below |
+
+A router reads the registry through a node of its own, which can lag. A name
+registered or changed after `lastBlockTs` still reads as it did before, so a client
+MUST NOT treat an answer as current without checking it.
+
+`registration`'s `type` selects which of the three answers it is.
+
+| `type` | Meaning |
+|---|---|
+| `registered` | held by someone until `expires`, renewable by its owner alone until `graceUntil`. It always carries `nameRecord`: where the owner set none, every field is unset and the resolver address is zero |
+| `available` | held by nobody and registrable now, at `pricing` |
+| `reserved` | held back by the registry and not registered. It carries no price, and a router MUST NOT quote one |
+
+| Field | On | JSON type | Constraints |
+|---|---|---|---|
+| `expires` | `registered` | number | absent only from a v20/v21 router, which sent the record alone |
+| `graceUntil` | `registered` | number | after `expires`; until here only the owner may renew. Absent on the same condition as `expires` |
+| `reservedReason_` | `registered` | string | a reason word, present only when the name is held back as well |
+| `nameRecord` | `registered` | object | the record, schema below |
+| `pricing` | `available` | object | `registrationPrices`, `basePrice` and `minLabelLength`, below |
+| `reservedReason` | `reserved` | string | a reason word |
+
+| `pricing` field | JSON type | Constraints |
+|---|---|---|
+| `registrationPrices` | object | label length, as a decimal string, to US cents per year, for the lengths the registry prices specially |
+| `basePrice` | number | US cents per year for every other length |
+| `minLabelLength` | number | characters; the registry refuses shorter labels |
+
+A reason word is `internal`, `trademark`, `community`, or a word a later version
+reserves under, at most 32 printable ASCII characters. A router truncates an
+unknown word to that and otherwise passes it through unchanged. A client MUST
+read a word it does not know as unknown and still treat the name as reserved.
+
+**Computing the price.** In US cents, for a duration in seconds:
+
+```
+price len duration = tier len * duration / 31536000
+tier len = the entry for len in registrationPrices, or basePrice when there is none
+```
+
+The registry's minimum registration is 730 days, a contract constant, so it is
+specified here rather than sent. `registrationPrices` omits any length below
+`minLabelLength`. A client MUST NOT show a quote for a label the registry
+refuses: a hashed query carries no length, so only the client can check it.
+
+Below v22, `RNAME` carries the bare record and nothing else, and every answer
+without one is `ERR NAME NOT_FOUND`. A v22 client reads such an answer as
+`registered` with no expiry, grace or reservation.
+
+From v22 a client MUST NOT read `ERR NAME NOT_FOUND` as "registrable": only
+`available` says that.
+
+A router that cannot state an answer completely MUST send `ERR NAME RESOLVER
+<detail>` rather than answer partially or guess. That covers a TLD with no
+registrar or price oracle, an unreachable chain, a timeout, a registration it
+could not date or resolve, and any status word it does not recognise.
+
+`nameRecord` MUST be a UTF-8 JSON object with the following schema:
 
 | Field | JSON type | Constraints |
 |---|---|---|
@@ -1513,34 +1614,16 @@ rname = %s"RNAME" SP json-bytes   ; json-bytes consumes the remainder of the tra
 | `owner` | string | `"0x"` followed by 40 lowercase hex characters (20 raw bytes) |
 | `resolver` | string | `"0x"` followed by 40 lowercase hex characters; the resolver contract address that produced the record |
 
-Text fields (`nickname`, `website`, `location`) use the empty string `""` as
-the "unset" sentinel: a backing resolver with no value for the field MUST emit
-an empty string, not JSON `null` and not an absent key. Link fields
-(`simplexContact`, `simplexChannel`) are arrays, primary link first, and use the
-empty array `[]` when unset. Coin fields (`eth`, `btc`, `xmr`, `dot`) use JSON
-`null` as the "unset" sentinel and MAY also be absent from the object entirely.
+Testnet-vs-mainnet status is derived from the queried TLD, not from the record.
 
-The backing resolver filters records that are expired or otherwise unavailable
-(the names router then returns `ERR NAME NOT_FOUND` to the client), so the wire
-format carries no expiry field. Testnet-vs-mainnet status is derived from the
-queried TLD rather than an in-record flag.
+Receivers MUST tolerate extra unknown fields; adding a required field is a
+breaking change requiring an SMP version bump. Receivers parse by key name, so
+peers MUST NOT rely on a byte-canonical form.
 
-Receivers MUST tolerate extra unknown fields (forward-compatibility for future
-field additions). Adding a required field is a breaking change requiring an
-SMP version bump.
-
-**Field order is not significant.** Receivers parse JSON by key name, so object
-key order, insignificant whitespace, and number formatting carry no meaning;
-records are interpreted by decoded value, never compared byte-for-byte. Peers
-MUST NOT rely on a byte-canonical form — a different resolver or server may emit
-the same record with different key order or spacing. This order-independence is
-what makes the format forward-compatible (see the unknown-field rule above).
-
-**Wire-size budget.** The names router caps the resolver response it will
-accept (`resolver_max_response_bytes`, ≤ 16000 bytes, the default) so the
-re-encoded `RNAME` stays within the SMP proxied transmission budget of 16224
-bytes; a response over the cap is rejected as `ERR NAME RESOLVER`. The link
-arrays are bounded by this overall budget rather than a fixed per-field count.
+The names router caps the resolver response it will accept
+(`resolver_max_response_bytes`, at most 16000 bytes) so the re-encoded `RNAME`
+stays within the SMP proxied transmission budget of 16224 bytes; a response over
+the cap is `ERR NAME RESOLVER`.
 
 ## Transport connection with the SMP router
 

protocol/xftp.md

@@ -1,4 +1,4 @@
-Version 3, 2025-01-24
+Version 4, 2026-08-08
 
 # SimpleX File Transfer Protocol
 
@@ -50,11 +50,12 @@ The objective of SimpleX File Transfer Protocol (XFTP) is to facilitate the secu
 
 XFTP is implemented as an application level protocol on top of HTTP2 and TLS.
 
-This document describes XFTP protocol version 3. The version history:
+This document describes XFTP protocol version 4. The version history:
 
 - v1: initial version
 - v2: authenticated commands - added basic auth support for commands
 - v3: blocked files - added BLOCKED error type for policy violations
+- v4: server public information and entitlement proof in handshake, file storage time in FNEW and SIDS
 
 The protocol describes the set of commands that senders and recipients can send to XFTP routers to create, upload, download and delete data packets of several pre-defined sizes. XFTP routers SHOULD support packets of 4 sizes: 64KB, 256KB, 1MB and 4MB (1KB = 1024 bytes, 1MB = 1024KB).
 
@@ -329,7 +330,7 @@ Once TLS handshake is complete, client and router will exchange blocks of fixed
 
 ```abnf
 paddedRouterHello = <padded(routerHello, 16384)>
-routerHello = xftpVersionRange sessionIdentifier routerCerts signedRouterKey ignoredPart
+routerHello = xftpVersionRange sessionIdentifier routerCerts signedRouterKey webIdentityProof serverInfo ignoredPart
 xftpVersionRange = minXftpVersion maxXftpVersion
 minXftpVersion = xftpVersion
 maxXftpVersion = xftpVersion
@@ -338,15 +339,27 @@ sessionIdentifier = shortString
 routerCerts = length 1*routerCert ; NonEmpty list of certificates in chain
 routerCert = originalLength <x509encoded>
 signedRouterKey = originalLength <x509encoded> ; signed by router certificate
+webIdentityProof = shortString ; signature over the web client challenge and sessionIdentifier, empty when the client sent no challenge
+serverInfo = %s"0" / (%s"1" largeString) ; JSON server public information, sent when maxXftpVersion is 4 or above
 
 paddedClientHello = <padded(clientHello, 16384)>
-clientHello = xftpVersion keyHash ignoredPart
+clientHello = xftpVersion keyHash entitlementProof ignoredPart
 ; chosen XFTP protocol version - must be the maximum supported version
 ; within the range offered by the router
 
+entitlementProof = %s"0" / (%s"1" issuerKeyIndex bbsProof entitlement)
+; proof of the user entitlement, encoded from v4, bound to sessionIdentifier
+issuerKeyIndex = 2*2OCTET ; Word16 index of the issuer public key
+bbsProof = largeString
+entitlement = entExpires entName entExtra
+entExpires = shortString ; expiration as an ISO8601 UTC timestamp
+entName = shortString ; e.g. "supporter", "legend"
+entExtra = largeString ; opaque to the router
+
 xftpVersion = 2*2OCTET ; Word16 version number
 keyHash = shortString
 shortString = length length*OCTET
+largeString = originalLength *OCTET
 length = 1*1OCTET
 originalLength = 2*2OCTET
 ignoredPart = *OCTET
@@ -432,7 +445,7 @@ Routers SHOULD support basic auth with this command, to allow only router owners
 The syntax is:
 
 ```abnf
-register = %s"FNEW " fileInfo rcvPublicAuthKeys basicAuth
+register = %s"FNEW " fileInfo rcvPublicAuthKeys basicAuth fileStorageTime
 fileInfo = sndKey size digest
 sndKey = length x509encoded
 size = 4*4 OCTET ; Word32 big-endian
@@ -440,19 +453,27 @@ digest = length *OCTET
 rcvPublicAuthKeys = length 1*rcvPublicAuthKey
 rcvPublicAuthKey = length x509encoded
 basicAuth = "0" / "1" length *OCTET
+fileStorageTime = %s"0" / (%s"1" storageHours)
+; encoded from v4; absent or zero requests the maximum the router allows
+storageHours = 4*4 OCTET ; Word32 big-endian
 
 x509encoded = <binary X509 key encoding>
 
 length = 1*1 OCTET
 ```
 
+`fileStorageTime` requests how long the router stores the data packet. The router grants the smaller of the request and the maximum it allows for the entitlement presented in the handshake, and returns the granted expiration in `sndIds`.
+
 If the data packet is registered successfully, the router must send `sndIds` response with the sender's and recipients' data packet IDs:
 
 ```abnf
-sndIds = %s"SIDS " senderId recipientIds
+sndIds = %s"SIDS " senderId recipientIds grantedStorageTime
 senderId = length *OCTET
 recipientIds = length 1*recipientId
 recipientId = length *OCTET
+grantedStorageTime = %s"0" / (%s"1" grantedExpires) ; encoded from v4
+grantedExpires = %s"T" expiresAt
+expiresAt = 8*8 OCTET ; Int64 big-endian, seconds since epoch
 ```
 
 #### Add data packet recipients

protocol/xrcp.md

@@ -121,9 +121,15 @@ During TLS handshake, parties validate certificate chains against previously kno
 
 ### Session verification and protocol negotiation
 
-Once TLS session is established, both the host and controller devices present a "session security code" to the user who must match them (e.g., visually or via QR code scan) and confirm on the host device. The session security code must be a digest of tlsunique channel binding. As it is computed as a digest of the TLS handshake for both the controller and the host, it will validate that the same TLS certificates are used on both sides, and that the same TLS session is established, mitigating the possibility of MITM attack in the connection.
+Once TLS session is established, both the host and controller devices present a "session security code" to the user. The session security code must be a digest of tlsunique channel binding. As it is computed as a digest of the TLS handshake for both the controller and the host, it will validate that the same TLS certificates are used on both sides, and that the same TLS session is established, mitigating the possibility of MITM attack in the connection.
 
-Once the session is confirmed by the user, the host sends HELLO block to the controller.
+To verify the session, the user matches the codes (e.g., visually or via QR code scan) and confirms the session on the host device.
+
+The user MUST verify the first session with a new controller. The host stores the long-term identity of the controller (CA fingerprint and Ed25519 public key) only after this session is verified. A controller with the stored long-term identity is a known controller.
+
+The user MAY verify the subsequent sessions with a known controller. In a session that the host confirms without user action, the controller is authenticated only by its long-term identity stored in the first session (see [Threat model](#threat-model)). SimpleX Chat mobile apps require user verification of sessions with known controllers only when the user enables the "Verify connections" option.
+
+Once the session is confirmed, the host sends HELLO block to the controller.
 
 XRCP blocks inside TLS are padded to 16384 bytes.
 
@@ -305,7 +311,7 @@ to receive: (rcvKey', sk, nonce) = HKDF(rcvKey, "SimpleXSbChain", 88)
 - prevent host and controller devices from establishing the session
 
 *cannot:*
-- same as passive adversary, provided that user visually verified session code out-of-band.
+- same as passive adversary, provided that user visually verified session code out-of-band in the first session with the controller.
 
 #### An active adversary with the access to the network:
 
@@ -324,6 +330,15 @@ to receive: (rcvKey', sk, nonce) = HKDF(rcvKey, "SimpleXSbChain", 88)
 *cannot:*
 - connect to the host or make host connect to itself.
 
+#### An active adversary with the access to the network who also obtained the long-term private keys of a known controller (e.g., from the controller device):
+
+*can:*
+- make host connect to itself instead of the controller.
+- access any data of the controlled host application, within the capabilities of the provided API, if the host confirms the session without user action.
+
+*cannot:*
+- access host application data, provided that user visually verified session code out-of-band.
+
 #### Compromised controller device:
 
 *can:*

rfcs/2026-07-11-service-rpc.md

@@ -0,0 +1,114 @@
+---
+Proposed: 2026-07-11
+Protocol: agent-protocol (new version)
+Depends on: 2026-07-12-address-pqdr-keys
+---
+
+# One-off requests to service addresses
+
+Implementation plan: to follow, after this and the address-DR RFC are reviewed.
+
+## Problem
+
+Client applications need to interact with services, for example: badge issuance, directory requests, telemetry submissions, blockchain reads and writes, LLM calls. The only communication primitive available today is a duplex connection, so each of these interactions requires the full connection procedure - creating queues, key agreement, double ratchet initialization - and leaves persistent state on both sides: queues, ratchet state, connection records, message history.
+
+This is the wrong primitive for most service interactions:
+
+1. Cost. To send the first request to a not yet connected service, the client and the service exchange multiple commands across two servers. For "search the directory" all of it is overhead. The setup cost also creates an incentive to keep connections open, and a service with N users who used it once permanently holds N sets of queues and ratchet states.
+
+2. Privacy. A connection is a stable pairwise pseudonym. If a service were to use a duplex connection, it could link all requests made over it into a profile: search history in the directory, blockchain operations linked even when different on-chain keys are used, telemetry that becomes longitudinal tracking. The client also accumulates history that can be recovered from the device. Where continuity is needed, it can be provided in the application protocol (e.g., a token included in requests), without a transport-level identity.
+
+3. Encryption. Messages sent to contact addresses outside an established connection have a single layer of X25519 encryption, with no post-quantum protection and no forward secrecy. This is not acceptable for service requests.
+
+In-app service addresses should be stored as names resolving to links via the existing addressing layer (server host in the link authority, current link data retrieved with `LGET`), so that service links can be changed without redeploying the apps. Name resolution is already supported, and out of scope.
+
+## Security objectives
+
+1. Requests from the same client must not be linkable to each other by the service or by servers, and no long term state is created on either side in the transport layer.
+2. Post-quantum resistant end-to-end encryption of requests and replies.
+3. Reply authenticity must be verifiable against the link; substitution, replay, dropping or reordering of replies by servers must be detectable.
+4. A repeated request for the same operation must not be executed twice.
+
+## Solution
+
+A service address is an ordinary short-link contact address. The client sends one request to the address queue and receives replies in a reply queue it creates for the request. The double ratchet is established from the address's published keys (see the address-DR RFC): the request is the first ratchet message, and replies are subsequent ratchet messages. So a request-response exchange is a short-lived one-directional double ratchet connection, established from the first message and removed after the last.
+
+The exchange:
+
+1. Retrieve the address link data (`LGET`, via proxy when IP protection is needed): the root key, the identity key, the prekey with its id, and the KEM key.
+2. Create a reply queue (`NEW`, subscribed).
+3. Establish the sending ratchet from the published keys (`pqX3dhSnd`, `initSndRatchet`). Build the request: the reply queue, the requester's X3DH parameters, and the payload encrypted under the ratchet. Encrypt the whole request to the address queue and send it once (unauthenticated `SEND`, via proxy when IP protection is needed). There is no transport retry; a reply is the success signal, and a hard error fails the request.
+4. The service establishes the receiving ratchet from its private keys and the request's X3DH parameters (`pqX3dhRcv`, `initRcvRatchet`), decrypts the payload, and delivers it to the service application. To reply it creates a send connection with the ratchet and sends reply messages to the reply queue, each encrypted under the ratchet.
+5. The client decrypts and delivers each reply message to the application. The first reply message returns from the request; later reply messages are delivered through a callback the application registered. The exchange ends on a reply marked final. The client deletes the reply queue and the ratchet on the final message, the deadline, or when the application cancels.
+
+How this meets the objectives:
+
+1. Unlinkability: fresh X3DH keys and a fresh reply queue per request; the sender's IP address and session are protected by existing private routing; the reply queue and both ratchets are removed after the exchange; nothing is shared between two requests.
+2. Encryption: the double ratchet with its sntrup761 KEM, from the first message.
+3. Authenticity: the ratchet is established against the identity key committed by the link hash, so a decryptable reply proves it came from the address owner; the ratchet message numbering detects dropped and reordered replies. No separate signature is needed.
+4. Single execution: the service identifies a request by the hash of its decrypted payload and, within a fixed retention period, re-sends the stored replies for a repeated request without running the operation again.
+
+## Design
+
+Syntax uses [ABNF][1] with [case-sensitive strings extension][2]. `SMPQueueInfo`, `agentVersion`, and the X3DH and ratchet types are as in the [agent protocol](../protocol/agent-protocol.md) and `Crypto.Ratchet`.
+
+### Correlation and chat
+
+A reply is connected to its request by the reply queue: each request has its own reply queue, and every message in that queue is a reply to that request. The request hash (of the decrypted payload) is used only for idempotency. The application sets an id inside the payload to make two requests the same operation or different ones; the agent does not read it.
+
+Both ends are chat bots on the chat library over the agent. The chat library serializes a service command into the request payload and deserializes the responses; the agent transports them, establishes and removes the ratchet, and correlates by reply queue.
+
+### Request
+
+A new agent envelope, shaped like `AgentConfirmation` (X3DH parameters to establish the ratchet, plus a body encrypted under it):
+
+```abnf
+agentRequest = agentVersion %s"Q" replyQueues prekeyId sndE2EParams encRequest
+replyQueues = length 1*SMPQueueInfo ; the first is used; more are for redundancy
+prekeyId = shortString ; the published prekey used, from link data
+sndE2EParams = <requester Snd X3DH parameters, see address-DR RFC>
+encRequest = <ratchet-encrypted request payload>
+```
+
+The whole `agentRequest` is encrypted to the address queue with the per-queue layer and sent with `SEND`, as an invitation is today. The reply queue keys and the X3DH parameters are not visible to servers. A request must fit one message; a larger payload is an XFTP file description in the payload.
+
+The request hash is the SHA3-256 of the decrypted payload - the same bytes on both sides. There is no transport retry; a hard error (`AUTH`, `QUOTA`) fails the request, and the application decides whether to send a new one.
+
+### Replies
+
+The service creates a send connection with the ratchet and sends reply messages to the reply queue, each encrypted under the ratchet. A reply message uses a new agent envelope:
+
+```abnf
+agentResponse = agentVersion %s"P" final responses
+final = %s"T" / %s"F" ; T - no more reply messages follow
+responses = length 1*responseItem ; non-empty list of application responses
+responseItem = largeString ; opaque application response
+```
+
+Each message includes a list of responses, so responses known together are sent in one message and responses that become known over time are sent in separate messages. The message is encrypted and numbered by the ratchet, which authenticates it against the committed identity key and detects dropped or reordered messages; no separate signature is used.
+
+The first reply message returns from the request. Later reply messages are delivered through the callback the application registered with the request, while the process runs. The exchange ends on a message with `final = T`. The client deletes the reply queue and the ratchet on that message, on the deadline, or when the application cancels. Deleting the reply queue stops further replies. The transport keeps no exchange across a client restart; the application keeps its own state and sends a new request when it needs to.
+
+### Rejection
+
+The service refuses a request with the `AgentRejection` envelope from the [communicating rejection RFC](../../simplex-chat/docs/rfcs/2024-03-22-communicating-reject.md), sent to the reply queue under the ratchet, with an opaque application reason. The same envelope communicates refusal of a connection request, where today it is dropped silently. A rejection ends the exchange like a final reply.
+
+### Idempotency
+
+The service keeps, for a fixed retention period it chooses (1 to 24 hours, in service configuration, not in link data), the request hash, the ordered response messages it produced, and the reply queues and ratchets subscribed under that hash. A repeat request with the same hash does not reach the service application:
+
+- while the first request is being answered, the repeat establishes its own ratchet and reply queue, is added to the record, receives the responses already produced, and receives each later response too.
+- after the operation completed, the repeat receives the whole stored sequence of responses, re-encrypted under its own ratchet.
+
+The stored responses are the application response bytes, not the ratchet ciphertext, because a repeat establishes a new ratchet and the responses are re-encrypted for it. This gives single execution over at-least-once delivery. After the retention period a request with the same hash is a new operation and runs again.
+
+### Out of scope
+
+- Recovery across restart: the transport keeps no exchange across a client restart; the application persists its own state and sends a new request when it needs to.
+- Service-initiated messages: there is no standing channel; use a connection where the service must reach the client without a request.
+- Abuse protection beyond existing queue quotas: services can require application-level credentials (e.g., a badge) in the request payload; rate limiting is a separate discussion.
+- Scaling request reception: a single address queue bounds service throughput; distributing reception across multiple queues or relays (the existing `relays` field in contact link data) is a separate question, but it would fit well with name resolving to multiple addresses, both for redundancy, reliability and higher throughput.
+- Name resolution: existing addressing layer.
+
+[1]: https://tools.ietf.org/html/rfc5234
+[2]: https://tools.ietf.org/html/rfc7405

rfcs/2026-07-12-address-pqdr-keys.md

@@ -0,0 +1,83 @@
+---
+Proposed: 2026-07-12
+Protocol: agent-protocol (new version)
+---
+
+# Establishing the double ratchet from address data
+
+## Problem
+
+A contact address receives a connection request as an `AgentInvitation` message, encrypted only with the per-queue X25519 layer. The double ratchet is established later: the address owner joins the requester's connection request, generates its X3DH keys, and sends them in the confirmation. So the first message to an address - the invitation and the profile in it - is not under the double ratchet, and has no post-quantum protection.
+
+The cause is that the address owner's X3DH contribution is generated per request and sent in the confirmation, so it cannot exist before the requester's first message.
+
+## Solution
+
+Publish the address owner's X3DH contribution in the address link data, so a requester can establish the double ratchet in its first message. The requester runs the existing `pqX3dhSnd` against the published keys, initializes a sending ratchet, and encrypts its first message under it. The owner runs the existing `pqX3dhRcv` against its stored private keys and the requester's X3DH keys from the message, initializes a receiving ratchet, and decrypts it.
+
+Publish the owner's X3DH contribution - two X448 keys and an optional sntrup761 KEM key, with the e2e version range - as one bundle in the mutable contact user data, signed by the root key. The bundle is the existing `RcvE2ERatchetParamsUri` type that a one-time invitation already advertises, with an id for rotation.
+
+This is backward compatible. A requester that does not use the published bundle sends a current `AgentInvitation` with its own X3DH keys, and the owner does what it does today: generates fresh X3DH keys and sends them in the confirmation. The owner branches on whether the incoming message uses the published bundle.
+
+Three properties follow. The first message, including the profile, is under the double ratchet, which closes the profile gap and gives it post-quantum protection through the ratchet's sntrup761 KEM. A decryptable message proves the sender established X3DH against the root-signed keys, so it authenticates the address owner without a separate signature. And because the bundle is in mutable data, an existing address can advertise the double ratchet by updating its mutable data - no new link.
+
+## Design
+
+Syntax uses [ABNF][1] with [case-sensitive strings extension][2]. Key and ratchet types are as in `Crypto.Ratchet` and the [agent protocol](../protocol/agent-protocol.md); all DH keys are X448, the KEM is sntrup761.
+
+### Published keys in link data
+
+The owner's X3DH contribution is a ratchet-keys bundle appended to the mutable contact user data, signed by the root key. Nothing is added to the immutable fixed link data:
+
+```abnf
+userContactData =/ ratchetKeys ; appended, ignored by earlier versions
+ratchetKeys = %s"0" / (%s"1" ratchetKeyId e2eParams)
+ratchetKeyId = shortString ; identifies this bundle, changes on rotation, echoed in the request
+e2eParams = <RcvE2ERatchetParamsUri: e2e version range, two X448 keys, optional sntrup761 key>
+```
+
+`e2eParams` is the existing `RcvE2ERatchetParamsUri` - the same type a one-time invitation advertises - so a requester reads it, negotiates the concrete e2e version against its own range, and runs `pqX3dhSnd` against it, exactly as it does for an invitation. The KEM key is optional (the params' KEM field is a `Maybe`), controlled by the address's initial-keys mode, the same 3-way choice as invitations: advertise the KEM (post-quantum from the first message), advertise X448-only but still support post-quantum if the requester proposes its own KEM (one message later, avoiding the ~1158-byte key in link data), or no post-quantum.
+
+The owner keeps the private side of each bundle - the two X448 private keys and, when PQ is on, the KEM keypair - indexed by `ratchetKeyId`. On rotation with `LSET` it publishes a new bundle with a new `ratchetKeyId` and keeps the previous private keys for a window covering queue message retention, so a request that used a just-rotated bundle still decrypts. Because the bundle is in mutable data, an existing address advertises the double ratchet by updating its mutable data - no new link, no re-creation.
+
+### Request confirmation
+
+A requester that uses the published keys establishes the sending ratchet before its first message, so it sends that message as a confirmation, not an invitation - the same envelope a joining party sends in a connection. The confirmation gains an optional `ratchetKeyId` naming the bundle the requester used, so the owner selects the matching private keys:
+
+```abnf
+agentConfirmation =/ ratchetKeyId ; the bundle the requester used, echoed; absent on other confirmations
+```
+
+The confirmation holds the requester's Snd X3DH parameters (so the owner runs `pqX3dhRcv`) and, encrypted under the ratchet, the first message. A confirmation with `ratchetKeyId` on a contact address takes the published-key path; an `AgentInvitation`, as today, takes the current path where the owner generates fresh X3DH keys and returns them in its own confirmation. A connection-request URI is unchanged: it advertises the requester's Rcv parameters and must not include Snd parameters.
+
+### Establishing the ratchet
+
+Requester:
+
+1. Retrieve link data (`LGET`), read the bundle - its `ratchetKeyId` and `e2eParams` (`RcvE2ERatchetParamsUri`) - and negotiate the concrete e2e version against its own range.
+2. `generateSndE2EParams` for its own X3DH contribution - encapsulating to the bundle's KEM if it advertises one, or proposing its own KEM if the requester wants post-quantum and the bundle is X448-only.
+3. `pqX3dhSnd` against the bundle's parameters, then `initSndRatchet` - the sending ratchet.
+4. Encrypt the first message under the ratchet, and send a confirmation with its Snd parameters and `ratchetKeyId`.
+
+Owner:
+
+1. On a confirmation with `ratchetKeyId` on a contact address, select the private X3DH keys and, if any, KEM keypair by `ratchetKeyId` (current or a retained previous generation).
+2. `pqX3dhRcv` against the requester's Snd parameters with those private keys, then `initRcvRatchet` - the receiving ratchet. Decrypting the first message advances the ratchet and gives it a send side, so the owner can reply.
+3. Decrypt, and reply under the ratchet.
+
+A request whose `ratchetKeyId` is no longer retained cannot be decrypted; the owner does not learn the requester or its reply address, and the requester's attempt fails at its own timeout.
+
+### Authentication
+
+The ratchet-keys bundle is in the mutable link data, signed by the root key. A decryptable message proves the sender established X3DH against those root-signed keys: an SMP server cannot substitute them without forging the root signature (`decryptLinkData` verifies it), which is the X3DH anti-substitution property. Where a message today relies on a separate signature over its content for authenticity, this establishment provides it, and the signature is not needed. The root Ed25519 key is the address's signing identity; the X3DH keys are separate DH keys (X448), and X3DH is over crypto_box, so deniability is preserved. A malicious server can still serve an older but validly-signed bundle (rollback to a retired generation); this is bounded by the retention window and by the ratchet advancing after the first message, and a per-key signature would not prevent it. Reusing a bundle across requesters is consistent with the address already being a shared identifier.
+
+## Uses
+
+- Invitations to an address, and the profile in them, are under the double ratchet from the first message.
+- An existing address gains the double ratchet when the app updates its mutable link data (`LSET`, with the user's confirmation and current profile, combined with the full→short address migration); no new link is issued.
+- The service RPC (see the RPC RFC) establishes the ratchet this way to send the request as the first ratchet message.
+
+This RFC depends on nothing else here. It replaces the need for the PQ-queue RFC in the address case, because the ratchet provides post-quantum protection for the first message; the PQ-queue RFC remains for first messages that do not establish a ratchet.
+
+[1]: https://tools.ietf.org/html/rfc5234
+[2]: https://tools.ietf.org/html/rfc7405

rfcs/2026-07-12-queue-pq-encryption.md

@@ -0,0 +1,42 @@
+---
+Proposed: 2026-07-12
+Protocol: smp-client (new version)
+---
+
+# Post-quantum encryption of the SMP queue layer
+
+## Problem
+
+A message sent to a queue outside an established double ratchet has one layer of end-to-end encryption: NaCl crypto_box over an X25519 DH secret. The sender generates an ephemeral X25519 key, computes the secret with the recipient's per-queue DH key, and puts its ephemeral key in the message public header (`agentCbEncryptOnce`). This layer protects invitations, confirmations, and the profile sent with them.
+
+It is not post-quantum. An adversary that records this traffic and later has a quantum computer can recover the X25519 secret and decrypt it. The double ratchet adds a post-quantum KEM once it is established, but the first message to a queue, before the ratchet, has only X25519.
+
+This RFC adds post-quantum protection to the single-shot queue encryption itself, for cases that do not establish a double ratchet from the first message. Where a double ratchet is established from the first message (see the address-DR RFC), the ratchet provides post-quantum protection and this layer is not needed.
+
+## Solution
+
+Extend the single-shot queue encryption to a hybrid X25519 + sntrup761 scheme, in a new SMP client version. The recipient publishes a KEM encapsulation key alongside its per-queue DH key. The sender encapsulates to it, combines the KEM shared secret with the X25519 DH secret, and encrypts the body with the combined secret. The KEM ciphertext travels in the message public header next to the ephemeral X25519 key.
+
+Recording the traffic and breaking X25519 later is not sufficient: without breaking sntrup761 as well, the combined secret is not recoverable.
+
+## Design
+
+The message public header (`PubHeader`) gains a hybrid variant, selected by a version and a tag, so older senders and the empty-header case are unchanged:
+
+```abnf
+smpPubHeaderHybrid = smpClientVersion %s"2" senderPublicDhKey kemCiphertext
+senderPublicDhKey = length x509encoded ; sender ephemeral X25519 key
+kemCiphertext = largeString ; sntrup761 ciphertext, 1039 bytes
+```
+
+The secret combines both shared secrets, and the body is encrypted with NaCl secret_box (the DH-only path uses crypto_box today; the combined secret is no longer a plain DH result, so it is used as a secret_box key), padded to the same lengths:
+
+```
+secret = HKDF(dh(recipient key, sender ephemeral key) || KEM shared secret)
+```
+
+The recipient's KEM encapsulation key is distributed the same way its per-queue DH key is today - in the queue address for a connection request, and in link data for a short link. The KEM ciphertext is stored the same way the ephemeral DH key is: in the public header, readable by the destination server (and not by the proxy with proxied sending), which cannot derive the secret without the recipient's KEM private key.
+
+The recipient stores the computed secret the way the per-queue DH secret is stored on receiving the first message (`setRcvQueueConfirmedE2E`), and reuses it for later messages on the queue.
+
+Sizes: the KEM ciphertext is 1039 bytes and the encapsulation key 1158 bytes. Link data user data is padded to 13784 bytes, so the key fits with application data. This RFC is independent of the RPC, SSND, and address-DR RFCs.

rfcs/2026-07-12-smp-secure-send.md

@@ -0,0 +1,55 @@
+---
+Proposed: 2026-07-12
+Protocol: smp (new version)
+---
+
+# SSND: combined secure-and-send command
+
+## Problem
+
+Two SMP flows secure a messaging queue and then immediately send the first message to it, as two commands and two round trips:
+
+- The fast connection handshake: the joining party secures the queue with `SKEY`, then sends the confirmation with `SEND`.
+- Any first send to a sender-securable queue where the sender both secures it and delivers the first message.
+
+The two commands express one intent - "this is my key, and here is my first message" - so they can be one command and one round trip. The combination must be idempotent, because the first send is retried on network failure and a queue is often secured before the response is known. `SKEY` is already idempotent (a repeat with the same key succeeds). `SEND` is not, so a naive combination would deliver a duplicate message on retry.
+
+## Solution
+
+A new command `SSND` combines `SKEY` and `SEND` in one transmission, idempotent in both parts:
+
+- Key part: as `SKEY` - a repeat with the same key succeeds, a different key fails with `AUTH`.
+- Send part: the server keeps the hash of the message until it is acknowledged, and reports a repeat of the same message as delivered without delivering it again.
+
+The server-side hash covers the common case, when the retry arrives before the message is acknowledged. A retry that arrives after the acknowledgement is delivered as a duplicate and discarded by the receiving agent by message hash, as duplicate messages are discarded today.
+
+## Design
+
+Syntax uses [ABNF][1] with [case-sensitive strings extension][2]. `senderAuthPublicKey`, `msgFlags` and `smpEncMessage` are as in the [SMP protocol](../protocol/simplex-messaging.md).
+
+```abnf
+secureSend = %s"SSND " senderAuthPublicKey SP msgFlags SP smpEncMessage
+senderAuthPublicKey = length x509encoded
+```
+
+`SSND` is a sender command, authorized with the key it sets, and accepted only on messaging-mode queues (`QMMessaging`), where the sender can secure the queue. The server responds `OK` or `ERR`.
+
+Server processing:
+
+1. Secure the queue with the key, as `SKEY`. A repeat with the same key succeeds; a different key returns `AUTH`.
+2. If the queue holds an unacknowledged message whose stored hash equals the hash of this message, respond `OK` without storing it again.
+3. Otherwise store the message, keep its hash with the queue until the message is acknowledged, and deliver it.
+
+The stored hash is one value per not-yet-acknowledged queue message. It is removed when the message is acknowledged. All queue store backends (in-memory, journal, PostgreSQL) keep it.
+
+`SSND` composes with the proxy protocol without change: `proxySMPCommand` forwards any sender command through `PFWD`/`RFWD`, so `SSND` is proxied as `SEND` and `SKEY` are today.
+
+## Uses
+
+- The fast connection handshake replaces `SKEY` then the `SEND` confirmation with one `SSND`.
+- The service RPC response (see the RPC RFC) secures the reply queue and sends the first reply with one `SSND`.
+
+This RFC is independent of the RPC, PQ-queue, and address-DR RFCs and can be implemented on its own.
+
+[1]: https://tools.ietf.org/html/rfc5234
+[2]: https://tools.ietf.org/html/rfc7405

rfcs/2026-08-09-fast-queue-rotation.md

@@ -0,0 +1,97 @@
+---
+Proposed: 2026-08-09
+Protocol: agent-protocol v8
+Diagram: ../protocol/diagrams/duplex-messaging/queue-rotation-fast.svg
+---
+
+# Fast queue rotation
+
+## Problem
+
+In the current rotation the peer returns `QKEY` to the initiator over the initiator's current
+receiving queue. When that queue's server is unavailable the rotation cannot complete, so a client
+cannot move away from a failed server.
+
+## Solution
+
+Both the current rotation and v8 add a queue, deliver to both queues while the rotation is in
+progress, and remove the old queue; the recipient drops duplicates in both. The main difference is where
+the new queue's secret is established. In the current rotation it is established over the current
+queue, by `QKEY`, so it cannot complete when the current server is down. In v8 the peer establishes the
+new queue's secret over the new queue itself — a confirmation it sends on R' — so establishing the
+secret no longer depends on the current queue, and the rotation completes even when the current server
+is down.
+
+v8 also starts writing to both queues earlier: from the moment the queue is added, including the
+current queue's not-yet-delivered backlog. So the initiator adds the new queue with `QADD`; from that
+point the peer writes every message to both the current queue and R'. Once R' is secured the peer
+writes new messages to it alone, while the current queue delivers whatever was already scheduled on it
+and a final `QEND`, and is then removed. Because the recipient drops duplicates, neither the order of
+arrival nor which queue carries a message matters, provided each message arrives on at least one queue
+— with one exception, the confirmation, which is always the first message on R'. A dead new queue does
+not stop delivery either, because the current queue keeps delivering until R' is secured.
+
+Roles: A initiates (its receiving queue rotates; A receives on the new queue R'). B is the peer (B
+holds the sending queue to A, secures R', and delivers to both).
+
+Sequence:
+
+    A -> R'       : create new queue (messaging mode, SKEY allowed)
+    A -> S -> B   : QADD(R')          (over A's sending queue; A's current server untouched)
+    B             : from QADD, schedule every message and the current backlog on both queues
+    B -> current  : deliver the scheduled messages (R' holds its copies while securing)
+    B -> R'       : SKEY              (authorize B as sender)
+    B -> R'       : confirmation      (empty; establishes R' secret; first message on R')
+    B -> R'       : deliver R''s held copies (A dedups), then new messages to R' only
+    B -> current  : deliver the remaining tail, then QEND(current)
+    B -> R'       : QEND(current)
+    A             : on QEND, delete the current queue; keep receiving on R'
+
+## Confirmation
+
+The confirmation is the only message a recipient can read on a queue that is not yet secured, and it
+establishes the queue's shared secret without depending on the current queue. Because a data message
+that reached R' before the confirmation could not be read, the confirmation is the first message the
+peer sends on R', and the peer does not start ordinary delivery on R' until the confirmation has been
+sent.
+
+The confirmation body is empty: both parties already know each other, so no profile or reply queue is
+sent. It is sealed by the queue's box (keyed by the shared secret being established) and is not
+additionally encrypted with the double ratchet, so rotation does not advance the message ratchet.
+
+## Termination
+
+`QEND` names a queue to remove and is delivered on both queues. On receipt the recipient deletes the
+named queue; on send the peer removes its sending queue of that address. `QEND` is a general
+queue-removal message — the peer can remove either queue with it — so on the wire a rotation is the
+addition of a queue (`QADD`) and the later removal of the replaced one (`QEND`), each an ordinary
+operation on the queue set rather than a `QTEST`-style completion. Delivering `QEND` on the removed
+queue is best effort; the copy on the surviving queue removes it and reaches the recipient even when
+the removed server is dead.
+
+## Per-queue secret
+
+R' secret is a fresh Diffie-Hellman between the peer's queue key, sent in the confirmation header, and
+the initiator's R' key. It does not depend on any current queue, so redundancy of current queues is
+unaffected.
+
+## Compatibility
+
+Fast rotation runs only when the connection's agreed agent protocol version is 8 or higher; the peer
+chooses it. Otherwise the `QKEY`/`QUSE` exchange is used. `QEND` is defined at version 8 and is only
+sent during fast rotation, so peers below version 8 never receive it.
+
+    new A / new B  : fast (QADD, confirmation on R', QEND)
+    new A / old B  : slow (old B returns QKEY; new A keeps the QKEY/QUSE handling)
+    old A / new B  : slow (agreed version below 8; new B returns QKEY)
+    old A / old B  : slow
+
+The recipient does not choose by version; it reacts to whichever message arrives — `QKEY`, or a
+confirmation on R' followed later by `QEND`.
+
+## Dead current server
+
+The initiator keeps reading messages on the new queue and removes the current queue when `QEND`
+arrives there, without waiting for the current server. Nothing is lost, because every message is
+scheduled on the new queue; the only cleanup that a dead current server delays is the deletion
+of its queue, which is retried a bounded number of times and then abandoned.

rfcs/2026-08-22-xftp-file-storage-time.md

@@ -0,0 +1,109 @@
+# XFTP variable file storage time
+
+## Summary
+
+The server stores a storage time for each file. The sender sets it in the FNEW command. The client may present a proof of an entitlement in the handshake to raise the maximum storage time the server allows. The proof is bound to the TLS session, so it cannot be reused for another session.
+
+An entitlement belongs to the user, not to a file: the client presents it once per connection, and the server applies it to everything the client does in that session. The server can also vary other limits, such as throttling, by the entitlement.
+
+## Entitlement
+
+An entitlement is a name, an expiration, and an extra string. It is the disclosed content of a BBS proof: the holder's secret remains undisclosed, and the three fields are revealed. The server reads `entName` to select a maximum storage time, checks `entExpires`, and ignores `entExtra`; the interpretation of `entExtra` is out of scope here. The protocol references only the entitlement, never a badge; chat maps its own badge to an entitlement before it asks the agent to send.
+
+The proof discloses the entitlement and includes the issuer key index and the BBS proof. The holder's secret and the BBS signature remain with the sender and are never transmitted. The origin of the sender's signed entitlement, from the entitlement service, is out of scope here.
+
+```
+entitlement = entExpires entName entExtra
+entExpires = shortString     ; expiration as a UTCTime ISO8601 string
+entName = shortString        ; e.g. "supporter", "legend"
+entExtra = largeString       ; opaque, interpretation out of scope
+
+entitlementProof = issuerKeyIndex bbsProof entitlement
+issuerKeyIndex = 2*2 OCTET   ; Word16, network byte order
+bbsProof = largeString       ; BBS proof bytes
+```
+
+The presentation header that the BBS proof is generated over is not transmitted; the server takes it from the session (see [Binding](#binding)), which is what binds the proof.
+
+### Issuer keys
+
+Client apps and servers share one list of issuer public keys, indexed by `issuerKeyIndex`. The secret key for the current index is held by the service that issues entitlements, on conditions that are out of scope here, such as payment.
+
+The list holds eight keys so that the issuing service can rotate its current key without an app or server release: the next index is already known to every app and server. Each rotation consumes one index, so a new key has to be added to client apps and servers eventually, and released before the list is exhausted.
+
+## Storage time
+
+```
+fileStorageTime = %s"0" / (%s"1" storageHours)
+storageHours = 4*4 OCTET     ; Word32, network byte order
+```
+
+The storage time is an optional number of hours. Absent (`%s"0"`), or present as zero hours, requests the maximum the server allows for the presented entitlement, or the default maximum when no proof is present. A value above zero requests that number of hours, and the server grants the smaller of it and the maximum.
+
+## Handshake, new XFTP version
+
+The client handshake carries the entitlement proof.
+
+```
+clientHandshake = xftpVersion keyHash optEntitlementProof
+optEntitlementProof = %s"0" / (%s"1" entitlementProof)
+```
+
+`xftpVersion` and `keyHash` are defined by the current XFTP protocol. Version 3 and earlier encode no proof.
+
+The server verifies the proof when it accepts the handshake and keeps the result for the session; further handshakes on the same connection keep that result and verify nothing, so a session costs one verification however many handshakes it makes. A proof that names an entitlement the server does not configure, or one whose expiration passed 24 hours ago or more, is ignored without verification; a proof that fails to verify is logged. In each case the session gets the default maximum. The response is the same in every case, but only a configured, unlapsed name costs a verification, so the handshake latency tells the client which names the server configures.
+
+## Commands
+
+The new protocol version extends FNEW with the storage time.
+
+```
+fnew = %s"FNEW " fileInfo rcvKeys optBasicAuth fileStorageTime
+```
+
+`fileInfo`, `rcvKeys`, and `optBasicAuth` are defined by the current XFTP protocol. Version 3 and earlier encode no `fileStorageTime`, and the server applies the default storage time.
+
+## Responses
+
+FNEW extends the SIDS response with the granted storage.
+
+```
+sndIds = %s"SIDS " senderId rcvIds optGrantedStorageTime
+optGrantedStorageTime = %s"0" / (%s"1" grantedStorageTime)
+grantedStorageTime = grantedExpires
+grantedExpires = %s"T" expiresAt
+expiresAt = 8*8 OCTET        ; Int64, seconds since epoch (absolute UTC instant), network byte order
+```
+
+`grantedExpires` returns the absolute expiration — the same value stored for the file. The sum encoding retains a one-character prefix so further variants can be added. Version 3 and earlier omit `optGrantedStorageTime` entirely; a client decoding such a response reads it as absent. `senderId` and `rcvIds` are defined by the current XFTP protocol.
+
+## Binding
+
+The presentation header binds the proof to the TLS session, so a proof presented on any other session fails to verify.
+
+```
+presHeader = sessionId
+```
+
+`sessionId` is the TLS session identifier, the TLS unique channel binding. Both sides take it from the connection: the client has it once TLS is established, and the client checks that the identifier the server sends in its handshake matches.
+
+Binding to the session is what stops a proof being replayed by another client. A proof is not bound to a file, because the entitlement belongs to the user and authorises everything the client does in that session.
+
+BBS proofs of one credential are unlinkable, but every proof discloses the same `entExpires`, `entName` and `entExtra`, so a server can link the sessions of one holder by that triple, and two servers can correlate them. The issuing service decides how identifying it is: `entExpires` set to the same instant for everyone who buys in the same period, and an `entExtra` that carries nothing per-holder, make the holders of that period indistinguishable.
+
+## Maximum storage time
+
+The server configures a maximum storage time for each entitlement name, and a default maximum for requests with no proof. Each maximum is a number of hours. The server exits at startup if any name's maximum is below the default, so a proof never reduces the allowed time. The server honours an entitlement for 24 hours after its expiration; past that grace it is treated as no proof.
+
+If the requested time exceeds the maximum, the server stores the file for the maximum and does not reject the request. The expiration is rounded up to the hour, stored, and returned as `grantedExpires`.
+
+## Encoding primitives
+
+```
+shortString = length *OCTET   ; 0-255 bytes
+largeString = length2 *OCTET
+length = 1*1 OCTET
+length2 = 2*2 OCTET           ; Word16, network byte order
+```
+
+`senderId`, `rcvIds`, `fileInfo`, `sndKey`, `digest`, `rcvKeys`, `optBasicAuth`, and `sessionId` are defined by the current XFTP and SMP protocols.

scripts/resolver/README.md

@@ -62,15 +62,26 @@ curl -s -X POST http://127.0.0.1:8545 \
 **2. resolver is healthy:**
 ```sh
 curl -s http://127.0.0.1:8000/health | jq
-# → {"ok": true, "rpc": "http://reth:8545", "registries": {"testing": "0x…", "simplex": ""}}
+# → {"ok": true, "rpc": "http://reth:8545", "registries": {"testing": "0x…", "simplex": ""},
+#    "blockNumber": 23400000, "chainLagSeconds": 12}
 ```
 
+`chainLagSeconds` is how far the node is behind the wall clock. A resolver that
+is reachable and answering can still be hours behind, and every name it reports
+is that stale. `null` means the node could not be reached.
+
 **3. resolver resolves a live name** (`foobar.testing` is a populated test name):
 ```sh
 curl -s http://127.0.0.1:8000/resolve/foobar.testing | jq
 # → {"name":"foobar.testing","nickname":"Foo","simplexContact":["https://smp16.simplex.im/a#…"], … }
 ```
 
+**4. the route your router will call** (check 3 passes on an older resolver too):
+```sh
+curl -s http://127.0.0.1:8000/v2/resolve/foobar.testing | jq
+# → {"type":"registered","expires":1780000000,"graceUntil":…,"nameRecord":{…}}
+```
+
 **Wire your smp-server:** in its `[NAMES]` section set
 `resolver_endpoint: http://127.0.0.1:8000` (no auth needed for loopback).
 
@@ -82,7 +93,7 @@ curl -s http://127.0.0.1:8000/resolve/foobar.testing | jq
 | reth p2p | `:30303` tcp/udp | Ethereum sync (open on firewall) |
 | nimbus p2p | `:9000` tcp/udp | beacon sync (open on firewall) |
 | nimbus REST | `127.0.0.1:5052` | beacon API |
-| **resolver** | `127.0.0.1:8000` | SNRC REST (`/resolve`, `/health`) |
+| **resolver** | `127.0.0.1:8000` | SNRC REST (`/v2/resolve`, `/resolve`, `/health`) |
 
 ## Caveats
 
@@ -110,7 +121,57 @@ standalone for local dev (no Docker), via [`uv`](https://docs.astral.sh/uv/):
 uv run scripts/resolver/service/snrc-resolve.py  # defaults to local reth + mainnet .testing
 ```
 
-### Response shape
+Three routes, versioned separately from the protocol so each only changes when
+its own shape does:
+
+| Route | Called by | Answers |
+|---|---|---|
+| `/v2/resolve/<query>` | routers from SMP v22 | a `NameRegistration` |
+| `/resolve/<name>` | routers before SMP v22 | a name record, flat |
+| `/health` | anyone | readiness |
+
+`/v1/resolve/<name>` is an alias for `/resolve/<name>`.
+
+### v2: `/v2/resolve/<query>`
+
+The body is the SMP protocol's `NameResponse`, which the router decodes as is
+and forwards; translating the registry's model to it is this resolver's job. It
+is the registration below, plus `lastBlockTs`, the timestamp of the block it was read
+at: a node that lags answers with names it has not seen registered yet.
+Its `type` is `registered`, `available` or `reserved`, and the fields each one
+carries are specified once, in the **Name response** section of
+[`protocol/simplex-messaging.md`](../../protocol/simplex-messaging.md). It is
+the wire format, so it is documented with the wire.
+
+Two things follow from that and are worth stating here. Expiry and grace belong
+to the registration, not to the record: `expires` and `graceUntil` sit beside
+`nameRecord`, not inside it. And a name nobody holds is not an error: it
+answers 200 with `type: available` and its price, so no status code from this
+route means "not registered".
+
+| Status | Meaning |
+|---|---|
+| 200 | a registration: `registered`, `available` or `reserved` |
+| 400 | `tldNotConfigured`, `notFullyQualified` |
+| 502 | `noPriceOracle`, `labelNotRecorded`, an unreadable status, or `upstreamError` |
+
+Error bodies carry `name` and a fixed `error` code to branch on. Only
+`upstreamError` adds a `message`; the v1 route always adds one.
+
+`labelNotRecorded` means the registrar holds the name but never recorded its
+label, so a hashed query cannot be answered with a name. See
+[Querying by labelhash](#querying-by-labelhash).
+
+A subname reports the expiry and grace of the 2LD above it, since that is what
+bounds its lifetime. A subname nobody created reports as not registered.
+
+### v1: `/resolve/<name>`
+
+What routers before SMP v22 call. Its shape is unrelated to v2's: the record is
+flat, and `status`, `expires`, `graceEnds`, `reasonCode` and `reason` sit
+alongside its fields.
+
+#### v1 response shape
 
 ```jsonc
 {
@@ -119,28 +180,197 @@ uv run scripts/resolver/service/snrc-resolve.py  # defaults to local reth + main
   "simplexContact": ["https://smp16.simplex.im/a#…", "https://smp11…"],  // primary first, fallbacks after
   "simplexChannel": [],
   "eth": null, "btc": "bc1q…", "xmr": "4ANz…", "dot": "139G…",
-  "owner": "0xd83b…", "resolver": "0x80fa…"
+  "owner": "0xd83b…", "resolver": "0x80fa…",
+  "status": "registered",      // registered | grace | expired | unregistered | unknown
+  "expires": 1780000000,       // Unix seconds; when the registration ends
+  "graceEnds": 1787776000,     // expires + GRACE_PERIOD; last moment the owner can renew
+  "reasonCode": null,          // set when the name is held back as well
+  "reason": null               // set when the name is held back as well
 }
 ```
 
 `simplexContact`/`simplexChannel` are arrays (a name can advertise multiple SMP
-servers; clients try them in order). On-chain they're a single comma-separated
+servers; clients try them in order). On-chain they're a single `;`-separated
 text record; the resolver splits/trims/drops-empties. Address encodings are
 canonical per chain (EIP-55 / bech32 / SS58 / Monero-base58). Subnames work
 identically (`bar.foobar.testing`).
 
-### Status codes
+#### v1 registration status and expiry
+
+A response carries `status`, `expires` and `graceEnds` whenever the resolver
+read them, a successful resolve included, so a client that has just resolved a
+name already knows when it expires. Both timestamps are Unix seconds, and
+`null` when they could not be read.
+
+| `status` | Meaning |
+|---|---|
+| `registered` | live; `expires` is when that ends |
+| `grace` | lapsed, but only the previous owner may renew it, until `graceEnds` |
+| `expired` | lapsed and past grace; anyone may register it |
+| `unregistered` | never registered, and free to take |
+| `unknown` | no `SNRC_REGISTRAR_<TLD>` configured, so status could not be read |
+
+A reservation is orthogonal to the status: a name held back by the registry
+carries `reasonCode` and `reason` whether or not it is registered.
+
+`grace` and `expired` are told apart by the registrar's own `available(id)`
+rule, `expires + GRACE_PERIOD < now`. `GRACE_PERIOD` is read from the contract
+rather than assumed, and `now` is the latest block's timestamp rather than the
+host clock, which the registrar compares against too, so a machine with a wrong
+clock cannot misreport a registration. That rule alone is not enough: it also
+holds for a name nobody ever registered (`0 + GRACE_PERIOD < now`), so a zero
+expiry is what separates *never registered* from *registered and since
+released*.
+
+A subname reports the status of the 2LD above it, which is only as good as the
+name it sits under. A subname nobody created answers 404 `unregistered`.
+
+#### v1 errors
+
+Every non-2xx body carries two fields: `error` is a fixed code to branch on,
+and `message` is a sentence for a human. Match on `error`, never on `message`,
+which is free to change.
+
+```jsonc
+{"name": "nope.testing", "error": "unregistered",
+ "message": "this name has never been registered",
+ "status": "unregistered", "expires": null, "graceEnds": null}
+```
+
+The codes are `tldNotConfigured`, `notFullyQualified`, `unregistered`,
+`expired`, `noSuchRoute` and `upstreamError`. When the registration is what went
+wrong, `error` and `status` hold the same value, so one field is enough to read.
+
+`upstreamError` says only which exception type the RPC call raised. The text
+goes to the resolver's log instead, because `SNRC_RPC` can carry a provider key
+and urlopen puts the URL it failed on into the message. It is also the answer
+when a registrar, controller or oracle address has no contract behind it: the
+empty reply is refused rather than read as zero, which would make every name
+look free.
+
+#### v1 status codes
 
 | Status | Meaning |
 |---|---|
-| 200 | resolved |
+| 200 | resolved (`status` is `registered` or `grace`, or `unknown` when no registrar is configured) |
 | 400 | TLD not configured, or not a fully-qualified name |
-| 404 | name has no resolver set on the registry |
+| 404 | `unregistered` |
+| 410 | `expired`: lapsed and past grace, so anyone may take it |
 | 502 | upstream RPC error / reth not synced |
 
-### Configuring registries
+### Querying by labelhash
 
-Defaults to mainnet `.testing` (`0x03f438…`); `.simplex` is unset until
-deployed. Override per TLD via env on the `resolver` service in
-`docker-compose.yml` (`SNRC_REGISTRY_TESTING` / `SNRC_REGISTRY_SIMPLEX`), or as
-env vars for the standalone script.
+A client asking whether a name is free is usually about to register it, and
+whoever runs the resolver could register it first. To avoid that, send the
+keccak hash of the label in ENS's `[<64 hex>]` form instead of the label:
+
+```sh
+# instead of /resolve/acme.testing
+curl -s "http://127.0.0.1:8000/resolve/[$(printf acme | keccak-256sum | cut -d' ' -f1)].testing"
+```
+
+namehash is `keccak(parent || keccak(label))`, so this reaches the same node and
+returns the same record. The registrar keys `nameExpires` and `reservedNames` on
+the labelhash too, so the status fields do not need the label either. The
+resolver learns the name only by guessing the label and hashing it.
+
+Only the second-level label is a registry key, and `status` decodes a bracket
+there at any depth. The record does not: a bracket is decoded only in a
+two-label name, so `sub.[<hash>].testing` is not a supported query. Subname
+labels stay text; a bracket label left of the 2LD is an ordinary label. Routers
+from v22 send every 2LD this way, so a registrable name normally never reaches
+this service.
+
+On v2, read `type`: only `available` means the name is free. On v1, read
+`status`: a name is free on `unregistered` (404) and on `expired` (410), every
+other status means somebody holds it, and a `reasonCode` means the registry will
+refuse it whatever the status says.
+
+The hash must be keccak-256. `openssl dgst -sha3-256` and `sha3sum` compute
+SHA3-256, a different function that returns 64 valid-looking hex characters
+pointing at the wrong node.
+
+The resolver lowercases the query before matching, so uppercase hex works too.
+Clients that refuse raw brackets in a path can percent-encode them as `%5B` and
+`%5D`.
+
+Brackets cannot collide with a real name: they are invalid in a normalised ENS
+name, and a `[<64 hex>]` label is 66 bytes against the registrar's
+`maxLabelLength` of 63. A plain `0x…` label is not treated as a hash, since that
+is an ordinary, registrable name.
+
+Only 2LDs can be queried this way, as only a 2LD can be raced for: subnames are
+created by the 2LD's owner. A bracket label in a subname is hashed as written,
+so it points at a node nobody can own. ENS tooling accepts the bracketed form at
+any depth; this resolver does not, on purpose.
+
+This hides interest in a name and nothing else: the registration itself is
+public, and commit-reveal covers that step. A short or well-known label is easy
+to guess by hashing candidates, and the reveal publishes the labelhash, so an
+operator who logged the query can match it to the name afterwards.
+
+### What a name costs
+
+The controller's `prices()` names the price oracle, so no extra configuration is
+needed beyond `SNRC_CONTROLLER_<TLD>`. A `SimplexPriceOracle` exposes its curve
+through `prices()`, in US cents per year, which is the unit this API carries.
+
+An ENS-shaped oracle exposes only `price1Letter()`..`price6Letter()`, in attoUSD
+per second, and charges a premium on a lapsed name that it does not expose. A
+quote from one is therefore only safe for a name that was never registered: an
+`expired` name gets no price rather than one below what the registrar charges.
+
+**Set `SNRC_CONTROLLER_<TLD>` wherever `SNRC_REGISTRAR_<TLD>` is.** Without a
+controller there is no oracle, so no name can be priced.
+
+**Upgrade this service before the routers that query it.** Routers from SMP v22
+call `/v2/resolve`, which an older resolver does not serve. Every name then
+answers `ERR NAME RESOLVER "HTTP 404"` until this service is upgraded, while
+`/health` still reports it as ready.
+
+### Why a name is reserved
+
+v2 carries the controller's reason as `reservedReason`. v1 carries the same word
+as `reasonCode`, plus `reason`, an English sentence for a human reading this API.
+Clients should branch on the word and phrase it themselves, in the user's
+language.
+
+| `reasonCode` | Meaning |
+|---|---|
+| `internal` | reserved for SimpleX |
+| `trademark` | reserved to protect a trademark |
+| `community` | reserved for the community |
+| `unknown` | a reason added to the contract after this resolver; still reserved |
+
+These are `SimplexController.Reason`, where 0 means not reserved. A controller
+from before the enum stores a boolean, whose `true` decodes as 1, which is why
+1 reads as `internal`, so nothing needs migrating.
+
+### Configuring addresses
+
+The resolver reads three contracts, each configured per TLD.
+
+The **registry** answers who owns a node, and `/resolve` reads the records from
+it. The **registrar** (ERC-721) holds `nameExpires` and `GRACE_PERIOD`, which
+is where every expiry field comes from, and `labelOf`, which is how a hashed
+query is answered with a name. A name registered without recording its label
+cannot answer one, and `/v2/resolve` refuses it rather than answer with a name
+the client will reject. With no registrar for a TLD, `/resolve` still works and
+reports `"status": "unknown"`. The **controller** holds
+`reservedNames`, which is where `reasonCode` comes from. With no controller a
+held-back name reads as not reserved, and no name can be priced.
+
+All three default to the mainnet `.testing` deployment. `.simplex` is unset
+until it is deployed.
+
+The controller default is the **proxy**, not `SimplexControllerImpl`. Storage
+lives in the proxy, so the implementation address answers nothing. The two
+deployment files use different names for that proxy:
+`deployments.mainnet.testing.json` records it under the ENS role name
+`ETHRegistrarController`, and `verification.mainnet.testing.json` calls it
+`SimplexControllerProxy`. Both are the same address, and it is the one used
+here.
+
+To override any of them, set `SNRC_REGISTRY_<TLD>`, `SNRC_REGISTRAR_<TLD>` or
+`SNRC_CONTROLLER_<TLD>` on the `resolver` service in `docker-compose.yml`, or
+as env vars when you run the script directly.

scripts/resolver/docker-compose.yml

@@ -150,6 +150,12 @@ services:
       # only if you're deploying against a different network or contract.
       # SNRC_REGISTRY_TESTING: 0x...
       # SNRC_REGISTRY_SIMPLEX: 0x...
+      # Registrar and controller, same cascade. Without the registrar `status`
+      # is "unknown"; without the controller a reserved name is "unregistered".
+      # SNRC_REGISTRAR_TESTING: 0x...
+      # SNRC_REGISTRAR_SIMPLEX: 0x...
+      # SNRC_CONTROLLER_TESTING: 0x...
+      # SNRC_CONTROLLER_SIMPLEX: 0x...
     ports:
       - "127.0.0.1:8000:8000"
     restart: unless-stopped

scripts/resolver/service/snrc-resolve.py

@@ -29,6 +29,7 @@ Usage:
   ./snrc-resolve.py                    # serve on :8000
 
   curl -s http://127.0.0.1:8000/resolve/foobar.testing | jq .
+  curl -s 'http://127.0.0.1:8000/resolve/[<64-hex labelhash>].testing' | jq .
   curl -s http://127.0.0.1:8000/health
 
 Environment:
@@ -38,6 +39,11 @@ Environment:
                           0x58fc46996d975c57883564648bda5206d1a0102b)
   SNRC_REGISTRY_SIMPLEX  ENSRegistry for the .simplex deployment
                          (default: empty — TLD not yet deployed)
+  SNRC_REGISTRAR_<TLD>   BaseRegistrar (ERC-721) for the TLD; expiry and status
+                         (default: mainnet for .testing, empty for .simplex)
+  SNRC_CONTROLLER_<TLD>  SimplexController (proxy) for the TLD; reservations,
+                         and through its `prices()` oracle what registering costs
+                         (default: mainnet for .testing, empty for .simplex)
   SNRC_PORT              Listen port (default: 8000)
   SNRC_BIND              Bind address (default: 0.0.0.0)
 
@@ -61,6 +67,7 @@ import hashlib
 import json
 import os
 import sys
+import time
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 from urllib.parse import unquote, urlparse
 from urllib.request import Request, urlopen
@@ -84,6 +91,31 @@ REGISTRIES = {
     "simplex": os.environ.get("SNRC_REGISTRY_SIMPLEX", ""),  # not deployed yet
 }
 
+REGISTRARS = {
+    "testing": os.environ.get("SNRC_REGISTRAR_TESTING", "")
+    or "0xef47eb4384b46c89e4482a677c2cbcbd2a6fd85a",  # mainnet .testing
+    "simplex": os.environ.get("SNRC_REGISTRAR_SIMPLEX", ""),  # not deployed yet
+}
+
+CONTROLLERS = {
+    "testing": os.environ.get("SNRC_CONTROLLER_TESTING", "")
+    # Proxy address, not SimplexControllerImpl: storage is held by the proxy.
+    # Recorded in deployments.json as ETHRegistrarController.
+    or "0xeeb9b6bf5fb68fb726005f7ba549c2f4b32f2dad",  # mainnet .testing
+    "simplex": os.environ.get("SNRC_CONTROLLER_SIMPLEX", ""),  # not deployed yet
+}
+
+# `reservedNames` holds a SimplexController.Reason; 0 means not reserved. A
+# controller from before the enum stores a bool, whose `true` decodes as 1,
+# which is why 1 reads as "internal".
+RESERVED_REASONS = {
+    1: ("internal", "reserved for SimpleX"),
+    2: ("trademark", "reserved to protect a trademark"),
+    3: ("community", "reserved for the community"),
+}
+# a Reason added to the contract after this resolver: still reserved, unworded
+UNKNOWN_REASON = ("unknown", "reserved")
+
 # SLIP-44 coin types (https://github.com/satoshilabs/slips/blob/master/slip-0044.md)
 COIN_ETH = 60
 COIN_BTC = 0
@@ -92,6 +124,8 @@ COIN_DOT = 354
 
 ZERO_ADDR = "0x0000000000000000000000000000000000000000"
 
+# The registry prices in attoUSD (1e-18 USD); the protocol carries US cents.
+
 
 # ---------- RPC + ABI helpers (mirrors ens-lookup.py shape) ----------
 
@@ -123,12 +157,216 @@ def namehash(name: str) -> bytes:
     return node
 
 
+# ENS's encoding for a label whose preimage is unknown. Brackets are outside
+# the normalised character set, so it cannot collide with a registrable name.
+ENCODED_LABELHASH_LEN = 66  # "[" + 64 hex + "]"
+
+
+def is_encoded_labelhash(label: str) -> bool:
+    return (
+        len(label) == ENCODED_LABELHASH_LEN
+        and label.startswith("[")
+        and label.endswith("]")
+        and all(c in "0123456789abcdef" for c in label[1:-1])
+    )
+
+
+def node_of(name: str) -> bytes:
+    """namehash, decoding a second-level labelhash so `[hash].tld` reaches the
+    node its name does. Only a second-level name is ever hashed; a bracket
+    anywhere else is hashed as written."""
+    labels = name.split(".")
+    if len(labels) != 2 or not is_encoded_labelhash(labels[0]):
+        return namehash(name)
+    return keccak(namehash(labels[1]) + bytes.fromhex(labels[0][1:-1]))
+
+
+# ---------- Registration status ----------
+
+
+def head_block():
+    """How far behind the node is. Unlike expiry, this is the one thing that has
+    to be measured against the host clock: a node that stops still has a block."""
+    try:
+        block = rpc("eth_getBlockByNumber", ["latest", False])
+        return {
+            "blockNumber": decode_uint(block["number"]),
+            "chainLagSeconds": int(time.time()) - decode_uint(block["timestamp"]),
+        }
+    except Exception:
+        return {"blockNumber": None, "chainLagSeconds": None}
+
+
+def chain_now() -> int:
+    """Expiry is compared against the block timestamp, never the host clock."""
+    block = rpc("eth_getBlockByNumber", ["latest", False])
+    return decode_uint(block["timestamp"])
+
+
+def grace_period(registrar: str) -> int:
+    """A deployment can configure a different window, so it is read on chain."""
+    return decode_uint(eth_call(registrar, selector("GRACE_PERIOD()")))
+
+
+def expiry_status(expires: int, grace: int, now: int) -> str:
+    """The registrar's `available(id)` is not enough on its own: it is also
+    true for a name nobody registered, since 0 + GRACE_PERIOD < now."""
+    if expires == 0:
+        return "unregistered"
+    if expires > now:
+        return "registered"
+    if expires + grace >= now:
+        return "grace"
+    return "expired"
+
+
+def reservation_reason(tld: str, token: int) -> int:
+    """The SimplexController.Reason held for the name, 0 when not reserved."""
+    controller = CONTROLLERS.get(tld)
+    if not controller:
+        return 0
+    raw = eth_call(controller, selector("reservedNames(bytes32)") + encode_uint(token))
+    return decode_uint(raw)
+
+
+def pricing_params(tld: str):
+    """What it costs to register a name under this TLD, in US cents, or None
+    when no controller or price oracle is configured."""
+    controller = CONTROLLERS.get(tld)
+    if not controller:
+        return None
+    oracle = decode_address(eth_call(controller, selector("prices()")))
+    if oracle == ZERO_ADDR:
+        return None
+    try:
+        return read_oracle_prices(controller, oracle)
+    except RuntimeError:
+        # An oracle that does not expose its curve cannot be quoted from. The
+        # name is still registrable; the price is simply not ours to state.
+        return None
+
+
+SECONDS_PER_YEAR = 31536000
+ATTO_PER_CENT = 10**16
+
+
+def read_oracle_prices(controller: str, oracle: str):
+    """SimplexPriceOracle keeps the curve in US cents per year, the unit the SMP
+    protocol carries. An ENS-shaped oracle prices in attoUSD per second and
+    charges a premium on lapsed names that it does not expose, so a quote from
+    it is only safe for a name that was never registered."""
+    try:
+        base, tiers = decode_prices(eth_call(oracle, selector("prices()")))
+        premium_unknown = False
+    except RuntimeError:
+        base, tiers = decode_letter_prices(oracle)
+        premium_unknown = True
+    min_len = decode_uint(eth_call(controller, selector("minCharLength()")))
+    return {
+        # lengths the registry refuses are left out rather than priced at zero
+        "registrationPrices": {n: c for n, c in tiers.items() if n >= min_len},
+        "basePrice": base,
+        "minLabelLength": min_len,
+        "_premiumUnknown": premium_unknown,
+    }
+
+
+def decode_letter_prices(oracle: str):
+    """`price1Letter()`..`price6Letter()`, in attoUSD per second. Quotes round
+    up, so one is never below what the registry charges. An oracle built before
+    the six-letter tier stops at five, and charges its highest tier for anything
+    longer, which is what basePrice means here."""
+    tiers = {}
+    for n in range(1, 7):
+        try:
+            rate = decode_uint(eth_call(oracle, selector(f"price{n}Letter()")))
+        except RuntimeError:
+            if n <= 5:
+                raise
+            break
+        tiers[n] = ceil_div(rate * SECONDS_PER_YEAR, ATTO_PER_CENT)
+    return tiers.pop(max(tiers)), tiers
+
+
+def ceil_div(a: int, b: int) -> int:
+    return -(-a // b)
+
+
+def decode_prices(hex_data: str):
+    """`prices()` returns the base price and the lengths priced differently."""
+    raw = bytes.fromhex(hex_data[2:] if hex_data.startswith("0x") else hex_data)
+    # a short answer is not a curve: decoding it would quote every name as free
+    if len(raw) < 96:
+        raise RuntimeError("prices(): short response")
+    base = int.from_bytes(raw[:32], "big")
+    at = int.from_bytes(raw[32:64], "big")
+    count = int.from_bytes(raw[at:at + 32], "big")
+    tiers = {}
+    for i in range(count):
+        item = at + 32 + i * 64
+        length = int.from_bytes(raw[item:item + 32], "big")
+        tiers[length] = int.from_bytes(raw[item + 32:item + 64], "big")
+    return base, tiers
+
+
+def name_status(name: str):
+    labels = name.split(".")
+    tld = labels[-1]
+    registrar = REGISTRARS.get(tld)
+    if not registrar or len(labels) < 2:
+        return {
+            "status": "unknown",
+            # nothing was read, so there is no block to report
+            "lastBlockTs": None,
+            "expires": None,
+            "graceEnds": None,
+            "reasonCode": None,
+            "reason": None,
+        }
+
+    # nameExpires and reservedNames are keyed on uint256(keccak(label)).
+    # The 2LD's label is that key at any depth. node_of decodes a bracket only
+    # in a two-label name, so a bracket subname gets a status but no record.
+    token = label_token(labels[-2])
+    expires = decode_uint(
+        eth_call(registrar, selector("nameExpires(uint256)") + encode_uint(token))
+    )
+    grace = grace_period(registrar) if expires else 0
+    now = chain_now()
+    status = expiry_status(expires, grace, now)
+
+    # A reservation is orthogonal to the registration: a registered name can be
+    # held back too.
+    code = reservation_reason(tld, token)
+    reason = RESERVED_REASONS.get(code, UNKNOWN_REASON) if code else None
+
+    out = {
+        "status": status,
+        # the block this was read at
+        "lastBlockTs": now,
+        "expires": expires or None,
+        "graceEnds": (expires + grace) if expires else None,
+        "reasonCode": reason[0] if reason else None,
+        "reason": reason[1] if reason else None,
+    }
+    if status in ("unregistered", "expired"):
+        pricing = pricing_params(tld)
+        # a lapsed name may carry a premium this resolver cannot read, and a
+        # quote without it would be below what the registry charges
+        if pricing and not (status == "expired" and pricing["_premiumUnknown"]):
+            out.update({k: v for k, v in pricing.items() if not k.startswith("_")})
+    return out
+
+
 def selector(signature: str) -> str:
     return "0x" + keccak(signature.encode())[:4].hex()
 
 
 def eth_call(to: str, data: str) -> str:
-    return rpc("eth_call", [{"to": to, "data": data}, "latest"])
+    result = rpc("eth_call", [{"to": to, "data": data}, "latest"])
+    if result == "0x":
+        raise RuntimeError(f"empty return from {to}: no contract at that address?")
+    return result
 
 
 def decode_address(hex_data: str) -> str:
@@ -143,6 +381,42 @@ def decode_bytes(hex_data: str) -> bytes:
     return raw[64:64 + length]
 
 
+def registered_label(registrar: str, token: int):
+    """The plaintext label the registrar recorded at registration, keyed by the
+    hash of that label. None when the name was registered without
+    registerWithLabel, so the registrar cannot name it."""
+    raw = decode_bytes(eth_call(registrar, selector("labelOf(uint256)") + encode_uint(token)))
+    return raw.decode("utf-8", errors="replace") if raw else None
+
+
+def canonical_name(name: str):
+    """The name to answer with: a hashed query does not carry one, so the
+    registrar's record of the label fills it in. None when it recorded none."""
+    labels = name.split(".")
+    registrar = REGISTRARS.get(labels[-1])
+    if not registrar or len(labels) != 2 or not is_encoded_labelhash(labels[0]):
+        return name
+    label = registered_label(registrar, label_token(labels[0]))
+    return label + "." + labels[1] if label else None
+
+
+def label_token(label: str) -> int:
+    """The registry key for a second-level label, whether it arrived as text or
+    already hashed."""
+    if is_encoded_labelhash(label):
+        return int(label[1:-1], 16)
+    return int.from_bytes(keccak(label.encode()), "big")
+
+
+def decode_uint(hex_data: str) -> int:
+    raw = hex_data[2:] if hex_data.startswith("0x") else hex_data
+    return int(raw[-64:], 16) if raw else 0
+
+
+def encode_uint(value: int) -> str:
+    return value.to_bytes(32, "big").hex()
+
+
 def encode_text_call(node: bytes, key: str) -> str:
     sel = selector("text(bytes32,string)")
     head = node.hex() + (0x40).to_bytes(32, "big").hex()
@@ -390,6 +664,117 @@ def split_links(value: str) -> list:
     return [item.strip() for item in value.split(LINK_SEPARATOR) if item.strip()]
 
 
+def upstream_error(subject: dict, e: Exception) -> dict:
+    """urlopen puts the failing URL into its message and SNRC_RPC can carry a
+    provider key, so the text goes to the log and only the type to the caller."""
+    print(f"upstream error: {type(e).__name__}: {e}", file=sys.stderr)
+    return {
+        **subject,
+        "error": "upstreamError",
+        "message": f"upstream RPC failed ({type(e).__name__})",
+    }
+
+
+def name_record(name: str):
+    """The NameRecord for a registered name. A name with no resolver set still
+    has one, with every field unset."""
+    registry = REGISTRIES[name.rsplit(".", 1)[-1]]
+    node = node_of(name)
+    node_hex = node.hex()
+    resolver_addr = decode_address(eth_call(registry, selector("resolver(bytes32)") + node_hex))
+    owner = decode_address(eth_call(registry, selector("owner(bytes32)") + node_hex))
+    rec = {
+        "name": canonical_name(name),
+        "nickname": "",
+        "website": "",
+        "location": "",
+        "simplexContact": [],
+        "simplexChannel": [],
+        "eth": None,
+        "btc": None,
+        "xmr": None,
+        "dot": None,
+        "owner": owner,
+        "resolver": resolver_addr,
+    }
+    if resolver_addr == ZERO_ADDR:
+        return rec
+    texts = {}
+    for k in TEXT_KEYS:
+        try:
+            v = text(resolver_addr, node, k)
+        except RuntimeError:
+            v = ""
+        if v:
+            texts[k] = v
+    rec.update(
+        {
+            "nickname": texts.get("nickname") or texts.get("name") or texts.get("description") or "",
+            "website": texts.get("url", ""),
+            "location": texts.get("location", ""),
+            "simplexContact": split_links(texts.get("simplex.contact", "")),
+            "simplexChannel": split_links(texts.get("simplex.channel", "")),
+            "eth": addr_multicoin(resolver_addr, node, COIN_ETH),
+            "btc": addr_multicoin(resolver_addr, node, COIN_BTC),
+            "xmr": addr_multicoin(resolver_addr, node, COIN_XMR),
+            "dot": addr_multicoin(resolver_addr, node, COIN_DOT),
+        }
+    )
+    return rec
+
+
+def name_response(reg, registration_body):
+    """The SMP protocol's NameResponse: the registration and the block read at."""
+    return 200, {"lastBlockTs": reg["lastBlockTs"], "registration": registration_body}
+
+
+def registration(name: str):
+    """The SMP protocol's NameRegistration, which the relay decodes as is.
+    Translating the contract's model to it is this resolver's job."""
+    tld = name.rsplit(".", 1)[-1]
+    if not REGISTRIES.get(tld):
+        return 400, {"name": name, "error": "tldNotConfigured"}
+    reg = name_status(name)
+    status = reg["status"]
+    if status in ("registered", "grace"):
+        rec = name_record(name)
+        # the client checks that the record names what it asked about, so a
+        # hashed query the registrar cannot name is refused rather than answered
+        if rec["name"] is None:
+            return 502, {"name": name, "error": "labelNotRecorded"}
+        # a subname inherits the 2LD's status, so only its node's owner says
+        # whether anyone created it
+        if len(name.split(".")) > 2 and rec["owner"] == ZERO_ADDR:
+            # name_status reads pricing only when the name was already
+            # unregistered, so read it here
+            status = "unregistered"
+            pricing = pricing_params(tld)
+            if pricing:
+                reg.update({k: v for k, v in pricing.items() if not k.startswith("_")})
+        else:
+            return name_response(reg, {
+                "type": "registered",
+                "expires": reg["expires"],
+                "graceUntil": reg["graceEnds"],
+                "reservedReason_": reg["reasonCode"],
+                "nameRecord": rec,
+            })
+    if reg["reasonCode"]:
+        return name_response(reg, {"type": "reserved", "reservedReason": reg["reasonCode"]})
+    if status in ("unregistered", "expired"):
+        if "basePrice" not in reg:
+            return 502, {"name": name, "error": "noPriceOracle"}
+        return name_response(reg, {
+            "type": "available",
+            "pricing": {
+                "registrationPrices": reg["registrationPrices"],
+                "basePrice": reg["basePrice"],
+                "minLabelLength": reg["minLabelLength"],
+            },
+        })
+    return 502, {"name": name, "error": status}
+
+
 def resolve(name: str):
     tld = name.rsplit(".", 1)[-1]
     registry = REGISTRIES.get(tld)
@@ -397,17 +782,60 @@ def resolve(name: str):
         configured = [k for k, v in REGISTRIES.items() if v]
         return 400, {
             "name": name,
-            "error": f"TLD '{tld}' is not configured on this resolver",
-            "configured_tlds": configured,
+            "error": "tldNotConfigured",
+            "message": f"TLD '{tld}' is not configured on this resolver",
+            "configuredTlds": configured,
         }
 
-    node = namehash(name)
+    node = node_of(name)
     node_hex = node.hex()
 
+    # Before the resolver lookup, so a lapsed name is not reported as noResolver.
+    reg = name_status(name)
+    if reg["status"] in ("unregistered", "expired"):
+        # A name in grace is not here: its record still resolves.
+        body = {
+            "name": name,
+            **reg,
+            "error": reg["status"],
+            "message": (
+                "this name has never been registered"
+                if reg["status"] == "unregistered"
+                else "this registration expired and is open to anyone"
+            ),
+        }
+        return (404 if reg["status"] == "unregistered" else 410), body
+
     resolver_raw = eth_call(registry, selector("resolver(bytes32)") + node_hex)
     resolver_addr = decode_address(resolver_raw)
     if resolver_addr == ZERO_ADDR:
-        return 404, {"name": name, "error": "no resolver set for this name"}
+        # A registered name always resolves: with no resolver set the record is
+        # still returned with every field unset, so "taken until <date>" stays
+        # answerable. For a subname, no owner means nobody created it.
+        owner = decode_address(eth_call(registry, selector("owner(bytes32)") + node_hex))
+        if len(name.split(".")) > 2 and owner == ZERO_ADDR:
+            return 404, {
+                "name": name,
+                **reg,
+                "status": "unregistered",
+                "error": "unregistered",
+                "message": "this subname has never been created",
+            }
+        return 200, {
+            "name": canonical_name(name) or name,
+            "nickname": "",
+            "website": "",
+            "location": "",
+            "simplexContact": [],
+            "simplexChannel": [],
+            "eth": None,
+            "btc": None,
+            "xmr": None,
+            "dot": None,
+            "owner": owner,
+            "resolver": ZERO_ADDR,
+            **reg,
+        }
 
     owner_raw = eth_call(registry, selector("owner(bytes32)") + node_hex)
     owner = decode_address(owner_raw)
@@ -431,7 +859,7 @@ def resolve(name: str):
     # use the ENSIP-5 dot convention (e.g. "simplex.contact") — only the
     # resolver's JSON surface camelCases them.
     return 200, {
-        "name": name,
+        "name": canonical_name(name) or name,
         "nickname": nickname,
         "website": texts.get("url", ""),
         "location": texts.get("location", ""),
@@ -443,6 +871,7 @@ def resolve(name: str):
         "dot": addr_multicoin(resolver_addr, node, COIN_DOT),
         "owner": owner,
         "resolver": resolver_addr,
+        **reg,
     }
 
 
@@ -454,33 +883,51 @@ class Handler(BaseHTTPRequestHandler):
         parts = [unquote(p) for p in path.split("/") if p]
 
         if parts == ["health"]:
-            self._respond(
-                200,
-                {"ok": True, "rpc": RPC, "registries": REGISTRIES},
-            )
+            self._respond(200, {"ok": True, "rpc": RPC, "registries": REGISTRIES, **head_block()})
             return
 
+        if len(parts) == 3 and parts[0] == "v2" and parts[1] == "resolve":
+            name = parts[2].strip().lower()
+            if not name or "." not in name:
+                self._respond(400, {"name": name, "error": "notFullyQualified"})
+                return
+            try:
+                status, body = registration(name)
+            except Exception as e:  # surface upstream errors as 502
+                status, body = 502, upstream_error({"name": name}, e)
+            self._respond(status, body)
+            return
+
+        # /v1/resolve is an alias: relays before SMP v22 call /resolve
+        if parts[:2] == ["v1", "resolve"] and len(parts) == 3:
+            parts = ["resolve", parts[2]]
+
         if len(parts) == 2 and parts[0] == "resolve":
             name = parts[1].strip().lower()
             if not name or "." not in name:
                 self._respond(
                     400,
                     {
-                        "error": "expected fully-qualified name, e.g. /resolve/alice.testing",
-                        "got": name,
+                        "name": name,
+                        "error": "notFullyQualified",
+                        "message": "expected a fully-qualified name, e.g. alice.testing",
                     },
                 )
                 return
             try:
                 status, body = resolve(name)
             except Exception as e:  # surface upstream errors as 502
-                status, body = 502, {"name": name, "error": f"{type(e).__name__}: {e}"}
+                status, body = 502, upstream_error({"name": name}, e)
             self._respond(status, body)
             return
 
         self._respond(
             404,
-            {"error": "not found", "routes": ["/health", "/resolve/<name>"]},
+            {
+                "error": "noSuchRoute",
+                "message": "not found",
+                "routes": ["/health", "/v2/resolve/<query>", "/v1/resolve/<name>", "/resolve/<name>"],
+            },
         )
 
     def _respond(self, status: int, body: dict):
@@ -505,7 +952,7 @@ def main():
     )
     for tld, addr in REGISTRIES.items():
         sys.stderr.write(f"    .{tld:<8s} = {addr or '(not configured)'}\n")
-    sys.stderr.write("  GET /resolve/<name>   GET /health\n")
+    sys.stderr.write("  GET /v2/resolve/<name>   GET /v1/resolve/<name>   GET /health\n")
     try:
         server.serve_forever()
     except KeyboardInterrupt:

scripts/resolver/service/test_snrc_resolve.py

@@ -4,8 +4,11 @@
 Run with `python3 -m unittest scripts/resolver/service/test_snrc_resolve.py`.
 """
 
+import contextlib
 import importlib.util
+import io
 import os
+import time
 import unittest
 
 # snrc-resolve.py has a hyphen, so import it via importlib instead of `import`.
@@ -17,6 +20,12 @@ snrc = importlib.util.module_from_spec(_SPEC)
 _SPEC.loader.exec_module(snrc)
 
 
+def registration(name):
+    """registration() answers a NameResponse; most tests assert what is in it."""
+    status, body = snrc.registration(name)
+    return status, (body["registration"] if status == 200 else body)
+
+
 class SplitLinksTests(unittest.TestCase):
     """`split_links` decodes the multi-URL convention for simplex.contact /
     simplex.channel text records. Reuses the same rule the dApp's
@@ -82,5 +91,949 @@ class SplitLinksTests(unittest.TestCase):
         )
 
 
+class EncodedLabelhashTests(unittest.TestCase):
+    # keccak-256("alice"), written out in full wherever a test needs it.
+    # 9c0257114eb9399a2985f8e75dad7600c5d89fe3824ffa99ec1c3eb8bf3b0501
+
+    REGISTRAR = "0xef47eb4384b46c89e4482a677c2cbcbd2a6fd85a"
+    GRACE = 90 * 86400
+
+    def setUp(self):
+        self._saved = (snrc.REGISTRARS, snrc.CONTROLLERS, snrc.eth_call, snrc.chain_now)
+        snrc.REGISTRARS = {"testing": self.REGISTRAR}
+        snrc.CONTROLLERS = {"testing": ""}
+        snrc.chain_now = lambda: int(time.time())
+
+    def tearDown(self):
+        snrc.REGISTRARS, snrc.CONTROLLERS, snrc.eth_call, snrc.chain_now = self._saved
+
+    def test_the_encoded_form_is_recognised(self):
+        self.assertTrue(
+            snrc.is_encoded_labelhash(
+                "[9c0257114eb9399a2985f8e75dad7600c5d89fe3824ffa99ec1c3eb8bf3b0501]"
+            )
+        )
+
+    def test_an_ordinary_label_is_not(self):
+        self.assertFalse(snrc.is_encoded_labelhash("alice"))
+        self.assertFalse(snrc.is_encoded_labelhash("[alice]"))
+        self.assertFalse(snrc.is_encoded_labelhash("9c0257114eb9399a2985f8e75dad7600c5d89fe3824ffa99ec1c3eb8bf3b0501"))
+
+    def test_non_hex_between_the_brackets_is_not(self):
+        self.assertFalse(snrc.is_encoded_labelhash("[" + "z" * 64 + "]"))
+        # uppercase is rejected because the handler lowercases the whole name
+        self.assertFalse(snrc.is_encoded_labelhash("[" + "A" * 64 + "]"))
+        self.assertFalse(snrc.is_encoded_labelhash("[0x9c0257114eb9399a2985f8e75dad7600c5d89fe3824ffa99ec1c3eb8bf3b0501]"))
+
+    def test_the_wrong_length_is_not(self):
+        self.assertFalse(snrc.is_encoded_labelhash("[" + "a" * 63 + "]"))
+        self.assertFalse(snrc.is_encoded_labelhash("[" + "a" * 65 + "]"))
+
+    def test_hash_and_label_reach_the_same_node(self):
+        self.assertEqual(
+            snrc.node_of("alice.testing"),
+            snrc.node_of(
+                "[9c0257114eb9399a2985f8e75dad7600c5d89fe3824ffa99ec1c3eb8bf3b0501]"
+                ".testing"
+            ),
+        )
+
+    def test_a_plain_name_is_unaffected(self):
+        self.assertEqual(snrc.node_of("alice.testing"), snrc.namehash("alice.testing"))
+
+    def test_a_bracket_subname_label_stays_literal(self):
+        """Only the 2LD is a key, so a bracket label left of it is hashed as
+        written."""
+        self.assertNotEqual(
+            snrc.node_of(
+                "[9c0257114eb9399a2985f8e75dad7600c5d89fe3824ffa99ec1c3eb8bf3b0501]"
+                ".alice.testing"
+            ),
+            snrc.namehash("alice.alice.testing"),
+        )
+
+    def test_a_0x_prefixed_label_is_taken_literally(self):
+        name = "0x9c0257114eb9399a2985f8e75dad7600c5d89fe3824ffa99ec1c3eb8bf3b0501.testing"
+        self.assertEqual(snrc.node_of(name), snrc.namehash(name))
+        self.assertNotEqual(snrc.node_of(name), snrc.node_of("alice.testing"))
+
+    def test_a_malformed_bracket_label_falls_back_to_a_literal_name(self):
+        name = "[nothex].testing"
+        self.assertEqual(snrc.node_of(name), snrc.namehash(name))
+
+    def test_status_by_hash_matches_status_by_name(self):
+        future = int(time.time()) + 86400
+        seen = []
+
+        def eth_call(to, data):
+            seen.append(data)
+            if data.startswith(snrc.selector("GRACE_PERIOD()")):
+                return "0x" + snrc.encode_uint(self.GRACE)
+            return "0x" + snrc.encode_uint(future)
+
+        snrc.eth_call = eth_call
+        by_name = snrc.name_status("alice.testing")
+        by_hash = snrc.name_status(
+            "[9c0257114eb9399a2985f8e75dad7600c5d89fe3824ffa99ec1c3eb8bf3b0501]"
+            ".testing"
+        )
+        self.assertEqual(by_name, by_hash)
+        self.assertEqual(by_name["status"], "registered")
+        # nothing in either request carried the label itself
+        self.assertTrue(all("alice".encode().hex() not in d for d in seen))
+
+
+class NameStatusTests(unittest.TestCase):
+    REGISTRAR = "0xef47eb4384b46c89e4482a677c2cbcbd2a6fd85a"
+
+    GRACE = 90 * 86400
+
+    def _expiry(self, value):
+        def eth_call(to, data):
+            if data.startswith(snrc.selector("GRACE_PERIOD()")):
+                return "0x" + snrc.encode_uint(self.GRACE)
+            self.assertTrue(data.startswith(snrc.selector("nameExpires(uint256)")))
+            return "0x" + snrc.encode_uint(value)
+
+        return eth_call
+
+    def _keys(self, status, expires, grace_ends, read_at=-1):
+        """Every branch answers with the same keys; only some carry values."""
+        return {
+            "status": status,
+            "lastBlockTs": self.now if read_at == -1 else read_at,
+            "expires": expires,
+            "graceEnds": grace_ends,
+            "reasonCode": None,
+            "reason": None,
+        }
+
+    def setUp(self):
+        self._saved = (
+            snrc.REGISTRARS,
+            snrc.CONTROLLERS,
+            snrc.eth_call,
+            snrc.chain_now,
+            snrc.rpc,
+        )
+        snrc.REGISTRARS = {"testing": self.REGISTRAR}
+        # Expiry alone; ReservedTests covers a configured controller.
+        snrc.CONTROLLERS = {"testing": ""}
+        self.now = int(time.time())
+        snrc.chain_now = lambda: self.now
+
+    def tearDown(self):
+        (
+            snrc.REGISTRARS,
+            snrc.CONTROLLERS,
+            snrc.eth_call,
+            snrc.chain_now,
+            snrc.rpc,
+        ) = self._saved
+
+    def test_now_is_the_latest_blocks_timestamp(self):
+        # setUp replaced chain_now with the fixture clock; test the real one
+        real_chain_now = self._saved[3]
+        snrc.rpc = lambda method, params: {"timestamp": "0x65f1a2c0", "number": "0x123"}
+        self.assertEqual(real_chain_now(), 0x65F1A2C0)
+
+    def test_status_reads_the_chain_clock_not_the_host_clock(self):
+        future = int(time.time()) + 3600
+        snrc.eth_call = self._expiry(future)
+        self.assertEqual(snrc.name_status("alice.testing")["status"], "registered")
+        snrc.chain_now = lambda: future + 3650 * 86400
+        self.assertEqual(snrc.name_status("alice.testing")["status"], "expired")
+
+    def test_a_registrar_that_is_not_a_contract_is_an_error_not_a_free_name(self):
+        """An address with no code answers eth_call with empty data. Read as
+        zero, that would make every name look free."""
+        snrc.eth_call = self._saved[2]  # the real one, so its guard runs
+        snrc.rpc = lambda method, params: "0x"
+        with self.assertRaises(RuntimeError):
+            snrc.name_status("alice.testing")
+
+    def test_zero_expiry_means_never_registered(self):
+        snrc.eth_call = self._expiry(0)
+        self.assertEqual(
+            snrc.name_status("alice.testing"),
+            self._keys("unregistered", None, None),
+        )
+
+    def test_recently_expired_is_in_grace_and_says_when_it_ends(self):
+        past = int(time.time()) - 3600
+        snrc.eth_call = self._expiry(past)
+        self.assertEqual(
+            snrc.name_status("alice.testing"),
+            self._keys("grace", past, past + self.GRACE),
+        )
+
+    def test_past_the_grace_window_it_is_expired_and_claimable(self):
+        past = int(time.time()) - self.GRACE - 3600
+        snrc.eth_call = self._expiry(past)
+        self.assertEqual(snrc.name_status("alice.testing")["status"], "expired")
+
+    def test_the_boundary_belongs_to_grace(self):
+        """The registrar frees a name only when expires + GRACE < now."""
+        now = int(time.time())
+        snrc.eth_call = self._expiry(now - self.GRACE)
+        self.assertEqual(snrc.name_status("alice.testing")["status"], "grace")
+
+    def test_future_expiry_is_registered(self):
+        future = int(time.time()) + 3600
+        snrc.eth_call = self._expiry(future)
+        self.assertEqual(
+            snrc.name_status("alice.testing"),
+            self._keys("registered", future, future + self.GRACE),
+        )
+
+    def test_never_registered_is_not_confused_with_claimable(self):
+        """`available(id)` is true for both, since 0 + GRACE < now."""
+        snrc.eth_call = self._expiry(0)
+        self.assertEqual(snrc.name_status("alice.testing")["status"], "unregistered")
+        self.assertNotEqual(snrc.name_status("alice.testing")["status"], "expired")
+
+    def test_a_subname_reports_the_status_of_its_2ld(self):
+        future = int(time.time()) + 3600
+        seen = []
+
+        def eth_call(to, data):
+            seen.append(data)
+            return "0x" + snrc.encode_uint(future)
+
+        snrc.eth_call = eth_call
+        self.assertEqual(snrc.name_status("x.alice.testing")["status"], "registered")
+        # the token asked about is keccak("alice"), not keccak("x")
+        self.assertTrue(seen[0].endswith(snrc.keccak(b"alice").hex()))
+
+    def test_a_hashed_2ld_is_queried_by_its_hash_at_any_depth(self):
+        """The token must come from the hash, not from hashing the brackets."""
+        seen = []
+
+        def eth_call(to, data):
+            seen.append(data)
+            return "0x" + snrc.encode_uint(0)
+
+        snrc.eth_call = eth_call
+        hashed = "[" + snrc.keccak(b"alice").hex() + "]"
+        snrc.name_status("x." + hashed + ".testing")
+        self.assertTrue(seen[0].endswith(snrc.keccak(b"alice").hex()))
+
+    def test_unconfigured_tld_is_unknown_rather_than_unregistered(self):
+        snrc.REGISTRARS = {"testing": ""}
+        snrc.eth_call = lambda *a: self.fail("must not reach the chain")
+        self.assertEqual(
+            snrc.name_status("alice.testing"),
+            self._keys("unknown", None, None, read_at=None),
+        )
+
+    def test_every_branch_returns_the_same_keys(self):
+        keys = {
+            "status",
+            "lastBlockTs",
+            "expires",
+            "graceEnds",
+            "reasonCode",
+            "reason",
+        }
+        snrc.eth_call = self._expiry(0)
+        self.assertEqual(set(snrc.name_status("alice.testing")), keys)
+        snrc.eth_call = self._expiry(int(time.time()) + 3600)
+        self.assertEqual(set(snrc.name_status("alice.testing")), keys)
+        snrc.REGISTRARS = {"testing": ""}
+        snrc.eth_call = lambda *a: self.fail("must not reach the chain")
+        self.assertEqual(set(snrc.name_status("alice.testing")), keys)
+
+
+class ReservedTests(unittest.TestCase):
+    REGISTRAR = "0xef47eb4384b46c89e4482a677c2cbcbd2a6fd85a"
+    CONTROLLER = "0x281ca41311c2aa808c917c4674639d7567b75714"
+
+    def setUp(self):
+        self._saved = (snrc.REGISTRARS, snrc.CONTROLLERS, snrc.eth_call, snrc.chain_now)
+        snrc.REGISTRARS = {"testing": self.REGISTRAR}
+        snrc.CONTROLLERS = {"testing": self.CONTROLLER}
+        snrc.chain_now = lambda: int(time.time())
+
+    def tearDown(self):
+        snrc.REGISTRARS, snrc.CONTROLLERS, snrc.eth_call, snrc.chain_now = self._saved
+
+    def _chain(self, expires, reserved):
+        def eth_call(to, data):
+            if data.startswith(snrc.selector("reservedNames(bytes32)")):
+                self.assertEqual(to, self.CONTROLLER)
+                return "0x" + snrc.encode_uint(1 if reserved else 0)
+            if data.startswith(snrc.selector("GRACE_PERIOD()")):
+                return "0x" + snrc.encode_uint(90 * 86400)
+            if data.startswith(snrc.selector("prices()")):
+                return "0x" + snrc.encode_uint(0)  # no price oracle, no auction
+            return "0x" + snrc.encode_uint(expires)
+
+        return eth_call
+
+    def test_unregistered_and_reserved_reports_the_reservation(self):
+        snrc.eth_call = self._chain(0, True)
+        reg = snrc.name_status("acme.testing")
+        self.assertEqual(reg["status"], "unregistered")
+        self.assertEqual(reg["reasonCode"], "internal")
+
+    def test_unregistered_and_not_reserved_reads_unregistered(self):
+        snrc.eth_call = self._chain(0, False)
+        self.assertEqual(snrc.name_status("acme.testing")["status"], "unregistered")
+
+    def test_a_lapsed_reserved_name_keeps_its_reservation(self):
+        past = int(time.time()) - 91 * 86400
+        snrc.eth_call = self._chain(past, True)
+        reg = snrc.name_status("acme.testing")
+        self.assertEqual(reg["status"], "expired")
+        self.assertEqual(reg["reasonCode"], "internal")
+
+    def test_a_live_name_is_registered_even_if_reserved(self):
+        snrc.eth_call = self._chain(int(time.time()) + 86400, True)
+        self.assertEqual(snrc.name_status("acme.testing")["status"], "registered")
+
+    def test_a_name_in_grace_belongs_to_its_owner_not_the_reserved_set(self):
+        snrc.eth_call = self._chain(int(time.time()) - 3600, True)
+        self.assertEqual(snrc.name_status("acme.testing")["status"], "grace")
+
+    def test_no_controller_configured_means_reserved_is_never_reported(self):
+        snrc.CONTROLLERS = {"testing": ""}
+        snrc.eth_call = self._chain(0, True)  # reserved on chain, but unread
+        self.assertEqual(snrc.name_status("acme.testing")["status"], "unregistered")
+
+    def test_reserved_is_asked_by_labelhash_so_a_hashed_query_works(self):
+        # keccak-256("acme")
+        hashed = "[e29dae06ef4c3e336b7538b6d4f52ca1ecec009b1df6fb501320e11b223aeeaf]"
+        snrc.eth_call = self._chain(0, True)
+        self.assertEqual(snrc.name_status(hashed + ".testing")["reasonCode"], "internal")
+
+
+class ReservedReasonTests(unittest.TestCase):
+    REGISTRY = "0x58fc46996d975c57883564648bda5206d1a0102b"
+    REGISTRAR = "0xef47eb4384b46c89e4482a677c2cbcbd2a6fd85a"
+    CONTROLLER = "0x281ca41311c2aa808c917c4674639d7567b75714"
+
+    def setUp(self):
+        self._saved = (
+            snrc.REGISTRIES,
+            snrc.REGISTRARS,
+            snrc.CONTROLLERS,
+            snrc.eth_call,
+            snrc.chain_now,
+        )
+        snrc.REGISTRIES = {"testing": self.REGISTRY}
+        snrc.REGISTRARS = {"testing": self.REGISTRAR}
+        snrc.CONTROLLERS = {"testing": self.CONTROLLER}
+        snrc.chain_now = lambda: int(time.time())
+
+    def tearDown(self):
+        (
+            snrc.REGISTRIES,
+            snrc.REGISTRARS,
+            snrc.CONTROLLERS,
+            snrc.eth_call,
+            snrc.chain_now,
+        ) = self._saved
+
+    def _chain(self, expires, reserved):
+        def eth_call(to, data):
+            if data.startswith(snrc.selector("reservedNames(bytes32)")):
+                return "0x" + snrc.encode_uint(1 if reserved else 0)
+            if data.startswith(snrc.selector("GRACE_PERIOD()")):
+                return "0x" + snrc.encode_uint(90 * 86400)
+            if data.startswith(snrc.selector("prices()")):
+                return "0x" + snrc.encode_uint(0)  # no price oracle, no auction
+            return "0x" + snrc.encode_uint(expires)
+
+        return eth_call
+
+    def _reserved_as(self, code):
+        def eth_call(to, data):
+            if data.startswith(snrc.selector("reservedNames(bytes32)")):
+                return "0x" + snrc.encode_uint(code)
+            if data.startswith(snrc.selector("GRACE_PERIOD()")):
+                return "0x" + snrc.encode_uint(90 * 86400)
+            if data.startswith(snrc.selector("prices()")):
+                return "0x" + snrc.encode_uint(0)
+            return "0x" + snrc.encode_uint(0)
+
+        return eth_call
+
+    def test_every_enum_value_has_a_code_and_a_sentence(self):
+        for code, (name, sentence) in snrc.RESERVED_REASONS.items():
+            snrc.eth_call = self._reserved_as(code)
+            reg = snrc.name_status("acme.testing")
+            self.assertEqual(reg["reasonCode"], name)
+            self.assertEqual(reg["reason"], sentence)
+
+    def test_a_trademark_reservation_says_so(self):
+        snrc.eth_call = self._reserved_as(2)
+        _, body = snrc.resolve("acme.testing")
+        self.assertEqual(body["reasonCode"], "trademark")
+
+    def test_a_controller_storing_a_bool_reads_as_internal(self):
+        """Before the enum `reservedNames` was a bool; its `true` decodes as 1."""
+        snrc.eth_call = self._reserved_as(1)
+        reg = snrc.name_status("acme.testing")
+        self.assertEqual(reg["reasonCode"], "internal")
+        self.assertEqual(reg["reason"], "reserved for SimpleX")
+
+    def test_an_enum_value_this_resolver_predates_is_not_dropped(self):
+        """A new Reason still reserves the name, and says it is unknown rather
+        than claiming the chain recorded none."""
+        snrc.eth_call = self._reserved_as(99)
+        reg = snrc.name_status("acme.testing")
+        self.assertEqual(reg["reasonCode"], "unknown")
+        self.assertEqual(reg["reason"], "reserved")
+
+    def test_a_reserved_name_carries_the_reason(self):
+        snrc.eth_call = self._chain(0, True)
+        status, body = snrc.resolve("acme.testing")
+        self.assertEqual(status, 404)
+        self.assertEqual(body["status"], "unregistered")
+        self.assertEqual(body["reason"], "reserved for SimpleX")
+
+    def test_the_message_does_not_claim_a_trademark(self):
+        snrc.eth_call = self._chain(0, True)
+        _, body = snrc.resolve("acme.testing")
+        self.assertNotIn("trademark", body["message"])
+
+    def test_an_unregistered_name_has_no_reason(self):
+        snrc.eth_call = self._chain(0, False)
+        status, body = snrc.resolve("acme.testing")
+        self.assertEqual(status, 404)
+        self.assertEqual(body["status"], "unregistered")
+        self.assertIsNone(body["reason"])
+
+    def test_an_expired_name_has_no_reason(self):
+        snrc.eth_call = self._chain(1, False)
+        status, body = snrc.resolve("acme.testing")
+        self.assertEqual(status, 410)
+        self.assertEqual(body["status"], "expired")
+        self.assertIsNone(body["reason"])
+
+    def test_a_hashed_query_gets_the_reason_too(self):
+        snrc.eth_call = self._chain(0, True)
+        # keccak-256("acme")
+        hashed = "[e29dae06ef4c3e336b7538b6d4f52ca1ecec009b1df6fb501320e11b223aeeaf]"
+        _, body = snrc.resolve(hashed + ".testing")
+        self.assertEqual(body["reason"], "reserved for SimpleX")
+
+
+class PricingTests(unittest.TestCase):
+    """The oracle keeps the curve in US cents per year, and a lapsed name costs
+    the ordinary price: this registry runs no auction."""
+
+    REGISTRY = "0x58fc46996d975c57883564648bda5206d1a0102b"
+    REGISTRAR = "0xef47eb4384b46c89e4482a677c2cbcbd2a6fd85a"
+    CONTROLLER = "0x281ca41311c2aa808c917c4674639d7567b75714"
+    ORACLE = "0x1e0c9a2b9d1a4c8f7b3e5d6a9c2f4b8e1d7a3c50"
+
+    GRACE = 90 * 86400
+    BASE = 200
+    EXCEPTIONS = {1: 64000, 2: 16000, 3: 1600, 4: 800, 5: 500}
+    MIN_LENGTH = 3
+
+    def setUp(self):
+        self._saved = (
+            snrc.REGISTRIES,
+            snrc.REGISTRARS,
+            snrc.CONTROLLERS,
+            snrc.eth_call,
+            snrc.chain_now,
+        )
+        snrc.REGISTRIES = {"testing": self.REGISTRY}
+        snrc.REGISTRARS = {"testing": self.REGISTRAR}
+        snrc.CONTROLLERS = {"testing": self.CONTROLLER}
+        self.now = int(time.time())
+        snrc.chain_now = lambda: self.now
+
+    def tearDown(self):
+        (
+            snrc.REGISTRIES,
+            snrc.REGISTRARS,
+            snrc.CONTROLLERS,
+            snrc.eth_call,
+            snrc.chain_now,
+        ) = self._saved
+
+    def _prices_return(self):
+        words = [snrc.encode_uint(self.BASE), snrc.encode_uint(0x40),
+                 snrc.encode_uint(len(self.EXCEPTIONS))]
+        for length, cents in self.EXCEPTIONS.items():
+            words += [snrc.encode_uint(length), snrc.encode_uint(cents)]
+        return "0x" + "".join(words)
+
+    def _chain(self, expires, oracle=None, reserved=0):
+        oracle = self.ORACLE if oracle is None else oracle
+        self.oracle_calls = []
+
+        def eth_call(to, data):
+            if data.startswith(snrc.selector("nameExpires(uint256)")):
+                return "0x" + snrc.encode_uint(expires)
+            if data.startswith(snrc.selector("GRACE_PERIOD()")):
+                return "0x" + snrc.encode_uint(self.GRACE)
+            if data.startswith(snrc.selector("reservedNames(bytes32)")):
+                return "0x" + snrc.encode_uint(reserved)
+            if data.startswith(snrc.selector("minCharLength()")):
+                self.assertEqual(to, self.CONTROLLER)
+                return "0x" + snrc.encode_uint(self.MIN_LENGTH)
+            if data.startswith(snrc.selector("prices()")):
+                if to == self.CONTROLLER:
+                    return "0x" + snrc.encode_uint(int(oracle, 16))
+                self.oracle_calls.append(data[:10])
+                self.assertEqual(to, oracle)
+                return self._prices_return()
+            return self.fail("unexpected call " + data[:10])
+
+        return eth_call
+
+    def _lapsed(self, days_past_grace):
+        """An expiry whose grace ended `days_past_grace` days ago. The extra
+        second clears the boundary, which counts as still in grace."""
+        return self.now - self.GRACE - 1 - days_past_grace * 86400
+
+    def test_the_prices_are_the_oracles_cents_per_year(self):
+        snrc.eth_call = self._chain(self._lapsed(0))
+        reg = snrc.name_status("acme.testing")
+        # 1 and 2 are below minCharLength
+        self.assertEqual(reg["registrationPrices"], {3: 1600, 4: 800, 5: 500})
+        self.assertEqual(reg["basePrice"], self.BASE)
+        self.assertEqual(reg["minLabelLength"], self.MIN_LENGTH)
+
+    def test_a_lapsed_name_costs_the_ordinary_price(self):
+        snrc.eth_call = self._chain(self._lapsed(0))
+        reg = snrc.name_status("acme.testing")
+        self.assertEqual(reg["status"], "expired")
+
+    def test_a_controller_with_no_oracle_leaves_the_name_merely_expired(self):
+        snrc.eth_call = self._chain(self._lapsed(0), oracle=snrc.ZERO_ADDR)
+        self.assertEqual(snrc.name_status("acme.testing")["status"], "expired")
+
+    def test_a_name_in_grace_never_reaches_the_oracle(self):
+        snrc.eth_call = self._chain(self.now - 3600)
+        self.assertEqual(snrc.name_status("acme.testing")["status"], "grace")
+        self.assertEqual(self.oracle_calls, [])
+
+    def test_a_reserved_lapsed_name_keeps_its_reservation(self):
+        snrc.eth_call = self._chain(self._lapsed(0), reserved=2)
+        reg = snrc.name_status("acme.testing")
+        self.assertEqual(reg["status"], "expired")
+        self.assertEqual(reg["reasonCode"], "trademark")
+
+    def test_resolve_reports_the_prices(self):
+        snrc.eth_call = self._chain(self._lapsed(1))
+        status, body = snrc.resolve("acme.testing")
+        self.assertEqual(status, 410)
+        self.assertEqual(body["status"], "expired")
+        self.assertEqual(body["basePrice"], self.BASE)
+
+    def test_a_hashed_query_is_priced_too(self):
+        # keccak-256("acme")
+        hashed = "[e29dae06ef4c3e336b7538b6d4f52ca1ecec009b1df6fb501320e11b223aeeaf]"
+        snrc.eth_call = self._chain(self._lapsed(0))
+        _, body = snrc.resolve(hashed + ".testing")
+        self.assertEqual(body["status"], "expired")
+        self.assertEqual(body["basePrice"], self.BASE)
+
+
+class EnsOracleTests(unittest.TestCase):
+    """.testing runs an ENS-shaped oracle: it prices in attoUSD per second and
+    charges a premium on lapsed names that it does not expose."""
+
+    REGISTRY = "0x58fc46996d975c57883564648bda5206d1a0102b"
+    REGISTRAR = "0xef47eb4384b46c89e4482a677c2cbcbd2a6fd85a"
+    CONTROLLER = "0x281ca41311c2aa808c917c4674639d7567b75714"
+    ORACLE = "0x1e0c9a2b9d1a4c8f7b3e5d6a9c2f4b8e1d7a3c50"
+    GRACE = 90 * 86400
+    MIN_LENGTH = 6
+
+    def setUp(self):
+        self._saved = (snrc.REGISTRIES, snrc.REGISTRARS, snrc.CONTROLLERS, snrc.eth_call, snrc.chain_now)
+        snrc.REGISTRIES = {"testing": self.REGISTRY}
+        snrc.REGISTRARS = {"testing": self.REGISTRAR}
+        snrc.CONTROLLERS = {"testing": self.CONTROLLER}
+        self.now = int(time.time())
+        snrc.chain_now = lambda: self.now
+
+    def tearDown(self):
+        (snrc.REGISTRIES, snrc.REGISTRARS, snrc.CONTROLLERS, snrc.eth_call, snrc.chain_now) = self._saved
+
+    def _chain(self, expires, letter_cents=0):
+        def eth_call(to, data):
+            if data.startswith(snrc.selector("nameExpires(uint256)")):
+                return "0x" + snrc.encode_uint(expires)
+            if data.startswith(snrc.selector("GRACE_PERIOD()")):
+                return "0x" + snrc.encode_uint(self.GRACE)
+            if data.startswith(snrc.selector("reservedNames(bytes32)")):
+                return "0x" + snrc.encode_uint(0)
+            if data.startswith(snrc.selector("minCharLength()")):
+                return "0x" + snrc.encode_uint(self.MIN_LENGTH)
+            if data.startswith(snrc.selector("prices()")):
+                if to == self.CONTROLLER:
+                    return "0x" + snrc.encode_uint(int(self.ORACLE, 16))
+                raise RuntimeError("eth_call returned 0x")  # no prices() on this oracle
+            for n in range(1, 7):
+                if data.startswith(snrc.selector(f"price{n}Letter()")):
+                    rate = letter_cents * snrc.ATTO_PER_CENT // snrc.SECONDS_PER_YEAR
+                    return "0x" + snrc.encode_uint(rate)
+            return self.fail("unexpected call " + data[:10])
+
+        return eth_call
+
+    def test_a_never_registered_name_is_priced_from_the_letter_curve(self):
+        snrc.eth_call = self._chain(0)
+        reg = snrc.name_status("ghost.testing")
+        self.assertEqual(reg["status"], "unregistered")
+        self.assertEqual(reg["basePrice"], 0)
+        self.assertEqual(reg["minLabelLength"], self.MIN_LENGTH)
+
+    def test_a_non_zero_letter_curve_converts_to_cents_per_year(self):
+        snrc.eth_call = self._chain(0, letter_cents=1200)
+        self.assertEqual(snrc.name_status("ghost.testing")["basePrice"], 1200)
+
+    def test_a_lapsed_name_is_not_priced_because_the_premium_is_unreadable(self):
+        snrc.eth_call = self._chain(self.now - self.GRACE - 1)
+        reg = snrc.name_status("acme.testing")
+        self.assertEqual(reg["status"], "expired")
+        self.assertNotIn("basePrice", reg)
+
+
+class ErrorCodeTests(unittest.TestCase):
+    REGISTRY = "0x58fc46996d975c57883564648bda5206d1a0102b"
+    REGISTRAR = "0xef47eb4384b46c89e4482a677c2cbcbd2a6fd85a"
+
+    def setUp(self):
+        self._saved = (
+            snrc.REGISTRIES,
+            snrc.REGISTRARS,
+            snrc.CONTROLLERS,
+            snrc.eth_call,
+            snrc.chain_now,
+        )
+        snrc.REGISTRIES = {"testing": self.REGISTRY, "simplex": ""}
+        snrc.REGISTRARS = {"testing": self.REGISTRAR}
+        snrc.CONTROLLERS = {"testing": ""}
+        snrc.chain_now = lambda: int(time.time())
+
+    def tearDown(self):
+        (
+            snrc.REGISTRIES,
+            snrc.REGISTRARS,
+            snrc.CONTROLLERS,
+            snrc.eth_call,
+            snrc.chain_now,
+        ) = self._saved
+
+    def _chain(self, expires, resolver=None):
+        def eth_call(to, data):
+            if data.startswith(snrc.selector("GRACE_PERIOD()")):
+                return "0x" + snrc.encode_uint(90 * 86400)
+            if data.startswith(snrc.selector("resolver(bytes32)")):
+                return "0x" + "00" * 12 + (resolver or "00" * 20)
+            return "0x" + snrc.encode_uint(expires)
+
+        return eth_call
+
+    def test_an_unconfigured_tld_names_the_ones_that_are(self):
+        status, body = snrc.resolve("alice.nosuchtld")
+        self.assertEqual(status, 400)
+        self.assertEqual(body["error"], "tldNotConfigured")
+        self.assertEqual(body["configuredTlds"], ["testing"])
+        self.assertIn("nosuchtld", body["message"])
+
+    def test_a_registration_problem_reports_the_status_as_the_code(self):
+        for expires, code in (
+            (0, "unregistered"),
+            (int(time.time()) - 91 * 86400, "expired"),
+        ):
+            with self.subTest(code=code):
+                snrc.eth_call = self._chain(expires)
+                _, body = snrc.resolve("alice.testing")
+                self.assertEqual(body["error"], code)
+                self.assertEqual(body["status"], code)
+
+    def test_a_name_in_grace_still_resolves(self):
+        snrc.eth_call = self._chain(int(time.time()) - 3600)
+        status, body = snrc.resolve("alice.testing")
+        self.assertEqual(status, 200)
+        self.assertEqual(body["status"], "grace")
+        self.assertNotIn("error", body)
+
+    def test_a_registered_name_pointing_nowhere_resolves_with_empty_records(self):
+        snrc.eth_call = self._chain(int(time.time()) + 86400)
+        status, body = snrc.resolve("alice.testing")
+        self.assertEqual(status, 200)
+        self.assertEqual(body["status"], "registered")
+        self.assertEqual(body["resolver"], snrc.ZERO_ADDR)
+        self.assertEqual(body["simplexContact"], [])
+
+    def test_every_error_body_carries_both_fields(self):
+        snrc.eth_call = self._chain(0)
+        for name in ("alice.nosuchtld", "alice.testing"):
+            with self.subTest(name=name):
+                _, body = snrc.resolve(name)
+                self.assertIsInstance(body["error"], str)
+                self.assertIsInstance(body["message"], str)
+                self.assertNotEqual(body["error"], body["message"])
+
+    def test_an_upstream_failure_does_not_echo_the_exception(self):
+        with contextlib.redirect_stderr(io.StringIO()) as log:
+            body = snrc.upstream_error(
+                {"name": "alice.testing"},
+                RuntimeError("http://user:[email protected]/kEy8 refused"),
+            )
+        # the operator still sees the detail in the log
+        self.assertIn("secret", log.getvalue())
+        self.assertEqual(body["error"], "upstreamError")
+        self.assertIn("RuntimeError", body["message"])
+        self.assertNotIn("secret", body["message"])
+        self.assertNotIn("kEy8", body["message"])
+
+
+class RegistrationV2Tests(unittest.TestCase):
+    """`/v2/resolve` answers with the SMP protocol's NameRegistration, which the
+    relay decodes as is. The key names are the wire contract, so they are pinned
+    here: renaming one without the Haskell side is a silent break."""
+
+    REGISTRY = "0x58fc46996d975c57883564648bda5206d1a0102b"
+    REGISTRAR = "0xef47eb4384b46c89e4482a677c2cbcbd2a6fd85a"
+    CONTROLLER = "0x281ca41311c2aa808c917c4674639d7567b75714"
+    ORACLE = "0x1e0c9a2b9d1a4c8f7b3e5d6a9c2f4b8e1d7a3c50"
+    OWNER = "0xd83bd7e0e6b8a4c1f2593a7b0c4e8d1a6f9b2c37"
+
+    GRACE = 90 * 86400
+    BASE = 200
+    EXCEPTIONS = {1: 64000, 2: 16000, 3: 1600, 4: 800, 5: 500}
+    MIN_LENGTH = 3
+
+    def setUp(self):
+        self._saved = (
+            snrc.REGISTRIES,
+            snrc.REGISTRARS,
+            snrc.CONTROLLERS,
+            snrc.eth_call,
+            snrc.chain_now,
+        )
+        snrc.REGISTRIES = {"testing": self.REGISTRY}
+        snrc.REGISTRARS = {"testing": self.REGISTRAR}
+        snrc.CONTROLLERS = {"testing": self.CONTROLLER}
+        self.now = int(time.time())
+        snrc.chain_now = lambda: self.now
+
+    def tearDown(self):
+        (
+            snrc.REGISTRIES,
+            snrc.REGISTRARS,
+            snrc.CONTROLLERS,
+            snrc.eth_call,
+            snrc.chain_now,
+        ) = self._saved
+
+    def _prices_return(self):
+        words = [snrc.encode_uint(self.BASE), snrc.encode_uint(0x40),
+                 snrc.encode_uint(len(self.EXCEPTIONS))]
+        for length, cents in self.EXCEPTIONS.items():
+            words += [snrc.encode_uint(length), snrc.encode_uint(cents)]
+        return "0x" + "".join(words)
+
+    @staticmethod
+    def _abi_bytes(value: bytes) -> str:
+        """head offset, length, then the payload padded to a 32-byte word."""
+        pad = (-len(value)) % 32
+        return ("0x" + snrc.encode_uint(0x20) + snrc.encode_uint(len(value))
+                + (value + b"\x00" * pad).hex())
+
+    def _chain(self, expires, reserved=0, oracle=None, label=b"acme", owner=None):
+        """The registry answers a zero resolver, so name_record returns the
+        empty record a registered name still has. `label` is what the registrar
+        recorded for the 2LD; b"" means it recorded none. `owner` is the owner of
+        the queried node; ZERO_ADDR means that node was never created."""
+        oracle = self.ORACLE if oracle is None else oracle
+        owner = self.OWNER if owner is None else owner
+
+        def eth_call(to, data):
+            if data.startswith(snrc.selector("labelOf(uint256)")):
+                return self._abi_bytes(label)
+            if data.startswith(snrc.selector("nameExpires(uint256)")):
+                return "0x" + snrc.encode_uint(expires)
+            if data.startswith(snrc.selector("GRACE_PERIOD()")):
+                return "0x" + snrc.encode_uint(self.GRACE)
+            if data.startswith(snrc.selector("reservedNames(bytes32)")):
+                return "0x" + snrc.encode_uint(reserved)
+            if data.startswith(snrc.selector("minCharLength()")):
+                return "0x" + snrc.encode_uint(self.MIN_LENGTH)
+            if data.startswith(snrc.selector("prices()")):
+                if to == self.CONTROLLER:
+                    return "0x" + snrc.encode_uint(int(oracle, 16))
+                return self._prices_return()
+            if data.startswith(snrc.selector("resolver(bytes32)")):
+                return "0x" + snrc.encode_uint(0)
+            if data.startswith(snrc.selector("owner(bytes32)")):
+                return "0x" + snrc.encode_uint(int(owner, 16))
+            return self.fail("unexpected call " + data[:10])
+
+        return eth_call
+
+    def _lapsed(self, days_past_grace):
+        return self.now - self.GRACE - 1 - days_past_grace * 86400
+
+    def test_a_live_name_is_registered_and_carries_its_record(self):
+        expires = self.now + 3600
+        snrc.eth_call = self._chain(expires)
+        status, body = registration("acme.testing")
+        self.assertEqual(status, 200)
+        self.assertEqual(body["type"], "registered")
+        self.assertEqual(body["expires"], expires)
+        self.assertEqual(body["graceUntil"], expires + self.GRACE)
+        self.assertIsNone(body["reservedReason_"])
+        self.assertEqual(body["nameRecord"]["name"], "acme.testing")
+
+    def test_a_name_in_grace_is_still_registered(self):
+        expires = self.now - 3600
+        snrc.eth_call = self._chain(expires)
+        _, body = registration("acme.testing")
+        self.assertEqual(body["type"], "registered")
+        self.assertGreater(body["graceUntil"], self.now)
+
+    def test_a_registered_name_that_is_held_back_says_so(self):
+        snrc.eth_call = self._chain(self.now + 3600, reserved=1)
+        _, body = registration("acme.testing")
+        self.assertEqual(body["type"], "registered")
+        self.assertEqual(body["reservedReason_"], "internal")
+
+    def test_an_unregistered_name_is_available_with_its_pricing(self):
+        snrc.eth_call = self._chain(0)
+        status, body = registration("acme.testing")
+        self.assertEqual(status, 200)
+        self.assertEqual(body["type"], "available")
+        # lengths below minCharLength are unregistrable, so they are not priced
+        self.assertEqual(body["pricing"]["registrationPrices"], {3: 1600, 4: 800, 5: 500})
+        self.assertEqual(body["pricing"]["basePrice"], self.BASE)
+        self.assertEqual(body["pricing"]["minLabelLength"], self.MIN_LENGTH)
+
+    def test_a_lapsed_name_is_available_at_the_ordinary_price(self):
+        snrc.eth_call = self._chain(self._lapsed(1))
+        _, body = registration("acme.testing")
+        self.assertEqual(body["type"], "available")
+        self.assertEqual(body["pricing"]["basePrice"], self.BASE)
+
+    def test_a_held_back_name_is_reserved_and_is_never_priced(self):
+        snrc.eth_call = self._chain(0, reserved=2)
+        status, body = registration("acme.testing")
+        self.assertEqual(status, 200)
+        self.assertEqual(body["type"], "reserved")
+        self.assertEqual(body["reservedReason"], "trademark")
+        self.assertNotIn("pricing", body)
+
+    def test_a_hashed_query_answers_the_same_as_the_name(self):
+        # keccak-256("acme")
+        hashed = "[e29dae06ef4c3e336b7538b6d4f52ca1ecec009b1df6fb501320e11b223aeeaf]"
+        snrc.eth_call = self._chain(0)
+        _, by_name = registration("acme.testing")
+        _, by_hash = registration(hashed + ".testing")
+        self.assertEqual(by_name, by_hash)
+
+    def test_an_unconfigured_tld_is_refused_not_answered(self):
+        snrc.REGISTRIES = {"testing": ""}
+        snrc.eth_call = lambda *a: self.fail("must not reach the chain")
+        status, body = registration("acme.testing")
+        self.assertEqual(status, 400)
+        self.assertEqual(body["error"], "tldNotConfigured")
+
+    def test_no_price_oracle_is_an_error_not_a_free_name(self):
+        snrc.eth_call = self._chain(0, oracle=snrc.ZERO_ADDR)
+        status, body = registration("acme.testing")
+        self.assertEqual(status, 502)
+        self.assertEqual(body["error"], "noPriceOracle")
+
+    def test_a_status_it_cannot_read_is_an_error_not_a_registration(self):
+        snrc.REGISTRARS = {"testing": ""}
+        snrc.eth_call = self._chain(0)
+        status, body = registration("acme.testing")
+        self.assertEqual(status, 502)
+        self.assertEqual(body["error"], "unknown")
+
+    def test_each_answer_carries_exactly_its_own_fields(self):
+        """The relay decodes by these names; an extra or missing one is a break."""
+        cases = {
+            "registered": (self._chain(self.now + 3600),
+                           {"type", "expires", "graceUntil", "reservedReason_", "nameRecord"}),
+            "available": (self._chain(0), {"type", "pricing"}),
+            "reserved": (self._chain(0, reserved=1), {"type", "reservedReason"}),
+        }
+        for expected_type, (chain, keys) in cases.items():
+            with self.subTest(type=expected_type):
+                snrc.eth_call = chain
+                _, body = registration("acme.testing")
+                self.assertEqual(body["type"], expected_type)
+                self.assertEqual(set(body), keys)
+    def test_a_hashed_query_the_registrar_cannot_name_is_refused(self):
+        """The client checks the record names what it asked about, so answering
+        with a record the registrar could not name would only fail there."""
+        hashed = "[e29dae06ef4c3e336b7538b6d4f52ca1ecec009b1df6fb501320e11b223aeeaf]"
+        snrc.eth_call = self._chain(self.now + 3600, label=b"")
+        status, body = registration(hashed + ".testing")
+        self.assertEqual(status, 502)
+        self.assertEqual(body["error"], "labelNotRecorded")
+
+    def test_a_hashed_query_is_answered_with_the_name_the_registrar_recorded(self):
+        hashed = "[e29dae06ef4c3e336b7538b6d4f52ca1ecec009b1df6fb501320e11b223aeeaf]"
+        snrc.eth_call = self._chain(self.now + 3600)
+        status, body = registration(hashed + ".testing")
+        self.assertEqual(status, 200)
+        self.assertEqual(body["nameRecord"]["name"], "acme.testing")
+    def test_a_subname_that_exists_is_registered_with_its_parents_dates(self):
+        expires = self.now + 3600
+        snrc.eth_call = self._chain(expires)
+        status, body = registration("sub.acme.testing")
+        self.assertEqual(status, 200)
+        self.assertEqual(body["type"], "registered")
+        self.assertEqual(body["expires"], expires)
+        self.assertEqual(body["nameRecord"]["name"], "sub.acme.testing")
+
+    def test_a_subname_nobody_created_is_not_registered(self):
+        """The registrar only tracks 2LDs, so the parent's registration says
+        nothing about a child that was never created: its node has no owner."""
+        snrc.eth_call = self._chain(self.now + 3600, owner=snrc.ZERO_ADDR)
+        status, body = registration("sub.acme.testing")
+        self.assertEqual(status, 200)
+        self.assertEqual(body["type"], "available")
+
+    def test_a_2ld_is_not_subject_to_the_owner_check(self):
+        """Only a subname can be absent under a registered parent."""
+        snrc.eth_call = self._chain(self.now + 3600, owner=snrc.ZERO_ADDR)
+        _, body = registration("acme.testing")
+        self.assertEqual(body["type"], "registered")
+
+    def test_v1_does_not_report_an_uncreated_subname_as_registered(self):
+        """v1 has no availability, so the only honest answer is not-found. The
+        2LD case is untouched: a registered name with no resolver still resolves."""
+        snrc.eth_call = self._chain(self.now + 3600, owner=snrc.ZERO_ADDR)
+        status, body = snrc.resolve("sub.acme.testing")
+        self.assertEqual(status, 404)
+        self.assertEqual(body["error"], "unregistered")
+
+    def test_v1_still_resolves_a_2ld_with_no_resolver_set(self):
+        snrc.eth_call = self._chain(self.now + 3600, owner=snrc.ZERO_ADDR)
+        status, body = snrc.resolve("acme.testing")
+        self.assertEqual(status, 200)
+        self.assertEqual(body["resolver"], snrc.ZERO_ADDR)
+
+    def test_the_answer_says_which_block_it_was_read_at(self):
+        """The resolver is only as current as its node. Without this a client
+        cannot tell an answer that predates its own registration."""
+        snrc.eth_call = self._chain(self.now + 3600)
+        _, res = snrc.registration("acme.testing")
+        self.assertEqual(res["lastBlockTs"], self.now)
+        self.assertEqual(res["registration"]["type"], "registered")
+
+    def test_an_available_name_says_so_too(self):
+        """This is the path that reads no block otherwise, and the one where
+        staleness matters most: the name may already be taken."""
+        snrc.eth_call = self._chain(0)
+        _, res = snrc.registration("acme.testing")
+        self.assertEqual(res["lastBlockTs"], self.now)
+        self.assertEqual(res["registration"]["type"], "available")
+
 if __name__ == "__main__":
     unittest.main()

simplexmq.cabal

@@ -1,7 +1,7 @@
 cabal-version: 3.0
 
 name:           simplexmq
-version:        7.0.0.6
+version:        7.1.0.9
 synopsis:       SimpleXMQ message broker
 description:    This package includes <./docs/Simplex-Messaging-Server.html server>,
                 <./docs/Simplex-Messaging-Client.html client> and
@@ -135,6 +135,7 @@ library
       Simplex.Messaging.Crypto.Lazy
       Simplex.Messaging.Crypto.Ratchet
       Simplex.Messaging.Crypto.BBS
+      Simplex.Messaging.Crypto.Entitlement
       Simplex.Messaging.Crypto.SNTRUP761
       Simplex.Messaging.Crypto.SNTRUP761.Bindings
       Simplex.Messaging.Crypto.SNTRUP761.Bindings.Defines
@@ -152,6 +153,7 @@ library
       Simplex.Messaging.Protocol
       Simplex.Messaging.Protocol.Types
       Simplex.Messaging.Server.Expiration
+      Simplex.Messaging.Server.Information
       Simplex.Messaging.Server.QueueStore.Postgres.Config
       Simplex.Messaging.Server.QueueStore.QueueInfo
       Simplex.Messaging.ServiceScheme
@@ -190,6 +192,9 @@ library
           Simplex.Messaging.Agent.Store.Postgres.Migrations.M20251230_strict_tables
           Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260410_receive_attempts
           Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260411_service_certs
+          Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260712_address_dr_rpc
+          Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260823_snd_files_entitlement
+          Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260919_ratchet_verify_codes
   else
       exposed-modules:
           Simplex.Messaging.Agent.Store.SQLite
@@ -242,6 +247,9 @@ library
           Simplex.Messaging.Agent.Store.SQLite.Migrations.M20251230_strict_tables
           Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260410_receive_attempts
           Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260411_service_certs
+          Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260712_address_dr_rpc
+          Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260823_snd_files_entitlement
+          Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260919_ratchet_verify_codes
           Simplex.Messaging.Agent.Store.SQLite.Util
   if flag(client_postgres) || flag(server_postgres)
       exposed-modules:
@@ -266,7 +274,6 @@ library
           Simplex.Messaging.Server.CLI
           Simplex.Messaging.Server.Control
           Simplex.Messaging.Server.Env.STM
-          Simplex.Messaging.Server.Information
           Simplex.Messaging.Server.Main
           Simplex.Messaging.Server.Main.GitCommit
           Simplex.Messaging.Server.Main.Init
@@ -652,3 +659,48 @@ test-suite simplexmq-test
   if flag(client_postgres) || flag(server_postgres)
     build-depends:
         postgresql-simple ==0.7.*
+
+executable smp-mem-bench
+  if flag(client_library)
+      buildable: False
+  main-is: MemBench.hs
+  other-modules:
+      NetLag
+      SMPClient
+      Util
+  hs-source-dirs:
+      bench
+      tests
+  default-extensions:
+      StrictData
+  ghc-options: -Wall -Wno-unused-imports -Wno-unused-top-binds -Wno-name-shadowing -threaded -rtsopts "-with-rtsopts=-T"
+  build-depends:
+      base
+    , async
+    , bytestring
+    , containers
+    , crypton
+    , crypton-x509
+    , crypton-x509-store
+    , crypton-x509-validation
+    , directory
+    , hspec ==2.11.*
+    , hspec-core ==2.11.*
+    , mtl
+    , network
+    , process
+    , simple-logger
+    , simplexmq
+    , sqlcipher-simple
+    , stm
+    , text
+    , time
+    , tls >=1.9.0 && <1.10
+    , transformers
+    , unliftio
+    , unliftio-core
+  if flag(server_postgres)
+    cpp-options: -DdbServerPostgres
+    build-depends:
+        postgresql-simple ==0.7.*
+  default-language: Haskell2010

src/Simplex/FileTransfer/Agent.hs

@@ -17,11 +17,16 @@ module Simplex.FileTransfer.Agent
     toFSFilePath,
     -- Receiving files
     xftpReceiveFile',
+    xftpPrepareReceiveFile',
+    xftpStartReceiveFile',
     xftpDeleteRcvFile',
     xftpDeleteRcvFiles',
     -- Sending files
     xftpSendFile',
     xftpSendDescription',
+    xftpPrepareSendFile',
+    xftpPrepareSendDescription',
+    xftpStartSendFile',
     deleteSndFileInternal,
     deleteSndFilesInternal,
     deleteSndFileRemote,
@@ -50,11 +55,12 @@ import qualified Data.Set as S
 import Data.Text (Text, pack)
 import Data.Time.Clock (getCurrentTime)
 import Data.Time.Format (defaultTimeLocale, formatTime)
+import Data.Word (Word32)
 import Simplex.FileTransfer.Chunks (toKB)
 import Simplex.FileTransfer.Client (XFTPChunkSpec (..), getChunkDigest, prepareChunkSizes, prepareChunkSpecs, singleChunkSize)
 import Simplex.FileTransfer.Crypto
 import Simplex.FileTransfer.Description
-import Simplex.FileTransfer.Protocol (FileParty (..), SFileParty (..))
+import Simplex.FileTransfer.Protocol (FileParty (..), GrantedStorageTime, SFileParty (..))
 import Simplex.FileTransfer.Transport (XFTPRcvChunkSpec (..))
 import qualified Simplex.FileTransfer.Transport as XFTP
 import Simplex.FileTransfer.Types
@@ -126,7 +132,12 @@ closeXFTPAgent a = do
     stopWorkers workers = atomically (swapTVar workers M.empty) >>= mapM_ (liftIO . cancelWorker)
 
 xftpReceiveFile' :: AgentClient -> UserId -> ValidFileDescription 'FRecipient -> Maybe CryptoFileArgs -> Bool -> AM RcvFileId
-xftpReceiveFile' c userId (ValidFileDescription fd@FileDescription {chunks, redirect}) cfArgs approvedRelays = do
+xftpReceiveFile' c userId vfd cfArgs approvedRelays = do
+  fId <- xftpPrepareReceiveFile' c userId vfd cfArgs approvedRelays
+  fId <$ xftpStartReceiveFile' c fId
+
+xftpPrepareReceiveFile' :: AgentClient -> UserId -> ValidFileDescription 'FRecipient -> Maybe CryptoFileArgs -> Bool -> AM RcvFileId
+xftpPrepareReceiveFile' c userId (ValidFileDescription fd@FileDescription {redirect}) cfArgs approvedRelays = do
   g <- asks random
   prefixPath <- lift $ getPrefixPath "rcv.xftp"
   createDirectory prefixPath
@@ -136,7 +147,7 @@ xftpReceiveFile' c userId (ValidFileDescription fd@FileDescription {chunks, redi
   lift $ createDirectory =<< toFSFilePath relTmpPath
   lift $ createEmptyFile =<< toFSFilePath relSavePath
   let saveFile = CryptoFile relSavePath cfArgs
-  fId <- case redirect of
+  case redirect of
     Nothing -> withStore c $ \db -> createRcvFile db g userId fd relPrefixPath relTmpPath saveFile approvedRelays
     Just _ -> do
       -- prepare description paths
@@ -148,8 +159,11 @@ xftpReceiveFile' c userId (ValidFileDescription fd@FileDescription {chunks, redi
       let saveFileRedirect = CryptoFile relSavePathRedirect $ Just cfArgsRedirect
       -- create download tasks
       withStore c $ \db -> createRcvFileRedirect db g userId fd relPrefixPath relTmpPathRedirect saveFileRedirect relTmpPath saveFile approvedRelays
-  forM_ chunks (downloadChunk c)
-  pure fId
+
+xftpStartReceiveFile' :: AgentClient -> RcvFileId -> AM ()
+xftpStartReceiveFile' c rcvFileEntityId = do
+  srvs <- withStore c (`startPreparedRcvFile` rcvFileEntityId)
+  lift $ forM_ srvs $ void . getXFTPRcvWorker True c . Just
 
 downloadChunk :: AgentClient -> FileChunk -> AM ()
 downloadChunk c FileChunk {replicas = (FileChunkReplica {server} : _)} = do
@@ -350,8 +364,13 @@ xftpDeleteRcvFiles' c rcvFileEntityIds = do
 notify :: forall m e. (MonadIO m, AEntityI e) => AgentClient -> AEntityId -> AEvent e -> m ()
 notify c entId cmd = atomically $ writeTBQueue (subQ c) ("", entId, AEvt (sAEntity @e) cmd)
 
-xftpSendFile' :: AgentClient -> UserId -> CryptoFile -> Int -> AM SndFileId
-xftpSendFile' c userId file numRecipients = do
+xftpSendFile' :: AgentClient -> UserId -> CryptoFile -> Int -> Maybe Word32 -> AM SndFileId
+xftpSendFile' c userId file numRecipients storageHours = do
+  fId <- xftpPrepareSendFile' c userId file numRecipients storageHours
+  fId <$ xftpStartSendFile' c fId
+
+xftpPrepareSendFile' :: AgentClient -> UserId -> CryptoFile -> Int -> Maybe Word32 -> AM SndFileId
+xftpPrepareSendFile' c userId file numRecipients storageHours = do
   g <- asks random
   prefixPath <- lift $ getPrefixPath "snd.xftp"
   createDirectory prefixPath
@@ -359,12 +378,15 @@ xftpSendFile' c userId file numRecipients = do
   key <- atomically $ C.randomSbKey g
   nonce <- atomically $ C.randomCbNonce g
   -- saving absolute filePath will not allow to restore file encryption after app update, but it's a short window
-  fId <- withStore c $ \db -> createSndFile db g userId file numRecipients relPrefixPath key nonce Nothing
-  lift . void $ getXFTPSndWorker True c Nothing
-  pure fId
+  withStore c $ \db -> createSndFile db g userId file numRecipients relPrefixPath key nonce Nothing storageHours
 
 xftpSendDescription' :: AgentClient -> UserId -> ValidFileDescription 'FRecipient -> Int -> AM SndFileId
-xftpSendDescription' c userId (ValidFileDescription fdDirect@FileDescription {size, digest}) numRecipients = do
+xftpSendDescription' c userId vfd numRecipients = do
+  fId <- xftpPrepareSendDescription' c userId vfd numRecipients
+  fId <$ xftpStartSendFile' c fId
+
+xftpPrepareSendDescription' :: AgentClient -> UserId -> ValidFileDescription 'FRecipient -> Int -> AM SndFileId
+xftpPrepareSendDescription' c userId (ValidFileDescription fdDirect@FileDescription {size, digest}) numRecipients = do
   g <- asks random
   prefixPath <- lift $ getPrefixPath "snd.xftp"
   createDirectory prefixPath
@@ -375,9 +397,12 @@ xftpSendDescription' c userId (ValidFileDescription fdDirect@FileDescription {si
   liftError (FILE . FILE_IO . show) $ CF.writeFile file (LB.fromStrict $ strEncode fdDirect)
   key <- atomically $ C.randomSbKey g
   nonce <- atomically $ C.randomCbNonce g
-  fId <- withStore c $ \db -> createSndFile db g userId file numRecipients relPrefixPath key nonce $ Just RedirectFileInfo {size, digest}
+  withStore c $ \db -> createSndFile db g userId file numRecipients relPrefixPath key nonce (Just RedirectFileInfo {size, digest}) Nothing
+
+xftpStartSendFile' :: AgentClient -> SndFileId -> AM ()
+xftpStartSendFile' c sndFileEntityId = do
+  withStore c (`startPreparedSndFile` sndFileEntityId)
   lift . void $ getXFTPSndWorker True c Nothing
-  pure fId
 
 resumeXFTPSndWork :: AgentClient -> Maybe XFTPServer -> AM' ()
 resumeXFTPSndWork = void .: getXFTPSndWorker False
@@ -405,7 +430,7 @@ runXFTPSndPrepareWorker c Worker {doWork} = do
     prepareFile _ SndFile {prefixPath = Nothing} =
       throwE $ INTERNAL "no prefix path"
     prepareFile cfg sndFile@SndFile {sndFileId, sndFileEntityId, userId, prefixPath = Just ppath, status} = do
-      SndFile {numRecipients, chunks} <-
+      SndFile {numRecipients, chunks, storageHours} <-
         if status /= SFSEncrypted -- status is SFSNew or SFSEncrypting
           then do
             fsEncPath <- lift . toFSFilePath $ sndFileEncPath ppath
@@ -424,7 +449,7 @@ runXFTPSndPrepareWorker c Worker {doWork} = do
       let (pendingChunks, preparedSrvs) = partitionEithers $ map srvOrPendingChunk chunks
       -- concurrently?
       -- separate worker to create chunks? record retries and delay on snd_file_chunks?
-      srvs <- forM pendingChunks $ createChunk numRecipients'
+      srvs <- forM pendingChunks $ createChunk numRecipients' storageHours
       let allSrvs = S.fromList $ preparedSrvs <> srvs
       lift $ forM_ allSrvs $ \srv -> getXFTPSndWorker True c (Just srv)
       withStore' c $ \db -> updateSndFileStatus db sndFileId SFSUploading
@@ -454,8 +479,8 @@ runXFTPSndPrepareWorker c Worker {doWork} = do
         srvOrPendingChunk ch@SndFileChunk {replicas} = case replicas of
           [] -> Left ch
           SndFileChunkReplica {server} : _ -> Right server
-        createChunk :: Int -> SndFileChunk -> AM (ProtocolServer 'PXFTP)
-        createChunk numRecipients' ch = do
+        createChunk :: Int -> Maybe Word32 -> SndFileChunk -> AM (ProtocolServer 'PXFTP)
+        createChunk numRecipients' storageHours ch = do
           liftIO $ assertAgentForeground c
           (replica, ProtoServerWithAuth srv _) <- tryCreate
           withStore' c $ \db -> createSndFileReplica db ch replica
@@ -482,7 +507,7 @@ runXFTPSndPrepareWorker c Worker {doWork} = do
               deleted <- withStore' c $ \db -> getSndFileDeleted db sndFileId
               when deleted $ throwE $ FILE NO_FILE
               withNextSrv c userId storageSrvs triedHosts [] $ \srvAuth -> do
-                replica <- agentXFTPNewChunk c ch numRecipients' srvAuth
+                replica <- agentXFTPNewChunk c ch numRecipients' srvAuth storageHours
                 pure (replica, srvAuth)
 
 sndWorkerInternalError :: AgentClient -> DBSndFileId -> SndFileId -> Maybe FilePath -> AgentErrorType -> AM ()
@@ -543,7 +568,7 @@ runXFTPSndWorker c srv Worker {doWork} = do
       notify c sndFileEntityId $ SFPROG uploaded total
       when complete $ do
         (sndDescr, rcvDescrs) <- sndFileToDescrs sf
-        notify c sndFileEntityId $ SFDONE sndDescr rcvDescrs
+        notify c sndFileEntityId $ SFDONE sndDescr rcvDescrs (sndFileExpiresAt chunks)
         lift . forM_ prefixPath $ removePath <=< toFSFilePath
         withStore' c $ \db -> updateSndFileComplete db sndFileId
       where
@@ -577,6 +602,10 @@ runXFTPSndWorker c srv Worker {doWork} = do
           let chunkSize = FileSize $ sndChunkSize ch
               replicas = [FileChunkReplica {server, replicaId, replicaKey}]
           pure FileChunk {chunkNo, digest = chDigest, chunkSize, replicas}
+        sndFileExpiresAt :: [SndFileChunk] -> Maybe GrantedStorageTime
+        sndFileExpiresAt chunks' = fmap minimum $ L.nonEmpty =<< mapM chunkExpiresAt chunks'
+          where
+            chunkExpiresAt SndFileChunk {replicas} = maximum <$> L.nonEmpty (mapMaybe (\SndFileChunkReplica {expiresAt} -> expiresAt) replicas)
         createRcvFileDescriptions :: FileDescription 'FRecipient -> [SndFileChunk] -> [FileDescription 'FRecipient]
         createRcvFileDescriptions fd sndChunks = map (\chunks -> (fd :: (FileDescription 'FRecipient)) {chunks}) rcvChunks
           where

src/Simplex/FileTransfer/Client.hs

@@ -36,8 +36,10 @@ import qualified Control.Exception as E
 import Control.Logger.Simple
 import Control.Monad
 import Control.Monad.Except
+import Control.Monad.IO.Class (liftIO)
 import Control.Monad.Trans.Except
 import Crypto.Random (ChaChaDRG)
+import qualified Data.Aeson as J
 import Data.Bifunctor (first)
 import Data.ByteString.Builder (Builder, byteString)
 import Data.ByteString.Char8 (ByteString)
@@ -57,6 +59,7 @@ import Network.Socket (HostName)
 import Simplex.FileTransfer.Chunks
 import Simplex.FileTransfer.Protocol
 import Simplex.FileTransfer.Transport
+import Simplex.Messaging.Crypto.Entitlement (EntitlementProof)
 import Simplex.Messaging.Client
   ( NetworkConfig (..),
     NetworkRequestMode (..),
@@ -84,7 +87,7 @@ import Simplex.Messaging.Protocol
     SenderId,
     pattern NoEntity,
   )
-import Simplex.Messaging.Transport (ALPN, CertChainPubKey (..), HandshakeError (..), THandleAuth (..), THandleParams (..), TransportError (..), TransportPeer (..), defaultSupportedParams)
+import Simplex.Messaging.Transport (ALPN, CertChainPubKey (..), HandshakeError (..), SessionId, THandleAuth (..), THandleParams (..), TransportError (..), TransportPeer (..), defaultSupportedParams)
 import Simplex.Messaging.Transport.Client (TransportClientConfig (..), TransportHost)
 import Simplex.Messaging.Transport.HTTP2
 import Simplex.Messaging.Transport.HTTP2.Client
@@ -109,12 +112,6 @@ data XFTPClientConfig = XFTPClientConfig
     clientALPN :: Maybe [ALPN]
   }
 
-data XFTPChunkBody = XFTPChunkBody
-  { chunkSize :: Int,
-    chunkPart :: Int -> IO ByteString,
-    http2Body :: HTTP2Body
-  }
-
 data XFTPChunkSpec = XFTPChunkSpec
   { filePath :: FilePath,
     chunkOffset :: Int64,
@@ -132,8 +129,8 @@ defaultXFTPClientConfig =
       clientALPN = Just alpnSupportedXFTPhandshakes
     }
 
-getXFTPClient :: TransportSession FileResponse -> XFTPClientConfig -> [HostName] -> UTCTime -> (XFTPClient -> IO ()) -> IO (Either XFTPClientError XFTPClient)
-getXFTPClient transportSession@(_, srv, _) config@XFTPClientConfig {clientALPN, xftpNetworkConfig, serverVRange} presetDomains proxySessTs disconnected = runExceptT $ do
+getXFTPClient :: TransportSession FileResponse -> XFTPClientConfig -> [HostName] -> UTCTime -> (SessionId -> IO (Maybe EntitlementProof)) -> (XFTPClient -> IO ()) -> IO (Either XFTPClientError XFTPClient)
+getXFTPClient transportSession@(_, srv, _) config@XFTPClientConfig {clientALPN, xftpNetworkConfig, serverVRange} presetDomains proxySessTs mkEntitlementProof disconnected = runExceptT $ do
   let socksCreds = clientSocksCredentials xftpNetworkConfig proxySessTs transportSession
       ProtocolServer _ host port keyHash = srv
       useALPN = if useWebPort xftpNetworkConfig presetDomains srv then Just [httpALPN11] else clientALPN
@@ -147,26 +144,28 @@ getXFTPClient transportSession@(_, srv, _) config@XFTPClientConfig {clientALPN,
   let HTTP2Client {sessionId, sessionALPN} = http2Client
       v = VersionXFTP 1
       thServerVRange = versionToRange v
-      thParams0 = THandleParams {sessionId, blockSize = xftpBlockSize, thVersion = v, thServerVRange, thAuth = Nothing, implySessId = False, encryptBlock = Nothing, batch = True, serviceAuth = False}
+      thParams0 = THandleParams {sessionId, blockSize = xftpBlockSize, thVersion = v, thServerVRange, thAuth = Nothing, implySessId = False, encryptBlock = Nothing, serviceAuth = False, serverInfo = Nothing}
   logDebug $ "Client negotiated handshake protocol: " <> tshow sessionALPN
   thParams@THandleParams {thVersion} <- case sessionALPN of
     Just alpn
       | alpn == xftpALPNv1 || alpn == httpALPN11 ->
-          xftpClientHandshakeV1 serverVRange keyHash http2Client thParams0
+          xftpClientHandshakeV1 serverVRange keyHash http2Client mkEntitlementProof thParams0
     _ -> pure thParams0
   logDebug $ "Client negotiated protocol: " <> tshow thVersion
   let c = XFTPClient {http2Client, thParams, transportSession, config}
   atomically $ writeTVar clientVar $ Just c
   pure c
 
-xftpClientHandshakeV1 :: VersionRangeXFTP -> C.KeyHash -> HTTP2Client -> THandleParamsXFTP 'TClient -> ExceptT XFTPClientError IO (THandleParamsXFTP 'TClient)
-xftpClientHandshakeV1 serverVRange keyHash@(C.KeyHash kh) c@HTTP2Client {sessionId, serverKey} thParams0 = do
-  shs@XFTPServerHandshake {authPubKey = ck} <- getServerHandshake
+xftpClientHandshakeV1 :: VersionRangeXFTP -> C.KeyHash -> HTTP2Client -> (SessionId -> IO (Maybe EntitlementProof)) -> THandleParamsXFTP 'TClient -> ExceptT XFTPClientError IO (THandleParamsXFTP 'TClient)
+xftpClientHandshakeV1 serverVRange keyHash@(C.KeyHash kh) c@HTTP2Client {sessionId, serverKey} mkEntitlementProof thParams0 = do
+  shs@XFTPServerHandshake {authPubKey = ck, serverInfoBytes} <- getServerHandshake
   (vr, sk) <- processServerHandshake shs
   let v = maxVersion vr
-  sendClientHandshake XFTPClientHandshake {xftpVersion = v, keyHash}
+  ep <- if v >= fileStorageTimeXFTPVersion then liftIO (mkEntitlementProof sessionId) else pure Nothing
+  sendClientHandshake XFTPClientHandshake {xftpVersion = v, keyHash, entitlementProof = ep}
   let thAuth = Just THAuthClient {peerServerPubKey = sk, peerServerCertKey = ck, clientService = Nothing, sessSecret = Nothing}
-  pure thParams0 {thAuth, thVersion = v, thServerVRange = vr}
+      serverInfo = J.eitherDecodeStrict' <$> serverInfoBytes
+  pure thParams0 {thAuth, thVersion = v, thServerVRange = vr, serverInfo}
   where
     getServerHandshake :: ExceptT XFTPClientError IO XFTPServerHandshake
     getServerHandshake = do
@@ -259,10 +258,11 @@ createXFTPChunk ::
   FileInfo ->
   NonEmpty C.APublicAuthKey ->
   Maybe BasicAuth ->
-  ExceptT XFTPClientError IO (SenderId, NonEmpty RecipientId)
-createXFTPChunk c spKey file rcps auth_ =
-  sendXFTPCommand c spKey NoEntity (FNEW file rcps auth_) Nothing >>= \case
-    (FRSndIds sId rIds, body) -> noFile body (sId, rIds)
+  Maybe Word32 ->
+  ExceptT XFTPClientError IO (SenderId, NonEmpty RecipientId, Maybe GrantedStorageTime)
+createXFTPChunk c spKey file rcps auth_ storageHours =
+  sendXFTPCommand c spKey NoEntity (FNEW file rcps auth_ storageHours) Nothing >>= \case
+    (FRSndIds sId rIds gs, body) -> noFile body (sId, rIds, gs)
     (r, _) -> throwE $ unexpectedResponse r
 
 addXFTPRecipients :: XFTPClient -> C.APrivateAuthKey -> XFTPFileId -> NonEmpty C.APublicAuthKey -> ExceptT XFTPClientError IO (NonEmpty RecipientId)

src/Simplex/FileTransfer/Client/Agent.hs

@@ -81,7 +81,7 @@ getXFTPServerClient XFTPClientAgent {xftpClients, startedAt, config} srv = do
     connectClient =
       ExceptT $
         first (XFTPClientAgentError srv)
-          <$> getXFTPClient (1, srv, Nothing) (xftpConfig config) [] startedAt clientDisconnected
+          <$> getXFTPClient (1, srv, Nothing) (xftpConfig config) [] startedAt (\_ -> pure Nothing) clientDisconnected
 
     clientDisconnected :: XFTPClient -> IO ()
     clientDisconnected _ = do

src/Simplex/FileTransfer/Client/Main.hs

@@ -328,7 +328,7 @@ cliSendFileOpts SendOptions {filePath, outputDir, numRecipients, xftpServers, re
           digest <- liftIO $ getChunkDigest chunkSpec
           let ch = FileInfo {sndKey, size = chunkSize, digest}
           c <- withRetry retryCount $ getXFTPServerClient a xftpServer
-          (sndId, rIds) <- withRetry retryCount $ createXFTPChunk c spKey ch (L.map fst rKeys) auth
+          (sndId, rIds, _) <- withRetry retryCount $ createXFTPChunk c spKey ch (L.map fst rKeys) auth Nothing
           withReconnect a xftpServer retryCount $ \c' -> uploadXFTPChunk c' spKey sndId chunkSpec
           logDebug $ "uploaded chunk " <> tshow chunkNo
           uploaded <- atomically . stateTVar uploadedChunks $ \cs ->
@@ -441,7 +441,6 @@ cliReceiveFile ReceiveOptions {fileDescription, filePath, retryCount, tempPath,
           | depth > 0 -> do
               CryptoFile tmpFile _ <- withExceptT cliCryptoError $ decryptChunks encSize chunkPaths key nonce $ \_ ->
                 fmap CF.plain $ uniqueCombine encPath "redirect.yaml"
-              forM_ chunks $ acknowledgeFileChunk a
               yaml <- liftIO $ B.readFile tmpFile
               whenM (doesPathExist encPath) $ removeDirectoryRecursive encPath
               innerVfd <- either (throwE . CLIError . ("Redirect: invalid file description: " <>)) pure $ strDecode yaml
@@ -450,7 +449,6 @@ cliReceiveFile ReceiveOptions {fileDescription, filePath, retryCount, tempPath,
         Nothing -> do
           liftIO $ printNoNewLine "Decrypting file..."
           CryptoFile path _ <- withExceptT cliCryptoError $ decryptChunks encSize chunkPaths key nonce $ fmap CF.plain . getFilePath
-          forM_ chunks $ acknowledgeFileChunk a
           whenM (doesPathExist encPath) $ removeDirectoryRecursive encPath
           liftIO $ do
             printNoNewLine $ "File downloaded: " <> path
@@ -479,12 +477,6 @@ cliReceiveFile ReceiveOptions {fileDescription, filePath, retryCount, tempPath,
       _ -> (`uniqueCombine` name') . (</> "Downloads") =<< getHomeDirectory
       where
         name' = T.unpack name
-    acknowledgeFileChunk :: XFTPClientAgent -> FileChunk -> ExceptT CLIError IO ()
-    acknowledgeFileChunk a FileChunk {replicas = replica : _} = do
-      let FileChunkReplica {server, replicaId, replicaKey} = replica
-      c <- withRetry retryCount $ getXFTPServerClient a server
-      withRetry retryCount $ ackXFTPChunk c replicaKey (unChunkReplicaId replicaId)
-    acknowledgeFileChunk _ _ = throwE $ CLIError "chunk has no replicas"
 
 printProgress :: String -> Int64 -> Int64 -> IO ()
 printProgress s part total = printNoNewLine $ s <> " " <> show ((part * 100) `div` total) <> "%"

src/Simplex/FileTransfer/Description.hs

@@ -29,6 +29,7 @@ module Simplex.FileTransfer.Description
     YAMLFileDescription (..), -- for tests
     YAMLServerReplicas (..), -- for tests
     validateFileDescription,
+    sharedDescriptionHash,
     groupReplicasByServer,
     fdSeparator,
     kb,
@@ -72,6 +73,7 @@ import Simplex.FileTransfer.Protocol
 import Simplex.Messaging.Agent.QueryString
 import Simplex.Messaging.Agent.Store.DB (Binary (..), FromField (..), ToField (..))
 import qualified Simplex.Messaging.Crypto as C
+import Simplex.Messaging.Encoding (Encoding (..))
 import Simplex.Messaging.Encoding.String
 import Simplex.Messaging.Parsers (defaultJSON, parseAll)
 import Simplex.Messaging.Protocol (XFTPServer)
@@ -227,6 +229,13 @@ validateFileDescription fd@FileDescription {size, chunks}
     chunkNos = map (\FileChunk {chunkNo} -> chunkNo) chunks
     chunksSize = foldl' (\(s :: Int64) FileChunk {chunkSize} -> s + fromIntegral (unFileSize chunkSize)) 0
 
+sharedDescriptionHash :: FileDescription p -> ByteString
+sharedDescriptionHash FileDescription {size, digest, key, nonce, chunkSize, chunks} =
+  C.sha512Hash $ smpEncode (unFileSize size, unFileDigest digest, C.unSbKey key, nonce, unFileSize chunkSize) <> foldMap chunkFields chunks
+  where
+    chunkFields FileChunk {chunkNo, chunkSize = cSize, digest = cDigest} =
+      smpEncode (fromIntegral chunkNo :: Int64, unFileSize cSize, unFileDigest cDigest)
+
 encodeFileDescription :: FileDescription p -> YAMLFileDescription
 encodeFileDescription FileDescription {party, size, digest, key, nonce, chunkSize, chunks, redirect} =
   YAMLFileDescription

src/Simplex/FileTransfer/Protocol.hs

@@ -22,6 +22,7 @@ module Simplex.FileTransfer.Protocol
     FileCommand (..),
     FileCmd (..),
     FileInfo (..),
+    GrantedStorageTime (..),
     XFTPFileId,
     FileResponse (..),
     xftpBlockSize,
@@ -38,12 +39,13 @@ import qualified Data.Aeson.TH as J
 import Data.Bifunctor (first)
 import Data.ByteString.Char8 (ByteString)
 import qualified Data.ByteString.Char8 as B
+import Data.Int (Int64)
 import Data.Kind (Type)
 import Data.List.NonEmpty (NonEmpty (..))
 import Data.Maybe (isNothing)
 import Data.Type.Equality
 import Data.Word (Word32)
-import Simplex.FileTransfer.Transport (XFTPErrorType (..), XFTPVersion, blockedFilesXFTPVersion, xftpClientHandshakeStub)
+import Simplex.FileTransfer.Transport (XFTPErrorType (..), XFTPVersion, blockedFilesXFTPVersion, fileStorageTimeXFTPVersion, xftpClientHandshakeStub)
 import Simplex.Messaging.Client (authTransmission)
 import qualified Simplex.Messaging.Crypto as C
 import Simplex.Messaging.Encoding
@@ -175,7 +177,7 @@ instance Protocol XFTPVersion XFTPErrorType FileResponse where
   {-# INLINE protocolError #-}
 
 data FileCommand (p :: FileParty) where
-  FNEW :: FileInfo -> NonEmpty RcvPublicAuthKey -> Maybe BasicAuth -> FileCommand FSender
+  FNEW :: FileInfo -> NonEmpty RcvPublicAuthKey -> Maybe BasicAuth -> Maybe Word32 -> FileCommand FSender
   FADD :: NonEmpty RcvPublicAuthKey -> FileCommand FSender
   FPUT :: FileCommand FSender
   FDEL :: FileCommand FSender
@@ -196,12 +198,27 @@ data FileInfo = FileInfo
   }
   deriving (Show)
 
+data GrantedStorageTime = GSTExpires {epochSeconds :: Int64}
+  deriving (Eq, Ord, Show)
+
+instance Encoding GrantedStorageTime where
+  smpEncode = \case
+    GSTExpires t -> smpEncode ('T', t)
+  smpP =
+    smpP >>= \case
+      'T' -> GSTExpires <$> smpP
+      _ -> fail "bad GrantedStorageTime"
+
 type XFTPFileId = EntityId
 
 instance FilePartyI p => ProtocolEncoding XFTPVersion XFTPErrorType (FileCommand p) where
   type Tag (FileCommand p) = FileCommandTag p
-  encodeProtocol _v = \case
-    FNEW file rKeys auth_ -> e (FNEW_, ' ', file, rKeys, auth_)
+  encodeProtocol v = \case
+    FNEW file rKeys auth_ st
+      | v >= fileStorageTimeXFTPVersion -> fnew <> e st
+      | otherwise -> fnew
+      where
+        fnew = e (FNEW_, ' ', file, rKeys, auth_)
     FADD rKeys -> e (FADD_, ' ', rKeys)
     FPUT -> e FPUT_
     FDEL -> e FDEL_
@@ -235,10 +252,14 @@ instance ProtocolEncoding XFTPVersion XFTPErrorType FileCmd where
   type Tag FileCmd = FileCmdTag
   encodeProtocol _v (FileCmd _ c) = encodeProtocol _v c
 
-  protocolP _v = \case
+  protocolP v = \case
     FCT SFSender tag ->
       FileCmd SFSender <$> case tag of
-        FNEW_ -> FNEW <$> _smpP <*> smpP <*> smpP
+        FNEW_
+          | v >= fileStorageTimeXFTPVersion -> fnewP smpP
+          | otherwise -> fnewP (pure Nothing)
+          where
+            fnewP stP = FNEW <$> _smpP <*> smpP <*> smpP <*> stP
         FADD_ -> FADD <$> _smpP
         FPUT_ -> pure FPUT
         FDEL_ -> pure FDEL
@@ -292,7 +313,7 @@ instance ProtocolMsgTag FileResponseTag where
     _ -> Nothing
 
 data FileResponse
-  = FRSndIds SenderId (NonEmpty RecipientId)
+  = FRSndIds SenderId (NonEmpty RecipientId) (Maybe GrantedStorageTime)
   | FRRcvIds (NonEmpty RecipientId)
   | FRFile RcvPublicDhKey C.CbNonce
   | FROk
@@ -303,7 +324,9 @@ data FileResponse
 instance ProtocolEncoding XFTPVersion XFTPErrorType FileResponse where
   type Tag FileResponse = FileResponseTag
   encodeProtocol v = \case
-    FRSndIds fId rIds -> e (FRSndIds_, ' ', fId, rIds)
+    FRSndIds fId rIds gs
+      | v >= fileStorageTimeXFTPVersion -> e (FRSndIds_, ' ', fId, rIds, gs)
+      | otherwise -> e (FRSndIds_, ' ', fId, rIds)
     FRRcvIds rIds -> e (FRRcvIds_, ' ', rIds)
     FRFile rDhKey nonce -> e (FRFile_, ' ', rDhKey, nonce)
     FROk -> e FROk_
@@ -315,8 +338,10 @@ instance ProtocolEncoding XFTPVersion XFTPErrorType FileResponse where
       e :: Encoding a => a -> ByteString
       e = smpEncode
 
-  protocolP _v = \case
-    FRSndIds_ -> FRSndIds <$> _smpP <*> smpP
+  protocolP v = \case
+    FRSndIds_
+      | v >= fileStorageTimeXFTPVersion -> FRSndIds <$> _smpP <*> smpP <*> smpP
+      | otherwise -> FRSndIds <$> _smpP <*> smpP <*> pure Nothing
     FRRcvIds_ -> FRRcvIds <$> _smpP
     FRFile_ -> FRFile <$> _smpP <*> smpP
     FROk_ -> pure FROk

src/Simplex/FileTransfer/Server.hs

@@ -23,18 +23,21 @@ import Control.Monad
 import Control.Monad.Except
 import Control.Monad.Reader
 import Control.Monad.Trans.Except
+import qualified Data.Aeson as J
 import Data.Bifunctor (first)
-import qualified Data.ByteString.Base64.URL as B64
 import Data.ByteString.Builder (Builder, byteString)
 import Data.ByteString.Char8 (ByteString)
 import qualified Data.ByteString.Char8 as B
+import qualified Data.ByteString.Lazy.Char8 as LB
 import Data.Int (Int64)
 import Data.List.NonEmpty (NonEmpty)
 import qualified Data.List.NonEmpty as L
+import qualified Data.Map.Strict as M
 import Data.Maybe (fromMaybe, isJust)
 import qualified Data.Text as T
 import qualified Data.Text.IO as T
 import Data.Time.Clock (UTCTime (..), diffTimeToPicoseconds, getCurrentTime)
+import Data.Time.Clock.System (systemSeconds, utcToSystemTime)
 import Data.Time.Format.ISO8601 (iso8601Show)
 import Data.Word (Word32)
 import qualified Data.X509 as X
@@ -54,10 +57,12 @@ import Simplex.FileTransfer.Server.Store
 import Simplex.FileTransfer.Server.StoreLog
 import Simplex.FileTransfer.Transport
 import qualified Simplex.Messaging.Crypto as C
+import Simplex.Messaging.Crypto.BBS (BBSPresHeader (..))
+import Simplex.Messaging.Crypto.Entitlement (Entitlement (..), EntitlementProof (..), EntitlementVerification (..), verifyEntitlement)
 import qualified Simplex.Messaging.Crypto.Lazy as LC
 import Simplex.Messaging.Encoding
 import Simplex.Messaging.Encoding.String
-import Simplex.Messaging.Protocol (BlockingInfo, EntityId (..), RcvPublicAuthKey, RcvPublicDhKey, RecipientId, SignedTransmission, pattern NoEntity)
+import Simplex.Messaging.Protocol (BlockingInfo, CommandError (..), EntityId (..), RcvPublicAuthKey, RcvPublicDhKey, RecipientId, SignedTransmission, pattern NoEntity)
 import Simplex.Messaging.Server (controlPortAuth, dummyVerifyCmd, verifyCmdAuthorization)
 import Simplex.Messaging.Server.Control (CPClientRole (..))
 import Simplex.Messaging.Server.Expiration
@@ -66,7 +71,7 @@ import Simplex.Messaging.Server.Stats
 import Simplex.Messaging.SystemTime
 import Simplex.Messaging.TMap (TMap)
 import qualified Simplex.Messaging.TMap as TM
-import Simplex.Messaging.Transport (CertChainPubKey (..), SessionId, THandleAuth (..), THandleParams (..), TransportPeer (..), defaultSupportedParams, defaultSupportedParamsHTTPS)
+import Simplex.Messaging.Transport (CertChainPubKey (..), EntitlementConfig (..), SessionEntitlement (..), SessionId, THandleAuth (..), THandleParams (..), TransportPeer (..), defaultSupportedParams, defaultSupportedParamsHTTPS)
 import Simplex.Messaging.Transport.Buffer (trimCR)
 import Simplex.Messaging.Transport.HTTP2
 import Simplex.Messaging.Transport.HTTP2.File (fileBlockSize)
@@ -120,17 +125,20 @@ runXFTPServerBlocking :: FileStoreClass s => TMVar Bool -> XFTPServerConfig s ->
 runXFTPServerBlocking started cfg = newXFTPServerEnv cfg >>= runReaderT (xftpServer cfg started)
 
 data Handshake
-  = HandshakeSent C.PrivateKeyX25519
+  = HandshakeSent C.PrivateKeyX25519 EntitlementChecked
   | HandshakeAccepted (THandleParams XFTPVersion 'TServer)
 
+-- | the entitlement of the session is resolved once, however many handshakes it has
+data EntitlementChecked = EntNotChecked | EntChecked (Maybe SessionEntitlement)
+
 xftpServer :: forall s. FileStoreClass s => XFTPServerConfig s -> TMVar Bool -> M s ()
-xftpServer cfg@XFTPServerConfig {xftpPort, transportConfig, inactiveClientExpiration, fileExpiration, xftpServerVRange} started = do
-  mapM_ (expireServerFiles Nothing) fileExpiration
+xftpServer cfg@XFTPServerConfig {xftpPort, transportConfig, inactiveClientExpiration, fileExpiration, xftpServerVRange, information} started = do
+  expireServerFiles Nothing fileExpiration
   restoreServerStats
   raceAny_
     ( runServer
-        : expireFilesThread_ cfg
-          <> serverStatsThread_ cfg
+        : expireFiles fileExpiration
+        : serverStatsThread_ cfg
           <> prometheusMetricsThread_ cfg
           <> controlPortThread_ cfg
     )
@@ -157,7 +165,7 @@ xftpServer cfg@XFTPServerConfig {xftpPort, transportConfig, inactiveClientExpira
             reqBody <- getHTTP2Body r xftpBlockSize
             let v = VersionXFTP 1
                 thServerVRange = versionToRange v
-                thParams0 = THandleParams {sessionId, blockSize = xftpBlockSize, thVersion = v, thServerVRange, thAuth = Nothing, implySessId = False, encryptBlock = Nothing, batch = True, serviceAuth = False}
+                thParams0 = THandleParams {sessionId, blockSize = xftpBlockSize, thVersion = v, thServerVRange, thAuth = Nothing, implySessId = False, encryptBlock = Nothing, serviceAuth = False, serverInfo = Nothing}
                 req0 = XFTPTransportRequest {thParams = thParams0, request = r, reqBody, sendResponse, sniUsed, addCORS = addCORS'}
             flip runReaderT env $ case sessionALPN of
               Nothing -> processRequest req0
@@ -174,12 +182,12 @@ xftpServer cfg@XFTPServerConfig {xftpPort, transportConfig, inactiveClientExpira
         Nothing
           | sniUsed && not webHello -> throwE SESSION
           | otherwise -> processHello Nothing
-        Just (HandshakeSent pk)
+        Just (HandshakeSent pk ent_)
           | webHello -> processHello (Just pk)
-          | otherwise -> processClientHandshake pk
+          | otherwise -> processClientHandshake pk ent_
         Just (HandshakeAccepted thParams)
           | webHello -> processHello (serverPrivKey <$> thAuth thParams)
-          | webHandshake, Just auth <- thAuth thParams -> processClientHandshake (serverPrivKey auth)
+          | webHandshake, Just auth <- thAuth thParams -> processClientHandshake (serverPrivKey auth) (EntChecked $ peerEntitlement auth)
           | otherwise -> pure $ Just thParams
       either sendError pure r
       where
@@ -196,28 +204,35 @@ xftpServer cfg@XFTPServerConfig {xftpPort, transportConfig, inactiveClientExpira
               | otherwise -> throwE HANDSHAKE
           rng <- asks random
           k <- atomically $ TM.lookup sessionId sessions >>= \case
-            Just (HandshakeSent pk') -> pure $ C.publicKey pk'
-            _ -> do
+            Just (HandshakeSent pk' _) -> pure $ C.publicKey pk'
+            s' -> do
               kp <- maybe (C.generateKeyPair rng) (\p -> pure (C.publicKey p, p)) pk_
-              fst kp <$ TM.insert sessionId (HandshakeSent $ snd kp) sessions
+              let ent_ = case s' of
+                    Just (HandshakeAccepted thParams) -> EntChecked $ peerEntitlement =<< thAuth thParams
+                    _ -> EntNotChecked
+              fst kp <$ TM.insert sessionId (HandshakeSent (snd kp) ent_) sessions
           let authPubKey = CertChainPubKey chain (C.signX509 serverSignKey $ C.publicToX509 k)
               webIdentityProof = C.sign serverSignKey . (<> sessionId) <$> challenge_
-          let hs = XFTPServerHandshake {xftpVersionRange = xftpServerVRange, sessionId, authPubKey, webIdentityProof}
+              serverInfoBytes = LB.toStrict . J.encode <$> information
+          let hs = XFTPServerHandshake {xftpVersionRange = xftpServerVRange, sessionId, authPubKey, webIdentityProof, serverInfoBytes}
           shs <- encodeXftp hs
 #ifdef slow_servers
           lift randomDelay
 #endif
           liftIO . sendResponse $ H.responseBuilder N.ok200 (corsHeaders addCORS) shs
           pure Nothing
-        processClientHandshake pk = do
+        processClientHandshake pk ent_ = do
           unless (B.length bodyHead == xftpBlockSize) $ throwE HANDSHAKE
           body <- liftHS $ C.unPad bodyHead
-          XFTPClientHandshake {xftpVersion = v, keyHash} <- liftHS $ smpDecode body
+          XFTPClientHandshake {xftpVersion = v, keyHash, entitlementProof} <- liftHS $ smpDecode body
           kh <- asks serverIdentity
           unless (keyHash == kh) $ throwE HANDSHAKE
           case compatibleVRange' xftpServerVRange v of
             Just (Compatible vr) -> do
-              let auth = THAuthServer {serverPrivKey = pk, peerClientService = Nothing, sessSecret' = Nothing}
+              ent <- case ent_ of
+                EntChecked ent -> pure ent
+                EntNotChecked -> lift $ verifiedEntitlement entitlementProof
+              let auth = THAuthServer {serverPrivKey = pk, peerClientService = Nothing, peerEntitlement = ent, sessSecret' = Nothing}
                   thParams = thParams0 {thAuth = Just auth, thVersion = v, thServerVRange = vr}
               atomically $ TM.insert sessionId (HandshakeAccepted thParams) sessions
 #ifdef slow_servers
@@ -226,6 +241,19 @@ xftpServer cfg@XFTPServerConfig {xftpPort, transportConfig, inactiveClientExpira
               liftIO . sendResponse $ H.responseNoBody N.ok200 (corsHeaders addCORS)
               pure Nothing
             Nothing -> throwE HANDSHAKE
+        verifiedEntitlement :: Maybe EntitlementProof -> M s (Maybe SessionEntitlement)
+        verifiedEntitlement ep =
+          pure ep $>>= \proof@EntitlementProof {entitlement = Entitlement {entitlementName, expiresAt = expiresAtTs}} -> do
+            entCfg <- asks $ M.lookup entitlementName . fileStorageEntitlements . config
+            now <- liftIO getSystemSeconds
+            let expiresAt = RoundedSystemTime $ systemSeconds $ utcToSystemTime expiresAtTs
+            case entCfg of
+              Just cfg | entitlementValid now expiresAt -> do
+                keys <- asks $ entitlementKeys . config
+                liftIO (verifyEntitlement keys (BBSPresHeader sessionId) proof) >>= \case
+                  EVValid -> pure $ Just SessionEntitlement {expiresAt, entConfig = cfg}
+                  r -> Nothing <$ logError ("entitlement not verified: " <> tshow r)
+              _ -> pure Nothing
         sendError :: XFTPErrorType -> M s (Maybe (THandleParams XFTPVersion 'TServer))
         sendError err = do
           runExceptT (encodeXftp err) >>= \case
@@ -243,10 +271,6 @@ xftpServer cfg@XFTPServerConfig {xftpPort, transportConfig, inactiveClientExpira
       saveServerStats
       logNote "Server stopped"
 
-    expireFilesThread_ :: XFTPServerConfig s -> [M s ()]
-    expireFilesThread_ XFTPServerConfig {fileExpiration = Just fileExp} = [expireFiles fileExp]
-    expireFilesThread_ _ = []
-
     expireFiles :: ExpirationConfig -> M s ()
     expireFiles expCfg = do
       let interval = checkInterval expCfg * 1000000
@@ -402,8 +426,9 @@ processRequest XFTPTransportRequest {thParams, reqBody = body@HTTP2Body {bodyHea
       case xftpDecodeTServer thParams bodyHead of
         Right (Right t@(_, _, (corrId, fId, _))) -> do
           let THandleParams {thAuth} = thParams
+              ent = peerEntitlement =<< thAuth
           verifyXFTPTransmission thAuth t >>= \case
-            VRVerified req -> uncurry send =<< processXFTPRequest body req
+            VRVerified req -> uncurry send =<< processXFTPRequest ent body req
             VRFailed e -> send (FRErr e) Nothing
           where
             send resp = sendXFTPResponse (corrId, fId, resp)
@@ -443,7 +468,7 @@ data VerificationResult = VRVerified XFTPRequest | VRFailed XFTPErrorType
 verifyXFTPTransmission :: forall s. FileStoreClass s => Maybe (THandleAuth 'TServer) -> SignedTransmission FileCmd -> M s VerificationResult
 verifyXFTPTransmission thAuth (tAuth, authorized, (corrId, fId, cmd)) =
   case cmd of
-    FileCmd SFSender (FNEW file rcps auth') -> pure $ XFTPReqNew file rcps auth' `verifyWith` sndKey file
+    FileCmd SFSender (FNEW file rcps auth' st) -> pure $ XFTPReqNew file rcps auth' st `verifyWith` sndKey file
     FileCmd SFRecipient PING -> pure $ VRVerified XFTPReqPing
     FileCmd party _ -> verifyCmd party
   where
@@ -464,9 +489,9 @@ verifyXFTPTransmission thAuth (tAuth, authorized, (corrId, fId, cmd)) =
     -- TODO verify with DH authorization
     req `verifyWith` k = if verifyCmdAuthorization thAuth tAuth authorized corrId k then VRVerified req else VRFailed AUTH
 
-processXFTPRequest :: forall s. FileStoreClass s => HTTP2Body -> XFTPRequest -> M s (FileResponse, Maybe ServerFile)
-processXFTPRequest HTTP2Body {bodyPart} = \case
-  XFTPReqNew file rks auth -> noFile =<< ifM allowNew (createFile file rks) (pure $ FRErr AUTH)
+processXFTPRequest :: forall s. FileStoreClass s => Maybe SessionEntitlement -> HTTP2Body -> XFTPRequest -> M s (FileResponse, Maybe ServerFile)
+processXFTPRequest ent HTTP2Body {bodyPart} = \case
+  XFTPReqNew file rks auth storageHours -> noFile =<< ifM allowNew (createFile file rks storageHours) (pure $ FRErr AUTH)
     where
       allowNew = do
         XFTPServerConfig {allowNewFiles, newFileBasicAuth} <- asks config
@@ -483,29 +508,42 @@ processXFTPRequest HTTP2Body {bodyPart} = \case
   XFTPReqPing -> noFile FRPong
   where
     noFile resp = pure (resp, Nothing)
-    createFile :: FileInfo -> NonEmpty RcvPublicAuthKey -> M s FileResponse
-    createFile file rks = do
+    createFile :: FileInfo -> NonEmpty RcvPublicAuthKey -> Maybe Word32 -> M s FileResponse
+    createFile file rks storageHours = do
       st <- asks fileStore
       r <- runExceptT $ do
         sizes <- asks $ allowedChunkSizes . config
         unless (size file `elem` sizes) $ throwE SIZE
-        ts <- liftIO getFileTime
+        now <- liftIO getSystemSeconds
+        maxSeconds <- lift $ storageMaxSeconds now
+        let nowSeconds = roundedSeconds now
+            ts = RoundedSystemTime $ (nowSeconds `div` fileTimePrecision) * fileTimePrecision
+            secs = case storageHours of
+              Just hours | hours > 0 -> min maxSeconds (fromIntegral hours * 3600)
+              _ -> maxSeconds
+            fileExpiresAt = RoundedSystemTime $ ((nowSeconds + secs + fileTimePrecision - 1) `div` fileTimePrecision) * fileTimePrecision
         -- TODO validate body empty
-        sId <- ExceptT $ addFileRetry st file 3 ts
+        sId <- ExceptT $ addFileRetry st file 3 ts (Just fileExpiresAt)
         rcps <- mapM (ExceptT . addRecipientRetry st 3 sId) rks
         lift $ withFileLog $ \sl -> do
-          logAddFile sl sId file ts EntityActive
+          logAddFile sl sId file ts (Just fileExpiresAt) EntityActive
           logAddRecipients sl sId rcps
         stats <- asks serverStats
         lift $ incFileStat filesCreated
         liftIO $ atomicModifyIORef'_ (fileRecipients stats) (+ length rks)
         let rIds = L.map (\(FileRecipient rId _) -> rId) rcps
-        pure $ FRSndIds sId rIds
+        pure $ FRSndIds sId rIds (Just (GSTExpires (roundedSeconds fileExpiresAt)))
       pure $ either FRErr id r
-    addFileRetry :: s -> FileInfo -> Int -> RoundedFileTime -> M s (Either XFTPErrorType XFTPFileId)
-    addFileRetry st file n ts =
+    storageMaxSeconds :: SystemSeconds -> M s Int64
+    storageMaxSeconds now = do
+      defaultMax <- asks $ ttl . fileExpiration . config
+      pure $ case ent of
+        Just SessionEntitlement {expiresAt, entConfig} | entitlementValid now expiresAt -> max (storageTime entConfig) defaultMax
+        _ -> defaultMax
+    addFileRetry :: s -> FileInfo -> Int -> RoundedFileTime -> Maybe RoundedFileTime -> M s (Either XFTPErrorType XFTPFileId)
+    addFileRetry st file n ts expiresAt =
       retryAdd n $ \sId -> runExceptT $ do
-        ExceptT $ addFile st sId file ts EntityActive
+        ExceptT $ addFile st sId file ts expiresAt EntityActive
         pure sId
     addRecipientRetry :: s -> Int -> XFTPFileId -> RcvPublicAuthKey -> M s (Either XFTPErrorType FileRecipient)
     addRecipientRetry st n sId rpk =
@@ -555,7 +593,7 @@ processXFTPRequest HTTP2Body {bodyPart} = \case
               \used -> let used' = used + fromIntegral size in if used' <= quota then (True, used') else (False, used)
           receive = do
             path <- asks $ filesPath . config
-            let fPath = path </> B.unpack (B64.encode $ unEntityId senderId)
+            let fPath = path </> serverFileName senderId
             receiveChunk (XFTPRcvChunkSpec fPath size digest) >>= \case
               Right () -> do
                 stats <- asks serverStats
@@ -607,11 +645,12 @@ processXFTPRequest HTTP2Body {bodyPart} = \case
 
     ackFileReception :: RecipientId -> FileRec -> M s FileResponse
     ackFileReception rId fr = do
-      withFileLog (`logAckFile` rId)
-      st <- asks fileStore
-      liftIO $ deleteRecipient st rId fr
-      incFileStat fileDownloadAcks
-      pure FROk
+      pure $ FRErr $ CMD PROHIBITED
+      -- withFileLog (`logAckFile` rId)
+      -- st <- asks fileStore
+      -- liftIO $ deleteRecipient st rId fr
+      -- incFileStat fileDownloadAcks
+      -- pure FROk
 
 deleteServerFile_ :: FileStoreClass s => FileRec -> M s (Either XFTPErrorType ())
 deleteServerFile_ fr@FileRec {senderId} = do
@@ -640,24 +679,25 @@ deleteOrBlockServerFile_ FileRec {filePath, fileInfo} stat storeAction = runExce
       liftIO $ atomicModifyIORef'_ (filesCount stats) (subtract 1)
       liftIO $ atomicModifyIORef'_ (filesSize stats) (subtract $ fromIntegral $ size fileInfo)
 
-getFileTime :: IO RoundedFileTime
-getFileTime = getRoundedSystemTime
+entitlementValid :: SystemSeconds -> SystemSeconds -> Bool
+entitlementValid now expiresAt = roundedSeconds expiresAt + 86400 > roundedSeconds now
 
 expireServerFiles :: FileStoreClass s => Maybe Int -> ExpirationConfig -> M s ()
 expireServerFiles itemDelay expCfg = do
   st <- asks fileStore
   us <- asks usedStorage
   usedStart <- readTVarIO us
+  now <- liftIO getSystemSeconds
   old <- liftIO $ expireBeforeEpoch expCfg
   filesCount <- liftIO $ getFileCount st
   logNote $ "Expiration check: " <> tshow filesCount <> " files"
-  expireLoop st us old
+  expireLoop st us now old
   usedEnd <- readTVarIO us
   logNote $ "Used " <> mbs usedStart <> " -> " <> mbs usedEnd <> ", " <> mbs (usedStart - usedEnd) <> " reclaimed."
   where
     mbs bs = tshow (bs `div` 1048576) <> "mb"
-    expireLoop st us old = do
-      expired <- liftIO $ expiredFiles st old 10000
+    expireLoop st us now old = do
+      expired <- liftIO $ expiredFiles st now old 10000
       forM_ expired $ \(sId, filePath_, fileSize) -> do
         mapM_ threadDelay itemDelay
         forM_ filePath_ $ \fp ->
@@ -670,7 +710,7 @@ expireServerFiles itemDelay expCfg = do
       unless (null sIds) $ do
         withFileLog $ \sl -> mapM_ (logDeleteFile sl) sIds
         liftIO $ deleteFiles st sIds
-        expireLoop st us old
+        expireLoop st us now old
 
 randomId :: Int -> M s ByteString
 randomId n = atomically . C.randomBytes n =<< asks random

src/Simplex/FileTransfer/Server/Env.hs

@@ -37,8 +37,11 @@ import Control.Monad
 import Crypto.Random
 import Data.Int (Int64)
 import Data.List.NonEmpty (NonEmpty)
+import Data.Map.Strict (Map)
+import qualified Data.Map.Strict as M
+import Data.Text (Text)
 import Data.Time.Clock (getCurrentTime)
-import Data.Word (Word32)
+import Data.Word (Word16, Word32)
 import Data.X509.Validation (Fingerprint (..))
 import Network.Socket
 import qualified Network.TLS as T
@@ -62,8 +65,11 @@ import System.Directory (doesFileExist)
 import Simplex.FileTransfer.Server.StoreLog
 import Simplex.FileTransfer.Transport (VersionRangeXFTP)
 import qualified Simplex.Messaging.Crypto as C
+import Simplex.Messaging.Crypto.BBS (BBSPublicKey)
 import Simplex.Messaging.Protocol (BasicAuth, RcvPublicAuthKey)
 import Simplex.Messaging.Server.Expiration
+import Simplex.Messaging.Server.Information (ServerPublicInfo)
+import Simplex.Messaging.Transport (EntitlementConfig (..))
 import Simplex.Messaging.Transport.Server (ServerCredentials (..), TransportServerConfig (..), loadFingerprint, loadServerCredential)
 import Simplex.Messaging.Util (tshow)
 import System.IO (IOMode (..))
@@ -88,7 +94,10 @@ data XFTPServerConfig s = XFTPServerConfig
     controlPortUserAuth :: Maybe BasicAuth,
     controlPortAdminAuth :: Maybe BasicAuth,
     -- | time after which the files can be removed and check interval, seconds
-    fileExpiration :: Maybe ExpirationConfig,
+    fileExpiration :: ExpirationConfig,
+    -- | what each entitlement name grants
+    fileStorageEntitlements :: Map Text EntitlementConfig,
+    entitlementKeys :: Map Word16 BBSPublicKey,
     -- | timeout to receive file
     fileTimeout :: Int,
     -- | time after which inactive clients can be disconnected and check interval, seconds
@@ -97,6 +106,8 @@ data XFTPServerConfig s = XFTPServerConfig
     httpCredentials :: Maybe ServerCredentials,
     -- | XFTP client-server protocol version range
     xftpServerVRange :: VersionRangeXFTP,
+    -- | server public information sent in handshake and used to generate static mini-site
+    information :: Maybe ServerPublicInfo,
     -- stats config - see SMP server config
     logStatsInterval :: Maybe Int64,
     logStatsStartTime :: Int64,
@@ -171,7 +182,13 @@ defaultFileExpiration =
     }
 
 newXFTPServerEnv :: FileStoreClass s => XFTPServerConfig s -> IO (XFTPEnv s)
-newXFTPServerEnv config@XFTPServerConfig {serverStoreCfg, fileSizeQuota, xftpCredentials, httpCredentials} = do
+newXFTPServerEnv config@XFTPServerConfig {serverStoreCfg, fileSizeQuota, fileExpiration, fileStorageEntitlements, xftpCredentials, httpCredentials} = do
+  let defaultMax = ttl fileExpiration
+      belowDefault = M.filter ((< defaultMax) . storageTime) fileStorageEntitlements
+  unless (M.null belowDefault) $ do
+    forM_ (M.assocs belowDefault) $ \(name, EntitlementConfig {storageTime}) ->
+      logError $ "expire_files_hours_for_" <> name <> " is " <> tshow (storageTime `div` 3600) <> " hours, below expire_files_hours " <> tshow (defaultMax `div` 3600) <> " hours, server not started"
+    exitFailure
   random <- C.newRandom
   (store, storeLog) <- case serverStoreCfg of
     XSCMemory storeLogPath -> do
@@ -196,7 +213,7 @@ newXFTPServerEnv config@XFTPServerConfig {serverStoreCfg, fileSizeQuota, xftpCre
   pure XFTPEnv {config, store, usedStorage, storeLog, random, tlsServerCreds, httpServerCreds, serverIdentity = C.KeyHash fp, serverStats}
 
 data XFTPRequest
-  = XFTPReqNew FileInfo (NonEmpty RcvPublicAuthKey) (Maybe BasicAuth)
+  = XFTPReqNew FileInfo (NonEmpty RcvPublicAuthKey) (Maybe BasicAuth) (Maybe Word32)
   | XFTPReqCmd XFTPFileId FileRec FileCmd
   | XFTPReqPing
 

src/Simplex/FileTransfer/Server/Main.hs

@@ -6,6 +6,7 @@
 {-# LANGUAGE OverloadedStrings #-}
 {-# LANGUAGE PatternSynonyms #-}
 {-# LANGUAGE RankNTypes #-}
+{-# LANGUAGE TupleSections #-}
 {-# LANGUAGE TypeApplications #-}
 
 module Simplex.FileTransfer.Server.Main
@@ -13,14 +14,16 @@ module Simplex.FileTransfer.Server.Main
     xftpServerCLI_,
   ) where
 
-import Control.Monad (unless, when)
+import Control.Monad (forM_, unless, when)
 import Data.Either (fromRight)
 import Data.Functor (($>))
-import Data.Ini (lookupValue, readIniFile)
+import Data.Ini (Ini, lookupValue, readIniFile)
 import Data.Int (Int64)
 import Data.List (find)
 import qualified Data.List.NonEmpty as L
-import Data.Maybe (fromMaybe, isJust)
+import Data.Map.Strict (Map)
+import qualified Data.Map.Strict as M
+import Data.Maybe (fromMaybe, isJust, mapMaybe)
 import Data.Text.Encoding (encodeUtf8)
 import qualified Data.Text as T
 import qualified Data.Text.IO as T
@@ -32,6 +35,7 @@ import Simplex.FileTransfer.Server (runXFTPServer)
 import Simplex.FileTransfer.Server.Env (XFTPServerConfig (..), XFTPStoreConfig, AFStoreType (..), defFileExpirationHours, defaultFileExpiration, defaultInactiveClientExpiration, readFileStoreType, runWithStoreConfig, checkFileStoreMode, importToDatabase, exportFromDatabase)
 import Simplex.FileTransfer.Transport (alpnSupportedXFTPhandshakes, supportedFileServerVRange)
 import qualified Simplex.Messaging.Crypto as C
+import Simplex.Messaging.Crypto.Entitlement (entitlementIssuerKeys)
 import Simplex.Messaging.Encoding.String
 import Simplex.Messaging.Protocol (ProtoServerWithAuth (..), pattern XFTPServer)
 import Simplex.Messaging.Agent.Store.Shared (MigrationConfirmation (..))
@@ -40,6 +44,7 @@ import Simplex.Messaging.Server.Expiration
 import Simplex.Messaging.Server.Information (ServerPublicInfo (..))
 import Simplex.Messaging.Server.Main (serverPublicInfo, printSourceCode)
 import Simplex.Messaging.Server.Web (EmbeddedWebParams (..), WebHttpsParams (..))
+import Simplex.Messaging.Transport (EntitlementConfig (..))
 import Simplex.Messaging.Transport.Client (TransportHost (..))
 import Simplex.Messaging.Transport.HTTP2 (httpALPN)
 import Simplex.Messaging.Transport.Server (ServerCredentials (..), TransportServerConfig (..), mkTransportServerConfig)
@@ -155,8 +160,14 @@ xftpServerCLI_ generateSite serveStaticFiles cfgPath logPath = do
                \# db_pool_size = 10\n\n\
                \# Write database changes to store log file\n\
                \# db_store_log = off\n\n"
-            <> "# Expire files after the specified number of hours.\n"
+            <> "# Expire files after the specified number of hours.\n\
+               \# The change only affects new files.\n"
             <> ("expire_files_hours = " <> tshow defFileExpirationHours <> "\n\n")
+            <> "# Expire files after the specified number of hours for the senders that present\n\
+               \# a proof of the entitlement. Must not be below expire_files_hours.\n\
+               \# The change only affects new files.\n\
+               \# expire_files_hours_for_supporter = 168\n\
+               \# expire_files_hours_for_legend = 504\n\n"
             <> "log_stats = off\n\
                \\n\
                \# Log interval for real-time Prometheus metrics\n\
@@ -235,13 +246,13 @@ xftpServerCLI_ generateSite serveStaticFiles cfgPath logPath = do
         enableStoreLog = settingIsOn "STORE_LOG" "enable" ini
         logStats = settingIsOn "STORE_LOG" "log_stats" ini
         c = combine cfgPath . ($ defaultX509Config)
-        printXFTPConfig XFTPServerConfig {allowNewFiles, newFileBasicAuth, xftpPort, storeLogFile, fileExpiration, inactiveClientExpiration} = do
+        printXFTPConfig XFTPServerConfig {allowNewFiles, newFileBasicAuth, xftpPort, storeLogFile, fileExpiration, fileStorageEntitlements, inactiveClientExpiration} = do
           putStrLn $ case storeLogFile of
             Just f -> "Store log: " <> f
             _ -> "Store log disabled."
-          putStrLn $ case fileExpiration of
-            Just ExpirationConfig {ttl} -> "expiring files after " <> showTTL ttl
-            _ -> "not expiring files"
+          putStrLn $ "expiring files after " <> showTTL (ttl fileExpiration)
+          forM_ (M.assocs fileStorageEntitlements) $ \(name, EntitlementConfig {storageTime}) ->
+            putStrLn $ "expiring files of " <> T.unpack name <> " after " <> showTTL storageTime
           putStrLn $ case inactiveClientExpiration of
             Just ExpirationConfig {ttl, checkInterval} -> "expiring clients inactive for " <> show ttl <> " seconds every " <> show checkInterval <> " seconds"
             _ -> "not expiring inactive clients"
@@ -287,10 +298,11 @@ xftpServerCLI_ generateSite serveStaticFiles cfgPath logPath = do
               controlPortAdminAuth = either error id <$> strDecodeIni "AUTH" "control_port_admin_password" ini,
               controlPortUserAuth = either error id <$> strDecodeIni "AUTH" "control_port_user_password" ini,
               fileExpiration =
-                Just
-                  defaultFileExpiration
-                    { ttl = 3600 * readIniDefault defFileExpirationHours "STORE_LOG" "expire_files_hours" ini
-                    },
+                defaultFileExpiration
+                  { ttl = 3600 * readIniDefault defFileExpirationHours "STORE_LOG" "expire_files_hours" ini
+                  },
+              fileStorageEntitlements = iniEntitlements ini,
+              entitlementKeys = entitlementIssuerKeys,
               fileTimeout = 5 * 60 * 1000000, -- 5 mins to send 4mb chunk
               inactiveClientExpiration =
                 settingIsOn "INACTIVE_CLIENTS" "disconnect" ini
@@ -306,6 +318,7 @@ xftpServerCLI_ generateSite serveStaticFiles cfgPath logPath = do
                   },
               httpCredentials = httpCredentials_,
               xftpServerVRange = supportedFileServerVRange,
+              information = serverPublicInfo ini,
               logStatsInterval = logStats $> 86400, -- seconds
               logStatsStartTime = 0, -- seconds from 00:00 UTC
               serverStatsLogFile = combine logPath "file-server-stats.daily.log",
@@ -437,3 +450,10 @@ cliCommandP cfgPath logPath iniFile =
         ( command "import" (info (pure SCImport) (progDesc "Import store log file into PostgreSQL database"))
             <> command "export" (info (pure SCExport) (progDesc "Export PostgreSQL database to store log file"))
         )
+
+iniEntitlements :: Ini -> Map T.Text EntitlementConfig
+iniEntitlements ini =
+  M.fromList $ mapMaybe readEntitlement [("supporter", "expire_files_hours_for_supporter"), ("legend", "expire_files_hours_for_legend")]
+  where
+    readEntitlement (name, key) = (name,) . EntitlementConfig . parseHours key <$> eitherToMaybe (lookupValue "STORE_LOG" key ini)
+    parseHours key t = maybe (error $ "Error: invalid " <> T.unpack key <> " value: " <> T.unpack t) (3600 *) (readMaybe (T.unpack (T.strip t)) :: Maybe Int64)

src/Simplex/FileTransfer/Server/Store.hs

@@ -55,6 +55,7 @@ data FileRec = FileRec
     filePath :: TVar (Maybe FilePath),
     recipientIds :: TVar (Set RecipientId),
     createdAt :: RoundedFileTime,
+    expiresAt :: Maybe RoundedFileTime,
     fileStatus :: TVar ServerEntityStatus
   }
 
@@ -74,7 +75,7 @@ class FileStoreClass s where
   type FileStoreConfig s
   newFileStore :: FileStoreConfig s -> IO s
   closeFileStore :: s -> IO ()
-  addFile :: s -> SenderId -> FileInfo -> RoundedFileTime -> ServerEntityStatus -> IO (Either XFTPErrorType ())
+  addFile :: s -> SenderId -> FileInfo -> RoundedFileTime -> Maybe RoundedFileTime -> ServerEntityStatus -> IO (Either XFTPErrorType ())
   setFilePath :: s -> SenderId -> FilePath -> IO (Either XFTPErrorType ())
   addRecipient :: s -> SenderId -> FileRecipient -> IO (Either XFTPErrorType ())
   deleteFile :: s -> SenderId -> IO (Either XFTPErrorType ())
@@ -84,7 +85,7 @@ class FileStoreClass s where
   deleteRecipient :: s -> RecipientId -> FileRec -> IO ()
   getFile :: s -> SFileParty p -> XFTPFileId -> IO (Either XFTPErrorType (FileRec, C.APublicAuthKey))
   ackFile :: s -> RecipientId -> IO (Either XFTPErrorType ())
-  expiredFiles :: s -> Int64 -> Int -> IO [(SenderId, Maybe FilePath, Word32)]
+  expiredFiles :: s -> SystemSeconds -> Int64 -> Int -> IO [(SenderId, Maybe FilePath, Word32)]
   getUsedStorage :: s -> IO Int64
   getFileCount :: s -> IO Int
 
@@ -107,9 +108,9 @@ instance FileStoreClass STMFileStore where
 
   closeFileStore STMFileStore {stmStoreLog} = readTVarIO stmStoreLog >>= mapM_ closeStoreLog
 
-  addFile STMFileStore {files} sId fileInfo createdAt status = atomically $
+  addFile STMFileStore {files} sId fileInfo createdAt expiresAt status = atomically $
     ifM (TM.member sId files) (pure $ Left DUPLICATE_) $ do
-      f <- newFileRec sId fileInfo createdAt status
+      f <- newFileRec sId fileInfo createdAt expiresAt status
       TM.insert sId f files
       pure $ Right ()
 
@@ -166,14 +167,17 @@ instance FileStoreClass STMFileStore where
           pure $ Right ()
       _ -> pure $ Left AUTH
 
-  expiredFiles STMFileStore {files} old _limit = do
+  expiredFiles STMFileStore {files} now old _limit = do
     fs <- readTVarIO files
-    fmap catMaybes . forM (M.toList fs) $ \(sId, FileRec {fileInfo = FileInfo {size}, filePath, createdAt = RoundedSystemTime createdAt}) ->
-      if createdAt + fileTimePrecision < old
-        then do
-          path <- readTVarIO filePath
-          pure $ Just (sId, path, size)
-        else pure Nothing
+    fmap catMaybes . forM (M.toList fs) $ \(sId, FileRec {fileInfo = FileInfo {size}, filePath, createdAt = RoundedSystemTime createdAt, expiresAt}) ->
+      let expired = case expiresAt of
+            Just e -> roundedSeconds e < roundedSeconds now
+            Nothing -> createdAt + fileTimePrecision < old
+       in if expired
+            then do
+              path <- readTVarIO filePath
+              pure $ Just (sId, path, size)
+            else pure Nothing
 
   getUsedStorage STMFileStore {files} = foldM addSize 0 =<< readTVarIO files
     where
@@ -184,12 +188,12 @@ instance FileStoreClass STMFileStore where
 
 -- Internal STM helpers
 
-newFileRec :: SenderId -> FileInfo -> RoundedFileTime -> ServerEntityStatus -> STM FileRec
-newFileRec senderId fileInfo createdAt status = do
+newFileRec :: SenderId -> FileInfo -> RoundedFileTime -> Maybe RoundedFileTime -> ServerEntityStatus -> STM FileRec
+newFileRec senderId fileInfo createdAt expiresAt status = do
   recipientIds <- newTVar S.empty
   filePath <- newTVar Nothing
   fileStatus <- newTVar status
-  pure FileRec {senderId, fileInfo, filePath, recipientIds, createdAt, fileStatus}
+  pure FileRec {senderId, fileInfo, filePath, recipientIds, createdAt, expiresAt, fileStatus}
 
 withFile :: STMFileStore -> SenderId -> (FileRec -> STM (Either XFTPErrorType a)) -> STM (Either XFTPErrorType a)
 withFile STMFileStore {files} sId a =

src/Simplex/FileTransfer/Server/Store/Postgres.hs

@@ -55,6 +55,7 @@ import Simplex.Messaging.Transport (EntityId (..))
 import Simplex.Messaging.Server.QueueStore (ServerEntityStatus (..))
 import Simplex.Messaging.Server.QueueStore.Postgres ()
 import Simplex.Messaging.Server.StoreLog (openWriteStoreLog)
+import Simplex.Messaging.SystemTime (roundedSeconds)
 import Simplex.Messaging.Util (firstRow, tshow)
 import System.Directory (renameFile)
 import System.Exit (exitFailure)
@@ -82,17 +83,17 @@ instance FileStoreClass PostgresFileStore where
     closeDBStore dbStore
     mapM_ closeStoreLog dbStoreLog
 
-  addFile st sId fileInfo@FileInfo {sndKey, size, digest} createdAt status =
+  addFile st sId fileInfo@FileInfo {sndKey, size, digest} createdAt expiresAt status =
     E.uninterruptibleMask_ $ runExceptT $ do
       void $ withDB "addFile" st $ \db ->
         E.try
           ( DB.execute
               db
-              "INSERT INTO files (sender_id, file_size, file_digest, sender_key, created_at, status) VALUES (?,?,?,?,?,?)"
-              (sId, (fromIntegral size :: Int32), Binary digest, Binary (C.encodePubKey sndKey), createdAt, status)
+              "INSERT INTO files (sender_id, file_size, file_digest, sender_key, created_at, expires_at, status) VALUES (?,?,?,?,?,?,?)"
+              (sId, (fromIntegral size :: Int32), Binary digest, Binary (C.encodePubKey sndKey), createdAt, expiresAt, status)
           )
           >>= either handleDuplicate (pure . Right)
-      withLog "addFile" st $ \s -> logAddFile s sId fileInfo createdAt status
+      withLog "addFile" st $ \s -> logAddFile s sId fileInfo createdAt expiresAt status
 
   setFilePath st sId fPath = E.uninterruptibleMask_ $ runExceptT $ do
     assertUpdated $ withDB' "setFilePath" st $ \db ->
@@ -131,13 +132,13 @@ instance FileStoreClass PostgresFileStore where
 
   getFile st party fId = runExceptT $ case party of
     SFSender -> do
-      row <- loadFileRow "SELECT sender_id, file_size, file_digest, sender_key, file_path, created_at, status FROM files WHERE sender_id = ?"
+      row <- loadFileRow "SELECT sender_id, file_size, file_digest, sender_key, file_path, created_at, expires_at, status FROM files WHERE sender_id = ?"
       fr <- ExceptT $ rowToFileRec row
       pure (fr, sndKey (fileInfo fr))
     SFRecipient -> do
       row :. Only rcpKeyBs <-
         loadFileRow
-          "SELECT f.sender_id, f.file_size, f.file_digest, f.sender_key, f.file_path, f.created_at, f.status, r.recipient_key FROM files f JOIN recipients r ON r.sender_id = f.sender_id WHERE r.recipient_id = ?"
+          "SELECT f.sender_id, f.file_size, f.file_digest, f.sender_key, f.file_path, f.created_at, f.expires_at, f.status, r.recipient_key FROM files f JOIN recipients r ON r.sender_id = f.sender_id WHERE r.recipient_id = ?"
       fr <- ExceptT $ rowToFileRec row
       rcpKey <- either (const $ throwE INTERNAL) pure $ C.decodePubKey rcpKeyBs
       pure (fr, rcpKey)
@@ -152,12 +153,12 @@ instance FileStoreClass PostgresFileStore where
       DB.execute db "DELETE FROM recipients WHERE recipient_id = ?" (Only rId)
     withLog "ackFile" st $ \s -> logAckFile s rId
 
-  expiredFiles st old limit =
+  expiredFiles st now old limit =
     fmap toResult $ withTransaction (dbStore st) $ \db ->
       DB.query
         db
-        "SELECT sender_id, file_path, file_size FROM files WHERE created_at + ? < ? ORDER BY created_at LIMIT ?"
-        (fileTimePrecision, old, limit)
+        "(SELECT sender_id, file_path, file_size FROM files WHERE expires_at < ? LIMIT ?) UNION ALL (SELECT sender_id, file_path, file_size FROM files WHERE expires_at IS NULL AND created_at < ? LIMIT ?)"
+        (roundedSeconds now, limit, old - fileTimePrecision, limit)
     where
       toResult :: [(SenderId, Maybe FilePath, Int32)] -> [(SenderId, Maybe FilePath, Word32)]
       toResult = map (\(sId, path, size) -> (sId, path, fromIntegral size))
@@ -174,21 +175,21 @@ instance FileStoreClass PostgresFileStore where
 
 -- Internal helpers
 
-mkFileRec :: SenderId -> FileInfo -> Maybe FilePath -> RoundedFileTime -> ServerEntityStatus -> IO FileRec
-mkFileRec senderId fileInfo path createdAt status = do
+mkFileRec :: SenderId -> FileInfo -> Maybe FilePath -> RoundedFileTime -> Maybe RoundedFileTime -> ServerEntityStatus -> IO FileRec
+mkFileRec senderId fileInfo path createdAt expiresAt status = do
   filePath <- newTVarIO path
   recipientIds <- newTVarIO S.empty
   fileStatus <- newTVarIO status
-  pure FileRec {senderId, fileInfo, filePath, recipientIds, createdAt, fileStatus}
+  pure FileRec {senderId, fileInfo, filePath, recipientIds, createdAt, expiresAt, fileStatus}
 
-type FileRecRow = (SenderId, Int32, ByteString, ByteString, Maybe FilePath, RoundedFileTime, ServerEntityStatus)
+type FileRecRow = (SenderId, Int32, ByteString, ByteString, Maybe FilePath, RoundedFileTime, Maybe RoundedFileTime, ServerEntityStatus)
 
 rowToFileRec :: FileRecRow -> IO (Either XFTPErrorType FileRec)
-rowToFileRec (sId, size, digest, sndKeyBs, path, createdAt, status) =
+rowToFileRec (sId, size, digest, sndKeyBs, path, createdAt, expiresAt, status) =
   case C.decodePubKey sndKeyBs of
     Right sndKey -> do
       let fileInfo = FileInfo {sndKey, size = fromIntegral size, digest}
-      Right <$> mkFileRec sId fileInfo path createdAt status
+      Right <$> mkFileRec sId fileInfo path createdAt expiresAt status
     Left _ -> pure $ Left INTERNAL
 
 -- DB helpers
@@ -243,7 +244,7 @@ importFileStore storeLogFilePath dbCfg = do
   fCnt <- withTransaction (dbStore pgStore) $ \db -> do
     DB.copy_
       db
-      "COPY files (sender_id, file_size, file_digest, sender_key, file_path, created_at, status) FROM STDIN WITH (FORMAT csv)"
+      "COPY files (sender_id, file_size, file_digest, sender_key, file_path, created_at, expires_at, status) FROM STDIN WITH (FORMAT csv)"
     iforM_ (M.toList allFiles) $ \i (sId, fr) -> do
       DB.putCopyData db =<< fileRecToCSV sId fr
       when (i > 0 && i `mod` 10000 == 0) $ putStr ("  " <> show i <> " files\r") >> hFlush stdout
@@ -282,13 +283,13 @@ exportFileStore storeLogFilePath dbCfg = do
   !fCnt <- withTransaction (dbStore pgStore) $ \db ->
     DB.fold_
       db
-      "SELECT sender_id, file_size, file_digest, sender_key, file_path, created_at, status FROM files ORDER BY created_at"
+      "SELECT sender_id, file_size, file_digest, sender_key, file_path, created_at, expires_at, status FROM files ORDER BY created_at"
       (0 :: Int)
-      ( \(!fc) (sId, size :: Int32, digest :: ByteString, sndKeyBs :: ByteString, path :: Maybe String, createdAt, status) ->
+      ( \(!fc) (sId, size :: Int32, digest :: ByteString, sndKeyBs :: ByteString, path :: Maybe String, createdAt, expiresAt, status) ->
           case C.decodePubKey sndKeyBs of
             Right sndKey -> do
               let fileInfo = FileInfo {sndKey, size = fromIntegral size, digest}
-              logAddFile sl sId fileInfo createdAt status
+              logAddFile sl sId fileInfo createdAt expiresAt status
               forM_ path $ logPutFile sl sId
               pure (fc + 1)
             Left _ -> do
@@ -326,7 +327,7 @@ iforM_ :: Monad m => [a] -> (Int -> a -> m ()) -> m ()
 iforM_ xs f = zipWithM_ f [0 ..] xs
 
 fileRecToCSV :: SenderId -> FileRec -> IO ByteString
-fileRecToCSV sId FileRec {fileInfo = FileInfo {sndKey, size, digest}, filePath, createdAt, fileStatus} = do
+fileRecToCSV sId FileRec {fileInfo = FileInfo {sndKey, size, digest}, filePath, createdAt, expiresAt, fileStatus} = do
   path <- readTVarIO filePath
   status <- readTVarIO fileStatus
   pure $ LB.toStrict $ BB.toLazyByteString $ mconcat (BB.char7 ',' `intersperse` fields path status) <> BB.char7 '\n'
@@ -338,6 +339,7 @@ fileRecToCSV sId FileRec {fileInfo = FileInfo {sndKey, size, digest}, filePath,
         renderField (toField (Binary (C.encodePubKey sndKey))),
         nullable (toField <$> path),
         renderField (toField createdAt),
+        nullable (toField <$> expiresAt),
         quotedField (toField status)
       ]
 

src/Simplex/FileTransfer/Server/Store/Postgres/Migrations.hs

@@ -14,7 +14,8 @@ import Text.RawString.QQ (r)
 
 xftpSchemaMigrations :: [(String, Text, Maybe Text)]
 xftpSchemaMigrations =
-  [ ("20260325_initial", m20260325_initial, Nothing)
+  [ ("20260325_initial", m20260325_initial, Nothing),
+    ("20260823_file_expiration", m20260823_file_expiration, Just down_m20260823_file_expiration)
   ]
 
 -- | The list of migrations in ascending order by date
@@ -45,3 +46,17 @@ CREATE TABLE recipients (
 CREATE INDEX idx_recipients_sender_id ON recipients (sender_id);
 CREATE INDEX idx_files_created_at ON files (created_at);
 |]
+
+m20260823_file_expiration :: Text
+m20260823_file_expiration =
+  [r|
+ALTER TABLE files ADD COLUMN expires_at BIGINT;
+CREATE INDEX idx_files_expiry ON files (expires_at, created_at);
+|]
+
+down_m20260823_file_expiration :: Text
+down_m20260823_file_expiration =
+  [r|
+DROP INDEX idx_files_expiry;
+ALTER TABLE files DROP COLUMN expires_at;
+|]

src/Simplex/FileTransfer/Server/StoreLog.hs

@@ -17,16 +17,18 @@ module Simplex.FileTransfer.Server.StoreLog
     logDeleteFile,
     logBlockFile,
     logAckFile,
+    serverFileName,
   )
 where
 
-import Control.Applicative ((<|>))
+import Control.Applicative (optional, (<|>))
 import Control.Concurrent.STM
 import Control.Monad.Except
 import qualified Data.Attoparsec.ByteString.Char8 as A
+import qualified Data.ByteString.Base64.URL as B64
 import qualified Data.ByteString.Char8 as B
 import qualified Data.ByteString.Lazy.Char8 as LB
-import Data.Composition ((.:), (.::))
+import Data.Composition ((.:), (.::.))
 import Data.List.NonEmpty (NonEmpty)
 import qualified Data.List.NonEmpty as L
 import Data.Map.Strict (Map)
@@ -35,14 +37,15 @@ import Simplex.FileTransfer.Protocol (FileInfo (..))
 import Simplex.FileTransfer.Server.Store
 import Simplex.FileTransfer.Transport (XFTPErrorType (..))
 import Simplex.Messaging.Encoding.String
-import Simplex.Messaging.Protocol (BlockingInfo, RcvPublicAuthKey, RecipientId, SenderId)
+import Simplex.Messaging.Protocol (BlockingInfo, EntityId (..), RcvPublicAuthKey, RecipientId, SenderId)
 import Simplex.Messaging.Server.QueueStore (ServerEntityStatus (..))
 import Simplex.Messaging.Server.StoreLog
 import Simplex.Messaging.Util (bshow)
+import System.FilePath (takeFileName)
 import System.IO
 
 data FileStoreLogRecord
-  = AddFile SenderId FileInfo RoundedFileTime ServerEntityStatus
+  = AddFile SenderId FileInfo RoundedFileTime (Maybe RoundedFileTime) ServerEntityStatus
   | PutFile SenderId FilePath
   | AddRecipients SenderId (NonEmpty FileRecipient)
   | DeleteFile SenderId
@@ -52,27 +55,37 @@ data FileStoreLogRecord
 
 instance StrEncoding FileStoreLogRecord where
   strEncode = \case
-    AddFile sId file createdAt status -> strEncode (Str "FNEW", sId, file, createdAt, status)
+    AddFile sId file createdAt expiresAt status -> B.concat [strEncode (Str "FNEW", sId, file, createdAt), expE expiresAt, " ", strEncode status]
     PutFile sId path -> strEncode (Str "FPUT", sId, path)
     AddRecipients sId rcps -> strEncode (Str "FADD", sId, rcps)
     DeleteFile sId -> strEncode (Str "FDEL", sId)
     BlockFile sId info -> strEncode (Str "FBLK", sId, info)
     AckFile rId -> strEncode (Str "FACK", rId)
+    where
+      expE = maybe "" ((" " <>) . strEncode)
   strP =
     A.choice
-      [ "FNEW " *> (AddFile <$> strP_ <*> strP_ <*> strP <*> (_strP <|> pure EntityActive)),
+      [ "FNEW " *> addFileP,
         "FPUT " *> (PutFile <$> strP_ <*> strP),
         "FADD " *> (AddRecipients <$> strP_ <*> strP),
         "FDEL " *> (DeleteFile <$> strP),
         "FBLK " *> (BlockFile <$> strP_ <*> strP),
         "FACK " *> (AckFile <$> strP)
       ]
+    where
+      addFileP = do
+        sId <- strP_
+        file <- strP_
+        createdAt <- strP
+        expiresAt <- optional _strP
+        status <- _strP <|> pure EntityActive
+        pure $ AddFile sId file createdAt expiresAt status
 
 logFileStoreRecord :: StoreLog 'WriteMode -> FileStoreLogRecord -> IO ()
 logFileStoreRecord = writeStoreLogRecord
 
-logAddFile :: StoreLog 'WriteMode -> SenderId -> FileInfo -> RoundedFileTime -> ServerEntityStatus -> IO ()
-logAddFile s = logFileStoreRecord s .:: AddFile
+logAddFile :: StoreLog 'WriteMode -> SenderId -> FileInfo -> RoundedFileTime -> Maybe RoundedFileTime -> ServerEntityStatus -> IO ()
+logAddFile s = logFileStoreRecord s .::. AddFile
 
 logPutFile :: StoreLog 'WriteMode -> SenderId -> FilePath -> IO ()
 logPutFile s = logFileStoreRecord s .: PutFile
@@ -89,6 +102,9 @@ logBlockFile s fId = logFileStoreRecord s . BlockFile fId
 logAckFile :: StoreLog 'WriteMode -> RecipientId -> IO ()
 logAckFile s = logFileStoreRecord s . AckFile
 
+serverFileName :: SenderId -> FilePath
+serverFileName = B.unpack . B64.encode . unEntityId
+
 readWriteFileStore :: FilePath -> STMFileStore -> IO (StoreLog 'WriteMode)
 readWriteFileStore = readWriteStoreLog readFileStore writeFileStore
 
@@ -102,10 +118,12 @@ readFileStore f st = mapM_ (addFileLogRecord . LB.toStrict) . LB.lines =<< LB.re
           Left e -> B.putStrLn $ "Log processing error (" <> bshow e <> "): " <> B.take 100 s
           _ -> pure ()
     addToStore = \case
-      AddFile sId file createdAt status
-        | size file > 0 -> addFile st sId file createdAt status
+      AddFile sId file createdAt expiresAt status
+        | size file > 0 -> addFile st sId file createdAt expiresAt status
         | otherwise -> pure $ Left SIZE
-      PutFile qId path -> setFilePath st qId path
+      PutFile qId path
+        | takeFileName path == serverFileName qId -> setFilePath st qId path
+        | otherwise -> pure $ Left AUTH
       AddRecipients sId rcps -> runExceptT $ addRecipients sId rcps
       DeleteFile sId -> deleteFile st sId
       BlockFile sId info -> blockFile st sId info True
@@ -118,9 +136,9 @@ writeFileStore s STMFileStore {files, recipients} = do
   readTVarIO files >>= mapM_ (logFile allRcps)
   where
     logFile :: Map RecipientId (SenderId, RcvPublicAuthKey) -> FileRec -> IO ()
-    logFile allRcps FileRec {senderId, fileInfo, filePath, recipientIds, createdAt, fileStatus} = do
+    logFile allRcps FileRec {senderId, fileInfo, filePath, recipientIds, createdAt, expiresAt, fileStatus} = do
       status <- readTVarIO fileStatus
-      logAddFile s senderId fileInfo createdAt status
+      logAddFile s senderId fileInfo createdAt expiresAt status
       (rcpErrs, rcps) <- M.mapEither getRcp . M.fromSet id <$> readTVarIO recipientIds
       mapM_ (logAddRecipients s senderId) $ L.nonEmpty $ M.elems rcps
       mapM_ (B.putStrLn . ("Error storing log: " <>)) rcpErrs

src/Simplex/FileTransfer/Transport.hs

@@ -12,6 +12,7 @@ module Simplex.FileTransfer.Transport
   ( supportedFileServerVRange,
     authCmdsXFTPVersion,
     blockedFilesXFTPVersion,
+    fileStorageTimeXFTPVersion,
     xftpClientHandshakeStub,
     alpnSupportedXFTPhandshakes,
     xftpALPNv1,
@@ -36,7 +37,6 @@ module Simplex.FileTransfer.Transport
   )
 where
 
-import Control.Applicative (optional)
 import qualified Control.Exception as E
 import Control.Logger.Simple
 import Control.Monad
@@ -56,13 +56,14 @@ import Data.Word (Word16, Word32)
 import Network.HTTP2.Client (HTTP2Error)
 import qualified Simplex.Messaging.Crypto as C
 import qualified Simplex.Messaging.Crypto.Lazy as LC
+import Simplex.Messaging.Crypto.Entitlement (EntitlementProof)
 import Simplex.Messaging.Encoding
 import Simplex.Messaging.Encoding.String
 import Simplex.Messaging.Parsers
 import Simplex.Messaging.Protocol (BlockingInfo, CommandError)
 import Simplex.Messaging.Transport (ALPN, CertChainPubKey, ServiceCredentials, SessionId, THandle (..), THandleParams (..), TransportError (..), TransportPeer (..))
 import Simplex.Messaging.Transport.HTTP2.File
-import Simplex.Messaging.Util (bshow, tshow, (<$?>))
+import Simplex.Messaging.Util (bshow, tshow, (<$?>), (<$$>))
 import Simplex.Messaging.Version
 import Simplex.Messaging.Version.Internal
 import System.IO (Handle, IOMode (..), withFile)
@@ -97,8 +98,11 @@ authCmdsXFTPVersion = VersionXFTP 2
 blockedFilesXFTPVersion :: VersionXFTP
 blockedFilesXFTPVersion = VersionXFTP 3
 
+fileStorageTimeXFTPVersion :: VersionXFTP
+fileStorageTimeXFTPVersion = VersionXFTP 4
+
 currentXFTPVersion :: VersionXFTP
-currentXFTPVersion = VersionXFTP 3
+currentXFTPVersion = VersionXFTP 4
 
 supportedFileServerVRange :: VersionRangeXFTP
 supportedFileServerVRange = mkVersionRange initialXFTPVersion currentXFTPVersion
@@ -124,14 +128,18 @@ data XFTPServerHandshake = XFTPServerHandshake
     -- | pub key to agree shared secrets for command authorization and entity ID encryption.
     authPubKey :: CertChainPubKey,
     -- | signed identity challenge from  XFTPClientHello
-    webIdentityProof :: Maybe C.ASignature
+    webIdentityProof :: Maybe C.ASignature,
+    -- | optional server public information (JSON-encoded ServerPublicInfo), sent when version >= fileStorageTimeXFTPVersion
+    serverInfoBytes :: Maybe ByteString
   }
 
 data XFTPClientHandshake = XFTPClientHandshake
   { -- | agreed XFTP server protocol version
     xftpVersion :: VersionXFTP,
     -- | server identity - CA certificate fingerprint
-    keyHash :: C.KeyHash
+    keyHash :: C.KeyHash,
+    -- | proof of the user entitlement bound to the session
+    entitlementProof :: Maybe EntitlementProof
   }
 
 instance Encoding XFTPClientHello where
@@ -143,21 +151,33 @@ instance Encoding XFTPClientHello where
     pure XFTPClientHello {webChallenge}
 
 instance Encoding XFTPClientHandshake where
-  smpEncode XFTPClientHandshake {xftpVersion, keyHash} =
-    smpEncode (xftpVersion, keyHash)
+  smpEncode XFTPClientHandshake {xftpVersion = v, keyHash, entitlementProof} =
+    smpEncode (v, keyHash) <> ifHasEntitlement v (smpEncode entitlementProof) ""
   smpP = do
-    (xftpVersion, keyHash) <- smpP
+    (v, keyHash) <- smpP
+    entitlementProof <- ifHasEntitlement v smpP (pure Nothing)
     Tail _compat <- smpP
-    pure XFTPClientHandshake {xftpVersion, keyHash}
+    pure XFTPClientHandshake {xftpVersion = v, keyHash, entitlementProof}
+
+ifHasEntitlement :: VersionXFTP -> a -> a -> a
+ifHasEntitlement v a b = if v >= fileStorageTimeXFTPVersion then a else b
 
 instance Encoding XFTPServerHandshake where
-  smpEncode XFTPServerHandshake {xftpVersionRange, sessionId, authPubKey, webIdentityProof} =
-    smpEncode (xftpVersionRange, sessionId, authPubKey, C.signatureBytes webIdentityProof)
+  smpEncode XFTPServerHandshake {xftpVersionRange, sessionId, authPubKey, webIdentityProof, serverInfoBytes} =
+    smpEncode (xftpVersionRange, sessionId, authPubKey, C.signatureBytes webIdentityProof) <> info
+    where
+      info = ifHasServerInfo (maxVersion xftpVersionRange) (smpEncode (Large <$> serverInfoBytes)) ""
   smpP = do
     (xftpVersionRange, sessionId, authPubKey) <- smpP
-    webIdentityProof <- optional $ C.decodeSignature <$?> smpP
+    -- decode the (length-prefixed) signature bytes deterministically: empty bytes decode to Nothing.
+    -- (Must not use `optional`, which would backtrack and leave the bytes for the parsers that follow.)
+    webIdentityProof <- C.decodeSignature <$?> smpP
+    serverInfoBytes <- ifHasServerInfo (maxVersion xftpVersionRange) (unLarge <$$> smpP) (pure Nothing)
     Tail _compat <- smpP
-    pure XFTPServerHandshake {xftpVersionRange, sessionId, authPubKey, webIdentityProof}
+    pure XFTPServerHandshake {xftpVersionRange, sessionId, authPubKey, webIdentityProof, serverInfoBytes}
+
+ifHasServerInfo :: VersionXFTP -> a -> a -> a
+ifHasServerInfo v a b = if v >= fileStorageTimeXFTPVersion then a else b
 
 sendEncFile :: Handle -> (Builder -> IO ()) -> LC.SbState -> Word32 -> IO ()
 sendEncFile h send = go

src/Simplex/FileTransfer/Types.hs

@@ -39,6 +39,7 @@ import Data.Text.Encoding (encodeUtf8)
 import Data.Word (Word32)
 import Simplex.FileTransfer.Client (XFTPChunkSpec (..))
 import Simplex.FileTransfer.Description
+import Simplex.FileTransfer.Protocol (GrantedStorageTime (..))
 import Simplex.Messaging.Agent.Store.DB (FromField (..), ToField (..), fromTextField_)
 import qualified Simplex.Messaging.Crypto as C
 import Simplex.Messaging.Crypto.File (CryptoFile (..))
@@ -92,7 +93,8 @@ data RcvFile = RcvFile
   deriving (Show)
 
 data RcvFileStatus
-  = RFSReceiving
+  = RFSPrepared
+  | RFSReceiving
   | RFSReceived
   | RFSDecrypting
   | RFSComplete
@@ -105,6 +107,7 @@ instance ToField RcvFileStatus where toField = toField . textEncode
 
 instance TextEncoding RcvFileStatus where
   textDecode = \case
+    "prepared" -> Just RFSPrepared
     "receiving" -> Just RFSReceiving
     "received" -> Just RFSReceived
     "decrypting" -> Just RFSDecrypting
@@ -112,6 +115,7 @@ instance TextEncoding RcvFileStatus where
     "error" -> Just RFSError
     _ -> Nothing
   textEncode = \case
+    RFSPrepared -> "prepared"
     RFSReceiving -> "receiving"
     RFSReceived -> "received"
     RFSDecrypting -> "decrypting"
@@ -167,7 +171,8 @@ data SndFile = SndFile
     prefixPath :: Maybe FilePath,
     status :: SndFileStatus,
     deleted :: Bool,
-    redirect :: Maybe RedirectFileInfo
+    redirect :: Maybe RedirectFileInfo,
+    storageHours :: Maybe Word32
   }
   deriving (Show)
 
@@ -175,7 +180,8 @@ sndFileEncPath :: FilePath -> FilePath
 sndFileEncPath prefixPath = prefixPath </> "xftp.encrypted"
 
 data SndFileStatus
-  = SFSNew -- db record created
+  = SFSPrepared
+  | SFSNew -- db record created
   | SFSEncrypting -- encryption started
   | SFSEncrypted -- encryption complete
   | SFSUploading -- all chunk replicas are created on servers
@@ -189,6 +195,7 @@ instance ToField SndFileStatus where toField = toField . textEncode
 
 instance TextEncoding SndFileStatus where
   textDecode = \case
+    "prepared" -> Just SFSPrepared
     "new" -> Just SFSNew
     "encrypting" -> Just SFSEncrypting
     "encrypted" -> Just SFSEncrypted
@@ -197,6 +204,7 @@ instance TextEncoding SndFileStatus where
     "error" -> Just SFSError
     _ -> Nothing
   textEncode = \case
+    SFSPrepared -> "prepared"
     SFSNew -> "new"
     SFSEncrypting -> "encrypting"
     SFSEncrypted -> "encrypted"
@@ -225,7 +233,8 @@ data NewSndChunkReplica = NewSndChunkReplica
   { server :: XFTPServer,
     replicaId :: ChunkReplicaId,
     replicaKey :: C.APrivateAuthKey,
-    rcvIdsKeys :: [(ChunkReplicaId, C.APrivateAuthKey)]
+    rcvIdsKeys :: [(ChunkReplicaId, C.APrivateAuthKey)],
+    expiresAt :: Maybe GrantedStorageTime
   }
   deriving (Show)
 
@@ -237,7 +246,8 @@ data SndFileChunkReplica = SndFileChunkReplica
     rcvIdsKeys :: [(ChunkReplicaId, C.APrivateAuthKey)],
     replicaStatus :: SndFileReplicaStatus,
     delay :: Maybe Int64,
-    retries :: Int
+    retries :: Int,
+    expiresAt :: Maybe GrantedStorageTime
   }
   deriving (Show)
 

src/Simplex/FileTransfer/Util.hs

@@ -1,19 +1,26 @@
 module Simplex.FileTransfer.Util
   ( uniqueCombine,
+    safeFileNameStr,
     removePath,
   )
 where
 
 import Simplex.Messaging.Util (ifM, whenM)
-import System.FilePath (splitExtensions, (</>))
+import System.FilePath (makeValid, splitExtensions, takeFileName, (</>))
 import UnliftIO
 import UnliftIO.Directory
 
+safeFileNameStr :: String -> String
+safeFileNameStr = notDots . makeValid . takeFileName
+  where
+    notDots n = if n == "." || n == ".." then "_" else n
+
+-- | The file name is sanitized, so the combined path cannot escape the folder.
 uniqueCombine :: MonadIO m => FilePath -> String -> m FilePath
 uniqueCombine filePath fileName = tryCombine (0 :: Int)
   where
     tryCombine n =
-      let (name, ext) = splitExtensions fileName
+      let (name, ext) = splitExtensions $ safeFileNameStr fileName
           suffix = if n == 0 then "" else "_" <> show n
           f = filePath </> (name <> suffix <> ext)
        in ifM (doesPathExist f) (tryCombine $ n + 1) (pure f)

src/Simplex/Messaging/Agent/Client.hs

@@ -41,6 +41,7 @@ module Simplex.Messaging.Agent.Client
     reconnectServerClients,
     reconnectSMPServer,
     closeXFTPServerClient,
+    closeUserXFTPClients,
     runSMPServerTest,
     runXFTPServerTest,
     runNTFServerTest,
@@ -227,7 +228,7 @@ import Data.Text (Text)
 import Data.Text.Encoding
 import Data.Time (UTCTime, addUTCTime, defaultTimeLocale, formatTime, getCurrentTime)
 import Data.Time.Clock.System (getSystemTime)
-import Data.Word (Word16)
+import Data.Word (Word16, Word32)
 import qualified Data.X509.Validation as XV
 import Network.Socket (HostName)
 import Simplex.FileTransfer.Client (XFTPChunkSpec (..), XFTPClient, XFTPClientConfig (..), XFTPClientError)
@@ -253,6 +254,8 @@ import Simplex.Messaging.Agent.TSessionSubs (TSessionSubs)
 import qualified Simplex.Messaging.Agent.TSessionSubs as SS
 import Simplex.Messaging.Client
 import qualified Simplex.Messaging.Crypto as C
+import Simplex.Messaging.Crypto.BBS (BBSPresHeader (..), BBSPublicKey)
+import Simplex.Messaging.Crypto.Entitlement (EntitlementCredential, EntitlementProof, generateEntitlementProof)
 import Simplex.Messaging.Encoding
 import Simplex.Messaging.Encoding.String
 import Simplex.Messaging.Notifications.Client
@@ -269,7 +272,7 @@ import Simplex.Messaging.Protocol
     NetworkError (..),
     MsgFlags (..),
     MsgId,
-    NameRecord,
+    NameResponse,
     NtfServer,
     NtfServerWithAuth,
     ProtoServer,
@@ -304,11 +307,12 @@ import Simplex.Messaging.Protocol
 import qualified Simplex.Messaging.Protocol as SMP
 import Simplex.Messaging.Protocol.Types
 import Simplex.Messaging.Server.QueueStore.QueueInfo
+import Simplex.Messaging.Server.Information (ServerPublicInfo)
 import Simplex.Messaging.Session
 import Simplex.Messaging.SystemTime
 import Simplex.Messaging.TMap (TMap)
 import qualified Simplex.Messaging.TMap as TM
-import Simplex.Messaging.Transport (HandshakeError (..), SMPServiceRole (..), SMPVersion, ServiceCredentials (..), SessionId, THClientService' (..), THandleAuth (..), THandleParams (sessionId, thAuth, thVersion), TransportError (..), TransportPeer (..), sndAuthKeySMPVersion, shortLinksSMPVersion, newNtfCredsSMPVersion)
+import Simplex.Messaging.Transport (HandshakeError (..), SMPServiceRole (..), SMPVersion, ServiceCredentials (..), SessionId, THClientService' (..), THandleAuth (..), THandleParams (sessionId, thAuth, thVersion, serverInfo), TransportError (..), TransportPeer (..), shortLinksSMPVersion, newNtfCredsSMPVersion)
 import Simplex.Messaging.Transport.Client (TransportHost (..))
 import Simplex.Messaging.Transport.Credentials
 import Simplex.Messaging.Util
@@ -353,6 +357,7 @@ data AgentClient = AgentClient
     ntfClients :: TMap NtfTransportSession NtfClientVar,
     xftpServers :: TMap UserId (UserServers 'PXFTP),
     xftpClients :: TMap XFTPTransportSession XFTPClientVar,
+    userEntitlements :: TMap UserId EntitlementCredential,
     useNetworkConfig :: TVar (NetworkConfig, NetworkConfig), -- (slow, fast) networks
     presetDomains :: [HostName],
     presetServers :: [SMPServer],
@@ -384,6 +389,7 @@ data AgentClient = AgentClient
     clientId :: Int,
     agentEnv :: Env,
     proxySessTs :: TVar UTCTime,
+    serviceRequests :: TMap ConnId (TMVar (Either AgentErrorType SMP.MsgBody)),
     smpServersStats :: TMap (UserId, SMPServer) AgentSMPServerStats,
     xftpServersStats :: TMap (UserId, XFTPServer) AgentXFTPServerStats,
     ntfServersStats :: TMap (UserId, NtfServer) AgentNtfServerStats,
@@ -509,7 +515,7 @@ data UserNetworkType = UNNone | UNCellular | UNWifi | UNEthernet | UNOther
 
 -- | Creates an SMP agent client instance that receives commands and sends responses via 'TBQueue's.
 newAgentClient :: Int -> InitialAgentServers -> UTCTime -> Map (Maybe SMPServer) (Maybe SystemSeconds) -> Env -> IO AgentClient
-newAgentClient clientId InitialAgentServers {smp, ntf, xftp, netCfg, useServices, presetDomains, presetServers} currentTs notices agentEnv = do
+newAgentClient clientId InitialAgentServers {smp, ntf, xftp, entitlements, netCfg, useServices, presetDomains, presetServers} currentTs notices agentEnv = do
   let cfg = config agentEnv
       qSize = tbqSize cfg
   proxySessTs <- newTVarIO =<< getCurrentTime
@@ -525,6 +531,7 @@ newAgentClient clientId InitialAgentServers {smp, ntf, xftp, netCfg, useServices
   ntfClients <- TM.emptyIO
   xftpServers <- newTVarIO $ M.map mkUserServers xftp
   xftpClients <- TM.emptyIO
+  userEntitlements <- newTVarIO entitlements
   useNetworkConfig <- newTVarIO (slowNetworkConfig netCfg, netCfg)
   userNetworkInfo <- newTVarIO $ UserNetworkInfo UNOther True
   userNetworkUpdated <- newTVarIO Nothing
@@ -547,6 +554,7 @@ newAgentClient clientId InitialAgentServers {smp, ntf, xftp, netCfg, useServices
   invLocks <- TM.emptyIO
   deleteLock <- createLockIO
   smpSubWorkers <- TM.emptyIO
+  serviceRequests <- TM.emptyIO
   smpServersStats <- TM.emptyIO
   xftpServersStats <- TM.emptyIO
   ntfServersStats <- TM.emptyIO
@@ -565,6 +573,7 @@ newAgentClient clientId InitialAgentServers {smp, ntf, xftp, netCfg, useServices
         ntfClients,
         xftpServers,
         xftpClients,
+        userEntitlements,
         useNetworkConfig,
         presetDomains,
         presetServers,
@@ -592,6 +601,7 @@ newAgentClient clientId InitialAgentServers {smp, ntf, xftp, netCfg, useServices
         clientId,
         agentEnv,
         proxySessTs,
+        serviceRequests,
         smpServersStats,
         xftpServersStats,
         ntfServersStats,
@@ -710,7 +720,7 @@ getSMPProxyClient c@AgentClient {active, smpClients, smpProxiedRelays, workerSeq
       pure (clnt, sess)
     newProxiedRelay :: SMPConnectedClient -> Maybe SMP.BasicAuth -> ProxiedRelayVar -> AM (Either AgentErrorType ProxiedRelay)
     newProxiedRelay (SMPConnectedClient smp prs) proxyAuth rv =
-      tryAllErrors (liftClient SMP (clientServer smp) $ connectSMPProxiedRelay smp nm destSrv proxyAuth) >>= \case
+      tryAllErrors (liftClient proxyRelayError (clientServer smp) $ connectSMPProxiedRelay smp nm destSrv proxyAuth) >>= \case
         Right sess -> do
           atomically $ putTMVar (sessionVar rv) (Right sess)
           pure $ Right sess
@@ -721,6 +731,18 @@ getSMPProxyClient c@AgentClient {active, smpClients, smpProxiedRelays, workerSeq
               TM.delete destSess smpProxiedRelays
             putTMVar (sessionVar rv) (Left e)
             pure $ Left e
+      where
+        -- proxy reports BROKER errors about the relay, not about its own connection,
+        -- so they include both addresses, same as PFWD errors.
+        proxyRelayError :: HostName -> ErrorType -> AgentErrorType
+        proxyRelayError proxyHost = \case
+          e@(SMP.PROXY (SMP.BROKER _)) ->
+            PROXY
+              { proxyServer = protocolClientServer smp,
+                relayServer = B.unpack $ strEncode destSrv,
+                proxyErr = ProxyProtocolError e
+              }
+          e -> SMP proxyHost e
     waitForProxiedRelay :: SMPTransportSession -> ProxiedRelayVar -> AM (Either AgentErrorType ProxiedRelay)
     waitForProxiedRelay (_, srv, _) rv = do
       NetworkConfig {tcpConnectTimeout} <- getNetworkConfig c
@@ -862,7 +884,7 @@ getNtfServerClient c@AgentClient {active, ntfClients, workerSeq, proxySessTs, pr
       logInfo . decodeUtf8 $ "Agent disconnected from " <> showServer srv
 
 getXFTPServerClient :: AgentClient -> XFTPTransportSession -> AM XFTPClient
-getXFTPServerClient c@AgentClient {active, xftpClients, workerSeq, proxySessTs, presetDomains} tSess@(_, srv, _) = do
+getXFTPServerClient c@AgentClient {active, xftpClients, userEntitlements, workerSeq, proxySessTs, presetDomains} tSess@(userId, srv, _) = do
   unlessM (readTVarIO active) $ throwE INACTIVE
   ts <- liftIO getCurrentTime
   withGetSessVar workerSeq tSess xftpClients ts (newProtocolClient c tSess xftpClients connectClient) (waitForProtocolClient c NRMBackground tSess xftpClients)
@@ -870,12 +892,28 @@ getXFTPServerClient c@AgentClient {active, xftpClients, workerSeq, proxySessTs,
     connectClient :: XFTPClientVar -> AM XFTPClient
     connectClient v = do
       cfg <- asks $ xftpCfg . config
+      keys <- asks $ entitlementKeys . config
       xftpNetworkConfig <- getNetworkConfig c
       ts <- readTVarIO proxySessTs
       liftError' (protocolClientError XFTP $ B.unpack $ strEncode srv) $
-        X.getXFTPClient tSess cfg {xftpNetworkConfig} presetDomains ts $
+        X.getXFTPClient tSess cfg {xftpNetworkConfig} presetDomains ts (mkEntitlementProof keys) $
           clientDisconnected v
 
+    mkEntitlementProof :: Map Word16 BBSPublicKey -> SessionId -> IO (Maybe EntitlementProof)
+    mkEntitlementProof keys sessId =
+      ifM knownServer proof (pure Nothing)
+      where
+        -- the entitlement is presented only to the servers of this user, matched by key hash that TLS pins,
+        -- so a file description of the sender cannot direct it to another server
+        knownServer = maybe False (any (sameKeyHash . snd) . storageSrvs) <$> TM.lookupIO userId (xftpServers c)
+        sameKeyHash (ProtoServerWithAuth srv' _) = srvKeyHash srv' == srvKeyHash srv
+        srvKeyHash (ProtocolServer _ _ _ kh) = kh
+        proof =
+          TM.lookupIO userId userEntitlements $>>= \cred ->
+            generateEntitlementProof keys cred (BBSPresHeader sessId) >>= \case
+              Right p -> pure $ Just p
+              Left e -> Nothing <$ logError ("entitlement proof error: " <> tshow e)
+
     clientDisconnected :: XFTPClientVar -> XFTPClient -> IO ()
     clientDisconnected v client = do
       atomically $ removeSessVar v tSess xftpClients
@@ -1021,6 +1059,13 @@ reconnectSMPServer c userId srv = do
       | userId == userId' && srv == srv' = (v :)
       | otherwise = id
 
+closeUserXFTPClients :: AgentClient -> UserId -> IO ()
+closeUserXFTPClients c userId = do
+  vs <- atomically $ stateTVar (xftpClients c) $ \cs ->
+    let (userCs, cs') = M.partitionWithKey (\(userId', _, _) _ -> userId == userId') cs
+     in (M.elems userCs, cs')
+  mapM_ (forkIO . closeClient_ c) vs
+
 closeClient :: ProtocolServerClient v err msg => AgentClient -> (AgentClient -> TMap (TransportSession msg) (ClientVar msg)) -> TransportSession msg -> IO ()
 closeClient c clientSel tSess =
   atomically (TM.lookupDelete tSess $ clientSel c) >>= mapM_ (closeClient_ c)
@@ -1281,7 +1326,7 @@ data ProtocolTestFailure = ProtocolTestFailure
   }
   deriving (Eq, Show)
 
-runSMPServerTest :: AgentClient -> NetworkRequestMode -> UserId -> SMPServerWithAuth -> AM' (Maybe ProtocolTestFailure)
+runSMPServerTest :: AgentClient -> NetworkRequestMode -> UserId -> SMPServerWithAuth -> AM' (Either ProtocolTestFailure (Maybe (Either String ServerPublicInfo)))
 runSMPServerTest c@AgentClient {presetDomains} nm userId (ProtoServerWithAuth srv auth) = do
   cfg <- getClientConfig c smpCfg
   C.AuthAlg ra <- asks $ rcvAuthAlg . config
@@ -1303,14 +1348,14 @@ runSMPServerTest c@AgentClient {presetDomains} nm userId (ProtoServerWithAuth sr
               _ -> secureSMPQueue smp nm rpKey rcvId sKey
           liftError (testErr TSDeleteQueue) $ deleteSMPQueue smp nm rpKey rcvId
         ok <- netTimeoutInt (tcpTimeout $ networkConfig cfg) nm `timeout` closeProtocolClient smp
-        pure $ either Just (const Nothing) r <|> maybe (Just (ProtocolTestFailure TSDisconnect $ BROKER addr TIMEOUT)) (const Nothing) ok
-      Left e -> pure (Just $ testErr TSConnect e)
+        pure $ r >> maybe (Left (ProtocolTestFailure TSDisconnect $ BROKER addr TIMEOUT)) (const $ Right $ serverInfo (thParams smp)) ok
+      Left e -> pure $ Left (testErr TSConnect e)
   where
     addr = B.unpack $ strEncode srv
     testErr :: ProtocolTestStep -> SMPClientError -> ProtocolTestFailure
     testErr step = ProtocolTestFailure step . protocolClientError SMP addr
 
-runXFTPServerTest :: AgentClient -> NetworkRequestMode -> UserId -> XFTPServerWithAuth -> AM' (Maybe ProtocolTestFailure)
+runXFTPServerTest :: AgentClient -> NetworkRequestMode -> UserId -> XFTPServerWithAuth -> AM' (Either ProtocolTestFailure (Maybe (Either String ServerPublicInfo)))
 runXFTPServerTest c@AgentClient {presetDomains} nm userId (ProtoServerWithAuth srv auth) = do
   cfg <- asks $ xftpCfg . config
   g <- asks random
@@ -1321,7 +1366,7 @@ runXFTPServerTest c@AgentClient {presetDomains} nm userId (ProtoServerWithAuth s
   liftIO $ do
     let tSess = (userId, srv, Nothing)
     ts <- readTVarIO $ proxySessTs c
-    X.getXFTPClient tSess cfg {xftpNetworkConfig} presetDomains ts (\_ -> pure ()) >>= \case
+    X.getXFTPClient tSess cfg {xftpNetworkConfig} presetDomains ts (\_ -> pure Nothing) (\_ -> pure ()) >>= \case
       Right xftp -> withTestChunk filePath $ do
         (sndKey, spKey) <- atomically $ C.generateAuthKeyPair C.SEd25519 g
         (rcvKey, rpKey) <- atomically $ C.generateAuthKeyPair C.SEd25519 g
@@ -1329,15 +1374,15 @@ runXFTPServerTest c@AgentClient {presetDomains} nm userId (ProtoServerWithAuth s
         let file = FileInfo {sndKey, size = chSize, digest}
             chunkSpec = X.XFTPChunkSpec {filePath, chunkOffset = 0, chunkSize = chSize}
         r <- runExceptT $ do
-          (sId, [rId]) <- liftError (testErr TSCreateFile) $ X.createXFTPChunk xftp spKey file [rcvKey] auth
+          (sId, [rId], _) <- liftError (testErr TSCreateFile) $ X.createXFTPChunk xftp spKey file [rcvKey] auth Nothing
           liftError (testErr TSUploadFile) $ X.uploadXFTPChunk xftp spKey sId chunkSpec
           liftError (testErr TSDownloadFile) $ X.downloadXFTPChunk g xftp rpKey rId $ XFTPRcvChunkSpec rcvPath chSize digest
           rcvDigest <- liftIO $ C.sha256Hash <$> B.readFile rcvPath
           unless (digest == rcvDigest) $ throwE $ ProtocolTestFailure TSCompareFile $ XFTP (B.unpack $ strEncode srv) DIGEST
           liftError (testErr TSDeleteFile) $ X.deleteXFTPChunk xftp spKey sId
         ok <- netTimeoutInt (tcpTimeout xftpNetworkConfig) nm `timeout` X.closeXFTPClient xftp
-        pure $ either Just (const Nothing) r <|> maybe (Just (ProtocolTestFailure TSDisconnect $ BROKER addr TIMEOUT)) (const Nothing) ok
-      Left e -> pure (Just $ testErr TSConnect e)
+        pure $ r >> maybe (Left (ProtocolTestFailure TSDisconnect $ BROKER addr TIMEOUT)) (const $ Right $ serverInfo (X.thParams xftp)) ok
+      Left e -> pure $ Left (testErr TSConnect e)
   where
     addr = B.unpack $ strEncode srv
     testErr :: ProtocolTestStep -> XFTPClientError -> ProtocolTestFailure
@@ -1504,9 +1549,7 @@ newRcvQueue_ c nm userId connId (ProtoServerWithAuth srv auth) vRange cqrd enabl
           if sndId == sndId' && lnkId == lnkId'
             then pure $ Just $ ShortLinkCreds lnkId linkKey privSigKey Nothing (fst d)
             else newErr "different sender or link IDs"
-      (_, Nothing) -> case linkId of
-        Nothing | v < sndAuthKeySMPVersion -> pure Nothing
-        _ -> newErr "unexpected link ID"
+      (_, Nothing) -> newErr "unexpected link ID"
       _ -> newErr "unexpected queue mode"
       where
         v = thVersion thParams'
@@ -1913,17 +1956,10 @@ sendConfirmation c nm sq@SndQueue {userId, server, connId, sndId, queueMode, snd
   sendOrProxySMPMessage c nm userId server connId "<CONF>" spKey sndId (MsgFlags {notification = True}) msg
 sendConfirmation _ _ _ _ = throwE $ INTERNAL "sendConfirmation called without snd_queue public key(s) in the database"
 
-sendInvitation :: AgentClient -> NetworkRequestMode -> UserId -> ConnId -> Compatible SMPQueueInfo -> Compatible VersionSMPA -> ConnectionRequestUri 'CMInvitation -> ConnInfo -> AM (Maybe SMPServer)
-sendInvitation c nm userId connId (Compatible (SMPQueueInfo v SMPQueueAddress {smpServer, senderId, dhPublicKey})) (Compatible agentVersion) connReq connInfo = do
-  msg <- mkInvitation
+sendInvitation :: AgentClient -> NetworkRequestMode -> UserId -> ConnId -> Compatible SMPQueueInfo -> AgentMsgEnvelope -> AM (Maybe SMPServer)
+sendInvitation c nm userId connId (Compatible (SMPQueueInfo v SMPQueueAddress {smpServer, senderId, dhPublicKey})) agentEnvelope = do
+  msg <- agentCbEncryptOnce v dhPublicKey . smpEncode $ SMP.ClientMessage SMP.PHEmpty (smpEncode agentEnvelope)
   sendOrProxySMPMessage c nm userId smpServer connId "<INV>" Nothing senderId (MsgFlags {notification = True}) msg
-  where
-    mkInvitation :: AM ByteString
-    -- this is only encrypted with per-queue E2E, not with double ratchet
-    mkInvitation = do
-      let agentEnvelope = AgentInvitation {agentVersion, connReq, connInfo}
-      agentCbEncryptOnce v dhPublicKey . smpEncode $
-        SMP.ClientMessage SMP.PHEmpty (smpEncode agentEnvelope)
 
 getQueueMessage :: AgentClient -> RcvQueue -> AM (Maybe SMPMsgMeta)
 getQueueMessage c rq@RcvQueue {server, rcvId, rcvPrivateKey} = do
@@ -1943,7 +1979,7 @@ getQueueMessage c rq@RcvQueue {server, rcvId, rcvPrivateKey} = do
 
 decryptSMPMessage :: RcvQueue -> SMP.RcvMessage -> AM SMP.ClientRcvMsgBody
 decryptSMPMessage rq SMP.RcvMessage {msgId, msgBody = SMP.EncRcvMsgBody body} =
-  liftEither $ parse SMP.clientRcvMsgBodyP (AGENT A_MESSAGE) =<< decrypt body
+  liftEither $ parse SMP.clientRcvMsgBodyP (AGENT $ A_MESSAGE "decrypt message") =<< decrypt body
   where
     decrypt = agentCbDecrypt (rcvDhSecret rq) (C.cbNonce msgId)
 
@@ -1986,7 +2022,7 @@ getQueueLink c nm userId server lnkId =
 -- resolver) and falls back to a direct send when the proxy is unavailable
 -- (faster but exposes the client IP). Mode selection is delegated to
 -- `sendOrProxySMPCommand`, which honours the network config (SPMNever etc.).
-resolveName :: AgentClient -> NetworkRequestMode -> UserId -> SMPServer -> SimplexDomain -> AM NameRecord
+resolveName :: AgentClient -> NetworkRequestMode -> UserId -> SMPServer -> SimplexDomain -> AM NameResponse
 resolveName c nm userId server domain =
   snd <$> sendOrProxySMPCommand c nm userId server "" "RSLV" NoEntity resolveViaProxy resolveDirectly
   where
@@ -2179,16 +2215,17 @@ agentXFTPDownloadChunk c userId (FileDigest chunkDigest) RcvFileChunkReplica {se
   g <- asks random
   withXFTPClient c (userId, server, chunkDigest) "FGET" $ \xftp -> X.downloadXFTPChunk g xftp replicaKey fId chunkSpec
 
-agentXFTPNewChunk :: AgentClient -> SndFileChunk -> Int -> XFTPServerWithAuth -> AM NewSndChunkReplica
-agentXFTPNewChunk c SndFileChunk {userId, chunkSpec = XFTPChunkSpec {chunkSize}, digest = FileDigest chunkDigest} n (ProtoServerWithAuth srv auth) = do
+agentXFTPNewChunk :: AgentClient -> SndFileChunk -> Int -> XFTPServerWithAuth -> Maybe Word32 -> AM NewSndChunkReplica
+agentXFTPNewChunk c SndFileChunk {userId, chunkSpec = XFTPChunkSpec {chunkSize}, digest = FileDigest chunkDigest} n (ProtoServerWithAuth srv auth) storageHours = do
   rKeys <- xftpRcvKeys n
   (sndKey, replicaKey) <- atomically . C.generateAuthKeyPair C.SEd25519 =<< asks random
   let fileInfo = FileInfo {sndKey, size = chunkSize, digest = chunkDigest}
   logServer "-->" c srv NoEntity "FNEW"
   tSess <- mkTransportSession c userId srv chunkDigest
-  (sndId, rIds) <- withClient c NRMBackground tSess $ \xftp -> X.createXFTPChunk xftp replicaKey fileInfo (L.map fst rKeys) auth
+  (sndId, rIds, expiresAt) <- withClient c NRMBackground tSess $ \xftp ->
+    X.createXFTPChunk xftp replicaKey fileInfo (L.map fst rKeys) auth storageHours
   logServer "<--" c srv NoEntity $ B.unwords ["SIDS", logSecret sndId]
-  pure NewSndChunkReplica {server = srv, replicaId = ChunkReplicaId sndId, replicaKey, rcvIdsKeys = L.toList $ xftpRcvIdsKeys rIds rKeys}
+  pure NewSndChunkReplica {server = srv, replicaId = ChunkReplicaId sndId, replicaKey, rcvIdsKeys = L.toList $ xftpRcvIdsKeys rIds rKeys, expiresAt}
 
 agentXFTPUploadChunk :: AgentClient -> UserId -> FileDigest -> SndFileChunkReplica -> XFTPChunkSpec -> AM ()
 agentXFTPUploadChunk c userId (FileDigest chunkDigest) SndFileChunkReplica {server, replicaId = ChunkReplicaId fId, replicaKey} chunkSpec =
@@ -2247,7 +2284,7 @@ agentCbDecrypt dhSecret nonce msg =
 cryptoError :: C.CryptoError -> AgentErrorType
 cryptoError = \case
   C.CryptoLargeMsgError -> CMD LARGE "CryptoLargeMsgError"
-  C.CryptoHeaderError _ -> AGENT A_MESSAGE -- parsing error
+  C.CryptoHeaderError e -> AGENT $ A_MESSAGE $ "parse msg header " <> e
   C.CERatchetDuplicateMessage -> AGENT $ A_DUPLICATE Nothing
   C.AESDecryptError -> c DECRYPT_AES
   C.CBDecryptError -> c DECRYPT_CB

src/Simplex/Messaging/Agent/Env/SQLite.hs

@@ -67,6 +67,8 @@ import Simplex.Messaging.Agent.Store.Interface (DBOpts)
 import Simplex.Messaging.Agent.Store.Shared (MigrationConfig (..), MigrationError (..))
 import Simplex.Messaging.Client
 import qualified Simplex.Messaging.Crypto as C
+import Simplex.Messaging.Crypto.BBS (BBSPublicKey)
+import Simplex.Messaging.Crypto.Entitlement (EntitlementCredential, entitlementIssuerKeys)
 import Simplex.Messaging.Crypto.Ratchet (VersionRangeE2E, supportedE2EEncryptVRange)
 import Simplex.Messaging.Notifications.Client (defaultNTFClientConfig)
 import Simplex.Messaging.Notifications.Transport (NTFVersion)
@@ -89,6 +91,7 @@ data InitialAgentServers = InitialAgentServers
   { smp :: Map UserId (NonEmpty (ServerCfg 'PSMP)),
     ntf :: [NtfServer],
     xftp :: Map UserId (NonEmpty (ServerCfg 'PXFTP)),
+    entitlements :: Map UserId EntitlementCredential,
     netCfg :: NetworkConfig,
     useServices :: Map UserId Bool,
     presetDomains :: [HostName],
@@ -148,14 +151,18 @@ data AgentConfig = AgentConfig
     smpCfg :: ProtocolClientConfig SMPVersion,
     ntfCfg :: ProtocolClientConfig NTFVersion,
     xftpCfg :: XFTPClientConfig,
+    entitlementKeys :: Map Word16 BBSPublicKey,
     reconnectInterval :: RetryInterval,
     messageRetryInterval :: RetryInterval2,
     userNetworkInterval :: Int,
     userOfflineDelay :: NominalDiffTime,
     messageTimeout :: NominalDiffTime,
+    serviceRequestTimeout :: NominalDiffTime, -- client side: default time the client waits for a service response (overridable per request)
+    serviceResponseTimeout :: NominalDiffTime, -- service side: time a received service request is valid to respond to
     connDeleteDeliveryTimeout :: NominalDiffTime,
     helloTimeout :: NominalDiffTime,
     quotaExceededTimeout :: NominalDiffTime,
+    commandQuotaRetryInterval :: RetryInterval,
     persistErrorInterval :: NominalDiffTime,
     initialCleanupDelay :: Int64,
     cleanupInterval :: Int64,
@@ -170,6 +177,7 @@ data AgentConfig = AgentConfig
     xftpConsecutiveRetries :: Int,
     xftpMaxRecipientsPerRequest :: Int,
     deleteErrorCount :: Int,
+    keepAddressKeys :: Int,
     ntfCron :: Word16,
     ntfBatchSize :: Int,
     ntfSubFirstCheckInterval :: NominalDiffTime,
@@ -223,14 +231,18 @@ defaultAgentConfig =
       smpCfg = defaultSMPClientConfig,
       ntfCfg = defaultNTFClientConfig,
       xftpCfg = defaultXFTPClientConfig,
+      entitlementKeys = entitlementIssuerKeys,
       reconnectInterval = defaultReconnectInterval,
       messageRetryInterval = defaultMessageRetryInterval,
       userNetworkInterval = 1800_000000, -- 30 minutes, should be less than Int32 max value
       userOfflineDelay = 2, -- if network offline event happens in less than 2 seconds after it was set online, it is ignored
       messageTimeout = 2 * nominalDay,
+      serviceRequestTimeout = 30,
+      serviceResponseTimeout = 180,
       connDeleteDeliveryTimeout = 2 * nominalDay,
       helloTimeout = 2 * nominalDay,
       quotaExceededTimeout = 7 * nominalDay,
+      commandQuotaRetryInterval = RetryInterval {initialInterval = 30_000000, increaseAfter = 0, maxInterval = 3600_000000},
       persistErrorInterval = 3, -- seconds
       initialCleanupDelay = 30 * 1000000, -- 30 seconds
       cleanupInterval = 5 * 60 * 1000000, -- 5 minutes
@@ -245,6 +257,7 @@ defaultAgentConfig =
       xftpConsecutiveRetries = 3,
       xftpMaxRecipientsPerRequest = 200,
       deleteErrorCount = 10,
+      keepAddressKeys = 3,
       ntfCron = 20, -- minutes
       ntfBatchSize = 150,
       ntfSubFirstCheckInterval = nominalDay,

src/Simplex/Messaging/Agent/Protocol.hs

@@ -1,6 +1,7 @@
 {-# LANGUAGE DataKinds #-}
 {-# LANGUAGE DeriveAnyClass #-}
 {-# LANGUAGE DerivingStrategies #-}
+{-# LANGUAGE DerivingVia #-}
 {-# LANGUAGE DuplicateRecordFields #-}
 {-# LANGUAGE FlexibleInstances #-}
 {-# LANGUAGE GADTs #-}
@@ -40,12 +41,8 @@ module Simplex.Messaging.Agent.Protocol
     VersionSMPA,
     VersionRangeSMPA,
     pattern VersionSMPA,
-    duplexHandshakeSMPAgentVersion,
-    ratchetSyncSMPAgentVersion,
-    deliveryRcptsSMPAgentVersion,
-    pqdrSMPAgentVersion,
-    sndAuthKeySMPAgentVersion,
     ratchetOnConfSMPAgentVersion,
+    rpcAddressSMPAgentVersion,
     currentSMPAgentVersion,
     supportedSMPAgentVRange,
     e2eEncConnInfoLength,
@@ -54,8 +51,10 @@ module Simplex.Messaging.Agent.Protocol
     -- * SMP agent protocol types
     ConnInfo,
     SndQueueSecured,
+    UseRatchetKeys,
     AEntityId,
     ACommand (..),
+    JoinRequest (..),
     AEvent (..),
     AEvt (..),
     ACommandTag (..),
@@ -80,6 +79,7 @@ module Simplex.Messaging.Agent.Protocol
     SMPConfirmation (..),
     AgentMsgEnvelope (..),
     AgentMessage (..),
+    RequestSignature (..),
     AgentMessageType (..),
     APrivHeader (..),
     AMessage (..),
@@ -107,6 +107,9 @@ module Simplex.Messaging.Agent.Protocol
     ConnectionModeI (..),
     ConnectionRequestUri (..),
     AConnectionRequestUri (..),
+    BinaryConnectionRequestUri (..),
+    ABinaryConnectionRequestUri (..),
+    binaryConnReq,
     ShortLinkCreds (..),
     ConnReqUriData (..),
     CRClientData,
@@ -118,6 +121,10 @@ module Simplex.Messaging.Agent.Protocol
     UserConnLinkData (..),
     UserContactData (..),
     UserLinkData (..),
+    AddressRatchetKeys,
+    NewRatchetKeys,
+    DRInvitation (..),
+    RatchetKeyId (..),
     OwnerAuth (..),
     OwnerId,
     ConnectionLink (..),
@@ -151,6 +158,7 @@ module Simplex.Messaging.Agent.Protocol
     ConnectionErrorType (..),
     BrokerErrorType (..),
     SMPAgentError (..),
+    AgentServiceError (..),
     DroppedMsg (..),
     AgentCryptoError (..),
     cryptoErrToSyncState,
@@ -158,6 +166,8 @@ module Simplex.Messaging.Agent.Protocol
     ConnId,
     ConfirmationId,
     InvitationId,
+    ConnVerifyCodes (..),
+    ContactRequestBinding (..),
     MsgIntegrity (..),
     MsgErrorType (..),
     QueueStatus (..),
@@ -200,6 +210,7 @@ import qualified Data.Attoparsec.ByteString.Char8 as A
 import qualified Data.ByteString.Base64.URL as B64
 import Data.ByteString.Char8 (ByteString)
 import qualified Data.ByteString.Char8 as B
+import qualified Data.ByteString.Lazy as LB
 import Data.Char (toLower, toUpper)
 import Data.Foldable (find)
 import Data.Functor (($>))
@@ -219,7 +230,7 @@ import Data.Type.Equality
 import Data.Typeable (Typeable)
 import Data.Word (Word16, Word32)
 import Simplex.FileTransfer.Description
-import Simplex.FileTransfer.Protocol (FileParty (..))
+import Simplex.FileTransfer.Protocol (FileParty (..), GrantedStorageTime)
 import Simplex.FileTransfer.Transport (XFTPErrorType)
 import Simplex.FileTransfer.Types (FileErrorType)
 import Simplex.Messaging.Agent.QueryString
@@ -230,9 +241,11 @@ import Simplex.Messaging.Crypto.Ratchet
   ( InitialKeys (..),
     PQEncryption (..),
     PQSupport,
+    RatchetX448,
     RcvE2ERatchetParams,
     RcvE2ERatchetParamsUri,
     SndE2ERatchetParams,
+    RcvE2EPrivRatchetParams,
     pattern PQSupportOff,
     pattern PQSupportOn,
   )
@@ -289,6 +302,7 @@ import UnliftIO.Exception (Exception)
 -- 5 - post-quantum double ratchet (3/14/2024)
 -- 6 - secure reply queues with provided keys (6/14/2024)
 -- 7 - initialize ratchet on processing confirmation (7/18/2024)
+-- 8 - agent RPC and double ratchet PQ encryption from first message to contact address (8/01/2026)
 
 data SMPAgentVersion
 
@@ -301,29 +315,20 @@ type VersionRangeSMPA = VersionRange SMPAgentVersion
 pattern VersionSMPA :: Word16 -> VersionSMPA
 pattern VersionSMPA v = Version v
 
-duplexHandshakeSMPAgentVersion :: VersionSMPA
-duplexHandshakeSMPAgentVersion = VersionSMPA 2
-
-ratchetSyncSMPAgentVersion :: VersionSMPA
-ratchetSyncSMPAgentVersion = VersionSMPA 3
-
-deliveryRcptsSMPAgentVersion :: VersionSMPA
-deliveryRcptsSMPAgentVersion = VersionSMPA 4
-
-pqdrSMPAgentVersion :: VersionSMPA
-pqdrSMPAgentVersion = VersionSMPA 5
-
-sndAuthKeySMPAgentVersion :: VersionSMPA
-sndAuthKeySMPAgentVersion = VersionSMPA 6
+_sndAuthKeySMPAgentVersion :: VersionSMPA
+_sndAuthKeySMPAgentVersion = VersionSMPA 6
 
 ratchetOnConfSMPAgentVersion :: VersionSMPA
 ratchetOnConfSMPAgentVersion = VersionSMPA 7
 
+rpcAddressSMPAgentVersion :: VersionSMPA
+rpcAddressSMPAgentVersion = VersionSMPA 8
+
 minSupportedSMPAgentVersion :: VersionSMPA
-minSupportedSMPAgentVersion = duplexHandshakeSMPAgentVersion
+minSupportedSMPAgentVersion = _sndAuthKeySMPAgentVersion
 
 currentSMPAgentVersion :: VersionSMPA
-currentSMPAgentVersion = VersionSMPA 7
+currentSMPAgentVersion = VersionSMPA 8
 
 supportedSMPAgentVRange :: VersionRangeSMPA
 supportedSMPAgentVRange = mkVersionRange minSupportedSMPAgentVersion currentSMPAgentVersion
@@ -331,17 +336,17 @@ supportedSMPAgentVRange = mkVersionRange minSupportedSMPAgentVersion currentSMPA
 -- it is shorter to allow all handshake headers,
 -- including E2E (double-ratchet) parameters and
 -- signing key of the sender for the server
-e2eEncConnInfoLength :: VersionSMPA -> PQSupport -> Int
-e2eEncConnInfoLength v = \case
+e2eEncConnInfoLength :: PQSupport -> Int
+e2eEncConnInfoLength = \case
   -- reduced by 3726 (roughly the increase of message ratchet header size + key and ciphertext in reply link)
-  PQSupportOn | v >= pqdrSMPAgentVersion -> 11106
-  _ -> 14832
+  PQSupportOn -> 11106
+  PQSupportOff -> 14832
 
-e2eEncAgentMsgLength :: VersionSMPA -> PQSupport -> Int
-e2eEncAgentMsgLength v = \case
+e2eEncAgentMsgLength :: PQSupport -> Int
+e2eEncAgentMsgLength = \case
   -- reduced by 2222 (the increase of message ratchet header size)
-  PQSupportOn | v >= pqdrSMPAgentVersion -> 13618
-  _ -> 15840
+  PQSupportOn -> 13618
+  PQSupportOff -> 15840
 
 -- | SMP agent event
 type ATransmission = (ACorrId, AEntityId, AEvt)
@@ -393,13 +398,18 @@ type ConnInfo = ByteString
 
 type SndQueueSecured = Bool
 
+type UseRatchetKeys = Bool
+
 -- | Parameterized type for SMP agent events
 data AEvent (e :: AEntity) where
   INV :: AConnectionRequestUri -> AEvent AEConn
   LINK :: ConnShortLink 'CMContact -> UserConnLinkData 'CMContact -> AEvent AEConn
-  LDATA :: FixedLinkData 'CMContact -> ConnLinkData 'CMContact -> AEvent AEConn
+  LDATA :: FixedLinkData 'CMContact -> ConnLinkData 'CMContact -> ConnectionRequestUri 'CMContact -> AEvent AEConn
   CONF :: ConfirmationId -> PQSupport -> [SMPServer] -> ConnInfo -> AEvent AEConn -- ConnInfo is from sender, [SMPServer] will be empty only in v1 handshake
-  REQ :: InvitationId -> PQSupport -> NonEmpty SMPServer -> ConnInfo -> AEvent AEConn -- ConnInfo is from sender
+  REQ :: InvitationId -> PQSupport -> NonEmpty SMPServer -> ConnInfo -> ContactRequestBinding -> Bool -> AEvent AEConn -- ConnInfo is from sender; Bool - rejection reason can be sent
+  SREQ :: InvitationId -> Maybe C.PublicKeyEd25519 -> MsgBody -> AEvent AEConn
+  SSENT :: AgentMsgId -> Maybe SMPServer -> AEvent AEConn
+  RJCT :: ConnInfo -> AEvent AEConn
   INFO :: PQSupport -> ConnInfo -> AEvent AEConn
   CON :: PQEncryption -> AEvent AEConn -- notification that connection is established
   END :: AEvent AEConn
@@ -436,7 +446,7 @@ data AEvent (e :: AEntity) where
   RFERR :: AgentErrorType -> AEvent AERcvFile
   RFWARN :: AgentErrorType -> AEvent AERcvFile
   SFPROG :: Int64 -> Int64 -> AEvent AESndFile
-  SFDONE :: ValidFileDescription 'FSender -> [ValidFileDescription 'FRecipient] -> AEvent AESndFile
+  SFDONE :: ValidFileDescription 'FSender -> [ValidFileDescription 'FRecipient] -> Maybe GrantedStorageTime -> AEvent AESndFile
   SFERR :: AgentErrorType -> AEvent AESndFile
   SFWARN :: AgentErrorType -> AEvent AESndFile
 
@@ -454,15 +464,15 @@ instance Eq AEvtTag where
 deriving instance Show AEvtTag
 
 data ACommand
-  = NEW Bool AConnectionMode InitialKeys SubscriptionMode -- response INV
+  = NEW Bool AConnectionMode InitialKeys SubscriptionMode UseRatchetKeys -- response INV
   | LSET (UserConnLinkData 'CMContact) (Maybe CRClientData) -- response LINK
   | LGET (ConnShortLink 'CMContact) -- response LDATA
-  | JOIN Bool AConnectionRequestUri PQSupport SubscriptionMode ConnInfo
+  | JOIN JoinRequest SubscriptionMode ConnInfo
   | LET ConfirmationId ConnInfo -- ConnInfo is from client
   | ACK AgentMsgId (Maybe MsgReceiptInfo)
   | SWCH
   | DEL
-  deriving (Eq, Show)
+  deriving (Show)
 
 data ACommandTag
   = NEW_
@@ -481,6 +491,9 @@ data AEventTag (e :: AEntity) where
   LDATA_ :: AEventTag AEConn
   CONF_ :: AEventTag AEConn
   REQ_ :: AEventTag AEConn
+  SREQ_ :: AEventTag AEConn
+  SSENT_ :: AEventTag AEConn
+  RJCT_ :: AEventTag AEConn
   INFO_ :: AEventTag AEConn
   CON_ :: AEventTag AEConn
   END_ :: AEventTag AEConn
@@ -544,6 +557,9 @@ aEventTag = \case
   LDATA {} -> LDATA_
   CONF {} -> CONF_
   REQ {} -> REQ_
+  SREQ {} -> SREQ_
+  SSENT {} -> SSENT_
+  RJCT {} -> RJCT_
   INFO {} -> INFO_
   CON _ -> CON_
   END -> END_
@@ -625,16 +641,22 @@ instance FromJSON RcvSwitchStatus where
 data SndSwitchStatus
   = SSSendingQKEY
   | SSSendingQTEST
+  | SSSecuringQueue
+  | SSSendingQEND
   deriving (Eq, Show)
 
 instance StrEncoding SndSwitchStatus where
   strEncode = \case
     SSSendingQKEY -> "sending_qkey"
     SSSendingQTEST -> "sending_qtest"
+    SSSecuringQueue -> "securing_queue"
+    SSSendingQEND -> "sending_qend"
   strP =
     A.takeTill (== ' ') >>= \case
       "sending_qkey" -> pure SSSendingQKEY
       "sending_qtest" -> pure SSSendingQTEST
+      "securing_queue" -> pure SSSecuringQueue
+      "sending_qend" -> pure SSSendingQEND
       _ -> fail "bad SndSwitchStatus"
 
 instance ToField SndSwitchStatus where toField = toField . decodeLatin1 . strEncode
@@ -843,6 +865,12 @@ data AgentMsgEnvelope
         connReq :: ConnectionRequestUri 'CMInvitation,
         connInfo :: ByteString -- this message is only encrypted with per-queue E2E, not with double ratchet,
       }
+  | AgentContactRequest -- DR request to a contact address that published DR keys: a contact invitation or a service (RPC) request
+      { agentVersion :: VersionSMPA,
+        e2eSndParams :: SndE2ERatchetParams 'C.X448,
+        ratchetKeyId :: RatchetKeyId,
+        encConnInfo :: ByteString
+      }
   | AgentRatchetKey
       { agentVersion :: VersionSMPA,
         e2eEncryption :: RcvE2ERatchetParams 'C.X448,
@@ -858,6 +886,8 @@ instance Encoding AgentMsgEnvelope where
       smpEncode (agentVersion, 'M', Tail encAgentMessage)
     AgentInvitation {agentVersion, connReq, connInfo} ->
       smpEncode (agentVersion, 'I', Large $ strEncode connReq, Tail connInfo)
+    AgentContactRequest {agentVersion, e2eSndParams, ratchetKeyId, encConnInfo} ->
+      smpEncode (agentVersion, 'A', e2eSndParams, ratchetKeyId, Tail encConnInfo)
     AgentRatchetKey {agentVersion, e2eEncryption, info} ->
       smpEncode (agentVersion, 'R', e2eEncryption, Tail info)
   smpP = do
@@ -873,12 +903,22 @@ instance Encoding AgentMsgEnvelope where
         connReq <- strDecode . unLarge <$?> smpP
         Tail connInfo <- smpP
         pure AgentInvitation {agentVersion, connReq, connInfo}
+      'A' -> do
+        (e2eSndParams, ratchetKeyId, Tail encConnInfo) <- smpP
+        pure AgentContactRequest {agentVersion, e2eSndParams, ratchetKeyId, encConnInfo}
       'R' -> do
         e2eEncryption <- smpP
         Tail info <- smpP
         pure AgentRatchetKey {agentVersion, e2eEncryption, info}
       _ -> fail "bad AgentMsgEnvelope"
 
+data RequestSignature = RequestSignature C.PublicKeyEd25519 (C.Signature 'C.Ed25519)
+  deriving (Eq, Show)
+
+instance Encoding RequestSignature where
+  smpEncode (RequestSignature k sig) = smpEncode (k, sig)
+  smpP = RequestSignature <$> smpP <*> smpP
+
 -- SMP agent message formats (after double ratchet decryption,
 -- or in case of AgentInvitation - in plain text body)
 -- AgentRatchetInfo is not encrypted with double ratchet, but with per-queue E2E encryption
@@ -890,6 +930,9 @@ data AgentMessage
     AgentConnInfoReply (NonEmpty SMPQueueInfo) ConnInfo
   | AgentRatchetInfo ByteString
   | AgentMessage APrivHeader AMessage
+  | AgentServiceRequest (NonEmpty SMPQueueInfo) (Maybe RequestSignature) MsgBody
+  | AgentServiceResponse MsgBody
+  | AgentRejection ByteString
   deriving (Show)
 
 instance Encoding AgentMessage where
@@ -898,12 +941,18 @@ instance Encoding AgentMessage where
     AgentConnInfoReply smpQueues cInfo -> smpEncode ('D', smpQueues, Tail cInfo) -- 'D' stands for "duplex"
     AgentRatchetInfo info -> smpEncode ('R', Tail info)
     AgentMessage hdr aMsg -> smpEncode ('M', hdr, aMsg)
+    AgentServiceRequest qs sig_ body -> smpEncode ('A', qs, sig_, Tail body)
+    AgentServiceResponse body -> smpEncode ('P', Tail body)
+    AgentRejection reason -> smpEncode ('J', Tail reason)
   smpP =
     smpP >>= \case
       'I' -> AgentConnInfo . unTail <$> smpP
       'D' -> AgentConnInfoReply <$> smpP <*> (unTail <$> smpP)
       'R' -> AgentRatchetInfo . unTail <$> smpP
       'M' -> AgentMessage <$> smpP <*> smpP
+      'A' -> AgentServiceRequest <$> smpP <*> smpP <*> (unTail <$> smpP)
+      'P' -> AgentServiceResponse . unTail <$> smpP
+      'J' -> AgentRejection . unTail <$> smpP
       _ -> fail "bad AgentMessage"
 
 -- internal type for storing message type in the database
@@ -919,7 +968,11 @@ data AgentMessageType
   | AM_QKEY_
   | AM_QUSE_
   | AM_QTEST_
+  | AM_QEND_
   | AM_EREADY_
+  | AM_SRV_REQ
+  | AM_SRV_RESP
+  | AM_RJCT
   deriving (Eq, Show)
 
 instance Encoding AgentMessageType where
@@ -935,7 +988,11 @@ instance Encoding AgentMessageType where
     AM_QKEY_ -> "QK"
     AM_QUSE_ -> "QU"
     AM_QTEST_ -> "QT"
+    AM_QEND_ -> "QE"
     AM_EREADY_ -> "E"
+    AM_SRV_REQ -> "A"
+    AM_SRV_RESP -> "P"
+    AM_RJCT -> "J"
   smpP =
     A.anyChar >>= \case
       'C' -> pure AM_CONN_INFO
@@ -951,8 +1008,12 @@ instance Encoding AgentMessageType where
           'K' -> pure AM_QKEY_
           'U' -> pure AM_QUSE_
           'T' -> pure AM_QTEST_
+          'E' -> pure AM_QEND_
           _ -> fail "bad AgentMessageType"
       'E' -> pure AM_EREADY_
+      'A' -> pure AM_SRV_REQ
+      'P' -> pure AM_SRV_RESP
+      'J' -> pure AM_RJCT
       _ -> fail "bad AgentMessageType"
 
 agentMessageType :: AgentMessage -> AgentMessageType
@@ -961,6 +1022,9 @@ agentMessageType = \case
   AgentConnInfoReply {} -> AM_CONN_INFO_REPLY
   AgentRatchetInfo _ -> AM_RATCHET_INFO
   AgentMessage _ aMsg -> aMessageType aMsg
+  AgentServiceRequest {} -> AM_SRV_REQ
+  AgentServiceResponse {} -> AM_SRV_RESP
+  AgentRejection {} -> AM_RJCT
 
 data APrivHeader = APrivHeader
   { -- | sequential ID assigned by the sending agent
@@ -984,6 +1048,7 @@ data AMsgType
   | QKEY_
   | QUSE_
   | QTEST_
+  | QEND_
   | EREADY_
   deriving (Eq)
 
@@ -997,6 +1062,7 @@ instance Encoding AMsgType where
     QKEY_ -> "QK"
     QUSE_ -> "QU"
     QTEST_ -> "QT"
+    QEND_ -> "QE"
     EREADY_ -> "E"
   smpP =
     A.anyChar >>= \case
@@ -1010,6 +1076,7 @@ instance Encoding AMsgType where
           'K' -> pure QKEY_
           'U' -> pure QUSE_
           'T' -> pure QTEST_
+          'E' -> pure QEND_
           _ -> fail "bad AMsgType"
       'E' -> pure EREADY_
       _ -> fail "bad AMsgType"
@@ -1034,6 +1101,8 @@ data AMessage
     QUSE (NonEmpty (SndQAddr, Bool))
   | -- sent by the sender to test new queues and to complete switching
     QTEST (NonEmpty SndQAddr)
+  | -- sent by the sender to remove queues from the connection (fast rotation, v8)
+    QEND (NonEmpty SndQAddr)
   | -- ratchet re-synchronization is complete, with last decrypted sender message id (recipient's `last_external_snd_msg_id`)
     EREADY AgentMsgId
   deriving (Show)
@@ -1052,6 +1121,7 @@ aMessageType = \case
   QKEY _ -> AM_QKEY_
   QUSE _ -> AM_QUSE_
   QTEST _ -> AM_QTEST_
+  QEND _ -> AM_QEND_
   EREADY _ -> AM_EREADY_
 
 -- | this type is used to send as part of the protocol between different clients
@@ -1104,6 +1174,7 @@ instance Encoding AMessage where
     QKEY qs -> smpEncode (QKEY_, qs)
     QUSE qs -> smpEncode (QUSE_, qs)
     QTEST qs -> smpEncode (QTEST_, qs)
+    QEND qs -> smpEncode (QEND_, qs)
     EREADY lastDecryptedMsgId -> smpEncode (EREADY_, lastDecryptedMsgId)
   smpP =
     smpP
@@ -1116,6 +1187,7 @@ instance Encoding AMessage where
         QKEY_ -> QKEY <$> smpP
         QUSE_ -> QUSE <$> smpP
         QTEST_ -> QTEST <$> smpP
+        QEND_ -> QEND <$> smpP
         EREADY_ -> EREADY <$> smpP
 
 instance ToField AMessage where toField = toField . Binary . smpEncode
@@ -1131,11 +1203,13 @@ instance Encoding AMessageReceipt where
 
 instance ConnectionModeI m => StrEncoding (ConnectionRequestUri m) where
   strEncode = \case
-    CRInvitationUri crData e2eParams -> crEncode "invitation" crData (Just e2eParams)
-    CRContactUri crData -> crEncode "contact" crData Nothing
+    CRInvitationUri crData e2eParams -> crEncode "invitation" crData (Just e2eParams, Nothing)
+    CRContactUri crData rks -> crEncode "contact" crData $ case rks of
+      Just (ratchetKeyId, e2eRcvParams) -> (Just e2eRcvParams, Just ratchetKeyId)
+      Nothing -> (Nothing, Nothing)
     where
-      crEncode :: ByteString -> ConnReqUriData -> Maybe (RcvE2ERatchetParamsUri 'C.X448) -> ByteString
-      crEncode crMode ConnReqUriData {crScheme, crAgentVRange, crSmpQueues, crClientData} e2eParams =
+      crEncode :: ByteString -> ConnReqUriData -> (Maybe (RcvE2ERatchetParamsUri 'C.X448), Maybe RatchetKeyId) -> ByteString
+      crEncode crMode ConnReqUriData {crScheme, crAgentVRange, crSmpQueues, crClientData} (e2eParams, rk) =
         strEncode crScheme <> "/" <> crMode <> "#/?" <> queryStr
         where
           queryStr =
@@ -1143,23 +1217,24 @@ instance ConnectionModeI m => StrEncoding (ConnectionRequestUri m) where
               -- semicolon is used to separate SMP queues because comma is used to separate server address hostnames
               [("v", strEncode crAgentVRange), ("smp", B.intercalate ";" $ map strEncode $ L.toList crSmpQueues)]
                 <> maybe [] (\e2e -> [("e2e", strEncode e2e)]) e2eParams
+                <> maybe [] (\k -> [("rk", strEncode k)]) rk
                 <> maybe [] (\cd -> [("data", encodeUtf8 cd)]) crClientData
   strP = connReqUriP' (Just SSSimplex)
 
-instance ConnectionModeI m => Encoding (ConnectionRequestUri m) where
+instance ConnectionModeI m => Encoding (BinaryConnectionRequestUri m) where
   smpEncode = \case
-    CRInvitationUri crData e2eParams -> smpEncode (CMInvitation, crData, e2eParams)
-    CRContactUri crData -> smpEncode (CMContact, crData)
-  smpP = (\(ACR _ cr) -> checkConnMode cr) <$?> smpP
+    BCRInvitationUri crData e2eParams -> smpEncode (CMInvitation, crData, e2eParams)
+    BCRContactUri crData -> smpEncode (CMContact, crData)
+  smpP = (\(ABCR _ cr) -> checkConnMode cr) <$?> smpP
   {-# INLINE smpP #-}
 
-instance Encoding AConnectionRequestUri where
-  smpEncode (ACR _ cr) = smpEncode cr
+instance Encoding ABinaryConnectionRequestUri where
+  smpEncode (ABCR _ cr) = smpEncode cr
   {-# INLINE smpEncode #-}
   smpP =
     smpP >>= \case
-      CMInvitation -> ACR SCMInvitation <$> (CRInvitationUri <$> smpP <*> smpP)
-      CMContact -> ACR SCMContact . CRContactUri <$> smpP
+      CMInvitation -> ABCR SCMInvitation <$> (BCRInvitationUri <$> smpP <*> smpP)
+      CMContact -> ABCR SCMContact . BCRContactUri <$> smpP
 
 instance Encoding ConnReqUriData where
   smpEncode ConnReqUriData {crAgentVRange, crSmpQueues, crClientData} =
@@ -1202,7 +1277,10 @@ connReqUriP overrideScheme = do
       pure . ACR SCMInvitation $ CRInvitationUri crData crE2eParams
     -- contact links are adjusted to the minimum version supported by the agent
     -- to preserve compatibility with the old links published online
-    CMContact -> pure . ACR SCMContact $ CRContactUri crData {crAgentVRange = adjustAgentVRange aVRange}
+    CMContact -> do
+      e2e_ <- queryParam_ "e2e" query
+      rk_ <- queryParam_ "rk" query
+      pure . ACR SCMContact $ CRContactUri crData {crAgentVRange = adjustAgentVRange aVRange} ((,) <$> rk_ <*> e2e_)
   where
     crModeP = "invitation" $> CMInvitation <|> "contact" $> CMContact
     -- semicolon is used to separate SMP queues because comma is used to separate server address hostnames
@@ -1296,6 +1374,15 @@ type ConfirmationId = ByteString
 
 type InvitationId = ByteString
 
+data ConnVerifyCodes = ConnVerifyCodes
+  { codeAD :: ByteString,
+    codePQ :: Maybe ByteString
+  }
+  deriving (Eq, Show)
+
+data ContactRequestBinding = CRBRatchet ConnVerifyCodes | CRBRequest ByteString
+  deriving (Eq, Show)
+
 extraSMPServerHosts :: Map TransportHost TransportHost
 extraSMPServerHosts =
   M.fromList
@@ -1328,6 +1415,7 @@ sameQueue addr q = sameQAddress addr (qAddress q)
 
 data SMPQueueInfo = SMPQueueInfo {clientVersion :: VersionSMPC, queueAddress :: SMPQueueAddress}
   deriving (Eq, Show)
+  deriving (ToJSON, FromJSON) via (StrJSON "SMPQueueInfo" SMPQueueInfo)
 
 instance Encoding SMPQueueInfo where
   smpEncode (SMPQueueInfo clientVersion SMPQueueAddress {smpServer, senderId, dhPublicKey, queueMode})
@@ -1433,6 +1521,10 @@ instance StrEncoding SMPQueueUri where
               _ -> Nothing
         pure (vr, maybe [] thList_ hs_, dhKey, queueMode)
 
+instance StrEncoding SMPQueueInfo where
+  strEncode (SMPQueueInfo v addr) = strEncode (SMPQueueUri (versionToRange v) addr)
+  strP = (\(SMPQueueUri vr addr) -> SMPQueueInfo (maxVersion vr) addr) <$> strP
+
 instance Encoding SMPQueueUri where
   smpEncode (SMPQueueUri clientVRange@(VersionRange minV maxV) SMPQueueAddress {smpServer, senderId, dhPublicKey, queueMode})
     -- The condition is for minVersion as earlier clients won't be able to support it.
@@ -1452,16 +1544,30 @@ instance Encoding SMPQueueUri where
 queueModeP :: Parser (Maybe QueueMode)
 queueModeP = Just <$> smpP <|> optional ((\case True -> QMMessaging; _ -> QMContact) <$> smpP)
 
+data BinaryConnectionRequestUri (m :: ConnectionMode) where
+  BCRInvitationUri :: ConnReqUriData -> RcvE2ERatchetParamsUri 'C.X448 -> BinaryConnectionRequestUri CMInvitation
+  BCRContactUri :: ConnReqUriData -> BinaryConnectionRequestUri CMContact
+
+deriving instance Eq (BinaryConnectionRequestUri m)
+
+deriving instance Show (BinaryConnectionRequestUri m)
+
+data ABinaryConnectionRequestUri = forall m. ConnectionModeI m => ABCR (SConnectionMode m) (BinaryConnectionRequestUri m)
+
 data ConnectionRequestUri (m :: ConnectionMode) where
   CRInvitationUri :: ConnReqUriData -> RcvE2ERatchetParamsUri 'C.X448 -> ConnectionRequestUri CMInvitation
-  -- contact connection request does NOT contain E2E encryption parameters for double ratchet -
-  -- they are passed in AgentInvitation message
-  CRContactUri :: ConnReqUriData -> ConnectionRequestUri CMContact
+  -- optional contact address DR keys for double ratchet e2e from message 1
+  CRContactUri :: ConnReqUriData -> Maybe AddressRatchetKeys -> ConnectionRequestUri CMContact
 
 simplexConnReqUri :: ConnectionRequestUri m -> ConnectionRequestUri m
 simplexConnReqUri = \case
   CRInvitationUri crData e2eParams -> CRInvitationUri crData {crScheme = SSSimplex} e2eParams
-  CRContactUri crData -> CRContactUri crData {crScheme = SSSimplex}
+  CRContactUri crData rk -> CRContactUri crData {crScheme = SSSimplex} rk
+
+binaryConnReq :: ConnectionRequestUri m -> BinaryConnectionRequestUri m
+binaryConnReq = \case
+  CRInvitationUri crData e2eParams -> BCRInvitationUri crData e2eParams
+  CRContactUri crData _ -> BCRContactUri crData
 
 deriving instance Eq (ConnectionRequestUri m)
 
@@ -1519,9 +1625,12 @@ data PreparedLinkParams = PreparedLinkParams
     -- | smpEncode of FixedLinkData (includes linkEntityId)
     plpSignedFixedData :: ByteString,
     -- | Server with basic auth (not stored in link)
-    plpSrvWithAuth :: SMPServerWithAuth
+    plpSrvWithAuth :: SMPServerWithAuth,
+    -- | Initial PQ keys
+    plpInitKeys :: InitialKeys,
+    -- | Contact address double ratchet keys
+    plpAddressKeys :: Maybe (RatchetKeyId, RcvE2EPrivRatchetParams 'C.X448)
   }
-  deriving (Show)
 
 instance ConnectionModeI c => ToField (ConnectionLink c) where toField = toField . Binary . strEncode
 
@@ -1733,7 +1842,7 @@ findPresetServer ProtocolServer {host = h :| _} = find (\ProtocolServer {host =
 {-# INLINE findPresetServer #-}
 
 sameConnReqContact :: ConnectionRequestUri 'CMContact -> ConnectionRequestUri 'CMContact -> Bool
-sameConnReqContact (CRContactUri ConnReqUriData {crSmpQueues = qs}) (CRContactUri ConnReqUriData {crSmpQueues = qs'}) =
+sameConnReqContact (CRContactUri ConnReqUriData {crSmpQueues = qs} _) (CRContactUri ConnReqUriData {crSmpQueues = qs'} _) =
   L.length qs == L.length qs' && all same (L.zip qs qs')
   where
     same (q, q') = sameQAddress (qAddress q) (qAddress q')
@@ -1772,7 +1881,7 @@ type CRClientData = Text
 data FixedLinkData c = FixedLinkData
   { agentVRange :: VersionRangeSMPA,
     rootKey :: C.PublicKeyEd25519,
-    linkConnReq :: ConnectionRequestUri c,
+    linkConnReq :: BinaryConnectionRequestUri c,
     linkEntityId :: Maybe ByteString
   }
   deriving (Eq, Show)
@@ -1785,6 +1894,31 @@ deriving instance Eq (ConnLinkData c)
 
 deriving instance Show (ConnLinkData c)
 
+newtype RatchetKeyId = RatchetKeyId ByteString
+  deriving (Eq, Show)
+  deriving newtype (Encoding, StrEncoding)
+
+-- | double ratchet keys in contact address
+type AddressRatchetKeys = (RatchetKeyId, RcvE2ERatchetParamsUri 'C.X448)
+
+-- | Whether to rotate double ratchet keys in contact address
+type NewRatchetKeys = Bool
+
+-- | stored invitation with double ratchet keys
+data DRInvitation = DRInvitation
+  { ratchetState :: RatchetX448,
+    replyQueue :: SMPQueueInfo,
+    agentVersion :: VersionSMPA,
+    pqSupport :: PQSupport
+  }
+  deriving (Show)
+
+data JoinRequest
+  = JRConnReq {enableNtfs :: Bool, joinConnReq :: AConnectionRequestUri, joinPQSupport :: PQSupport}
+  | JRServiceReq {contactReq :: ConnectionRequestUri 'CMContact, joinPQSupport :: PQSupport, requestKey :: Maybe (C.StoredPrivateKey C.Ed25519)}
+  | JRInvitationDR DRInvitation
+  deriving (Show)
+
 data UserContactData = UserContactData
   { -- direct connection via connReq in fixed data is allowed.
     direct :: Bool,
@@ -1792,7 +1926,8 @@ data UserContactData = UserContactData
     owners :: [OwnerAuth],
     -- alternative addresses of chat relays that receive requests for this contact address.
     relays :: [ConnShortLink 'CMContact],
-    userData :: UserLinkData
+    userData :: UserLinkData,
+    ratchetKeys :: Maybe AddressRatchetKeys
   }
   deriving (Eq, Show)
 
@@ -1870,10 +2005,12 @@ validateLinkOwners rootKey = go []
 
 instance ConnectionModeI c => Encoding (FixedLinkData c) where
   smpEncode FixedLinkData {agentVRange, rootKey, linkConnReq, linkEntityId} =
+    -- TODO this encoding is not extensible, replace with smpEncode (fromMaybe "" linkEntityId) - safe to do it in 2027
     smpEncode (agentVRange, rootKey, linkConnReq) <> maybe "" smpEncode linkEntityId
   smpP = do
     (agentVRange, rootKey, linkConnReq) <- smpP
-    linkEntityId <- optional smpP <* A.takeByteString -- ignoring tail for forward compatibility with the future link data encoding
+    linkEntityId <- ((\s -> if B.null s then Nothing else Just s) =<<) <$> optional smpP
+    _ <- A.takeByteString -- ignoring tail for forward compatibility with the future link data encoding (added in January 2026)
     pure FixedLinkData {agentVRange, rootKey, linkConnReq, linkEntityId}
 
 instance ConnectionModeI c => Encoding (ConnLinkData c) where
@@ -1920,14 +2057,13 @@ instance ConnectionModeI c => StrEncoding (UserConnLinkData c) where
   {-# INLINE strP #-}
 
 instance Encoding UserContactData where
-  smpEncode UserContactData {direct, owners, relays, userData} =
-    B.concat [smpEncode direct, smpEncodeList owners, smpEncodeList relays, smpEncode userData]
+  smpEncode UserContactData {direct, owners, relays, userData, ratchetKeys} =
+    smpEncode (direct, EncList owners, EncList relays, userData, ratchetKeys)
   smpP = do
-    direct <- smpP
-    owners <- smpListP
-    relays <- smpListP
-    userData <- smpP <* A.takeByteString -- ignoring tail for forward compatibility with the future link data encoding
-    pure UserContactData {direct, owners, relays, userData}
+    (direct, EncList owners, EncList relays, userData) <- smpP
+    ratchetKeys <- smpP <|> pure Nothing
+    _ <- A.takeByteString -- ignoring tail for forward compatibility with the future link data encoding
+    pure UserContactData {direct, owners, relays, userData, ratchetKeys}
 
 instance Encoding UserLinkData where
   smpEncode (UserLinkData s) = if B.length s <= 254 then smpEncode s else smpEncode ('\255', Large s)
@@ -2075,7 +2211,7 @@ data ConnectionErrorType
 -- | Errors of another SMP agent.
 data SMPAgentError
   = -- | client or agent message that failed to parse
-    A_MESSAGE
+    A_MESSAGE {messageErr :: String}
   | -- | prohibited SMP/agent message
     A_PROHIBITED {prohibitedErr :: String}
   | -- | incompatible version of SMP client, agent or encryption protocols
@@ -2090,8 +2226,17 @@ data SMPAgentError
     A_DUPLICATE {droppedMsg_ :: Maybe DroppedMsg}
   | -- | error in the message to add/delete/etc queue in connection
     A_QUEUE {queueErr :: String}
+  | A_SERVICE {serviceError :: AgentServiceError}
   deriving (Eq, Show, Exception)
 
+data AgentServiceError
+  = ASERejected {rejectReason :: Text}
+  | ASETimeout
+  | ASENoPendingRequest
+  | ASENotDRAddress
+  | ASEBadSignature
+  deriving (Eq, Show)
+
 data AgentCryptoError
   = -- | AES decryption error
     DECRYPT_AES
@@ -2116,6 +2261,19 @@ cryptoErrToSyncState = \case
   RATCHET_SKIPPED _ -> RSRequired
   RATCHET_SYNC -> RSRequired
 
+$(J.deriveJSON defaultJSON ''DRInvitation)
+
+-- JRConnReq is identical to JOIN before DR support for old/new client compatibility
+instance StrEncoding JoinRequest where
+  strEncode = \case
+    JRConnReq ntfs cReq pqSup -> strEncode (ntfs, cReq, pqSup)
+    JRServiceReq cReq pqSup signKey -> strEncode ('S', cReq, pqSup, signKey)
+    JRInvitationDR dr -> serializeBinary $ LB.toStrict (J'.encode dr)
+  strP =
+    (A.char 'S' *> (JRServiceReq <$> _strP <*> _strP <*> _strP))
+      <|> (JRConnReq <$> strP <*> _strP <*> (_strP <|> pure PQSupportOff))
+      <|> (JRInvitationDR <$> (J'.eitherDecodeStrict' <$?> (A.take =<< (A.decimal <* "\n"))))
+
 -- | SMP agent command and response parser for commands stored in db (fully parses binary bodies)
 dbCommandP :: Parser ACommand
 dbCommandP = commandP $ A.take =<< (A.decimal <* "\n")
@@ -2146,10 +2304,11 @@ commandP :: Parser ByteString -> Parser ACommand
 commandP binaryP =
   strP
     >>= \case
-      NEW_ -> s (NEW <$> strP_ <*> strP_ <*> pqIKP <*> (strP <|> pure SMP.SMSubscribe))
+      -- useDR is a trailing field defaulting to False, so NEW persisted before it was added still parses
+      NEW_ -> s (NEW <$> strP_ <*> strP_ <*> pqIKP <*> (strP <|> pure SMP.SMSubscribe) <*> (_strP <|> pure False))
       LSET_ -> s (LSET <$> strP <*> optional (A.space *> strP))
       LGET_ -> s (LGET <$> strP)
-      JOIN_ -> s (JOIN <$> strP_ <*> strP_ <*> pqSupP <*> (strP_ <|> pure SMP.SMSubscribe) <*> binaryP)
+      JOIN_ -> s (JOIN <$> strP_ <*> (strP_ <|> pure SMP.SMSubscribe) <*> binaryP)
       LET_ -> s (LET <$> A.takeTill (== ' ') <* A.space <*> binaryP)
       ACK_ -> s (ACK <$> A.decimal <*> optional (A.space *> binaryP))
       SWCH_ -> pure SWCH
@@ -2159,16 +2318,14 @@ commandP binaryP =
     s p = A.space *> p
     pqIKP :: Parser InitialKeys
     pqIKP = strP_ <|> pure (IKLinkPQ PQSupportOff)
-    pqSupP :: Parser PQSupport
-    pqSupP = strP_ <|> pure PQSupportOff
 
 -- | Serialize SMP agent command.
 serializeCommand :: ACommand -> ByteString
 serializeCommand = \case
-  NEW ntfs cMode pqIK subMode -> s (NEW_, ntfs, cMode, pqIK, subMode)
+  NEW ntfs cMode pqIK subMode useDR -> s (NEW_, ntfs, cMode, pqIK, subMode, useDR)
   LSET uld cd_ -> s (LSET_, uld) <> maybe "" (B.cons ' ' . s) cd_
   LGET sl -> s (LGET_, sl)
-  JOIN ntfs cReq pqSup subMode cInfo -> s (JOIN_, ntfs, cReq, pqSup, subMode, Str $ serializeBinary cInfo)
+  JOIN joinReq subMode cInfo -> s (JOIN_, joinReq, subMode, Str $ serializeBinary cInfo)
   LET confId cInfo -> B.unwords [s LET_, confId, serializeBinary cInfo]
   ACK mId rcptInfo_ -> s (ACK_, mId) <> maybe "" (B.cons ' ' . serializeBinary) rcptInfo_
   SWCH -> s SWCH_
@@ -2200,6 +2357,8 @@ $(J.deriveJSON (sumTypeJSON id) ''ConnectionErrorType)
 
 $(J.deriveJSON (sumTypeJSON id) ''AgentCryptoError)
 
+$(J.deriveJSON (sumTypeJSON $ dropPrefix "ASE") ''AgentServiceError)
+
 $(J.deriveJSON defaultJSON ''DroppedMsg)
 
 $(J.deriveJSON (sumTypeJSON id) ''SMPAgentError)

src/Simplex/Messaging/Agent/RetryInterval.hs

@@ -10,6 +10,7 @@ module Simplex.Messaging.Agent.RetryInterval
     withRetryInterval,
     withRetryIntervalCount,
     withRetryForeground,
+    withRetryInterval2,
     withRetryLock2,
     updateRetryInterval2,
     nextRetryDelay,
@@ -86,9 +87,24 @@ withRetryForeground ri isForeground isOnline action = callAction 0 $ initialInte
                 | otherwise = (elapsed + delay, nextRetryDelay elapsed' delay ri)
           callAction elapsed' delay'
 
+withRetryInterval2 :: forall m. MonadIO m => RetryInterval2 -> (RI2State -> (RetryIntervalMode -> m ()) -> m ()) -> m ()
+withRetryInterval2 = withRetryWait2 $ liftIO . threadDelay'
+
 -- This function allows action to toggle between slow and fast retry intervals.
 withRetryLock2 :: forall m. MonadIO m => RetryInterval2 -> TMVar () -> (RI2State -> (RetryIntervalMode -> m ()) -> m ()) -> m ()
-withRetryLock2 RetryInterval2 {riSlow, riFast} lock action =
+withRetryLock2 ri lock = withRetryWait2 wait ri
+  where
+    wait delay = do
+      waiting <- newTVarIO True
+      _ <- liftIO . forkIO $ do
+        threadDelay' delay
+        atomically $ whenM (readTVar waiting) $ void $ tryPutTMVar lock ()
+      atomically $ do
+        takeTMVar lock
+        writeTVar waiting False
+
+withRetryWait2 :: forall m. Monad m => (Int64 -> m ()) -> RetryInterval2 -> (RI2State -> (RetryIntervalMode -> m ()) -> m ()) -> m ()
+withRetryWait2 wait RetryInterval2 {riSlow, riFast} action =
   callAction (0, initialInterval riSlow) (0, initialInterval riFast)
   where
     callAction :: (Int64, Int64) -> (Int64, Int64) -> m ()
@@ -102,14 +118,6 @@ withRetryLock2 RetryInterval2 {riSlow, riFast} lock action =
           let elapsed' = elapsed + delay
               delay' = nextRetryDelay elapsed' delay ri
           call (elapsed', delay')
-        wait delay = do
-          waiting <- newTVarIO True
-          _ <- liftIO . forkIO $ do
-            threadDelay' delay
-            atomically $ whenM (readTVar waiting) $ void $ tryPutTMVar lock ()
-          atomically $ do
-            takeTMVar lock
-            writeTVar waiting False
 
 nextRetryDelay :: Int64 -> Int64 -> RetryInterval -> Int64
 nextRetryDelay elapsed delay RetryInterval {increaseAfter, maxInterval} =

src/Simplex/Messaging/Agent/Store.hs

@@ -45,6 +45,8 @@ module Simplex.Messaging.Agent.Store
     AcceptedConfirmation (..),
     NewInvitation (..),
     Invitation (..),
+    ContactRequest (..),
+    DRInvitation (..),
     PrevExternalSndId,
     PrevRcvMsgHash,
     PrevSndMsgHash,
@@ -205,12 +207,13 @@ rcvSMPQueueAddress :: RcvQueue -> SMPQueueAddress
 rcvSMPQueueAddress RcvQueue {server, sndId, e2ePrivKey, queueMode} =
   SMPQueueAddress server sndId (C.publicKey e2ePrivKey) queueMode
 
-canAbortRcvSwitch :: RcvQueue -> Bool
-canAbortRcvSwitch = maybe False canAbort . rcvSwchStatus
+canAbortRcvSwitch :: ConnData -> RcvQueue -> Bool
+canAbortRcvSwitch ConnData {connAgentVersion} = maybe False canAbort . rcvSwchStatus
   where
     canAbort = \case
       RSSwitchStarted -> True
-      RSSendingQADD -> True
+      -- at agent version 8 and above the peer always chooses fast rotation, so a sent QADD is committed
+      RSSendingQADD -> connAgentVersion < rpcAddressSMPAgentVersion
       -- if switch is in RSSendingQUSE, a race condition with sender deleting the original queue is possible
       RSSendingQUSE -> False
       -- if switch is in RSReceivedMessage status, aborting switch (deleting new queue)
@@ -463,7 +466,9 @@ data ConnData = ConnData
     lastExternalSndId :: PrevExternalSndId,
     deleted :: Bool,
     ratchetSyncState :: RatchetSyncState,
-    pqSupport :: PQSupport
+    pqSupport :: PQSupport,
+    -- client side: set on the requester's connection for a service request; Nothing otherwise. The time the client stops waiting for the response (created + serviceRequestTimeout or per-call override).
+    serviceRequestExpiresAt :: Maybe UTCTime
   }
   deriving (Eq, Show)
 
@@ -471,8 +476,8 @@ type NoticeId = Int64
 
 -- this function should be mirrored in the clients
 ratchetSyncAllowed :: ConnData -> Bool
-ratchetSyncAllowed ConnData {ratchetSyncState, connAgentVersion} =
-  connAgentVersion >= ratchetSyncSMPAgentVersion && (ratchetSyncState `elem` ([RSAllowed, RSRequired] :: [RatchetSyncState]))
+ratchetSyncAllowed ConnData {ratchetSyncState} =
+  ratchetSyncState `elem` ([RSAllowed, RSRequired] :: [RatchetSyncState])
 
 -- this function should be mirrored in the clients
 ratchetSyncSendProhibited :: ConnData -> Bool
@@ -484,7 +489,8 @@ data PendingCommand = PendingCommand
     corrId :: ACorrId,
     userId :: UserId,
     connId :: ConnId,
-    command :: AgentCommand
+    command :: AgentCommand,
+    createdAt :: UTCTime
   }
 
 data AgentCmdType = ACClient | ACInternal
@@ -534,7 +540,9 @@ data InternalCommand
   | ICDeleteConn
   | ICDeleteRcvQueue SMP.RecipientId
   | ICQSecure SMP.RecipientId SMP.SndPublicAuthKey
+  | ICQSndSecure SMP.SenderId
   | ICQDelete SMP.RecipientId
+  | ICReplyDel
 
 data InternalCommandTag
   = ICAck_
@@ -544,7 +552,9 @@ data InternalCommandTag
   | ICDeleteConn_
   | ICDeleteRcvQueue_
   | ICQSecure_
+  | ICQSndSecure_
   | ICQDelete_
+  | ICReplyDel_
   deriving (Show)
 
 instance StrEncoding InternalCommand where
@@ -556,7 +566,9 @@ instance StrEncoding InternalCommand where
     ICDeleteConn -> strEncode ICDeleteConn_
     ICDeleteRcvQueue rId -> strEncode (ICDeleteRcvQueue_, rId)
     ICQSecure rId senderKey -> strEncode (ICQSecure_, rId, senderKey)
+    ICQSndSecure sId -> strEncode (ICQSndSecure_, sId)
     ICQDelete rId -> strEncode (ICQDelete_, rId)
+    ICReplyDel -> strEncode ICReplyDel_
   strP =
     strP >>= \case
       ICAck_ -> ICAck <$> _strP <*> _strP
@@ -566,7 +578,9 @@ instance StrEncoding InternalCommand where
       ICDeleteConn_ -> pure ICDeleteConn
       ICDeleteRcvQueue_ -> ICDeleteRcvQueue <$> _strP
       ICQSecure_ -> ICQSecure <$> _strP <*> _strP
+      ICQSndSecure_ -> ICQSndSecure <$> _strP
       ICQDelete_ -> ICQDelete <$> _strP
+      ICReplyDel_ -> pure ICReplyDel
 
 instance StrEncoding InternalCommandTag where
   strEncode = \case
@@ -577,7 +591,9 @@ instance StrEncoding InternalCommandTag where
     ICDeleteConn_ -> "DELETE_CONN"
     ICDeleteRcvQueue_ -> "DELETE_RCV_QUEUE"
     ICQSecure_ -> "QSECURE"
+    ICQSndSecure_ -> "QSND_SECURE"
     ICQDelete_ -> "QDELETE"
+    ICReplyDel_ -> "REPLY_DEL"
   strP =
     A.takeTill (== ' ') >>= \case
       "ACK" -> pure ICAck_
@@ -587,7 +603,9 @@ instance StrEncoding InternalCommandTag where
       "DELETE_CONN" -> pure ICDeleteConn_
       "DELETE_RCV_QUEUE" -> pure ICDeleteRcvQueue_
       "QSECURE" -> pure ICQSecure_
+      "QSND_SECURE" -> pure ICQSndSecure_
       "QDELETE" -> pure ICQDelete_
+      "REPLY_DEL" -> pure ICReplyDel_
       _ -> fail "bad InternalCommandTag"
 
 agentCommandTag :: AgentCommand -> AgentCommandTag
@@ -604,7 +622,9 @@ internalCmdTag = \case
   ICDeleteConn -> ICDeleteConn_
   ICDeleteRcvQueue {} -> ICDeleteRcvQueue_
   ICQSecure {} -> ICQSecure_
+  ICQSndSecure {} -> ICQSndSecure_
   ICQDelete _ -> ICQDelete_
+  ICReplyDel -> ICReplyDel_
 
 -- * Confirmation types
 
@@ -626,19 +646,28 @@ data AcceptedConfirmation = AcceptedConfirmation
 
 data NewInvitation = NewInvitation
   { contactConnId :: ConnId,
-    connReq :: ConnectionRequestUri 'CMInvitation,
-    recipientConnInfo :: ConnInfo
+    connReq :: ContactRequest,
+    recipientConnInfo :: ConnInfo,
+    -- service side: the received request is a service request (SREQ) not a contact request (REQ)
+    serviceRequest :: Bool
   }
 
 data Invitation = Invitation
   { invitationId :: InvitationId,
     contactConnId_ :: Maybe ConnId,
-    connReq :: ConnectionRequestUri 'CMInvitation,
+    connReq :: ContactRequest,
     recipientConnInfo :: ConnInfo,
     ownConnInfo :: Maybe ConnInfo,
-    accepted :: Bool
+    accepted :: Bool,
+    -- service side: the received request is a service request (SREQ) not a contact request (REQ)
+    serviceRequest :: Bool,
+    createdAt :: UTCTime
   }
 
+data ContactRequest
+  = CRInvitation (ConnectionRequestUri 'CMInvitation)
+  | CRInvitationDR DRInvitation
+
 -- * Message integrity validation types
 
 -- | Corresponds to `last_external_snd_msg_id` in `connections` table

src/Simplex/Messaging/Agent/Store/AgentStore.hs

@@ -73,6 +73,8 @@ module Simplex.Messaging.Agent.Store.AgentStore
     setConnPQSupport,
     updateNewConnJoin,
     getDeletedConnIds,
+    getExpiredServiceConns,
+    deleteExpiredServiceRequests,
     getDeletedWaitingDeliveryConnIds,
     setConnRatchetSync,
     addProcessedRatchetKeyHash,
@@ -129,6 +131,8 @@ module Simplex.Messaging.Agent.Store.AgentStore
     createSndMsg,
     updateSndMsgHash,
     createSndMsgDelivery,
+    copyPendingSndDeliveries,
+    countSndQueueDeliveries,
     getSndMsgViaRcpt,
     updateSndMsgRcpt,
     getPendingQueueMsg,
@@ -152,11 +156,17 @@ module Simplex.Messaging.Agent.Store.AgentStore
     createRatchetX3dhKeys,
     getRatchetX3dhKeys,
     setRatchetX3dhKeys,
+    createAddressRatchetKeys,
+    getCurrentAddressRatchetKeys,
+    getAddressRatchetKeys,
+    deleteOldAddressRatchetKeys,
     createSndRatchet,
     getSndRatchet,
     createRatchet,
     deleteRatchet,
     getRatchet,
+    getRatchetVerifyCodes,
+    ratchetVerifyCodes,
     getRatchetForUpdate,
     getSkippedMsgKeys,
     updateRatchet,
@@ -205,6 +215,7 @@ module Simplex.Messaging.Agent.Store.AgentStore
     -- Rcv files
     createRcvFile,
     createRcvFileRedirect,
+    startPreparedRcvFile,
     lockRcvFileForUpdate,
     getRcvFile,
     getRcvFileByEntityId,
@@ -226,6 +237,7 @@ module Simplex.Messaging.Agent.Store.AgentStore
     getRcvFilesExpired,
     -- Snd files
     createSndFile,
+    startPreparedSndFile,
     lockSndFileForUpdate,
     getSndFile,
     getSndFileByEntityId,
@@ -278,9 +290,12 @@ import Control.Monad.IO.Class
 import Control.Monad.Trans.Except
 import Crypto.Random (ChaChaDRG)
 import Data.Bifunctor (first)
+import qualified Data.Aeson as J
 import Data.ByteString (ByteString)
 import qualified Data.ByteString.Base64.URL as U
 import qualified Data.ByteString.Char8 as B
+import qualified Data.ByteString.Lazy as LB
+import Data.Either (partitionEithers)
 import Data.Functor (($>))
 import Data.Int (Int64)
 import Data.List (foldl', sortBy)
@@ -299,7 +314,7 @@ import Network.Socket (ServiceName)
 import qualified Network.TLS as TLS
 import Simplex.FileTransfer.Client (XFTPChunkSpec (..))
 import Simplex.FileTransfer.Description
-import Simplex.FileTransfer.Protocol (FileParty (..), SFileParty (..))
+import Simplex.FileTransfer.Protocol (FileParty (..), GrantedStorageTime (..), SFileParty (..))
 import Simplex.FileTransfer.Types
 import Simplex.Messaging.Agent.Protocol
 import Simplex.Messaging.Agent.RetryInterval (RI2State (..))
@@ -558,14 +573,16 @@ createSndConn db gVar cData q@SndQueue {server} =
       insertSndQueue_ db connId q serverKeyHash_
 
 createConnRecord :: DB.Connection -> ConnId -> ConnData -> SConnectionMode c -> IO ()
-createConnRecord db connId ConnData {userId, connAgentVersion, enableNtfs, pqSupport} cMode =
+createConnRecord db connId ConnData {userId, connAgentVersion, enableNtfs, pqSupport, serviceRequestExpiresAt} cMode = do
+  createdAt <- getCurrentTime
   DB.execute
     db
     [sql|
       INSERT INTO connections
-        (user_id, conn_id, conn_mode, smp_agent_version, enable_ntfs, pq_support, duplex_handshake) VALUES (?,?,?,?,?,?,?)
+        (user_id, conn_id, conn_mode, smp_agent_version, enable_ntfs, pq_support, service_request_expires_at, duplex_handshake, created_at)
+        VALUES (?,?,?,?,?,?,?,?,?)
     |]
-    (userId, connId, cMode, connAgentVersion, BI enableNtfs, pqSupport, BI True)
+    (userId, connId, cMode, connAgentVersion, BI enableNtfs, pqSupport, serviceRequestExpiresAt, BI True, createdAt)
 
 deleteConnRecord :: DB.Connection -> ConnId -> IO ()
 deleteConnRecord db connId = DB.execute db "DELETE FROM connections WHERE conn_id = ?" (Only connId)
@@ -868,15 +885,15 @@ removeConfirmations db connId =
     (Only connId)
 
 createInvitation :: DB.Connection -> TVar ChaChaDRG -> NewInvitation -> IO (Either StoreError InvitationId)
-createInvitation db gVar NewInvitation {contactConnId, connReq, recipientConnInfo} =
+createInvitation db gVar NewInvitation {contactConnId, connReq, recipientConnInfo, serviceRequest} =
   createWithRandomId db gVar $ \invitationId ->
     DB.execute
       db
       [sql|
         INSERT INTO conn_invitations
-        (invitation_id, contact_conn_id, cr_invitation, recipient_conn_info, accepted) VALUES (?, ?, ?, ?, 0);
+        (invitation_id, contact_conn_id, cr_invitation, recipient_conn_info, accepted, service_request) VALUES (?, ?, ?, ?, 0, ?);
       |]
-      (Binary invitationId, contactConnId, connReq, Binary recipientConnInfo)
+      (Binary invitationId, contactConnId, connReq, Binary recipientConnInfo, BI serviceRequest)
 
 getInvitation :: DB.Connection -> String -> InvitationId -> IO (Either StoreError Invitation)
 getInvitation db cxt invitationId =
@@ -884,15 +901,15 @@ getInvitation db cxt invitationId =
     DB.query
       db
       [sql|
-        SELECT contact_conn_id, cr_invitation, recipient_conn_info, own_conn_info, accepted
+        SELECT contact_conn_id, cr_invitation, recipient_conn_info, own_conn_info, accepted, service_request, created_at
         FROM conn_invitations
         WHERE invitation_id = ?
           AND accepted = 0
       |]
       (Only (Binary invitationId))
   where
-    invitation (contactConnId_, connReq, recipientConnInfo, ownConnInfo, BI accepted) =
-      Invitation {invitationId, contactConnId_, connReq, recipientConnInfo, ownConnInfo, accepted}
+    invitation (contactConnId_, connReq, recipientConnInfo, ownConnInfo, BI accepted, BI serviceRequest, createdAt) =
+      Invitation {invitationId, contactConnId_, connReq, recipientConnInfo, ownConnInfo, accepted, serviceRequest, createdAt}
 
 acceptInvitation :: DB.Connection -> InvitationId -> ConnInfo -> IO ()
 acceptInvitation db invitationId ownConnInfo =
@@ -1031,6 +1048,24 @@ createSndMsgDelivery :: DB.Connection -> SndQueue -> InternalId -> IO ()
 createSndMsgDelivery db SndQueue {connId, dbQueueId} msgId =
   DB.execute db "INSERT INTO snd_message_deliveries (conn_id, snd_queue_id, internal_id) VALUES (?, ?, ?)" (connId, dbQueueId, msgId)
 
+-- copies every undelivered (failed = 0) delivery from one snd queue to another, for redundant delivery during fast rotation
+copyPendingSndDeliveries :: DB.Connection -> SndQueue -> SndQueue -> IO ()
+copyPendingSndDeliveries db SndQueue {connId, dbQueueId = fromQueueId} SndQueue {dbQueueId = toQueueId} =
+  DB.execute
+    db
+    [sql|
+      INSERT INTO snd_message_deliveries (conn_id, snd_queue_id, internal_id)
+      SELECT conn_id, ?, internal_id
+      FROM snd_message_deliveries
+      WHERE conn_id = ? AND snd_queue_id = ? AND failed = 0
+    |]
+    (toQueueId, connId, fromQueueId)
+
+countSndQueueDeliveries :: DB.Connection -> SndQueue -> IO Int
+countSndQueueDeliveries db SndQueue {connId, dbQueueId} =
+  maybeFirstRow' 0 fromOnly $
+    DB.query db "SELECT count(1) FROM snd_message_deliveries WHERE conn_id = ? AND snd_queue_id = ? AND failed = 0" (connId, dbQueueId)
+
 getSndMsgViaRcpt :: DB.Connection -> ConnId -> InternalSndId -> IO (Either StoreError SndMsg)
 getSndMsgViaRcpt db connId sndMsgId =
   firstRow toSndMsg (SEMsgNotFound "getSndMsgViaRcpt") $
@@ -1384,15 +1419,71 @@ setRatchetX3dhKeys db connId (x3dhPrivKey1, x3dhPrivKey2, pqPrivKem) =
     |]
     (x3dhPrivKey1, x3dhPrivKey2, pqPrivKem, connId)
 
+createAddressRatchetKeys :: DB.Connection -> ConnId -> (RatchetKeyId, CR.RcvE2EPrivRatchetParams 'C.X448) -> IO ()
+createAddressRatchetKeys db connId (ratchetKeyId, (x3dhPrivKey1, x3dhPrivKey2, pqPrivKem)) =
+  DB.execute
+    db
+    [sql|
+      INSERT INTO address_ratchet_keys
+        (conn_id, ratchet_key_id, x3dh_priv_key_1, x3dh_priv_key_2, pq_priv_kem)
+      VALUES (?, ?, ?, ?, ?)
+    |]
+    (connId, ratchetKeyId, x3dhPrivKey1, x3dhPrivKey2, pqPrivKem)
+
+getCurrentAddressRatchetKeys :: DB.Connection -> ConnId -> IO (Either StoreError (RatchetKeyId, CR.RcvE2EPrivRatchetParams 'C.X448))
+getCurrentAddressRatchetKeys db connId =
+  firstRow (\(Only rkId :. pks) -> (rkId, pks)) SEX3dhKeysNotFound $
+    DB.query
+      db
+      [sql|
+        SELECT ratchet_key_id, x3dh_priv_key_1, x3dh_priv_key_2, pq_priv_kem
+        FROM address_ratchet_keys
+        WHERE conn_id = ?
+        ORDER BY address_ratchet_key_id DESC
+        LIMIT 1
+      |]
+      (Only connId)
+
+getAddressRatchetKeys :: DB.Connection -> ConnId -> RatchetKeyId -> IO (Either StoreError (CR.RcvE2EPrivRatchetParams 'C.X448))
+getAddressRatchetKeys db connId ratchetKeyId =
+  firstRow id SEX3dhKeysNotFound $
+    DB.query
+      db
+      [sql|
+        SELECT x3dh_priv_key_1, x3dh_priv_key_2, pq_priv_kem
+        FROM address_ratchet_keys
+        WHERE conn_id = ? AND ratchet_key_id = ?
+      |]
+      (connId, ratchetKeyId)
+
+deleteOldAddressRatchetKeys :: DB.Connection -> ConnId -> Int -> IO ()
+deleteOldAddressRatchetKeys db connId keep =
+  DB.execute
+    db
+    [sql|
+      DELETE FROM address_ratchet_keys
+      WHERE conn_id = ?
+        AND address_ratchet_key_id NOT IN (
+          SELECT address_ratchet_key_id
+          FROM address_ratchet_keys
+          WHERE conn_id = ?
+          ORDER BY address_ratchet_key_id DESC
+          LIMIT ?
+        )
+    |]
+    (connId, connId, keep)
+
 createSndRatchet :: DB.Connection -> ConnId -> RatchetX448 -> CR.AE2ERatchetParams 'C.X448 -> IO ()
 createSndRatchet db connId ratchetState (CR.AE2ERatchetParams s (CR.E2ERatchetParams _ x3dhPubKey1 x3dhPubKey2 pqPubKem)) =
   DB.execute
     db
     [sql|
       INSERT INTO ratchets
-        (conn_id, ratchet_state, x3dh_pub_key_1, x3dh_pub_key_2, pq_pub_kem) VALUES (?, ?, ?, ?, ?)
+        (conn_id, ratchet_state, rc_verify_code_ad, rc_verify_code_pq, x3dh_pub_key_1, x3dh_pub_key_2, pq_pub_kem) VALUES (?, ?, ?, ?, ?, ?, ?)
       ON CONFLICT (conn_id) DO UPDATE SET
         ratchet_state = EXCLUDED.ratchet_state,
+        rc_verify_code_ad = EXCLUDED.rc_verify_code_ad,
+        rc_verify_code_pq = EXCLUDED.rc_verify_code_pq,
         x3dh_priv_key_1 = NULL,
         x3dh_priv_key_2 = NULL,
         x3dh_pub_key_1 = EXCLUDED.x3dh_pub_key_1,
@@ -1400,7 +1491,13 @@ createSndRatchet db connId ratchetState (CR.AE2ERatchetParams s (CR.E2ERatchetPa
         pq_priv_kem = NULL,
         pq_pub_kem = EXCLUDED.pq_pub_kem
     |]
-    (connId, ratchetState, x3dhPubKey1, x3dhPubKey2, CR.ARKP s <$> pqPubKem)
+    ((connId, ratchetState) :. verifyCodesRow (ratchetVerifyCodes ratchetState) :. (x3dhPubKey1, x3dhPubKey2, CR.ARKP s <$> pqPubKem))
+
+ratchetVerifyCodes :: RatchetX448 -> ConnVerifyCodes
+ratchetVerifyCodes CR.Ratchet {rcAD, rcVCPQ} = ConnVerifyCodes {codeAD = C.sha256Hash $ unStr rcAD, codePQ = unStr <$> rcVCPQ}
+
+verifyCodesRow :: ConnVerifyCodes -> (Binary ByteString, Maybe (Binary ByteString))
+verifyCodesRow ConnVerifyCodes {codeAD, codePQ} = (Binary codeAD, Binary <$> codePQ)
 
 getSndRatchet :: DB.Connection -> ConnId -> CR.VersionE2E -> IO (Either StoreError (RatchetX448, CR.AE2ERatchetParams 'C.X448))
 getSndRatchet db connId v =
@@ -1421,10 +1518,12 @@ createRatchet db connId rc =
   DB.execute
     db
     [sql|
-      INSERT INTO ratchets (conn_id, ratchet_state)
-      VALUES (?, ?)
+      INSERT INTO ratchets (conn_id, ratchet_state, rc_verify_code_ad, rc_verify_code_pq)
+      VALUES (?, ?, ?, ?)
       ON CONFLICT (conn_id) DO UPDATE SET
-        ratchet_state = ?,
+        ratchet_state = EXCLUDED.ratchet_state,
+        rc_verify_code_ad = EXCLUDED.rc_verify_code_ad,
+        rc_verify_code_pq = EXCLUDED.rc_verify_code_pq,
         x3dh_priv_key_1 = NULL,
         x3dh_priv_key_2 = NULL,
         x3dh_pub_key_1 = NULL,
@@ -1432,7 +1531,42 @@ createRatchet db connId rc =
         pq_priv_kem = NULL,
         pq_pub_kem = NULL
     |]
-    (connId, rc, rc)
+    ((connId, rc) :. verifyCodesRow (ratchetVerifyCodes rc))
+
+getRatchetVerifyCodes :: DB.Connection -> [ConnId] -> IO (Map ConnId ConnVerifyCodes)
+getRatchetVerifyCodes db connIds = do
+  (saved, computed) <- partitionEithers . mapMaybe rowCodes <$> selectCodes
+  stored <- saveCodes computed
+  pure $ M.fromList $ saved <> stored
+  where
+    rowCodes (connId, codeAD_, codePQ_, rc_) = case codeAD_ of
+      Just codeAD -> Just $ Left $ toCodes (connId, codeAD, codePQ_)
+      Nothing -> Right . (connId,) . ratchetVerifyCodes <$> rc_
+    toCodes (connId, Binary codeAD, codePQ_) = (connId, ConnVerifyCodes {codeAD, codePQ = fromBinary <$> codePQ_})
+    codesRow (connId, cs) = verifyCodesRow cs :. Only connId
+    codesQuery :: Query
+    codesQuery = "SELECT conn_id, rc_verify_code_ad, rc_verify_code_pq, CASE WHEN rc_verify_code_ad IS NULL THEN ratchet_state END FROM ratchets"
+    selectCodes :: IO [(ConnId, Maybe (Binary ByteString), Maybe (Binary ByteString), Maybe RatchetX448)]
+    saveCodes :: [(ConnId, ConnVerifyCodes)] -> IO [(ConnId, ConnVerifyCodes)]
+#if defined(dbPostgres)
+    selectCodes = DB.query db (codesQuery <> " WHERE conn_id IN ?") (Only (In connIds))
+    saveCodes cs =
+      map toCodes
+        <$> DB.returning
+          db
+          [sql|
+            UPDATE ratchets r
+            SET rc_verify_code_ad = COALESCE(r.rc_verify_code_ad, (upd.rc_verify_code_ad :: BYTEA)),
+              rc_verify_code_pq = CASE WHEN r.rc_verify_code_ad IS NULL THEN (upd.rc_verify_code_pq :: BYTEA) ELSE r.rc_verify_code_pq END
+            FROM (VALUES(?, ?, ?)) AS upd(rc_verify_code_ad, rc_verify_code_pq, conn_id)
+            WHERE r.conn_id = (upd.conn_id :: BYTEA)
+            RETURNING r.conn_id, r.rc_verify_code_ad, r.rc_verify_code_pq
+          |]
+          (map codesRow cs)
+#else
+    selectCodes = concat <$> mapM (DB.query db (codesQuery <> " WHERE conn_id = ?") . Only) connIds
+    saveCodes cs = cs <$ unless (null cs) (DB.executeMany db "UPDATE ratchets SET rc_verify_code_ad = ?, rc_verify_code_pq = ? WHERE conn_id = ?" $ map codesRow cs)
+#endif
 
 deleteRatchet :: DB.Connection -> ConnId -> IO ()
 deleteRatchet db connId =
@@ -1584,7 +1718,7 @@ getPendingServerCommand db connId srv_ = getWorkItem "command" getCmdId getComma
         DB.query
           db
           [sql|
-            SELECT c.corr_id, cs.user_id, c.command
+            SELECT c.corr_id, cs.user_id, c.command, c.created_at
             FROM commands c
             JOIN connections cs USING (conn_id)
             WHERE c.command_id = ?
@@ -1592,7 +1726,7 @@ getPendingServerCommand db connId srv_ = getWorkItem "command" getCmdId getComma
           (Only cmdId)
       where
         err = SEInternal $ "command  " <> bshow cmdId <> " returned []"
-        pendingCommand (corrId, userId, command) = PendingCommand {cmdId, corrId, userId, connId, command}
+        pendingCommand (corrId, userId, command, createdAt) = PendingCommand {cmdId, corrId, userId, connId, command, createdAt}
     markCommandFailed cmdId = DB.execute db "UPDATE commands SET failed = 1 WHERE command_id = ?" (Only cmdId)
 
 updateCommandServer :: DB.Connection -> AsyncCmdId -> SMPServer -> IO (Either StoreError ())
@@ -2075,6 +2209,21 @@ instance ConnectionModeI c => ToField (ConnectionRequestUri c) where toField = t
 
 instance (E.Typeable c, ConnectionModeI c) => FromField (ConnectionRequestUri c) where fromField = blobFieldDecoder strDecode
 
+instance ToField RatchetKeyId where toField (RatchetKeyId s) = toField $ Binary s
+
+instance FromField RatchetKeyId where fromField = blobFieldDecoder $ Right . RatchetKeyId
+
+instance ToField ContactRequest where
+  toField = toField . Binary . \case
+    CRInvitation cr -> strEncode cr
+    CRInvitationDR dr -> LB.toStrict $ J.encode dr
+
+instance FromField ContactRequest where
+  fromField = blobFieldDecoder $ \bs ->
+    if "{" `B.isPrefixOf` bs
+      then CRInvitationDR <$> J.eitherDecodeStrict' bs
+      else CRInvitation <$> strDecode bs
+
 instance ToField ConnectionMode where toField = toField . decodeLatin1 . strEncode
 
 instance FromField ConnectionMode where fromField = fromTextField_ connModeT
@@ -2351,12 +2500,12 @@ getSubscriptionServers db onlyNeeded =
 
 -- TODO [certs rcv] check index for getting queues with service present
 getUserServerRcvQueueSubs :: DB.Connection -> UserId -> SMPServer -> Bool -> ServiceAssoc -> Int -> Maybe SMP.RecipientId -> IO [RcvQueueSub]
-getUserServerRcvQueueSubs db userId (SMPServer h p kh) onlyNeeded hasService limit cursor_ =
-  map toRcvQueueSub <$> case cursor_ of
+getUserServerRcvQueueSubs db userId srv@(SMPServer h p kh) onlyNeeded hasService limit cursor_ =
+  map (toServerRcvQueueSub srv) <$> case cursor_ of
     Nothing -> DB.query db (q <> orderLimit) (userId, h, p, kh, limit)
     Just cursor -> DB.query db (q <> " AND q.rcv_id > ? " <> orderLimit) (userId, h, p, kh, cursor, limit)
   where
-    q = rcvQueueSubQuery <> toSubscribe <> " c.deleted = 0 AND q.deleted = 0 AND c.user_id = ? AND q.host = ? AND q.port = ? AND COALESCE(q.server_key_hash, s.key_hash) = ?" <> serviceCond
+    q = serverRcvQueueSubQuery <> toSubscribe <> " c.deleted = 0 AND q.deleted = 0 AND c.user_id = ? AND q.host = ? AND q.port = ? AND COALESCE(q.server_key_hash, s.key_hash) = ?" <> serviceCond
     orderLimit = " ORDER BY q.rcv_id LIMIT ?"
     toSubscribe
       | onlyNeeded = " WHERE q.to_subscribe = 1 AND "
@@ -2369,7 +2518,7 @@ unassocUserServerRcvQueueSubs :: DB.Connection -> UserId -> SMPServer -> IO [Rcv
 unassocUserServerRcvQueueSubs db userId srv@(SMPServer h p kh) = do
   deleteClientService db userId srv
 #if defined(dbPostgres)
-  map toRcvQueueSub
+  map (toServerRcvQueueSub srv)
     <$> DB.query
       db
       (removeRcvAssocsQuery <> " " <> returningColumns)
@@ -2377,15 +2526,15 @@ unassocUserServerRcvQueueSubs db userId srv@(SMPServer h p kh) = do
   where
     returningColumns =
       [sql|
-        RETURNING c.user_id, rcv_queues.conn_id, rcv_queues.host, rcv_queues.port, COALESCE(rcv_queues.server_key_hash, s.key_hash),
+        RETURNING c.user_id, rcv_queues.conn_id,
           rcv_queues.rcv_id, rcv_queues.rcv_private_key, rcv_queues.status, c.enable_ntfs, rcv_queues.client_notice_id,
           rcv_queues.rcv_queue_id, rcv_queues.rcv_primary, rcv_queues.replace_rcv_queue_id
       |]
 #else
-  qs <- map toRcvQueueSub
+  qs <- map (toServerRcvQueueSub srv)
     <$> DB.query
       db
-      (rcvQueueSubQuery <> " WHERE c.user_id = ? AND q.host = ? AND q.port = ? AND COALESCE(q.server_key_hash, s.key_hash) = ? AND q.rcv_service_assoc = 1")
+      (serverRcvQueueSubQuery <> " WHERE c.user_id = ? AND q.host = ? AND q.port = ? AND COALESCE(q.server_key_hash, s.key_hash) = ? AND q.rcv_service_assoc = 1")
       (userId, h, p, kh)
   DB.execute db removeRcvAssocsQuery (h, p, userId, kh)
   pure qs
@@ -2549,7 +2698,7 @@ getConnsData_ deleted' db connIds =
       db
       [sql|
         SELECT user_id, conn_id, conn_mode, smp_agent_version, enable_ntfs,
-          last_external_snd_msg_id, deleted, ratchet_sync_state, pq_support
+          last_external_snd_msg_id, deleted, ratchet_sync_state, pq_support, service_request_expires_at
         FROM connections
         WHERE conn_id IN ? AND deleted = ?
       |]
@@ -2584,7 +2733,7 @@ getConnData deleted' forUpdate db connId' =
       db
       ( [sql|
           SELECT user_id, conn_id, conn_mode, smp_agent_version, enable_ntfs,
-            last_external_snd_msg_id, deleted, ratchet_sync_state, pq_support
+            last_external_snd_msg_id, deleted, ratchet_sync_state, pq_support, service_request_expires_at
           FROM connections
           WHERE conn_id = ? AND deleted = ?
         |]
@@ -2601,9 +2750,9 @@ lockConnForUpdate db connId = do
 #endif
   pure ()
 
-rowToConnData :: (UserId, ConnId, ConnectionMode, VersionSMPA, Maybe BoolInt, PrevExternalSndId, BoolInt, RatchetSyncState, PQSupport) -> (ConnData, ConnectionMode)
-rowToConnData (userId, connId, cMode, connAgentVersion, enableNtfs_, lastExternalSndId, BI deleted, ratchetSyncState, pqSupport) =
-  (ConnData {userId, connId, connAgentVersion, enableNtfs = maybe True unBI enableNtfs_, lastExternalSndId, deleted, ratchetSyncState, pqSupport}, cMode)
+rowToConnData :: (UserId, ConnId, ConnectionMode, VersionSMPA, Maybe BoolInt, PrevExternalSndId, BoolInt, RatchetSyncState, PQSupport, Maybe UTCTime) -> (ConnData, ConnectionMode)
+rowToConnData (userId, connId, cMode, connAgentVersion, enableNtfs_, lastExternalSndId, BI deleted, ratchetSyncState, pqSupport, serviceRequestExpiresAt) =
+  (ConnData {userId, connId, connAgentVersion, enableNtfs = maybe True unBI enableNtfs_, lastExternalSndId, deleted, ratchetSyncState, pqSupport, serviceRequestExpiresAt}, cMode)
 
 setConnDeleted :: DB.Connection -> Bool -> ConnId -> IO ()
 setConnDeleted db waitDelivery connId
@@ -2632,6 +2781,14 @@ updateNewConnJoin db connId aVersion pqSupport enableNtfs =
 getDeletedConnIds :: DB.Connection -> IO [ConnId]
 getDeletedConnIds db = map fromOnly <$> DB.query db "SELECT conn_id FROM connections WHERE deleted = ?" (Only (BI True))
 
+getExpiredServiceConns :: DB.Connection -> UTCTime -> IO [ConnId]
+getExpiredServiceConns db now =
+  map fromOnly <$> DB.query db "SELECT conn_id FROM connections WHERE service_request_expires_at < ? AND deleted = 0 AND deleted_at_wait_delivery IS NULL" (Only now)
+
+deleteExpiredServiceRequests :: DB.Connection -> UTCTime -> IO ()
+deleteExpiredServiceRequests db expireTs =
+  DB.execute db "DELETE FROM conn_invitations WHERE service_request = 1 AND created_at < ?" (Only expireTs)
+
 getDeletedWaitingDeliveryConnIds :: DB.Connection -> IO [ConnId]
 getDeletedWaitingDeliveryConnIds db =
   map fromOnly <$> DB.query_ db "SELECT conn_id FROM connections WHERE deleted_at_wait_delivery IS NOT NULL"
@@ -2728,17 +2885,32 @@ getRcvQueueSubsByConnId_ db connId =
 rcvQueueSubQuery :: Query
 rcvQueueSubQuery =
   [sql|
-    SELECT c.user_id, q.conn_id, q.host, q.port, COALESCE(q.server_key_hash, s.key_hash), q.rcv_id, q.rcv_private_key, q.status, c.enable_ntfs, q.client_notice_id,
+    SELECT q.host, q.port, COALESCE(q.server_key_hash, s.key_hash), c.user_id, q.conn_id, q.rcv_id, q.rcv_private_key, q.status, c.enable_ntfs, q.client_notice_id,
       q.rcv_queue_id, q.rcv_primary, q.replace_rcv_queue_id
     FROM rcv_queues q
     JOIN servers s ON q.host = s.host AND q.port = s.port
     JOIN connections c ON q.conn_id = c.conn_id
   |]
 
-toRcvQueueSub :: (UserId, ConnId, NonEmpty TransportHost, ServiceName, C.KeyHash, SMP.RecipientId, SMP.RcvPrivateAuthKey) :. (QueueStatus, Maybe BoolInt, Maybe NoticeId, Int64, BoolInt, Maybe Int64) -> RcvQueueSub
-toRcvQueueSub ((userId, connId, host, port, keyHash, rcvId, rcvPrivateKey) :. (status, enableNtfs_, clientNoticeId, dbQueueId, BI primary, dbReplaceQueueId)) =
+serverRcvQueueSubQuery :: Query
+serverRcvQueueSubQuery =
+  [sql|
+    SELECT c.user_id, q.conn_id, q.rcv_id, q.rcv_private_key, q.status, c.enable_ntfs, q.client_notice_id,
+      q.rcv_queue_id, q.rcv_primary, q.replace_rcv_queue_id
+    FROM rcv_queues q
+    JOIN servers s ON q.host = s.host AND q.port = s.port
+    JOIN connections c ON q.conn_id = c.conn_id
+  |]
+
+type RcvQueueSubRow = (UserId, ConnId, SMP.RecipientId, SMP.RcvPrivateAuthKey) :. (QueueStatus, Maybe BoolInt, Maybe NoticeId, Int64, BoolInt, Maybe Int64)
+
+toRcvQueueSub :: (NonEmpty TransportHost, ServiceName, C.KeyHash) :. RcvQueueSubRow -> RcvQueueSub
+toRcvQueueSub ((host, port, keyHash) :. r) = toServerRcvQueueSub (SMPServer host port keyHash) r
+
+toServerRcvQueueSub :: SMPServer -> RcvQueueSubRow -> RcvQueueSub
+toServerRcvQueueSub server ((userId, connId, rcvId, rcvPrivateKey) :. (status, enableNtfs_, clientNoticeId, dbQueueId, BI primary, dbReplaceQueueId)) =
   let enableNtfs = maybe True unBI enableNtfs_
-   in RcvQueueSub {userId, connId, server = SMPServer host port keyHash, rcvId, rcvPrivateKey, status, enableNtfs, clientNoticeId, dbQueueId, primary, dbReplaceQueueId}
+   in RcvQueueSub {userId, connId, server, rcvId, rcvPrivateKey, status, enableNtfs, clientNoticeId, dbQueueId, primary, dbReplaceQueueId}
 
 getRcvQueueById :: DB.Connection -> ConnId -> Int64 -> IO (Either StoreError RcvQueue)
 getRcvQueueById db connId dbRcvId =
@@ -3008,6 +3180,29 @@ createRcvFileRedirect db gVar userId redirectFd@FileDescription {chunks = redire
           chunks = []
         }
 
+startPreparedRcvFile :: DB.Connection -> RcvFileId -> IO (Either StoreError [XFTPServer])
+startPreparedRcvFile db rcvFileEntityId = runExceptT $ do
+  rcvFileId <- ExceptT $ getRcvFileIdByEntityId_ db rcvFileEntityId
+  liftIO $ do
+    updatedAt <- getCurrentTime
+    DB.execute
+      db
+      "UPDATE rcv_files SET status = ?, updated_at = ? WHERE (rcv_file_id = ? OR redirect_id = ?) AND status = ?"
+      (RFSReceiving, updatedAt, rcvFileId, rcvFileId, RFSPrepared)
+    map toXFTPServer
+      <$> DB.query
+        db
+        [sql|
+          SELECT DISTINCT
+            s.xftp_host, s.xftp_port, s.xftp_key_hash
+          FROM rcv_file_chunk_replicas r
+          JOIN xftp_servers s ON s.xftp_server_id = r.xftp_server_id
+          JOIN rcv_file_chunks c ON c.rcv_file_chunk_id = r.rcv_file_chunk_id
+          JOIN rcv_files f ON f.rcv_file_id = c.rcv_file_id
+          WHERE (f.rcv_file_id = ? OR f.redirect_id = ?) AND r.replica_number = 1
+        |]
+        (rcvFileId, rcvFileId)
+
 insertRcvFile :: DB.Connection -> TVar ChaChaDRG -> UserId -> FileDescription 'FRecipient -> FilePath -> FilePath -> CryptoFile -> Maybe DBRcvFileId -> Maybe RcvFileId -> Bool -> IO (Either StoreError (RcvFileId, DBRcvFileId))
 insertRcvFile db gVar userId FileDescription {size, digest, key, nonce, chunkSize, redirect} prefixPath tmpPath (CryptoFile savePath cfArgs) redirectId_ redirectEntityId_ approvedRelays = runExceptT $ do
   let (redirectDigest_, redirectSize_) = case redirect of
@@ -3018,7 +3213,7 @@ insertRcvFile db gVar userId FileDescription {size, digest, key, nonce, chunkSiz
       DB.execute
         db
         "INSERT INTO rcv_files (rcv_file_entity_id, user_id, size, digest, key, nonce, chunk_size, prefix_path, tmp_path, save_path, save_file_key, save_file_nonce, status, redirect_id, redirect_entity_id, redirect_digest, redirect_size, approved_relays) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
-        ((Binary rcvFileEntityId, userId, size, digest, key, nonce, chunkSize, prefixPath, tmpPath) :. (savePath, fileKey <$> cfArgs, fileNonce <$> cfArgs, RFSReceiving, redirectId_, Binary <$> redirectEntityId_, redirectDigest_, redirectSize_, BI approvedRelays))
+        ((Binary rcvFileEntityId, userId, size, digest, key, nonce, chunkSize, prefixPath, tmpPath) :. (savePath, fileKey <$> cfArgs, fileNonce <$> cfArgs, RFSPrepared, redirectId_, Binary <$> redirectEntityId_, redirectDigest_, redirectSize_, BI approvedRelays))
   rcvFileId <- liftIO $ insertedRowId db
   pure (rcvFileEntityId, rcvFileId)
 
@@ -3317,19 +3512,29 @@ getRcvFilesExpired db ttl = do
     |]
     (Only cutoffTs)
 
-createSndFile :: DB.Connection -> TVar ChaChaDRG -> UserId -> CryptoFile -> Int -> FilePath -> C.SbKey -> C.CbNonce -> Maybe RedirectFileInfo -> IO (Either StoreError SndFileId)
-createSndFile db gVar userId (CryptoFile path cfArgs) numRecipients prefixPath key nonce redirect_ =
+createSndFile :: DB.Connection -> TVar ChaChaDRG -> UserId -> CryptoFile -> Int -> FilePath -> C.SbKey -> C.CbNonce -> Maybe RedirectFileInfo -> Maybe Word32 -> IO (Either StoreError SndFileId)
+createSndFile db gVar userId (CryptoFile path cfArgs) numRecipients prefixPath key nonce redirect_ storageHours =
   createWithRandomId db gVar $ \sndFileEntityId ->
     DB.execute
       db
-      "INSERT INTO snd_files (snd_file_entity_id, user_id, path, src_file_key, src_file_nonce, num_recipients, prefix_path, key, nonce, status, redirect_size, redirect_digest) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)"
-      ((Binary sndFileEntityId, userId, path, fileKey <$> cfArgs, fileNonce <$> cfArgs, numRecipients) :. (prefixPath, key, nonce, SFSNew, redirectSize_, redirectDigest_))
+      "INSERT INTO snd_files (snd_file_entity_id, user_id, path, src_file_key, src_file_nonce, num_recipients, prefix_path, key, nonce, status, redirect_size, redirect_digest, storage_time) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)"
+      ((Binary sndFileEntityId, userId, path, fileKey <$> cfArgs, fileNonce <$> cfArgs, numRecipients) :. (prefixPath, key, nonce, SFSPrepared, redirectSize_, redirectDigest_, storageHours))
   where
     (redirectSize_, redirectDigest_) =
       case redirect_ of
         Nothing -> (Nothing, Nothing)
         Just RedirectFileInfo {size, digest} -> (Just size, Just digest)
 
+startPreparedSndFile :: DB.Connection -> SndFileId -> IO (Either StoreError ())
+startPreparedSndFile db sndFileEntityId = runExceptT $ do
+  sndFileId <- ExceptT $ getSndFileIdByEntityId_ db sndFileEntityId
+  liftIO $ do
+    updatedAt <- getCurrentTime
+    DB.execute
+      db
+      "UPDATE snd_files SET status = ?, updated_at = ? WHERE snd_file_id = ? AND status = ?"
+      (SFSNew, updatedAt, sndFileId, SFSPrepared)
+
 getSndFileByEntityId :: DB.Connection -> SndFileId -> IO (Either StoreError SndFile)
 getSndFileByEntityId db sndFileEntityId = runExceptT $ do
   sndFileId <- ExceptT $ getSndFileIdByEntityId_ db sndFileEntityId
@@ -3359,7 +3564,7 @@ getSndFile db sndFileId = runExceptT $ do
         DB.query
           db
           ( [sql|
-              SELECT snd_file_entity_id, user_id, path, src_file_key, src_file_nonce, num_recipients, digest, prefix_path, key, nonce, status, deleted, redirect_size, redirect_digest
+              SELECT snd_file_entity_id, user_id, path, src_file_key, src_file_nonce, num_recipients, digest, prefix_path, key, nonce, status, deleted, redirect_size, redirect_digest, storage_time
               FROM snd_files
               WHERE snd_file_id = ?
             |]
@@ -3369,12 +3574,12 @@ getSndFile db sndFileId = runExceptT $ do
           )
           (Only sndFileId)
       where
-        toFile :: (SndFileId, UserId, FilePath, Maybe C.SbKey, Maybe C.CbNonce, Int, Maybe FileDigest, Maybe FilePath, C.SbKey, C.CbNonce) :. (SndFileStatus, BoolInt, Maybe (FileSize Int64), Maybe FileDigest) -> SndFile
-        toFile ((sndFileEntityId, userId, srcPath, srcKey_, srcNonce_, numRecipients, digest, prefixPath, key, nonce) :. (status, BI deleted, redirectSize_, redirectDigest_)) =
+        toFile :: (SndFileId, UserId, FilePath, Maybe C.SbKey, Maybe C.CbNonce, Int, Maybe FileDigest, Maybe FilePath, C.SbKey, C.CbNonce) :. (SndFileStatus, BoolInt, Maybe (FileSize Int64), Maybe FileDigest, Maybe Word32) -> SndFile
+        toFile ((sndFileEntityId, userId, srcPath, srcKey_, srcNonce_, numRecipients, digest, prefixPath, key, nonce) :. (status, BI deleted, redirectSize_, redirectDigest_, storageHours)) =
           let cfArgs = CFArgs <$> srcKey_ <*> srcNonce_
               srcFile = CryptoFile srcPath cfArgs
               redirect = RedirectFileInfo <$> redirectSize_ <*> redirectDigest_
-           in SndFile {sndFileId, sndFileEntityId, userId, srcFile, numRecipients, digest, prefixPath, key, nonce, status, deleted, redirect, chunks = []}
+           in SndFile {sndFileId, sndFileEntityId, userId, srcFile, numRecipients, digest, prefixPath, key, nonce, status, deleted, redirect, storageHours, chunks = []}
     getChunks :: SndFileId -> UserId -> Int -> FilePath -> IO [SndFileChunk]
     getChunks sndFileEntityId userId numRecipients filePrefixPath = do
       chunks <-
@@ -3403,7 +3608,7 @@ getSndFile db sndFileId = runExceptT $ do
             db
             [sql|
               SELECT
-                r.snd_file_chunk_replica_id, r.replica_id, r.replica_key, r.replica_status, r.delay, r.retries,
+                r.snd_file_chunk_replica_id, r.replica_id, r.replica_key, r.replica_status, r.delay, r.retries, r.replica_expires_at,
                 s.xftp_host, s.xftp_port, s.xftp_key_hash
               FROM snd_file_chunk_replicas r
               JOIN xftp_servers s ON s.xftp_server_id = r.xftp_server_id
@@ -3414,10 +3619,10 @@ getSndFile db sndFileId = runExceptT $ do
         rcvIdsKeys <- getChunkReplicaRecipients_ db sndChunkReplicaId
         pure (replica :: SndFileChunkReplica) {rcvIdsKeys}
       where
-        toReplica :: (Int64, ChunkReplicaId, C.APrivateAuthKey, SndFileReplicaStatus, Maybe Int64, Int, NonEmpty TransportHost, ServiceName, C.KeyHash) -> SndFileChunkReplica
-        toReplica (sndChunkReplicaId, replicaId, replicaKey, replicaStatus, delay, retries, host, port, keyHash) =
+        toReplica :: (Int64, ChunkReplicaId, C.APrivateAuthKey, SndFileReplicaStatus, Maybe Int64, Int, Maybe Int64, NonEmpty TransportHost, ServiceName, C.KeyHash) -> SndFileChunkReplica
+        toReplica (sndChunkReplicaId, replicaId, replicaKey, replicaStatus, delay, retries, expiresAtSec, host, port, keyHash) =
           let server = XFTPServer host port keyHash
-           in SndFileChunkReplica {sndChunkReplicaId, server, replicaId, replicaKey, replicaStatus, delay, retries, rcvIdsKeys = []}
+           in SndFileChunkReplica {sndChunkReplicaId, server, replicaId, replicaKey, replicaStatus, delay, retries, expiresAt = GSTExpires <$> expiresAtSec, rcvIdsKeys = []}
 
 getChunkReplicaRecipients_ :: DB.Connection -> Int64 -> IO [(ChunkReplicaId, C.APrivateAuthKey)]
 getChunkReplicaRecipients_ db replicaId =
@@ -3498,16 +3703,16 @@ createSndFileReplica :: DB.Connection -> SndFileChunk -> NewSndChunkReplica -> I
 createSndFileReplica db SndFileChunk {sndChunkId} = createSndFileReplica_ db sndChunkId
 
 createSndFileReplica_ :: DB.Connection -> Int64 -> NewSndChunkReplica -> IO ()
-createSndFileReplica_ db sndChunkId NewSndChunkReplica {server, replicaId, replicaKey, rcvIdsKeys} = do
+createSndFileReplica_ db sndChunkId NewSndChunkReplica {server, replicaId, replicaKey, rcvIdsKeys, expiresAt} = do
   srvId <- createXFTPServer_ db server
   DB.execute
     db
     [sql|
       INSERT INTO snd_file_chunk_replicas
-        (snd_file_chunk_id, replica_number, xftp_server_id, replica_id, replica_key, replica_status)
-      VALUES (?,?,?,?,?,?)
+        (snd_file_chunk_id, replica_number, xftp_server_id, replica_id, replica_key, replica_status, replica_expires_at)
+      VALUES (?,?,?,?,?,?,?)
     |]
-    (sndChunkId, 1 :: Int, srvId, replicaId, replicaKey, SFRSCreated)
+    (sndChunkId, 1 :: Int, srvId, replicaId, replicaKey, SFRSCreated, epochSeconds <$> expiresAt)
   rId <- insertedRowId db
   forM_ rcvIdsKeys $ \(rcvId, rcvKey) -> do
     DB.execute
@@ -3553,7 +3758,7 @@ getNextSndChunkToUpload db server@ProtocolServer {host, port, keyHash} ttl = do
               SELECT
                 f.snd_file_id, f.snd_file_entity_id, f.user_id, f.num_recipients, f.prefix_path,
                 c.snd_file_chunk_id, c.chunk_no, c.chunk_offset, c.chunk_size, c.digest,
-                r.snd_file_chunk_replica_id, r.replica_id, r.replica_key, r.replica_status, r.delay, r.retries
+                r.snd_file_chunk_replica_id, r.replica_id, r.replica_key, r.replica_status, r.delay, r.retries, r.replica_expires_at
               FROM snd_file_chunk_replicas r
               JOIN xftp_servers s ON s.xftp_server_id = r.xftp_server_id
               JOIN snd_file_chunks c ON c.snd_file_chunk_id = r.snd_file_chunk_id
@@ -3567,8 +3772,8 @@ getNextSndChunkToUpload db server@ProtocolServer {host, port, keyHash} ttl = do
           pure (replica :: SndFileChunkReplica) {rcvIdsKeys}
         pure (chunk {replicas = replicas'} :: SndFileChunk)
       where
-        toChunk :: ((DBSndFileId, SndFileId, UserId, Int, FilePath) :. (Int64, Int, Int64, Word32, FileDigest) :. (Int64, ChunkReplicaId, C.APrivateAuthKey, SndFileReplicaStatus, Maybe Int64, Int)) -> SndFileChunk
-        toChunk ((sndFileId, sndFileEntityId, userId, numRecipients, filePrefixPath) :. (sndChunkId, chunkNo, chunkOffset, chunkSize, digest) :. (sndChunkReplicaId, replicaId, replicaKey, replicaStatus, delay, retries)) =
+        toChunk :: ((DBSndFileId, SndFileId, UserId, Int, FilePath) :. (Int64, Int, Int64, Word32, FileDigest) :. (Int64, ChunkReplicaId, C.APrivateAuthKey, SndFileReplicaStatus, Maybe Int64, Int, Maybe Int64)) -> SndFileChunk
+        toChunk ((sndFileId, sndFileEntityId, userId, numRecipients, filePrefixPath) :. (sndChunkId, chunkNo, chunkOffset, chunkSize, digest) :. (sndChunkReplicaId, replicaId, replicaKey, replicaStatus, delay, retries, expiresAtSec)) =
           let chunkSpec = XFTPChunkSpec {filePath = sndFileEncPath filePrefixPath, chunkOffset, chunkSize}
            in SndFileChunk
                 { sndFileId,
@@ -3580,7 +3785,7 @@ getNextSndChunkToUpload db server@ProtocolServer {host, port, keyHash} ttl = do
                   chunkSpec,
                   digest,
                   filePrefixPath,
-                  replicas = [SndFileChunkReplica {sndChunkReplicaId, server, replicaId, replicaKey, replicaStatus, delay, retries, rcvIdsKeys = []}]
+                  replicas = [SndFileChunkReplica {sndChunkReplicaId, server, replicaId, replicaKey, replicaStatus, delay, retries, expiresAt = GSTExpires <$> expiresAtSec, rcvIdsKeys = []}]
                 }
 
 updateSndChunkReplicaDelay :: DB.Connection -> Int64 -> Int64 -> IO ()

src/Simplex/Messaging/Agent/Store/Postgres/DB.hs

@@ -13,6 +13,7 @@ module Simplex.Messaging.Agent.Store.Postgres.DB
     execute,
     execute_,
     executeMany,
+    returning,
     query,
     query_,
     blobFieldDecoder,
@@ -60,6 +61,10 @@ executeMany :: ToRow q => PSQL.Connection -> Query -> [q] -> IO ()
 executeMany db q qs = void $ PSQL.executeMany db q qs `E.catch` addSql q
 {-# INLINE executeMany #-}
 
+returning :: (ToRow q, FromRow r) => PSQL.Connection -> Query -> [q] -> IO [r]
+returning db q qs = PSQL.returning db q qs `E.catch` addSql q
+{-# INLINE returning #-}
+
 query :: (ToRow q, FromRow r) => PSQL.Connection -> Query -> q -> IO [r]
 query db q qs = PSQL.query db q qs `E.catch` addSql q
 {-# INLINE query #-}

src/Simplex/Messaging/Agent/Store/Postgres/Migrations/App.hs

@@ -13,6 +13,9 @@ import Simplex.Messaging.Agent.Store.Postgres.Migrations.M20251010_client_notice
 import Simplex.Messaging.Agent.Store.Postgres.Migrations.M20251230_strict_tables
 import Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260410_receive_attempts
 import Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260411_service_certs
+import Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260712_address_dr_rpc
+import Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260823_snd_files_entitlement
+import Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260919_ratchet_verify_codes
 import Simplex.Messaging.Agent.Store.Shared (Migration (..))
 
 schemaMigrations :: [(String, Text, Maybe Text)]
@@ -25,7 +28,10 @@ schemaMigrations =
     ("20251010_client_notices", m20251010_client_notices, Just down_m20251010_client_notices),
     ("20251230_strict_tables", m20251230_strict_tables, Just down_m20251230_strict_tables),
     ("20260410_receive_attempts", m20260410_receive_attempts, Just down_m20260410_receive_attempts),
-    ("20260411_service_certs", m20260411_service_certs, Just down_m20260411_service_certs)
+    ("20260411_service_certs", m20260411_service_certs, Just down_m20260411_service_certs),
+    ("20260712_address_dr_rpc", m20260712_address_dr_rpc, Just down_m20260712_address_dr_rpc),
+    ("20260823_snd_files_entitlement", m20260823_snd_files_entitlement, Just down_m20260823_snd_files_entitlement),
+    ("20260919_ratchet_verify_codes", m20260919_ratchet_verify_codes, Just down_m20260919_ratchet_verify_codes)
   ]
 
 -- | The list of migrations in ascending order by date

src/Simplex/Messaging/Agent/Store/Postgres/Migrations/M20260712_address_dr_rpc.hs

@@ -0,0 +1,42 @@
+{-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE QuasiQuotes #-}
+
+module Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260712_address_dr_rpc where
+
+import Data.Text (Text)
+import Text.RawString.QQ (r)
+
+m20260712_address_dr_rpc :: Text
+m20260712_address_dr_rpc =
+  [r|
+CREATE TABLE address_ratchet_keys(
+  address_ratchet_key_id BIGSERIAL PRIMARY KEY,
+  conn_id BYTEA NOT NULL REFERENCES connections ON DELETE CASCADE,
+  ratchet_key_id BYTEA NOT NULL,
+  x3dh_priv_key_1 BYTEA NOT NULL,
+  x3dh_priv_key_2 BYTEA NOT NULL,
+  pq_priv_kem BYTEA,
+  created_at TIMESTAMPTZ NOT NULL DEFAULT (now())
+);
+
+CREATE UNIQUE INDEX idx_address_ratchet_keys ON address_ratchet_keys(conn_id, ratchet_key_id);
+
+ALTER TABLE conn_invitations ADD COLUMN service_request SMALLINT NOT NULL DEFAULT 0; -- service side: received request is a service request (SREQ) not a contact request (REQ)
+ALTER TABLE connections ADD COLUMN created_at TIMESTAMPTZ NOT NULL DEFAULT '1970-01-01 00:00:00';
+ALTER TABLE connections ADD COLUMN service_request_expires_at TIMESTAMPTZ; -- client side: requester's outstanding service request; the time the client stops waiting for the response
+
+CREATE INDEX idx_connections_deleted ON connections(deleted);
+CREATE INDEX idx_connections_service_request_expires_at ON connections(service_request_expires_at);
+  |]
+
+down_m20260712_address_dr_rpc :: Text
+down_m20260712_address_dr_rpc =
+  [r|
+DROP INDEX idx_connections_service_request_expires_at;
+DROP INDEX idx_connections_deleted;
+ALTER TABLE connections DROP COLUMN service_request_expires_at;
+ALTER TABLE connections DROP COLUMN created_at;
+ALTER TABLE conn_invitations DROP COLUMN service_request;
+DROP INDEX idx_address_ratchet_keys;
+DROP TABLE address_ratchet_keys;
+  |]

src/Simplex/Messaging/Agent/Store/Postgres/Migrations/M20260823_snd_files_entitlement.hs

@@ -0,0 +1,21 @@
+{-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE QuasiQuotes #-}
+
+module Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260823_snd_files_entitlement where
+
+import Data.Text (Text)
+import Text.RawString.QQ (r)
+
+m20260823_snd_files_entitlement :: Text
+m20260823_snd_files_entitlement =
+  [r|
+ALTER TABLE snd_files ADD COLUMN storage_time BIGINT;
+ALTER TABLE snd_file_chunk_replicas ADD COLUMN replica_expires_at BIGINT;
+|]
+
+down_m20260823_snd_files_entitlement :: Text
+down_m20260823_snd_files_entitlement =
+  [r|
+ALTER TABLE snd_file_chunk_replicas DROP COLUMN replica_expires_at;
+ALTER TABLE snd_files DROP COLUMN storage_time;
+|]

src/Simplex/Messaging/Agent/Store/Postgres/Migrations/M20260919_ratchet_verify_codes.hs

@@ -0,0 +1,21 @@
+{-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE QuasiQuotes #-}
+
+module Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260919_ratchet_verify_codes where
+
+import Data.Text (Text)
+import Text.RawString.QQ (r)
+
+m20260919_ratchet_verify_codes :: Text
+m20260919_ratchet_verify_codes =
+  [r|
+ALTER TABLE ratchets ADD COLUMN rc_verify_code_ad BYTEA;
+ALTER TABLE ratchets ADD COLUMN rc_verify_code_pq BYTEA;
+|]
+
+down_m20260919_ratchet_verify_codes :: Text
+down_m20260919_ratchet_verify_codes =
+  [r|
+ALTER TABLE ratchets DROP COLUMN rc_verify_code_pq;
+ALTER TABLE ratchets DROP COLUMN rc_verify_code_ad;
+|]

src/Simplex/Messaging/Agent/Store/Postgres/Migrations/agent_postgres_schema.sql

@@ -104,6 +104,31 @@ CREATE AGGREGATE smp_agent_test_protocol_schema.xor_aggregate(bytea) (
 SET default_table_access_method = heap;
 
 
+CREATE TABLE smp_agent_test_protocol_schema.address_ratchet_keys (
+    address_ratchet_key_id bigint NOT NULL,
+    conn_id bytea NOT NULL,
+    ratchet_key_id bytea NOT NULL,
+    x3dh_priv_key_1 bytea NOT NULL,
+    x3dh_priv_key_2 bytea NOT NULL,
+    pq_priv_kem bytea,
+    created_at timestamp with time zone DEFAULT now() NOT NULL
+);
+
+
+
+CREATE SEQUENCE smp_agent_test_protocol_schema.address_ratchet_keys_address_ratchet_key_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+
+
+ALTER SEQUENCE smp_agent_test_protocol_schema.address_ratchet_keys_address_ratchet_key_id_seq OWNED BY smp_agent_test_protocol_schema.address_ratchet_keys.address_ratchet_key_id;
+
+
+
 CREATE TABLE smp_agent_test_protocol_schema.client_notices (
     client_notice_id bigint NOT NULL,
     protocol text NOT NULL,
@@ -194,7 +219,8 @@ CREATE TABLE smp_agent_test_protocol_schema.conn_invitations (
     recipient_conn_info bytea NOT NULL,
     accepted smallint DEFAULT 0 NOT NULL,
     own_conn_info bytea,
-    created_at timestamp with time zone DEFAULT now() NOT NULL
+    created_at timestamp with time zone DEFAULT now() NOT NULL,
+    service_request smallint DEFAULT 0 NOT NULL
 );
 
 
@@ -215,7 +241,9 @@ CREATE TABLE smp_agent_test_protocol_schema.connections (
     user_id bigint NOT NULL,
     ratchet_sync_state text DEFAULT 'ok'::text NOT NULL,
     deleted_at_wait_delivery timestamp with time zone,
-    pq_support smallint DEFAULT 0 NOT NULL
+    pq_support smallint DEFAULT 0 NOT NULL,
+    created_at timestamp with time zone DEFAULT '1970-01-01 00:00:00+01'::timestamp with time zone NOT NULL,
+    service_request_expires_at timestamp with time zone
 );
 
 
@@ -420,7 +448,9 @@ CREATE TABLE smp_agent_test_protocol_schema.ratchets (
     x3dh_pub_key_1 bytea,
     x3dh_pub_key_2 bytea,
     pq_priv_kem bytea,
-    pq_pub_kem bytea
+    pq_pub_kem bytea,
+    rc_verify_code_ad bytea,
+    rc_verify_code_pq bytea
 );
 
 
@@ -654,7 +684,8 @@ CREATE TABLE smp_agent_test_protocol_schema.snd_file_chunk_replicas (
     delay bigint,
     retries bigint DEFAULT 0 NOT NULL,
     created_at timestamp with time zone DEFAULT now() NOT NULL,
-    updated_at timestamp with time zone DEFAULT now() NOT NULL
+    updated_at timestamp with time zone DEFAULT now() NOT NULL,
+    replica_expires_at bigint
 );
 
 
@@ -713,7 +744,8 @@ CREATE TABLE smp_agent_test_protocol_schema.snd_files (
     src_file_nonce bytea,
     failed smallint DEFAULT 0,
     redirect_size bigint,
-    redirect_digest bytea
+    redirect_digest bytea,
+    storage_time bigint
 );
 
 
@@ -847,6 +879,15 @@ ALTER TABLE smp_agent_test_protocol_schema.xftp_servers ALTER COLUMN xftp_server
 
 
 
+ALTER TABLE ONLY smp_agent_test_protocol_schema.address_ratchet_keys ALTER COLUMN address_ratchet_key_id SET DEFAULT nextval('smp_agent_test_protocol_schema.address_ratchet_keys_address_ratchet_key_id_seq'::regclass);
+
+
+
+ALTER TABLE ONLY smp_agent_test_protocol_schema.address_ratchet_keys
+    ADD CONSTRAINT address_ratchet_keys_pkey PRIMARY KEY (address_ratchet_key_id);
+
+
+
 ALTER TABLE ONLY smp_agent_test_protocol_schema.client_notices
     ADD CONSTRAINT client_notices_pkey PRIMARY KEY (client_notice_id);
 
@@ -1032,6 +1073,10 @@ ALTER TABLE ONLY smp_agent_test_protocol_schema.xftp_servers
 
 
 
+CREATE UNIQUE INDEX idx_address_ratchet_keys ON smp_agent_test_protocol_schema.address_ratchet_keys USING btree (conn_id, ratchet_key_id);
+
+
+
 CREATE UNIQUE INDEX idx_client_notices_entity ON smp_agent_test_protocol_schema.client_notices USING btree (protocol, host, port, entity_id);
 
 
@@ -1056,6 +1101,14 @@ CREATE INDEX idx_conn_invitations_contact_conn_id ON smp_agent_test_protocol_sch
 
 
 
+CREATE INDEX idx_connections_deleted ON smp_agent_test_protocol_schema.connections USING btree (deleted);
+
+
+
+CREATE INDEX idx_connections_service_request_expires_at ON smp_agent_test_protocol_schema.connections USING btree (service_request_expires_at);
+
+
+
 CREATE INDEX idx_connections_user ON smp_agent_test_protocol_schema.connections USING btree (user_id);
 
 
@@ -1268,6 +1321,11 @@ CREATE TRIGGER tr_rcv_queue_update AFTER UPDATE ON smp_agent_test_protocol_schem
 
 
 
+ALTER TABLE ONLY smp_agent_test_protocol_schema.address_ratchet_keys
+    ADD CONSTRAINT address_ratchet_keys_conn_id_fkey FOREIGN KEY (conn_id) REFERENCES smp_agent_test_protocol_schema.connections(conn_id) ON DELETE CASCADE;
+
+
+
 ALTER TABLE ONLY smp_agent_test_protocol_schema.client_services
     ADD CONSTRAINT client_services_host_port_fkey FOREIGN KEY (host, port) REFERENCES smp_agent_test_protocol_schema.servers(host, port) ON DELETE RESTRICT;
 

src/Simplex/Messaging/Agent/Store/SQLite.hs

@@ -50,7 +50,6 @@ import Data.Bits (xor)
 import Data.ByteArray (ScrubbedBytes)
 import qualified Data.ByteArray as BA
 import Data.ByteString (ByteString)
-import qualified Data.ByteString as B
 import Data.Functor (($>))
 import Data.IORef
 import Data.Maybe (fromMaybe)

src/Simplex/Messaging/Agent/Store/SQLite/Migrations/App.hs

@@ -49,6 +49,9 @@ import Simplex.Messaging.Agent.Store.SQLite.Migrations.M20251010_client_notices
 import Simplex.Messaging.Agent.Store.SQLite.Migrations.M20251230_strict_tables
 import Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260410_receive_attempts
 import Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260411_service_certs
+import Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260712_address_dr_rpc
+import Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260823_snd_files_entitlement
+import Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260919_ratchet_verify_codes
 import Simplex.Messaging.Agent.Store.Shared (Migration (..))
 
 schemaMigrations :: [(String, Query, Maybe Query)]
@@ -97,7 +100,10 @@ schemaMigrations =
     ("m20251010_client_notices", m20251010_client_notices, Just down_m20251010_client_notices),
     ("m20251230_strict_tables", m20251230_strict_tables, Just down_m20251230_strict_tables),
     ("m20260410_receive_attempts", m20260410_receive_attempts, Just down_m20260410_receive_attempts),
-    ("m20260411_service_certs", m20260411_service_certs, Just down_m20260411_service_certs)
+    ("m20260411_service_certs", m20260411_service_certs, Just down_m20260411_service_certs),
+    ("m20260712_address_dr_rpc", m20260712_address_dr_rpc, Just down_m20260712_address_dr_rpc),
+    ("m20260823_snd_files_entitlement", m20260823_snd_files_entitlement, Just down_m20260823_snd_files_entitlement),
+    ("m20260919_ratchet_verify_codes", m20260919_ratchet_verify_codes, Just down_m20260919_ratchet_verify_codes)
   ]
 
 -- | The list of migrations in ascending order by date

src/Simplex/Messaging/Agent/Store/SQLite/Migrations/M20260712_address_dr_rpc.hs

@@ -0,0 +1,41 @@
+{-# LANGUAGE QuasiQuotes #-}
+
+module Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260712_address_dr_rpc where
+
+import Database.SQLite.Simple (Query)
+import Database.SQLite.Simple.QQ (sql)
+
+m20260712_address_dr_rpc :: Query
+m20260712_address_dr_rpc =
+  [sql|
+CREATE TABLE address_ratchet_keys(
+  address_ratchet_key_id INTEGER PRIMARY KEY AUTOINCREMENT,
+  conn_id BLOB NOT NULL REFERENCES connections ON DELETE CASCADE,
+  ratchet_key_id BLOB NOT NULL,
+  x3dh_priv_key_1 BLOB NOT NULL,
+  x3dh_priv_key_2 BLOB NOT NULL,
+  pq_priv_kem BLOB,
+  created_at TEXT NOT NULL DEFAULT(datetime('now'))
+) STRICT;
+
+CREATE UNIQUE INDEX idx_address_ratchet_keys ON address_ratchet_keys(conn_id, ratchet_key_id);
+
+ALTER TABLE conn_invitations ADD COLUMN service_request INTEGER NOT NULL DEFAULT 0; -- service side: received request is a service request (SREQ) not a contact request (REQ)
+ALTER TABLE connections ADD COLUMN created_at TEXT NOT NULL DEFAULT('1970-01-01 00:00:00');
+ALTER TABLE connections ADD COLUMN service_request_expires_at TEXT; -- client side: requester's outstanding service request; the time the client stops waiting for the response
+
+CREATE INDEX idx_connections_deleted ON connections(deleted);
+CREATE INDEX idx_connections_service_request_expires_at ON connections(service_request_expires_at);
+  |]
+
+down_m20260712_address_dr_rpc :: Query
+down_m20260712_address_dr_rpc =
+  [sql|
+DROP INDEX idx_connections_service_request_expires_at;
+DROP INDEX idx_connections_deleted;
+ALTER TABLE connections DROP COLUMN service_request_expires_at;
+ALTER TABLE connections DROP COLUMN created_at;
+ALTER TABLE conn_invitations DROP COLUMN service_request;
+DROP INDEX idx_address_ratchet_keys;
+DROP TABLE address_ratchet_keys;
+  |]

src/Simplex/Messaging/Agent/Store/SQLite/Migrations/M20260823_snd_files_entitlement.hs

@@ -0,0 +1,20 @@
+{-# LANGUAGE QuasiQuotes #-}
+
+module Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260823_snd_files_entitlement where
+
+import Database.SQLite.Simple (Query)
+import Database.SQLite.Simple.QQ (sql)
+
+m20260823_snd_files_entitlement :: Query
+m20260823_snd_files_entitlement =
+  [sql|
+ALTER TABLE snd_files ADD COLUMN storage_time INTEGER;
+ALTER TABLE snd_file_chunk_replicas ADD COLUMN replica_expires_at INTEGER;
+  |]
+
+down_m20260823_snd_files_entitlement :: Query
+down_m20260823_snd_files_entitlement =
+  [sql|
+ALTER TABLE snd_file_chunk_replicas DROP COLUMN replica_expires_at;
+ALTER TABLE snd_files DROP COLUMN storage_time;
+  |]

src/Simplex/Messaging/Agent/Store/SQLite/Migrations/M20260919_ratchet_verify_codes.hs

@@ -0,0 +1,20 @@
+{-# LANGUAGE QuasiQuotes #-}
+
+module Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260919_ratchet_verify_codes where
+
+import Database.SQLite.Simple (Query)
+import Database.SQLite.Simple.QQ (sql)
+
+m20260919_ratchet_verify_codes :: Query
+m20260919_ratchet_verify_codes =
+  [sql|
+ALTER TABLE ratchets ADD COLUMN rc_verify_code_ad BLOB;
+ALTER TABLE ratchets ADD COLUMN rc_verify_code_pq BLOB;
+  |]
+
+down_m20260919_ratchet_verify_codes :: Query
+down_m20260919_ratchet_verify_codes =
+  [sql|
+ALTER TABLE ratchets DROP COLUMN rc_verify_code_pq;
+ALTER TABLE ratchets DROP COLUMN rc_verify_code_ad;
+  |]

src/Simplex/Messaging/Agent/Store/SQLite/Migrations/agent_schema.sql

@@ -27,7 +27,9 @@ CREATE TABLE connections(
   REFERENCES users ON DELETE CASCADE,
   ratchet_sync_state TEXT NOT NULL DEFAULT 'ok',
   deleted_at_wait_delivery TEXT,
-  pq_support INTEGER NOT NULL DEFAULT 0
+  pq_support INTEGER NOT NULL DEFAULT 0,
+  created_at TEXT NOT NULL DEFAULT('1970-01-01 00:00:00'),
+  service_request_expires_at TEXT
 ) WITHOUT ROWID, STRICT;
 CREATE TABLE rcv_queues(
   host TEXT NOT NULL,
@@ -164,6 +166,8 @@ CREATE TABLE conn_invitations(
   accepted INTEGER NOT NULL DEFAULT 0,
   own_conn_info BLOB,
   created_at TEXT NOT NULL DEFAULT(datetime('now'))
+  ,
+  service_request INTEGER NOT NULL DEFAULT 0
 ) WITHOUT ROWID, STRICT;
 CREATE TABLE ratchets(
   conn_id BLOB NOT NULL PRIMARY KEY REFERENCES connections
@@ -178,7 +182,9 @@ CREATE TABLE ratchets(
   x3dh_pub_key_1 BLOB,
   x3dh_pub_key_2 BLOB,
   pq_priv_kem BLOB,
-  pq_pub_kem BLOB
+  pq_pub_kem BLOB,
+  rc_verify_code_ad BLOB,
+  rc_verify_code_pq BLOB
 ) WITHOUT ROWID, STRICT;
 CREATE TABLE skipped_messages(
   skipped_message_id INTEGER PRIMARY KEY,
@@ -348,7 +354,8 @@ CREATE TABLE snd_files(
   src_file_nonce BLOB,
   failed INTEGER DEFAULT 0,
   redirect_size INTEGER,
-  redirect_digest BLOB
+  redirect_digest BLOB,
+  storage_time INTEGER
 ) STRICT;
 CREATE TABLE snd_file_chunks(
   snd_file_chunk_id INTEGER PRIMARY KEY,
@@ -372,6 +379,8 @@ CREATE TABLE snd_file_chunk_replicas(
   retries INTEGER NOT NULL DEFAULT 0,
   created_at TEXT NOT NULL DEFAULT(datetime('now')),
   updated_at TEXT NOT NULL DEFAULT(datetime('now'))
+  ,
+  replica_expires_at INTEGER
 ) STRICT;
 CREATE TABLE snd_file_chunk_replica_recipients(
   snd_file_chunk_replica_recipient_id INTEGER PRIMARY KEY,
@@ -465,6 +474,15 @@ CREATE TABLE client_services(
   service_queue_ids_hash BLOB NOT NULL DEFAULT x'00000000000000000000000000000000',
   FOREIGN KEY(host, port) REFERENCES servers ON UPDATE CASCADE ON DELETE RESTRICT
 ) STRICT;
+CREATE TABLE address_ratchet_keys(
+  address_ratchet_key_id INTEGER PRIMARY KEY AUTOINCREMENT,
+  conn_id BLOB NOT NULL REFERENCES connections ON DELETE CASCADE,
+  ratchet_key_id BLOB NOT NULL,
+  x3dh_priv_key_1 BLOB NOT NULL,
+  x3dh_priv_key_2 BLOB NOT NULL,
+  pq_priv_kem BLOB,
+  created_at TEXT NOT NULL DEFAULT(datetime('now'))
+) STRICT;
 CREATE UNIQUE INDEX idx_rcv_queues_ntf ON rcv_queues(host, port, ntf_id);
 CREATE UNIQUE INDEX idx_rcv_queue_id ON rcv_queues(conn_id, rcv_queue_id);
 CREATE UNIQUE INDEX idx_snd_queue_id ON snd_queues(conn_id, snd_queue_id);
@@ -615,6 +633,14 @@ CREATE UNIQUE INDEX idx_server_certs_user_id_host_port ON client_services(
   server_key_hash
 );
 CREATE INDEX idx_server_certs_host_port ON client_services(host, port);
+CREATE UNIQUE INDEX idx_address_ratchet_keys ON address_ratchet_keys(
+  conn_id,
+  ratchet_key_id
+);
+CREATE INDEX idx_connections_deleted ON connections(deleted);
+CREATE INDEX idx_connections_service_request_expires_at ON connections(
+  service_request_expires_at
+);
 CREATE TRIGGER tr_rcv_queue_insert
 AFTER INSERT ON rcv_queues
 FOR EACH ROW

src/Simplex/Messaging/Client.hs

@@ -35,6 +35,8 @@ module Simplex.Messaging.Client
     ProxiedRelay (..),
     getProtocolClient,
     closeProtocolClient,
+    closeTimedOutClient,
+    pClientSentCommandsCount,
     protocolClientServer,
     protocolClientServer',
     transportHost',
@@ -150,7 +152,8 @@ import Data.Int (Int64)
 import Data.List (find, isSuffixOf)
 import Data.List.NonEmpty (NonEmpty (..))
 import qualified Data.List.NonEmpty as L
-import Data.Maybe (catMaybes, fromMaybe)
+import qualified Data.Map.Strict as M
+import Data.Maybe (catMaybes, fromMaybe, isNothing)
 import Data.Text (Text)
 import qualified Data.Text as T
 import Data.Time.Clock (UTCTime (..), diffUTCTime, getCurrentTime)
@@ -166,7 +169,7 @@ import Simplex.Messaging.Parsers (defaultJSON, dropPrefix, enumJSON, sumTypeJSON
 import Simplex.Messaging.Protocol
 import Simplex.Messaging.Protocol.Types
 import Simplex.Messaging.Server.QueueStore.QueueInfo
-import Simplex.Messaging.SimplexName (SimplexDomain)
+import Simplex.Messaging.SimplexName (SimplexDomain, fullDomainName)
 import Simplex.Messaging.TMap (TMap)
 import qualified Simplex.Messaging.TMap as TM
 import Simplex.Messaging.Transport
@@ -227,10 +230,10 @@ smpClientStub g sessionId thVersion thAuth = do
               thServerVRange = supportedServerSMPRelayVRange,
               thAuth,
               blockSize = smpBlockSize,
-              implySessId = thVersion >= authCmdsSMPVersion,
+              implySessId = True,
               encryptBlock = Nothing,
-              batch = True,
-              serviceAuth = thVersion >= serviceCertsSMPVersion
+              serviceAuth = thVersion >= serviceCertsSMPVersion,
+              serverInfo = Nothing
             },
         sessionTs = ts,
         client_ =
@@ -658,10 +661,9 @@ getProtocolClient g nm transportSession@(_, srv, _) cfg@ProtocolClientConfig {qS
               responseErr = atomically . putTMVar responseVar . Left . PCETransportError
 
     receive :: Transport c => ProtocolClient v err msg -> THandle v c 'TClient -> IO ()
-    receive ProtocolClient {client_ = PClient {rcvQ, lastReceived, timeoutErrorCount}} h = forever $ do
+    receive ProtocolClient {client_ = PClient {rcvQ, lastReceived}} h = forever $ do
       tGetClient h >>= atomically . writeTBQueue rcvQ
       getCurrentTime >>= atomically . writeTVar lastReceived
-      atomically $ writeTVar timeoutErrorCount 0
 
     monitor :: ProtocolClient v err msg -> IO ()
     monitor c@ProtocolClient {client_ = PClient {sendPings, lastReceived, timeoutErrorCount}} = loop smpPingInterval
@@ -737,6 +739,10 @@ useWebPort cfg presetDomains ProtocolServer {host = h :| _} = case smpWebPortSer
   SWPPreset -> isPresetDomain presetDomains h
   SWPOff -> False
 
+-- | Count of in-flight (awaiting-response) commands on a client - for leak diagnostics.
+pClientSentCommandsCount :: ProtocolClient v err msg -> IO Int
+pClientSentCommandsCount ProtocolClient {client_ = PClient {sentCommands}} = M.size <$> readTVarIO sentCommands
+
 isPresetDomain :: [HostName] -> TransportHost -> Bool
 isPresetDomain presetDomains = \case
   THDomainName h -> any (`isSuffixOf` h) presetDomains
@@ -750,6 +756,12 @@ closeProtocolClient :: ProtocolClient v err msg -> IO ()
 closeProtocolClient = mapM_ (deRefWeak >=> mapM_ killThread) . action
 {-# INLINE closeProtocolClient #-}
 
+-- | Disconnects client when maxCnt commands in a row timed out, 0 to disable.
+closeTimedOutClient :: Int -> ProtocolClient v err msg -> IO ()
+closeTimedOutClient maxCnt c@ProtocolClient {client_ = PClient {timeoutErrorCount}} = do
+  cnt <- readTVarIO timeoutErrorCount
+  when (maxCnt > 0 && cnt >= maxCnt) $ closeProtocolClient c
+
 -- | SMP client error type.
 data ProtocolClientError err
   = -- | Correctly parsed SMP server ERR response.
@@ -974,7 +986,7 @@ deleteSMPQueueLink :: SMPClient -> NetworkRequestMode -> RcvPrivateAuthKey -> Re
 deleteSMPQueueLink = okSMPCommand LDEL
 {-# INLINE deleteSMPQueueLink #-}
 
--- | Get 1-time inviation SMP queue link data and secure the queue via queue link ID.
+-- | Get 1-time invitation SMP queue link data and secure the queue via queue link ID.
 secureGetSMPQueueLink :: SMPClient -> NetworkRequestMode -> SndPrivateAuthKey -> LinkId -> ExceptT SMPClientError IO (SenderId, QueueLinkData)
 secureGetSMPQueueLink c nm spKey lnkId =
   sendSMPCommand c nm (Just spKey) lnkId (LKEY $ C.toPublic spKey) >>= \case
@@ -1054,11 +1066,11 @@ proxySMPMessage c nm proxiedRelay spKey sId flags msg = proxyOKSMPCommand c nm p
 -- through `proxySMPCommand` and pattern-matches the expected RNAME response.
 -- Version-gated on the destination relay (mirrors `connectSMPProxiedRelay`):
 -- the client never sends RSLV to a relay that predates names support.
-proxyResolveName :: SMPClient -> NetworkRequestMode -> ProxiedRelay -> SimplexDomain -> ExceptT SMPClientError IO (Either ProxyClientError NameRecord)
+proxyResolveName :: SMPClient -> NetworkRequestMode -> ProxiedRelay -> SimplexDomain -> ExceptT SMPClientError IO (Either ProxyClientError NameResponse)
 proxyResolveName c nm proxiedRelay name
   | prVersion proxiedRelay >= namesSMPVersion =
-      proxySMPCommand c nm proxiedRelay Nothing NoEntity (RSLV name) >>= \case
-        Right (RNAME nr) -> pure $ Right nr
+      proxySMPCommand c nm proxiedRelay Nothing NoEntity (RSLV (NQDomain name)) >>= \case
+        Right (RNAME reg) | resolvedNameOrNotFound name reg -> pure $ Right reg
         Right r -> throwE $ unexpectedResponse r
         Left e -> pure $ Left e
   | otherwise = throwE $ PCETransportError TEVersion
@@ -1066,16 +1078,21 @@ proxyResolveName c nm proxiedRelay name
 -- | Direct (non-PFWD) name resolution. Exposes the client IP to the resolver;
 -- callers that want anonymity should use `proxyResolveName` via the standard
 -- proxy fallback in the agent. RSLV requires no entity ID or authorization
--- (see `noAuthCmd` in Protocol.hs). Version-gated on the session here, not the
--- encoder, so an old server never receives RSLV.
-directResolveName :: SMPClient -> NetworkRequestMode -> SimplexDomain -> ExceptT SMPClientError IO NameRecord
+-- (see `noAuthCmd` in Protocol.hs). Gated on the session version, below which
+-- the server has no RSLV at all; the encoder gates the query format separately.
+directResolveName :: SMPClient -> NetworkRequestMode -> SimplexDomain -> ExceptT SMPClientError IO NameResponse
 directResolveName c nm name
   | thVersion (thParams c) >= namesSMPVersion =
-      sendProtocolCommand c nm Nothing NoEntity (Cmd SResolver (RSLV name)) >>= \case
-        RNAME nr -> pure nr
+      sendProtocolCommand c nm Nothing NoEntity (Cmd SResolver (RSLV (NQDomain name))) >>= \case
+        RNAME reg | resolvedNameOrNotFound name reg -> pure reg
         r -> throwE $ unexpectedResponse r
   | otherwise = throwE $ PCETransportError TEVersion
 
+resolvedNameOrNotFound :: SimplexDomain -> NameResponse -> Bool
+resolvedNameOrNotFound d NameResponse {registration} = case registration of
+  NRRegistered {nameRecord} -> T.toLower (nrName nameRecord) == fullDomainName d
+  _ -> True
+
 -- | Acknowledge message delivery (server deletes the message).
 --
 -- https://github.com/simplex-chat/simplexmq/blob/master/protocol/simplex-messaging.md#acknowledge-message-delivery
@@ -1110,17 +1127,15 @@ deleteSMPQueues = okSMPCommands DEL
 -- send PRXY :: SMPServer -> Maybe BasicAuth -> Command Sender
 -- receives PKEY :: SessionId -> X.CertificateChain -> X.SignedExact X.PubKey -> BrokerMsg
 connectSMPProxiedRelay :: SMPClient -> NetworkRequestMode -> SMPServer -> Maybe BasicAuth -> ExceptT SMPClientError IO ProxiedRelay
-connectSMPProxiedRelay c@ProtocolClient {client_ = PClient {tcpConnectTimeout, tcpTimeout}} nm relayServ@ProtocolServer {port = relayPort, keyHash = C.KeyHash kh} proxyAuth
-  | thVersion (thParams c) >= sendingProxySMPVersion =
-      sendProtocolCommand_ c nm Nothing tOut Nothing NoEntity (Cmd SProxiedClient (PRXY relayServ proxyAuth)) >>= \case
-        PKEY sId vr (CertChainPubKey chain key) ->
-          case supportedClientSMPRelayVRange `compatibleVersion` vr of
-            Nothing -> throwE $ transportErr TEVersion
-            Just (Compatible v) -> do
-              relayKey <- liftEitherWith (const $ transportErr $ TEHandshake IDENTITY) =<< liftIO (runExceptT $ validateRelay chain key)
-              pure $ ProxiedRelay sId v proxyAuth relayKey
-        r -> throwE $ unexpectedResponse r
-  | otherwise = throwE $ PCETransportError TEVersion
+connectSMPProxiedRelay c@ProtocolClient {client_ = PClient {tcpConnectTimeout, tcpTimeout}} nm relayServ@ProtocolServer {port = relayPort, keyHash = C.KeyHash kh} proxyAuth =
+  sendProtocolCommand_ c nm Nothing tOut Nothing NoEntity (Cmd SProxiedClient (PRXY relayServ proxyAuth)) >>= \case
+    PKEY sId vr (CertChainPubKey chain key) ->
+      case supportedClientSMPRelayVRange `compatibleVersion` vr of
+        Nothing -> throwE $ transportErr TEVersion
+        Just (Compatible v) -> do
+          relayKey <- liftEitherWith (const $ transportErr $ TEHandshake IDENTITY) =<< liftIO (runExceptT $ validateRelay chain key)
+          pure $ ProxiedRelay sId v proxyAuth relayKey
+    r -> throwE $ unexpectedResponse r
   where
     tOut = Just $ netTimeoutInt tcpConnectTimeout nm + netTimeoutInt tcpTimeout nm
     transportErr = PCEProtocolError . PROXY . BROKER . TRANSPORT
@@ -1351,22 +1366,22 @@ sendProtocolCommand c nm = sendProtocolCommand_ c nm Nothing Nothing
 --
 -- Please note: if nonce is passed it is also used as a correlation ID
 sendProtocolCommand_ :: forall v err msg. Protocol v err msg => ProtocolClient v err msg -> NetworkRequestMode -> Maybe C.CbNonce -> Maybe Int -> Maybe C.APrivateAuthKey -> EntityId -> ProtoCommand msg -> ExceptT (ProtocolClientError err) IO msg
-sendProtocolCommand_ c@ProtocolClient {client_ = PClient {sndQ}, thParams = THandleParams {batch, blockSize, serviceAuth}} nm nonce_ tOut pKey entId cmd =
+sendProtocolCommand_ c@ProtocolClient {client_ = PClient {sndQ, sentCommands}, thParams = THandleParams {blockSize, serviceAuth}} nm nonce_ tOut pKey entId cmd =
   ExceptT $ uncurry sendRecv =<< mkTransmission_ c nonce_ (entId, pKey, cmd)
   where
     -- two separate "atomically" needed to avoid blocking
     sendRecv :: Either TransportError SentRawTransmission -> Request err msg -> IO (Either (ProtocolClientError err) msg)
-    sendRecv t_ r = case t_ of
-      Left e -> pure . Left $ PCETransportError e
+    sendRecv t_ r@Request {corrId} = case t_ of
+      Left e -> notSent e
       Right t
-        | B.length s > blockSize - 2 -> pure . Left $ PCETransportError TELargeMsg
+        | B.length s > blockSize - 2 -> notSent TELargeMsg
         | otherwise -> do
             nonBlockingWriteTBQueue sndQ (Just r, s)
             response <$> getResponse c nm tOut r
         where
-          s
-            | batch = tEncodeBatch1 serviceAuth t
-            | otherwise = tEncode serviceAuth t
+          s = tEncodeBatch1 serviceAuth t
+      where
+        notSent e = Left (PCETransportError e) <$ atomically (TM.delete corrId sentCommands)
 
 nonBlockingWriteTBQueue :: TBQueue a -> a -> IO ()
 nonBlockingWriteTBQueue q x = do
@@ -1374,7 +1389,7 @@ nonBlockingWriteTBQueue q x = do
   unless sent $ void $ forkIO $ atomically $ writeTBQueue q x
 
 getResponse :: ProtocolClient v err msg -> NetworkRequestMode -> Maybe Int -> Request err msg -> IO (Response err msg)
-getResponse ProtocolClient {client_ = PClient {tcpTimeout, timeoutErrorCount}} nm tOut Request {entityId, pending, responseVar} = do
+getResponse ProtocolClient {client_ = PClient {tcpTimeout, timeoutErrorCount, sentCommands, msgQ}} nm tOut Request {corrId, entityId, pending, responseVar} = do
   r <- fromMaybe (netTimeoutInt tcpTimeout nm) tOut `timeout` atomically (takeTMVar responseVar)
   response <- atomically $ do
     writeTVar pending False
@@ -1383,7 +1398,11 @@ getResponse ProtocolClient {client_ = PClient {tcpTimeout, timeoutErrorCount}} n
     -- See `processMsg`.
     ((r <|>) <$> tryTakeTMVar responseVar) >>= \case
       Just r' -> writeTVar timeoutErrorCount 0 $> r'
-      Nothing -> modifyTVar' timeoutErrorCount (+ 1) $> Left PCEResponseTimeout
+      Nothing -> do
+        modifyTVar' timeoutErrorCount (+ 1)
+        -- a late response is delivered to msgQ, without msgQ it is only logged
+        when (isNothing msgQ) $ TM.delete corrId sentCommands
+        pure $ Left PCEResponseTimeout
   pure Response {entityId, response}
 
 mkTransmission :: Protocol v err msg => ProtocolClient v err msg ->  ClientCommand msg -> IO (PCTransmission err msg)

src/Simplex/Messaging/Client/Agent.hs

@@ -31,6 +31,8 @@ module Simplex.Messaging.Client.Agent
     removeActiveSubs,
     removePendingSub,
     removePendingSubs,
+    AgentLeakStats (..),
+    getAgentLeakStats,
   )
 where
 
@@ -107,7 +109,7 @@ data SMPClientAgentConfig = SMPClientAgentConfig
   { smpCfg :: ProtocolClientConfig SMPVersion,
     reconnectInterval :: RetryInterval,
     persistErrorInterval :: NominalDiffTime,
-    msgQSize :: Natural,
+    msgQSize :: Maybe Natural,
     agentQSize :: Natural,
     agentSubsBatchSize :: Int,
     ownServerDomains :: [ByteString]
@@ -124,7 +126,7 @@ defaultSMPClientAgentConfig =
             maxInterval = 10 * second
           },
       persistErrorInterval = 30, -- seconds
-      msgQSize = 2048,
+      msgQSize = Just 2048,
       agentQSize = 2048,
       agentSubsBatchSize = 1360,
       ownServerDomains = []
@@ -138,7 +140,7 @@ data SMPClientAgent p = SMPClientAgent
     dbService :: Maybe DBService,
     active :: TVar Bool,
     startedAt :: UTCTime,
-    msgQ :: TBQueue (ServerTransmissionBatch SMPVersion ErrorType BrokerMsg),
+    msgQ :: Maybe (TBQueue (ServerTransmissionBatch SMPVersion ErrorType BrokerMsg)),
     agentQ :: TBQueue SMPClientAgentEvent,
     randomDrg :: TVar ChaChaDRG,
     smpClients :: TMap SMPServer SMPClientVar,
@@ -158,11 +160,41 @@ data SMPClientAgent p = SMPClientAgent
 
 type OwnServer = Bool
 
+-- | Sizes of every per-server/per-session map in the client agent, plus total in-flight
+-- forwarded commands - for leak diagnostics on the proxy path.
+data AgentLeakStats = AgentLeakStats
+  { alSmpClients :: Int,
+    alSmpSessions :: Int,
+    alActiveServiceSubs :: Int,
+    alActiveQueueSubs :: Int,
+    alPendingServiceSubs :: Int,
+    alPendingQueueSubs :: Int,
+    alSmpSubWorkers :: Int,
+    alSentCommands :: Int,
+    alMsgQ :: Int
+  }
+
+getAgentLeakStats :: SMPClientAgent p -> IO AgentLeakStats
+getAgentLeakStats SMPClientAgent {smpClients, smpSessions, activeServiceSubs, activeQueueSubs, pendingServiceSubs, pendingQueueSubs, smpSubWorkers, msgQ} = do
+  alSmpClients <- msize smpClients
+  sess <- readTVarIO smpSessions
+  alActiveServiceSubs <- msize activeServiceSubs
+  alActiveQueueSubs <- msize activeQueueSubs
+  alPendingServiceSubs <- msize pendingServiceSubs
+  alPendingQueueSubs <- msize pendingQueueSubs
+  alSmpSubWorkers <- msize smpSubWorkers
+  alSentCommands <- foldM (\ !a (_, c) -> (a +) <$> pClientSentCommandsCount c) 0 (M.elems sess)
+  alMsgQ <- fromIntegral <$> maybe (pure 0) (atomically . lengthTBQueue) msgQ
+  pure AgentLeakStats {alSmpClients, alSmpSessions = M.size sess, alActiveServiceSubs, alActiveQueueSubs, alPendingServiceSubs, alPendingQueueSubs, alSmpSubWorkers, alSentCommands, alMsgQ}
+  where
+    msize m = M.size <$> readTVarIO m
+
 newSMPClientAgent :: SParty p -> SMPClientAgentConfig -> Maybe DBService -> TVar ChaChaDRG -> IO (SMPClientAgent p)
 newSMPClientAgent agentParty agentCfg@SMPClientAgentConfig {msgQSize, agentQSize} dbService randomDrg = do
   active <- newTVarIO True
   startedAt <- getCurrentTime
-  msgQ <- newTBQueueIO msgQSize
+  -- Only subscribing agents receive server transmissions, should not be created until processed to prevent deadlock.
+  msgQ <- mapM newTBQueueIO msgQSize
   agentQ <- newTBQueueIO agentQSize
   smpClients <- TM.emptyIO
   smpSessions <- TM.emptyIO
@@ -264,7 +296,7 @@ connectClient ca@SMPClientAgent {agentCfg, dbService, smpClients, smpSessions, m
   Nothing -> getClient cfg
   where
     cfg = smpCfg agentCfg
-    getClient cfg' = getProtocolClient randomDrg NRMBackground (1, srv, Nothing) cfg' [] (Just msgQ) startedAt clientDisconnected
+    getClient cfg' = getProtocolClient randomDrg NRMBackground (1, srv, Nothing) cfg' [] msgQ startedAt clientDisconnected
 
     clientDisconnected :: SMPClient -> IO ()
     clientDisconnected smp = do

src/Simplex/Messaging/Crypto.hs

@@ -61,6 +61,7 @@ module Simplex.Messaging.Crypto
     APublicAuthKey (..),
     CryptoPublicKey (..),
     CryptoPrivateKey (..),
+    StoredPrivateKey (..),
     AAuthKeyPair,
     KeyPair,
     KeyPairX25519,
@@ -342,8 +343,17 @@ deriving instance Eq (PrivateKey a)
 
 deriving instance Show (PrivateKey a)
 
--- Do not enable, to avoid leaking key data
--- instance StrEncoding (PrivateKey Ed25519) where
+-- Do not enable, to avoid leaking key data, use StoredPrivateKey instead
+-- instance StrEncoding (PrivateKey a) where
+
+newtype StoredPrivateKey a = StoredPrivateKey {unStored :: PrivateKey a}
+  deriving (Show)
+
+instance AlgorithmI a => StrEncoding (StoredPrivateKey a) where
+  strEncode = strEncode . encodePrivKey . unStored
+  {-# INLINE strEncode #-}
+  strDecode = fmap StoredPrivateKey . decodePrivKey
+  {-# INLINE strDecode #-}
 
 -- Used in notification store log
 instance StrEncoding (PrivateKey X25519) where

src/Simplex/Messaging/Crypto/BBS.hs

@@ -17,6 +17,7 @@ module Simplex.Messaging.Crypto.BBS
     BBSProof (..),
     BBSHeader (..),
     BBSPresHeader (..),
+    FixedBS (..),
     bbsKeyGen,
     bbsPublicKey,
     bbsSign,
@@ -33,7 +34,9 @@ import Data.Proxy (Proxy (..))
 import Foreign
 import Foreign.C
 import GHC.TypeLits (KnownNat, KnownSymbol, Nat, Symbol, natVal, symbolVal)
+import Simplex.Messaging.Encoding (Encoding (..), Large (..))
 import Simplex.Messaging.Encoding.String
+import Simplex.Messaging.Util ((<$?>))
 import System.IO.Unsafe (unsafePerformIO)
 
 -- Note: the data constructors below are unchecked escape hatches for trusted,
@@ -71,8 +74,8 @@ newtype BBSPresHeader = BBSPresHeader ByteString
   deriving (ToJSON, FromJSON) via (StrJSON "BBSPresHeader" BBSPresHeader)
 
 -- | A ByteString validated to be exactly @n@ bytes when parsed via StrEncoding
--- (and the JSON derived from it). Local to BBS, where every key/signature is a
--- fixed size; @name@ appears in the decode error only.
+-- (and the JSON derived from it), for keys and signatures of a fixed size;
+-- @name@ appears in the decode error only.
 newtype FixedBS (name :: Symbol) (n :: Nat) = FixedBS ByteString
 
 instance forall name n. (KnownSymbol name, KnownNat n) => StrEncoding (FixedBS name n) where
@@ -97,14 +100,20 @@ bbsProofLen :: Int -> Int
 bbsProofLen numUndisclosed = bbsProofBaseLen + numUndisclosed * bbsProofUdElemLen
 
 -- | A proof is @bbsProofBaseLen + 32 * numUndisclosed@ bytes; reject anything else.
+mkBBSProof :: ByteString -> Either String BBSProof
+mkBBSProof bs
+  | len >= bbsProofBaseLen && (len - bbsProofBaseLen) `mod` bbsProofUdElemLen == 0 = Right $ BBSProof bs
+  | otherwise = Left $ "BBS: invalid proof length " <> show len
+  where
+    len = B.length bs
+
 instance StrEncoding BBSProof where
   strEncode (BBSProof bs) = strEncode bs
-  strP = do
-    bs <- base64urlP
-    let len = B.length bs
-    if len >= bbsProofBaseLen && (len - bbsProofBaseLen) `mod` bbsProofUdElemLen == 0
-      then pure (BBSProof bs)
-      else fail $ "BBS: invalid proof length " <> show len
+  strP = mkBBSProof <$?> base64urlP
+
+instance Encoding BBSProof where
+  smpEncode (BBSProof p) = smpEncode (Large p)
+  smpP = mkBBSProof . unLarge <$?> smpP
 
 -- FFI
 

src/Simplex/Messaging/Crypto/Entitlement.hs

@@ -0,0 +1,151 @@
+{-# LANGUAGE DataKinds #-}
+{-# LANGUAGE DerivingVia #-}
+{-# LANGUAGE DuplicateRecordFields #-}
+{-# LANGUAGE GeneralizedNewtypeDeriving #-}
+{-# LANGUAGE NamedFieldPuns #-}
+{-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE TemplateHaskell #-}
+
+module Simplex.Messaging.Crypto.Entitlement
+  ( Entitlement (..),
+    EntitlementCredential (..),
+    EntitlementProof (..),
+    EntitlementVerification (..),
+    MasterKey (..),
+    randomMasterKey,
+    entitlementBBSHeader,
+    entitlementIssuerKeys,
+    signEntitlement,
+    verifyCredential,
+    generateEntitlementProof,
+    verifyEntitlement,
+  )
+where
+
+import Control.Concurrent.STM
+import Crypto.Random (ChaChaDRG)
+import Data.Aeson (FromJSON (..), ToJSON (..))
+import qualified Data.Aeson.TH as JQ
+import Data.ByteString.Char8 (ByteString)
+import qualified Data.ByteString.Char8 as B
+import Data.Either (fromRight)
+import Data.Map.Strict (Map)
+import qualified Data.Map.Strict as M
+import Data.Text (Text)
+import Data.Text.Encoding (decodeUtf8', encodeUtf8)
+import Data.Time.Clock (UTCTime)
+import Data.Word (Word16)
+import qualified Simplex.Messaging.Crypto as C
+import Simplex.Messaging.Crypto.BBS
+import Simplex.Messaging.Encoding
+import Simplex.Messaging.Encoding.String
+import Simplex.Messaging.Parsers (defaultJSON)
+import Simplex.Messaging.Util ((<$$>))
+
+newtype MasterKey = MasterKey ByteString
+  deriving newtype (Eq, Show)
+  deriving (StrEncoding) via (FixedBS "MasterKey" 32)
+  deriving (ToJSON, FromJSON) via (StrJSON "MasterKey" MasterKey)
+
+data Entitlement = Entitlement
+  { expiresAt :: UTCTime,
+    entitlementName :: Text,
+    extraInfo :: Text
+  }
+  deriving (Eq, Show)
+
+data EntitlementCredential = EntitlementCredential
+  { issuerKeyIdx :: Word16,
+    masterKey :: MasterKey,
+    entitlement :: Entitlement,
+    issuerSignature :: BBSSignature
+  }
+  deriving (Eq, Show)
+
+data EntitlementProof = EntitlementProof
+  { issuerKeyIdx :: Word16,
+    entitlement :: Entitlement,
+    entProof :: BBSProof
+  }
+  deriving (Eq, Show)
+
+data EntitlementVerification = EVValid | EVInvalid | EVUnknownIssuer
+  deriving (Eq, Show)
+
+instance Encoding Entitlement where
+  smpEncode Entitlement {expiresAt, entitlementName, extraInfo} =
+    smpEncode (strEncode expiresAt, entitlementName, Large $ encodeUtf8 extraInfo)
+  smpP = do
+    (expBs, entitlementName, Large extraBs) <- smpP
+    expiresAt <- either fail pure $ strDecode (expBs :: ByteString)
+    extraInfo <- either (fail . show) pure $ decodeUtf8' extraBs
+    pure Entitlement {expiresAt, entitlementName, extraInfo}
+
+instance Encoding EntitlementProof where
+  smpEncode EntitlementProof {issuerKeyIdx, entProof, entitlement} =
+    smpEncode (issuerKeyIdx, entProof, entitlement)
+  smpP = do
+    (issuerKeyIdx, entProof, entitlement) <- smpP
+    pure EntitlementProof {issuerKeyIdx, entProof, entitlement}
+
+entitlementBBSHeader :: BBSHeader
+entitlementBBSHeader = BBSHeader "SimpleX badges v1"
+
+entitlementMessageCount :: Int
+entitlementMessageCount = 4
+
+entitlementDisclosedIndexes :: [Int]
+entitlementDisclosedIndexes = [1, 2, 3]
+
+entitlementMessages :: MasterKey -> Entitlement -> [ByteString]
+entitlementMessages (MasterKey mk) ent = mk : disclosedMessages ent
+
+disclosedMessages :: Entitlement -> [ByteString]
+disclosedMessages Entitlement {expiresAt, entitlementName, extraInfo} =
+  [strEncode expiresAt, encodeUtf8 entitlementName, encodeUtf8 extraInfo]
+
+randomMasterKey :: TVar ChaChaDRG -> STM MasterKey
+randomMasterKey g = MasterKey <$> C.randomBytes 32 g
+
+signEntitlement :: BBSSecretKey -> Word16 -> MasterKey -> Entitlement -> IO (Either String EntitlementCredential)
+signEntitlement sk keyIdx mk ent =
+  EntitlementCredential keyIdx mk ent <$$> bbsSign sk entitlementBBSHeader (entitlementMessages mk ent)
+
+verifyCredential :: BBSPublicKey -> EntitlementCredential -> IO Bool
+verifyCredential pk EntitlementCredential {masterKey, issuerSignature, entitlement} =
+  bbsVerify pk issuerSignature entitlementBBSHeader (entitlementMessages masterKey entitlement)
+
+generateEntitlementProof :: Map Word16 BBSPublicKey -> EntitlementCredential -> BBSPresHeader -> IO (Either String EntitlementProof)
+generateEntitlementProof keys EntitlementCredential {issuerKeyIdx, masterKey, issuerSignature, entitlement} ph =
+  case M.lookup issuerKeyIdx keys of
+    Nothing -> pure $ Left $ "no issuer key " <> show issuerKeyIdx
+    Just pk -> EntitlementProof issuerKeyIdx entitlement <$$> bbsProofGen pk issuerSignature entitlementBBSHeader ph entitlementDisclosedIndexes (entitlementMessages masterKey entitlement)
+
+verifyEntitlement :: Map Word16 BBSPublicKey -> BBSPresHeader -> EntitlementProof -> IO EntitlementVerification
+verifyEntitlement keys ph EntitlementProof {issuerKeyIdx, entProof, entitlement} =
+  case M.lookup issuerKeyIdx keys of
+    Nothing -> pure EVUnknownIssuer
+    Just pk ->
+      (\valid -> if valid then EVValid else EVInvalid)
+        <$> bbsProofVerify pk entProof entitlementBBSHeader ph entitlementDisclosedIndexes entitlementMessageCount (disclosedMessages entitlement)
+
+entitlementIssuerKeys :: Map Word16 BBSPublicKey
+entitlementIssuerKeys =
+  M.fromList
+    [ (1, key "mW_5Zp1wHnXDF56wOZwFcRjGrf0GLLsfyymIQDqYoWfjfvS7oQWSfi7hH65N8JhuE9x8wbKXHidnQLO4GnOSMP_bRKUMH1qIzv5SQKFHNM8G4PaWcTcri8iZLc-3xhSI"),
+      (2, key "odGCB7uVDXTURsHgSvSciByV4Q3-3ZvEB8myDsDJqm-PwOYc5-At36uc7n_pyUDxEQEHr9i4RJgFih2FSArPW-EQBXNPNf4wTtA0znn74qLEGc4fh9pVYPEIm_ZGbnsJ"),
+      (3, key "txkT2003WMjc43KvYvPKEcR970NLmw5UZY51eUqgk91sgp53idt1HTlKYvnrEttJDFMlctYf1-bpri0e9DhBQ-xk1J4WoLN2uif_1OcA1pGCobpk9lwtsq1Idek4biy0"),
+      (4, key "q_YzegihaLYrEm9z3cAghsfDGNZfXuEpQGMJERJQS4M0Szl4gvSC_fV_muKc3NIMA_8iYuBN8qyvb5U55RctCRn3kleFQ4sqf-WBgoydX6UVo7BsYcUbXWWEFZXlOGIH"),
+      (5, key "oqymHASH_okefShrnz4HnTooUNlE1WoDRnSrgd0bTCpOacgJWBsMpwZpdmYlX-vQAKAC_zmI4VdKoOznnhW-sdUXZw6bthCi5JYjGxCR1Co27i1tix5UXCTbR5Jp901-"),
+      (6, key "kDqaB6zKSRp_97QPFj5JPDlo0vzfSTLSp9goFx1qajv4q4H6dR6BbkmWZ4xx_9Q2AxmcpqcV0ethz1OH-Jk_Sz2J1mIz1PUVM9LkdLhi_PNtqhezzO5dbVs-HJ1fNqe6"),
+      (7, key "rl36D5mg2N3NmmEybxE_RBeU9YZ_zeXNPfp7ZMLtUEuf2Mo4OQM_Up1v5rX_IqICD-AIJcuyptEBsELx_PJQzpmiNuG5I4cWO6HkRKtc6fVFvgZMrDJjaascPd1CIyxX"),
+      (8, key "joM3Bnt7JPt5JiwQwERHGjro2iVZ0mPD_clUh4hzkhxvbjuFrWuTmfSNA8PWBqGKEGNl13aRi1pMf6yY14E27c5C71JxWm7T-rZaBrGPEUWifhD-qidWuf3PU7KJCCWd")
+    ]
+  where
+    key = fromRight (error "bad base64 in BBSPublicKey") . strDecode . B.pack
+
+$(JQ.deriveJSON defaultJSON ''Entitlement)
+
+$(JQ.deriveJSON defaultJSON ''EntitlementCredential)
+
+$(JQ.deriveJSON defaultJSON ''EntitlementProof)

src/Simplex/Messaging/Crypto/Ratchet.hs

@@ -37,6 +37,7 @@ module Simplex.Messaging.Crypto.Ratchet
     AUseKEM (..),
     RatchetKEMState (..),
     SRatchetKEMState (..),
+    RatchetKEMStateI (..),
     RcvPrivRKEMParams,
     APrivRKEMParams (..),
     RcvE2ERatchetParamsUri,
@@ -50,8 +51,6 @@ module Simplex.Messaging.Crypto.Ratchet
     VersionRangeE2E,
     pattern VersionE2E,
     RatchetVersions (..),
-    kdfX3DHE2EEncryptVersion,
-    pqRatchetE2EEncryptVersion,
     currentE2EEncryptVersion,
     supportedE2EEncryptVRange,
     generateRcvE2EParams,
@@ -86,8 +85,6 @@ module Simplex.Messaging.Crypto.Ratchet
     RatchetKey (..),
     fullHeaderLen,
     applySMDiff,
-    encodeMsgHeader,
-    msgHeaderP,
   )
 where
 
@@ -101,7 +98,6 @@ import Crypto.Random (ChaChaDRG)
 import Data.Aeson (FromJSON (..), ToJSON (..))
 import qualified Data.Aeson as J
 import qualified Data.Aeson.TH as JQ
-import Data.Attoparsec.ByteString (Parser, peekWord8')
 import qualified Data.Attoparsec.ByteString.Char8 as A
 import qualified Data.ByteArray as BA
 import Data.ByteString.Char8 (ByteString)
@@ -131,6 +127,7 @@ import UnliftIO.STM
 -- e2e encryption headers version history:
 -- 1 - binary protocol encoding (1/1/2022)
 -- 2 - use KDF in x3dh (10/20/2022)
+-- 3 - PQDR (3/14/2024)
 
 data E2EVersion
 
@@ -143,17 +140,17 @@ type VersionRangeE2E = VersionRange E2EVersion
 pattern VersionE2E :: Word16 -> VersionE2E
 pattern VersionE2E v = Version v
 
-kdfX3DHE2EEncryptVersion :: VersionE2E
-kdfX3DHE2EEncryptVersion = VersionE2E 2
+_pqRatchetE2EEncryptVersion :: VersionE2E
+_pqRatchetE2EEncryptVersion = VersionE2E 3
 
-pqRatchetE2EEncryptVersion :: VersionE2E
-pqRatchetE2EEncryptVersion = VersionE2E 3
+minSupportedE2EEncryptVersion :: VersionE2E
+minSupportedE2EEncryptVersion = _pqRatchetE2EEncryptVersion
 
 currentE2EEncryptVersion :: VersionE2E
 currentE2EEncryptVersion = VersionE2E 3
 
 supportedE2EEncryptVRange :: VersionRangeE2E
-supportedE2EEncryptVRange = mkVersionRange kdfX3DHE2EEncryptVersion currentE2EEncryptVersion
+supportedE2EEncryptVRange = mkVersionRange minSupportedE2EEncryptVersion currentE2EEncryptVersion
 
 data RatchetKEMState
   = RKSProposed -- only KEM encapsulation key
@@ -237,9 +234,7 @@ data AnyE2ERatchetParams
 deriving instance Show AnyE2ERatchetParams
 
 instance (RatchetKEMStateI s, AlgorithmI a) => Encoding (E2ERatchetParams s a) where
-  smpEncode (E2ERatchetParams v k1 k2 kem_)
-    | v >= pqRatchetE2EEncryptVersion = smpEncode (v, k1, k2, kem_)
-    | otherwise = smpEncode (v, k1, k2)
+  smpEncode (E2ERatchetParams v k1 k2 kem_) = smpEncode (v, k1, k2, kem_)
   smpP = toParams <$?> smpP
     where
       toParams :: AE2ERatchetParams a -> Either String (E2ERatchetParams s a)
@@ -260,14 +255,9 @@ instance Encoding AnyE2ERatchetParams where
     case testEquality a a' of
       Nothing -> fail "bad e2e params: different key algorithms"
       Just Refl ->
-        kemP v >>= \case
+        smpP >>= \case
           Just (ARKP s kem) -> pure $ AnyE2ERatchetParams s a $ E2ERatchetParams v k1 k2 (Just kem)
           Nothing -> pure $ AnyE2ERatchetParams SRKSProposed a $ E2ERatchetParams v k1 k2 Nothing
-    where
-      kemP :: VersionE2E -> Parser (Maybe ARKEMParams)
-      kemP v
-        | v >= pqRatchetE2EEncryptVersion = smpP
-        | otherwise = pure Nothing
 
 instance VersionI E2EVersion (E2ERatchetParams s a) where
   type VersionRangeT E2EVersion (E2ERatchetParams s a) = E2ERatchetParamsUri s a
@@ -306,11 +296,10 @@ instance (RatchetKEMStateI s, AlgorithmI a) => StrEncoding (E2ERatchetParamsUri
       [("v", strEncode vs), ("x3dh", strEncodeList [key1, key2])]
         <> maybe [] encodeKem kem_
     where
-      encodeKem kem
-        | maxVersion vs < pqRatchetE2EEncryptVersion = []
-        | otherwise = case kem of
-            RKParamsProposed k -> [("kem_key", strEncode k)]
-            RKParamsAccepted ct k -> [("kem_ct", strEncode ct), ("kem_key", strEncode k)]
+      encodeKem :: RKEMParams s -> [(ByteString, ByteString)]
+      encodeKem kem  = case kem of
+        RKParamsProposed k -> [("kem_key", strEncode k)]
+        RKParamsAccepted ct k -> [("kem_ct", strEncode ct), ("kem_key", strEncode k)]
   strP = toE2ERatchetParamsUri <$?> strP
   {-# INLINE strP #-}
 
@@ -327,25 +316,26 @@ instance StrEncoding AnyE2ERatchetParamsUri where
   strEncode (AnyE2ERatchetParamsUri _ _ ps) = strEncode ps
   strP = do
     query <- strP
-    vr :: VersionRangeE2E <- queryParam "v" query
+    vr :: VersionRangeE2E <- adjustE2EVRange <$> queryParam "v" query
     keys <- L.toList <$> queryParam "x3dh" query
     case keys of
       [APublicDhKey a k1, APublicDhKey a' k2] -> case testEquality a a' of
         Nothing -> fail "bad e2e params: different key algorithms"
         Just Refl ->
-          kemP vr query >>= \case
+          kemP query >>= \case
             Just (ARKP s kem) -> pure $ AnyE2ERatchetParamsUri s a $ E2ERatchetParamsUri vr k1 k2 (Just kem)
             Nothing -> pure $ AnyE2ERatchetParamsUri SRKSProposed a $ E2ERatchetParamsUri vr k1 k2 Nothing
       _ -> fail "bad e2e params"
     where
-      kemP vr query
-        | maxVersion vr >= pqRatchetE2EEncryptVersion =
-            queryParam_ "kem_key" query
-              $>>= \k -> Just . kemParams k <$> queryParam_ "kem_ct" query
-        | otherwise = pure Nothing
+      kemP query =
+        queryParam_ "kem_key" query
+          $>>= \k -> Just . kemParams k <$> queryParam_ "kem_ct" query
       kemParams k = \case
         Nothing -> ARKP SRKSProposed $ RKParamsProposed k
         Just ct -> ARKP SRKSAccepted $ RKParamsAccepted ct k
+      adjustE2EVRange vr =
+        let v = max minSupportedE2EEncryptVersion $ minVersion vr
+        in fromMaybe vr $ safeVersionRange v (max v $ maxVersion vr)
 
 instance (RatchetKEMStateI s, AlgorithmI a) => Encoding (E2ERatchetParamsUri s a) where
   smpEncode (E2ERatchetParamsUri vr k1 k2 kem_) = smpEncode (vr, k1, k2, kem_)
@@ -431,16 +421,14 @@ generateE2EParams g v useKEM_ = do
   where
     kemParams :: IO (Maybe (RKEMParams s, PrivRKEMParams s))
     kemParams = case useKEM_ of
-      Just useKem
-        | v >= pqRatchetE2EEncryptVersion ->
-            Just <$> do
-              ks@(k, _) <- sntrup761Keypair g
-              case useKem of
-                ProposeKEM -> pure (RKParamsProposed k, PrivateRKParamsProposed ks)
-                AcceptKEM k' -> do
-                  (ct, shared) <- sntrup761Enc g k'
-                  pure (RKParamsAccepted ct k, PrivateRKParamsAccepted ct shared ks)
-      _ -> pure Nothing
+      Just useKem -> Just <$> do
+        ks@(k, _) <- sntrup761Keypair g
+        case useKem of
+          ProposeKEM -> pure (RKParamsProposed k, PrivateRKParamsProposed ks)
+          AcceptKEM k' -> do
+            (ct, shared) <- sntrup761Enc g k'
+            pure (RKParamsAccepted ct k, PrivateRKParamsAccepted ct shared ks)
+      Nothing -> pure Nothing
 
 -- used by party initiating connection, Bob in double-ratchet spec
 generateRcvE2EParams :: (AlgorithmI a, DhAlgorithm a) => TVar ChaChaDRG -> VersionE2E -> PQSupport -> IO (RcvE2EPrivRatchetParams a, RcvE2ERatchetParams a)
@@ -463,6 +451,7 @@ generateSndE2EParams g v = \case
 
 data RatchetInitParams = RatchetInitParams
   { assocData :: Str,
+    rcVerifyCodePQ :: Str,
     ratchetKey :: RatchetKey,
     sndHK :: HeaderKey,
     rcvNextHK :: HeaderKey,
@@ -473,30 +462,30 @@ data RatchetInitParams = RatchetInitParams
 -- this is used by the peer joining the connection
 pqX3dhSnd :: DhAlgorithm a => AE2EPrivRatchetParams a -> E2ERatchetParams 'RKSProposed a -> Either CryptoError (RatchetInitParams, Maybe KEMKeyPair)
 --        3. replied       2. received
-pqX3dhSnd (spk1, spk2, spKem_) (E2ERatchetParams v rk1 rk2 rKem_) = do
+pqX3dhSnd (spk1, spk2, spKem_) (E2ERatchetParams _ rk1 rk2 rKem_) = do
   (ks_, kem_) <- sndPq
   let initParams = pqX3dh (publicKey spk1, rk1) (dh' rk1 spk2) (dh' rk2 spk1) (dh' rk2 spk2) kem_
   pure (initParams, ks_)
   where
     sndPq :: Either CryptoError (Maybe KEMKeyPair, Maybe RatchetKEMAccepted)
     sndPq = case spKem_ of
-      Just (APRKP _ ps) | v >= pqRatchetE2EEncryptVersion -> case (ps, rKem_) of
+      Just (APRKP _ ps) -> case (ps, rKem_) of
         (PrivateRKParamsAccepted ct shared ks, Just (RKParamsProposed k)) -> Right (Just ks, Just $ RatchetKEMAccepted k shared ct)
         (PrivateRKParamsProposed ks, _) -> Right (Just ks, Nothing) -- both parties can send "proposal" in case of ratchet renegotiation
         _ -> Left CERatchetKEMState
-      _ -> Right (Nothing, Nothing)
+      Nothing -> Right (Nothing, Nothing)
 
 -- this is used by the peer that created new connection, after receiving the reply
 pqX3dhRcv :: forall s a. (RatchetKEMStateI s, DhAlgorithm a) => RcvE2EPrivRatchetParams a -> E2ERatchetParams s a -> ExceptT CryptoError IO (RatchetInitParams, Maybe KEMKeyPair)
 --        1. sent          4. received in reply
-pqX3dhRcv (rpk1, rpk2, rpKem_) (E2ERatchetParams v sk1 sk2 sKem_) = do
+pqX3dhRcv (rpk1, rpk2, rpKem_) (E2ERatchetParams _ sk1 sk2 sKem_) = do
   kem_ <- rcvPq
   let initParams = pqX3dh (sk1, publicKey rpk1) (dh' sk2 rpk1) (dh' sk1 rpk2) (dh' sk2 rpk2) (snd <$> kem_)
   pure (initParams, fst <$> kem_)
   where
     rcvPq :: ExceptT CryptoError IO (Maybe (KEMKeyPair, RatchetKEMAccepted))
     rcvPq = case sKem_ of
-      Just (RKParamsAccepted ct k') | v >= pqRatchetE2EEncryptVersion -> case rpKem_ of
+      Just (RKParamsAccepted ct k') -> case rpKem_ of
         Just (PrivateRKParamsProposed ks@(_, pk)) -> do
           shared <- liftIO $ sntrup761Dec ct pk
           pure $ Just (ks, RatchetKEMAccepted k' shared ct)
@@ -505,14 +494,14 @@ pqX3dhRcv (rpk1, rpk2, rpKem_) (E2ERatchetParams v sk1 sk2 sKem_) = do
 
 pqX3dh :: DhAlgorithm a => (PublicKey a, PublicKey a) -> DhSecret a -> DhSecret a -> DhSecret a -> Maybe RatchetKEMAccepted -> RatchetInitParams
 pqX3dh (sk1, rk1) dh1 dh2 dh3 kemAccepted =
-  RatchetInitParams {assocData, ratchetKey = RatchetKey sk, sndHK = Key hk, rcvNextHK = Key nhk, kemAccepted}
+  RatchetInitParams {assocData, rcVerifyCodePQ = Str vcPQ, ratchetKey = RatchetKey sk, sndHK = Key hk, rcvNextHK = Key nhk, kemAccepted}
   where
     assocData = Str $ pubKeyBytes sk1 <> pubKeyBytes rk1
     dhs = dhBytes' dh1 <> dhBytes' dh2 <> dhBytes' dh3 <> pq
     pq = maybe "" (\RatchetKEMAccepted {rcPQRss = KEMSharedKey ss} -> BA.convert ss) kemAccepted
-    (hk, nhk, sk) =
-      let salt = B.replicate 64 '\0'
-       in hkdf3 salt dhs "SimpleXX3DH"
+    salt = B.replicate 64 '\0'
+    (hk, nhk, sk) = hkdf3 salt dhs "SimpleXX3DH"
+    vcPQ = hkdf salt dhs "SimpleXVerifyCode" 32
 
 type RatchetX448 = Ratchet 'X448
 
@@ -521,6 +510,8 @@ data Ratchet a = Ratchet
     rcVersion :: RatchetVersions,
     -- associated data - must be the same in both parties ratchets
     rcAD :: Str,
+    -- verification code covering all handshake keys, absent in the ratchets created before it was added
+    rcVCPQ :: Maybe Str,
     rcDHRs :: PrivateKey a,
     rcKEM :: Maybe RatchetKEM,
     rcSupportKEM :: PQSupport, -- defines header size, can only be enabled once
@@ -649,13 +640,14 @@ instance FromField MessageKey where fromField = blobFieldDecoder smpDecode
 -- @
 initSndRatchet ::
   forall a. (AlgorithmI a, DhAlgorithm a) => RatchetVersions -> PublicKey a -> PrivateKey a -> (RatchetInitParams, Maybe KEMKeyPair) -> Ratchet a
-initSndRatchet rcVersion rcDHRr rcDHRs (RatchetInitParams {assocData, ratchetKey, sndHK, rcvNextHK, kemAccepted}, rcPQRs_) = do
+initSndRatchet rcVersion rcDHRr rcDHRs (RatchetInitParams {assocData, rcVerifyCodePQ, ratchetKey, sndHK, rcvNextHK, kemAccepted}, rcPQRs_) = do
   -- state.RK, state.CKs, state.NHKs = KDF_RK_HE(SK, DH(state.DHRs, state.DHRr) || state.PQRss)
   let (rcRK, rcCKs, rcNHKs) = rootKdf ratchetKey rcDHRr rcDHRs (rcPQRss <$> kemAccepted)
       pqOn = isJust rcPQRs_
    in Ratchet
         { rcVersion,
           rcAD = assocData,
+          rcVCPQ = Just rcVerifyCodePQ,
           rcDHRs,
           rcKEM = (`RatchetKEM` kemAccepted) <$> rcPQRs_,
           rcSupportKEM = PQSupport pqOn,
@@ -680,10 +672,11 @@ initSndRatchet rcVersion rcDHRr rcDHRs (RatchetInitParams {assocData, ratchetKey
 -- as part of the connection request and random salt was received from the sender.
 initRcvRatchet ::
   forall a. (AlgorithmI a, DhAlgorithm a) => RatchetVersions -> PrivateKey a -> (RatchetInitParams, Maybe KEMKeyPair) -> PQSupport -> Ratchet a
-initRcvRatchet rcVersion rcDHRs (RatchetInitParams {assocData, ratchetKey, sndHK, rcvNextHK, kemAccepted}, rcPQRs_) pqSupport =
+initRcvRatchet rcVersion rcDHRs (RatchetInitParams {assocData, rcVerifyCodePQ, ratchetKey, sndHK, rcvNextHK, kemAccepted}, rcPQRs_) pqSupport =
   Ratchet
     { rcVersion,
       rcAD = assocData,
+      rcVCPQ = Just rcVerifyCodePQ,
       rcDHRs,
       -- rcKEM:
       -- state.PQRs = bob_pq_kem_key_pair
@@ -720,31 +713,22 @@ data MsgHeader a = MsgHeader
 -- to allow extension without increasing the size, the actual header length is:
 -- 69 = 2 (original size) + 2 + 1+56 (Curve448) + 4 + 4
 -- The exact size is 2288, added reserve
-paddedHeaderLen :: VersionE2E -> PQSupport -> Int
-paddedHeaderLen v = \case
-  PQSupportOn | v >= pqRatchetE2EEncryptVersion -> 2310
-  _ -> 88
+paddedHeaderLen :: PQSupport -> Int
+paddedHeaderLen = \case
+  PQSupportOn -> 2310
+  PQSupportOff -> 88
 
 -- only used in tests to validate correct padding
 -- (2 bytes - version size, 1 byte - header size)
-fullHeaderLen :: VersionE2E -> PQSupport -> Int
-fullHeaderLen v pq = 2 + 1 + paddedHeaderLen v pq + authTagSize + ivSize @AES256
+fullHeaderLen :: PQSupport -> Int
+fullHeaderLen pq = 2 + 1 + paddedHeaderLen pq + authTagSize + ivSize @AES256
 
--- pass the current version, as MsgHeader only includes the max supported version that can be different from the current
-encodeMsgHeader :: AlgorithmI a => VersionE2E -> MsgHeader a -> ByteString
-encodeMsgHeader v MsgHeader {msgMaxVersion, msgDHRs, msgKEM, msgPN, msgNs}
-  | v >= pqRatchetE2EEncryptVersion = smpEncode (msgMaxVersion, msgDHRs, msgKEM, msgPN, msgNs)
-  | otherwise = smpEncode (msgMaxVersion, msgDHRs, msgPN, msgNs)
-
--- pass the current version, as MsgHeader only includes the max supported version that can be different from the current
-msgHeaderP :: AlgorithmI a => VersionE2E -> Parser (MsgHeader a)
-msgHeaderP v = do
-  msgMaxVersion <- smpP
-  msgDHRs <- smpP
-  msgKEM <- if v >= pqRatchetE2EEncryptVersion then smpP else pure Nothing
-  msgPN <- smpP
-  msgNs <- smpP
-  pure MsgHeader {msgMaxVersion, msgDHRs, msgKEM, msgPN, msgNs}
+instance AlgorithmI a => Encoding (MsgHeader a) where
+  smpEncode MsgHeader {msgMaxVersion, msgDHRs, msgKEM, msgPN, msgNs} =
+    smpEncode (msgMaxVersion, msgDHRs, msgKEM, msgPN, msgNs)
+  smpP = do
+    (msgMaxVersion, msgDHRs, msgKEM, msgPN, msgNs) <- smpP
+    pure MsgHeader {msgMaxVersion, msgDHRs, msgKEM, msgPN, msgNs}
 
 data EncMessageHeader = EncMessageHeader
   { ehVersion :: VersionE2E, -- this is current ratchet version
@@ -756,26 +740,11 @@ data EncMessageHeader = EncMessageHeader
 -- this encoding depends on version in EncMessageHeader because it is "current" ratchet version
 instance Encoding EncMessageHeader where
   smpEncode EncMessageHeader {ehVersion, ehIV, ehAuthTag, ehBody} =
-    smpEncode (ehVersion, ehIV, ehAuthTag) <> encodeLarge ehVersion ehBody
+    smpEncode (ehVersion, ehIV, ehAuthTag, Large ehBody)
   smpP = do
-    (ehVersion, ehIV, ehAuthTag) <- smpP
-    ehBody <- largeP
+    (ehVersion, ehIV, ehAuthTag, Large ehBody) <- smpP
     pure EncMessageHeader {ehVersion, ehIV, ehAuthTag, ehBody}
 
--- the encoder always uses 2-byte lengths for the new version, even for short headers without PQ keys.
-encodeLarge :: VersionE2E -> ByteString -> ByteString
-encodeLarge v s
-  | v >= pqRatchetE2EEncryptVersion = smpEncode $ Large s
-  | otherwise = smpEncode s
-
--- This parser relies on the fact that header cannot be shorter than 32 bytes (it is ~69 bytes without PQ KEM),
--- therefore if the first byte is less or equal to 31 (x1F), then we have 2 byte-length limited to 8191.
--- This allows upgrading the current version in one message.
-largeP :: Parser ByteString
-largeP = do
-  len1 <- peekWord8'
-  if len1 < 32 then unLarge <$> smpP else smpP
-
 -- the header is length-prefixed to parse it as string and use as part of associated data for authenticated encryption
 data EncRatchetMessage = EncRatchetMessage
   { emHeader :: ByteString,
@@ -783,15 +752,12 @@ data EncRatchetMessage = EncRatchetMessage
     emBody :: ByteString
   }
 
-encodeEncRatchetMessage :: VersionE2E -> EncRatchetMessage -> ByteString
-encodeEncRatchetMessage v EncRatchetMessage {emHeader, emBody, emAuthTag} =
-  encodeLarge v emHeader <> smpEncode (emAuthTag, Tail emBody)
-
-encRatchetMessageP :: Parser EncRatchetMessage
-encRatchetMessageP = do
-  emHeader <- largeP
-  (emAuthTag, Tail emBody) <- smpP
-  pure EncRatchetMessage {emHeader, emBody, emAuthTag}
+instance Encoding EncRatchetMessage where
+  smpEncode EncRatchetMessage {emHeader, emBody, emAuthTag} =
+    smpEncode (Large emHeader, emAuthTag, Tail emBody)
+  smpP = do
+    (Large emHeader, emAuthTag, Tail emBody) <- smpP
+    pure EncRatchetMessage {emHeader, emBody, emAuthTag}
 
 newtype PQEncryption = PQEncryption {enablePQ :: Bool}
   deriving (Eq, Show)
@@ -840,15 +806,15 @@ pqEncToSupport (PQEncryption pq) = PQSupport pq
 pqSupportAnd :: PQSupport -> PQSupport -> PQSupport
 pqSupportAnd (PQSupport s1) (PQSupport s2) = PQSupport $ s1 && s2
 
-pqEnableSupport :: VersionE2E -> PQSupport -> PQEncryption -> PQSupport
-pqEnableSupport v (PQSupport sup) (PQEncryption enc) = PQSupport $ sup || (v >= pqRatchetE2EEncryptVersion && enc)
+pqEnableSupport :: PQSupport -> PQEncryption -> PQSupport
+pqEnableSupport (PQSupport sup) (PQEncryption enc) = PQSupport $ sup || enc
 
-replyKEM_ :: VersionE2E -> Maybe (RKEMParams 'RKSProposed) -> PQSupport -> Maybe AUseKEM
-replyKEM_ v kem_ = \case
-  PQSupportOn | v >= pqRatchetE2EEncryptVersion -> Just $ case kem_ of
+replyKEM_ :: Maybe (RKEMParams 'RKSProposed) -> PQSupport -> Maybe AUseKEM
+replyKEM_ kem_ = \case
+  PQSupportOn -> Just $ case kem_ of
     Just (RKParamsProposed k) -> AUseKEM SRKSAccepted $ AcceptKEM k
     Nothing -> AUseKEM SRKSProposed ProposeKEM
-  _ -> Nothing
+  PQSupportOff -> Nothing
 
 instance StrEncoding PQEncryption where
   strEncode pqMode
@@ -897,9 +863,9 @@ connPQEncryption = \case
   IKUsePQ -> PQSupportOn
   IKLinkPQ pq -> pq -- default for creating connection is IKLinkPQ PQEncOn
 
-joinContactInitialKeys :: Bool -> PQSupport -> InitialKeys
-joinContactInitialKeys pqCompatible = \case
-  PQSupportOn | pqCompatible -> IKUsePQ
+joinContactInitialKeys :: PQSupport -> InitialKeys
+joinContactInitialKeys = \case
+  PQSupportOn -> IKUsePQ
   pqEnc -> IKLinkPQ pqEnc
 
 rcCheckCanPad :: Int -> ByteString -> ExceptT CryptoError IO ()
@@ -915,14 +881,14 @@ rcEncryptHeader rc@Ratchet {rcSnd = Just sr@SndRatchet {rcCKs, rcHKs}, rcDHRs, r
       -- PQ encryption can be enabled or disabled
       rcEnableKEM' = fromMaybe rcEnableKEM pqEnc_
       -- support for PQ encryption (and therefore large headers/small envelopes) can only be enabled, it cannot be disabled
-      rcSupportKEM' = pqEnableSupport v rcSupportKEM rcEnableKEM'
+      rcSupportKEM' = pqEnableSupport rcSupportKEM rcEnableKEM'
       -- This sets max version to support PQ encryption.
       -- Current version upgrade happens when peer decrypts the message.
       -- TODO note that maxSupported will not downgrade here below current (v).
-      maxSupported' = max supportedE2EVersion $ if pqEnc_ == Just PQEncOn then pqRatchetE2EEncryptVersion else v
+      maxSupported' = max supportedE2EVersion $ if pqEnc_ == Just PQEncOn then minSupportedE2EEncryptVersion else v
       rcVersion' = rcVersion {maxSupported = maxSupported'}
   -- enc_header = HENCRYPT(state.HKs, header)
-  (ehAuthTag, ehBody) <- encryptAEAD rcHKs ehIV (paddedHeaderLen v rcSupportKEM') rcAD (msgHeader v maxSupported')
+  (ehAuthTag, ehBody) <- encryptAEAD rcHKs ehIV (paddedHeaderLen rcSupportKEM') rcAD (msgHeader maxSupported')
   -- return enc_header
   let emHeader = smpEncode EncMessageHeader {ehVersion = v, ehBody, ehAuthTag, ehIV}
       msgEncryptKey =
@@ -950,9 +916,8 @@ rcEncryptHeader rc@Ratchet {rcSnd = Just sr@SndRatchet {rcCKs, rcHKs}, rcDHRs, r
     --   pn = state.PN,
     --   n = state.Ns
     -- )
-    msgHeader v maxSupported' =
-      encodeMsgHeader
-        v
+    msgHeader maxSupported' =
+      smpEncode
         MsgHeader
           { msgMaxVersion = maxSupported',
             msgDHRs = publicKey rcDHRs,
@@ -975,11 +940,10 @@ data MsgEncryptKey a = MsgEncryptKey
   deriving (Show)
 
 rcEncryptMsg :: AlgorithmI a => MsgEncryptKey a -> Int -> ByteString -> ExceptT CryptoError IO ByteString
-rcEncryptMsg MsgEncryptKey {msgKey = MessageKey mk iv, msgRcAD, msgEncHeader, msgRcVersion = v} paddedMsgLen msg = do
+rcEncryptMsg MsgEncryptKey {msgKey = MessageKey mk iv, msgRcAD, msgEncHeader} paddedMsgLen msg = do
   -- return ENCRYPT(mk, plaintext, CONCAT(AD, enc_header))
   (emAuthTag, emBody) <- encryptAEAD mk iv paddedMsgLen (msgRcAD <> msgEncHeader) msg
-  let msg' = encodeEncRatchetMessage v EncRatchetMessage {emHeader = msgEncHeader, emBody, emAuthTag}
-  pure msg'
+  pure $ smpEncode EncRatchetMessage {emHeader = msgEncHeader, emBody, emAuthTag}
 
 data SkippedMessage a
   = SMMessage (DecryptResult a)
@@ -1003,7 +967,7 @@ rcDecrypt ::
   ByteString ->
   ExceptT CryptoError IO (DecryptResult a)
 rcDecrypt g rc@Ratchet {rcRcv, rcAD = Str rcAD, rcVersion} rcMKSkipped msg' = do
-  encMsg@EncRatchetMessage {emHeader} <- parseE CryptoHeaderError encRatchetMessageP msg'
+  encMsg@EncRatchetMessage {emHeader} <- parseE CryptoHeaderError smpP msg'
   encHdr <- parseE CryptoHeaderError smpP emHeader
   -- plaintext = TrySkippedMessageKeysHE(state, enc_header, cipher-text, AD)
   decryptSkipped encHdr encMsg >>= \case
@@ -1048,7 +1012,7 @@ rcDecrypt g rc@Ratchet {rcRcv, rcAD = Str rcAD, rcVersion} rcMKSkipped msg' = do
         smkDiff :: SkippedMsgKeys -> SkippedMsgDiff
         smkDiff smks = if M.null smks then SMDNoChange else SMDAdd smks
         ratchetStep :: Ratchet a -> MsgHeader a -> ExceptT CryptoError IO (Ratchet a)
-        ratchetStep rc'@Ratchet {rcDHRs, rcRK, rcNHKs, rcNHKr, rcSupportKEM, rcVersion = rv} MsgHeader {msgDHRs, msgKEM} = do
+        ratchetStep rc'@Ratchet {rcDHRs, rcRK, rcNHKs, rcNHKr, rcSupportKEM} MsgHeader {msgDHRs, msgKEM} = do
           (kemSS, kemSS', rcKEM') <- pqRatchetStep rc' msgKEM
           -- state.DHRs = GENERATE_DH()
           (_, rcDHRs') <- atomically $ generateKeyPair @a g
@@ -1063,7 +1027,7 @@ rcDecrypt g rc@Ratchet {rcRcv, rcAD = Str rcAD, rcVersion} rcMKSkipped msg' = do
             rc'
               { rcDHRs = rcDHRs',
                 rcKEM = rcKEM',
-                rcSupportKEM = pqEnableSupport (current rv) rcSupportKEM rcEnableKEM',
+                rcSupportKEM = pqEnableSupport rcSupportKEM rcEnableKEM',
                 rcEnableKEM = rcEnableKEM',
                 rcSndKEM = PQEncryption sndKEM,
                 rcRcvKEM = PQEncryption rcvKEM,
@@ -1077,17 +1041,17 @@ rcDecrypt g rc@Ratchet {rcRcv, rcAD = Str rcAD, rcVersion} rcMKSkipped msg' = do
                 rcNHKr = rcNHKr'
               }
         pqRatchetStep :: Ratchet a -> Maybe ARKEMParams -> ExceptT CryptoError IO (Maybe KEMSharedKey, Maybe KEMSharedKey, Maybe RatchetKEM)
-        pqRatchetStep Ratchet {rcKEM, rcEnableKEM = PQEncryption pqEnc, rcVersion = rv} = \case
+        pqRatchetStep Ratchet {rcKEM, rcEnableKEM = PQEncryption pqEnc} = \case
           -- received message does not have KEM in header,
           -- but the user enabled KEM when sending previous message
           Nothing -> case rcKEM of
-            Nothing | pqEnc && current rv >= pqRatchetE2EEncryptVersion -> do
+            Nothing | pqEnc -> do
               rcPQRs <- liftIO $ sntrup761Keypair g
               pure (Nothing, Nothing, Just RatchetKEM {rcPQRs, rcKEMs = Nothing})
             _ -> pure (Nothing, Nothing, Nothing)
           -- received message has KEM in header.
           Just (ARKP _ ps)
-            | pqEnc && current rv >= pqRatchetE2EEncryptVersion -> do
+            | pqEnc -> do
                 -- state.PQRr = header.kem
                 (ss, rcPQRr) <- sharedSecret
                 -- state.PQRct = PQKEM-ENC(state.PQRr, state.PQRss) // encapsulated additional shared secret KEM #1
@@ -1155,9 +1119,9 @@ rcDecrypt g rc@Ratchet {rcRcv, rcAD = Str rcAD, rcVersion} rcMKSkipped msg' = do
         e -> throwE e
     -- header = HDECRYPT(state.NHKr, enc_header)
     decryptNextHeader hdr = (AdvanceRatchet,) <$> decryptHeader (rcNHKr rc) hdr
-    decryptHeader k EncMessageHeader {ehVersion, ehBody, ehAuthTag, ehIV} = do
+    decryptHeader k EncMessageHeader {ehBody, ehAuthTag, ehIV} = do
       header <- decryptAEAD k ehIV rcAD ehBody ehAuthTag `catchE` \_ -> throwE CERatchetHeader
-      parseE' CryptoHeaderError (msgHeaderP ehVersion) header
+      parseE' CryptoHeaderError smpP header
     decryptMessage :: MessageKey -> EncRatchetMessage -> ExceptT CryptoError IO (Either CryptoError ByteString)
     decryptMessage (MessageKey mk iv) EncRatchetMessage {emHeader, emBody, emAuthTag} =
       -- DECRYPT(mk, cipher-text, CONCAT(AD, enc_header))

src/Simplex/Messaging/Crypto/ShortLink.hs

@@ -53,14 +53,14 @@ invShortLinkKdf :: LinkKey -> C.SbKey
 invShortLinkKdf (LinkKey k) = C.unsafeSbKey $ C.hkdf "" k "SimpleXInvLink" 32
 
 encodeSignLinkData :: forall c. ConnectionModeI c => C.KeyPairEd25519 -> VersionRangeSMPA -> ConnectionRequestUri c -> Maybe ByteString -> UserConnLinkData c -> (LinkKey, (ByteString, ByteString))
-encodeSignLinkData keys@(_, pk) agentVRange linkConnReq linkEntityId userData =
-  let (linkKey, fd) = encodeSignFixedData keys agentVRange linkConnReq linkEntityId
+encodeSignLinkData keys@(_, pk) agentVRange connReq linkEntityId userData =
+  let (linkKey, fd) = encodeSignFixedData keys agentVRange connReq linkEntityId
       md = encodeSignUserData (sConnectionMode @c) pk agentVRange userData
    in (linkKey, (fd, md))
 
 encodeSignFixedData :: ConnectionModeI c => C.KeyPairEd25519 -> VersionRangeSMPA -> ConnectionRequestUri c -> Maybe ByteString -> (LinkKey, ByteString)
-encodeSignFixedData (rootKey, pk) agentVRange linkConnReq linkEntityId =
-  let fd = smpEncode FixedLinkData {agentVRange, rootKey, linkConnReq, linkEntityId}
+encodeSignFixedData (rootKey, pk) agentVRange connReq linkEntityId =
+  let fd = smpEncode FixedLinkData {agentVRange, rootKey, linkConnReq = binaryConnReq connReq, linkEntityId}
    in (LinkKey (C.sha3_256 fd), encodeSign pk fd)
 
 encodeSignUserData :: ConnectionModeI c => SConnectionMode c -> C.PrivateKeyEd25519 -> VersionRangeSMPA -> UserConnLinkData c -> ByteString
@@ -120,6 +120,6 @@ decryptLinkData linkKey k (encFD, encMD) = do
       pure (sig, s)
     decode :: Encoding a => ByteString -> Either AgentErrorType a
     decode = msgErr . smpDecode
-    msgErr = first (const $ AGENT A_MESSAGE)
+    msgErr = first (const $ AGENT $ A_MESSAGE "parse link data")
     linkErr :: String -> Either AgentErrorType ()
     linkErr = Left . AGENT . A_LINK

src/Simplex/Messaging/Encoding.hs

@@ -11,6 +11,7 @@ module Simplex.Messaging.Encoding
   ( Encoding (..),
     Tail (..),
     Large (..),
+    EncList (..),
     _smpP,
     smpEncodeList,
     smpListP,
@@ -177,6 +178,12 @@ instance Encoding a => Encoding (L.NonEmpty a) where
       0 -> fail "empty list"
       n -> L.fromList <$> A.count n smpP
 
+newtype EncList a = EncList [a]
+
+instance Encoding a => Encoding (EncList a) where
+  smpEncode (EncList xs) = smpEncodeList xs
+  smpP = EncList <$> smpListP
+
 instance (Encoding a, Encoding b) => Encoding (a, b) where
   smpEncode (a, b) = smpEncode a <> smpEncode b
   {-# INLINE smpEncode #-}

src/Simplex/Messaging/Names/Record.hs

@@ -1,3 +1,6 @@
+{-# LANGUAGE DerivingStrategies #-}
+{-# LANGUAGE GeneralizedNewtypeDeriving #-}
+{-# LANGUAGE LambdaCase #-}
 {-# LANGUAGE NamedFieldPuns #-}
 {-# LANGUAGE OverloadedStrings #-}
 {-# LANGUAGE StrictData #-}
@@ -5,13 +8,24 @@
 
 module Simplex.Messaging.Names.Record
   ( NameRecord (..),
+    NameResponse (..),
+    NameRegistration (..),
+    NamePricing (..),
+    USDCents (..),
+    NameReservedReason (..),
   )
 where
 
+import Data.Aeson (FromJSON (..), ToJSON (..))
 import qualified Data.Aeson as J
 import qualified Data.Aeson.TH as JQ
+import Data.Int (Int64)
+import Data.Map.Strict (Map)
 import Data.Text (Text)
-import Simplex.Messaging.Parsers (defaultJSON, dropPrefix)
+import qualified Data.Text as T
+import Simplex.Messaging.Encoding.String
+import Simplex.Messaging.Parsers (defaultJSON, dropPrefix, taggedObjectJSON)
+import Simplex.Messaging.SystemTime (SystemSeconds)
 
 -- | Resolved name record returned by the names role. JSON keys match the
 --   resolver REST output; both FromJSON (resolver -> server) and ToJSON
@@ -44,3 +58,87 @@ $( JQ.deriveJSON
     defaultJSON {J.omitNothingFields = False, J.fieldLabelModifier = dropPrefix "nr"}
     ''NameRecord
  )
+
+-- | US cents.
+newtype USDCents = USDCents Int64
+  deriving (Eq, Ord, Show)
+  deriving newtype (ToJSON, FromJSON)
+
+-- | What the registry holds for a name, and the block it was read at.
+data NameResponse = NameResponse
+  { -- | absent only from a v20/v21 router, which sent the record alone
+    lastBlockTs :: Maybe SystemSeconds,
+    registration :: NameRegistration
+  }
+  deriving (Eq, Show)
+
+-- | What the registry holds for a name.
+data NameRegistration
+  = -- | Held by someone. Always carries a record, empty where none was set.
+    NRRegistered
+      { -- | absent only from a v20/v21 router, which sent the record alone
+        expires :: Maybe SystemSeconds,
+        -- | unix seconds, > expires: until here only the owner may renew
+        graceUntil :: Maybe SystemSeconds,
+        -- | held back as well, which is why it will not free up at expiry
+        reservedReason_ :: Maybe NameReservedReason,
+        nameRecord :: NameRecord
+      }
+  | -- | Held by nobody, and registrable now.
+    NRAvailable {pricing :: NamePricing}
+  | -- | Held back by the registry, and not for sale at its price.
+    NRReserved {reservedReason :: NameReservedReason}
+  deriving (Eq, Show)
+
+-- | Enough to price the name locally, which the router cannot do behind a hash.
+data NamePricing = NamePricing
+  { -- | US cents per year, for the lengths the registry prices specially
+    registrationPrices :: Map Int USDCents,
+    -- | US cents per year for every other length
+    basePrice :: USDCents,
+    -- | characters; the registry refuses shorter labels
+    minLabelLength :: Int
+  }
+  deriving (Eq, Show)
+
+-- | Why the registry holds a name back.
+data NameReservedReason
+  = -- | held for SimpleX
+    NRRInternal
+  | NRRTrademark
+  | NRRCommunity
+  | -- | added to the registry after this version, and still reserved
+    NRRUnknown Text
+  deriving (Eq, Show)
+
+instance TextEncoding NameReservedReason where
+  textEncode = \case
+    NRRInternal -> "internal"
+    NRRTrademark -> "trademark"
+    NRRCommunity -> "community"
+    NRRUnknown t -> t
+  textDecode = Just . reservedReasonOf
+
+-- | An unknown reason is kept as text, capped: it reaches a client as a word.
+-- Capping precedes the match, so what is kept encodes back to what it decoded.
+reservedReasonOf :: Text -> NameReservedReason
+reservedReasonOf t = case T.take 32 $ T.takeWhile (\c -> c > ' ' && c < '\DEL') t of
+  "internal" -> NRRInternal
+  "trademark" -> NRRTrademark
+  "community" -> NRRCommunity
+  r -> NRRUnknown r
+
+instance ToJSON NameReservedReason where
+  toJSON = textToJSON
+  toEncoding = textToEncoding
+
+instance FromJSON NameReservedReason where
+  parseJSON = textParseJSON "NameReservedReason"
+
+$(JQ.deriveJSON defaultJSON ''NamePricing)
+
+-- taggedObjectJSON, not sumTypeJSON: this JSON is the RNAME payload and the
+-- resolver contract, so it must not vary with the swift build flag.
+$(JQ.deriveJSON (taggedObjectJSON $ dropPrefix "NR") ''NameRegistration)
+
+$(JQ.deriveJSON defaultJSON ''NameResponse)

src/Simplex/Messaging/Notifications/Protocol.hs

@@ -61,7 +61,7 @@ import Simplex.Messaging.Agent.Store.DB (FromField (..), ToField (..), fromTextF
 import qualified Simplex.Messaging.Crypto as C
 import Simplex.Messaging.Encoding
 import Simplex.Messaging.Encoding.String
-import Simplex.Messaging.Notifications.Transport (NTFVersion, invalidReasonNTFVersion, ntfClientHandshake)
+import Simplex.Messaging.Notifications.Transport (NTFVersion, ntfClientHandshake)
 import Simplex.Messaging.Protocol hiding (Command (..), CommandTag (..))
 import Simplex.Messaging.Util (eitherToMaybe, (<$?>))
 
@@ -329,18 +329,12 @@ data NtfResponse
 
 instance ProtocolEncoding NTFVersion ErrorType NtfResponse where
   type Tag NtfResponse = NtfResponseTag
-  encodeProtocol v = \case
+  encodeProtocol _v = \case
     NRTknId entId dhKey -> e (NRTknId_, ' ', entId, dhKey)
     NRSubId entId -> e (NRSubId_, ' ', entId)
     NROk -> e NROk_
     NRErr err -> e (NRErr_, ' ', err)
-    NRTkn stat -> e (NRTkn_, ' ', stat')
-      where
-        stat'
-          | v >= invalidReasonNTFVersion = stat
-          | otherwise = case stat of
-              NTInvalid _ -> NTInvalid Nothing
-              _ -> stat
+    NRTkn stat -> e (NRTkn_, ' ', stat)
     NRSub stat -> e (NRSub_, ' ', stat)
     NRPong -> e NRPong_
     where

src/Simplex/Messaging/Notifications/Server.hs

@@ -34,6 +34,7 @@ import Data.ByteString.Char8 (ByteString)
 import qualified Data.ByteString.Char8 as B
 import Data.Either (partitionEithers)
 import Data.Functor (($>))
+import Data.Hashable (hash)
 import Data.IORef
 import Data.Int (Int64)
 import qualified Data.IntSet as IS
@@ -526,10 +527,10 @@ subscribeNtfs NtfSubscriber {smpSubscribers, subscriberSeq, smpAgent = ca} st sm
       subscribeQueuesNtfs ca smpServer' [sub]
 
 ntfSubscriber :: NtfSubscriber -> M ()
-ntfSubscriber NtfSubscriber {smpAgent = ca@SMPClientAgent {msgQ, agentQ}} =
+ntfSubscriber NtfSubscriber {smpAgent = ca@SMPClientAgent {msgQ = msgQ_, agentQ}} =
   race_ receiveSMP receiveAgent
   where
-    receiveSMP = do
+    receiveSMP = forM_ msgQ_ $ \msgQ -> do
       st <- asks store
       ps <- asks pushServer
       stats <- asks serverStats
@@ -640,11 +641,13 @@ showServer' :: SMPServer -> Text
 showServer' = decodeLatin1 . strEncode . host
 
 pushNotification :: NtfPushServer -> Maybe T.Text -> OwnServer -> NtfTknRec -> PushNotification -> M ()
-pushNotification s srvHost_ isOwn tkn@NtfTknRec {token = token@(DeviceToken pp _)} ntf =
+pushNotification s srvHost_ isOwn tkn@NtfTknRec {ntfTknId, token = token@(DeviceToken pp _)} ntf =
   ifM
     (pushProviderAllowed token)
-    (getOrCreatePushWorker s (srvHost_, pp) isOwn >>= atomically . (`writeTBQueue` (tkn, ntf)))
+    (getOrCreatePushWorker s (srvHost_, pp, hash (unEntityId ntfTknId) `mod` pushWorkersPerServer) isOwn >>= atomically . (`writeTBQueue` (tkn, ntf)))
     (logWarn "skipping disabled APNS test push provider")
+  where
+    pushWorkersPerServer = 8
 
 pushProviderAllowed :: DeviceToken -> M Bool
 pushProviderAllowed (DeviceToken PPApnsTest _) = asks (allowTestPushProvider . config)
@@ -657,8 +660,8 @@ guardPushProvider token action =
     action
     (pure $ NRErr $ CMD SMP.PROHIBITED)
 
-getOrCreatePushWorker :: NtfPushServer -> (Maybe T.Text, PushProvider) -> OwnServer -> M (TBQueue (NtfTknRec, PushNotification))
-getOrCreatePushWorker s@NtfPushServer {pushWorkers, pushWorkerSeq, pushQSize} key@(srvHost_, _) isOwn = do
+getOrCreatePushWorker :: NtfPushServer -> (Maybe T.Text, PushProvider, Int) -> OwnServer -> M (TBQueue (NtfTknRec, PushNotification))
+getOrCreatePushWorker s@NtfPushServer {pushWorkers, pushWorkerSeq, pushQSize} key@(srvHost_, _, _) isOwn = do
   ts <- liftIO getCurrentTime
   withGetSessVar' pushWorkerSeq key pushWorkers ts createWorker existingWorker
   where
@@ -731,7 +734,7 @@ runPushWorker s srvHost_ isOwn q = forever $ do
               _ -> err e
         err e = logError ("Push provider error (" <> tshow pp <> ", " <> tshow ntfTknId <> "): " <> tshow e) $> Left e
 
-pushWorkersQLength :: TMap (Maybe T.Text, PushProvider) PushWorkerVar -> IO Natural
+pushWorkersQLength :: TMap (Maybe T.Text, PushProvider, Int) PushWorkerVar -> IO Natural
 pushWorkersQLength workers = do
   ws <- readTVarIO workers
   foldM addQLength 0 ws

src/Simplex/Messaging/Notifications/Server/Env.hs

@@ -174,7 +174,7 @@ data SMPSubscriber = SMPSubscriber
   }
 
 data NtfPushServer = NtfPushServer
-  { pushWorkers :: TMap (Maybe T.Text, PushProvider) PushWorkerVar,
+  { pushWorkers :: TMap (Maybe T.Text, PushProvider, Int) PushWorkerVar, -- Int is the worker shard
     pushWorkerSeq :: TVar Int,
     pushQSize :: Natural,
     pushClients :: TMap PushProvider PushClientVar,

src/Simplex/Messaging/Notifications/Transport.hs

@@ -12,7 +12,6 @@ module Simplex.Messaging.Notifications.Transport
     VersionRangeNTF,
     pattern VersionNTF,
     THandleNTF,
-    invalidReasonNTFVersion,
     supportedClientNTFVRange,
     supportedServerNTFVRange,
     alpnSupportedNTFHandshakes,
@@ -20,12 +19,8 @@ module Simplex.Messaging.Notifications.Transport
     ntfClientHandshake,
   ) where
 
-import Control.Monad (forM)
 import Control.Monad.Except
 import Control.Monad.Trans.Except
-import Data.Attoparsec.ByteString.Char8 (Parser)
-import Data.ByteString.Char8 (ByteString)
-import qualified Data.ByteString.Char8 as B
 import Data.Word (Word16)
 import qualified Data.X509 as X
 import qualified Simplex.Messaging.Crypto as C
@@ -50,13 +45,10 @@ pattern VersionNTF :: Word16 -> VersionNTF
 pattern VersionNTF v = Version v
 
 initialNTFVersion :: VersionNTF
-initialNTFVersion = VersionNTF 1
+initialNTFVersion = VersionNTF 3
 
-authBatchCmdsNTFVersion :: VersionNTF
-authBatchCmdsNTFVersion = VersionNTF 2
-
-invalidReasonNTFVersion :: VersionNTF
-invalidReasonNTFVersion = VersionNTF 3
+_invalidReasonNTFVersion :: VersionNTF
+_invalidReasonNTFVersion = VersionNTF 3
 
 currentClientNTFVersion :: VersionNTF
 currentClientNTFVersion = VersionNTF 3
@@ -67,9 +59,6 @@ currentServerNTFVersion = VersionNTF 3
 supportedClientNTFVRange :: VersionRangeNTF
 supportedClientNTFVRange = mkVersionRange initialNTFVersion currentClientNTFVersion
 
-legacyServerNTFVRange :: VersionRangeNTF
-legacyServerNTFVRange = mkVersionRange initialNTFVersion initialNTFVersion
-
 supportedServerNTFVRange :: VersionRangeNTF
 supportedServerNTFVRange = mkVersionRange initialNTFVersion currentServerNTFVersion
 
@@ -82,7 +71,7 @@ data NtfServerHandshake = NtfServerHandshake
   { ntfVersionRange :: VersionRangeNTF,
     sessionId :: SessionId,
     -- pub key to agree shared secrets for command authorization and entity ID encryption.
-    authPubKey :: Maybe (X.SignedExact X.PubKey)
+    authPubKey :: X.SignedExact X.PubKey
   }
 
 data NtfClientHandshake = NtfClientHandshake
@@ -94,25 +83,13 @@ data NtfClientHandshake = NtfClientHandshake
 
 instance Encoding NtfServerHandshake where
   smpEncode NtfServerHandshake {ntfVersionRange, sessionId, authPubKey} =
-    B.concat
-      [ smpEncode (ntfVersionRange, sessionId),
-        encodeAuthEncryptCmds (maxVersion ntfVersionRange) $ C.SignedObject <$> authPubKey
-      ]
+    smpEncode (ntfVersionRange, sessionId, C.SignedObject authPubKey)
 
   smpP = do
     (ntfVersionRange, sessionId) <- smpP
-    -- TODO drop SMP v6: remove special parser and make key non-optional
-    authPubKey <- authEncryptCmdsP (maxVersion ntfVersionRange) $ C.getSignedExact <$> smpP
+    authPubKey <- C.getSignedExact <$> smpP
     pure NtfServerHandshake {ntfVersionRange, sessionId, authPubKey}
 
-encodeAuthEncryptCmds :: Encoding a => VersionNTF -> Maybe a -> ByteString
-encodeAuthEncryptCmds v k
-  | v >= authBatchCmdsNTFVersion = maybe "" smpEncode k
-  | otherwise = ""
-
-authEncryptCmdsP :: VersionNTF -> Parser a -> Parser (Maybe a)
-authEncryptCmdsP v p = if v >= authBatchCmdsNTFVersion then Just <$> p else pure Nothing
-
 instance Encoding NtfClientHandshake where
   smpEncode NtfClientHandshake {ntfVersion, keyHash} =
     smpEncode (ntfVersion, keyHash)
@@ -122,11 +99,10 @@ instance Encoding NtfClientHandshake where
 
 -- | Notifcations server transport handshake.
 ntfServerHandshake :: forall c. Transport c => C.APrivateSignKey -> c 'TServer -> C.KeyPairX25519 -> C.KeyHash -> VersionRangeNTF -> ExceptT TransportError IO (THandleNTF c 'TServer)
-ntfServerHandshake serverSignKey c (k, pk) kh ntfVRange = do
+ntfServerHandshake serverSignKey c (k, pk) kh ntfVersionRange = do
   let th@THandle {params = THandleParams {sessionId}} = ntfTHandle c
-  let sk = C.signX509 serverSignKey $ C.publicToX509 k
-  let ntfVersionRange = maybe legacyServerNTFVRange (const ntfVRange) $ getSessionALPN c
-  sendHandshake th $ NtfServerHandshake {sessionId, ntfVersionRange, authPubKey = Just sk}
+      authPubKey = C.signX509 serverSignKey $ C.publicToX509 k
+  sendHandshake th $ NtfServerHandshake {sessionId, ntfVersionRange, authPubKey}
   getHandshake th >>= \case
     NtfClientHandshake {ntfVersion = v, keyHash}
       | keyHash /= kh ->
@@ -140,36 +116,35 @@ ntfServerHandshake serverSignKey c (k, pk) kh ntfVRange = do
 ntfClientHandshake :: forall c. Transport c => c 'TClient -> C.KeyHash -> VersionRangeNTF -> Bool -> Maybe (ServiceCredentials, C.KeyPairEd25519) -> ExceptT TransportError IO (THandleNTF c 'TClient)
 ntfClientHandshake c keyHash ntfVRange _proxyServer _serviceKeys = do
   let th@THandle {params = THandleParams {sessionId}} = ntfTHandle c
-  NtfServerHandshake {sessionId = sessId, ntfVersionRange, authPubKey = sk'} <- getHandshake th
+  NtfServerHandshake {sessionId = sessId, ntfVersionRange, authPubKey} <- getHandshake th
   if sessionId /= sessId
     then throwE TEBadSession
     else case ntfVersionRange `compatibleVRange` ntfVRange of
       Just (Compatible vr) -> do
-        ck_ <- forM sk' $ \signedKey -> liftEitherWith (const $ TEHandshake BAD_AUTH) $ do
+        ck <- liftEitherWith (const $ TEHandshake BAD_AUTH) $ do
           serverKey <- getServerVerifyKey c
-          pubKey <- C.verifyX509 serverKey signedKey
-          (,CertChainPubKey (getPeerCertChain c) signedKey) <$> C.x509ToPublic' pubKey
+          pubKey <- C.verifyX509 serverKey authPubKey
+          (,CertChainPubKey (getPeerCertChain c) authPubKey) <$> C.x509ToPublic' pubKey
         let v = maxVersion vr
         sendHandshake th $ NtfClientHandshake {ntfVersion = v, keyHash}
-        pure $ ntfThHandleClient th v vr ck_
+        pure $ ntfThHandleClient th v vr ck
       Nothing -> throwE TEVersion
 
 ntfThHandleServer :: forall c. THandleNTF c 'TServer -> VersionNTF -> VersionRangeNTF -> C.PrivateKeyX25519 -> THandleNTF c 'TServer
 ntfThHandleServer th v vr pk =
-  let thAuth = THAuthServer {serverPrivKey = pk, peerClientService = Nothing, sessSecret' = Nothing}
+  let thAuth = THAuthServer {serverPrivKey = pk, peerClientService = Nothing, peerEntitlement = Nothing, sessSecret' = Nothing}
    in ntfThHandle_ th v vr (Just thAuth)
 
-ntfThHandleClient :: forall c. THandleNTF c 'TClient -> VersionNTF -> VersionRangeNTF -> Maybe (C.PublicKeyX25519, CertChainPubKey) -> THandleNTF c 'TClient
+ntfThHandleClient :: forall c. THandleNTF c 'TClient -> VersionNTF -> VersionRangeNTF -> (C.PublicKeyX25519, CertChainPubKey) -> THandleNTF c 'TClient
 ntfThHandleClient th v vr ck_ =
-  let thAuth = clientTHParams <$> ck_
+  let thAuth = Just $ clientTHParams ck_
       clientTHParams (k, ck) = THAuthClient {peerServerPubKey = k, peerServerCertKey = ck, clientService = Nothing, sessSecret = Nothing}
    in ntfThHandle_ th v vr thAuth
 
 ntfThHandle_ :: forall c p. THandleNTF c p -> VersionNTF -> VersionRangeNTF -> Maybe (THandleAuth p) -> THandleNTF c p
 ntfThHandle_ th@THandle {params} v vr thAuth =
   -- TODO drop SMP v6: make thAuth non-optional
-  let v3 = v >= authBatchCmdsNTFVersion
-      params' = params {thVersion = v, thServerVRange = vr, thAuth, implySessId = v3, batch = v3}
+  let params' = params {thVersion = v, thServerVRange = vr, thAuth}
    in (th :: THandleNTF c p) {params = params'}
 
 ntfTHandle :: Transport c => c p -> THandleNTF c p
@@ -183,8 +158,8 @@ ntfTHandle c = THandle {connection = c, params}
           thVersion = v,
           thServerVRange = versionToRange v,
           thAuth = Nothing,
-          implySessId = False,
+          implySessId = True,
           encryptBlock = Nothing,
-          batch = False,
-          serviceAuth = False
+          serviceAuth = False,
+          serverInfo = Nothing
         }

src/Simplex/Messaging/Protocol.hs

@@ -80,6 +80,12 @@ module Simplex.Messaging.Protocol
     ErrorType (..),
     CommandError (..),
     ProxyError (..),
+    NameQuery (..),
+    NameResponse (..),
+    NameRegistration (..),
+    NamePricing (..),
+    USDCents (..),
+    NameReservedReason (..),
     NameErrorType (..),
     BrokerErrorType (..),
     NetworkError (..),
@@ -235,6 +241,7 @@ import Data.Attoparsec.ByteString.Char8 (Parser, (<?>))
 import qualified Data.Attoparsec.ByteString.Char8 as A
 import Data.Bifunctor (bimap, first)
 import Data.Bits (xor)
+import qualified Data.ByteArray as BA
 import qualified Data.ByteString as BS
 import qualified Data.ByteString.Base64 as B64
 import Data.ByteString.Char8 (ByteString)
@@ -265,12 +272,12 @@ import Simplex.Messaging.Agent.Store.DB (Binary (..), FromField (..), ToField (.
 import qualified Simplex.Messaging.Crypto as C
 import Simplex.Messaging.Encoding
 import Simplex.Messaging.Encoding.String
-import Simplex.Messaging.Names.Record (NameRecord (..))
+import Simplex.Messaging.Names.Record
 import Simplex.Messaging.Parsers
 import Simplex.Messaging.Protocol.Types
 import Simplex.Messaging.Server.QueueStore.QueueInfo
 import Simplex.Messaging.ServiceScheme
-import Simplex.Messaging.SimplexName (SimplexDomain)
+import Simplex.Messaging.SimplexName (LabelHash, SimplexDomain (..), SimplexTLD (..), fullDomainName, labelHash)
 import Simplex.Messaging.Transport
 import Simplex.Messaging.Transport.Client (TransportHost, TransportHosts (..))
 import Simplex.Messaging.Util (bshow, eitherToMaybe, safeDecodeUtf8, (<$?>))
@@ -312,18 +319,13 @@ currentSMPClientVersion = VersionSMPC 4
 supportedSMPClientVRange :: VersionRangeSMPC
 supportedSMPClientVRange = mkVersionRange initialSMPClientVersion currentSMPClientVersion
 
--- TODO v6.0 remove dependency on version
-maxMessageLength :: VersionSMP -> Int
-maxMessageLength v
-  | v >= encryptedBlockSMPVersion = 16048 -- max 16048
-  | v >= sendingProxySMPVersion = 16064 -- max 16067
-  | otherwise = 16088 -- 16048 - always use this size to determine allowed ranges
+maxMessageLength :: Int
+maxMessageLength = 16048 -- max 16048
 
 paddedProxiedTLength :: Int
 paddedProxiedTLength = 16226 -- 16225 .. 16227
 
--- TODO v7.0 change to 16048
-type MaxMessageLen = 16088
+type MaxMessageLen = 16048
 
 -- 16 extra bytes: 8 for timestamp and 8 for flags (7 flags and the space, only 1 flag is currently used)
 type MaxRcvMessageLen = MaxMessageLen + 16 -- 16104, the padded size is 16106
@@ -608,7 +610,7 @@ data Command (p :: Party) where
   -- - corrId: unique correlation ID between proxy and relay, also used as a nonce to encrypt forwarded transmission
   RFWD :: EncFwdTransmission -> Command ProxyService -- use CorrId as CbNonce, proxy to relay
   -- Resolve SimpleX name.
-  RSLV :: SimplexDomain -> Command Resolver
+  RSLV :: NameQuery -> Command Resolver
 
 deriving instance Show (Command p)
 
@@ -744,8 +746,8 @@ data BrokerMsg where
   OK :: BrokerMsg
   ERR :: ErrorType -> BrokerMsg
   PONG :: BrokerMsg
-  -- Resolved SimpleX name.
-  RNAME :: NameRecord -> BrokerMsg
+  -- What the router knows about a SimpleX name.
+  RNAME :: NameResponse -> BrokerMsg
   deriving (Eq, Show)
 
 data RcvMessage = RcvMessage
@@ -1322,7 +1324,10 @@ instance ProtocolTypeI p => FromJSON (ProtocolServer p) where
   parseJSON = strParseJSON "ProtocolServer"
 
 newtype BasicAuth = BasicAuth {unBasicAuth :: ByteString}
-  deriving (Eq, Ord, Show)
+  deriving (Ord, Show)
+
+instance Eq BasicAuth where
+  BasicAuth s == BasicAuth s' = BA.constEq s s'
 
 instance IsString BasicAuth where fromString = BasicAuth . B.pack
 
@@ -1582,7 +1587,7 @@ data ErrorType
     STORE {storeErr :: Text}
   | -- | ACK command is sent without message to be acknowledged
     NO_MSG
-  | -- | sent message is too large (> maxMessageLength = 16088 bytes)
+  | -- | sent message is too large (> maxMessageLength = 16048 bytes)
     LARGE_MSG
   | -- | relay public key is expired
     EXPIRED
@@ -1594,11 +1599,28 @@ data ErrorType
     DUPLICATE_ -- not part of SMP protocol, used internally
   deriving (Eq, Show)
 
+-- | What RSLV asks about: a name, or the hash of a second-level label.
+data NameQuery = NQDomain SimplexDomain | NQHash LabelHash SimplexTLD
+  deriving (Eq, Show)
+
+instance Encoding NameQuery where
+  smpEncode = \case
+    NQDomain d -> encodeUtf8 $ fullDomainName d
+    NQHash h tld -> strEncode h <> strEncode tld
+  smpP = NQHash <$> strP <*> strP <|> NQDomain <$> strP
+
+-- | Hashed from v22, except a name with subnames or a web TLD.
+hashedQuery :: NameQuery -> NameQuery
+hashedQuery q = case q of
+  NQDomain SimplexDomain {nameTLD, domain, subDomain}
+    | null subDomain && nameTLD /= TLDWeb -> NQHash (labelHash domain) nameTLD
+  _ -> q
+
 -- | Name resolution error
 data NameErrorType
   = -- | the names role / resolver is not configured on this server
     NO_RESOLVER
-  | -- | the name is not registered (resolver returned not-found)
+  | -- | the name does not resolve; sent only to a session below v22
     NOT_FOUND
   | -- | backing resolver/RPC failure - contains the diagnostic detail
     RESOLVER {resolverErr :: Text}
@@ -1795,13 +1817,11 @@ instance PartyI p => ProtocolEncoding SMPVersion ErrorType (Command p) where
   type Tag (Command p) = CommandTag p
   encodeProtocol v = \case
     NEW NewQueueReq {rcvAuthKey = rKey, rcvDhKey = dhKey, auth_, subMode, queueReqData, ntfCreds}
-      | v >= newNtfCredsSMPVersion -> new <> e (auth_, subMode, queueReqData, ntfCreds)
-      | v >= shortLinksSMPVersion -> new <> e (auth_, subMode, queueReqData)
-      | v >= sndAuthKeySMPVersion -> new <> e (auth_, subMode, senderCanSecure (queueReqMode <$> queueReqData))
-      | otherwise -> new <> auth <> e subMode
+      | v >= newNtfCredsSMPVersion -> new <> e (subMode, queueReqData, ntfCreds)
+      | v >= shortLinksSMPVersion -> new <> e (subMode, queueReqData)
+      | otherwise -> new <> e (subMode, senderCanSecure (queueReqMode <$> queueReqData))
       where
-        new = e (NEW_, ' ', rKey, dhKey)
-        auth = maybe "" (e . ('A',)) auth_
+        new = e (NEW_, ' ', rKey, dhKey, auth_)
     SUB -> e SUB_
     SUBS n idsHash
       | v >= rcvServiceSMPVersion -> e (SUBS_, ' ', n, idsHash)
@@ -1829,7 +1849,7 @@ instance PartyI p => ProtocolEncoding SMPVersion ErrorType (Command p) where
     PRXY host auth_ -> e (PRXY_, ' ', host, auth_)
     PFWD fwdV pubKey (EncTransmission s) -> e (PFWD_, ' ', fwdV, pubKey, Tail s)
     RFWD (EncFwdTransmission s) -> e (RFWD_, ' ', Tail s)
-    RSLV d -> e (RSLV_, ' ', d)
+    RSLV q -> e (RSLV_, ' ', if v >= nameAvailSMPVersion then hashedQuery q else q)
     where
       e :: Encoding a => a -> ByteString
       e = smpEncode
@@ -1886,21 +1906,19 @@ instance ProtocolEncoding SMPVersion ErrorType Cmd where
     CT SCreator NEW_ -> Cmd SCreator <$> newCmd
       where
         newCmd
-          | v >= newNtfCredsSMPVersion = new smpP smpP smpP
-          | v >= shortLinksSMPVersion = new smpP smpP nothing
-          | v >= sndAuthKeySMPVersion = new smpP (qReq <$> smpP) nothing
-          | otherwise = new auth nothing nothing
+          | v >= newNtfCredsSMPVersion = new smpP smpP
+          | v >= shortLinksSMPVersion = new smpP nothing
+          | otherwise = new (qReq <$> smpP) nothing
           where
             nothing = pure Nothing
-            new p1 p2 p3 = NEW <$> do
+            new p2 p3 = NEW <$> do
               rcvAuthKey <- _smpP
               rcvDhKey <- smpP
-              auth_ <- p1
+              auth_ <- smpP
               subMode <- smpP
               queueReqData <- p2
               ntfCreds <- p3
               pure NewQueueReq {rcvAuthKey, rcvDhKey, auth_, subMode, queueReqData, ntfCreds}
-            auth = optional (A.char 'A' *> smpP)
             qReq sndSecure = Just $ if sndSecure then QRMessaging Nothing else QRContact Nothing
     CT SRecipient tag ->
       Cmd SRecipient <$> case tag of
@@ -1938,7 +1956,7 @@ instance ProtocolEncoding SMPVersion ErrorType Cmd where
     CT SNotifierService NSUBS_
       | v >= rcvServiceSMPVersion -> Cmd SNotifierService <$> (NSUBS <$> _smpP <*> smpP)
       | otherwise -> pure $ Cmd SNotifierService $ NSUBS (-1) mempty
-    CT SResolver RSLV_ -> Cmd SResolver . RSLV <$> _smpP <* A.takeByteString
+    CT SResolver RSLV_ -> Cmd SResolver . RSLV <$> _smpP
 
   fromProtocolError = fromProtocolError @SMPVersion @ErrorType @BrokerMsg
   {-# INLINE fromProtocolError #-}
@@ -1950,11 +1968,10 @@ instance ProtocolEncoding SMPVersion ErrorType BrokerMsg where
   type Tag BrokerMsg = BrokerMsgTag
   encodeProtocol v = \case
     IDS QIK {rcvId, sndId, rcvPublicDhKey = srvDh, queueMode, linkId, serviceId, serverNtfCreds}
-      | v >= newNtfCredsSMPVersion -> ids <> e queueMode <> e linkId <> e serviceId <> e serverNtfCreds
-      | v >= serviceCertsSMPVersion -> ids <> e queueMode <> e linkId <> e serviceId
-      | v >= shortLinksSMPVersion -> ids <> e queueMode <> e linkId
-      | v >= sndAuthKeySMPVersion -> ids <> e (senderCanSecure queueMode)
-      | otherwise -> ids
+      | v >= newNtfCredsSMPVersion -> ids <> e (queueMode, linkId, serviceId, serverNtfCreds)
+      | v >= serviceCertsSMPVersion -> ids <> e (queueMode, linkId, serviceId)
+      | v >= shortLinksSMPVersion -> ids <> e (queueMode, linkId)
+      | otherwise -> ids <> e (senderCanSecure queueMode)
       where
         ids = e (IDS_, ' ', rcvId, sndId, srvDh)
     LNK sId d -> e (LNK_, ' ', sId, d)
@@ -1972,20 +1989,21 @@ instance ProtocolEncoding SMPVersion ErrorType BrokerMsg where
     PRES (EncResponse encBlock) -> e (PRES_, ' ', Tail encBlock)
     END -> e END_
     ENDS n idsHash -> serviceResp ENDS_ n idsHash
-    DELD
-      | v >= deletedEventSMPVersion -> e DELD_
-      | otherwise -> e END_
+    DELD -> e DELD_
     INFO info -> e (INFO_, ' ', info)
     OK -> e OK_
     ERR err -> e (ERR_, ' ', err')
       where
         err' = case err of
           BLOCKED info
-            | v < blockedEntitySMPVersion -> AUTH
             | v < clientNoticesSMPVersion -> BLOCKED info {notice = Nothing}
           _ -> err
     PONG -> e PONG_
-    RNAME rec -> e (RNAME_, ' ', Tail $ LB.toStrict $ J.encode rec)
+    RNAME res
+      | v >= nameAvailSMPVersion -> e (RNAME_, ' ', Tail $ LB.toStrict $ J.encode res)
+      | otherwise -> case registration res of
+          NRRegistered {nameRecord} -> e (RNAME_, ' ', Tail $ LB.toStrict $ J.encode nameRecord)
+          _ -> e (ERR_, ' ', NAME NOT_FOUND)
     where
       e :: Encoding a => a -> ByteString
       e = smpEncode
@@ -2004,8 +2022,7 @@ instance ProtocolEncoding SMPVersion ErrorType BrokerMsg where
       | v >= newNtfCredsSMPVersion -> ids smpP smpP smpP smpP
       | v >= serviceCertsSMPVersion -> ids smpP smpP smpP nothing
       | v >= shortLinksSMPVersion -> ids smpP smpP nothing nothing
-      | v >= sndAuthKeySMPVersion -> ids (qm <$> smpP) nothing nothing nothing
-      | otherwise -> ids nothing nothing nothing nothing
+      | otherwise -> ids (qm <$> smpP) nothing nothing nothing
       where
         qm sndSecure = Just $ if sndSecure then QMMessaging else QMContact
         nothing = pure Nothing
@@ -2033,8 +2050,12 @@ instance ProtocolEncoding SMPVersion ErrorType BrokerMsg where
     OK_ -> pure OK
     ERR_ -> ERR <$> _smpP
     PONG_ -> pure PONG
-    RNAME_ -> fmap RNAME . J.eitherDecodeStrict . unTail <$?> _smpP
+    RNAME_
+      | v >= nameAvailSMPVersion -> fmap RNAME . J.eitherDecodeStrict . unTail <$?> _smpP
+      | otherwise -> fmap (RNAME . oldResponse) . J.eitherDecodeStrict . unTail <$?> _smpP
     where
+      oldResponse nameRecord =
+        NameResponse {lastBlockTs = Nothing, registration = NRRegistered {expires = Nothing, graceUntil = Nothing, reservedReason_ = Nothing, nameRecord}}
       serviceRespP resp
         | v >= rcvServiceSMPVersion = resp <$> _smpP <*> smpP
         | otherwise = resp <$> _smpP <*> pure mempty
@@ -2056,7 +2077,7 @@ instance ProtocolEncoding SMPVersion ErrorType BrokerMsg where
     PKEY {} -> noEntityMsg
     RRES _ -> noEntityMsg
     ALLS -> noEntityMsg
-    RNAME _ -> noEntityMsg
+    RNAME {} -> noEntityMsg
     -- other broker responses must have queue ID
     _
       | B.null entId -> Left $ CMD NO_ENTITY
@@ -2290,19 +2311,8 @@ batchTransmissions params = batchTransmissions' params . L.map (,())
 
 -- | encodes and batches transmissions into blocks
 batchTransmissions' :: forall v p r. THandleParams v p -> NonEmpty (Either TransportError SentRawTransmission, r) -> [TransportBatch r]
-batchTransmissions' THandleParams {batch, blockSize = bSize, serviceAuth} ts
-  | batch = batchTransmissions_ bSize $ L.map (first $ fmap $ tEncodeForBatch serviceAuth) ts
-  | otherwise = map mkBatch1 $ L.toList ts
-  where
-    mkBatch1 :: (Either TransportError SentRawTransmission, r) -> TransportBatch r
-    mkBatch1 (t_, r) = case t_ of
-      Left e -> TBError e r
-      Right t
-        -- 2 bytes are reserved for pad size
-        | B.length s <= bSize - 2 -> TBTransmission s r
-        | otherwise -> TBError TELargeMsg r
-        where
-          s = tEncode serviceAuth t
+batchTransmissions' THandleParams {blockSize, serviceAuth} ts =
+  batchTransmissions_ blockSize $ L.map (first $ fmap $ tEncodeForBatch serviceAuth) ts
 
 -- | Pack encoded transmissions into batches
 batchTransmissions_ :: Int -> NonEmpty (Either TransportError ByteString, r) -> [TransportBatch r]
@@ -2366,9 +2376,8 @@ tGetParse th@THandle {params} = eitherList (tParse params) <$> tGetBlock th
 {-# INLINE tGetParse #-}
 
 tParse :: THandleParams v p -> ByteString -> NonEmpty (Either TransportError RawTransmission)
-tParse thParams@THandleParams {batch} s
-  | batch = eitherList (L.map (\(Large t) -> tParse1 t)) ts
-  | otherwise = [tParse1 s]
+tParse thParams s =
+  eitherList (L.map (tParse1 . unLarge)) ts
   where
     tParse1 = parse (transmissionP thParams) TEBadBlock
     ts = parse smpP TEBadBlock s
@@ -2402,7 +2411,9 @@ tDecodeServer THandleParams {sessionId, thVersion = v, implySessId} = \case
           where
             cmdOrErr = parseProtocol @v @err @cmd v command >>= checkCredentials tAuth entityId
             t :: a -> (CorrId, EntityId, a)
-            t = (corrId,entityId,)
+            -- IDs are slices of the ~16 KB received block and are kept as subscription keys,
+            -- so without a copy one live key retains the whole block
+            t = (CorrId $ B.copy $ bs corrId,EntityId $ B.copy $ unEntityId entityId,)
         Left _ -> tError corrId PEBlock
     | otherwise -> tError corrId PESession
   Left _ -> tError "" PEBlock
@@ -2438,3 +2449,4 @@ $(J.deriveJSON defaultJSON ''BlockingInfo)
 
 -- run deriveJSON in one TH splice to allow mutual instance
 $(concat <$> mapM @[] (J.deriveJSON (sumTypeJSON id)) [''ProxyError, ''NameErrorType, ''ErrorType])
+

src/Simplex/Messaging/Server.hs

@@ -56,6 +56,7 @@ import Control.Monad.Trans.Except
 import Control.Monad.STM (retry)
 import Crypto.Random (ChaChaDRG)
 import Data.Bifunctor (first, second)
+import qualified Data.Aeson as J
 import Data.ByteString.Base64 (encode)
 import qualified Data.ByteString.Builder as BLD
 import Data.ByteString.Char8 (ByteString)
@@ -75,7 +76,7 @@ import Data.List.NonEmpty (NonEmpty (..), (<|))
 import qualified Data.List.NonEmpty as L
 import Data.Map.Strict (Map)
 import qualified Data.Map.Strict as M
-import Data.Maybe (fromMaybe, isJust, isNothing)
+import Data.Maybe (fromMaybe, isJust, isNothing, listToMaybe)
 import Data.Semigroup (Sum (..))
 import qualified Data.Set as S
 import Data.Text (Text)
@@ -97,13 +98,12 @@ import Network.Socket (ServiceName, Socket, socketToHandle)
 import qualified Network.TLS as TLS
 import Numeric.Natural (Natural)
 import Simplex.Messaging.Agent.Lock
-import Simplex.Messaging.Client (ProtocolClient (thParams), ProtocolClientError (..), SMPClient, SMPClientError, clientHandlers, forwardSMPTransmission, smpProxyError, temporaryClientError)
-import Simplex.Messaging.Client.Agent (OwnServer, SMPClientAgent (..), SMPClientAgentEvent (..), closeSMPClientAgent, getSMPServerClient'', isOwnServer, lookupSMPServerClient, getConnectedSMPServerClient)
+import Simplex.Messaging.Client (NetworkConfig (..), ProtocolClient (thParams), ProtocolClientConfig (..), ProtocolClientError (..), SMPClient, SMPClientError, clientHandlers, closeTimedOutClient, forwardSMPTransmission, smpProxyError, temporaryClientError, transportHost')
+import Simplex.Messaging.Client.Agent (AgentLeakStats (..), OwnServer, SMPClientAgent (..), SMPClientAgentConfig (..), SMPClientAgentEvent (..), closeSMPClientAgent, getAgentLeakStats, getSMPServerClient'', isOwnServer, lookupSMPServerClient, getConnectedSMPServerClient)
 import qualified Simplex.Messaging.Crypto as C
 import Simplex.Messaging.Encoding
 import Simplex.Messaging.Encoding.String
 import Simplex.Messaging.Protocol
-import Simplex.Messaging.SimplexName (SimplexDomain)
 import Simplex.Messaging.Server.Control
 import Simplex.Messaging.Server.Env.STM as Env
 import Simplex.Messaging.Server.Expiration
@@ -128,6 +128,7 @@ import Simplex.Messaging.Transport.Server
 import Simplex.Messaging.Util
 import Simplex.Messaging.Version
 import System.Environment (lookupEnv)
+import Text.Read (readMaybe)
 import System.Exit (exitFailure, exitSuccess)
 import System.IO (hPrint, hPutStrLn, hSetNewlineMode, universalNewlineMode)
 import System.Mem.Weak (deRefWeak)
@@ -174,6 +175,23 @@ data ClientSubAction
 
 type PrevClientSub s = (Client s, ClientSubAction, (EntityId, BrokerMsg))
 
+-- accumulator for per-client leak diagnostics (summed across all connected clients)
+data ClientAgg = ClientAgg
+  { aggEndThreads :: !Int, -- Weak ThreadId registrations in endThreads (forkClient leak)
+    aggEndThreadSeq :: !Int, -- total forkClient forks ever (fork rate)
+    aggProcThreads :: !Int,
+    aggSubs :: !Int, -- entries in per-client subscriptions map
+    aggNoSub :: !Int,
+    aggPending :: !Int, -- SubPending: forked delivery threads not yet completed (blocked-thread candidates)
+    aggThread :: !Int, -- SubThread: live delivery threads
+    aggProhibit :: !Int,
+    aggNtfSubs :: !Int,
+    aggSvcSubsCount :: !Int,
+    aggRcvQ :: !Int,
+    aggSndQ :: !Int, -- full sndQ => forkDeliver blocks (leak trigger)
+    aggMsgQ :: !Int
+  }
+
 smpServer :: forall s. MsgStoreClass s => TMVar Bool -> ServerConfig s -> Maybe AttachHTTP -> M s ()
 smpServer started cfg@ServerConfig {transports, transportConfig = tCfg, startOptions} attachHTTP_ = do
   s <- asks server
@@ -191,6 +209,7 @@ smpServer started cfg@ServerConfig {transports, transportConfig = tCfg, startOpt
     ( serverThread "server subscribers" s subscribers subscriptions serviceSubsCount (Just cancelSub)
         : serverThread "server ntfSubscribers" s ntfSubscribers ntfSubscriptions ntfServiceSubsCount Nothing
         : deliverNtfsThread s
+        : leakDiagnosticsThread s
         : sendPendingEvtsThread s
         : receiveFromProxyAgent pa
         : expireNtfsThread cfg
@@ -369,7 +388,7 @@ smpServer started cfg@ServerConfig {transports, transportConfig = tCfg, startOpt
         runDeliverNtfs ms ns' stats
       where
         runDeliverNtfs :: s -> NtfStore -> ServerStats -> IO ()
-        runDeliverNtfs ms (NtfStore ns) stats = do
+        runDeliverNtfs ms ns'@(NtfStore ns) stats = do
           ntfs <- M.assocs <$> readTVarIO ns
           unless (null ntfs) $
             getQueueNtfServices @(StoreQueue s) (queueStore ms) ntfs >>= \case
@@ -381,6 +400,7 @@ smpServer started cfg@ServerConfig {transports, transportConfig = tCfg, startOpt
                     cIds <- IS.toList <$> readTVarIO subClients
                     forM_ cIds $ \cId -> getServerClient cId srv >>= mapM_ (deliverQueueNtfs ntfs')
                 atomically $ modifyTVar' ns (`M.withoutKeys` S.fromList (map fst deleted))
+                deleteEmptyNtfs ns' ntfs
           where
             deliverQueueNtfs ntfs' c@Client {ntfSubscriptions} =
               whenM (currentClient readTVarIO c) $ do
@@ -496,6 +516,101 @@ smpServer started cfg@ServerConfig {transports, transportConfig = tCfg, startOpt
               printMessageStats "STORE: messages" msgStats
             Left e -> logError $ "STORE: expireOldMessages, error expiring messages, " <> tshow e
 
+    -- Periodic comprehensive leak diagnostics: sizes of every growable structure in the
+    -- server, summed across clients, plus the proxy agent. Interval seconds via SMP_LEAKDIAG_SEC
+    -- (default 60). A single greppable "LEAKDIAG ..." line per interval; whichever counter grows
+    -- monotonically over time is the leak.
+    leakDiagnosticsThread :: Server s -> M s ()
+    leakDiagnosticsThread srv = do
+      secStr <- liftIO $ lookupEnv "SMP_LEAKDIAG_SEC"
+      let sec = max 5 $ fromMaybe 60 (secStr >>= readMaybe)
+      ms <- asks msgStore
+      ns <- asks ntfStore
+      ProxyAgent {smpAgent} <- asks proxyAgent
+      -- listening port identifies the server when several run in one process (bench topologies)
+      let srvPort = maybe "?" (\(p, _, _) -> p) $ listToMaybe transports
+      labelMyThread "leakDiagnosticsThread"
+      -- never let a diagnostics error crash the server (this thread is in raceAny_)
+      liftIO $ forever $ do
+        threadDelay $ sec * 1000000
+        tryAny (logLeakStats srvPort srv ms ns smpAgent) >>= either (logError . ("LEAKDIAG error: " <>) . tshow) (const $ pure ())
+
+    logLeakStats :: ServiceName -> Server s -> s -> NtfStore -> SMPClientAgent 'Sender -> IO ()
+    logLeakStats srvPort srv ms (NtfStore nsv) smpAgent = do
+#if MIN_VERSION_base(4,18,0)
+      nThreads <- length <$> listThreads
+#else
+      let nThreads = 0 :: Int
+#endif
+      cls <- IM.elems <$> getServerClients srv
+      cc <- foldM accClient emptyAgg cls
+      (smpQ, smpS, smpC, smpT, smpP) <- subAgg (subscribers srv)
+      (ntfQ, ntfS, ntfC, ntfT, ntfP) <- subAgg (ntfSubscribers srv)
+      ntfMap <- readTVarIO nsv
+      ntfMsgs <- foldM (\ !a v -> (a +) . length <$> readTVarIO v) (0 :: Int) (M.elems ntfMap)
+      EntityCounts {queueCount, notifierCount, rcvServiceCount, ntfServiceCount, rcvServiceQueuesCount, ntfServiceQueuesCount} <- getEntityCounts @(StoreQueue s) (queueStore ms)
+      LoadedQueueCounts {loadedQueueCount, loadedNotifierCount, openJournalCount, queueLockCount, notifierLockCount} <- loadedQueueCounts ms
+      AgentLeakStats {alSmpClients, alSmpSessions, alActiveServiceSubs, alActiveQueueSubs, alPendingServiceSubs, alPendingQueueSubs, alSmpSubWorkers, alSentCommands, alMsgQ} <- getAgentLeakStats smpAgent
+      logNote $
+        T.concat
+          [ "LEAKDIAG",
+            " srv=" <> T.pack srvPort,
+            f "threads" nThreads, f "clients" (length cls),
+            f "endThreads" (aggEndThreads cc), f "endThreadSeq" (aggEndThreadSeq cc), f "procThreads" (aggProcThreads cc),
+            f "subs" (aggSubs cc), f "subs_nosub" (aggNoSub cc), f "subs_pending" (aggPending cc), f "subs_thread" (aggThread cc), f "subs_prohibit" (aggProhibit cc),
+            f "ntfSubsClient" (aggNtfSubs cc), f "svcSubsCount" (aggSvcSubsCount cc),
+            f "rcvQ" (aggRcvQ cc), f "sndQ" (aggSndQ cc), f "msgQ" (aggMsgQ cc),
+            f "smp_qSubscribers" smpQ, f "smp_svcSubscribers" smpS, f "smp_subClients" smpC, f "smp_totalSvcSubs" smpT, f "smp_pendingEvents" smpP,
+            f "ntf_qSubscribers" ntfQ, f "ntf_svcSubscribers" ntfS, f "ntf_subClients" ntfC, f "ntf_totalSvcSubs" ntfT, f "ntf_pendingEvents" ntfP,
+            f "ntfStore_keys" (M.size ntfMap), f "ntfStore_msgs" ntfMsgs,
+            f "store_queues" queueCount, f "store_notifiers" notifierCount, f "store_rcvServices" rcvServiceCount, f "store_ntfServices" ntfServiceCount, f "store_rcvSvcQueues" rcvServiceQueuesCount, f "store_ntfSvcQueues" ntfServiceQueuesCount,
+            f "loaded_queues" loadedQueueCount, f "loaded_notifiers" loadedNotifierCount, f "open_journals" openJournalCount, f "queue_locks" queueLockCount, f "notifier_locks" notifierLockCount,
+            f "proxy_smpClients" alSmpClients, f "proxy_smpSessions" alSmpSessions, f "proxy_activeSvcSubs" alActiveServiceSubs, f "proxy_activeQSubs" alActiveQueueSubs, f "proxy_pendingSvcSubs" alPendingServiceSubs, f "proxy_pendingQSubs" alPendingQueueSubs, f "proxy_subWorkers" alSmpSubWorkers, f "proxy_sentCommands" alSentCommands, f "proxy_msgQ" alMsgQ
+          ]
+      where
+        f :: Show a => Text -> a -> Text
+        f k v = " " <> k <> "=" <> tshow v
+        emptyAgg = ClientAgg 0 0 0 0 0 0 0 0 0 0 0 0 0
+        accClient !agg Client {subscriptions, ntfSubscriptions, serviceSubsCount, procThreads, endThreads, endThreadSeq, rcvQ, sndQ, msgQ} = do
+          et <- IM.size <$> readTVarIO endThreads
+          es <- readTVarIO endThreadSeq
+          pt <- readTVarIO procThreads
+          subs <- readTVarIO subscriptions
+          (no, pe, th, pr) <- foldM accSub (0, 0, 0, 0) (M.elems subs)
+          nt <- M.size <$> readTVarIO ntfSubscriptions
+          sc <- fst <$> readTVarIO serviceSubsCount
+          (rl, sl, ml) <- atomically $ (,,) <$> lengthTBQueue rcvQ <*> lengthTBQueue sndQ <*> lengthTBQueue msgQ
+          pure
+            agg
+              { aggEndThreads = aggEndThreads agg + et,
+                aggEndThreadSeq = aggEndThreadSeq agg + es,
+                aggProcThreads = aggProcThreads agg + pt,
+                aggSubs = aggSubs agg + M.size subs,
+                aggNoSub = aggNoSub agg + no,
+                aggPending = aggPending agg + pe,
+                aggThread = aggThread agg + th,
+                aggProhibit = aggProhibit agg + pr,
+                aggNtfSubs = aggNtfSubs agg + nt,
+                aggSvcSubsCount = aggSvcSubsCount agg + fromIntegral sc,
+                aggRcvQ = aggRcvQ agg + fromIntegral rl,
+                aggSndQ = aggSndQ agg + fromIntegral sl,
+                aggMsgQ = aggMsgQ agg + fromIntegral ml
+              }
+        accSub (no, pe, th, pr) Sub {subThread} = case subThread of
+          ServerSub t ->
+            readTVarIO t >>= \case
+              NoSub -> pure (no + 1, pe, th, pr)
+              SubPending -> pure (no, pe + 1, th, pr)
+              SubThread _ -> pure (no, pe, th + 1, pr)
+          ProhibitSub -> pure (no, pe, th, pr + 1)
+        subAgg ServerSubscribers {queueSubscribers, serviceSubscribers, subClients, totalServiceSubs, pendingEvents} = do
+          q <- M.size <$> getSubscribedClients queueSubscribers
+          s' <- M.size <$> getSubscribedClients serviceSubscribers
+          sc <- IS.size <$> readTVarIO subClients
+          ts <- fst <$> readTVarIO totalServiceSubs
+          pe <- IM.size <$> readTVarIO pendingEvents
+          pure (q, s', sc, ts, pe)
+
     expireNtfsThread :: ServerConfig s -> M s ()
     expireNtfsThread ServerConfig {notificationExpiration = expCfg} = do
       ns <- asks ntfStore
@@ -739,9 +854,10 @@ smpServer started cfg@ServerConfig {transports, transportConfig = tCfg, startOpt
       idSize <- asks $ queueIdBytes . config
       kh <- asks serverIdentity
       ks <- atomically . C.generateKeyPair =<< asks random
-      ServerConfig {smpServerVRange, smpHandshakeTimeout} <- asks config
+      ServerConfig {smpServerVRange, smpHandshakeTimeout, information} <- asks config
+      let serverInfo = LB.toStrict . J.encode <$> information
       labelMyThread $ "smp handshake for " <> transportName tp
-      liftIO (timeout smpHandshakeTimeout . runExceptT $ smpServerHandshake srvCert srvSignKey h ks kh smpServerVRange $ getClientService ms g idSize) >>= \case
+      liftIO (timeout smpHandshakeTimeout . runExceptT $ smpServerHandshake srvCert srvSignKey h ks kh smpServerVRange serverInfo $ getClientService ms g idSize) >>= \case
         Just (Right th) -> runClientTransport th
         _ -> pure ()
 
@@ -1091,6 +1207,7 @@ controlPortAuth h user admin role auth = do
   readTVarIO role >>= \case
     CPRNone -> do
       atomically $ writeTVar role $! newRole
+      when (newRole == CPRNone) $ logWarn "ControlPort: failed auth"
       hPutStrLn h $ currentRole newRole
     r -> hPutStrLn h $ currentRole r <> if r == newRole then "" else ", start new session to change."
   where
@@ -1377,11 +1494,10 @@ client
   ms
   clnt@Client {clientId, rcvQ, sndQ, msgQ, clientTHParams = thParams'@THandleParams {sessionId}, procThreads} = do
     labelMyThread . B.unpack $ "client $" <> encode sessionId <> " commands"
-    let THandleParams {thVersion} = thParams'
-        clntServiceId = (\THClientService {serviceId} -> serviceId) <$> (peerClientService =<< thAuth thParams')
+    let clntServiceId = (\THClientService {serviceId} -> serviceId) <$> (peerClientService =<< thAuth thParams')
         process batchSubs t acc@(rs, msgs) =
           (maybe acc (\(!r, !msg_) -> (r : rs, maybe msgs (: msgs) msg_)))
-            <$> processCommand clntServiceId thVersion batchSubs t
+            <$> processCommand clntServiceId batchSubs t
     forever $ do
       batch <- atomically (readTBQueue rcvQ)
       batchSubs <- prepareBatchSubs clntServiceId batch
@@ -1444,11 +1560,11 @@ client
                 pure . ERR $ smpProxyError e
             where
               proxyResp smp =
-                let THandleParams {sessionId = srvSessId, thVersion, thServerVRange, thAuth} = thParams smp
+                let THandleParams {sessionId = srvSessId, thServerVRange, thAuth} = thParams smp
                   in case compatibleVRange thServerVRange proxiedSMPRelayVRange of
                       -- Cap the destination relay version range to prevent client version fingerprinting.
                       -- See comment for proxiedSMPRelayVersion.
-                      Just (Compatible vr) | thVersion >= sendingProxySMPVersion -> case thAuth of
+                      Just (Compatible vr) -> case thAuth of
                         Just THAuthClient {peerServerCertKey} -> PKEY srvSessId vr peerServerCertKey
                         Nothing -> ERR $ transportErr TENoServerAuth
                       _ -> ERR $ transportErr TEVersion
@@ -1459,16 +1575,17 @@ client
         liftIO (lookupSMPServerClient a sessId) >>= \case
           Just (own, smp) -> do
             inc own pRequests
-            if v >= sendingProxySMPVersion
-              then forkProxiedCmd $ do
-                liftIO (runExceptT (forwardSMPTransmission smp corrId fwdV pubKey encBlock) `E.catches` clientHandlers)  >>= \case
-                  Right r -> PRES r <$ inc own pSuccesses
-                  Left e -> ERR (smpProxyError e) <$ case e of
-                    PCEProtocolError {} -> inc own pSuccesses
-                    _ -> inc own pErrorsOther
-              else Just (ERR $ transportErr TEVersion) <$ inc own pErrorsCompat
-            where
-              THandleParams {thVersion = v} = thParams smp
+            forkProxiedCmd $ do
+              liftIO (runExceptT (forwardSMPTransmission smp corrId fwdV pubKey encBlock) `E.catches` clientHandlers)  >>= \case
+                Right r -> PRES r <$ inc own pSuccesses
+                Left e -> ERR (smpProxyError e) <$ case e of
+                  PCEProtocolError {} -> inc own pSuccesses
+                  _ -> do
+                    logWarn $ "Error forwarding to relay: " <> decodeLatin1 (strEncode $ transportHost' smp) <> " own=" <> tshow own <> " " <> tshow e
+                    inc own pErrorsOther
+                    case e of
+                      PCEResponseTimeout -> liftIO $ closeTimedOutClient (smpPingCount $ networkConfig $ smpCfg $ agentCfg a) smp
+                      _ -> pure ()
           Nothing -> inc False pRequests >> inc False pErrorsConnect $> Just (ERR $ PROXY NO_SESSION)
       where
         forkProxiedCmd :: M s BrokerMsg -> M s (Maybe BrokerMsg)
@@ -1476,11 +1593,15 @@ client
     -- Run a slow command on a thread
     forkCmd :: (ServerConfig s -> Int) -> CorrId -> EntityId -> M s BrokerMsg -> M s (Maybe a)
     forkCmd concurrency corrId entId cmdAction = do
-      bracket_ wait signal . forkClient clnt (B.unpack $ "client $" <> encode sessionId <> " cmd") $
-        -- commands MUST be processed under a reasonable timeout or the client would halt
-        cmdAction >>= \t -> atomically $ writeTBQueue sndQ ([(corrId, entId, t)], [])
+      -- the forked thread releases the slot when the command completes, the caller only if the fork failed
+      mask $ \restore -> do
+        wait
+        forkClient clnt (B.unpack $ "client $" <> encode sessionId <> " cmd") (restore cmd `finally` signal)
+          `onException` signal
       pure Nothing
       where
+        -- commands MUST be processed under a reasonable timeout or the client would halt
+        cmd = cmdAction >>= \t -> atomically $ writeTBQueue sndQ ([(corrId, entId, t)], [])
         wait = do
           limit <- asks (concurrency . config)
           atomically $ do
@@ -1496,24 +1617,28 @@ client
         Nothing -> incStat (rslvDisabled st) $> Nothing
         Just nenv -> pure (Just nenv)
     -- Runs on a forked thread so RSLV does not block other commands;
-    -- concurrency is limited by serverResolverConcurrency in forkCmd.
-    resolveNameMsg :: NamesEnv -> SimplexDomain -> M s BrokerMsg
-    resolveNameMsg nenv d = do
+    -- concurrency is limited per connection by serverResolverConcurrency in forkCmd and globally in resolveName.
+    resolveNameMsg :: VersionSMP -> NamesEnv -> NameQuery -> M s BrokerMsg
+    resolveNameMsg v nenv q = do
       st <- asks (rslvStats . serverStats)
       (selector, msg) <-
-        liftIO (resolveName nenv d) <&> \case
-          Right rec -> (rslvSucc, RNAME rec)
-          Left e@NOT_FOUND -> (rslvNotFound, ERR $ NAME e)
+        liftIO (resolveName nenv q) <&> \case
+          Right res -> (if answered (registration res) then rslvSucc else rslvNotFound, RNAME res)
           Left e -> (rslvResolverErrs, ERR $ NAME e)
       incStat (selector st) $> msg
+      where
+        -- below v22 the encoder answers anything but a record as NAME NOT_FOUND
+        answered = \case
+          NRRegistered {} -> True
+          _ -> v >= nameAvailSMPVersion
     transportErr :: TransportError -> ErrorType
     transportErr = PROXY . BROKER . TRANSPORT
     mkIncProxyStats :: MonadIO m => ProxyStats -> ProxyStats -> OwnServer -> (ProxyStats -> IORef Int) -> m ()
     mkIncProxyStats ps psOwn own sel = do
       incStat $ sel ps
       when own $ incStat $ sel psOwn
-    processCommand :: Maybe ServiceId -> VersionSMP -> Either ErrorType (Map RecipientId Message, Map RecipientId (Either ErrorType ()), Map RecipientId (Either ErrorType ())) -> VerifiedTransmission s -> M s (Maybe ResponseAndMessage)
-    processCommand clntServiceId clntVersion batchSubs (q_, (corrId, entId, cmd)) = case cmd of
+    processCommand :: Maybe ServiceId -> Either ErrorType (Map RecipientId Message, Map RecipientId (Either ErrorType ()), Map RecipientId (Either ErrorType ())) -> VerifiedTransmission s -> M s (Maybe ResponseAndMessage)
+    processCommand clntServiceId batchSubs (q_, (corrId, entId, cmd)) = case cmd of
       Cmd SProxiedClient command -> processProxiedCmd (corrId, entId, command)
       Cmd SSender command -> case command of
         SKEY k -> withQueue $ \q qr -> checkMode QMMessaging qr $ secureQueue_ q k
@@ -1522,7 +1647,7 @@ client
       Cmd SProxyService (RFWD encBlock) -> (response . (corrId, NoEntity,) =<<) <$> processForwardedCommand encBlock
       Cmd SResolver (RSLV d) -> rslvNamesEnv >>= \case
         Nothing -> pure $ response (corrId, NoEntity, ERR (NAME NO_RESOLVER))
-        Just nenv -> forkCmd serverResolverConcurrency corrId NoEntity (resolveNameMsg nenv d)
+        Just nenv -> forkCmd serverResolverConcurrency corrId NoEntity (resolveNameMsg (thVersion thParams') nenv d)
       Cmd SSenderLink command -> case command of
         LKEY k -> withQueue $ \q qr -> checkMode QMMessaging qr $ secureQueue_ q k $>> getQueueLink_ q qr
         LGET -> withQueue $ \q qr -> checkContact qr $ getQueueLink_ q qr
@@ -1980,7 +2105,7 @@ client
 
         sendMessage :: MsgFlags -> MsgBody -> StoreQueue s -> QueueRec -> M s (Transmission BrokerMsg)
         sendMessage msgFlags msgBody q qr
-          | B.length msgBody > maxMessageLength clntVersion = do
+          | B.length msgBody > maxMessageLength = do
               stats <- asks serverStats
               incStat $ msgSentLarge stats
               pure $ err LARGE_MSG
@@ -2127,6 +2252,7 @@ client
           let proxyNonce = C.cbNonce $ bs corrId
           s' <- liftEitherWith (const CRYPTO) $ C.cbDecryptNoPad sessSecret proxyNonce s
           FwdTransmission {fwdCorrId, fwdVersion, fwdKey, fwdTransmission = EncTransmission et} <- liftEitherWith (const $ CMD SYNTAX) $ smpDecode s'
+          unless (fwdVersion `isCompatible` thServerVRange thParams') $ throwE $ transportErr TEVersion
           let clientSecret = C.dh' fwdKey serverPrivKey
               clientNonce = C.cbNonce $ bs fwdCorrId
           b <- liftEitherWith (const CRYPTO) $ C.cbDecrypt clientSecret clientNonce et
@@ -2135,7 +2261,7 @@ client
           t' <- case tParse clntTHParams b of
             t :| [] -> pure $ tDecodeServer clntTHParams t
             _ -> throwE BLOCK
-          let clntThAuth = Just $ THAuthServer {serverPrivKey, peerClientService = Nothing, sessSecret' = Just clientSecret}
+          let clntThAuth = Just $ THAuthServer {serverPrivKey, peerClientService = Nothing, peerEntitlement = Nothing, sessSecret' = Just clientSecret}
               encodeResp r = do
                 r' <- case batchTransmissions clntTHParams [Right (Nothing, encodeTransmission clntTHParams r)] of
                   [] -> throwE INTERNAL -- at least 1 item is guaranteed from NonEmpty/Right
@@ -2153,11 +2279,11 @@ client
               Cmd SResolver (RSLV d) -> lift $ rslvNamesEnv >>= \case
                 Nothing -> pure $ Just (corrId', entId', ERR (NAME NO_RESOLVER))
                 Just nenv -> forkCmd serverResolverConcurrency corrId NoEntity $ do
-                  msg <- resolveNameMsg nenv d
+                  msg <- resolveNameMsg (thVersion clntTHParams) nenv d
                   either ERR id <$> runExceptT (encodeResp (corrId', entId', msg))
               -- INTERNAL because processCommand never returns Nothing for sender commands;
               -- `fst` drops the empty message only returned for SUB.
-              _ -> Just . maybe (corrId', entId', ERR INTERNAL) fst <$> lift (processCommand Nothing fwdVersion (Right (M.empty, M.empty, M.empty)) t'')
+              _ -> Just . maybe (corrId', entId', ERR INTERNAL) fst <$> lift (processCommand Nothing (Right (M.empty, M.empty, M.empty)) t'')
           stats <- asks serverStats
           incStat $ pMsgFwdsRecv stats
           traverse encodeResp r_

src/Simplex/Messaging/Server/Information.hs

@@ -26,7 +26,6 @@ import qualified Data.Attoparsec.ByteString.Char8 as A
 import Data.Int (Int64)
 import Data.Maybe (isJust)
 import Data.Text (Text)
-import Simplex.Messaging.Agent.Protocol (ConnectionLink, ConnectionMode (..))
 import Simplex.Messaging.Encoding.String
 import Simplex.Messaging.Parsers (defaultJSON, dropPrefix, enumJSON)
 
@@ -112,7 +111,7 @@ data Entity = Entity {name :: Text, country :: Maybe Text}
   deriving (Show)
 
 data ServerContactAddress = ServerContactAddress
-  { simplex :: Maybe (ConnectionLink 'CMContact),
+  { simplex :: Maybe Text,
     email :: Maybe Text, -- it is recommended that it matches DNS email address, if either is present
     pgp :: Maybe PGPKey
   }

src/Simplex/Messaging/Server/Main.hs

@@ -59,7 +59,7 @@ import qualified Data.Text as T
 import Data.Text.Encoding (decodeLatin1, encodeUtf8)
 import qualified Data.Text.IO as T
 import Options.Applicative
-import Simplex.Messaging.Agent.Protocol (ConnectionLink (..), connReqUriP')
+import Simplex.Messaging.Agent.Protocol (ConnectionLink (..), ConnectionMode (..), connReqUriP')
 import Simplex.Messaging.Agent.Store.Postgres.Options (DBOpts (..))
 import Simplex.Messaging.Agent.Store.Shared (MigrationConfirmation (..))
 import Simplex.Messaging.Client (HostMode (..), NetworkConfig (..), ProtocolClientConfig (..), SMPWebPortServers (..), SocksMode (..), defaultNetworkConfig, textToHostMode)
@@ -85,7 +85,7 @@ import Simplex.Messaging.Transport (supportedProxyClientSMPRelayVRange, alpnSupp
 import Simplex.Messaging.Transport.Client (TransportHost (..), defaultSocksProxy)
 import Simplex.Messaging.Transport.HTTP2 (httpALPN)
 import Simplex.Messaging.Transport.Server (ServerCredentials (..), mkTransportServerConfig)
-import Simplex.Messaging.Util (eitherToMaybe, ifM)
+import Simplex.Messaging.Util (eitherToMaybe, ifM, safeDecodeUtf8)
 import System.Directory (createDirectoryIfMissing, doesDirectoryExist, doesFileExist)
 import System.Exit (exitFailure)
 import System.FilePath (combine)
@@ -604,6 +604,7 @@ smpServerCLI_ generateSite serveStaticFiles attachStaticFiles cfgPath logPath =
                               }
                         },
                     ownServerDomains = either (const []) textToOwnServers $ lookupValue "PROXY" "own_server_domains" ini,
+                    msgQSize = Nothing, -- to prevent accumulation of late responses, and deadlocks in SMP proxy
                     persistErrorInterval = 30 -- seconds
                   },
               allowSMPProxy = True,
@@ -786,12 +787,13 @@ serverPublicInfo ini = serverInfo <$!> infoValue "source_code"
         <$!> infoValue nameField
     countryValue field = (either error id . validCountryValue (T.unpack field) . T.unpack) <$!> infoValue field
     iniContacts simplexField emailField pgpKeyUriField pgpKeyFingerprintField =
-      let simplex = either error id . parseAll linkP . encodeUtf8 <$!> eitherToMaybe (lookupValue "INFORMATION" simplexField ini)
+      let addr :: Maybe (ConnectionLink 'CMContact) = either error id . parseAll linkP . encodeUtf8 <$!> eitherToMaybe (lookupValue "INFORMATION" simplexField ini)
+          simplex = safeDecodeUtf8 . strEncode <$> addr
           linkP = CLFull <$> connReqUriP' Nothing <|> CLShort <$> strP
           email = infoValue emailField
           pkURI_ = infoValue pgpKeyUriField
           pkFingerprint_ = infoValue pgpKeyFingerprintField
-       in case (simplex, email, pkURI_, pkFingerprint_) of
+       in case (addr, email, pkURI_, pkFingerprint_) of
             (Nothing, Nothing, Nothing, _) -> Nothing
             (Nothing, Nothing, _, Nothing) -> Nothing
             (_, _, pkURI, pkFingerprint) -> Just ServerContactAddress {simplex, email, pgp = PGPKey <$> pkURI <*> pkFingerprint}
@@ -812,7 +814,8 @@ readNamesConfig ini
               { resolverEndpoint = either (error . ("[NAMES] resolver_endpoint: " <>)) id (validateUrl endpoint resolverAuth_),
                 resolverAuth = resolverAuth_,
                 resolverTimeoutMs = boundedIniInt 3000 100 60000 "resolver_timeout_ms",
-                resolverMaxResponseBytes = boundedIniInt 16000 1024 16000 "resolver_max_response_bytes"
+                resolverMaxResponseBytes = boundedIniInt 16000 1024 16000 "resolver_max_response_bytes",
+                resolverGlobalConcurrency = boundedIniInt 32 1 1000 "resolver_global_concurrency"
               }
   where
     enabled = fromMaybe False (iniOnOff "NAMES" "enable" ini)

src/Simplex/Messaging/Server/Main/GitCommit.hs

@@ -6,14 +6,25 @@ module Simplex.Messaging.Server.Main.GitCommit
   ) where
 
 import Language.Haskell.TH
+import Language.Haskell.TH.Syntax (addDependentFile)
 import System.Process
 import Control.Exception
+import Control.Monad (filterM)
+import System.Directory (doesFileExist)
 import System.Exit
+import System.FilePath ((</>))
 
 gitCommit :: Q Exp
-gitCommit = stringE . commit =<< runIO (try $ readProcessWithExitCode "git" ["rev-parse", "HEAD"] "")
+gitCommit =
+  runIO gitHead >>= \case
+    Right (ExitSuccess, out, _)
+      | [commit, gitDir] <- lines out -> do
+          -- without these dependencies GHC recompilation check would not know that the commit changed.
+          -- HEAD reflog is updated on any HEAD change and, unlike branch ref files, git gc cannot remove it
+          -- mid-build; it is absent when reflogs are disabled, and addDependentFile fails on a missing file
+          mapM_ addDependentFile =<< runIO (filterM doesFileExist [gitDir </> "HEAD", gitDir </> "logs" </> "HEAD"])
+          stringE commit
+    _ -> stringE ""
   where
-    commit :: Either SomeException (ExitCode, String, String) -> String
-    commit = \case
-      Right (ExitSuccess, out, _) -> take 40 out
-      _ -> ""
+    gitHead :: IO (Either SomeException (ExitCode, String, String))
+    gitHead = try $ readProcessWithExitCode "git" ["rev-parse", "HEAD", "--git-dir"] ""

src/Simplex/Messaging/Server/Main/Init.hs

@@ -167,6 +167,8 @@ iniFileContent cfgPath logPath opts host basicAuth controlPortPwds =
         \# resolver_auth: basic <username>:<password>\n\
         \# resolver_timeout_ms: 3000\n\
         \# resolver_max_response_bytes: 16000\n\
+        \# Max concurrent requests to the resolver from all connections.\n\
+        \# resolver_global_concurrency: 32\n\
         \# Max concurrent name resolutions per connection (forwarded RSLVs from many\n\
         \# clients share one proxy connection, so this is much higher than PROXY client_concurrency).\n"
     <> ("# resolver_concurrency = " <> tshow defaultNameResolverConcurrency)

src/Simplex/Messaging/Server/MsgStore/Postgres.hs

@@ -110,10 +110,9 @@ instance MsgStoreClass PostgresMsgStore where
   expireOldMessages :: Bool -> PostgresMsgStore -> Int64 -> Int64 -> IO MessageStats
   expireOldMessages _tty ms now ttl =
     maybeFirstRow' newMessageStats toMessageStats $ withConnection st $ \db ->
-      DB.query db "CALL expire_old_messages(?,?,?,0,0,0)" (oldQueue, oldMsg, batchSize)
+      DB.query db "CALL expire_old_messages(?,?,0,0,0)" (oldMsg, batchSize)
     where
       st = dbStore $ queueStore_ ms
-      oldQueue = 0 :: Int64 -- expire all queues
       oldMsg = now - ttl
       batchSize = 10000 :: Int
       toMessageStats (expiredMsgsCount, storedMsgsCount, storedQueues) =

src/Simplex/Messaging/Server/Names.hs

@@ -15,12 +15,15 @@ module Simplex.Messaging.Server.Names
   )
 where
 
+import Control.Concurrent.QSem (QSem, newQSem, signalQSem, waitQSem)
 import qualified Control.Exception as E
 import Control.Logger.Simple (logError)
 import Data.Bifunctor (first)
 import Data.Maybe (fromMaybe)
 import qualified Data.Text as T
-import Simplex.Messaging.Protocol (NameErrorType (..), NameRecord)
+import Data.Text.Encoding (decodeLatin1)
+import Simplex.Messaging.Encoding
+import Simplex.Messaging.Protocol (NameErrorType (..), NameQuery, NameResponse)
 import Simplex.Messaging.Server.Names.HttpResolver
   ( ResolverEnv,
     ResolverError (..),
@@ -30,26 +33,28 @@ import Simplex.Messaging.Server.Names.HttpResolver
     newResolverEnv,
     resolveHttp,
   )
-import Simplex.Messaging.SimplexName (SimplexDomain, fullDomainName)
 import System.Timeout (timeout)
 
 data NamesConfig = NamesConfig
   { resolverEndpoint :: String,
     resolverAuth :: Maybe RpcAuth,
     resolverTimeoutMs :: Int,
-    resolverMaxResponseBytes :: Int
+    resolverMaxResponseBytes :: Int,
+    resolverGlobalConcurrency :: Int
   }
   deriving (Show)
 
 data NamesEnv = NamesEnv
   { config :: NamesConfig,
-    resolverEnv :: ResolverEnv
+    resolverEnv :: ResolverEnv,
+    resolverSlots :: QSem
   }
 
 newNamesEnv :: NamesConfig -> IO NamesEnv
 newNamesEnv config = do
-  resolverEnv <- newResolverEnv (resolverEndpoint config) (resolverAuth config) (resolverTimeoutMs config) (resolverMaxResponseBytes config)
-  pure NamesEnv {config, resolverEnv}
+  resolverEnv <- newResolverEnv (resolverEndpoint config) (resolverAuth config) (resolverTimeoutMs config) (resolverMaxResponseBytes config) (resolverGlobalConcurrency config)
+  resolverSlots <- newQSem (resolverGlobalConcurrency config)
+  pure NamesEnv {config, resolverEnv, resolverSlots}
 
 closeNamesEnv :: NamesEnv -> IO ()
 closeNamesEnv NamesEnv {resolverEnv} = closeResolverEnv resolverEnv
@@ -58,9 +63,10 @@ pingEndpoint :: NamesEnv -> IO (Either ResolverError ())
 pingEndpoint NamesEnv {resolverEnv, config} =
   fromMaybe (Left ResolverTimeout) <$> timeout (resolverTimeoutMs config * 1000) (healthHttp resolverEnv)
 
-resolveName :: NamesEnv -> SimplexDomain -> IO (Either NameErrorType NameRecord)
-resolveName env d = do
-  r <- E.try (timeout (resolverTimeoutMs (config env) * 1000) (fetch env d))
+resolveName :: NamesEnv -> NameQuery -> IO (Either NameErrorType NameResponse)
+resolveName env@NamesEnv {resolverSlots} q = do
+  -- waiting for a slot counts towards the timeout, so lookups do not queue longer than resolverTimeoutMs
+  r <- E.try (timeout (resolverTimeoutMs (config env) * 1000) (E.bracket_ (waitQSem resolverSlots) (signalQSem resolverSlots) (fetch env q)))
   case r of
     Right result -> pure (fromMaybe (Left (RESOLVER "timeout")) result)
     Left e
@@ -69,14 +75,12 @@ resolveName env d = do
           logError $ "[NAMES] resolver fetch raised " <> T.pack (E.displayException e)
           pure (Left (RESOLVER "resolver error"))
 
-fetch :: NamesEnv -> SimplexDomain -> IO (Either NameErrorType NameRecord)
-fetch NamesEnv {resolverEnv} d =
-  first mapResolverError <$> resolveHttp resolverEnv (fullDomainName d)
+fetch :: NamesEnv -> NameQuery -> IO (Either NameErrorType NameResponse)
+fetch NamesEnv {resolverEnv} q =
+  first mapResolverError <$> resolveHttp resolverEnv (decodeLatin1 $ smpEncode q)
 
 mapResolverError :: ResolverError -> NameErrorType
 mapResolverError = \case
-  HttpStatusErr 404 -> NOT_FOUND
-  HttpStatusErr 400 -> NOT_FOUND
   HttpStatusErr code -> RESOLVER ("HTTP " <> T.pack (show code))
   HttpFailure _ -> RESOLVER "transport failure"
   BodyTooLarge -> RESOLVER "response too large"

src/Simplex/Messaging/Server/Names/HttpResolver.hs

@@ -8,10 +8,12 @@
 --
 -- The Python REST resolver (see scripts/resolver/snrc-resolve.py) exposes
 --
---   GET /resolve/<name>   -> 200 with a NameRecord JSON document
---                            404 / 400 for unknown names / TLDs
---                            502 for upstream RPC failures
---   GET /health           -> 200 when the resolver process is ready
+--   GET /v2/resolve/<query> -> 200 with a NameRegistration JSON document, for
+--                              all three registration shapes; 400 for unknown
+--                              TLDs, 502 for upstream RPC failures
+--   GET /v1/resolve/<name>  -> 200 with a NameRecord, what relays before SMP
+--                              v22 call as /resolve
+--   GET /health             -> 200 when the resolver process is ready
 --
 -- Boundary properties:
 --   * Response body read with `brReadSome maxResponseBytes` — adversarial
@@ -57,7 +59,7 @@ import qualified Network.HTTP.Client as HC
 import Network.HTTP.Client.TLS (tlsManagerSettings)
 import qualified Network.HTTP.Types as HT
 import Network.HTTP.Types.URI (urlEncode)
-import Simplex.Messaging.Names.Record (NameRecord)
+import Simplex.Messaging.Names.Record (NameResponse)
 
 data RpcAuth = AuthBearer Text | AuthBasic Text Text
 
@@ -83,9 +85,9 @@ data ResolverError
   | ResolverTimeout
   deriving (Show)
 
-newResolverEnv :: String -> Maybe RpcAuth -> Int -> Int -> IO ResolverEnv
-newResolverEnv baseUrl auth_ timeoutMs maxResponseBytes = do
-  manager <- HC.newManager tlsManagerSettings {managerConnCount = 10}
+newResolverEnv :: String -> Maybe RpcAuth -> Int -> Int -> Int -> IO ResolverEnv
+newResolverEnv baseUrl auth_ timeoutMs maxResponseBytes maxConcurrency = do
+  manager <- HC.newManager tlsManagerSettings {managerConnCount = maxConcurrency}
   pure
     ResolverEnv
       { manager,
@@ -108,14 +110,12 @@ authHeader = \case
     let encoded = BAE.convertToBase BAE.Base64 (encodeUtf8 u <> ":" <> encodeUtf8 p) :: ByteString
      in ("Authorization", "Basic " <> encoded)
 
--- | GET <baseUrl>/resolve/<percent-encoded name>, decoding the 200 body
--- directly into a NameRecord in one pass (no intermediate Aeson Value). The
--- name is percent-encoded (every non-unreserved byte per RFC 3986): the
--- resolver expects raw labels, so slashes/punctuation must not alter the path.
-resolveHttp :: ResolverEnv -> Text -> IO (Either ResolverError NameRecord)
-resolveHttp env name =
+-- | The query is a name or a bracketed label hash, percent-encoded (every
+-- non-unreserved byte per RFC 3986) so it cannot alter the path.
+resolveHttp :: ResolverEnv -> Text -> IO (Either ResolverError NameResponse)
+resolveHttp env q =
   (>>= first InvalidJson . J.eitherDecodeStrict . BL.toStrict)
-    <$> httpGet env ("/resolve/" <> B.unpack (urlEncode True (encodeUtf8 name)))
+    <$> httpGet env ("/v2/resolve/" <> B.unpack (urlEncode True (encodeUtf8 q)))
 
 -- | GET <baseUrl>/health; success = reachable with status < 400. The body is
 -- size-capped but NOT decoded — the probe only checks reachability.
@@ -136,8 +136,9 @@ httpGet ResolverEnv {manager, baseUrl, authHdr, timeoutMicro, maxResponseBytes}
           }
   result <- E.try $ withResponse req manager $ \res -> do
     let status = HT.statusCode (responseStatus res)
+    -- http-client closes the connection unless the body is read to the end
     if status >= 400
-      then pure (Left (HttpStatusErr status))
+      then Left (HttpStatusErr status) <$ brReadSome (responseBody res) (maxResponseBytes + 1)
       else do
         bs <- brReadSome (responseBody res) (maxResponseBytes + 1)
         pure $ if BL.length bs > fromIntegral maxResponseBytes then Left BodyTooLarge else Right bs

src/Simplex/Messaging/Server/NtfStore.hs

@@ -10,6 +10,7 @@ module Simplex.Messaging.Server.NtfStore
     storeNtf,
     deleteNtfs,
     deleteExpiredNtfs,
+    deleteEmptyNtfs,
   ) where
 
 import Control.Concurrent.STM
@@ -22,6 +23,7 @@ import Simplex.Messaging.Encoding.String
 import Simplex.Messaging.Protocol (EncNMsgMeta, MsgId, NotifierId)
 import Simplex.Messaging.TMap (TMap)
 import qualified Simplex.Messaging.TMap as TM
+import Simplex.Messaging.Util (whenM)
 
 newtype NtfStore = NtfStore (TMap NotifierId (TVar [MsgNtf]))
 
@@ -33,13 +35,10 @@ data MsgNtf = MsgNtf
   }
 
 storeNtf :: NtfStore -> NotifierId -> MsgNtf -> IO ()
-storeNtf (NtfStore ns) nId ntf = do
-  TM.lookupIO nId ns >>= atomically . maybe newNtfs (`modifyTVar'` (ntf :))
+storeNtf (NtfStore ns) nId ntf =
   -- TODO [ntfdb] coalesce messages here once the client is updated to process multiple messages
   -- for single notification.
-  -- when (isJust prevNtf) $ incStat $ msgNtfReplaced stats
-  where
-    newNtfs = TM.lookup nId ns >>= maybe (TM.insertM nId (newTVar [ntf]) ns) (`modifyTVar'` (ntf :))
+  atomically $ TM.lookup nId ns >>= maybe (TM.insertM nId (newTVar [ntf]) ns) (`modifyTVar'` (ntf :))
 
 deleteNtfs :: NtfStore -> NotifierId -> IO Int
 deleteNtfs (NtfStore ns) nId = atomically (TM.lookupDelete nId ns) >>= maybe (pure 0) (fmap length . readTVarIO)
@@ -48,18 +47,25 @@ deleteExpiredNtfs :: NtfStore -> Int64 -> IO Int
 deleteExpiredNtfs (NtfStore ns) old =
   foldM (\expired -> fmap (expired +) . expireQueue) 0 . M.keys =<< readTVarIO ns
   where
-    expireQueue nId = TM.lookupIO nId ns >>= maybe (pure 0) expire
-    expire v = readTVarIO v >>= \case
-      [] -> pure 0
-      _ ->
-        atomically $ readTVar v >>= \case
-          [] -> pure 0
-          -- check the last message first, it is the earliest
-          ntfs | systemSeconds (ntfTs $ last $ ntfs) < old -> do
+    expireQueue nId = atomically $ TM.lookup nId ns >>= maybe (pure 0) (expire nId)
+    expire nId v = readTVar v >>= \case
+      [] -> TM.delete nId ns >> pure 0
+      -- check the last message first, it is the earliest
+      ntfs
+        | systemSeconds (ntfTs $ last ntfs) < old -> do
             let !ntfs' = filter (\MsgNtf {ntfTs = ts} -> systemSeconds ts >= old) ntfs
-            writeTVar v ntfs'
-            pure $! length ntfs - length ntfs'
-          _ -> pure 0
+            if null ntfs'
+              then TM.delete nId ns >> pure (length ntfs)
+              else writeTVar v ntfs' >> pure (length ntfs - length ntfs')
+        | otherwise -> pure 0
+
+deleteEmptyNtfs :: NtfStore -> [(NotifierId, TVar [MsgNtf])] -> IO ()
+deleteEmptyNtfs (NtfStore ns) = mapM_ deleteEmpty
+  where
+    deleteEmpty (nId, v) =
+      whenM (null <$> readTVarIO v) $
+        atomically $
+          TM.lookup nId ns >>= mapM_ (\v' -> whenM (null <$> readTVar v') $ TM.delete nId ns)
 
 data NtfLogRecord = NLRv1 NotifierId MsgNtf
 

src/Simplex/Messaging/Server/Prometheus.hs

@@ -469,11 +469,11 @@ prometheusMetrics sm rtm ts =
           \# TYPE simplex_smp_names_reqs counter\n\
           \simplex_smp_names_reqs " <> mshow _rslvReqs <> "\n# rslvReqs\n\
           \\n\
-          \# HELP simplex_smp_names_success NameRecord successfully resolved and returned.\n\
+          \# HELP simplex_smp_names_success NameRecord resolved, or availability answered.\n\
           \# TYPE simplex_smp_names_success counter\n\
           \simplex_smp_names_success " <> mshow _rslvSucc <> "\n# rslvSucc\n\
           \\n\
-          \# HELP simplex_smp_names_not_found Name not registered (resolver returned 404 / 400).\n\
+          \# HELP simplex_smp_names_not_found Answers a client below v22 reads as NOT_FOUND.\n\
           \# TYPE simplex_smp_names_not_found counter\n\
           \simplex_smp_names_not_found " <> mshow _rslvNotFound <> "\n# rslvNotFound\n\
           \\n\

src/Simplex/Messaging/Server/QueueStore/Postgres.hs

@@ -159,7 +159,8 @@ instance StoreQueueClass q => QueueStoreClass q (PostgresQueueStore q) where
           db
           [sql|
             SELECT
-              (SELECT COUNT(1) FROM msg_queues WHERE deleted_at IS NULL) AS queue_count,
+              -- estimate via reltuples to avoid a full heap scan on every scrape
+              (SELECT GREATEST(reltuples, 0)::bigint FROM pg_class WHERE oid = 'msg_queues'::regclass) AS queue_count,
               (SELECT COUNT(1) FROM msg_queues WHERE deleted_at IS NULL AND notifier_id IS NOT NULL) AS notifier_count,
               (SELECT COUNT(1) FROM services WHERE service_role = ?) AS rcv_service_count,
               (SELECT COUNT(1) FROM services WHERE service_role = ?) AS ntf_service_count,
@@ -342,7 +343,7 @@ instance StoreQueueClass q => QueueStoreClass q (PostgresQueueStore q) where
     withQueueRec sq "secureQueue" $ \q -> do
       verify q
       assertUpdated $ withDB' "secureQueue" st $ \db ->
-        DB.execute db "UPDATE msg_queues SET sender_key = ? WHERE recipient_id = ? AND deleted_at IS NULL" (sKey, rId)
+        DB.execute db "UPDATE msg_queues SET sender_key = ? WHERE recipient_id = ? AND deleted_at IS NULL AND (sender_key IS NULL OR sender_key = ?)" (sKey, rId, sKey)
       atomically $ writeTVar (queueRec sq) $ Just q {senderKey = Just sKey}
       withLog "secureQueue" st $ \s -> logSecureQueue s rId sKey
     where

src/Simplex/Messaging/Server/QueueStore/Postgres/Migrations.hs

@@ -20,7 +20,9 @@ serverSchemaMigrations =
     ("20250320_short_links", m20250320_short_links, Just down_m20250320_short_links),
     ("20250514_service_certs", m20250514_service_certs, Just down_m20250514_service_certs),
     ("20250903_store_messages", m20250903_store_messages, Just down_m20250903_store_messages),
-    ("20250915_queue_ids_hash", m20250915_queue_ids_hash, Just down_m20250915_queue_ids_hash)
+    ("20250915_queue_ids_hash", m20250915_queue_ids_hash, Just down_m20250915_queue_ids_hash),
+    ("20260916_prometheus_indexes", m20260916_prometheus_indexes, Just down_m20260916_prometheus_indexes),
+    ("20260917_msg_queues_hot", m20260917_msg_queues_hot, Just down_m20260917_msg_queues_hot)
   ]
 
 -- | The list of migrations in ascending order by date
@@ -588,3 +590,147 @@ ALTER TABLE services
   DROP COLUMN queue_ids_hash;
     |]
     <> dropXorHashFuncs
+
+m20260916_prometheus_indexes :: Text
+m20260916_prometheus_indexes =
+  [r|
+CREATE INDEX idx_msg_queues_expire ON msg_queues (recipient_id) WHERE deleted_at IS NULL AND msg_queue_expire;
+CREATE INDEX idx_msg_queues_notifier_active ON msg_queues (notifier_id) WHERE deleted_at IS NULL AND notifier_id IS NOT NULL;
+    |]
+
+down_m20260916_prometheus_indexes :: Text
+down_m20260916_prometheus_indexes =
+  [r|
+DROP INDEX idx_msg_queues_expire;
+DROP INDEX idx_msg_queues_notifier_active;
+    |]
+
+m20260917_msg_queues_hot :: Text
+m20260917_msg_queues_hot =
+  [r|
+DROP INDEX idx_msg_queues_updated_at_recipient_id;
+
+ALTER TABLE msg_queues SET (
+  fillfactor = 80,
+  autovacuum_vacuum_scale_factor = 0.02,
+  autovacuum_analyze_scale_factor = 0.01,
+  autovacuum_vacuum_cost_limit = 1000
+);
+
+ALTER TABLE messages SET (
+  autovacuum_vacuum_scale_factor = 0.02,
+  autovacuum_analyze_scale_factor = 0.01,
+  toast.autovacuum_vacuum_scale_factor = 0.02
+);
+
+ALTER TABLE services SET (
+  fillfactor = 70,
+  autovacuum_vacuum_threshold = 1000,
+  autovacuum_vacuum_scale_factor = 0
+);
+
+DROP PROCEDURE expire_old_messages(bigint, bigint, integer);
+
+CREATE PROCEDURE expire_old_messages(IN p_old_ts bigint, IN batch_size integer, OUT r_expired_msgs_count bigint, OUT r_stored_msgs_count bigint, OUT r_stored_queues bigint)
+    LANGUAGE plpgsql
+    AS $$
+DECLARE
+  rids BYTEA[];
+  rid BYTEA;
+  last_rid BYTEA := '\x';
+  del_count BIGINT;
+  total_deleted BIGINT := 0;
+BEGIN
+  LOOP
+    SELECT array_agg(recipient_id)
+    INTO rids
+    FROM (
+      SELECT recipient_id
+      FROM msg_queues
+      WHERE deleted_at IS NULL
+        AND msg_queue_expire = TRUE
+        AND recipient_id > last_rid
+      ORDER BY recipient_id ASC
+      LIMIT batch_size
+    ) qs;
+
+    EXIT WHEN rids IS NULL OR cardinality(rids) = 0;
+
+    FOREACH rid IN ARRAY rids
+    LOOP
+      BEGIN
+        del_count := delete_expired_msgs(rid, p_old_ts);
+        total_deleted := total_deleted + del_count;
+      EXCEPTION WHEN OTHERS THEN
+        RAISE WARNING 'STORE, expire_old_messages, error expiring queue %: %', encode(rid, 'base64'), SQLERRM;
+        CONTINUE;
+      END;
+      COMMIT;
+    END LOOP;
+    last_rid := rids[cardinality(rids)];
+  END LOOP;
+
+  r_expired_msgs_count := total_deleted;
+  r_stored_msgs_count := (SELECT COUNT(1) FROM messages);
+  r_stored_queues := (SELECT COUNT(1) FROM msg_queues WHERE deleted_at IS NULL);
+END;
+$$;
+    |]
+
+down_m20260917_msg_queues_hot :: Text
+down_m20260917_msg_queues_hot =
+  [r|
+DROP PROCEDURE expire_old_messages(bigint, integer);
+
+CREATE PROCEDURE expire_old_messages(IN p_old_queue bigint, IN p_old_ts bigint, IN batch_size integer, OUT r_expired_msgs_count bigint, OUT r_stored_msgs_count bigint, OUT r_stored_queues bigint)
+    LANGUAGE plpgsql
+    AS $$
+DECLARE
+  rids BYTEA[];
+  rid BYTEA;
+  last_rid BYTEA := '\x';
+  del_count BIGINT;
+  total_deleted BIGINT := 0;
+BEGIN
+  LOOP
+    SELECT array_agg(recipient_id)
+    INTO rids
+    FROM (
+      SELECT recipient_id
+      FROM msg_queues
+      WHERE deleted_at IS NULL
+        AND updated_at > p_old_queue
+        AND msg_queue_expire = TRUE
+        AND recipient_id > last_rid
+      ORDER BY recipient_id ASC
+      LIMIT batch_size
+    ) qs;
+
+    EXIT WHEN rids IS NULL OR cardinality(rids) = 0;
+
+    FOREACH rid IN ARRAY rids
+    LOOP
+      BEGIN
+        del_count := delete_expired_msgs(rid, p_old_ts);
+        total_deleted := total_deleted + del_count;
+      EXCEPTION WHEN OTHERS THEN
+        RAISE WARNING 'STORE, expire_old_messages, error expiring queue %: %', encode(rid, 'base64'), SQLERRM;
+        CONTINUE;
+      END;
+      COMMIT;
+    END LOOP;
+    last_rid := rids[cardinality(rids)];
+  END LOOP;
+
+  r_expired_msgs_count := total_deleted;
+  r_stored_msgs_count := (SELECT COUNT(1) FROM messages);
+  r_stored_queues := (SELECT COUNT(1) FROM msg_queues WHERE deleted_at IS NULL);
+END;
+$$;
+
+CREATE INDEX idx_msg_queues_updated_at_recipient_id ON msg_queues (deleted_at, updated_at, msg_queue_expire, recipient_id);
+
+ALTER TABLE msg_queues RESET (fillfactor, autovacuum_vacuum_scale_factor, autovacuum_analyze_scale_factor, autovacuum_vacuum_cost_limit);
+ALTER TABLE messages RESET (autovacuum_vacuum_scale_factor, autovacuum_analyze_scale_factor, toast.autovacuum_vacuum_scale_factor);
+ALTER TABLE services RESET (fillfactor, autovacuum_vacuum_threshold, autovacuum_vacuum_scale_factor);
+    |]

src/Simplex/Messaging/Server/QueueStore/Postgres/server_schema.sql

@@ -1,5 +1,6 @@
 
 
+
 SET statement_timeout = 0;
 SET lock_timeout = 0;
 SET idle_in_transaction_session_timeout = 0;
@@ -56,7 +57,7 @@ $$;
 
 
 
-CREATE PROCEDURE smp_server.expire_old_messages(IN p_old_queue bigint, IN p_old_ts bigint, IN batch_size integer, OUT r_expired_msgs_count bigint, OUT r_stored_msgs_count bigint, OUT r_stored_queues bigint)
+CREATE PROCEDURE smp_server.expire_old_messages(IN p_old_ts bigint, IN batch_size integer, OUT r_expired_msgs_count bigint, OUT r_stored_msgs_count bigint, OUT r_stored_queues bigint)
     LANGUAGE plpgsql
     AS $$
 DECLARE
@@ -73,7 +74,6 @@ BEGIN
       SELECT recipient_id
       FROM msg_queues
       WHERE deleted_at IS NULL
-        AND updated_at > p_old_queue
         AND msg_queue_expire = TRUE
         AND recipient_id > last_rid
       ORDER BY recipient_id ASC
@@ -397,7 +397,8 @@ CREATE TABLE smp_server.messages (
     msg_quota boolean NOT NULL,
     msg_ntf_flag boolean NOT NULL,
     msg_body bytea NOT NULL
-);
+)
+WITH (autovacuum_vacuum_scale_factor='0.02', autovacuum_analyze_scale_factor='0.01', toast.autovacuum_vacuum_scale_factor='0.02');
 
 
 
@@ -441,7 +442,8 @@ CREATE TABLE smp_server.msg_queues (
     msg_can_write boolean DEFAULT true NOT NULL,
     msg_queue_expire boolean DEFAULT false NOT NULL,
     msg_queue_size bigint DEFAULT 0 NOT NULL
-);
+)
+WITH (fillfactor='80', autovacuum_vacuum_scale_factor='0.02', autovacuum_analyze_scale_factor='0.01', autovacuum_vacuum_cost_limit='1000');
 
 
 
@@ -453,7 +455,8 @@ CREATE TABLE smp_server.services (
     created_at bigint NOT NULL,
     queue_count bigint DEFAULT 0 NOT NULL,
     queue_ids_hash bytea DEFAULT '\x00000000000000000000000000000000'::bytea NOT NULL
-);
+)
+WITH (fillfactor='70', autovacuum_vacuum_threshold='1000', autovacuum_vacuum_scale_factor='0');
 
 
 
@@ -494,10 +497,18 @@ CREATE INDEX idx_messages_recipient_id_msg_ts ON smp_server.messages USING btree
 
 
 
+CREATE INDEX idx_msg_queues_expire ON smp_server.msg_queues USING btree (recipient_id) WHERE ((deleted_at IS NULL) AND msg_queue_expire);
+
+
+
 CREATE UNIQUE INDEX idx_msg_queues_link_id ON smp_server.msg_queues USING btree (link_id);
 
 
 
+CREATE INDEX idx_msg_queues_notifier_active ON smp_server.msg_queues USING btree (notifier_id) WHERE ((deleted_at IS NULL) AND (notifier_id IS NOT NULL));
+
+
+
 CREATE UNIQUE INDEX idx_msg_queues_notifier_id ON smp_server.msg_queues USING btree (notifier_id);
 
 
@@ -514,10 +525,6 @@ CREATE UNIQUE INDEX idx_msg_queues_sender_id ON smp_server.msg_queues USING btre
 
 
 
-CREATE INDEX idx_msg_queues_updated_at_recipient_id ON smp_server.msg_queues USING btree (deleted_at, updated_at, msg_queue_expire, recipient_id);
-
-
-
 CREATE INDEX idx_services_service_role ON smp_server.services USING btree (service_role);
 
 
@@ -549,3 +556,4 @@ ALTER TABLE ONLY smp_server.msg_queues
 
 
 
+

src/Simplex/Messaging/Server/Web.hs

@@ -270,14 +270,14 @@ serverInfoSubsts simplexmqSource information =
       ]
     admin ServerContactAddress {simplex, email, pgp} =
       [ ("admin", Just ""),
-        ("adminSimplex", strEncode <$> simplex),
+        ("adminSimplex", encodeUtf8 <$> simplex),
         ("adminEmail", encodeUtf8 <$> email),
         ("adminPGP", encodeUtf8 . pkURI <$> pgp),
         ("adminPGPFingerprint", encodeUtf8 . pkFingerprint <$> pgp)
       ]
     complaints ServerContactAddress {simplex, email, pgp} =
       [ ("complaints", Just ""),
-        ("complaintsSimplex", strEncode <$> simplex),
+        ("complaintsSimplex", encodeUtf8 <$> simplex),
         ("complaintsEmail", encodeUtf8 <$> email),
         ("complaintsPGP", encodeUtf8 . pkURI <$> pgp),
         ("complaintsPGPFingerprint", encodeUtf8 . pkFingerprint <$> pgp)

src/Simplex/Messaging/SimplexName.hs

@@ -10,14 +10,21 @@ module Simplex.Messaging.SimplexName
     SimplexTLD (..),
     SimplexNameType (..),
     fullDomainName,
+    LabelHash (..),
+    labelHash,
+    boundedNonSpace,
     shortNameInfoStr,
   )
 where
 
 import Control.Applicative (optional, (<|>))
+import Crypto.Hash (Digest, hash)
+import Crypto.Hash.Algorithms (Keccak_256)
 import qualified Data.Aeson.TH as J
 import qualified Data.Attoparsec.ByteString.Char8 as A
 import qualified Data.Attoparsec.Text as AT
+import qualified Data.ByteArray as BA
+import qualified Data.ByteArray.Encoding as BAE
 import Data.ByteString.Char8 (ByteString)
 import qualified Data.ByteString.Char8 as B
 import Data.Char (isDigit)
@@ -70,6 +77,20 @@ nameLabelP = do
     -- (Cyrillic а vs ASCII a hash to different on-chain records).
     isNameLetter c = c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z'
 
+-- | The registry's key for a label: 32 bytes, as labelOf takes it.
+newtype LabelHash = LabelHash ByteString
+  deriving (Eq, Show)
+
+-- | keccak-256 of the lowercased label, as the registry keys it.
+labelHash :: Text -> LabelHash
+labelHash label = LabelHash $ BA.convert (hash (encodeUtf8 (T.toLower label)) :: Digest Keccak_256)
+
+instance StrEncoding LabelHash where
+  strEncode (LabelHash h) = '[' `B.cons` (BAE.convertToBase BAE.Base16 h `B.snoc` ']')
+  strP = do
+    h <- BAE.convertFromBase BAE.Base16 <$?> (A.char '[' *> A.takeWhile (/= ']') <* A.char ']')
+    if B.length h == 32 then pure $ LabelHash h else fail "bad LabelHash"
+
 -- | Cap the name at 253 bytes (DNS full-domain limit)
 boundedNonSpace :: A.Parser ByteString
 boundedNonSpace = do
@@ -108,12 +129,15 @@ instance Encoding SimplexDomain where
   smpP = strP
 
 fullDomainName :: SimplexDomain -> Text
-fullDomainName SimplexDomain {nameTLD, domain, subDomain} = T.intercalate "." (reverse subDomain ++ [domain] ++ tld')
-  where
-    tld' = case nameTLD of
-      TLDSimplex -> ["simplex"]
-      TLDTesting -> ["testing"]
-      TLDWeb -> []
+fullDomainName SimplexDomain {nameTLD, domain, subDomain} = T.intercalate "." (reverse subDomain ++ [domain]) <> decodeLatin1 (strEncode nameTLD)
+
+instance StrEncoding SimplexTLD where
+  strEncode = \case
+    TLDSimplex -> ".simplex"
+    TLDTesting -> ".testing"
+    TLDWeb -> ""
+  strP =
+    ".simplex" $> TLDSimplex <|> ".testing" $> TLDTesting <|> pure TLDWeb
 
 shortNameInfoStr :: SimplexNameInfo -> Text
 shortNameInfoStr = \case

src/Simplex/Messaging/Transport.hs

@@ -46,18 +46,14 @@ module Simplex.Messaging.Transport
     minServerSMPRelayVersion,
     currentClientSMPRelayVersion,
     currentServerSMPRelayVersion,
-    authCmdsSMPVersion,
-    sendingProxySMPVersion,
-    sndAuthKeySMPVersion,
-    deletedEventSMPVersion,
-    encryptedBlockSMPVersion,
-    blockedEntitySMPVersion,
     shortLinksSMPVersion,
     serviceCertsSMPVersion,
     newNtfCredsSMPVersion,
     clientNoticesSMPVersion,
     rcvServiceSMPVersion,
     namesSMPVersion,
+    serverInfoSMPVersion,
+    nameAvailSMPVersion,
     simplexMQVersion,
     smpBlockSize,
     TransportConfig (..),
@@ -89,6 +85,8 @@ module Simplex.Messaging.Transport
     THandle (..),
     THandleParams (..),
     THandleAuth (..),
+    SessionEntitlement (..),
+    EntitlementConfig (..),
     CertChainPubKey (..),
     ServiceCredentials (..),
     THClientService' (..),
@@ -114,8 +112,8 @@ import Control.Monad
 import Control.Monad.Except
 import Control.Monad.IO.Class
 import Control.Monad.Trans.Except (throwE)
+import qualified Data.Aeson as J
 import qualified Data.Aeson.TH as J
-import Data.Attoparsec.ByteString.Char8 (Parser)
 import qualified Data.Attoparsec.ByteString.Char8 as A
 import Data.Bifunctor (first)
 import Data.Bitraversable (bimapM)
@@ -124,6 +122,7 @@ import qualified Data.ByteString.Char8 as B
 import qualified Data.ByteString.Lazy.Char8 as LB
 import Data.Default (def)
 import Data.Functor (($>))
+import Data.Int (Int64)
 import Data.Kind (Type)
 import Data.Tuple (swap)
 import Data.Typeable (Typeable)
@@ -140,9 +139,11 @@ import qualified Simplex.Messaging.Crypto as C
 import Simplex.Messaging.Encoding
 import Simplex.Messaging.Encoding.String
 import Simplex.Messaging.Parsers (dropPrefix, parseRead1, sumTypeJSON)
+import Simplex.Messaging.Server.Information
+import Simplex.Messaging.SystemTime (SystemSeconds)
 import Simplex.Messaging.Transport.Buffer
 import Simplex.Messaging.Transport.Shared
-import Simplex.Messaging.Util (bshow, catchAll, catchAll_, liftEitherWith)
+import Simplex.Messaging.Util (bshow, catchAll, catchAll_, liftEitherWith, (<$$>))
 import Simplex.Messaging.Version
 import Simplex.Messaging.Version.Internal
 import System.IO.Error (isEOFError)
@@ -173,6 +174,9 @@ smpBlockSize = 16384
 -- 17 - create notification credentials with NEW (7/12/2025)
 -- 18 - support client notices (10/10/2025)
 -- 19 - service subscriptions to messages (10/20/2025)
+-- 20 - public namespaces resolver, RSLV command (6/20/2026)
+-- 21 - server public information in handshake (7/5/2026)
+-- 22 - RNAME answers name availability as well as the record (7/25/2026)
 
 data SMPVersion
 
@@ -185,29 +189,8 @@ type VersionRangeSMP = VersionRange SMPVersion
 pattern VersionSMP :: Word16 -> VersionSMP
 pattern VersionSMP v = Version v
 
-_subModeSMPVersion :: VersionSMP
-_subModeSMPVersion = VersionSMP 6
-
-authCmdsSMPVersion :: VersionSMP
-authCmdsSMPVersion = VersionSMP 7
-
-sendingProxySMPVersion :: VersionSMP
-sendingProxySMPVersion = VersionSMP 8
-
-sndAuthKeySMPVersion :: VersionSMP
-sndAuthKeySMPVersion = VersionSMP 9
-
-deletedEventSMPVersion :: VersionSMP
-deletedEventSMPVersion = VersionSMP 10
-
-encryptedBlockSMPVersion :: VersionSMP
-encryptedBlockSMPVersion = VersionSMP 11
-
-blockedEntitySMPVersion :: VersionSMP
-blockedEntitySMPVersion = VersionSMP 12
-
-proxyServerHandshakeSMPVersion :: VersionSMP
-proxyServerHandshakeSMPVersion = VersionSMP 14
+_proxyServerHandshakeSMPVersion :: VersionSMP
+_proxyServerHandshakeSMPVersion = VersionSMP 14
 
 shortLinksSMPVersion :: VersionSMP
 shortLinksSMPVersion = VersionSMP 15
@@ -227,39 +210,40 @@ rcvServiceSMPVersion = VersionSMP 19
 namesSMPVersion :: VersionSMP
 namesSMPVersion = VersionSMP 20
 
+serverInfoSMPVersion :: VersionSMP
+serverInfoSMPVersion = VersionSMP 21
+
+-- | RNAME carries availability. A server below this answers RSLV with the
+-- record alone, and ERR NAME NOT_FOUND for a name that does not resolve.
+nameAvailSMPVersion :: VersionSMP
+nameAvailSMPVersion = VersionSMP 22
+
 minClientSMPRelayVersion :: VersionSMP
-minClientSMPRelayVersion = VersionSMP 6
+minClientSMPRelayVersion = VersionSMP 14
 
 minServerSMPRelayVersion :: VersionSMP
-minServerSMPRelayVersion = VersionSMP 6
+minServerSMPRelayVersion = VersionSMP 14
 
 currentClientSMPRelayVersion :: VersionSMP
-currentClientSMPRelayVersion = VersionSMP 20
-
-legacyServerSMPRelayVersion :: VersionSMP
-legacyServerSMPRelayVersion = VersionSMP 6
+currentClientSMPRelayVersion = VersionSMP 22
 
 currentServerSMPRelayVersion :: VersionSMP
-currentServerSMPRelayVersion = VersionSMP 20
+currentServerSMPRelayVersion = VersionSMP 22
 
 -- Max SMP protocol version to be used in e2e encrypted connection between
 -- client and server, as defined by SMP proxy. Normally set below the current
 -- version to prevent client version fingerprinting by the destination relays
--- when clients upgrade at different times. Pinned to the current version (20)
--- for this release because proxied name resolution is gated on namesSMPVersion
--- (20), so the one-version anti-fingerprinting buffer does not apply yet; it
--- reappears once the current version advances past 20.
+-- when clients upgrade at different times. Pinned to the current version (22)
+-- for this release because a proxied RSLV only carries availability from
+-- nameAvailSMPVersion (22), so the one-version anti-fingerprinting buffer does
+-- not apply yet; it reappears once the current version advances past 22.
 proxiedSMPRelayVersion :: VersionSMP
-proxiedSMPRelayVersion = VersionSMP 20
+proxiedSMPRelayVersion = VersionSMP 22
 
--- minimal supported protocol version is 6
--- TODO remove code that supports sending commands without batching
+-- minimal supported protocol version is 14
 supportedClientSMPRelayVRange :: VersionRangeSMP
 supportedClientSMPRelayVRange = mkVersionRange minClientSMPRelayVersion currentClientSMPRelayVersion
 
-legacyServerSMPRelayVRange :: VersionRangeSMP
-legacyServerSMPRelayVRange = mkVersionRange minServerSMPRelayVersion legacyServerSMPRelayVersion
-
 supportedServerSMPRelayVRange :: VersionRangeSMP
 supportedServerSMPRelayVRange = mkVersionRange minServerSMPRelayVersion currentServerSMPRelayVersion
 
@@ -267,7 +251,7 @@ supportedProxyClientSMPRelayVRange :: VersionRangeSMP
 supportedProxyClientSMPRelayVRange = mkVersionRange minServerSMPRelayVersion currentServerSMPRelayVersion
 
 proxiedSMPRelayVRange :: VersionRangeSMP
-proxiedSMPRelayVRange = mkVersionRange sendingProxySMPVersion proxiedSMPRelayVersion
+proxiedSMPRelayVRange = mkVersionRange minServerSMPRelayVersion proxiedSMPRelayVersion
 
 alpnSupportedSMPHandshakes :: [ALPN]
 alpnSupportedSMPHandshakes = ["smp/1"]
@@ -489,14 +473,14 @@ data THandleParams v p = THandleParams
     thAuth :: Maybe (THandleAuth p),
     -- | do NOT send session ID in transmission, but include it into signed message
     -- based on protocol version
+    -- This is True for SMP and NTF servers, and False for XFTP
     implySessId :: Bool,
     -- | keys for additional transport encryption
     encryptBlock :: Maybe TSbChainKeys,
-    -- | send multiple transmissions in a single block
-    -- based on protocol version
-    batch :: Bool,
     -- | include service signature (or '0' if it is absent), based on protocol version
-    serviceAuth :: Bool
+    serviceAuth :: Bool,
+    -- | JSON-encoded ServerPublicInfo from handshake, present when server version >= serverInfoSMPVersion
+    serverInfo :: Maybe (Either String ServerPublicInfo)
   }
 
 data THandleAuth (p :: TransportPeer) where
@@ -510,10 +494,15 @@ data THandleAuth (p :: TransportPeer) where
   THAuthServer ::
     { serverPrivKey :: C.PrivateKeyX25519, -- used by the server to combine with client's public per-queue key
       peerClientService :: Maybe THPeerClientService,
+      peerEntitlement :: Maybe SessionEntitlement, -- verified in the handshake, applies to the whole session
       sessSecret' :: Maybe C.DhSecretX25519 -- session secret (will be used in SMP proxy only)
     } ->
     THandleAuth 'TServer
 
+data SessionEntitlement = SessionEntitlement {expiresAt :: SystemSeconds, entConfig :: EntitlementConfig}
+
+newtype EntitlementConfig = EntitlementConfig {storageTime :: Int64}
+
 type THClientService = THClientService' C.PrivateKeyEd25519
 
 type THPeerClientService = THClientService' C.PublicKeyEd25519
@@ -548,7 +537,9 @@ data SMPServerHandshake = SMPServerHandshake
     sessionId :: SessionId,
     -- pub key to agree shared secrets for command authorization and entity ID encryption.
     -- todo C.PublicKeyX25519
-    authPubKey :: Maybe CertChainPubKey
+    authPubKey :: CertChainPubKey,
+    -- | optional server public information (JSON-encoded ServerPublicInfo), sent when version >= serverInfoSMPVersion
+    serverInfoBytes :: Maybe ByteString
   }
 
 -- This is the third handshake message that SMP server sends to services
@@ -602,14 +593,13 @@ data SMPServiceRole = SRMessaging | SRNotifier | SRProxy deriving (Eq, Show)
 instance Encoding SMPClientHandshake where
   smpEncode SMPClientHandshake {smpVersion = v, keyHash, authPubKey, proxyServer, clientService} =
     smpEncode (v, keyHash)
-      <> encodeAuthEncryptCmds v authPubKey
-      <> ifHasProxy v (smpEncode proxyServer) ""
+      <> maybe "" smpEncode authPubKey
+      <> smpEncode proxyServer
       <> ifHasService v (smpEncode clientService) ""
   smpP = do
     (v, keyHash) <- smpP
-    -- TODO drop SMP v6: remove special parser and make key non-optional
-    authPubKey <- authEncryptCmdsP v smpP
-    proxyServer <- ifHasProxy v smpP (pure False)
+    authPubKey <- optional smpP
+    proxyServer <- smpP
     clientService <- ifHasService v smpP (pure Nothing)
     pure SMPClientHandshake {smpVersion = v, keyHash, authPubKey, proxyServer, clientService}
 
@@ -632,22 +622,21 @@ instance Encoding SMPServiceRole where
       'P' -> pure SRProxy
       _ -> fail "bad SMPServiceRole"
 
-ifHasProxy :: VersionSMP -> a -> a -> a
-ifHasProxy v a b = if v >= proxyServerHandshakeSMPVersion then a else b
-
 ifHasService :: VersionSMP -> a -> a -> a
 ifHasService v a b = if v >= serviceCertsSMPVersion then a else b
 
+ifHasServerInfo :: VersionSMP -> a -> a -> a
+ifHasServerInfo v a b = if v >= serverInfoSMPVersion then a else b
+
 instance Encoding SMPServerHandshake where
-  smpEncode SMPServerHandshake {smpVersionRange, sessionId, authPubKey} =
-    smpEncode (smpVersionRange, sessionId) <> auth
+  smpEncode SMPServerHandshake {smpVersionRange, sessionId, authPubKey, serverInfoBytes} =
+    smpEncode (smpVersionRange, sessionId, authPubKey) <> info
     where
-      auth = encodeAuthEncryptCmds (maxVersion smpVersionRange) authPubKey
+      info = ifHasServerInfo (maxVersion smpVersionRange) (smpEncode (Large <$> serverInfoBytes)) ""
   smpP = do
-    (smpVersionRange, sessionId) <- smpP
-    -- TODO drop SMP v6: remove special parser and make key non-optional
-    authPubKey <- authEncryptCmdsP (maxVersion smpVersionRange) smpP
-    pure SMPServerHandshake {smpVersionRange, sessionId, authPubKey}
+    (smpVersionRange, sessionId, authPubKey) <- smpP
+    serverInfoBytes <- ifHasServerInfo (maxVersion smpVersionRange) (unLarge <$$> smpP) (pure Nothing)
+    pure SMPServerHandshake {smpVersionRange, sessionId, authPubKey, serverInfoBytes}
 
 -- newtype for CertificateChain and a session key signed with this certificate
 data CertChainPubKey = CertChainPubKey
@@ -663,14 +652,6 @@ instance Encoding CertChainPubKey where
     C.SignedObject signedPubKey <- smpP
     pure CertChainPubKey {certChain, signedPubKey}
 
-encodeAuthEncryptCmds :: Encoding a => VersionSMP -> Maybe a -> ByteString
-encodeAuthEncryptCmds v k
-  | v >= authCmdsSMPVersion = maybe "" smpEncode k
-  | otherwise = ""
-
-authEncryptCmdsP :: VersionSMP -> Parser a -> Parser (Maybe a)
-authEncryptCmdsP v p = if v >= authCmdsSMPVersion then optional p else pure Nothing
-
 instance Encoding SMPServerHandshakeResponse where
   smpEncode = \case
     SMPServerHandshakeResponse serviceId -> smpEncode ('R', serviceId)
@@ -763,12 +744,12 @@ smpServerHandshake ::
   C.KeyPairX25519 ->
   C.KeyHash ->
   VersionRangeSMP ->
+  Maybe ByteString ->
   (SMPServiceRole -> X.CertificateChain -> XV.Fingerprint -> ExceptT TransportError IO ServiceId) ->
   ExceptT TransportError IO (THandleSMP c 'TServer)
-smpServerHandshake srvCert srvSignKey c (k, pk) kh smpVRange getService = do
+smpServerHandshake srvCert srvSignKey c (k, pk) kh smpVersionRange serverInfoBytes getService = do
   let sk = C.signX509 srvSignKey $ C.publicToX509 k
-      smpVersionRange = maybe legacyServerSMPRelayVRange (const smpVRange) $ getSessionALPN c
-  sendHandshake th $ SMPServerHandshake {sessionId, smpVersionRange, authPubKey = Just (CertChainPubKey srvCert sk)}
+  sendHandshake th $ SMPServerHandshake {sessionId, smpVersionRange, authPubKey = CertChainPubKey srvCert sk, serverInfoBytes}
   SMPClientHandshake {smpVersion = v, keyHash, authPubKey = k', proxyServer, clientService} <- getHandshake th
   when (keyHash /= kh) $ throwE $ TEHandshake IDENTITY
   case compatibleVRange' smpVersionRange v of
@@ -800,35 +781,17 @@ smpServerHandshake srvCert srvSignKey c (k, pk) kh smpVRange getService = do
 --
 -- See https://github.com/simplex-chat/simplexmq/blob/master/protocol/simplex-messaging.md#appendix-a
 smpClientHandshake :: forall c. Transport c => c 'TClient -> Maybe C.KeyPairX25519 -> C.KeyHash -> VersionRangeSMP -> Bool -> Maybe (ServiceCredentials, C.KeyPairEd25519) -> ExceptT TransportError IO (THandleSMP c 'TClient)
-smpClientHandshake c ks_ keyHash@(C.KeyHash kh) vRange proxyServer serviceKeys_ = do
-  SMPServerHandshake {sessionId = sessId, smpVersionRange, authPubKey} <- getHandshake th
+smpClientHandshake c ks_ keyHash@(C.KeyHash kh) smpVRange proxyServer serviceKeys_ = do
+  SMPServerHandshake {sessionId = sessId, smpVersionRange, authPubKey = certKey@(CertChainPubKey chain exact), serverInfoBytes} <- getHandshake th
   when (sessionId /= sessId) $ throwE TEBadSession
-  -- Below logic downgrades version range in case the "client" is SMP proxy server and it is
-  -- connected to the destination server of the version 11 or older.
-  -- It disables transport encryption between SMP proxy and destination relay.
-  --
-  -- Prior to version v6.3 the version between proxy and destination was capped at 8,
-  -- by mistake, which also disables transport encryption and the latest features.
-  --
-  -- Transport encryption between proxy and destination breaks clients with version 10 or earlier,
-  -- because of a larger message size (see maxMessageLength).
-  --
-  -- To summarize:
-  -- - proxy and relay version 12: the agreed version is 12, transport encryption disabled (see blockEncryption with proxyServer == True).
-  -- - proxy is v 12, relay is 11: the agreed version is 10, because of this logic, transport encryption is disabled.
-  let smpVRange =
-        if proxyServer && maxVersion smpVersionRange < proxyServerHandshakeSMPVersion
-          then vRange {maxVersion = max (minVersion vRange) deletedEventSMPVersion}
-          else vRange
   case smpVersionRange `compatibleVRange` smpVRange of
     Just (Compatible vr) -> do
-      ck_ <- forM authPubKey $ \certKey@(CertChainPubKey chain exact) ->
-        liftEitherWith (const $ TEHandshake BAD_AUTH) $ do
-          case chainIdCaCerts chain of
-            CCValid {idCert} | XV.Fingerprint kh == XV.getFingerprint idCert X.HashSHA256 -> pure ()
-            _ -> throwError "bad certificate"
-          serverKey <- getServerVerifyKey c
-          (,certKey) <$> (C.x509ToPublic' =<< C.verifyX509 serverKey exact)
+      ck <- liftEitherWith (const $ TEHandshake BAD_AUTH) $ do
+        case chainIdCaCerts chain of
+          CCValid {idCert} | XV.Fingerprint kh == XV.getFingerprint idCert X.HashSHA256 -> pure ()
+          _ -> throwError "bad certificate"
+        serverKey <- getServerVerifyKey c
+        (,certKey) <$> (C.x509ToPublic' =<< C.verifyX509 serverKey exact)
       let v = maxVersion vr
           serviceVersion ServiceCredentials {serviceRole} = if serviceRole == SRMessaging then rcvServiceSMPVersion else serviceCertsSMPVersion
           serviceKeys = case serviceKeys_ of
@@ -838,7 +801,7 @@ smpClientHandshake c ks_ keyHash@(C.KeyHash kh) vRange proxyServer serviceKeys_
           hs = SMPClientHandshake {smpVersion = v, keyHash, authPubKey = fst <$> ks_, proxyServer, clientService}
       sendHandshake th hs
       service <- mapM getClientService serviceKeys
-      liftIO $ smpTHandleClient th v vr (snd <$> ks_) ck_ proxyServer service
+      liftIO $ smpTHandleClient th v vr (snd <$> ks_) ck proxyServer service serverInfoBytes
     Nothing -> throwE TEVersion
   where
     th@THandle {params = THandleParams {sessionId}} = smpTHandle c
@@ -854,18 +817,18 @@ smpClientHandshake c ks_ keyHash@(C.KeyHash kh) vRange proxyServer serviceKeys_
 
 smpTHandleServer :: forall c. THandleSMP c 'TServer -> VersionSMP -> VersionRangeSMP -> C.PrivateKeyX25519 -> Maybe C.PublicKeyX25519 -> Bool -> Maybe THPeerClientService -> IO (THandleSMP c 'TServer)
 smpTHandleServer th v vr pk k_ proxyServer peerClientService = do
-  let thAuth = Just THAuthServer {serverPrivKey = pk, peerClientService, sessSecret' = (`C.dh'` pk) <$!> k_}
-  be <- blockEncryption th v proxyServer thAuth
-  pure $ smpTHandle_ th v vr thAuth $ uncurry TSbChainKeys <$> be
+  let thAuth = Just THAuthServer {serverPrivKey = pk, peerClientService, peerEntitlement = Nothing, sessSecret' = (`C.dh'` pk) <$!> k_}
+  be <- blockEncryption th proxyServer thAuth
+  pure $ smpTHandle_ th v vr thAuth (uncurry TSbChainKeys <$> be) Nothing
 
-smpTHandleClient :: forall c. THandleSMP c 'TClient -> VersionSMP -> VersionRangeSMP -> Maybe C.PrivateKeyX25519 -> Maybe (C.PublicKeyX25519, CertChainPubKey) -> Bool -> Maybe THClientService -> IO (THandleSMP c 'TClient)
-smpTHandleClient th v vr pk_ ck_ proxyServer clientService = do
-  let thAuth = clientTHParams <$!> ck_
-  be <- blockEncryption th v proxyServer thAuth
+smpTHandleClient :: forall c. THandleSMP c 'TClient -> VersionSMP -> VersionRangeSMP -> Maybe C.PrivateKeyX25519 -> (C.PublicKeyX25519, CertChainPubKey) -> Bool -> Maybe THClientService -> Maybe ByteString -> IO (THandleSMP c 'TClient)
+smpTHandleClient th v vr pk_ (k, ck) proxyServer clientService serverInfoBytes = do
+  let thAuth = Just $! clientTHParams
+  be <- blockEncryption th proxyServer thAuth
   -- swap is needed to use client's sndKey as server's rcvKey and vice versa
-  pure $ smpTHandle_ th v vr thAuth $ uncurry TSbChainKeys . swap <$> be
+  pure $ smpTHandle_ th v vr thAuth (uncurry TSbChainKeys . swap <$> be) serverInfoBytes
   where
-    clientTHParams (k, ck) =
+    clientTHParams =
       THAuthClient
         { peerServerPubKey = k,
           peerServerCertKey = forceCertChain ck,
@@ -873,9 +836,9 @@ smpTHandleClient th v vr pk_ ck_ proxyServer clientService = do
           sessSecret = C.dh' k <$!> pk_
         }
 
-blockEncryption :: THandleSMP c p -> VersionSMP -> Bool -> Maybe (THandleAuth p) -> IO (Maybe (TVar C.SbChainKey, TVar C.SbChainKey))
-blockEncryption THandle {params = THandleParams {sessionId}} v proxyServer = \case
-  Just thAuth | not proxyServer && v >= encryptedBlockSMPVersion -> case thAuth of
+blockEncryption :: THandleSMP c p -> Bool -> Maybe (THandleAuth p) -> IO (Maybe (TVar C.SbChainKey, TVar C.SbChainKey))
+blockEncryption THandle {params = THandleParams {sessionId}} proxyServer = \case
+  Just thAuth | not proxyServer -> case thAuth of
     THAuthClient {sessSecret} -> be sessSecret
     THAuthServer {sessSecret'} -> be sessSecret'
   _ -> pure Nothing
@@ -883,8 +846,8 @@ blockEncryption THandle {params = THandleParams {sessionId}} v proxyServer = \ca
     be :: Maybe C.DhSecretX25519 -> IO (Maybe (TVar C.SbChainKey, TVar C.SbChainKey))
     be = mapM $ \(C.DhSecretX25519 secret) -> bimapM newTVarIO newTVarIO $ C.sbcInit sessionId secret
 
-smpTHandle_ :: forall c p. THandleSMP c p -> VersionSMP -> VersionRangeSMP -> Maybe (THandleAuth p) -> Maybe TSbChainKeys -> THandleSMP c p
-smpTHandle_ th@THandle {params} v vr thAuth encryptBlock =
+smpTHandle_ :: forall c p. THandleSMP c p -> VersionSMP -> VersionRangeSMP -> Maybe (THandleAuth p) -> Maybe TSbChainKeys -> Maybe ByteString -> THandleSMP c p
+smpTHandle_ th@THandle {params} v vr thAuth encryptBlock serverInfoBytes =
   -- TODO drop SMP v6: make thAuth non-optional
   -- * Note: update version-based parameters in smpTHParamsSetVersion as well.
   let params' =
@@ -892,9 +855,9 @@ smpTHandle_ th@THandle {params} v vr thAuth encryptBlock =
           { thVersion = v,
             thServerVRange = vr,
             thAuth,
-            implySessId = v >= authCmdsSMPVersion,
             encryptBlock,
-            serviceAuth = v >= serviceCertsSMPVersion -- optional service signature will be encoded for all commands and responses
+            serviceAuth = v >= serviceCertsSMPVersion, -- optional service signature will be encoded for all commands and responses
+            serverInfo = J.eitherDecodeStrict' <$> serverInfoBytes
           }
    in (th :: THandleSMP c p) {params = params'}
 
@@ -902,7 +865,6 @@ forceCertChain :: CertChainPubKey -> CertChainPubKey
 forceCertChain cert@(CertChainPubKey (X.CertificateChain cc) signedKey) = length (show cc) `seq` show signedKey `seq` cert
 {-# INLINE forceCertChain #-}
 
--- This function is only used with v >= 8, so currently it's a simple record update.
 -- * Note: it requires updating version-based parameters, to be consistent with smpTHandle_.
 smpTHParamsSetVersion :: VersionSMP -> THandleParams SMPVersion p -> THandleParams SMPVersion p
 smpTHParamsSetVersion v params =
@@ -930,10 +892,10 @@ smpTHandle c = THandle {connection = c, params}
           thServerVRange = versionToRange v,
           thVersion = v,
           thAuth = Nothing,
-          implySessId = False,
+          implySessId = True,
           encryptBlock = Nothing,
-          batch = True,
-          serviceAuth = False
+          serviceAuth = False,
+          serverInfo = Nothing
         }
 
 $(J.deriveJSON (sumTypeJSON id) ''HandshakeError)

tests/AgentTests/ConnectionRequestTests.hs

@@ -18,6 +18,7 @@ module AgentTests.ConnectionRequestTests
     invConnRequest,
   ) where
 
+import AgentTests.EqInstances ()
 import Data.ByteString (ByteString)
 import Network.HTTP.Types (urlEncode)
 import Simplex.Messaging.Agent.Protocol
@@ -145,8 +146,8 @@ connReqData1 = connReqData {crSmpQueues = [queue1]}
 connReqDataV1 :: ConnReqUriData
 connReqDataV1 = connReqData {crAgentVRange = mkVersionRange (VersionSMPA 1) (VersionSMPA 1)}
 
-connReqDataV2 :: ConnReqUriData
-connReqDataV2 = connReqData {crAgentVRange = mkVersionRange (VersionSMPA 2) (VersionSMPA 2)}
+connReqDataV6 :: ConnReqUriData
+connReqDataV6 = connReqData {crAgentVRange = mkVersionRange (VersionSMPA 6) (VersionSMPA 6)}
 
 connReqDataNew :: ConnReqUriData
 connReqDataNew = connReqData {crSmpQueues = [queueNew]}
@@ -158,10 +159,10 @@ testDhPubKey :: C.PublicKeyX448
 testDhPubKey = "MEIwBQYDK2VvAzkAmKuSYeQ/m0SixPDS8Wq8VBaTS1cW+Lp0n0h4Diu+kUpR+qXx4SDJ32YGEFoGFGSbGPry5Ychr6U="
 
 testE2ERatchetParams :: RcvE2ERatchetParamsUri 'C.X448
-testE2ERatchetParams = E2ERatchetParamsUri (mkVersionRange (VersionE2E 1) (VersionE2E 1)) testDhPubKey testDhPubKey Nothing
+testE2ERatchetParams = E2ERatchetParamsUri (mkVersionRange (VersionE2E 3) (VersionE2E 3)) testDhPubKey testDhPubKey Nothing
 
 testE2ERatchetParamsStrUri :: ByteString
-testE2ERatchetParamsStrUri = "v%3D1%26x3dh%3DMEIwBQYDK2VvAzkAmKuSYeQ_m0SixPDS8Wq8VBaTS1cW-Lp0n0h4Diu-kUpR-qXx4SDJ32YGEFoGFGSbGPry5Ychr6U%3D%2CMEIwBQYDK2VvAzkAmKuSYeQ_m0SixPDS8Wq8VBaTS1cW-Lp0n0h4Diu-kUpR-qXx4SDJ32YGEFoGFGSbGPry5Ychr6U%3D"
+testE2ERatchetParamsStrUri = "v%3D3%26x3dh%3DMEIwBQYDK2VvAzkAmKuSYeQ_m0SixPDS8Wq8VBaTS1cW-Lp0n0h4Diu-kUpR-qXx4SDJ32YGEFoGFGSbGPry5Ychr6U%3D%2CMEIwBQYDK2VvAzkAmKuSYeQ_m0SixPDS8Wq8VBaTS1cW-Lp0n0h4Diu-kUpR-qXx4SDJ32YGEFoGFGSbGPry5Ychr6U%3D"
 
 testE2ERatchetParams12 :: RcvE2ERatchetParamsUri 'C.X448
 testE2ERatchetParams12 = E2ERatchetParamsUri supportedE2EEncryptVRange testDhPubKey testDhPubKey Nothing
@@ -176,7 +177,7 @@ connectionRequestNoQM :: AConnectionRequestUri
 connectionRequestNoQM = ACR SCMInvitation $ CRInvitationUri connReqDataNoQM testE2ERatchetParams
 
 connectionRequestContact :: AConnectionRequestUri
-connectionRequestContact = ACR SCMContact $ CRContactUri connReqDataContact
+connectionRequestContact = ACR SCMContact $ CRContactUri connReqDataContact Nothing
 
 connectionRequestV1 :: AConnectionRequestUri
 connectionRequestV1 = ACR SCMInvitation $ CRInvitationUri connReqDataV1 testE2ERatchetParams
@@ -194,13 +195,16 @@ contactAddress :: AConnectionRequestUri
 contactAddress = ACR SCMContact $ contactConnRequest
 
 contactConnRequest :: ConnectionRequestUri 'CMContact
-contactConnRequest = CRContactUri connReqData
+contactConnRequest = CRContactUri connReqData Nothing
 
-contactAddressV2 :: AConnectionRequestUri
-contactAddressV2 = ACR SCMContact $ CRContactUri connReqDataV2
+contactAddressDR :: AConnectionRequestUri
+contactAddressDR = ACR SCMContact $ CRContactUri connReqData (Just (RatchetKeyId "0123456789abcdef", testE2ERatchetParams))
+
+contactAddressV6 :: AConnectionRequestUri
+contactAddressV6 = ACR SCMContact $ CRContactUri connReqDataV6 Nothing
 
 contactAddressNew :: AConnectionRequestUri
-contactAddressNew = ACR SCMContact $ CRContactUri connReqDataNew
+contactAddressNew = ACR SCMContact $ CRContactUri connReqDataNew Nothing
 
 connectionRequest2queues :: AConnectionRequestUri
 connectionRequest2queues = ACR SCMInvitation $ CRInvitationUri connReqData {crSmpQueues = [queue, queue]} testE2ERatchetParams
@@ -209,16 +213,20 @@ connectionRequest2queuesNew :: AConnectionRequestUri
 connectionRequest2queuesNew = ACR SCMInvitation $ CRInvitationUri connReqDataNew {crSmpQueues = [queueNew, queueNew]} testE2ERatchetParams
 
 contactAddress2queues :: AConnectionRequestUri
-contactAddress2queues = ACR SCMContact $ CRContactUri connReqData {crSmpQueues = [queue, queue]}
+contactAddress2queues = ACR SCMContact $ CRContactUri connReqData {crSmpQueues = [queue, queue]} Nothing
 
 contactAddress2queuesNew :: AConnectionRequestUri
-contactAddress2queuesNew = ACR SCMContact $ CRContactUri connReqDataNew {crSmpQueues = [queueNew, queueNew]}
+contactAddress2queuesNew = ACR SCMContact $ CRContactUri connReqDataNew {crSmpQueues = [queueNew, queueNew]} Nothing
 
 connectionRequestClientDataEmpty :: AConnectionRequestUri
 connectionRequestClientDataEmpty = ACR SCMInvitation $ CRInvitationUri connReqData {crClientData = Just "{}"} testE2ERatchetParams
 
 contactAddressClientData :: AConnectionRequestUri
-contactAddressClientData = ACR SCMContact $ CRContactUri connReqData {crClientData = Just "{\"type\":\"group_link\", \"group_link_id\":\"abc\"}"}
+contactAddressClientData = ACR SCMContact $ CRContactUri connReqData {crClientData = Just "{\"type\":\"group_link\", \"group_link_id\":\"abc\"}"} Nothing
+
+-- binary encoding is defined only for BinaryConnectionRequestUri; drop the address keys for the round-trip
+aBinaryConnReq :: AConnectionRequestUri -> ABinaryConnectionRequestUri
+aBinaryConnReq (ACR m cr) = ABCR m (binaryConnReq cr)
 
 url :: ByteString -> ByteString
 url = urlEncode True
@@ -256,26 +264,27 @@ connectionRequestTests =
       queueV1NoPort #== ("smp://[email protected]/3456-w==#/?v=1-1&dh=" <> url testDhKeyStr <> "&srv=jjbyvoemxysm7qxap7m5d5m35jzv5qq6gnlv7s4rsn7tdwwmuqciwpid.onion")
       queueV1NoPort #== ("smp://[email protected],jjbyvoemxysm7qxap7m5d5m35jzv5qq6gnlv7s4rsn7tdwwmuqciwpid.onion/3456-w==#" <> testDhKeyStr)
     it "should serialize and parse connection invitations and contact addresses" $ do
-      connectionRequest #==# ("simplex:/invitation#/?v=2-7&smp=" <> url queueStr <> "&e2e=" <> testE2ERatchetParamsStrUri)
-      connectionRequest #== ("https://simplex.chat/invitation#/?v=2-7&smp=" <> url queueStr <> "&e2e=" <> testE2ERatchetParamsStrUri)
-      connectionRequestNoQM #==# ("simplex:/invitation#/?v=2-7&smp=" <> url queueStrNoQM <> "&e2e=" <> testE2ERatchetParamsStrUri)
-      connectionRequest1 #==# ("simplex:/invitation#/?v=2-7&smp=" <> url queue1Str <> "&e2e=" <> testE2ERatchetParamsStrUri)
-      connectionRequest2queues #==# ("simplex:/invitation#/?v=2-7&smp=" <> url (queueStr <> ";" <> queueStr) <> "&e2e=" <> testE2ERatchetParamsStrUri)
-      connectionRequestNew #==# ("simplex:/invitation#/?v=2-7&smp=" <> url queueNewStr <> "&e2e=" <> testE2ERatchetParamsStrUri)
-      connectionRequestNew1 #==# ("simplex:/invitation#/?v=2-7&smp=" <> url queueNew1Str <> "&e2e=" <> testE2ERatchetParamsStrUri)
-      connectionRequest2queuesNew #==# ("simplex:/invitation#/?v=2-7&smp=" <> url (queueNewStr <> ";" <> queueNewStr) <> "&e2e=" <> testE2ERatchetParamsStrUri)
+      connectionRequest #==# ("simplex:/invitation#/?v=6-8&smp=" <> url queueStr <> "&e2e=" <> testE2ERatchetParamsStrUri)
+      connectionRequest #== ("https://simplex.chat/invitation#/?v=6-8&smp=" <> url queueStr <> "&e2e=" <> testE2ERatchetParamsStrUri)
+      connectionRequestNoQM #==# ("simplex:/invitation#/?v=6-8&smp=" <> url queueStrNoQM <> "&e2e=" <> testE2ERatchetParamsStrUri)
+      connectionRequest1 #==# ("simplex:/invitation#/?v=6-8&smp=" <> url queue1Str <> "&e2e=" <> testE2ERatchetParamsStrUri)
+      connectionRequest2queues #==# ("simplex:/invitation#/?v=6-8&smp=" <> url (queueStr <> ";" <> queueStr) <> "&e2e=" <> testE2ERatchetParamsStrUri)
+      connectionRequestNew #==# ("simplex:/invitation#/?v=6-8&smp=" <> url queueNewStr <> "&e2e=" <> testE2ERatchetParamsStrUri)
+      connectionRequestNew1 #==# ("simplex:/invitation#/?v=6-8&smp=" <> url queueNew1Str <> "&e2e=" <> testE2ERatchetParamsStrUri)
+      connectionRequest2queuesNew #==# ("simplex:/invitation#/?v=6-8&smp=" <> url (queueNewStr <> ";" <> queueNewStr) <> "&e2e=" <> testE2ERatchetParamsStrUri)
       connectionRequestV1 #== ("https://simplex.chat/invitation#/?v=1&smp=" <> url queueStr <> "&e2e=" <> testE2ERatchetParamsStrUri)
-      connectionRequestClientDataEmpty #==# ("simplex:/invitation#/?v=2-7&smp=" <> url queueStr <> "&e2e=" <> testE2ERatchetParamsStrUri <> "&data=" <> url "{}")
-      contactAddress #==# ("simplex:/contact#/?v=2-7&smp=" <> url queueStr)
-      contactAddress #== ("https://simplex.chat/contact#/?v=2-7&smp=" <> url queueStr)
-      contactAddress2queues #==# ("simplex:/contact#/?v=2-7&smp=" <> url (queueStr <> ";" <> queueStr))
-      contactAddressNew #==# ("simplex:/contact#/?v=2-7&smp=" <> url queueNewStr)
-      contactAddress2queuesNew #==# ("simplex:/contact#/?v=2-7&smp=" <> url (queueNewStr <> ";" <> queueNewStr))
-      contactAddressV2 #==# ("simplex:/contact#/?v=2&smp=" <> url queueStr)
-      contactAddressV2 #== ("https://simplex.chat/contact#/?v=1&smp=" <> url queueStr) -- adjusted to v2
-      contactAddressV2 #== ("https://simplex.chat/contact#/?v=1-2&smp=" <> url queueStr) -- adjusted to v2
-      contactAddressV2 #== ("https://simplex.chat/contact#/?v=2-2&smp=" <> url queueStr)
-      contactAddressClientData #==# ("simplex:/contact#/?v=2-7&smp=" <> url queueStr <> "&data=" <> url "{\"type\":\"group_link\", \"group_link_id\":\"abc\"}")
+      connectionRequestClientDataEmpty #==# ("simplex:/invitation#/?v=6-8&smp=" <> url queueStr <> "&e2e=" <> testE2ERatchetParamsStrUri <> "&data=" <> url "{}")
+      contactAddress #==# ("simplex:/contact#/?v=6-8&smp=" <> url queueStr)
+      contactAddressDR #==# ("simplex:/contact#/?v=6-8&smp=" <> url queueStr <> "&e2e=" <> testE2ERatchetParamsStrUri <> "&rk=MDEyMzQ1Njc4OWFiY2RlZg%3D%3D")
+      contactAddress #== ("https://simplex.chat/contact#/?v=6-8&smp=" <> url queueStr)
+      contactAddress2queues #==# ("simplex:/contact#/?v=6-8&smp=" <> url (queueStr <> ";" <> queueStr))
+      contactAddressNew #==# ("simplex:/contact#/?v=6-8&smp=" <> url queueNewStr)
+      contactAddress2queuesNew #==# ("simplex:/contact#/?v=6-8&smp=" <> url (queueNewStr <> ";" <> queueNewStr))
+      contactAddressV6 #==# ("simplex:/contact#/?v=6&smp=" <> url queueStr)
+      contactAddressV6 #== ("https://simplex.chat/contact#/?v=1&smp=" <> url queueStr) -- adjusted to v6
+      contactAddressV6 #== ("https://simplex.chat/contact#/?v=1-2&smp=" <> url queueStr) -- adjusted to v6
+      contactAddressV6 #== ("https://simplex.chat/contact#/?v=2-2&smp=" <> url queueStr)
+      contactAddressClientData #==# ("simplex:/contact#/?v=6-8&smp=" <> url queueStr <> "&data=" <> url "{\"type\":\"group_link\", \"group_link_id\":\"abc\"}")
     it "should serialize / parse queue address, connection invitations and contact addresses as binary" $ do
       smpEncodingTest queue
       smpEncodingTest queueNoQM -- this passes, no queue mode patch in SMPQueueUri encoding
@@ -287,21 +296,21 @@ connectionRequestTests =
       smpEncodingTest queueNew1NoPort
       smpEncodingTest queueV1
       smpEncodingTest queueV1NoPort
-      smpEncodingTest connectionRequest
+      smpEncodingTest (aBinaryConnReq connectionRequest)
       -- smpEncodingTest connectionRequestNoQM -- this fails, because of queue mode patch
-      smpEncodingTest connectionRequestContact -- this passes because of queue mode patch in ConnReqUriData encoding
-      smpEncodingTest connectionRequest1
-      smpEncodingTest connectionRequest2queues
-      smpEncodingTest connectionRequestNew
-      smpEncodingTest connectionRequestNew1
-      smpEncodingTest connectionRequest2queuesNew
-      smpEncodingTest connectionRequestClientDataEmpty
-      smpEncodingTest contactAddress
-      smpEncodingTest contactAddress2queues
-      smpEncodingTest contactAddressNew
-      smpEncodingTest contactAddress2queuesNew
-      smpEncodingTest contactAddressV2
-      smpEncodingTest contactAddressClientData
+      smpEncodingTest (aBinaryConnReq connectionRequestContact) -- this passes because of queue mode patch in ConnReqUriData encoding
+      smpEncodingTest (aBinaryConnReq connectionRequest1)
+      smpEncodingTest (aBinaryConnReq connectionRequest2queues)
+      smpEncodingTest (aBinaryConnReq connectionRequestNew)
+      smpEncodingTest (aBinaryConnReq connectionRequestNew1)
+      smpEncodingTest (aBinaryConnReq connectionRequest2queuesNew)
+      smpEncodingTest (aBinaryConnReq connectionRequestClientDataEmpty)
+      smpEncodingTest (aBinaryConnReq contactAddress)
+      smpEncodingTest (aBinaryConnReq contactAddress2queues)
+      smpEncodingTest (aBinaryConnReq contactAddressNew)
+      smpEncodingTest (aBinaryConnReq contactAddress2queuesNew)
+      smpEncodingTest (aBinaryConnReq contactAddressV6)
+      smpEncodingTest (aBinaryConnReq contactAddressClientData)
     it "should serialize / parse short links" $ do
       CSLContact SLSServer CCTContact srv (LinkKey "0123456789abcdef0123456789abcdef") #==# "https://smp.simplex.im/a#MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY?h=jjbyvoemxysm7qxap7m5d5m35jzv5qq6gnlv7s4rsn7tdwwmuqciwpid.onion&p=5223&c=1234-w"
       CSLContact SLSServer CCTGroup srv (LinkKey "0123456789abcdef0123456789abcdef") #==# "https://smp.simplex.im/g#MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY?h=jjbyvoemxysm7qxap7m5d5m35jzv5qq6gnlv7s4rsn7tdwwmuqciwpid.onion&p=5223&c=1234-w"
@@ -343,6 +352,11 @@ connectionRequestTests =
       Right (inv' :: ConnShortLink 'CMInvitation) <- pure $ strDecode "https://localhost/i#tnUaHYp8saREmyEHR93SBpl8ySHBchOt/LJ1ZQUzxH9Udb0jw5wmJACv5o6oe8e7BsX_hUCUMTSY"
       shortenShortLink [presetSrv] inv `shouldBe` inv'
       restoreShortLink [presetSrv] inv' `shouldBe` inv
+    it "should serialize and parse service RPC agent messages" $ do
+      let qInfo = SMPQueueInfo currentSMPClientVersion queueAddr
+      smpEncodingTest $ AgentServiceRequest [qInfo] Nothing "service request payload"
+      smpEncodingTest $ AgentServiceResponse "service response payload"
+      smpEncodingTest $ AgentRejection "rejected: not allowed"
   where
     smpEncodingTest :: (Encoding a, Eq a, Show a, HasCallStack) => a -> Expectation
     smpEncodingTest a = smpDecode (smpEncode a) `shouldBe` Right a