Merge remote-tracking branch 'origin/master' into sh/queue-map-leak

# Conflicts: # tests/AgentTests/FunctionalAPITests.hs
smp: reduce min supported version to 15 (short links support, 3/30/2025) (#1931 )
2026-10-12 02:22:45 +00:00 · 2026-10-06 07:32:52 +00:00 · 2026-10-05 11:31:57 +01:00 · 2026-10-05 07:57:57 +00:00 · 2026-10-05 07:44:32 +00:00 · 2026-10-05 08:07:04 +01:00
79 changed files with 3930 additions and 3031 deletions
@@ -320,15 +320,14 @@ jobs:
      - name: Clone project
        uses: actions/checkout@v3

-      - name: Set up Python
-        # Matches the runtime stage of scripts/resolver/service/Dockerfile.
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.13"
+      - name: Set up uv
+        # uv takes Python from .python-version, the version of the Dockerfile's runtime stage.
+        uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0

-      - name: Install resolver dependencies
-        # Must match scripts/resolver/service/pyproject.toml.
-        run: python -m pip install "eth-hash[pycryptodome]>=0.7"
+      - name: Lint
+        working-directory: scripts/resolver/service
+        run: uv run --frozen ruff check .

      - name: Test
-        run: python -m unittest discover -s scripts/resolver/service -v
+        working-directory: scripts/resolver/service
+        run: uv run --frozen pytest -v
@@ -1,4 +1,8 @@
 *.lock
+# the resolver's Docker image and CI install exactly these
+!/scripts/resolver/service/uv.lock
+# local overrides of scripts/resolver/.env.example
+/scripts/resolver/.env
 *.db
 *.db.bak
 *.session.sql
@@ -719,23 +719,32 @@ Small_random (small * out, void *random_ctx, sntrup761_random_func * random)

 /* ----- Streamlined NTRU Prime Core */

+/* x^p-x-1 has a degree-19 factor mod 3, so a random g is not invertible in R3
+   with probability about 3^-19; KeyGen_attempts failures in a row mean a broken RNG */
+#define KeyGen_attempts 10
+
 /* h,(f,ginv) = KeyGen() */
-static void
+/* returns 0 if KeyGen succeeded; else -1 */
+static int
 KeyGen (Fq * h, small * f, small * ginv, void *random_ctx,
        sntrup761_random_func * random)
 {
  small g[p];
  Fq finv[p];
+  int i;

-  for (;;)
+  for (i = 0; i < KeyGen_attempts; ++i)
    {
      Small_random (g, random_ctx, random);
      if (R3_recip (ginv, g) == 0)
        break;
    }
+  if (i == KeyGen_attempts)
+    return -1;
  Short_random (f, random_ctx, random);
  Rq_recip3 (finv, f);          /* always works */
  Rq_mult_small (h, finv, g);
+  return 0;
 }

 /* c = Encrypt(r,h) */
@@ -884,18 +893,21 @@ typedef small Inputs[p];        /* passed by reference */
 #define PublicKeys_bytes Rq_bytes

 /* pk,sk = ZKeyGen() */
-static void
+/* returns 0 if KeyGen succeeded; else -1 */
+static int
 ZKeyGen (unsigned char *pk, unsigned char *sk, void *random_ctx,
         sntrup761_random_func * random)
 {
  Fq h[p];
  small f[p], v[p];

-  KeyGen (h, f, v, random_ctx, random);
+  if (KeyGen (h, f, v, random_ctx, random) != 0)
+    return -1;
  Rq_encode (pk, h);
  Small_encode (sk, f);
  sk += Small_bytes;
  Small_encode (sk, v);
+  return 0;
 }

 /* C = ZEncrypt(r,pk) */
@@ -960,19 +972,21 @@ HashSession (unsigned char *k, int b, const unsigned char *y,
 /* ----- Streamlined NTRU Prime */

 /* pk,sk = KEM_KeyGen() */
-void
+int
 sntrup761_keypair (unsigned char *pk, unsigned char *sk, void *random_ctx,
                   sntrup761_random_func * random)
 {
  int i;

-  ZKeyGen (pk, sk, random_ctx, random);
+  if (ZKeyGen (pk, sk, random_ctx, random) != 0)
+    return -1;
  sk += SecretKeys_bytes;
  for (i = 0; i < PublicKeys_bytes; ++i)
    *sk++ = pk[i];
  random (random_ctx, Inputs_bytes, sk);
  sk += Inputs_bytes;
  Hash_prefix (sk, 4, pk, PublicKeys_bytes);
+  return 0;
 }

 /* c,r_enc = Hide(r,pk,cache); cache is Hash4(pk) */
@@ -19,7 +19,8 @@

 typedef void sntrup761_random_func (void *ctx, size_t length, uint8_t *dst);

-void
+/* returns 0 on success, -1 if the RNG never produced an invertible polynomial */
+int
 sntrup761_keypair (uint8_t *pk, uint8_t *sk,
                   void *random_ctx, sntrup761_random_func *random);

@@ -7,6 +7,7 @@ This file provides guidance on coding style and approaches and on building the c
 When designing code and planning implementations:
 - Apply adversarial thinking, and consider what may happen if one of the communicating parties is malicious.
 - Formulate an explicit threat model for each change - who can do which undesirable things and under which circumstances.
+- Use the default cryptographic primitive for each purpose from the [primitive registry](../protocol/crypto-registry.md); any exception requires review, and the registry must be updated with every new primitive use or domain-separation string.

 ## Code Quality Standards

@@ -144,9 +144,7 @@ space, so it is workload-dependent; `fillfactor = 80` reached 100% for this frac
 larger share of queues at once would need a lower value. `fillfactor` applies to pages rewritten after
 the change, so the ratio ramps up as the heap turns over, or immediately after `pg_repack`.

-Dropping the index regresses the deprecated journal message store's expiration (`foldRecentQueueRecs`,
-`Journal.hs:434`) to a sequential scan; this is accepted because that store is being retired. The
-Postgres message store does not read `updated_at`.
+The Postgres message store does not read `updated_at`, so no query depends on the dropped index.

 Autovacuum reloptions applied by the migration:

@@ -85,7 +85,7 @@ SMP agent protocol has 2 main parts:

 ![Duplex connection procedure](./diagrams/duplex-messaging/duplex-creating.svg)

-The procedure of establishing a duplex connection is explained on the example of Alice and Bob creating a bi-directional connection consisting of two unidirectional (simplex) queues, using SMP agents (A and B) to facilitate it, and two different SMP routers (which could be the same router). It is shown on the diagram above and has these steps:
+The procedure of establishing a duplex connection is explained on the example of Alice (the initiating party) and Bob (the joining party) creating a bi-directional connection consisting of two unidirectional (simplex) queues, using SMP agents (A and B) to facilitate it, and two different SMP routers (which could be the same router). It is shown on the diagram above and has these steps:

 1. Alice requests the new connection from the SMP agent A using agent `createConnection` api function.
 2. Agent A creates an SMP queue on the router (using [SMP protocol](./simplex-messaging.md) `NEW` command) and responds to Alice with the invitation that contains queue information and the encryption keys Bob's agent B should use. The invitation format is described in [Connection link](connection-link-1-time-invitation-and-contact-address).
@@ -208,7 +208,7 @@ This syntax of decrypted SMP client message body is defined by `decryptedAgentMe
 Decrypted SMP message client body can be one of 4 types:
 - `agentConnInfo` - used by the initiating party when confirming reply queue - sent in `agentConfirmation` envelope.
 - `agentConnInfoReply` - used by accepting party, includes reply queue(s) in the initial confirmation - sent in `agentConfirmation` envelope.
- `agentRatchetInfo` - used to pass additional information when renegotiating double ratchet encryption - sent in `agentRatchetKey` envelope.
+- `agentRatchetInfo` - used to pass additional information when renegotiating double ratchet encryption - sent in `agentRatchetKey` envelope. A key sent in reply to another key includes the hash of that key; agents do not reply to such keys.
 - `agentMessage` - all other agent messages.

 `agentMessage` contains these parts:
@@ -233,7 +233,8 @@ connInfo = *OCTET
 agentConnInfoReply = %s"D" smpQueues connInfo
 smpQueues = length 1*newQueueInfo ; NonEmpty list of reply queues
 agentRatchetInfo = %s"R" ratchetInfo
-ratchetInfo = *OCTET
+ratchetInfo = [answeredKeyHash *OCTET] ; bytes after answeredKeyHash are ignored
+answeredKeyHash = %s"0" / (%s"1" shortString) ; "0" in a key that starts renegotiation, otherwise SHA-256 of the two raw public keys of the answered key

 agentMessage = %s"M" agentMsgHeader aMessage
 agentMsgHeader = agentMsgId prevMsgHash
@@ -0,0 +1,189 @@
+Revision 1, 2026-10-01
+
+# SimpleX Network: cryptographic primitive registry
+
+The cryptographic primitives, constructions and domain-separation strings used by this repository, and which of them new code may use. For the threat model see [Security](./security.md).
+
+Module aliases follow the codebase: `C` = [Simplex.Messaging.Crypto](../src/Simplex/Messaging/Crypto.hs), `LC` = [Crypto.Lazy](../src/Simplex/Messaging/Crypto/Lazy.hs), `CR` = [Crypto.Ratchet](../src/Simplex/Messaging/Crypto/Ratchet.hs), `SL` = [Crypto.ShortLink](../src/Simplex/Messaging/Crypto/ShortLink.hs), `KEM` = [Crypto.SNTRUP761](../src/Simplex/Messaging/Crypto/SNTRUP761.hs) and [its bindings](../src/Simplex/Messaging/Crypto/SNTRUP761/Bindings.hs), `BBS` = [Crypto.BBS](../src/Simplex/Messaging/Crypto/BBS.hs). Lengths are in bytes.
+
+## Table of contents
+
+- [Policy](#policy)
+- [Defaults for new code](#defaults-for-new-code)
+- [Primitives](#primitives)
+- [SimpleX box constructions](#simplex-box-constructions)
+- [Domain separation and KDF inputs](#domain-separation-and-kdf-inputs)
+- [Nonces and IVs](#nonces-and-ivs)
+- [Implementation sources](#implementation-sources)
+- [xftp-web (TypeScript)](#xftp-web-typescript)
+- [Tests](#tests)
+
+## Policy
+
+- New code uses the default primitive for its purpose from [Defaults for new code](#defaults-for-new-code).
+- Using any other primitive, or a primitive marked *restricted* outside its listed purpose, requires a written justification in the pull request and review by a maintainer responsible for cryptography.
+- *Legacy-only* primitives stay only for the existing wire formats and stored data listed below. Do not add call sites.
+- Every new HKDF info string or hash prefix must be unique, start with `SimpleX`, and be added to [Domain separation and KDF inputs](#domain-separation-and-kdf-inputs).
+- Changing a primitive, key length, KDF input, info string or nonce construction changes the wire format. It requires a protocol version and an update of this registry and of the affected `protocol/` specification.
+
+Status values used below:
+
+| Status | Meaning |
+|---|---|
+| Default | Approved and preferred for new code for this purpose |
+| Approved | Approved for new code for this purpose when the default does not fit |
+| Restricted | Approved only for the listed purpose (external interoperability or a single protocol) |
+| Legacy-only | Kept for existing wire formats or stored data; no new call sites |
+
+## Defaults for new code
+
+| Purpose | Default | Functions |
+|---|---|---|
+| Signature | Ed25519 | `C.sign'`, `C.verify'` with `C.SEd25519` |
+| Key agreement | X25519 | `C.generateKeyPair`, `C.dh'` |
+| Post-quantum KEM | sntrup761 | `KEM.sntrup761Keypair`, `sntrup761Enc`, `sntrup761Dec` |
+| Hybrid DH + KEM secret | HKDF-SHA512 over `dh \|\| kemSecret` with a new info string, as in `CR.rootKdf` | `C.hkdf` |
+| Authenticated encryption | XSalsa20-Poly1305 [SimpleX secretbox](#simplex-box-constructions) | `C.sbEncrypt`/`C.sbDecrypt`, `LC.sbEncryptTailTag` for streams |
+| Authenticated encryption with a DH secret | XSalsa20-Poly1305 [SimpleX crypto_box](#simplex-box-constructions) | `C.cbEncrypt`/`C.cbDecrypt` |
+| Forward-secret symmetric session | HKDF-SHA512 key chain | `C.sbcInit`, `C.sbcHkdf` |
+| KDF | HKDF-SHA512 | `C.hkdf` |
+| Hash commitment, derived identifier | SHA3-256 | `C.sha3_256` |
+| Certificate fingerprint | SHA-256 X.509 fingerprint | `C.signedFingerprint`, `C.certificateFingerprint` |
+| Randomness | ChaChaDRG seeded from system entropy | `C.newRandom`, `C.randomBytes` and the `random*` helpers |
+
+## Primitives
+
+| Primitive | Status | Purpose and protocol | Implementation | Lengths |
+|---|---|---|---|---|
+| Ed25519 | Default | SMP/NTF/XFTP queue and file keys (agent default `rcvAuthAlg`, `sndAuthAlg` in [Agent/Env/SQLite.hs](../src/Simplex/Messaging/Agent/Env/SQLite.hs)), short-link signatures and owner auth (`SL.encodeSign`, `SL.newOwnerAuth`), XRCP identity and session keys ([RemoteControl/Invitation.hs](../src/Simplex/RemoteControl/Invitation.hs)), agent service request signatures, client/service TLS certificates ([Transport/Credentials.hs](../src/Simplex/Messaging/Transport/Credentials.hs)) | crypton `Crypto.PubKey.Ed25519` via `C.sign'`/`C.verify'` | pub 32, priv 32, sig 64; X.509 SPKI 44 |
+| Ed448 | Approved | Server CA and online certificates (default `signAlgorithm = ED448` in [Server/CLI.hs](../src/Simplex/Messaging/Server/CLI.hs)); accepted for SMP command signatures and TLS | crypton `Crypto.PubKey.Ed448`; server key generation by the `openssl genpkey` CLI | pub 57, priv 57, sig 114; SPKI 69 |
+| X25519 | Default | Per-queue e2e and server-to-recipient `crypto_box` keys, SMP session DH, command authenticator (SMP v7+), proxy (PRXY/PFWD), notification tokens, XFTP chunk transport, XRCP hello and announcements | crypton `Crypto.PubKey.Curve25519` via `C.dh'` | pub 32, priv 32, secret 32; SPKI 44 |
+| X448 | Restricted | E2E double ratchet only: X3DH and DH ratchet (`CR.RatchetX448`, E2E v3) | crypton `Crypto.PubKey.Curve448` | pub 56, priv 56, secret 56 |
+| sntrup761 | Default | PQ KEM in double ratchet (E2E v3, agent v5+) and XRCP v1 hello | Vendored C [cbits/sntrup761.c](../cbits/sntrup761.c) via FFI; randomness from ChaChaDRG through `haskell_rng_func` ([Bindings/RNG.hs](../src/Simplex/Messaging/Crypto/SNTRUP761/Bindings/RNG.hs)) | pk 1158, sk 1763, ct 1039, shared 32 |
+| XSalsa20-Poly1305, SimpleX crypto_box | Default | SMP message bodies (server to recipient), per-queue e2e envelope, confirmations, proxy forwarding, notifications, XRCP hello, XFTP chunk transport | crypton `Crypto.Cipher.XSalsa`, `Crypto.MAC.Poly1305`; `C.cryptoBox`, `LC.cbInit` | key 32 (DH secret), nonce 24, tag 16 |
+| XSalsa20-Poly1305, SimpleX secretbox | Default | SMP transport block encryption (SMP v11+), short-link data (SMP v15+), XFTP file encryption, local file encryption ([Crypto/File.hs](../src/Simplex/Messaging/Crypto/File.hs)), XRCP session | same; `C.sbEncrypt`, `LC.sbEncryptTailTag`, `LC.sbInit` | key 32, nonce 24, tag 16 |
+| crypto_box authenticator | Approved | Deniable sender command authorization with X25519 queue keys (SMP v7+); agent currently creates Ed25519 keys | `C.cbAuthenticate`, `C.cbVerify`: `crypto_box(sha512(msg))` | 80 = 64 + 16 |
+| AES-256-GCM, 16-byte IV | Legacy-only | Double ratchet header and body (E2E v3, [pqdr.md](./pqdr.md)) | crypton `Crypto.Cipher.AES`, `Crypto.Cipher.Types`; `C.encryptAEAD`, `C.decryptAEAD`, `C.initAEAD`; J0 = GHASH(IV) as NIST SP 800-38D defines for non-96-bit IVs | key 32, IV 16, tag 16; padded header 88 (PQ off) or 2310 (PQ on), `CR.paddedHeaderLen` |
+| AES-256-GCM, 12-byte IV | Restricted | WebRTC frame encryption in simplex-chat (`Simplex.Chat.Mobile.WebRTC`), for WebCrypto interoperability | `C.encryptAESNoPad`, `C.decryptAESNoPad`, `C.initAEADGCM`, `C.GCMIV` | key 32, IV 12, tag 16 |
+| HKDF-SHA512 | Default | All KDFs listed in [Domain separation](#domain-separation-and-kdf-inputs) | crypton `Crypto.KDF.HKDF` with `SHA512`; `C.hkdf` (extract + expand) | output per use, at most 255 * 64 |
+| SHA-256 | Default for X.509 fingerprints, otherwise Restricted | X.509 fingerprints (`C.KeyHash`, server identity, XRCP CA, service certificate hash, SMP v16+), XFTP chunk digests, agent message hashes, `requestCode`, ratchet key dedup hash, security code `codeAD = sha256(rcAD)` | crypton; `C.sha256Hash`, `LC.sha256Hash`, `getFingerprint ... HashSHA256` | 32 |
+| SHA-512 | Restricted | XFTP file digests and file description hash, input of the crypto_box authenticator | crypton; `C.sha512Hash`, `LC.sha512Hash` | 64 |
+| SHA-512 (inside sntrup761) | Restricted | sntrup761 internal hash | [cbits/sha512.c](../cbits/sha512.c) calls OpenSSL `SHA512()` from the system `libcrypto` (`extra-libraries: crypto`) | 64 |
+| SHA3-256 | Default | Short-link key `sha3_256(fixedData)` (SMP v15+), XRCP hybrid secret, service request binding (agent v8) | crypton; `C.sha3_256`, `KEM.kemHybridSecret` | 32 |
+| SHA3-384 | Restricted | Client-supplied sender ID: first 24 bytes of `sha3_384(corrId)`, computed by the agent (`prepareConnectionLink'`, `newRcvConnSrv`) and the server (`createQueue`) | crypton; `C.sha3_384` | 48, truncated to 24 |
+| Keccak-256 | Restricted | Label hash for SMP name queries (`NameQuery NQHash`), must match the on-chain registry | crypton `Keccak_256`; `labelHash` in [SimplexName.hs](../src/Simplex/Messaging/SimplexName.hs) | 32 |
+| MD5 | Legacy-only, non-security | XOR-aggregated queue ID hash `IdsHash` for service subscription reconciliation (SUBS, NSUBS, SOKS, ENDS) | crypton (`C.md5Hash`, `Protocol.queueIdHash`); SQLite UDF `simplex_xor_md5_combine` ([Agent/Store/SQLite.hs](../src/Simplex/Messaging/Agent/Store/SQLite.hs)); PostgreSQL pgcrypto `digest(..., 'md5')` in server and NTF schemas | 16 |
+| ChaChaDRG | Default | Keys, nonces, correlation IDs, random IDs, sntrup761 randomness | crypton `Crypto.Random`; `C.newRandom` seeds with `drgNew` from system entropy | n/a |
+| BBS+ over BLS12-381, SHA-256 suite | Restricted | Badge entitlement credentials and proofs ([Crypto/Entitlement.hs](../src/Simplex/Messaging/Crypto/Entitlement.hs)), XFTP storage-time extension | Vendored submodules libbbs and blst (C and assembly) via FFI; randomness from libbbs `getentropy`, `SecRandomCopyBytes` with the `commoncrypto` flag, `BCryptGenRandom` on Windows ([cbits/getentropy_win.c](../cbits/getentropy_win.c)) | sk 32, pk 96, sig 80, proof 272 + 32 per undisclosed message |
+| ECDSA with SHA-256 (ES256) | Restricted | APNS provider JWT ([Push/APNS.hs](../src/Simplex/Messaging/Notifications/Server/Push/APNS.hs)) | crypton `Crypto.PubKey.ECC.ECDSA.sign`; key read by cryptostore `Crypto.Store.PKCS8`; curve taken from the key file | signature DER `SEQUENCE {r, s}`, base64url |
+| TLS 1.3 / 1.2 | Restricted | SMP, XFTP, NTF, XRCP transport: `TLS_CHACHA20_POLY1305_SHA256` (1.3), `ECDHE_ECDSA_CHACHA20POLY1305_SHA256` (1.2), groups X448 and X25519, Ed448 and Ed25519 signatures (`defaultSupportedParams`) | `tls` 1.9 | n/a |
+| TLS for browsers | Restricted | XFTP server with HTTPS credentials (`defaultSupportedParamsHTTPS`: `ciphersuite_strong`, FFDHE, P-521, ECDSA and RSA signatures); SMP server HTTPS requires RSA-4096 (`checkHTTPSCredentials`) | `tls`, `warp-tls` | n/a |
+| X.509 | Restricted | Certificate chains, fingerprints, signed session keys (`C.signX509`, `C.verifyX509`) | `crypton-x509`, `crypton-x509-validation`, `cryptostore` | n/a |
+| SQLCipher | Restricted | Agent SQLite database at rest (`PRAGMA key`); cipher parameters are SQLCipher defaults, not set in this repository | `direct-sqlcipher` (git dependency, [cabal.project](../cabal.project)) | n/a |
+
+## SimpleX box constructions
+
+`C.cryptoBox` and `LC.sbInit_` compute, for a 32-byte key `k` and 24-byte nonce `n`:
+
+```
+k1 = HSalsa20(k, 0^16)
+subkey = HSalsa20(k1, n[0..16])
+stream = Salsa20(subkey, n[16..24])
+polyKey = stream[0..32]
+ct = msg XOR stream[32..]
+tag = Poly1305(polyKey, ct)
+```
+
+| Construction | Key `k` | Equivalent libsodium call | Layout |
+|---|---|---|---|
+| SimpleX crypto_box (`C.cbEncrypt`, `LC.cbInit`) | X25519 shared secret | `crypto_box_easy` (`crypto_box_beforenm` is `HSalsa20(dh, 0^16)`) | strict: `tag \|\| ct`; lazy tail-tag: `ct \|\| tag` |
+| SimpleX secretbox (`C.sbEncrypt`, `LC.sbInit`, `KEM.kcb*`) | 32-byte symmetric key | `crypto_secretbox_easy` with key `crypto_core_hsalsa20(0^16, k)`, not with `k` | same |
+
+Padding before encryption: `C.pad` prefixes a 2-byte big-endian length and fills with `#` (message at most 65533 bytes); `LC.pad` prefixes an 8-byte length. `*NoPad` variants skip padding.
+
+## Domain separation and KDF inputs
+
+HKDF is `C.hkdf salt ikm info len` (HKDF-SHA512).
+
+| Info string | Salt | IKM | Output (split) | Function | Use |
+|---|---|---|---|---|---|
+| `"SimpleXX3DH"` | 64 zero bytes | `dh1 \|\| dh2 \|\| dh3 [\|\| kemShared]` | 96: `hk`, `nhk`, root key (32 each) | `CR.pqX3dh` | Ratchet initialization, added in E2E v2; KEM secret from E2E v3 (current minimum) |
+| `"SimpleXVerifyCode"` | 64 zero bytes | same as `SimpleXX3DH` | 32: `rcVCPQ` | `CR.pqX3dh` | Verification code covering all handshake keys, new ratchets |
+| `"SimpleXRootRatchet"` | root key | `dh [\|\| kemShared]` | 96: root key, chain key, next header key | `CR.rootKdf` | DH/PQ ratchet step |
+| `"SimpleXChainRatchet"` | empty | chain key | 96: chain key, message key (32 each), message IV (16), header IV (16) | `CR.chainKdf` | Per-message keys |
+| `"SimpleXSbChainInit"` | SMP session ID | X25519 session secret | 64: two 32-byte chain keys | `C.sbcInit` from `Transport.blockEncryption` | SMP transport block encryption, SMP v11+ |
+| `"SimpleXSbChainInit"` | empty | XRCP hybrid secret (below) | 64: two 32-byte chain keys | `C.sbcInit` in [RemoteControl/Client.hs](../src/Simplex/RemoteControl/Client.hs) | XRCP v1 session |
+| `"SimpleXSbChain"` | empty | chain key | 88: chain key (32), secretbox key (32), nonce (24) | `C.sbcHkdf` | Each SMP block and XRCP message |
+| `"SimpleXContactLink"` | empty | link key (32) | 56: link ID (24), secretbox key (32) | `SL.contactShortLinkKdf` | Contact short links, SMP v15+ |
+| `"SimpleXInvLink"` | empty | link key (32) | 32: secretbox key | `SL.invShortLinkKdf` | Invitation short links, SMP v15+ |
+
+Other derivations:
+
+| Value | Construction | Function | Use |
+|---|---|---|---|
+| Service request binding | `sha3_256("SimpleXService" \|\| rcAD)` | `serviceReqBinding` in [Agent.hs](../src/Simplex/Messaging/Agent.hs) | Agent RPC, agent v8 |
+| BBS header | `"SimpleX badges v1"` | `entitlementBBSHeader` | Badge credentials |
+| XRCP hybrid secret | `sha3_256(x25519Dh \|\| kemShared)` | `KEM.kemHybridSecret` | XRCP v1 |
+| Short-link key | `sha3_256(smpEncode FixedLinkData)` | `SL.encodeSignFixedData`, checked in `SL.decryptLinkData` | SMP v15+ |
+| Sender ID | `take 24 (sha3_384 corrId)` | `prepareConnectionLink'`, `newRcvConnSrv` in Agent.hs; `createQueue` in [Server.hs](../src/Simplex/Messaging/Server.hs) | Client-supplied sender ID with link data, SMP v15+; the server rejects a mismatch to prevent an ID oracle |
+| Ratchet associated data | `pubKeyBytes sk1 \|\| pubKeyBytes rk1` (raw X448, 112) | `CR.pqX3dh` | AD of every ratchet AEAD |
+| Security code | `sha256(rcAD)` | `ratchetVerifyCodes` in [AgentStore.hs](../src/Simplex/Messaging/Agent/Store/AgentStore.hs) | Connection verification |
+| Contact request code | `sha256(smpEncode (k1, k2, kem, sndId))` | `requestCode` in Agent.hs | Contact request binding |
+| Command authenticator | `crypto_box(sha512(authorized))` | `C.cbAuthenticate` | SMP v7+ |
+| Queue IDs hash | `xor` of `md5(queueId)` | `Protocol.queueIdsHash` | Service subscriptions |
+
+`"SimpleXSbChainInit"` and `"SimpleXSbChain"` are shared by SMP block encryption and XRCP. Their outputs are separated only by the IKM (and the salt for `sbcInit`).
+
+## Nonces and IVs
+
+| Use | Construction |
+|---|---|
+| SMP command authenticator, proxied command | `C.cbNonce corrId`, where `corrId` is a random 24-byte nonce (`C.randomCbNonce`) |
+| SMP proxied responses | `C.reverseNonce` of the request nonce |
+| Server-to-recipient message body | `C.cbNonce msgId`; `msgIdBytes = 24` in [Server/Main.hs](../src/Simplex/Messaging/Server/Main.hs) |
+| Per-queue e2e envelope, short-link data, notifications, XRCP hello, XFTP chunk download | Random 24-byte nonce, sent with the ciphertext |
+| XFTP file, local encrypted file | Random key and nonce per file (`C.randomSbKey`, `C.randomCbNonce`) |
+| SMP block, XRCP session messages | Key and nonce from `C.sbcHkdf`, one per block |
+| Ratchet header, body | 16-byte IVs from `CR.chainKdf`; header IV is sent, body IV is not |
+| WebRTC frames | 12-byte `C.GCMIV` supplied by the caller in simplex-chat |
+
+## Implementation sources
+
+| Source | Version or pin | Provides |
+|---|---|---|
+| crypton | 0.34 | Ed25519, Ed448, X25519, X448, XSalsa20, Poly1305, AES-GCM, HKDF, SHA-2, SHA-3, Keccak, MD5, ChaChaDRG, ECDSA |
+| tls, crypton-x509, crypton-x509-validation, cryptostore | 1.9.0, 1.7.6, 1.6.12, 0.3.0.1 | TLS, X.509, PKCS#8 |
+| [cbits/sntrup761.c](../cbits/sntrup761.c) | Copy of draft-josefsson-ntruprime-streamlined-00 ([cbits/README.md](../cbits/README.md)) | sntrup761 |
+| [cbits/sha512.c](../cbits/sha512.c) and system OpenSSL `libcrypto` | system | SHA-512 for sntrup761 |
+| `cbits/libbbs` submodule ([simplex-chat/libbbs](https://github.com/simplex-chat/libbbs)) | `59a0f4bf` | BBS+ (`bbs_sha256_ciphersuite`), SHA-256, SHAKE256 |
+| `cbits/blst` submodule ([supranational/blst](https://github.com/supranational/blst)) | `db3defd0` | BLS12-381 (C and `build/assembly.S`, `-D__BLST_PORTABLE__`) |
+| direct-sqlcipher | git `f814ee68` | SQLCipher |
+| `openssl` CLI | system | Server CA and certificate key generation (`createServerX509_`) |
+
+## xftp-web (TypeScript)
+
+[xftp-web](../xftp-web/) reimplements a subset for the browser XFTP client. It has no AES-GCM, HKDF, SHA-3, MD5, X448 or sntrup761 code.
+
+| Primitive | Implementation | Haskell counterpart |
+|---|---|---|
+| Ed25519 sign, verify, keys | libsodium-wrappers-sumo (`crypto_sign_*`), [src/crypto/keys.ts](../xftp-web/src/crypto/keys.ts) | `C.sign'`, `C.verify'` |
+| Ed448 verify | `@noble/curves/ed448`, keys.ts `verifyEd448` | `C.verify'` |
+| X25519 | libsodium `crypto_scalarmult`, `crypto_box_keypair` | `C.dh'` |
+| SimpleX crypto_box, secretbox, tail-tag streaming | Salsa20 block function written in TypeScript, libsodium `crypto_core_hsalsa20` and `crypto_onetimeauth_*`, [src/crypto/secretbox.ts](../xftp-web/src/crypto/secretbox.ts) | `C.cryptoBox`, `LC.sbInit_` |
+| crypto_box authenticator | [src/protocol/client.ts](../xftp-web/src/protocol/client.ts) `cbAuthenticate`, `cbVerify` | `C.cbAuthenticate`, `C.cbVerify` |
+| SHA-256, SHA-512 | libsodium `crypto_hash_sha256`, `crypto_hash_sha512*`, [src/crypto/digest.ts](../xftp-web/src/crypto/digest.ts) | `C.sha256Hash`, `C.sha512Hash` |
+| Random | WebCrypto `crypto.getRandomValues`; libsodium for key generation | `C.randomBytes` |
+
+## Tests
+
+None of the tests below compares against published reference vectors (NIST, RFC 8032, RFC 7748, RFC 5869, NaCl, the sntrup761 draft or the BBS draft). Interoperability is tested between this repository's own implementations.
+
+| Area | Test module (hspec group) | Kind |
+|---|---|---|
+| Ed25519, Ed448, X25519 crypto_box, secretbox, lazy and tail-tag secretbox, AES-GCM 12-byte IV, X.509 key encoding, X.509 chains, sntrup761, BBS+, entitlements, padding | [tests/CoreTests/CryptoTests.hs](../tests/CoreTests/CryptoTests.hs) | Round-trip; fixed lengths for BBS+; fixed bytes for padding |
+| Double ratchet (X25519 and X448), PQ KEM agreement, AES-GCM 16-byte IV | [tests/AgentTests/DoubleRatchetTests.hs](../tests/AgentTests/DoubleRatchetTests.hs) | Round-trip; decoding of a stored v2 ratchet JSON |
+| Short links (HKDF info strings, SHA3-256 link key) | [tests/AgentTests/ShortLinkTests.hs](../tests/AgentTests/ShortLinkTests.hs) | Round-trip, tamper rejection |
+| File encryption | [tests/CoreTests/CryptoFileTests.hs](../tests/CoreTests/CryptoFileTests.hs) | Round-trip |
+| XRCP session (SHA3-256 hybrid, sb chain) | [tests/RemoteControl.hs](../tests/RemoteControl.hs) | End-to-end |
+| SMP authenticators, block encryption, `IdsHash` (MD5) | [tests/ServerTests.hs](../tests/ServerTests.hs) | End-to-end; with the PostgreSQL queue store this checks Haskell MD5 against pgcrypto |
+| Haskell and xftp-web: SHA-256/512, Ed25519, Ed448 identity proof, X25519, DER keys, crypto_box, secretbox, tail-tag, authenticator, file encryption | [tests/XFTPWebTests.hs](../tests/XFTPWebTests.hs) (`XFTP Web Client`) | Byte-identical cross-language output |
@@ -17,11 +17,11 @@ note over CC, HC: TLS handshake
 par
    note over CC: validate client X509 credentials
    CC->>CI: session code from tlsUnique
-    CI-->>CC: user confirmation
+    CI-->>CC: user confirmation (optional if known)
 and
    note over HC: validate server X509 credentials
    HC->>HI: session code from tlsUnique
-    HI-->>HC: user confirmation
+    HI-->>HC: user confirmation (optional if known)
 end

 note over CI, HI: 3. Session verification and protocol negotiation
@@ -62,14 +62,16 @@ It is possible to reduce size overhead by using only one KEM agreement and makin

 ## Double ratchet with encrypted headers augmented with double PQ KEM

-Algorithm below assumes that in addition to shared secret from the initial key agreement, there will be an encapsulation key available from the party that published its keys (Bob).
+Algorithm below assumes that in addition to shared secret from the initial key agreement, there will be an encapsulation key available from the initiating party, that sent its keys in the connection invitation to the joining party (see [agent protocol](./agent-protocol.md)). Following the double ratchet specification, the pseudo-code below names the joining party Alice and the initiating party Bob.
+
+Unlike X3DH prekey bundles, these keys are not reusable keys published for any party to use. The initiating party generates two X448 key pairs and, optionally, a sntrup761 KEM key pair for each connection, sends the public keys in the invitation, and deletes the stored private keys once the ratchet is initialized; only the second X448 private key and the agreed KEM key pair remain in the ratchet state as its initial ratchet keys. The joining party generates its keys when joining and keeps only its KEM key pair, in the ratchet state. The exception is a contact address that publishes ratchet keys in its link data: all requesters use these keys until the address owner rotates them, and the owner keeps the private keys of a few recent generations.

 ### Initialization

 The double ratchet initialization is defined in pseudo-code. This pseudo-code is identical to Signal algorithm specification except for that parts that add post-quantum key agreement.

 ```
-// Alice obtained Bob's keys and initializes ratchet first
+// Alice (joining party) received Bob's keys in the invitation and initializes ratchet first
 def RatchetInitAlicePQ2HE(state, SK, bob_dh_public_key, shared_hka, shared_nhkb, bob_pq_kem_encapsulation_key):
    state.DHRs = GENERATE_DH()
    state.DHRr = bob_dh_public_key
@@ -89,7 +91,7 @@ def RatchetInitAlicePQ2HE(state, SK, bob_dh_public_key, shared_hka, shared_nhkb,
    state.HKr = None
    state.NHKr = shared_nhkb

-// Bob initializes ratchet second, having received Alice's connection request
+// Bob (initiating party) initializes ratchet second, having received Alice's first message
 def RatchetInitBobPQ2HE(state, SK, bob_dh_key_pair, shared_hka, shared_nhkb, bob_pq_kem_key_pair):
    state.DHRs = bob_dh_key_pair
    state.DHRr = None
@@ -210,6 +212,8 @@ The outer envelope contains the encrypted header (used as associated data for bo

 The message body is encrypted with AES-256-GCM using the message key derived from the sending chain key (`KDF_CK`). The associated data for body encryption is the concatenation of the ratchet associated data and the encoded encrypted header.

+`KDF_CK(CK)` is HKDF-SHA512 with empty salt, `CK` as input key material and info `"SimpleXChainRatchet"`, producing 96 bytes split into the next chain key (32 bytes), the message key (32 bytes), the message body IV (16 bytes, not transmitted) and `headerIV` (16 bytes). Both IVs are used as 16-byte AES-256-GCM IVs, not the 12-byte IVs recommended by NIST SP 800-38D, so the initial counter block is J0 = GHASH(IV || 0^64 || [128]_64) as defined there for non-96-bit IVs. WebCrypto and other conforming implementations compute it when given the full 16-byte IV; truncating the IV to 12 bytes produces different ciphertext.
+
 ```abnf
 encRatchetMessage = versionedLength encMessageHeader msgAuthTag encMsgBody
 ; encMessageHeader is used as associated data for body decryption: AD = rcAD || encMessageHeader
@@ -274,7 +278,7 @@ As SimpleX Messaging Protocol pads messages to a fixed size, using 16kb transpor

 Sharing the initial keys in case of SimpleX Chat it is equivalent to sharing the invitation link. As encapsulation key is large, it may be inconvenient to share it in the link in some contexts, e.g. when QR codes are used.

-It is possible to postpone sharing the encapsulation key until the first message from Alice (confirmation message in SMP protocol), the party sending connection request. The upside here is that the invitation link size would not increase. The downside is that the user profile shared in this confirmation will not be encrypted with PQ-resistant algorithm.
+It is possible to postpone sharing the encapsulation key until the first message from the joining party (confirmation message in SMP protocol). The upside here is that the invitation link size would not increase. The downside is that the user profile shared in this confirmation will not be encrypted with PQ-resistant algorithm.

 Another consideration is pairwise ratchets in groups. Key generation in sntrup761 is quite slow - on slow devices it can be as slow as 10-20 keys per second, so using this primitive in groups larger than 10-20 members would result in slow performance.

@@ -15,7 +15,7 @@ This document describes the cryptographic primitives and threat model for the Si
  - [SimpleX Messaging Protocol router that proxies the messages to another SMP router](#simplex-messaging-protocol-router-that-proxies-the-messages-to-another-smp-router)
  - [An attacker who obtained Alice's (decrypted) chat database](#an-attacker-who-obtained-alices-decrypted-chat-database)
  - [A user's contact](#a-users-contact)
-  - [An attacker who observes Alice showing an introduction message to Bob](#an-attacker-who-observes-alice-showing-an-introduction-message-to-bob)
+  - [An attacker who observes the initiating party showing an introduction message to the joining party](#an-attacker-who-observes-the-initiating-party-showing-an-introduction-message-to-the-joining-party)
  - [An attacker with Internet access](#an-attacker-with-internet-access)


@@ -37,6 +37,8 @@ This document describes the cryptographic primitives and threat model for the Si
  - AES-GCM AEAD cipher,
  - SHA512-based HKDF for key derivation.

+All primitives in use, their lengths, domain-separation strings and the policy for new code are listed in the [cryptographic primitive registry](./crypto-registry.md).
+

 ## Threat Model

@@ -168,7 +170,7 @@ This document describes the cryptographic primitives and threat model for the Si

 *cannot:*

- impersonate a sender and send messages to the user whose database was stolen. Doing so requires also compromising the router (to place the message in the queue, that is possible until the Double-Ratchet advances forward) or the user's device at a subsequent time (to place the message in the database).
+- impersonate a sender and send messages to the user whose database was stolen, unless the database contains a new rotation queue that is not secured yet. Doing so requires also compromising the router (to place the message in the queue, that is possible until the Double-Ratchet advances forward) or the user's device at a subsequent time (to place the message in the database).

 - undetectably communicate at the same time as Alice with her contacts. Doing so would result in the contact getting different messages with repeated IDs.

@@ -190,15 +192,15 @@ This document describes the cryptographic primitives and threat model for the Si

 - cannot collaborate with another of the user's contacts to confirm they are communicating with the same user.

-### An attacker who observes Alice showing an introduction message to Bob
+### An attacker who observes the initiating party showing an introduction message to the joining party

 *can:*

- Impersonate Bob to Alice.
+- Impersonate the joining party to the initiating party.

 *cannot:*

- Impersonate Alice to Bob.
+- Impersonate the initiating party to the joining party.

 ### An attacker with Internet access

@@ -86,7 +86,7 @@ It's designed with the focus on communication security and integrity, under the

 It is designed as a low level protocol for other application protocols to solve the problem of secure and private message transmission, making [MITM attack][1] very difficult at any part of the message transmission system.

-This document describes SMP protocol version 22. Versions 1-5 are discontinued. The version history:
+This document describes SMP protocol version 23. Versions 1-5 are discontinued. The version history:

 - v1: binary protocol encoding
 - v2: message flags (used to control notifications)
@@ -109,6 +109,7 @@ This document describes SMP protocol version 22. Versions 1-5 are discontinued.
 - v20: public namespaces resolver (RSLV command, RNAME response) — direct or forwarded via PFWD
 - v21: server public information in handshake
 - v22: `RNAME` says whether a name can be registered, not only what it resolves to
+- v23: version in nonces of forwarded commands, random nonces for forwarded responses

 ## Introduction

@@ -1120,13 +1121,15 @@ When the client receives PKEY response it MUST validate that:
 - the fingerprint of the received certificate matches fingerprint in the router address - it mitigates MITM attack by proxy.
 - the router session key is correctly signed with the received certificate.

+`smpVersionRange` in `PKEY` is not signed.
+
 The proxy router may respond with error response in case the destination router is not available or in case it has an earlier version that does not support proxied commands.

 #### Send command via proxy

 Sender can send `SKEY` and `SEND` commands via proxy after obtaining the session ID with `PRXY` command (see [Request proxied session](#request-proxied-session)).

-Transmission sent to proxy router should use session ID as entity ID and use a random correlation ID of 24 bytes as a nonce for crypto_box encryption of transmission to the destination router. The random ephemeral X25519 key to encrypt transmission should be unique per command, and it should be combined with the key sent by the router in the handshake header to proxy and to the client in `PKEY` command.
+Transmission sent to proxy router should use session ID as entity ID and use a random correlation ID of 24 bytes as a nonce for crypto_box encryption of transmission to the destination router. When `smpVersion` in `PFWD` is 23 or higher, the first 2 bytes of this nonce are XOR-ed with `smpVersion`. The random ephemeral X25519 key to encrypt transmission should be unique per command, and it should be combined with the key sent by the router in the handshake header to proxy and to the client in `PKEY` command.

 Encrypted transmission should use the received session ID from the connection between proxy router and destination router in the authorized body.

@@ -1138,10 +1141,11 @@ commandKey = length x509encoded

 The proxy router will forward the encrypted transmission in `RFWD` command (see below).

-Having received the `RRES` response from the destination router, proxy router will forward `PRES` response to the client. `PRES` response should use the same correlation ID as `PFWD` command. The destination router will use this correlation ID increased by 1 as a nonce for encryption of the response.
+Having received the `RRES` response from the destination router, proxy router will forward `PRES` response to the client. `PRES` response should use the same correlation ID as `PFWD` command. The destination router will use this correlation ID increased by 1 as a nonce for encryption of the response. When `smpVersion` in `PFWD` is 23 or higher, the destination router uses a random nonce instead, and sends it before the encrypted response.

 ```abnf
-proxyResponse = %s"PRES" SP <encrypted padded(forwardedResponse, 16226)>
+proxyResponse = %s"PRES" SP [responseNonce] <encrypted padded(forwardedResponse, 16226)>
+responseNonce = %s"0" / (%s"1" 24*24 OCTET) ; from v23
 forwardedResponse = *OCTET ; client-encrypted SMP response, decrypted by client using per-command DH secret
 ```

@@ -1168,7 +1172,7 @@ The shared secret for encrypting transmission bodies between proxy router and de


 ```abnf
-relayResponse = %s"RRES" SP <encrypted(responseTransmission)>
+relayResponse = %s"RRES" SP [responseNonce] <encrypted(responseTransmission)>
 responseTransmission = fwdCorrId forwardedResponse
  ; fwdCorrId and forwardedResponse defined above in RFWD section
 ```
@@ -71,7 +71,7 @@ The session invitation contains this data:

 Host application decrypts (except the first session) and validates the invitation:
 - Session signature is valid.
- Timestamp is within some window from the current time.
+- Timestamp of a multicast announcement is not earlier than 3660 seconds before and not later than 3600 seconds after the current time of the host. The host ignores announcements outside of this interval and continues listening.
 - Long-term key signature is valid.
 - Long-term CA and signature key are the same as in the first session.
 - Some version in the offered range is supported.
@@ -121,9 +121,15 @@ During TLS handshake, parties validate certificate chains against previously kno

 ### Session verification and protocol negotiation

-Once TLS session is established, both the host and controller devices present a "session security code" to the user who must match them (e.g., visually or via QR code scan) and confirm on the host device. The session security code must be a digest of tlsunique channel binding. As it is computed as a digest of the TLS handshake for both the controller and the host, it will validate that the same TLS certificates are used on both sides, and that the same TLS session is established, mitigating the possibility of MITM attack in the connection.
+Once TLS session is established, both the host and controller devices present a "session security code" to the user. The session security code must be a digest of tlsunique channel binding. As it is computed as a digest of the TLS handshake for both the controller and the host, it will validate that the same TLS certificates are used on both sides, and that the same TLS session is established, mitigating the possibility of MITM attack in the connection.

-Once the session is confirmed by the user, the host sends HELLO block to the controller.
+To verify the session, the user matches the codes (e.g., visually or via QR code scan) and confirms the session on the host device.
+
+The user MUST verify the first session with a new controller. The host stores the long-term identity of the controller (CA fingerprint and Ed25519 public key) only after this session is verified. A controller with the stored long-term identity is a known controller.
+
+The user MAY verify the subsequent sessions with a known controller. In a session that the host confirms without user action, the controller is authenticated only by its long-term identity stored in the first session (see [Threat model](#threat-model)). SimpleX Chat mobile apps require user verification of sessions with known controllers only when the user enables the "Verify connections" option.
+
+Once the session is confirmed, the host sends HELLO block to the controller.

 XRCP blocks inside TLS are padded to 16384 bytes.

@@ -305,7 +311,7 @@ to receive: (rcvKey', sk, nonce) = HKDF(rcvKey, "SimpleXSbChain", 88)
 - prevent host and controller devices from establishing the session

 *cannot:*
- same as passive adversary, provided that user visually verified session code out-of-band.
+- same as passive adversary, provided that user visually verified session code out-of-band in the first session with the controller.

 #### An active adversary with the access to the network:

@@ -324,6 +330,15 @@ to receive: (rcvKey', sk, nonce) = HKDF(rcvKey, "SimpleXSbChain", 88)
 *cannot:*
 - connect to the host or make host connect to itself.

+#### An active adversary with the access to the network who also obtained the long-term private keys of a known controller (e.g., from the controller device):
+
+*can:*
+- make host connect to itself instead of the controller.
+- access any data of the controlled host application, within the capabilities of the provided API, if the host confirms the session without user action.
+
+*cannot:*
+- access host application data, provided that user visually verified session code out-of-band.
+
 #### Compromised controller device:

 *can:*
@@ -1,5 +1,8 @@
+# Copy to .env (next to docker-compose.yml) only to change a setting; without it
+# docker-compose.yml uses the values shown here.
+
 # ============================================================================
-# Required settings — the stack will not start without these.
+# Chain settings.
 # ============================================================================

 # Ethereum network: mainnet (the SNRC `.testing` contracts live on mainnet)
@@ -14,9 +17,16 @@ NETWORK=mainnet
 TRUSTED_NODE_URL=https://mainnet-checkpoint-sync.attestant.io

 # ============================================================================
-# Optional overrides — sensible defaults are baked into docker-compose.yml,
-# so leave these commented unless you need to change them.
+# Optional overrides — leave commented unless you need to change them.
 # ============================================================================

 # Nimbus NAT (default: any). For a stable public node set an explicit IP:
 #   NAT=extip:1.2.3.4        # your public IPv4: curl -s ifconfig.me
+
+# Resolver (defaults in the README's "Load and scaling" and "Logs" sections)
+#   SNRC_WORKERS=4                          # worker processes; default CPU count, at most 4
+#   SNRC_RPC_TIMEOUT=5                      # seconds per request to reth
+#   SNRC_LOG_FORMAT=json                    # default text (key=value)
+#   SNRC_LOG_COLOR=always                   # coloured `docker compose logs`
+#   SNRC_LOG_LEVEL=debug                    # also log health checks and idle connections
+#   SNRC_TRUSTED_PROXIES=172.16.0.0/12      # behind a reverse proxy on the host; see README
@@ -19,15 +19,15 @@ against **Ethereum mainnet** (where the `.testing` contracts live):

 ## 1. Configure

-Edit `.env` — the defaults work as-is; override only if needed:
+The defaults in `docker-compose.yml` work as-is:

 ```sh
 NETWORK=mainnet                                               # default
 TRUSTED_NODE_URL=https://mainnet-checkpoint-sync.attestant.io # default
 ```

-Everything else (NAT) has a working default baked into `docker-compose.yml`;
-uncomment the hints in `.env` only to override.
+To override these or NAT, copy `.env.example` to `.env` and edit it; `.env` is
+not committed.

 ## 2. Run

@@ -121,6 +121,15 @@ standalone for local dev (no Docker), via [`uv`](https://docs.astral.sh/uv/):
 uv run scripts/resolver/service/snrc-resolve.py  # defaults to local reth + mainnet .testing
 ```

+Tests and lint run from `scripts/resolver/service`:
+
+```sh
+uv run pytest
+uv run ruff check .
+```
+
+Dependencies are locked in `uv.lock`; the Docker image installs exactly those.
+
 Three routes, versioned separately from the protocol so each only changes when
 its own shape does:

@@ -243,7 +252,7 @@ wrong, `error` and `status` hold the same value, so one field is enough to read.

 `upstreamError` says only which exception type the RPC call raised. The text
 goes to the resolver's log instead, because `SNRC_RPC` can carry a provider key
-and urlopen puts the URL it failed on into the message. It is also the answer
+and the exception can carry the URL it failed on. It is also the answer
 when a registrar, controller or oracle address has no contract behind it: the
 empty reply is refused rather than read as zero, which would make every name
 look free.
@@ -374,3 +383,87 @@ here.
 To override any of them, set `SNRC_REGISTRY_<TLD>`, `SNRC_REGISTRAR_<TLD>` or
 `SNRC_CONTROLLER_<TLD>` on the `resolver` service in `docker-compose.yml`, or
 as env vars when you run the script directly.
+
+### Load and scaling
+
+A lookup reads the chain in at most three rounds: the name's status and
+registry entries, then its record from its resolver, then prices for a name
+that is free. The node runs the calls of a JSON-RPC batch one after another,
+so the contract reads of a round go to the chain as one `eth_call` to
+[Multicall3](https://github.com/mds1/multicall). A node that is slow for a
+moment therefore delays a lookup a few times, not once per read. Connections to
+the node are kept open and reused.
+
+The resolver answers HTTP/1.1, so each smp-server keeps its connections to it
+open instead of connecting for every lookup. An idle connection is closed after
+60 s; the smp-server drops its own idle ones after 30 s, so it closes them first.
+
+Each access log line ends with how long the request took, so slow requests
+show up in `docker compose logs resolver`.
+
+| Variable | Default | Meaning |
+|---|---|---|
+| `SNRC_WORKERS` | CPU count, at most 4 | processes sharing the port. If one exits, the others stop, so the container restarts |
+| `SNRC_RPC_TIMEOUT` | `5` | seconds to wait for each request to the node; the smp-server gives up after 3 |
+| `SNRC_MULTICALL` | `0xcA11bde05977b3631167028862bE2a173976CA11` | Multicall3 address. If it does not answer, each round is sent as a plain batch and the log says so once |
+
+### Logs
+
+Each event is one line: UTC time, level, event name, then `key=value` fields.
+
+```
+2026-09-26T08:06:19.992Z INFO  request  client=203.0.113.7 worker=12 method=GET path=/v2/resolve/[4fdd…].testing status=200 bytes=701 ms=35
+2026-09-26T08:06:20.051Z WARN  upstream_error  name=foobar.testing error=ConnectionRefusedError message="[Errno 111] Connection refused"
+```
+
+| Event | Level | Meaning |
+|---|---|---|
+| `listening` | info | started: address, workers, RPC endpoint, registries, trusted proxies |
+| `request` | info | one answered request; `/health` only at debug |
+| `upstream_error` | warn | a read from the node failed; the caller got 502 |
+| `multicall_unavailable` | warn | Multicall3 did not answer; rounds are sent as plain batches |
+| `client_gone` | warn | the caller hung up before the answer, usually an smp-server past its timeout |
+| `http_error` | warn | a malformed request; idle keep-alive timeouts only at debug |
+| `request_failed` | error | a request failed unexpectedly, with its traceback |
+| `worker_exited`, `worker_failed` | error | a worker stopped; the others stop too, so the container restarts |
+| `stopping` | info | stopped by a signal |
+
+| Variable | Default | Meaning |
+|---|---|---|
+| `SNRC_LOG_FORMAT` | `text` | `text` (`key=value`) or `json`, one object per line |
+| `SNRC_LOG_COLOR` | `auto` | `auto` colours only a terminal, `always`, `never`. Docker output is no terminal, so set `always` for coloured `docker compose logs` |
+| `SNRC_LOG_LEVEL` | `info` | `debug` also logs health checks and idle connections closing |
+| `SNRC_TRUSTED_PROXIES` | none | addresses or CIDRs of reverse proxies whose `X-Forwarded-For` names the client |
+
+With docker compose, set these, and those in "Load and scaling", in `.env`
+(`.env.example` lists them).
+
+Behind a reverse proxy such as Caddy on the host, the resolver sees the Docker
+network's gateway rather than the client. Trust that gateway, and the logged
+client is the last `X-Forwarded-For` address no trusted proxy added, so a caller
+through the proxy cannot claim another address. Everything else on the host that
+connects to `127.0.0.1:8000`, the smp-servers included, arrives through the same
+gateway and could set the logged address too; run the proxy in the compose network
+and trust only its address to avoid that.
+
+The gateway is
+`docker network inspect resolver_default --format '{{(index .IPAM.Config 0).Gateway}}'`,
+with the network named after the compose project. It changes when the network is
+recreated, as `docker compose down` does, so trust one of, in `.env`:
+
+- `SNRC_TRUSTED_PROXIES=172.16.0.0/12`: Docker's default pools for compose
+  networks, whatever subnet it picks. This also trusts every other container on the
+  host. Add the gateway too if Docker ever gives the network a `192.168.x.x` one.
+- a pinned subnet, so the gateway stays fixed, and only that gateway:
+
+  ```yaml
+  networks:
+    default:
+      ipam:
+        config:
+          - subnet: 172.30.0.0/24
+            gateway: 172.30.0.1
+  ```
+
+  in `docker-compose.yml`, with `SNRC_TRUSTED_PROXIES=172.30.0.1`, on a subnet no
+  other network on the host uses.
@@ -38,14 +38,14 @@ services:
      - sh
      - -c
      - |
-        if [ -d /home/user/nimbus-eth2/build/data/${NETWORK}/db ]; then
+        if [ -d /home/user/nimbus-eth2/build/data/${NETWORK:-mainnet}/db ]; then
          echo "Nimbus data dir already initialised — skipping checkpoint sync";
          exit 0;
        fi;
        /home/user/nimbus-eth2/build/nimbus_beacon_node trustedNodeSync \
-          --network=${NETWORK} \
-          --data-dir=/home/user/nimbus-eth2/build/data/${NETWORK} \
-          --trusted-node-url=${TRUSTED_NODE_URL} \
+          --network=${NETWORK:-mainnet} \
+          --data-dir=/home/user/nimbus-eth2/build/data/${NETWORK:-mainnet} \
+          --trusted-node-url=${TRUSTED_NODE_URL:-https://mainnet-checkpoint-sync.attestant.io} \
          --backfill=false
    restart: "no"

@@ -69,8 +69,8 @@ services:
          echo "Reth data already initialised — skipping snapshot download";
          exit 0;
        fi;
-        echo "Downloading Reth ${NETWORK} --minimal snapshot...";
-        reth download --datadir /data --chain ${NETWORK} --minimal && \
+        echo "Downloading Reth ${NETWORK:-mainnet} --minimal snapshot...";
+        reth download --datadir /data --chain ${NETWORK:-mainnet} --minimal && \
        touch /data/.snapshot-done && \
        echo "Snapshot download complete"
    restart: "no"
@@ -92,7 +92,7 @@ services:
    command: >
      node
      --datadir /data
-      --chain ${NETWORK}
+      --chain ${NETWORK:-mainnet}
      --minimal
      --authrpc.jwtsecret /jwt/jwt.hex
      --authrpc.addr 0.0.0.0 --authrpc.port 8551
@@ -117,8 +117,8 @@ services:
      - "9000:9000/udp"
      - "127.0.0.1:5052:5052"
    command: >
-      --network=${NETWORK}
-      --data-dir=/home/user/nimbus-eth2/build/data/${NETWORK}
+      --network=${NETWORK:-mainnet}
+      --data-dir=/home/user/nimbus-eth2/build/data/${NETWORK:-mainnet}
      --el=http://reth:8551
      --jwt-secret=/jwt/jwt.hex
      --non-interactive
@@ -131,6 +131,7 @@ services:
  # smp-server points its [NAMES] resolver_endpoint at this URL.
  # To change the host port, edit the LEFT side of the port mapping below.
  resolver:
+    image: snrc-resolve:local
    build:
      context: ./service
      dockerfile: Dockerfile
@@ -156,8 +157,18 @@ services:
      # SNRC_REGISTRAR_SIMPLEX: 0x...
      # SNRC_CONTROLLER_TESTING: 0x...
      # SNRC_CONTROLLER_SIMPLEX: 0x...
+      # Set in .env (see .env.example); unset or empty means the resolver's default.
+      SNRC_WORKERS: ${SNRC_WORKERS:-}
+      SNRC_RPC_TIMEOUT: ${SNRC_RPC_TIMEOUT:-}
+      SNRC_LOG_FORMAT: ${SNRC_LOG_FORMAT:-}
+      SNRC_LOG_COLOR: ${SNRC_LOG_COLOR:-}
+      SNRC_LOG_LEVEL: ${SNRC_LOG_LEVEL:-}
+      SNRC_TRUSTED_PROXIES: ${SNRC_TRUSTED_PROXIES:-}
    ports:
      - "127.0.0.1:8000:8000"
+    # a line per lookup; the local driver keeps 5 rotated files of 20 MB
+    logging:
+      driver: local
    restart: unless-stopped

 volumes:
@@ -0,0 +1,5 @@
+.venv
+**/__pycache__
+.pytest_cache
+.ruff_cache
+test_snrc_resolve.py
@@ -0,0 +1 @@
+3.13
@@ -1,9 +1,10 @@
 # syntax=docker/dockerfile:1.7
 # ---------- builder ----------
-# Use the official uv image (Astral) on top of a slim Python base.
-# uv resolves and installs the lockfile-free pyproject.toml in seconds and
-# produces a portable .venv we can copy into the runtime stage.
-FROM ghcr.io/astral-sh/uv:python3.13-bookworm-slim AS builder
+# The official uv image (Astral) on top of a slim Python base installs the
+# locked dependencies into a portable .venv for the runtime stage.
+# Pinned to the uv that wrote uv.lock (its bookworm images stopped at 0.9.30), on the
+# runtime's Debian release, so the .venv's Python path exists there.
+FROM ghcr.io/astral-sh/uv:0.12.16-python3.13-trixie-slim AS builder

 ENV UV_LINK_MODE=copy \
    UV_COMPILE_BYTECODE=1 \
@@ -12,17 +13,17 @@ ENV UV_LINK_MODE=copy \

 WORKDIR /app

-# Install deps first (separate layer) — script edits won't bust this cache.
-COPY pyproject.toml ./
+# Dependencies first (separate layer), exactly as locked: script edits don't bust this cache.
+COPY pyproject.toml uv.lock .python-version ./
 RUN --mount=type=cache,target=/root/.cache/uv \
-    uv sync --no-dev --no-install-project
+    uv sync --frozen --no-dev --no-install-project

 # Script is added after the dep layer for cache friendliness.
 COPY snrc-resolve.py ./

 # ---------- runtime ----------
 # Slim runtime — only the venv + script. No uv, no apt.
-FROM python:3.13-slim AS runtime
+FROM python:3.13-slim-trixie AS runtime

 ENV PYTHONUNBUFFERED=1 \
    PYTHONDONTWRITEBYTECODE=1 \
@@ -33,15 +34,15 @@ RUN groupadd --system --gid 10001 snrc && \
    useradd  --system --uid 10001 --gid snrc --no-create-home --shell /usr/sbin/nologin snrc

 WORKDIR /app
-COPY --from=builder --chown=snrc:snrc /app /app
+# owned by root, so the service cannot rewrite its own code
+COPY --from=builder /app /app

 USER snrc:snrc

 EXPOSE 8000

-# Liveness check hits the script's own /health route. ThreadingHTTPServer is
-# fast enough that 3s is generous for a localhost probe; restart if it stops
-# responding entirely.
+# Liveness check hits the script's own /health route. Three failures mark the
+# container unhealthy; compose does not restart it for that, but `docker ps` shows it.
 HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
    CMD ["python", "-c", "import urllib.request, sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=3).status == 200 else 1)"]

@@ -9,5 +9,14 @@ dependencies = [
  "eth-hash[pycryptodome]>=0.7",
 ]

+[dependency-groups]
+dev = [
+  "pytest>=8",
+  "ruff>=0.6",
+]
+
 [tool.uv]
 package = false
+
+[tool.ruff.lint]
+select = ["E9", "F", "B"]
@@ -46,6 +46,15 @@ Environment:
                         (default: mainnet for .testing, empty for .simplex)
  SNRC_PORT              Listen port (default: 8000)
  SNRC_BIND              Bind address (default: 0.0.0.0)
+  SNRC_WORKERS           Worker processes sharing the port (default: CPU count, at most 4)
+  SNRC_RPC_TIMEOUT       Seconds to wait for each RPC request (default: 5)
+  SNRC_MULTICALL         Multicall3 contract that runs a round of reads as one call
+                         (default: 0xcA11bde05977b3631167028862bE2a173976CA11)
+  SNRC_LOG_FORMAT        text (key=value) or json (default: text)
+  SNRC_LOG_COLOR         auto (on a terminal), always or never (default: auto)
+  SNRC_LOG_LEVEL         debug, info, warning or error (default: info)
+  SNRC_TRUSTED_PROXIES   Comma-separated addresses or CIDRs of reverse proxies whose
+                         X-Forwarded-For names the client (default: none)

 Each TLD is a separate SNRC deployment with its own ENSRegistry; the
 resolver dispatches by the queried name's rightmost label.
@@ -64,19 +73,45 @@ Unrecognised payloads fall back to `0x`-prefixed raw hex.
 """

 import hashlib
+import ipaddress
 import json
+import logging
 import os
+import queue
+import signal
+import socket
 import sys
+import threading
 import time
+from contextlib import contextmanager
+from datetime import datetime, timezone
+from functools import lru_cache
+from http.client import BadStatusLine, HTTPConnection, HTTPSConnection
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from urllib.error import HTTPError
 from urllib.parse import unquote, urlparse
-from urllib.request import Request, urlopen

 from eth_hash.auto import keccak

 RPC = os.environ.get("SNRC_RPC", "http://127.0.0.1:8545")
 BIND = os.environ.get("SNRC_BIND", "0.0.0.0")
 PORT = int(os.environ.get("SNRC_PORT", "8000"))
+# The smp-server gives up after 3 s, so a slower call only holds a thread.
+RPC_TIMEOUT_S = float(os.environ.get("SNRC_RPC_TIMEOUT", "") or 5)
+# The node and the beacon client usually share the host, so not every core.
+MAX_DEFAULT_WORKERS = 4
+WORKERS = int(os.environ.get("SNRC_WORKERS", "") or min(MAX_DEFAULT_WORKERS, os.cpu_count() or 1))
+# Multicall3, at this address on mainnet and most chains. A node runs a JSON-RPC
+# batch one call after another, so a round of reads is sent as one eth_call.
+MULTICALL = os.environ.get("SNRC_MULTICALL", "") or "0xcA11bde05977b3631167028862bE2a173976CA11"
+LOG_FORMAT = os.environ.get("SNRC_LOG_FORMAT", "") or "text"
+LOG_COLOR = os.environ.get("SNRC_LOG_COLOR", "") or "auto"
+LOG_LEVEL = os.environ.get("SNRC_LOG_LEVEL", "") or "info"
+# Peers whose X-Forwarded-For is believed, such as the Docker gateway a reverse proxy on the host
+# connects through. Anyone else could put any address there.
+TRUSTED_PROXIES = tuple(
+    ipaddress.ip_network(p.strip(), strict=False) for p in os.environ.get("SNRC_TRUSTED_PROXIES", "").split(",") if p.strip()
+)

 # Each TLD is its own SNRC deployment with its own ENSRegistry. Dispatch
 # happens on the rightmost label of the queried name. Empty / unset means
@@ -121,34 +156,291 @@ COIN_ETH = 60
 COIN_BTC = 0
 COIN_XMR = 128
 COIN_DOT = 354
+RECORD_COINS = (COIN_ETH, COIN_BTC, COIN_XMR, COIN_DOT)

 ZERO_ADDR = "0x0000000000000000000000000000000000000000"

 # The registry prices in attoUSD (1e-18 USD); the protocol carries US cents.


+# ---------- Logging ----------
+
+LOGGER = logging.getLogger("snrc_resolve")
+LEVEL_NAMES = {logging.DEBUG: "DEBUG", logging.INFO: "INFO", logging.WARNING: "WARN", logging.ERROR: "ERROR"}
+# ANSI SGR codes
+DIM, BOLD, GREEN, YELLOW, RED = "2", "1", "32", "33", "31"
+LEVEL_COLORS = {"DEBUG": DIM, "INFO": GREEN, "WARN": YELLOW, "ERROR": RED}
+STATUS_COLORS = {2: GREEN, 3: GREEN, 4: YELLOW, 5: RED}
+
+
+def log_event(level: int, name: str, /, exc_info=None, **fields):
+    LOGGER.log(level, name, exc_info=exc_info, extra={"fields": fields})
+
+
+def _log_time(record: logging.LogRecord) -> str:
+    return datetime.fromtimestamp(record.created, timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z")
+
+
+def _log_value(value) -> str:
+    """A logfmt value: bare when it cannot be misread, JSON-quoted otherwise."""
+    if value is None:
+        return "-"
+    s = str(value)
+    if not s or any(c in s for c in ' ="\\') or not s.isprintable():
+        return json.dumps(s)
+    return s
+
+
+class TextFormatter(logging.Formatter):
+    def __init__(self, color: bool):
+        super().__init__()
+        self.color = color
+
+    def _paint(self, code: str, text: str) -> str:
+        return f"\033[{code}m{text}\033[0m" if self.color and code else text
+
+    def format(self, record: logging.LogRecord) -> str:
+        level = LEVEL_NAMES.get(record.levelno, record.levelname)
+        fields = " ".join(
+            self._paint(DIM, f"{k}=") + self._paint(STATUS_COLORS.get(v // 100, "") if k == "status" and isinstance(v, int) else "", _log_value(v))
+            for k, v in getattr(record, "fields", {}).items()
+        )
+        line = f"{self._paint(DIM, _log_time(record))} {self._paint(LEVEL_COLORS.get(level, ''), f'{level:<5}')} {self._paint(BOLD, record.getMessage())}"
+        if fields:
+            line += "  " + fields
+        if record.exc_info:
+            line += "\n" + self.formatException(record.exc_info)
+        return line
+
+
+class JsonFormatter(logging.Formatter):
+    def format(self, record: logging.LogRecord) -> str:
+        out = {
+            "time": _log_time(record),
+            "level": LEVEL_NAMES.get(record.levelno, record.levelname).lower(),
+            "event": record.getMessage(),
+            **getattr(record, "fields", {}),
+        }
+        if record.exc_info:
+            out["exception"] = self.formatException(record.exc_info)
+        return json.dumps(out, default=str)
+
+
+def setup_logging(stream=None):
+    """Sends the resolver's events to stderr in the configured format."""
+    stream = stream or sys.stderr
+    if LOG_FORMAT not in ("text", "json"):
+        raise ValueError(f"SNRC_LOG_FORMAT must be text or json, not {LOG_FORMAT!r}")
+    if LOG_COLOR not in ("auto", "always", "never"):
+        raise ValueError(f"SNRC_LOG_COLOR must be auto, always or never, not {LOG_COLOR!r}")
+    level = logging.getLevelName(LOG_LEVEL.upper())
+    if not isinstance(level, int):
+        raise ValueError(f"SNRC_LOG_LEVEL must be debug, info, warning or error, not {LOG_LEVEL!r}")
+    color = LOG_COLOR == "always" or (LOG_COLOR == "auto" and stream.isatty())
+    handler = logging.StreamHandler(stream)
+    handler.setFormatter(JsonFormatter() if LOG_FORMAT == "json" else TextFormatter(color))
+    LOGGER.handlers[:] = [handler]
+    LOGGER.setLevel(level)
+    LOGGER.propagate = False
+
+
 # ---------- RPC + ABI helpers (mirrors ens-lookup.py shape) ----------

+RPC_URL = urlparse(RPC)
+# Set a non-default User-Agent; Cloudflare-fronted public RPCs (drpc,
+# publicnode, etc.) reject `Python-urllib/3.x` with 403.
+RPC_HEADERS = {"Content-Type": "application/json", "User-Agent": "snrc-resolve/1.0"}
+
+# Idle keep-alive connections to SNRC_RPC. A connection per call costs most of
+# a lookup's CPU and leaves a TIME_WAIT socket per call, which exhausts local
+# ports at a few dozen lookups per second.
+# Idle connections kept per worker; more are closed after use. 4 workers stay well
+# under reth's default limit of 500 connections.
+RPC_POOL_SIZE = 32
+_rpc_pool = queue.LifoQueue(maxsize=RPC_POOL_SIZE)
+
+
+def _new_rpc_connection():
+    conn_class = HTTPSConnection if RPC_URL.scheme == "https" else HTTPConnection
+    return conn_class(RPC_URL.hostname, RPC_URL.port, timeout=RPC_TIMEOUT_S)
+
+
+def _post_rpc(conn, body: bytes) -> bytes:
+    path = (RPC_URL.path or "/") + (f"?{RPC_URL.query}" if RPC_URL.query else "")
+    conn.request("POST", path, body, RPC_HEADERS)
+    res = conn.getresponse()
+    data = res.read()
+    # HTTPError, as urlopen raised: RuntimeError means the call itself failed
+    if not 200 <= res.status < 300:
+        raise HTTPError(RPC, res.status, res.reason, res.headers, None)
+    return data
+
+
+def _post_pooled(conn, body: bytes) -> bytes:
+    try:
+        data = _post_rpc(conn, body)
+    except BaseException:
+        conn.close()
+        raise
+    try:
+        _rpc_pool.put_nowait(conn)
+    except queue.Full:
+        conn.close()
+    return data
+
+
+def _send_rpc(payload) -> object:
+    body = json.dumps(payload).encode()
+    try:
+        idle = _rpc_pool.get_nowait()
+    except queue.Empty:
+        return json.loads(_post_pooled(_new_rpc_connection(), body))
+    try:
+        return json.loads(_post_pooled(idle, body))
+    except (ConnectionError, BadStatusLine):
+        # the node closed the idle connection; every call is a read, so resending is safe
+        return json.loads(_post_pooled(_new_rpc_connection(), body))
+
+
+# Reads prefetched for the request being answered, keyed by _read_key. Only set
+# inside request_reads(), so code called outside a request reads one call at a time.
+_request = threading.local()
+_UNREAD = object()
+
+
+def _read_key(method, params) -> str:
+    return method + json.dumps(params, sort_keys=True)
+
+
+@contextmanager
+def request_reads():
+    _request.reads = {}
+    try:
+        yield
+    finally:
+        del _request.reads
+
+
 def rpc(method, params):
-    body = json.dumps(
-        {"jsonrpc": "2.0", "method": method, "params": params, "id": 1}
-    ).encode()
-    # Set a non-default User-Agent; Cloudflare-fronted public RPCs (drpc,
-    # publicnode, etc.) reject `Python-urllib/3.x` with 403.
-    req = Request(
-        RPC,
-        data=body,
-        headers={
-            "Content-Type": "application/json",
-            "User-Agent": "snrc-resolve/1.0",
-        },
-    )
-    res = json.loads(urlopen(req, timeout=15).read())
+    reads = getattr(_request, "reads", None)
+    if reads is not None:
+        read = reads.get(_read_key(method, params), _UNREAD)
+        if isinstance(read, RuntimeError):
+            raise read
+        if read is not _UNREAD:
+            return read
+    res = _send_rpc({"jsonrpc": "2.0", "method": method, "params": params, "id": 1})
    if "error" in res:
        raise RuntimeError(res["error"])
    return res["result"]


+def _send_batch(requests):
+    """Answers of a JSON-RPC batch in request order, None for a request left
+    unanswered; all None when the node does not batch."""
+    res = _send_rpc([{"jsonrpc": "2.0", "id": i, "method": m, "params": p} for i, (m, p) in enumerate(requests)])
+    # a node that does not batch answers with a single error object
+    by_id = {r.get("id"): r for r in res if isinstance(r, dict)} if isinstance(res, list) else {}
+    return [by_id.get(i) for i in range(len(requests))]
+
+
+def _is_latest_call(method, params) -> bool:
+    return method == "eth_call" and params[1] == "latest"
+
+
+_multicall_failed_logged = False
+
+
+def _remember(reads, requests, answers):
+    for (m, p), r in zip(requests, answers, strict=True):
+        if r is not None and ("result" in r or "error" in r):
+            reads[_read_key(m, p)] = RuntimeError(r["error"]) if "error" in r else r["result"]
+
+
+def prefetch(requests):
+    """Sends reads the request will make in one round trip, with its contract
+    reads as one multicall. A read left unanswered is made on its own when the
+    code gets to it."""
+    global _multicall_failed_logged
+    reads = getattr(_request, "reads", None)
+    if reads is None:
+        return
+    todo = [(m, p) for m, p in requests if _read_key(m, p) not in reads]
+    if len(todo) < 2:
+        return
+    calls = [(m, p) for m, p in todo if _is_latest_call(m, p)]
+    if len(calls) < 2:
+        _remember(reads, todo, _send_batch(todo))
+        return
+    others = [(m, p) for m, p in todo if not _is_latest_call(m, p)]
+    multicall = eth_call_read(MULTICALL, encode_aggregate3([(p[0]["to"], p[0]["data"]) for _, p in calls]))
+    answers = _send_batch(others + [multicall])
+    if all(r is None for r in answers):
+        return
+    _remember(reads, others, answers[:-1])
+    try:
+        result = (answers[-1] or {}).get("result")
+        if not isinstance(result, str):
+            raise ValueError(f"multicall answered {answers[-1]!r}")
+        results = decode_aggregate3(result)
+        if len(results) != len(calls):
+            raise ValueError("multicall answered a different number of calls")
+    except (KeyError, TypeError, ValueError) as e:
+        if not _multicall_failed_logged:
+            _multicall_failed_logged = True
+            log_event(logging.WARNING, "multicall_unavailable", multicall=MULTICALL, error=repr(e), fallback="batch")
+        _remember(reads, calls, _send_batch(calls))
+        return
+    for (m, p), (success, data) in zip(calls, results, strict=True):
+        reads[_read_key(m, p)] = "0x" + data.hex() if success else RuntimeError("execution reverted")
+
+
+AGGREGATE3 = "0x82ad56cb"  # aggregate3((address,bool,bytes)[])
+
+
+def encode_aggregate3(calls) -> str:
+    """Calldata for Multicall3.aggregate3 with every call allowed to fail."""
+    tuples = []
+    for to, data in calls:
+        b = bytes.fromhex(data[2:])
+        tuples.append(
+            int(to, 16).to_bytes(32, "big")
+            + (1).to_bytes(32, "big")
+            + (0x60).to_bytes(32, "big")
+            + len(b).to_bytes(32, "big")
+            + b
+            + b"\x00" * ((-len(b)) % 32)
+        )
+    offsets, at = [], 32 * len(tuples)
+    for t in tuples:
+        offsets.append(at.to_bytes(32, "big"))
+        at += len(t)
+    body = (0x20).to_bytes(32, "big") + len(tuples).to_bytes(32, "big") + b"".join(offsets) + b"".join(tuples)
+    return AGGREGATE3 + body.hex()
+
+
+def decode_aggregate3(hex_data: str):
+    """Multicall3.aggregate3's (bool success, bytes returnData)[]."""
+    raw = bytes.fromhex(hex_data[2:] if hex_data.startswith("0x") else hex_data)
+
+    def word(at: int) -> int:
+        if at + 32 > len(raw):
+            raise ValueError("multicall answer is truncated")
+        return int.from_bytes(raw[at:at + 32], "big")
+
+    array = word(0)
+    base = array + 32
+    out = []
+    for i in range(word(array)):
+        item = base + word(base + 32 * i)
+        data = item + word(item + 32)
+        length = word(data)
+        if data + 32 + length > len(raw):
+            raise ValueError("multicall answer is truncated")
+        out.append((word(item) != 0, raw[data + 32:data + 32 + length]))
+    return out
+
+
 def namehash(name: str) -> bytes:
    node = b"\x00" * 32
    if name:
@@ -184,11 +476,14 @@ def node_of(name: str) -> bytes:
 # ---------- Registration status ----------


+BLOCK_READ = ("eth_getBlockByNumber", ["latest", False])
+
+
 def head_block():
    """How far behind the node is. Unlike expiry, this is the one thing that has
    to be measured against the host clock: a node that stops still has a block."""
    try:
-        block = rpc("eth_getBlockByNumber", ["latest", False])
+        block = rpc(*BLOCK_READ)
        return {
            "blockNumber": decode_uint(block["number"]),
            "chainLagSeconds": int(time.time()) - decode_uint(block["timestamp"]),
@@ -199,13 +494,13 @@ def head_block():

 def chain_now() -> int:
    """Expiry is compared against the block timestamp, never the host clock."""
-    block = rpc("eth_getBlockByNumber", ["latest", False])
+    block = rpc(*BLOCK_READ)
    return decode_uint(block["timestamp"])


 def grace_period(registrar: str) -> int:
    """A deployment can configure a different window, so it is read on chain."""
-    return decode_uint(eth_call(registrar, selector("GRACE_PERIOD()")))
+    return decode_uint(eth_call(*grace_call(registrar)))


 def expiry_status(expires: int, grace: int, now: int) -> str:
@@ -225,8 +520,7 @@ def reservation_reason(tld: str, token: int) -> int:
    controller = CONTROLLERS.get(tld)
    if not controller:
        return 0
-    raw = eth_call(controller, selector("reservedNames(bytes32)") + encode_uint(token))
-    return decode_uint(raw)
+    return decode_uint(eth_call(*reserved_call(controller, token)))


 def pricing_params(tld: str):
@@ -235,7 +529,8 @@ def pricing_params(tld: str):
    controller = CONTROLLERS.get(tld)
    if not controller:
        return None
-    oracle = decode_address(eth_call(controller, selector("prices()")))
+    prefetch([eth_call_read(*prices_call(controller)), eth_call_read(*min_length_call(controller))])
+    oracle = decode_address(eth_call(*prices_call(controller)))
    if oracle == ZERO_ADDR:
        return None
    try:
@@ -247,6 +542,8 @@ def pricing_params(tld: str):


 SECONDS_PER_YEAR = 31536000
+# an ENS-shaped oracle prices names by length up to six letters
+LETTER_TIERS = range(1, 7)
 ATTO_PER_CENT = 10**16


@@ -255,13 +552,15 @@ def read_oracle_prices(controller: str, oracle: str):
    protocol carries. An ENS-shaped oracle prices in attoUSD per second and
    charges a premium on lapsed names that it does not expose, so a quote from
    it is only safe for a name that was never registered."""
+    # either oracle shape is answered in the same round trip
+    prefetch([eth_call_read(*prices_call(oracle))] + [eth_call_read(*letter_price_call(oracle, n)) for n in LETTER_TIERS])
    try:
-        base, tiers = decode_prices(eth_call(oracle, selector("prices()")))
+        base, tiers = decode_prices(eth_call(*prices_call(oracle)))
        premium_unknown = False
    except RuntimeError:
        base, tiers = decode_letter_prices(oracle)
        premium_unknown = True
-    min_len = decode_uint(eth_call(controller, selector("minCharLength()")))
+    min_len = decode_uint(eth_call(*min_length_call(controller)))
    return {
        # lengths the registry refuses are left out rather than priced at zero
        "registrationPrices": {n: c for n, c in tiers.items() if n >= min_len},
@@ -277,9 +576,9 @@ def decode_letter_prices(oracle: str):
    the six-letter tier stops at five, and charges its highest tier for anything
    longer, which is what basePrice means here."""
    tiers = {}
-    for n in range(1, 7):
+    for n in LETTER_TIERS:
        try:
-            rate = decode_uint(eth_call(oracle, selector(f"price{n}Letter()")))
+            rate = decode_uint(eth_call(*letter_price_call(oracle, n)))
        except RuntimeError:
            if n <= 5:
                raise
@@ -301,6 +600,9 @@ def decode_prices(hex_data: str):
    base = int.from_bytes(raw[:32], "big")
    at = int.from_bytes(raw[32:64], "big")
    count = int.from_bytes(raw[at:at + 32], "big")
+    # the count comes from the answer, so it is checked against the answer's length
+    if at + 32 + count * 64 > len(raw):
+        raise RuntimeError("prices(): short response")
    tiers = {}
    for i in range(count):
        item = at + 32 + i * 64
@@ -328,9 +630,7 @@ def name_status(name: str):
    # The 2LD's label is that key at any depth. node_of decodes a bracket only
    # in a two-label name, so a bracket subname gets a status but no record.
    token = label_token(labels[-2])
-    expires = decode_uint(
-        eth_call(registrar, selector("nameExpires(uint256)") + encode_uint(token))
-    )
+    expires = decode_uint(eth_call(*expires_call(registrar, token)))
    grace = grace_period(registrar) if expires else 0
    now = chain_now()
    status = expiry_status(expires, grace, now)
@@ -358,12 +658,17 @@ def name_status(name: str):
    return out


+@lru_cache(maxsize=None)
 def selector(signature: str) -> str:
    return "0x" + keccak(signature.encode())[:4].hex()


+def eth_call_read(to: str, data: str):
+    return "eth_call", [{"to": to, "data": data}, "latest"]
+
+
 def eth_call(to: str, data: str) -> str:
-    result = rpc("eth_call", [{"to": to, "data": data}, "latest"])
+    result = rpc(*eth_call_read(to, data))
    if result == "0x":
        raise RuntimeError(f"empty return from {to}: no contract at that address?")
    return result
@@ -385,7 +690,7 @@ def registered_label(registrar: str, token: int):
    """The plaintext label the registrar recorded at registration, keyed by the
    hash of that label. None when the name was registered without
    registerWithLabel, so the registrar cannot name it."""
-    raw = decode_bytes(eth_call(registrar, selector("labelOf(uint256)") + encode_uint(token)))
+    raw = decode_bytes(eth_call(*label_call(registrar, token)))
    return raw.decode("utf-8", errors="replace") if raw else None


@@ -427,7 +732,7 @@ def encode_text_call(node: bytes, key: str) -> str:


 def text(resolver: str, node: bytes, key: str) -> str:
-    raw = decode_bytes(eth_call(resolver, encode_text_call(node, key)))
+    raw = decode_bytes(eth_call(*text_call(resolver, node, key)))
    return raw.decode("utf-8", errors="replace") if raw else ""


@@ -446,7 +751,7 @@ def addr_multicoin(resolver: str, node: bytes, coin_type: int):
    payload doesn't match any recognised on-chain shape. Returns None when
    the record is unset."""
    try:
-        raw = decode_bytes(eth_call(resolver, encode_addr_multicoin_call(node, coin_type)))
+        raw = decode_bytes(eth_call(*addr_call(resolver, node, coin_type)))
    except RuntimeError:
        return None
    if not raw:
@@ -465,6 +770,81 @@ def addr_multicoin(resolver: str, node: bytes, coin_type: int):
        return "0x" + raw.hex()


+# ---------- Contract calls, as (to, data) ----------
+# Shared by the reads and prefetch, so a prefetched read is found by the code that makes it.
+
+
+def expires_call(registrar: str, token: int):
+    return registrar, selector("nameExpires(uint256)") + encode_uint(token)
+
+
+def grace_call(registrar: str):
+    return registrar, selector("GRACE_PERIOD()")
+
+
+def label_call(registrar: str, token: int):
+    return registrar, selector("labelOf(uint256)") + encode_uint(token)
+
+
+def reserved_call(controller: str, token: int):
+    return controller, selector("reservedNames(bytes32)") + encode_uint(token)
+
+
+def prices_call(contract: str):
+    return contract, selector("prices()")
+
+
+def letter_price_call(oracle: str, letters: int):
+    return oracle, selector(f"price{letters}Letter()")
+
+
+def min_length_call(controller: str):
+    return controller, selector("minCharLength()")
+
+
+def resolver_call(registry: str, node: bytes):
+    return registry, selector("resolver(bytes32)") + node.hex()
+
+
+def owner_call(registry: str, node: bytes):
+    return registry, selector("owner(bytes32)") + node.hex()
+
+
+def text_call(resolver: str, node: bytes, key: str):
+    return resolver, encode_text_call(node, key)
+
+
+def addr_call(resolver: str, node: bytes, coin_type: int):
+    return resolver, encode_addr_multicoin_call(node, coin_type)
+
+
+def lookup_reads(name: str):
+    """The reads a lookup makes before it knows the name's resolver."""
+    labels = name.split(".")
+    tld = labels[-1]
+    reads = []
+    registrar = REGISTRARS.get(tld)
+    if registrar and len(labels) >= 2:
+        token = label_token(labels[-2])
+        reads += [BLOCK_READ, eth_call_read(*expires_call(registrar, token)), eth_call_read(*grace_call(registrar))]
+        if len(labels) == 2 and is_encoded_labelhash(labels[0]):
+            reads.append(eth_call_read(*label_call(registrar, token)))
+        if CONTROLLERS.get(tld):
+            reads.append(eth_call_read(*reserved_call(CONTROLLERS[tld], token)))
+    registry = REGISTRIES.get(tld)
+    if registry:
+        node = node_of(name)
+        reads += [eth_call_read(*resolver_call(registry, node)), eth_call_read(*owner_call(registry, node))]
+    return reads
+
+
+def record_reads(resolver: str, node: bytes):
+    """The reads of a name's record from its resolver."""
+    return [eth_call_read(*text_call(resolver, node, k)) for k in TEXT_KEYS] + [
+        eth_call_read(*addr_call(resolver, node, coin)) for coin in RECORD_COINS
+    ]
+
+
 # ---------- Coin-specific address encoders ----------
 # Each takes raw bytes as stored under ENSIP-9 and returns the canonical
 # user-facing string for that chain (EIP-55 for ETH, bech32/base58check
@@ -665,9 +1045,9 @@ def split_links(value: str) -> list:


 def upstream_error(subject: dict, e: Exception) -> dict:
-    """urlopen puts the failing URL into its message and SNRC_RPC can carry a
-    provider key, so the text goes to the log and only the type to the caller."""
-    print(f"upstream error: {type(e).__name__}: {e}", file=sys.stderr)
+    """The exception can carry the failing URL and SNRC_RPC can carry a provider
+    key, so the text goes to the log and only the type to the caller."""
+    log_event(logging.WARNING, "upstream_error", **subject, error=type(e).__name__, message=str(e))
    return {
        **subject,
        "error": "upstreamError",
@@ -680,9 +1060,8 @@ def name_record(name: str):
    has one, with every field unset."""
    registry = REGISTRIES[name.rsplit(".", 1)[-1]]
    node = node_of(name)
-    node_hex = node.hex()
-    resolver_addr = decode_address(eth_call(registry, selector("resolver(bytes32)") + node_hex))
-    owner = decode_address(eth_call(registry, selector("owner(bytes32)") + node_hex))
+    resolver_addr = decode_address(eth_call(*resolver_call(registry, node)))
+    owner = decode_address(eth_call(*owner_call(registry, node)))
    rec = {
        "name": canonical_name(name),
        "nickname": "",
@@ -699,6 +1078,7 @@ def name_record(name: str):
    }
    if resolver_addr == ZERO_ADDR:
        return rec
+    prefetch(record_reads(resolver_addr, node))
    texts = {}
    for k in TEXT_KEYS:
        try:
@@ -734,6 +1114,7 @@ def registration(name: str):
    tld = name.rsplit(".", 1)[-1]
    if not REGISTRIES.get(tld):
        return 400, {"name": name, "error": "tldNotConfigured"}
+    prefetch(lookup_reads(name))
    reg = name_status(name)
    status = reg["status"]
    if status in ("registered", "grace"):
@@ -788,7 +1169,7 @@ def resolve(name: str):
        }

    node = node_of(name)
-    node_hex = node.hex()
+    prefetch(lookup_reads(name))

    # Before the resolver lookup, so a lapsed name is not reported as noResolver.
    reg = name_status(name)
@@ -806,13 +1187,12 @@ def resolve(name: str):
        }
        return (404 if reg["status"] == "unregistered" else 410), body

-    resolver_raw = eth_call(registry, selector("resolver(bytes32)") + node_hex)
-    resolver_addr = decode_address(resolver_raw)
+    resolver_addr = decode_address(eth_call(*resolver_call(registry, node)))
    if resolver_addr == ZERO_ADDR:
        # A registered name always resolves: with no resolver set the record is
        # still returned with every field unset, so "taken until <date>" stays
        # answerable. For a subname, no owner means nobody created it.
-        owner = decode_address(eth_call(registry, selector("owner(bytes32)") + node_hex))
+        owner = decode_address(eth_call(*owner_call(registry, node)))
        if len(name.split(".")) > 2 and owner == ZERO_ADDR:
            return 404, {
                "name": name,
@@ -837,9 +1217,9 @@ def resolve(name: str):
            **reg,
        }

-    owner_raw = eth_call(registry, selector("owner(bytes32)") + node_hex)
-    owner = decode_address(owner_raw)
+    owner = decode_address(eth_call(*owner_call(registry, node)))

+    prefetch(record_reads(resolver_addr, node))
    texts = {}
    for k in TEXT_KEYS:
        try:
@@ -878,12 +1258,34 @@ def resolve(name: str):
 # ---------- HTTP layer ----------

 class Handler(BaseHTTPRequestHandler):
+    # the smp-server reuses a connection only after an HTTP/1.1 response
+    protocol_version = "HTTP/1.1"
+    # idle seconds before a kept-alive connection is closed; the smp-server closes
+    # its idle ones after 30-35 s, so it is the one to close them
+    timeout = 60
+
+    def setup(self):
+        super().setup()
+        # headers and body go out in separate writes, which Nagle holds for the client's delayed ACK
+        self.request.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+
+    def handle_one_request(self):
+        # on a kept-alive connection these still hold the previous request's values
+        self._started = self._sent = self.path = self.headers = None
+        super().handle_one_request()
+
    def do_GET(self):  # noqa: N802 - http.server contract
+        self._started = time.monotonic()
+        if self._has_body():
+            # an unread body would be taken for the next request on this connection, which a
+            # reverse proxy reusing it would pass to another client (request smuggling)
+            self._respond(400, {"error": "unexpectedBody"}, close=True)
+            return
        path = urlparse(self.path).path
        parts = [unquote(p) for p in path.split("/") if p]

        if parts == ["health"]:
-            self._respond(200, {"ok": True, "rpc": RPC, "registries": REGISTRIES, **head_block()})
+            self._respond(200, {"ok": True, "rpc": public_endpoint(RPC), "registries": REGISTRIES, **head_block()})
            return

        if len(parts) == 3 and parts[0] == "v2" and parts[1] == "resolve":
@@ -892,7 +1294,8 @@ class Handler(BaseHTTPRequestHandler):
                self._respond(400, {"name": name, "error": "notFullyQualified"})
                return
            try:
-                status, body = registration(name)
+                with request_reads():
+                    status, body = registration(name)
            except Exception as e:  # surface upstream errors as 502
                status, body = 502, upstream_error({"name": name}, e)
            self._respond(status, body)
@@ -915,7 +1318,8 @@ class Handler(BaseHTTPRequestHandler):
                )
                return
            try:
-                status, body = resolve(name)
+                with request_reads():
+                    status, body = resolve(name)
            except Exception as e:  # surface upstream errors as 502
                status, body = 502, upstream_error({"name": name}, e)
            self._respond(status, body)
@@ -930,34 +1334,176 @@ class Handler(BaseHTTPRequestHandler):
            },
        )

-    def _respond(self, status: int, body: dict):
+    def _has_body(self) -> bool:
+        # a malformed header line ends parsing, leaving the rest, Content-Length included, as payload
+        if self.headers.defects or self.headers.get_payload():
+            return True
+        if self.headers.get_all("Transfer-Encoding"):
+            return True
+        return any(length.strip() != "0" for length in self.headers.get_all("Content-Length") or [])
+
+    def _respond(self, status: int, body: dict, close: bool = False):
        data = json.dumps(body, indent=2).encode()
+        # send_response logs the request, so the size is known before it
+        self._sent = len(data)
        self.send_response(status)
        self.send_header("Content-Type", "application/json")
        self.send_header("Content-Length", str(len(data)))
+        if close:
+            # also sets close_connection, so the server closes it after this response
+            self.send_header("Connection", "close")
        self.end_headers()
        self.wfile.write(data)

+    def address_string(self) -> str:
+        forwarded = self.headers.get_all("X-Forwarded-For") if self.headers else None
+        return client_address(self.client_address[0], ",".join(forwarded) if forwarded else None)
+
+    def log_request(self, code="-", size="-"):
+        started = self._started
+        path = urlparse(self.path).path if self.path else None
+        log_event(
+            # the container's health check runs every 30 s
+            logging.DEBUG if path == "/health" else logging.INFO,
+            "request",
+            client=self.address_string(),
+            worker=os.getpid(),
+            method=getattr(self, "command", None),
+            path=unquote(self.path) if self.path else None,
+            status=getattr(code, "value", code),
+            bytes=self._sent,
+            ms=round((time.monotonic() - started) * 1000) if started else None,
+        )
+
+    def log_error(self, fmt, *args):
+        message = fmt % args
+        # a kept-alive connection left idle is closed on purpose
+        level = logging.DEBUG if message.startswith("Request timed out") else logging.WARNING
+        log_event(level, "http_error", client=self.address_string(), worker=os.getpid(), message=message)
+
    def log_message(self, fmt, *args):
-        # Quiet the default per-request access log; route to stderr in one line.
-        sys.stderr.write(f"{self.address_string()} - {fmt % args}\n")
+        log_event(logging.INFO, "http", client=self.address_string(), message=fmt % args)
+
+
+class ResolverServer(ThreadingHTTPServer):
+    # socketserver's default of 5 drops simultaneous connections, each retried by
+    # TCP after 1 s, and the smp-server gives up after 3 s. The kernel caps it at somaxconn.
+    request_queue_size = 128
+
+    def handle_error(self, request, client_address):
+        error = sys.exc_info()[1]
+        if isinstance(error, ConnectionError):
+            # the smp-server hung up, usually after its own timeout
+            log_event(logging.WARNING, "client_gone", client=client_address[0], worker=os.getpid(), error=type(error).__name__)
+        else:
+            log_event(logging.ERROR, "request_failed", client=client_address[0], worker=os.getpid(), exc_info=True)
+
+
+def public_endpoint(url: str) -> str:
+    """The RPC endpoint without credentials, path or query, where a provider key would be."""
+    parsed = urlparse(url)
+    return f"{parsed.scheme}://{parsed.netloc.rpartition('@')[2]}"
+
+
+def client_address(peer: str, forwarded_for: str | None) -> str:
+    """The peer, or behind trusted proxies the last X-Forwarded-For address none of them added."""
+    if not forwarded_for or not _trusted(peer):
+        return peer
+    hops = [hop.strip() for hop in forwarded_for.split(",")]
+    for hop in reversed(hops):
+        # an address it cannot read, and anything claimed before it, is not believed
+        if _address(hop) is None:
+            return peer
+        if not _trusted(hop):
+            return hop
+    return hops[0]
+
+
+def _address(text: str):
+    try:
+        ip = ipaddress.ip_address(text)
+    except ValueError:
+        return None
+    return ip.ipv4_mapped or ip if ip.version == 6 else ip
+
+
+def _trusted(text: str) -> bool:
+    ip = _address(text)
+    return ip is not None and any(ip in net for net in TRUSTED_PROXIES)
+
+
+def serve(reuse_port: bool):
+    server = ResolverServer((BIND, PORT), Handler, bind_and_activate=False)
+    server.allow_reuse_port = reuse_port
+    try:
+        server.server_bind()
+        server.server_activate()
+        server.serve_forever()
+    finally:
+        server.server_close()
+
+
+def stop_on(signum, _frame):
+    log_event(logging.INFO, "stopping", signal=signal.Signals(signum).name)
+    sys.exit(0)
+
+
+def supervise(workers: int):
+    """Runs the workers, each with its own socket on the shared port. One worker
+    exiting stops the others, so the container restarts instead of running short."""
+    children = []
+
+    def stop(*_):
+        for pid in children:
+            try:
+                os.kill(pid, signal.SIGTERM)
+            except ProcessLookupError:
+                pass
+
+    def stop_and_exit(signum, frame):
+        stop()
+        stop_on(signum, frame)
+
+    # before forking, so a signal during startup cannot orphan the workers
+    signal.signal(signal.SIGTERM, stop_and_exit)
+    signal.signal(signal.SIGINT, stop_and_exit)
+    for _ in range(workers):
+        pid = os.fork()
+        if pid == 0:
+            signal.signal(signal.SIGTERM, signal.SIG_DFL)
+            signal.signal(signal.SIGINT, signal.SIG_DFL)
+            code = 0
+            try:
+                serve(reuse_port=True)
+            except BaseException:
+                log_event(logging.ERROR, "worker_failed", worker=os.getpid(), exc_info=True)
+                code = 1
+            os._exit(code)
+        children.append(pid)
+    pid, status = os.wait()
+    log_event(logging.ERROR, "worker_exited", worker=pid, status=status, action="stopping")
+    stop()
+    sys.exit(1)


 def main():
-    server = ThreadingHTTPServer((BIND, PORT), Handler)
-    sys.stderr.write(
-        f"snrc-resolve listening on {BIND}:{PORT}\n"
-        f"  RPC = {RPC}\n"
-        f"  Registries:\n"
+    setup_logging()
+    log_event(
+        logging.INFO,
+        "listening",
+        bind=BIND,
+        port=PORT,
+        workers=WORKERS,
+        rpc=public_endpoint(RPC),
+        registries=",".join(f"{tld}={addr or '-'}" for tld, addr in REGISTRIES.items()),
+        trusted_proxies=",".join(map(str, TRUSTED_PROXIES)) or None,
    )
-    for tld, addr in REGISTRIES.items():
-        sys.stderr.write(f"    .{tld:<8s} = {addr or '(not configured)'}\n")
-    sys.stderr.write("  GET /v2/resolve/<name>   GET /v1/resolve/<name>   GET /health\n")
-    try:
-        server.serve_forever()
-    except KeyboardInterrupt:
-        sys.stderr.write("\nshutting down\n")
-        server.server_close()
+    if WORKERS > 1:
+        supervise(WORKERS)
+    else:
+        signal.signal(signal.SIGTERM, stop_on)
+        signal.signal(signal.SIGINT, stop_on)
+        serve(reuse_port=False)


 if __name__ == "__main__":
@@ -1,15 +1,30 @@
 #!/usr/bin/env python3
 """Unit tests for snrc-resolve helpers.

-Run with `python3 -m unittest scripts/resolver/service/test_snrc_resolve.py`.
+Run with `uv run pytest` from scripts/resolver/service.
 """

 import contextlib
+import http.client
 import importlib.util
 import io
+import ipaddress
+import json
+import logging
 import os
+import queue
+import re
+import signal
+import socket
+import subprocess
+import sys
+import threading
 import time
 import unittest
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from urllib.error import HTTPError
+from urllib.parse import urlparse
+from urllib.request import Request, urlopen

 # snrc-resolve.py has a hyphen, so import it via importlib instead of `import`.
 _HERE = os.path.dirname(os.path.abspath(__file__))
@@ -636,6 +651,25 @@ class PricingTests(unittest.TestCase):
        self.assertEqual(body["basePrice"], self.BASE)


+class DecodePricesTests(unittest.TestCase):
+    def test_a_count_longer_than_the_answer_is_refused(self):
+        """The count comes from the oracle; decoding it unchecked could loop for ever."""
+        huge = "0x" + snrc.encode_uint(200) + snrc.encode_uint(0x40) + snrc.encode_uint(2**64)
+        outcome = []
+
+        def decode():
+            try:
+                snrc.decode_prices(huge)
+            except RuntimeError as e:
+                outcome.append(e)
+
+        worker = threading.Thread(target=decode, daemon=True)
+        worker.start()
+        worker.join(5)
+        self.assertFalse(worker.is_alive(), "decoding did not stop")
+        self.assertRegex(str(outcome[0]), "short response")
+
+
 class EnsOracleTests(unittest.TestCase):
    """.testing runs an ENS-shaped oracle: it prices in attoUSD per second and
    charges a premium on lapsed names that it does not expose."""
@@ -777,13 +811,17 @@ class ErrorCodeTests(unittest.TestCase):
                self.assertNotEqual(body["error"], body["message"])

    def test_an_upstream_failure_does_not_echo_the_exception(self):
-        with contextlib.redirect_stderr(io.StringIO()) as log:
+        with self.assertLogs("snrc_resolve", "WARNING") as logs:
            body = snrc.upstream_error(
                {"name": "alice.testing"},
                RuntimeError("http://user:[email protected]/kEy8 refused"),
            )
        # the operator still sees the detail in the log
-        self.assertIn("secret", log.getvalue())
+        [record] = logs.records
+        self.assertEqual(record.getMessage(), "upstream_error")
+        self.assertEqual(record.fields["name"], "alice.testing")
+        self.assertEqual(record.fields["error"], "RuntimeError")
+        self.assertIn("secret", record.fields["message"])
        self.assertEqual(body["error"], "upstreamError")
        self.assertIn("RuntimeError", body["message"])
        self.assertNotIn("secret", body["message"])
@@ -1035,5 +1073,797 @@ class RegistrationV2Tests(unittest.TestCase):
        self.assertEqual(res["lastBlockTs"], self.now)
        self.assertEqual(res["registration"]["type"], "available")

-if __name__ == "__main__":
-    unittest.main()
+
+@unittest.skipUnless(sys.platform.startswith("linux"), "Linux drops SYNs on a full accept queue")
+class ListenBacklogTests(unittest.TestCase):
+    """The smp-server opens a connection per lookup and gives up after 3 s, so a
+    burst the accept queue cannot hold fails: TCP retries a dropped SYN after 1 s."""
+
+    BURST = 50
+
+    def test_a_burst_of_connections_is_queued_while_the_server_is_busy(self):
+        # never accepts, so every connection must wait in the queue
+        server = snrc.ResolverServer(("127.0.0.1", 0), snrc.Handler)
+        clients = []
+        try:
+            for i in range(self.BURST):
+                c = socket.socket()
+                clients.append(c)
+                c.settimeout(0.5)
+                try:
+                    c.connect(server.server_address)
+                except TimeoutError:
+                    self.fail(f"connection {i + 1} of {self.BURST} was not queued")
+        finally:
+            for c in clients:
+                c.close()
+            server.server_close()
+
+
+def _word(value: int) -> bytes:
+    return value.to_bytes(32, "big")
+
+
+def _decode_aggregate3_calls(data: str):
+    """Multicall3.aggregate3 calldata back to (to, data) pairs, written apart
+    from the resolver's encoder so the two check each other."""
+    raw = bytes.fromhex(data[len(snrc.AGGREGATE3):])
+
+    def word(at):
+        return int.from_bytes(raw[at:at + 32], "big")
+
+    array = word(0)
+    base = array + 32
+    calls = []
+    for i in range(word(array)):
+        item = base + word(base + 32 * i)
+        call = item + word(item + 64)
+        calls.append(("0x" + raw[item + 12:item + 32].hex(), "0x" + raw[call + 32:call + 32 + word(call)].hex()))
+    return calls
+
+
+def _encode_aggregate3_results(results) -> str:
+    tuples = [
+        _word(int(ok)) + _word(0x40) + _word(len(data)) + data + b"\x00" * ((-len(data)) % 32)
+        for ok, data in results
+    ]
+    offsets, at = b"", 32 * len(tuples)
+    for t in tuples:
+        offsets += _word(at)
+        at += len(t)
+    return "0x" + (_word(0x20) + _word(len(tuples)) + offsets + b"".join(tuples)).hex()
+
+
+class FakeChain:
+    """Contract state for one registered name and one free name. Any other
+    call reverts, as a view function asked for something unset does."""
+
+    REGISTRY = "0x58fc46996d975c57883564648bda5206d1a0102b"
+    REGISTRAR = "0xef47eb4384b46c89e4482a677c2cbcbd2a6fd85a"
+    CONTROLLER = "0x281ca41311c2aa808c917c4674639d7567b75714"
+    ORACLE = "0x1e0c9a2b9d1a4c8f7b3e5d6a9c2f4b8e1d7a3c50"
+    OWNER = "0xd83bd7e0e6b8a4c1f2593a7b0c4e8d1a6f9b2c37"
+    RESOLVER = "0x80fa2b1c3d4e5f60718293a4b5c6d7e8f9012345"
+    GRACE = 90 * 86400
+    TEXTS = {"nickname": "Acme", "url": "https://acme.example", "simplex.channel": "https://a.example/c#1;https://b.example/c#2"}
+
+    def __init__(self):
+        self.now = int(time.time())
+        acme, free = snrc.label_token("acme"), snrc.label_token("free")
+        node = snrc.node_of("acme.testing")
+        abi_bytes = RegistrationV2Tests._abi_bytes
+        prices = RegistrationV2Tests._prices_return(RegistrationV2Tests())
+        self.answers = {
+            snrc.expires_call(self.REGISTRAR, acme): "0x" + snrc.encode_uint(self.now + 3600),
+            snrc.expires_call(self.REGISTRAR, free): "0x" + snrc.encode_uint(0),
+            snrc.grace_call(self.REGISTRAR): "0x" + snrc.encode_uint(self.GRACE),
+            snrc.label_call(self.REGISTRAR, acme): abi_bytes(b"acme"),
+            snrc.reserved_call(self.CONTROLLER, acme): "0x" + snrc.encode_uint(0),
+            snrc.reserved_call(self.CONTROLLER, free): "0x" + snrc.encode_uint(0),
+            snrc.resolver_call(self.REGISTRY, node): "0x" + snrc.encode_uint(int(self.RESOLVER, 16)),
+            snrc.owner_call(self.REGISTRY, node): "0x" + snrc.encode_uint(int(self.OWNER, 16)),
+            snrc.addr_call(self.RESOLVER, node, snrc.COIN_ETH): abi_bytes(bytes.fromhex(self.OWNER[2:])),
+            snrc.prices_call(self.CONTROLLER): "0x" + snrc.encode_uint(int(self.ORACLE, 16)),
+            snrc.prices_call(self.ORACLE): prices,
+            snrc.min_length_call(self.CONTROLLER): "0x" + snrc.encode_uint(3),
+        }
+        for key, value in self.TEXTS.items():
+            self.answers[snrc.text_call(self.RESOLVER, node, key)] = abi_bytes(value.encode())
+
+    def call(self, to, data):
+        answer = self.answers.get((to.lower(), data))
+        if answer is None:
+            raise RuntimeError("execution reverted")
+        return answer
+
+
+class FakeNode(ThreadingHTTPServer):
+    """A JSON-RPC node over HTTP/1.1 keep-alive, serving FakeChain, with
+    batches and Multicall3, each of which a test can take away."""
+
+    daemon_threads = True
+
+    def __init__(self):
+        super().__init__(("127.0.0.1", 0), _FakeNodeHandler)
+        self.chain = FakeChain()
+        self.block = 100
+        self.requests = 0
+        self.connections = 0
+        self.batch = True
+        self.multicall = True
+        self.multicall_null = False
+        self.status = 200
+        self.hang_up = False
+        self.drop_after_reply = False
+        threading.Thread(target=self.serve_forever, args=(0.05,), daemon=True).start()
+
+    @property
+    def url(self):
+        return f"http://127.0.0.1:{self.server_address[1]}/"
+
+    def stop(self):
+        self.shutdown()
+        self.server_close()
+
+    def answer(self, req):
+        out = {"jsonrpc": "2.0", "id": req.get("id")}
+        method, params = req["method"], req["params"]
+        if method == "eth_blockNumber":
+            out["result"] = hex(self.block)
+        elif method == "eth_getBlockByNumber":
+            out["result"] = {"number": hex(self.block), "timestamp": hex(self.chain.now)}
+        elif method == "eth_call" and params[0]["to"].lower() == snrc.MULTICALL.lower():
+            if not self.multicall:
+                out["error"] = {"code": -32000, "message": "no contract code"}
+            elif self.multicall_null:
+                out["result"] = None
+            else:
+                results = []
+                for to, data in _decode_aggregate3_calls(params[0]["data"]):
+                    try:
+                        results.append((True, bytes.fromhex(self.chain.call(to, data)[2:])))
+                    except RuntimeError:
+                        results.append((False, b""))
+                out["result"] = _encode_aggregate3_results(results)
+        elif method == "eth_call":
+            try:
+                out["result"] = self.chain.call(params[0]["to"], params[0]["data"])
+            except RuntimeError:
+                out["error"] = {"code": 3, "message": "execution reverted"}
+        else:
+            out["error"] = {"code": -32601, "message": "method not found"}
+        return out
+
+
+class _FakeNodeHandler(BaseHTTPRequestHandler):
+    protocol_version = "HTTP/1.1"
+
+    def setup(self):
+        super().setup()
+        # headers and body go out in separate writes, which Nagle holds for the client's delayed ACK
+        self.request.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+        self.server.connections += 1
+
+    def do_POST(self):  # noqa: N802 - http.server contract
+        request = json.loads(self.rfile.read(int(self.headers["Content-Length"])))
+        node = self.server
+        node.requests += 1
+        if node.hang_up:
+            self.close_connection = True
+            return
+        if node.status != 200:
+            reply = {"error": "unavailable"}
+        elif isinstance(request, list):
+            reply = [node.answer(r) for r in request] if node.batch else {"jsonrpc": "2.0", "id": None, "error": {"code": -32600, "message": "batch not supported"}}
+        else:
+            reply = node.answer(request)
+        data = json.dumps(reply).encode()
+        self.send_response(node.status)
+        self.send_header("Content-Type", "application/json")
+        self.send_header("Content-Length", str(len(data)))
+        self.end_headers()
+        self.wfile.write(data)
+        # closes without `Connection: close`, as a node dropping an idle connection does
+        self.close_connection = node.drop_after_reply
+
+    def log_message(self, fmt, *args):
+        pass
+
+
+class FakeNodeTestCase(unittest.TestCase):
+    """Points the resolver at a FakeNode and at FakeChain's contracts."""
+
+    def setUp(self):
+        self.node = FakeNode()
+        self._saved = (snrc.RPC, snrc.RPC_URL, snrc.REGISTRIES, snrc.REGISTRARS, snrc.CONTROLLERS)
+        snrc.RPC = self.node.url
+        snrc.RPC_URL = urlparse(snrc.RPC)
+        snrc.REGISTRIES = {"testing": FakeChain.REGISTRY, "simplex": ""}
+        snrc.REGISTRARS = {"testing": FakeChain.REGISTRAR}
+        snrc.CONTROLLERS = {"testing": FakeChain.CONTROLLER}
+        snrc._multicall_failed_logged = False
+        self._drain_pool()
+
+    def tearDown(self):
+        self._drain_pool()
+        snrc.RPC, snrc.RPC_URL, snrc.REGISTRIES, snrc.REGISTRARS, snrc.CONTROLLERS = self._saved
+        self.node.stop()
+
+    def _drain_pool(self):
+        while not snrc._rpc_pool.empty():
+            snrc._rpc_pool.get_nowait().close()
+
+    def requests_made(self, action):
+        before = self.node.requests
+        result = action()
+        return result, self.node.requests - before
+
+
+class RpcTransportTests(FakeNodeTestCase):
+    """A lookup makes several reads, and a new connection per read costs CPU
+    and leaves a TIME_WAIT socket each, which exhausts local ports under load."""
+
+    def test_reads_share_one_connection(self):
+        for _ in range(18):
+            self.assertEqual(snrc.rpc("eth_blockNumber", []), hex(self.node.block))
+        self.assertEqual(self.node.connections, 1)
+
+    def test_a_connection_the_node_closed_is_replaced(self):
+        self.node.drop_after_reply = True
+        for _ in range(3):
+            self.assertEqual(snrc.rpc("eth_blockNumber", []), hex(self.node.block))
+        self.assertEqual(self.node.connections, 3)
+
+    def test_a_fresh_connection_that_fails_is_not_retried(self):
+        """Only a pooled connection can be stale; a new one failing means the
+        node is down, and resending would only double the wait."""
+        self.node.hang_up = True
+        with self.assertRaises(ConnectionError):
+            snrc.rpc("eth_blockNumber", [])
+        self.assertEqual(self.node.requests, 1)
+
+    def test_a_node_failure_is_not_a_reverted_call(self):
+        """Callers read RuntimeError as the call reverting and fall back to an
+        empty value, so a node failure must not look like one."""
+        self.node.status = 502
+        with self.assertRaises(HTTPError) as cm:
+            snrc.rpc("eth_blockNumber", [])
+        self.assertNotIsInstance(cm.exception, RuntimeError)
+        self.assertEqual(cm.exception.code, 502)
+
+    def test_a_reverted_call_is_a_runtime_error_and_keeps_the_connection(self):
+        with self.assertRaises(RuntimeError):
+            snrc.eth_call("0x" + "11" * 20, "0xdeadbeef")
+        snrc.rpc("eth_blockNumber", [])
+        self.assertEqual(self.node.connections, 1)
+
+
+class RpcPoolTests(FakeNodeTestCase):
+    def test_connections_beyond_the_pool_are_closed_after_use(self):
+        """Every worker would otherwise keep its peak concurrency open to the node."""
+        saved, snrc._rpc_pool = snrc._rpc_pool, queue.LifoQueue(maxsize=1)
+        try:
+            body = json.dumps({"jsonrpc": "2.0", "id": 1, "method": "eth_blockNumber", "params": []}).encode()
+            kept, extra = snrc._new_rpc_connection(), snrc._new_rpc_connection()
+            snrc._post_pooled(kept, body)
+            # returning a connection to a full pool must not wait for a free slot
+            done = threading.Thread(target=snrc._post_pooled, args=(extra, body), daemon=True)
+            done.start()
+            done.join(5)
+            self.assertFalse(done.is_alive(), "returning a connection to a full pool blocked")
+            self.assertEqual(snrc._rpc_pool.qsize(), 1)
+            self.assertIsNotNone(kept.sock)
+            self.assertIsNone(extra.sock)
+        finally:
+            while not snrc._rpc_pool.empty():
+                snrc._rpc_pool.get_nowait().close()
+            snrc._rpc_pool = saved
+
+
+class BatchedReadsTests(FakeNodeTestCase):
+    """Inside a request a round of reads is one round trip, and its contract
+    reads one multicall, because a node runs the calls of a JSON-RPC batch one
+    after another. Answers must be exactly those of reading one call at a time."""
+
+    def batched(self, answer, name):
+        def action():
+            with snrc.request_reads():
+                return answer(name)
+        return self.requests_made(action)
+
+    def assert_same_answer(self, answer, name, round_trips):
+        one_by_one, one_by_one_trips = self.requests_made(lambda: answer(name))
+        batched, batched_trips = self.batched(answer, name)
+        self.assertEqual(batched, one_by_one)
+        self.assertEqual(batched_trips, round_trips)
+        self.assertGreater(one_by_one_trips, round_trips)
+        return batched
+
+    def test_a_registered_name_takes_two_round_trips(self):
+        status, body = self.assert_same_answer(snrc.registration, "acme.testing", 2)
+        record = body["registration"]["nameRecord"]
+        self.assertEqual(record["nickname"], "Acme")
+        self.assertEqual(record["simplexChannel"], ["https://a.example/c#1", "https://b.example/c#2"])
+        self.assertIsNone(record["btc"])
+
+    def test_a_hashed_query_is_named_from_the_same_round_trip(self):
+        hashed = "[" + snrc.keccak(b"acme").hex() + "].testing"
+        status, body = self.assert_same_answer(snrc.registration, hashed, 2)
+        self.assertEqual(body["registration"]["nameRecord"]["name"], "acme.testing")
+
+    def test_an_available_name_is_priced_in_three_round_trips(self):
+        status, body = self.assert_same_answer(snrc.registration, "free.testing", 3)
+        self.assertEqual(body["registration"]["type"], "available")
+
+    def test_v1_answers_the_same(self):
+        self.assert_same_answer(snrc.resolve, "acme.testing", 2)
+
+    def test_without_multicall_a_round_is_still_one_batch(self):
+        self.node.multicall = False
+        with self.assertLogs("snrc_resolve", "WARNING") as logs:
+            self.assert_same_answer(snrc.registration, "acme.testing", 4)
+        [record] = logs.records
+        self.assertEqual((record.getMessage(), record.fields["fallback"]), ("multicall_unavailable", "batch"))
+
+    def test_a_multicall_without_a_result_falls_back_to_a_batch(self):
+        self.node.multicall_null = True
+        with self.assertLogs("snrc_resolve", "WARNING") as logs:
+            self.assert_same_answer(snrc.registration, "acme.testing", 4)
+        self.assertEqual([r.getMessage() for r in logs.records], ["multicall_unavailable"])
+
+    def test_an_answer_with_neither_result_nor_error_is_not_taken_as_one(self):
+        reads = {}
+        snrc._remember(reads, [("eth_blockNumber", [])], [{"jsonrpc": "2.0", "id": 0}])
+        self.assertEqual(reads, {})
+
+    def test_a_node_that_does_not_batch_is_read_one_call_at_a_time(self):
+        self.node.batch = False
+        one_by_one, one_by_one_trips = self.requests_made(lambda: snrc.registration("acme.testing"))
+        with self.assertNoLogs("snrc_resolve"):
+            batched, batched_trips = self.batched(snrc.registration, "acme.testing")
+        self.assertEqual(batched, one_by_one)
+        # one refused batch per round, then the reads the batch would have made
+        self.assertEqual(batched_trips, one_by_one_trips + 2)
+
+    def test_a_read_reverted_in_the_multicall_is_a_reverted_call(self):
+        with snrc.request_reads():
+            snrc.prefetch([snrc.eth_call_read(*snrc.grace_call(FakeChain.REGISTRAR)), snrc.eth_call_read(FakeChain.REGISTRY, "0xdeadbeef")])
+            _, trips = self.requests_made(lambda: self.assertRaises(RuntimeError, snrc.eth_call, FakeChain.REGISTRY, "0xdeadbeef"))
+        self.assertEqual(trips, 0)
+
+    def test_outside_a_request_nothing_is_prefetched(self):
+        _, trips = self.requests_made(lambda: snrc.prefetch(snrc.lookup_reads("acme.testing")))
+        self.assertEqual(trips, 0)
+
+
+class Aggregate3Tests(unittest.TestCase):
+    def test_calls_are_encoded_as_multicall3_reads_them(self):
+        calls = [(FakeChain.REGISTRY, "0x0178b8bf" + "11" * 32), (FakeChain.RESOLVER, "0x59d1d43c" + "22" * 100)]
+        data = snrc.encode_aggregate3(calls)
+        self.assertTrue(data.startswith(snrc.AGGREGATE3))
+        self.assertEqual(_decode_aggregate3_calls(data), calls)
+
+    def test_results_are_decoded_with_their_success_flags(self):
+        results = [(True, b"\x01" * 40), (False, b""), (True, b"")]
+        self.assertEqual(snrc.decode_aggregate3(_encode_aggregate3_results(results)), results)
+
+    def test_a_truncated_answer_is_refused(self):
+        whole = _encode_aggregate3_results([(True, b"\x01" * 40)])
+        with self.assertRaises(ValueError):
+            snrc.decode_aggregate3(whole[:-64])
+        with self.assertRaises(ValueError):
+            snrc.decode_aggregate3("0x")
+
+
+class RequestLogTests(unittest.TestCase):
+    """Each request is one event with the client it came from, behind a reverse
+    proxy the client the proxy names rather than the proxy itself."""
+
+    def setUp(self):
+        self.server = snrc.ResolverServer(("127.0.0.1", 0), snrc.Handler)
+        threading.Thread(target=self.server.serve_forever, args=(0.05,), daemon=True).start()
+        self._saved = snrc.TRUSTED_PROXIES
+
+    def tearDown(self):
+        snrc.TRUSTED_PROXIES = self._saved
+        self.server.shutdown()
+        self.server.server_close()
+
+    def request(self, path, headers=None, level="INFO"):
+        url = f"http://127.0.0.1:{self.server.server_address[1]}{path}"
+        with self.assertLogs("snrc_resolve", level) as logs:
+            try:
+                urlopen(Request(url, headers=headers or {}), timeout=5).read()
+            except HTTPError:
+                pass
+            deadline = time.monotonic() + 2
+            while not any(r.getMessage() == "request" for r in logs.records) and time.monotonic() < deadline:
+                time.sleep(0.01)
+        [record] = [r for r in logs.records if r.getMessage() == "request"]
+        return record
+
+    def test_a_request_is_logged_with_its_outcome(self):
+        record = self.request("/v2/resolve/x.simplex")
+        fields = record.fields
+        self.assertEqual(record.levelname, "INFO")
+        self.assertEqual(
+            (fields["client"], fields["method"], fields["path"], fields["status"], fields["worker"]),
+            ("127.0.0.1", "GET", "/v2/resolve/x.simplex", 400, os.getpid()),
+        )
+        self.assertGreater(fields["bytes"], 0)
+        self.assertGreaterEqual(fields["ms"], 0)
+
+    def test_a_trusted_proxy_names_the_client(self):
+        snrc.TRUSTED_PROXIES = (ipaddress.ip_network("127.0.0.1/32"),)
+        record = self.request("/v2/resolve/x.simplex", {"X-Forwarded-For": "203.0.113.7"})
+        self.assertEqual(record.fields["client"], "203.0.113.7")
+
+    def test_any_other_peer_cannot_name_itself(self):
+        record = self.request("/v2/resolve/x.simplex", {"X-Forwarded-For": "203.0.113.7"})
+        self.assertEqual(record.fields["client"], "127.0.0.1")
+
+    def test_a_bad_request_is_not_logged_with_the_previous_ones_fields(self):
+        """On a kept-alive connection the handler still held the last request's path and size."""
+        with self.assertLogs("snrc_resolve", "INFO") as logs:
+            with socket.create_connection(self.server.server_address, timeout=5) as sock:
+                sock.sendall(b"GET /v2/resolve/x.simplex HTTP/1.1\r\nHost: x\r\n\r\n")
+                sock.recv(65536)
+                sock.sendall(b"BOGUS\r\n\r\n")
+                while sock.recv(65536):
+                    pass
+        first, second = [r for r in logs.records if r.getMessage() == "request"]
+        self.assertEqual(first.fields["path"], "/v2/resolve/x.simplex")
+        self.assertEqual((second.fields["status"], second.fields["path"], second.fields["bytes"], second.fields["ms"]), (400, None, None, None))
+
+    def test_health_checks_are_logged_only_at_debug(self):
+        """The container checks /health every 30 s."""
+        record = self.request("/health", level="DEBUG")
+        self.assertEqual(record.levelname, "DEBUG")
+
+
+class RequestBodyTests(unittest.TestCase):
+    """Behind a reverse proxy that reuses connections, a body left unread would be
+    answered as another request, and that answer given to the proxy's next client."""
+
+    def setUp(self):
+        self.server = snrc.ResolverServer(("127.0.0.1", 0), snrc.Handler)
+        threading.Thread(target=self.server.serve_forever, args=(0.05,), daemon=True).start()
+        self._log = contextlib.redirect_stderr(io.StringIO())
+        self._log.__enter__()
+
+    def tearDown(self):
+        self._log.__exit__(None, None, None)
+        self.server.shutdown()
+        self.server.server_close()
+
+    def exchange(self, head: bytes, body: bytes = b"") -> bytes:
+        with socket.create_connection(self.server.server_address, timeout=5) as sock:
+            sock.sendall(head + body)
+            data = b""
+            while chunk := sock.recv(65536):
+                data += chunk
+                if data.count(b"HTTP/1.1 ") > 1:
+                    break
+            return data
+
+    SMUGGLED = b"GET /v2/resolve/y.simplex HTTP/1.1\r\nHost: x\r\n\r\n"
+
+    def test_a_body_is_refused_and_the_connection_closed(self):
+        data = self.exchange(b"GET /v2/resolve/x.simplex HTTP/1.1\r\nHost: x\r\nContent-Length: %d\r\n\r\n" % len(self.SMUGGLED), self.SMUGGLED)
+        self.assertEqual(data.count(b"HTTP/1.1 "), 1)
+        self.assertIn(b"HTTP/1.1 400 ", data)
+        self.assertIn(b"Connection: close", data)
+        self.assertNotIn(b"y.simplex", data)
+
+    def test_a_length_after_a_malformed_header_line_is_still_seen(self):
+        """Python stops parsing headers at a malformed line and keeps the rest as payload."""
+        data = self.exchange(b"GET /v2/resolve/x.simplex HTTP/1.1\r\nX-Junk : 1\r\nContent-Length: %d\r\n\r\n" % len(self.SMUGGLED), self.SMUGGLED)
+        self.assertEqual(data.count(b"HTTP/1.1 "), 1)
+        self.assertIn(b"unexpectedBody", data)
+        self.assertNotIn(b"y.simplex", data)
+
+    def test_a_chunked_body_is_refused_and_the_connection_closed(self):
+        chunked = b"%x\r\n" % len(self.SMUGGLED) + self.SMUGGLED + b"\r\n0\r\n\r\n"
+        data = self.exchange(b"GET /v2/resolve/x.simplex HTTP/1.1\r\nHost: x\r\nTransfer-Encoding: chunked\r\n\r\n", chunked)
+        self.assertEqual(data.count(b"HTTP/1.1 "), 1)
+        self.assertNotIn(b"y.simplex", data)
+
+    def test_an_empty_body_keeps_the_connection(self):
+        conn = http.client.HTTPConnection(*self.server.server_address, timeout=5)
+        answers = []
+        for _ in range(2):
+            conn.request("GET", "/v2/resolve/x.simplex", headers={"Content-Length": "0"})
+            res = conn.getresponse()
+            answers.append((res.status, json.loads(res.read())["error"], conn.sock))
+        conn.close()
+        self.assertEqual([a[:2] for a in answers], [(400, "tldNotConfigured")] * 2)
+        self.assertIs(answers[0][2], answers[1][2])
+
+
+class PublicEndpointTests(unittest.TestCase):
+    def test_credentials_path_and_query_are_not_shown(self):
+        self.assertEqual(snrc.public_endpoint("https://user:[email protected]:8443/v2/KEY?apikey=K"), "https://rpc.example:8443")
+
+    def test_a_plain_endpoint_is_unchanged(self):
+        self.assertEqual(snrc.public_endpoint("http://reth:8545"), "http://reth:8545")
+
+    def test_an_ipv6_host_keeps_its_brackets(self):
+        self.assertEqual(snrc.public_endpoint("http://u:p@[::1]:8545/key"), "http://[::1]:8545")
+
+    def test_a_malformed_port_does_not_break_health(self):
+        self.assertEqual(snrc.public_endpoint("http://reth:port"), "http://reth:port")
+
+
+class ClientAddressTests(unittest.TestCase):
+    PROXY = "172.18.0.1"
+
+    def setUp(self):
+        self._saved = snrc.TRUSTED_PROXIES
+        snrc.TRUSTED_PROXIES = (ipaddress.ip_network("172.16.0.0/12"),)
+
+    def tearDown(self):
+        snrc.TRUSTED_PROXIES = self._saved
+
+    def test_a_peer_that_is_no_proxy_is_the_client(self):
+        self.assertEqual(snrc.client_address("198.51.100.9", "203.0.113.7"), "198.51.100.9")
+
+    def test_without_the_header_the_proxy_is_the_client(self):
+        self.assertEqual(snrc.client_address(self.PROXY, None), self.PROXY)
+
+    def test_the_address_the_proxy_added_is_the_client(self):
+        self.assertEqual(snrc.client_address(self.PROXY, "203.0.113.7"), "203.0.113.7")
+
+    def test_addresses_a_client_sent_itself_are_not_believed(self):
+        """A proxy appends the address it saw, so only the last untrusted one is known."""
+        self.assertEqual(snrc.client_address(self.PROXY, "6.6.6.6, 203.0.113.7"), "203.0.113.7")
+
+    def test_trusted_proxies_in_a_chain_are_skipped(self):
+        self.assertEqual(snrc.client_address(self.PROXY, "203.0.113.7, 172.18.0.5"), "203.0.113.7")
+
+    def test_an_address_that_does_not_parse_is_not_believed(self):
+        self.assertEqual(snrc.client_address(self.PROXY, "203.0.113.7, not-an-ip"), self.PROXY)
+
+    def test_an_ipv4_mapped_peer_is_matched_as_ipv4(self):
+        self.assertEqual(snrc.client_address("::ffff:172.18.0.1", "203.0.113.7"), "203.0.113.7")
+
+    def test_an_ipv6_client_is_kept(self):
+        self.assertEqual(snrc.client_address(self.PROXY, "2001:db8::7"), "2001:db8::7")
+
+
+class RequestErrorTests(unittest.TestCase):
+    def setUp(self):
+        self.server = snrc.ResolverServer(("127.0.0.1", 0), snrc.Handler)
+
+    def tearDown(self):
+        self.server.server_close()
+
+    def test_a_client_that_hung_up_is_a_warning_without_a_traceback(self):
+        """What an smp-server that gave up after its timeout leaves behind."""
+        with self.assertLogs("snrc_resolve", "WARNING") as logs:
+            try:
+                raise BrokenPipeError
+            except BrokenPipeError:
+                self.server.handle_error(None, ("198.51.100.9", 4000))
+        [record] = logs.records
+        self.assertEqual((record.levelname, record.getMessage(), record.fields["error"]), ("WARNING", "client_gone", "BrokenPipeError"))
+        self.assertIsNone(record.exc_info)
+
+    def test_a_failure_is_an_error_with_its_traceback(self):
+        with self.assertLogs("snrc_resolve", "ERROR") as logs:
+            try:
+                raise KeyError("boom")
+            except KeyError:
+                self.server.handle_error(None, ("198.51.100.9", 4000))
+        [record] = logs.records
+        self.assertEqual((record.getMessage(), record.fields["client"]), ("request_failed", "198.51.100.9"))
+        self.assertIsNotNone(record.exc_info)
+
+
+class KeepAliveTests(unittest.TestCase):
+    """The smp-server keeps a resolver connection only after an HTTP/1.1
+    response, and otherwise connects for every lookup."""
+
+    def setUp(self):
+        self.server = snrc.ResolverServer(("127.0.0.1", 0), snrc.Handler)
+        threading.Thread(target=self.server.serve_forever, args=(0.05,), daemon=True).start()
+        self._saved_timeout = snrc.Handler.timeout
+        self._log = contextlib.redirect_stderr(io.StringIO())
+        self._log.__enter__()
+
+    def tearDown(self):
+        self._log.__exit__(None, None, None)
+        snrc.Handler.timeout = self._saved_timeout
+        self.server.shutdown()
+        self.server.server_close()
+
+    def test_requests_share_one_connection_without_delay(self):
+        conn = http.client.HTTPConnection(*self.server.server_address, timeout=5)
+        start = time.monotonic()
+        for i in range(20):
+            conn.request("GET", "/v2/resolve/x.simplex")
+            res = conn.getresponse()
+            res.read()
+            self.assertEqual((res.version, res.status), (11, 400))
+            if i == 0:
+                sock = conn.sock
+            self.assertIs(conn.sock, sock)
+        conn.close()
+        # a response held by Nagle for the delayed ACK takes ~40 ms, 20 of them over 0.8 s
+        self.assertLess(time.monotonic() - start, 0.5)
+
+    def test_idle_connections_outlast_the_smp_servers(self):
+        """http-client drops a connection idle for 30 s, checked every 5 s. A
+        resolver that closed sooner would race the client reusing it, and one
+        that never closed would keep a thread per dead connection."""
+        self.assertIsNotNone(self._saved_timeout)
+        self.assertGreater(self._saved_timeout, 35)
+
+    def test_an_idle_connection_is_closed(self):
+        snrc.Handler.timeout = 0.2
+        with socket.create_connection(self.server.server_address, timeout=5) as sock:
+            time.sleep(0.5)
+            self.assertEqual(sock.recv(1), b"")
+
+
+@unittest.skipUnless(sys.platform.startswith("linux"), "reads worker processes from /proc")
+class WorkerProcessesTests(unittest.TestCase):
+    """Workers share the port, and the service stops as a whole, so the
+    container restarts rather than serving on fewer workers."""
+
+    def setUp(self):
+        self.node = FakeNode()
+        with socket.socket() as s:
+            s.bind(("127.0.0.1", 0))
+            self.port = s.getsockname()[1]
+        env = dict(os.environ, SNRC_RPC=self.node.url, SNRC_BIND="127.0.0.1", SNRC_PORT=str(self.port), SNRC_WORKERS="2",
+                   SNRC_REGISTRY_TESTING=FakeChain.REGISTRY, SNRC_REGISTRAR_TESTING=FakeChain.REGISTRAR, SNRC_CONTROLLER_TESTING=FakeChain.CONTROLLER)
+        self.service = subprocess.Popen([sys.executable, os.path.join(_HERE, "snrc-resolve.py")], env=env, stderr=subprocess.PIPE, text=True)
+        self.workers = self._wait_for_workers(2)
+
+    def tearDown(self):
+        if self.service.poll() is None:
+            self.service.kill()
+            self.service.wait()
+        self.service.stderr.close()
+        for pid in self.workers:
+            with contextlib.suppress(ProcessLookupError):
+                os.kill(pid, signal.SIGKILL)
+        self.node.stop()
+
+    def _wait_for_workers(self, count):
+        deadline = time.monotonic() + 10
+        while time.monotonic() < deadline:
+            workers = [int(p) for p in os.listdir("/proc") if p.isdigit() and self._parent(p) == self.service.pid]
+            if len(workers) == count and self._serving():
+                return workers
+            time.sleep(0.05)
+        self.fail("workers did not start")
+
+    @staticmethod
+    def _parent(pid):
+        try:
+            with open(f"/proc/{pid}/stat") as f:
+                return int(f.read().rsplit(")", 1)[1].split()[1])
+        except (FileNotFoundError, ProcessLookupError):
+            return None
+
+    def _serving(self):
+        try:
+            with socket.create_connection(("127.0.0.1", self.port), timeout=1):
+                return True
+        except OSError:
+            return False
+
+    def _gone(self, pid):
+        deadline = time.monotonic() + 5
+        while time.monotonic() < deadline:
+            if self._parent(pid) != self.service.pid:
+                return True
+            time.sleep(0.05)
+        return False
+
+    def test_workers_answer_on_the_shared_port(self):
+        for _ in range(20):
+            with urlopen(f"http://127.0.0.1:{self.port}/v2/resolve/acme.testing", timeout=5) as res:
+                self.assertEqual(json.loads(res.read())["registration"]["type"], "registered")
+
+    def test_stopping_the_service_stops_every_worker(self):
+        self.service.send_signal(signal.SIGTERM)
+        self.assertEqual(self.service.wait(timeout=5), 0)
+        self.assertTrue(all(self._gone(pid) for pid in self.workers))
+        self.assertRegex(self.service.stderr.read(), r"INFO  stopping  signal=SIGTERM")
+
+    def test_a_worker_exiting_stops_the_service(self):
+        os.kill(self.workers[0], signal.SIGKILL)
+        self.assertEqual(self.service.wait(timeout=5), 1)
+        self.assertTrue(self._gone(self.workers[1]))
+
+
+TIME = r"\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{3}Z"
+
+
+class LogTestCase(unittest.TestCase):
+    def setUp(self):
+        self._saved = (snrc.LOG_FORMAT, snrc.LOG_COLOR, snrc.LOG_LEVEL)
+        snrc.LOG_FORMAT, snrc.LOG_COLOR, snrc.LOG_LEVEL = "text", "never", "info"
+        self.out = io.StringIO()
+
+    def tearDown(self):
+        snrc.LOG_FORMAT, snrc.LOG_COLOR, snrc.LOG_LEVEL = self._saved
+        snrc.LOGGER.handlers[:] = []
+        snrc.LOGGER.setLevel(logging.NOTSET)
+        snrc.LOGGER.propagate = True
+
+    def emit(self, level=logging.INFO, name="request", **fields):
+        snrc.setup_logging(self.out)
+        snrc.log_event(level, name, **fields)
+        return self.out.getvalue()
+
+
+class TextFormatTests(LogTestCase):
+    def test_a_line_is_utc_time_level_event_and_fields(self):
+        line = self.emit(client="203.0.113.7", path="/v2/resolve/[4fdd].testing", status=200, ms=7)
+        self.assertRegex(line, rf"^{TIME} INFO  request  client=203\.0\.113\.7 path=/v2/resolve/\[4fdd\]\.testing status=200 ms=7\n$")
+
+    def test_values_that_would_be_misread_are_quoted(self):
+        line = self.emit(message='says "hi" here', empty="", missing=None, eq="a=b")
+        self.assertIn('message="says \\"hi\\" here" empty="" missing=- eq="a=b"', line)
+
+    def test_levels_are_named_in_five_columns(self):
+        self.assertRegex(self.emit(logging.WARNING, "upstream_error"), rf"^{TIME} WARN  upstream_error\n$")
+
+    def test_below_the_configured_level_nothing_is_written(self):
+        self.assertEqual(self.emit(logging.DEBUG), "")
+
+    def test_an_exception_follows_its_line(self):
+        snrc.setup_logging(self.out)
+        try:
+            raise KeyError("boom")
+        except KeyError:
+            snrc.log_event(logging.ERROR, "request_failed", exc_info=True)
+        self.assertRegex(self.out.getvalue(), rf"(?s)^{TIME} ERROR request_failed\nTraceback .*KeyError: 'boom'\n$")
+
+
+class ColorTests(LogTestCase):
+    def test_colours_mark_the_level_and_the_status_class(self):
+        snrc.LOG_COLOR = "always"
+        line = self.emit(status=503)
+        self.assertIn("\033[32mINFO ", line)
+        self.assertIn("\033[31m503\033[0m", line)
+
+    def test_auto_leaves_output_that_is_no_terminal_plain(self):
+        snrc.LOG_COLOR = "auto"
+        self.assertNotIn("\033[", self.emit(status=200))
+
+    def test_never_is_plain(self):
+        self.assertNotIn("\033[", self.emit(status=200))
+
+
+class JsonFormatTests(LogTestCase):
+    def test_a_line_is_one_json_object(self):
+        snrc.LOG_FORMAT = "json"
+        record = json.loads(self.emit(client="203.0.113.7", status=200, missing=None))
+        self.assertRegex(record.pop("time"), rf"^{TIME}$")
+        self.assertEqual(record, {"level": "info", "event": "request", "client": "203.0.113.7", "status": 200, "missing": None})
+
+    def test_an_exception_is_a_field(self):
+        snrc.LOG_FORMAT = "json"
+        snrc.setup_logging(self.out)
+        try:
+            raise KeyError("boom")
+        except KeyError:
+            snrc.log_event(logging.ERROR, "request_failed", exc_info=True)
+        self.assertIn("KeyError: 'boom'", json.loads(self.out.getvalue())["exception"])
+
+
+class SetupTests(LogTestCase):
+    def test_unknown_settings_are_refused_at_start(self):
+        for setting, value in (("LOG_FORMAT", "yaml"), ("LOG_COLOR", "sometimes"), ("LOG_LEVEL", "loud")):
+            with self.subTest(setting=setting):
+                saved = getattr(snrc, setting)
+                setattr(snrc, setting, value)
+                try:
+                    with self.assertRaisesRegex(ValueError, re.escape(f"SNRC_{setting}")):
+                        snrc.setup_logging(self.out)
+                finally:
+                    setattr(snrc, setting, saved)
@@ -0,0 +1,156 @@
+version = 1
+revision = 3
+requires-python = ">=3.11"
+
+[[package]]
+name = "colorama"
+version = "0.4.6"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" },
+]
+
+[[package]]
+name = "eth-hash"
+version = "0.8.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/3c/f5/c67fc24f2f676aa9b7ab29679d44f113f314c817207cd4319353356f62da/eth_hash-0.8.0.tar.gz", hash = "sha256:b009752b620da2e9c7668014849d1f5fadbe4f138603f1871cc5d4ca706896b1", size = 12225, upload-time = "2026-03-25T16:36:55.099Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/87/87/b36792150ca0b28e4df683a34be15a61461ca0e349e5b5cf3ec8f694edb9/eth_hash-0.8.0-py3-none-any.whl", hash = "sha256:523718a51b369ab89866b929a5c93c52978cd866ea309192ad980dd8271f9fac", size = 7965, upload-time = "2026-03-25T16:36:54.205Z" },
+]
+
+[package.optional-dependencies]
+pycryptodome = [
+    { name = "pycryptodome" },
+]
+
+[[package]]
+name = "iniconfig"
+version = "2.3.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" },
+]
+
+[[package]]
+name = "packaging"
+version = "26.3"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" },
+]
+
+[[package]]
+name = "pluggy"
+version = "1.6.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" },
+]
+
+[[package]]
+name = "pycryptodome"
+version = "3.23.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/8e/a6/8452177684d5e906854776276ddd34eca30d1b1e15aa1ee9cefc289a33f5/pycryptodome-3.23.0.tar.gz", hash = "sha256:447700a657182d60338bab09fdb27518f8856aecd80ae4c6bdddb67ff5da44ef", size = 4921276, upload-time = "2025-05-17T17:21:45.242Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/04/5d/bdb09489b63cd34a976cc9e2a8d938114f7a53a74d3dd4f125ffa49dce82/pycryptodome-3.23.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:0011f7f00cdb74879142011f95133274741778abba114ceca229adbf8e62c3e4", size = 2495152, upload-time = "2025-05-17T17:20:20.833Z" },
+    { url = "https://files.pythonhosted.org/packages/a7/ce/7840250ed4cc0039c433cd41715536f926d6e86ce84e904068eb3244b6a6/pycryptodome-3.23.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:90460fc9e088ce095f9ee8356722d4f10f86e5be06e2354230a9880b9c549aae", size = 1639348, upload-time = "2025-05-17T17:20:23.171Z" },
+    { url = "https://files.pythonhosted.org/packages/ee/f0/991da24c55c1f688d6a3b5a11940567353f74590734ee4a64294834ae472/pycryptodome-3.23.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4764e64b269fc83b00f682c47443c2e6e85b18273712b98aa43bcb77f8570477", size = 2184033, upload-time = "2025-05-17T17:20:25.424Z" },
+    { url = "https://files.pythonhosted.org/packages/54/16/0e11882deddf00f68b68dd4e8e442ddc30641f31afeb2bc25588124ac8de/pycryptodome-3.23.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eb8f24adb74984aa0e5d07a2368ad95276cf38051fe2dc6605cbcf482e04f2a7", size = 2270142, upload-time = "2025-05-17T17:20:27.808Z" },
+    { url = "https://files.pythonhosted.org/packages/d5/fc/4347fea23a3f95ffb931f383ff28b3f7b1fe868739182cb76718c0da86a1/pycryptodome-3.23.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d97618c9c6684a97ef7637ba43bdf6663a2e2e77efe0f863cce97a76af396446", size = 2309384, upload-time = "2025-05-17T17:20:30.765Z" },
+    { url = "https://files.pythonhosted.org/packages/6e/d9/c5261780b69ce66d8cfab25d2797bd6e82ba0241804694cd48be41add5eb/pycryptodome-3.23.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9a53a4fe5cb075075d515797d6ce2f56772ea7e6a1e5e4b96cf78a14bac3d265", size = 2183237, upload-time = "2025-05-17T17:20:33.736Z" },
+    { url = "https://files.pythonhosted.org/packages/5a/6f/3af2ffedd5cfa08c631f89452c6648c4d779e7772dfc388c77c920ca6bbf/pycryptodome-3.23.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:763d1d74f56f031788e5d307029caef067febf890cd1f8bf61183ae142f1a77b", size = 2343898, upload-time = "2025-05-17T17:20:36.086Z" },
+    { url = "https://files.pythonhosted.org/packages/9a/dc/9060d807039ee5de6e2f260f72f3d70ac213993a804f5e67e0a73a56dd2f/pycryptodome-3.23.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:954af0e2bd7cea83ce72243b14e4fb518b18f0c1649b576d114973e2073b273d", size = 2269197, upload-time = "2025-05-17T17:20:38.414Z" },
+    { url = "https://files.pythonhosted.org/packages/f9/34/e6c8ca177cb29dcc4967fef73f5de445912f93bd0343c9c33c8e5bf8cde8/pycryptodome-3.23.0-cp313-cp313t-win32.whl", hash = "sha256:257bb3572c63ad8ba40b89f6fc9d63a2a628e9f9708d31ee26560925ebe0210a", size = 1768600, upload-time = "2025-05-17T17:20:40.688Z" },
+    { url = "https://files.pythonhosted.org/packages/e4/1d/89756b8d7ff623ad0160f4539da571d1f594d21ee6d68be130a6eccb39a4/pycryptodome-3.23.0-cp313-cp313t-win_amd64.whl", hash = "sha256:6501790c5b62a29fcb227bd6b62012181d886a767ce9ed03b303d1f22eb5c625", size = 1799740, upload-time = "2025-05-17T17:20:42.413Z" },
+    { url = "https://files.pythonhosted.org/packages/5d/61/35a64f0feaea9fd07f0d91209e7be91726eb48c0f1bfc6720647194071e4/pycryptodome-3.23.0-cp313-cp313t-win_arm64.whl", hash = "sha256:9a77627a330ab23ca43b48b130e202582e91cc69619947840ea4d2d1be21eb39", size = 1703685, upload-time = "2025-05-17T17:20:44.388Z" },
+    { url = "https://files.pythonhosted.org/packages/db/6c/a1f71542c969912bb0e106f64f60a56cc1f0fabecf9396f45accbe63fa68/pycryptodome-3.23.0-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:187058ab80b3281b1de11c2e6842a357a1f71b42cb1e15bce373f3d238135c27", size = 2495627, upload-time = "2025-05-17T17:20:47.139Z" },
+    { url = "https://files.pythonhosted.org/packages/6e/4e/a066527e079fc5002390c8acdd3aca431e6ea0a50ffd7201551175b47323/pycryptodome-3.23.0-cp37-abi3-macosx_10_9_x86_64.whl", hash = "sha256:cfb5cd445280c5b0a4e6187a7ce8de5a07b5f3f897f235caa11f1f435f182843", size = 1640362, upload-time = "2025-05-17T17:20:50.392Z" },
+    { url = "https://files.pythonhosted.org/packages/50/52/adaf4c8c100a8c49d2bd058e5b551f73dfd8cb89eb4911e25a0c469b6b4e/pycryptodome-3.23.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:67bd81fcbe34f43ad9422ee8fd4843c8e7198dd88dd3d40e6de42ee65fbe1490", size = 2182625, upload-time = "2025-05-17T17:20:52.866Z" },
+    { url = "https://files.pythonhosted.org/packages/5f/e9/a09476d436d0ff1402ac3867d933c61805ec2326c6ea557aeeac3825604e/pycryptodome-3.23.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c8987bd3307a39bc03df5c8e0e3d8be0c4c3518b7f044b0f4c15d1aa78f52575", size = 2268954, upload-time = "2025-05-17T17:20:55.027Z" },
+    { url = "https://files.pythonhosted.org/packages/f9/c5/ffe6474e0c551d54cab931918127c46d70cab8f114e0c2b5a3c071c2f484/pycryptodome-3.23.0-cp37-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aa0698f65e5b570426fc31b8162ed4603b0c2841cbb9088e2b01641e3065915b", size = 2308534, upload-time = "2025-05-17T17:20:57.279Z" },
+    { url = "https://files.pythonhosted.org/packages/18/28/e199677fc15ecf43010f2463fde4c1a53015d1fe95fb03bca2890836603a/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:53ecbafc2b55353edcebd64bf5da94a2a2cdf5090a6915bcca6eca6cc452585a", size = 2181853, upload-time = "2025-05-17T17:20:59.322Z" },
+    { url = "https://files.pythonhosted.org/packages/ce/ea/4fdb09f2165ce1365c9eaefef36625583371ee514db58dc9b65d3a255c4c/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_i686.whl", hash = "sha256:156df9667ad9f2ad26255926524e1c136d6664b741547deb0a86a9acf5ea631f", size = 2342465, upload-time = "2025-05-17T17:21:03.83Z" },
+    { url = "https://files.pythonhosted.org/packages/22/82/6edc3fc42fe9284aead511394bac167693fb2b0e0395b28b8bedaa07ef04/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:dea827b4d55ee390dc89b2afe5927d4308a8b538ae91d9c6f7a5090f397af1aa", size = 2267414, upload-time = "2025-05-17T17:21:06.72Z" },
+    { url = "https://files.pythonhosted.org/packages/59/fe/aae679b64363eb78326c7fdc9d06ec3de18bac68be4b612fc1fe8902693c/pycryptodome-3.23.0-cp37-abi3-win32.whl", hash = "sha256:507dbead45474b62b2bbe318eb1c4c8ee641077532067fec9c1aa82c31f84886", size = 1768484, upload-time = "2025-05-17T17:21:08.535Z" },
+    { url = "https://files.pythonhosted.org/packages/54/2f/e97a1b8294db0daaa87012c24a7bb714147c7ade7656973fd6c736b484ff/pycryptodome-3.23.0-cp37-abi3-win_amd64.whl", hash = "sha256:c75b52aacc6c0c260f204cbdd834f76edc9fb0d8e0da9fbf8352ef58202564e2", size = 1799636, upload-time = "2025-05-17T17:21:10.393Z" },
+    { url = "https://files.pythonhosted.org/packages/18/3d/f9441a0d798bf2b1e645adc3265e55706aead1255ccdad3856dbdcffec14/pycryptodome-3.23.0-cp37-abi3-win_arm64.whl", hash = "sha256:11eeeb6917903876f134b56ba11abe95c0b0fd5e3330def218083c7d98bbcb3c", size = 1703675, upload-time = "2025-05-17T17:21:13.146Z" },
+]
+
+[[package]]
+name = "pygments"
+version = "2.21.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz", hash = "sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", size = 5005329, upload-time = "2026-08-17T08:02:48.824Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" },
+]
+
+[[package]]
+name = "pytest"
+version = "9.1.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "colorama", marker = "sys_platform == 'win32'" },
+    { name = "iniconfig" },
+    { name = "packaging" },
+    { name = "pluggy" },
+    { name = "pygments" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" },
+]
+
+[[package]]
+name = "ruff"
+version = "0.16.9"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/96/bf/c935ca98e73fe8ce65b87ef08a280c0c1e85295d569228c15e87d8fdfaf1/ruff-0.16.9.tar.gz", hash = "sha256:12b625c6cfba78d285d9f48eda5f053374f1e53cb10ef17342a383750db99161", size = 4948764, upload-time = "2026-09-24T20:37:49.416Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/0d/26/df51322b52ee1ada7eff2d071ea09d11d5c2d1dcc9f02594f5785c4d1635/ruff-0.16.9-py3-none-linux_armv6l.whl", hash = "sha256:95e6f022090368ab3b824c36276839c53b2adf1a3f4c09fefc33dfc400f6da96", size = 10082922, upload-time = "2026-09-24T20:37:13.045Z" },
+    { url = "https://files.pythonhosted.org/packages/a5/27/7bf51f5a7aa375e9f339280a303aab44ca75dc1525f1cdc5991761685b0f/ruff-0.16.9-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:a5f27be168556594a86d2f415db0cf43f5291917849318f873c7e2791f7a8c67", size = 10236360, upload-time = "2026-09-24T20:37:16.049Z" },
+    { url = "https://files.pythonhosted.org/packages/b6/63/09659283f92f02dff45809da194a70da2f688d87c55d8875c4fae3536072/ruff-0.16.9-py3-none-macosx_11_0_arm64.whl", hash = "sha256:1632eb1d6197f33bd00b1acbc5b71009e89a8895c158e2d2b03a834fac964ab6", size = 9892940, upload-time = "2026-09-24T20:37:17.957Z" },
+    { url = "https://files.pythonhosted.org/packages/24/58/98de1b72ec172f5f8f1731236fe21585b3998bf7dfe9fcc44ae9ba626012/ruff-0.16.9-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b3f951b14d865d5952c89d40a5ca07e87abe24fa5453299878411e127748fb1c", size = 10032114, upload-time = "2026-09-24T20:37:19.942Z" },
+    { url = "https://files.pythonhosted.org/packages/c8/7d/f1e17c54ab59d4bad1dce8ee3e22a7a1d0ef4745240decacdcf3832b5bb2/ruff-0.16.9-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:447fc07e1573afff7cb02803462b12b6c8ece7cf10e2cd78565fa6d7a1c0bf8d", size = 9910227, upload-time = "2026-09-24T20:37:21.872Z" },
+    { url = "https://files.pythonhosted.org/packages/34/19/436f647a65075bbd3bab2668b3bdaa5120559b294694018cdcefabbbf30b/ruff-0.16.9-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8a3e039a6a40ed976c491722b60e0ae4a4aa1a86057f540ee7a37a5d19ae9120", size = 10547484, upload-time = "2026-09-24T20:37:24.229Z" },
+    { url = "https://files.pythonhosted.org/packages/03/59/38430a6bc2f6d8095447ac39625cf8b6e9344a47e6c26225c8ba1bff3ffb/ruff-0.16.9-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4684dded7db60aa57cb118fa158630f5feade4af5782903b6053484bdf9bd129", size = 11412367, upload-time = "2026-09-24T20:37:26.307Z" },
+    { url = "https://files.pythonhosted.org/packages/c8/bd/bbb6d7fc7f208c8b8c50dd5dc8206e4cfdb1e7a8fb852606a3adf370c880/ruff-0.16.9-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d29c934357e45642fda2f34c0b1f4025b4a6c01e15e4bf0016879d60078a142c", size = 10869787, upload-time = "2026-09-24T20:37:28.35Z" },
+    { url = "https://files.pythonhosted.org/packages/bc/b8/9c543074918061abbefc3bd139bee22de35abedb00dde0f2d27288838962/ruff-0.16.9-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a21713e629d3e5bdb2f5c2def1cc7f04f47fa8e1a7eb0571b4a28e1da64bc728", size = 10406494, upload-time = "2026-09-24T20:37:30.624Z" },
+    { url = "https://files.pythonhosted.org/packages/35/7a/5a8851bd146e7ccf8fd4b003f6c75c11f8fbdb0e60673b45097986b6bf41/ruff-0.16.9-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:7baa24ef5fc8e77aa93879e1d3f43754a01ae488e869f1ae30cf431afd4d2452", size = 10590083, upload-time = "2026-09-24T20:37:32.439Z" },
+    { url = "https://files.pythonhosted.org/packages/87/f0/4c3467188f23f806960b46fa76575a7cd0514c9ba90562650b71efc96980/ruff-0.16.9-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:a41aac6230aadfaa133bdfa1614488531ffa3e0837567ae04c0da2058a9c0f9e", size = 10119151, upload-time = "2026-09-24T20:37:34.581Z" },
+    { url = "https://files.pythonhosted.org/packages/15/34/5a4def5adea572ce6aea0bb64f21f928ee317b80e0db747d7979b01d7261/ruff-0.16.9-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:c2529fb5896d49115b0e9aa8f887490b34bbe76baf879ec2264ac59406869ce7", size = 9911544, upload-time = "2026-09-24T20:37:36.796Z" },
+    { url = "https://files.pythonhosted.org/packages/62/5d/d15ebea7499eef9373318c0ee6ca127832927c6529731f6d48e18dce7ca9/ruff-0.16.9-py3-none-musllinux_1_2_i686.whl", hash = "sha256:41e3870277694177429b56406d65dfbdb2c2802c52b715edaf6a0b829c69d4ee", size = 10269884, upload-time = "2026-09-24T20:37:38.857Z" },
+    { url = "https://files.pythonhosted.org/packages/d1/56/c5d3cd119ded7a3c7aba0e961b69cb3df701c91662c98ad694467d060ce1/ruff-0.16.9-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:8adbe4e58af167f767d7b2ba5e83c42e878350796cf78c2f5e14ab9903a92588", size = 10749366, upload-time = "2026-09-24T20:37:41.042Z" },
+    { url = "https://files.pythonhosted.org/packages/ac/fe/734ec7527029ac757ecf821f143c9f3fcf69149c21f53a044a899b430f5a/ruff-0.16.9-py3-none-win32.whl", hash = "sha256:0e1dbc2073624dee6618d41d0098690a7244654af746704b64759e12b6b6b385", size = 10152355, upload-time = "2026-09-24T20:37:43.025Z" },
+    { url = "https://files.pythonhosted.org/packages/14/21/26e4643629b3ebb44f0a06f9c9a53058d63d989415f63a9a3c28e2ee7f22/ruff-0.16.9-py3-none-win_amd64.whl", hash = "sha256:6bd40fec8cd4c8a3d4dd589bd8ad4e6320c13c29234159bfd959a40d529d597b", size = 10592965, upload-time = "2026-09-24T20:37:44.944Z" },
+    { url = "https://files.pythonhosted.org/packages/51/60/5fb1a39dbb5ae314d5f59bc7348a63c1d5c20f3cd83914c4b5cb0be31d2d/ruff-0.16.9-py3-none-win_arm64.whl", hash = "sha256:ed1a252039200f57a59eebc063b54beabea67bfbaaca0eeaa7f54b5fbcda2284", size = 10458649, upload-time = "2026-09-24T20:37:46.882Z" },
+]
+
+[[package]]
+name = "snrc-resolve"
+version = "0.1.0"
+source = { virtual = "." }
+dependencies = [
+    { name = "eth-hash", extra = ["pycryptodome"] },
+]
+
+[package.dev-dependencies]
+dev = [
+    { name = "pytest" },
+    { name = "ruff" },
+]
+
+[package.metadata]
+requires-dist = [{ name = "eth-hash", extras = ["pycryptodome"], specifier = ">=0.7" }]
+
+[package.metadata.requires-dev]
+dev = [
+    { name = "pytest", specifier = ">=8" },
+    { name = "ruff", specifier = ">=0.6" },
+]
@@ -1,7 +1,7 @@
 cabal-version: 3.0

 name:           simplexmq
-version:        7.1.0.9
+version:        7.1.0.10
 synopsis:       SimpleXMQ message broker
 description:    This package includes <./docs/Simplex-Messaging-Server.html server>,
                <./docs/Simplex-Messaging-Client.html client> and
@@ -195,6 +195,7 @@ library
          Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260712_address_dr_rpc
          Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260823_snd_files_entitlement
          Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260919_ratchet_verify_codes
+          Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260929_ratchet_indexes
  else
      exposed-modules:
          Simplex.Messaging.Agent.Store.SQLite
@@ -250,6 +251,7 @@ library
          Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260712_address_dr_rpc
          Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260823_snd_files_entitlement
          Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260919_ratchet_verify_codes
+          Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260929_ratchet_indexes
          Simplex.Messaging.Agent.Store.SQLite.Util
  if flag(client_postgres) || flag(server_postgres)
      exposed-modules:
@@ -279,8 +281,6 @@ library
          Simplex.Messaging.Server.Main.Init
          Simplex.Messaging.Server.Web
          Simplex.Messaging.Server.MsgStore
-          Simplex.Messaging.Server.MsgStore.Journal
-          Simplex.Messaging.Server.MsgStore.Journal.SharedLock
          Simplex.Messaging.Server.MsgStore.STM
          Simplex.Messaging.Server.MsgStore.Types
          Simplex.Messaging.Server.Names
@@ -25,7 +25,6 @@ import Control.Monad.Reader
 import Control.Monad.Trans.Except
 import qualified Data.Aeson as J
 import Data.Bifunctor (first)
-import qualified Data.ByteString.Base64.URL as B64
 import Data.ByteString.Builder (Builder, byteString)
 import Data.ByteString.Char8 (ByteString)
 import qualified Data.ByteString.Char8 as B
@@ -594,7 +593,7 @@ processXFTPRequest ent HTTP2Body {bodyPart} = \case
              \used -> let used' = used + fromIntegral size in if used' <= quota then (True, used') else (False, used)
          receive = do
            path <- asks $ filesPath . config
-            let fPath = path </> B.unpack (B64.encode $ unEntityId senderId)
+            let fPath = path </> serverFileName senderId
            receiveChunk (XFTPRcvChunkSpec fPath size digest) >>= \case
              Right () -> do
                stats <- asks serverStats
@@ -17,6 +17,7 @@ module Simplex.FileTransfer.Server.StoreLog
    logDeleteFile,
    logBlockFile,
    logAckFile,
+    serverFileName,
  )
 where

@@ -24,6 +25,7 @@ import Control.Applicative (optional, (<|>))
 import Control.Concurrent.STM
 import Control.Monad.Except
 import qualified Data.Attoparsec.ByteString.Char8 as A
+import qualified Data.ByteString.Base64.URL as B64
 import qualified Data.ByteString.Char8 as B
 import qualified Data.ByteString.Lazy.Char8 as LB
 import Data.Composition ((.:), (.::.))
@@ -35,10 +37,11 @@ import Simplex.FileTransfer.Protocol (FileInfo (..))
 import Simplex.FileTransfer.Server.Store
 import Simplex.FileTransfer.Transport (XFTPErrorType (..))
 import Simplex.Messaging.Encoding.String
-import Simplex.Messaging.Protocol (BlockingInfo, RcvPublicAuthKey, RecipientId, SenderId)
+import Simplex.Messaging.Protocol (BlockingInfo, EntityId (..), RcvPublicAuthKey, RecipientId, SenderId)
 import Simplex.Messaging.Server.QueueStore (ServerEntityStatus (..))
 import Simplex.Messaging.Server.StoreLog
 import Simplex.Messaging.Util (bshow)
+import System.FilePath (takeFileName)
 import System.IO

 data FileStoreLogRecord
@@ -99,6 +102,9 @@ logBlockFile s fId = logFileStoreRecord s . BlockFile fId
 logAckFile :: StoreLog 'WriteMode -> RecipientId -> IO ()
 logAckFile s = logFileStoreRecord s . AckFile

+serverFileName :: SenderId -> FilePath
+serverFileName = B.unpack . B64.encode . unEntityId
+
 readWriteFileStore :: FilePath -> STMFileStore -> IO (StoreLog 'WriteMode)
 readWriteFileStore = readWriteStoreLog readFileStore writeFileStore

@@ -115,7 +121,9 @@ readFileStore f st = mapM_ (addFileLogRecord . LB.toStrict) . LB.lines =<< LB.re
      AddFile sId file createdAt expiresAt status
        | size file > 0 -> addFile st sId file createdAt expiresAt status
        | otherwise -> pure $ Left SIZE
-      PutFile qId path -> setFilePath st qId path
+      PutFile qId path
+        | takeFileName path == serverFileName qId -> setFilePath st qId path
+        | otherwise -> pure $ Left AUTH
      AddRecipients sId rcps -> runExceptT $ addRecipients sId rcps
      DeleteFile sId -> deleteFile st sId
      BlockFile sId info -> blockFile st sId info True
@@ -1325,7 +1325,7 @@ newRcvConnSrv c nm userId connId enableNtfs cMode userLinkData_ clientData pqIni
    Just d -> do
      (nonce, qUri, cReq, qd) <- prepareLinkData addrKeys_ (setLinkDataRatchetKeys addrKeys_ d) $ fst e2eKeys
      (rq, qUri') <- createRcvQueue c nm userId connId srvWithAuth enableNtfs subMode (Just nonce) qd e2eKeys
-      connReqWithShortLink qUri cReq qUri' (shortLink rq)
+      connReqWithShortLink pqInitKeys qUri cReq qUri' rq
    Nothing -> do
      let qd = case cMode of SCMContact -> CQRContact Nothing; SCMInvitation -> CQRMessaging Nothing
      (_rq, qUri) <- createRcvQueue c nm userId connId srvWithAuth enableNtfs subMode Nothing qd e2eKeys
@@ -1372,23 +1372,19 @@ newRcvConnSrv c nm userId connId enableNtfs cMode userLinkData_ clientData pqIni
          srvData <- liftError id $ SL.encryptLinkData g k linkData
          pure $ CQRMessaging $ Just CQRData {linkKey, privSigKey, srvReq = (sndId, srvData)}
      pure (nonce, qUri, connReq, qd)
-    connReqWithShortLink :: SMPQueueUri -> ConnectionRequestUri c -> SMPQueueUri -> Maybe ShortLinkCreds -> AM (CreatedConnLink c)
-    connReqWithShortLink qUri cReq qUri' shortLink = case shortLink of
-      Just ShortLinkCreds {shortLinkId, shortLinkKey}
-        | qUri == qUri' -> pure $ case cReq of
-            CRContactUri _ _ -> CCLink cReq $ Just $ CSLContact SLSServer CCTContact srv shortLinkKey
-            CRInvitationUri crData (CR.E2ERatchetParamsUri vr k1 k2 _) ->
-              let cReq' = case pqInitKeys of
-                    CR.IKPQOn -> CRInvitationUri crData $ CR.E2ERatchetParamsUri vr k1 k2 Nothing -- remove PQ keys
-                    _ -> cReq -- either PQ is disabled, or disabled for initial request because there is no short link
-               in CCLink cReq' $ Just $ CSLInvitation SLSServer srv shortLinkId shortLinkKey
-        | otherwise -> throwE $ INTERNAL "different rcv queue address"
-      Nothing ->
-        let updated (ConnReqUriData _ vr _ _) = (ConnReqUriData SSSimplex vr [qUri'] clientData)
-            cReq' = case cReq of
-              CRContactUri crData rk -> CRContactUri (updated crData) rk
-              CRInvitationUri crData e2eParams -> CRInvitationUri (updated crData) e2eParams
-         in pure $ CCLink cReq' Nothing
+
+connReqWithShortLink :: CR.InitialKeys -> SMPQueueUri -> ConnectionRequestUri c -> SMPQueueUri -> RcvQueue -> AM (CreatedConnLink c)
+connReqWithShortLink pqInitKeys qUri cReq qUri' RcvQueue {server = srv, shortLink} = case shortLink of
+  Just ShortLinkCreds {shortLinkId, shortLinkKey}
+    | qUri == qUri' -> pure $ case cReq of
+        CRContactUri _ _ -> CCLink cReq $ Just $ CSLContact SLSServer CCTContact srv shortLinkKey
+        CRInvitationUri crData (CR.E2ERatchetParamsUri vr k1 k2 _) ->
+          let cReq' = case pqInitKeys of
+                CR.IKPQOn -> CRInvitationUri crData $ CR.E2ERatchetParamsUri vr k1 k2 Nothing -- remove PQ keys
+                _ -> cReq -- either PQ is disabled, or disabled for initial request because there is no short link
+           in CCLink cReq' $ Just $ CSLInvitation SLSServer srv shortLinkId shortLinkKey
+    | otherwise -> throwE $ INTERNAL "different rcv queue address"
+  Nothing -> throwE $ INTERNAL "no short link credentials"

 newQueueNtfServer :: AM (Maybe NtfServer)
 newQueueNtfServer = fmap ntfServer_ . readTVarIO . ntfTkn =<< asks ntfSupervisor
@@ -2435,62 +2431,66 @@ enqueueMessageB c reqs = do
  cfg <- asks config
  (_, reqMids) <- unsafeWithStore c $ \db -> do
    mapAccumLM (\ids r -> storeSentMsg db cfg ids r `E.catchAny` \e -> (ids,) <$> handleInternal e) IM.empty reqs
-  forME reqMids $ \((csqs_, _, _, _), InternalId msgId, pqSecr) -> forM csqs_ $ \(cData, sq :| sqs) -> do
-    submitPendingMsg c sq
-    let sqs' = filter (isActiveSndQ cData) sqs
-    pure ((msgId, pqSecr), if null sqs' then Nothing else Just (sqs', msgId))
+  forME reqMids $ submitSentMsg c
  where
-    storeSentMsg ::
-      DB.Connection ->
-      AgentConfig ->
-      IntMap (Maybe Int64, AMessage) ->
-      Either AgentErrorType (Either AgentErrorType (ConnData, NonEmpty SndQueue), Maybe PQEncryption, MsgFlags, ValueOrRef AMessage) ->
-      IO (IntMap (Maybe Int64, AMessage), Either AgentErrorType ((Either AgentErrorType (ConnData, NonEmpty SndQueue), Maybe PQEncryption, MsgFlags, ValueOrRef AMessage), InternalId, PQEncryption))
-    storeSentMsg db cfg aMessageIds = \case
-      Left e -> pure (aMessageIds, Left e)
-      Right req@(csqs_, pqEnc_, msgFlags, mbr) -> case mbr of
-        VRValue i_ aMessage -> case i_ >>= (`IM.lookup` aMessageIds) of
-          Just _ -> pure (aMessageIds, Left $ INTERNAL "enqueueMessageB: storeSentMsg duplicate saved message body")
-          Nothing -> do
-            (mbId_, r) <- case csqs_ of
-              Left e -> pure (Nothing, Left e)
-              Right (cData, sq :| _) -> do
-                mbId <- createSndMsgBody db aMessage
-                (Just mbId,) <$> storeSentMsg_ cData sq mbId aMessage
-            let aMessageIds' = maybe id (`IM.insert` (mbId_, aMessage)) i_ aMessageIds
-            pure (aMessageIds', r)
-        VRRef i -> case csqs_ of
-          Left e -> pure $ (aMessageIds, Left e)
-          Right (cData, sq :| _) -> case IM.lookup i aMessageIds of
-            Just (Just mbId, aMessage) -> (aMessageIds,) <$> storeSentMsg_ cData sq mbId aMessage
-            Just (Nothing, aMessage) -> do
-              mbId <- createSndMsgBody db aMessage
-              let aMessageIds' = IM.insert i (Just mbId, aMessage) aMessageIds
-              (aMessageIds',) <$> storeSentMsg_ cData sq mbId aMessage
-            Nothing -> pure (aMessageIds, Left $ INTERNAL "enqueueMessageB: storeSentMsg missing saved message body id")
-        where
-          storeSentMsg_ cData@ConnData {connId} sq sndMsgBodyId aMessage = fmap (first storeError) $ runExceptT $ do
-            let AgentConfig {e2eEncryptVRange} = cfg
-            internalTs <- liftIO getCurrentTime
-            (internalId, internalSndId, prevMsgHash) <- ExceptT $ updateSndIds db connId
-            -- We need to do pre-flight encoding that is not stored in database
-            -- to calculate its hash and remember it on connection (createSndMsg -> updateSndMsgHash)
-            -- to enable next enqueue.
-            -- (As encoding is different per connection, we can't store shared body, so it's repeated on delivery)
-            let agentMsgStr = encodeAgentMsgStr aMessage internalSndId prevMsgHash
-                internalHash = C.sha256Hash agentMsgStr
-                currentE2EVersion = maxVersion e2eEncryptVRange
-            (mek, paddedLen, pqEnc) <- agentRatchetEncryptHeader db cData e2eEncAgentMsgLength pqEnc_ currentE2EVersion
-            withExceptT (SEAgentError . cryptoError) $ CR.rcCheckCanPad paddedLen agentMsgStr
-            let msgType = aMessageType aMessage
-                -- msgBody is empty, because snd_messages record is linked to snd_message_bodies
-                msgData = SndMsgData {internalId, internalSndId, internalTs, msgType, msgFlags, msgBody = "", pqEncryption = pqEnc, internalHash, prevMsgHash, sndMsgPrepData_ = Just SndMsgPrepData {encryptKey = mek, paddedLen, sndMsgBodyId}}
-            liftIO $ createSndMsg db connId msgData
-            liftIO $ createSndMsgDelivery db sq internalId
-            pure (req, internalId, pqEnc)
    handleInternal :: E.SomeException -> IO (Either AgentErrorType b)
    handleInternal = pure . Left . INTERNAL . show

+submitSentMsg :: AgentClient -> ((Either AgentErrorType (ConnData, NonEmpty SndQueue), Maybe PQEncryption, MsgFlags, ValueOrRef AMessage), InternalId, PQEncryption) -> AM' (Either AgentErrorType ((AgentMsgId, PQEncryption), Maybe ([SndQueue], AgentMsgId)))
+submitSentMsg c ((csqs_, _, _, _), InternalId msgId, pqSecr) = forM csqs_ $ \(cData, sq :| sqs) -> do
+  submitPendingMsg c sq
+  let sqs' = filter (isActiveSndQ cData) sqs
+  pure ((msgId, pqSecr), if null sqs' then Nothing else Just (sqs', msgId))
+
+storeSentMsg ::
+  DB.Connection ->
+  AgentConfig ->
+  IntMap (Maybe Int64, AMessage) ->
+  Either AgentErrorType (Either AgentErrorType (ConnData, NonEmpty SndQueue), Maybe PQEncryption, MsgFlags, ValueOrRef AMessage) ->
+  IO (IntMap (Maybe Int64, AMessage), Either AgentErrorType ((Either AgentErrorType (ConnData, NonEmpty SndQueue), Maybe PQEncryption, MsgFlags, ValueOrRef AMessage), InternalId, PQEncryption))
+storeSentMsg db cfg aMessageIds = \case
+  Left e -> pure (aMessageIds, Left e)
+  Right req@(csqs_, pqEnc_, msgFlags, mbr) -> case mbr of
+    VRValue i_ aMessage -> case i_ >>= (`IM.lookup` aMessageIds) of
+      Just _ -> pure (aMessageIds, Left $ INTERNAL "enqueueMessageB: storeSentMsg duplicate saved message body")
+      Nothing -> do
+        (mbId_, r) <- case csqs_ of
+          Left e -> pure (Nothing, Left e)
+          Right (cData, sq :| _) -> do
+            mbId <- createSndMsgBody db aMessage
+            (Just mbId,) <$> storeSentMsg_ cData sq mbId aMessage
+        let aMessageIds' = maybe id (`IM.insert` (mbId_, aMessage)) i_ aMessageIds
+        pure (aMessageIds', r)
+    VRRef i -> case csqs_ of
+      Left e -> pure $ (aMessageIds, Left e)
+      Right (cData, sq :| _) -> case IM.lookup i aMessageIds of
+        Just (Just mbId, aMessage) -> (aMessageIds,) <$> storeSentMsg_ cData sq mbId aMessage
+        Just (Nothing, aMessage) -> do
+          mbId <- createSndMsgBody db aMessage
+          let aMessageIds' = IM.insert i (Just mbId, aMessage) aMessageIds
+          (aMessageIds',) <$> storeSentMsg_ cData sq mbId aMessage
+        Nothing -> pure (aMessageIds, Left $ INTERNAL "enqueueMessageB: storeSentMsg missing saved message body id")
+    where
+      storeSentMsg_ cData@ConnData {connId} sq sndMsgBodyId aMessage = fmap (first storeError) $ runExceptT $ do
+        let AgentConfig {e2eEncryptVRange} = cfg
+        internalTs <- liftIO getCurrentTime
+        (internalId, internalSndId, prevMsgHash) <- ExceptT $ updateSndIds db connId
+        -- We need to do pre-flight encoding that is not stored in database
+        -- to calculate its hash and remember it on connection (createSndMsg -> updateSndMsgHash)
+        -- to enable next enqueue.
+        -- (As encoding is different per connection, we can't store shared body, so it's repeated on delivery)
+        let agentMsgStr = encodeAgentMsgStr aMessage internalSndId prevMsgHash
+            internalHash = C.sha256Hash agentMsgStr
+            currentE2EVersion = maxVersion e2eEncryptVRange
+        (mek, paddedLen, pqEnc) <- agentRatchetEncryptHeader db cData e2eEncAgentMsgLength pqEnc_ currentE2EVersion
+        withExceptT (SEAgentError . cryptoError) $ CR.rcCheckCanPad paddedLen agentMsgStr
+        let msgType = aMessageType aMessage
+            -- msgBody is empty, because snd_messages record is linked to snd_message_bodies
+            msgData = SndMsgData {internalId, internalSndId, internalTs, msgType, msgFlags, msgBody = "", pqEncryption = pqEnc, internalHash, prevMsgHash, sndMsgPrepData_ = Just SndMsgPrepData {encryptKey = mek, paddedLen, sndMsgBodyId}}
+        liftIO $ createSndMsg db connId msgData
+        liftIO $ createSndMsgDelivery db sq internalId
+        pure (req, internalId, pqEnc)
+
 encodeAgentMsgStr :: AMessage -> InternalSndId -> PrevSndMsgHash -> ByteString
 encodeAgentMsgStr aMessage internalSndId prevMsgHash = do
  let privHeader = APrivHeader (unSndId internalSndId) prevMsgHash
@@ -2865,15 +2865,18 @@ synchronizeRatchet' c connId pqSupport' force = withConnLock c connId "synchroni
    SomeConn _ (DuplexConnection cData@ConnData {pqSupport} rqs sqs)
      | ratchetSyncAllowed cData || force -> do
          -- check queues are not switching?
-          when (pqSupport' /= pqSupport) $ withStore' c $ \db -> setConnPQSupport db connId pqSupport'
          let cData' = cData {pqSupport = pqSupport'} :: ConnData
-          AgentConfig {e2eEncryptVRange} <- asks config
+          AgentConfig {e2eEncryptVRange, smpAgentVRange} <- asks config
          g <- asks random
          (pks, e2eParams) <- liftIO $ CR.generateRcvE2EParams g (maxVersion e2eEncryptVRange) pqSupport'
-          enqueueRatchetKeyMsgs c cData' sqs e2eParams
-          withStore' c $ \db -> do
-            setConnRatchetSync db connId RSStarted
-            setRatchetX3dhKeys db connId pks
+          msgId <- withStore c $ \db -> runExceptT $ do
+            msgId <- storeRatchetKey db (L.head sqs) (maxVersion smpAgentVRange) e2eParams Nothing
+            liftIO $ do
+              when (pqSupport' /= pqSupport) $ setConnPQSupport db connId pqSupport'
+              setConnRatchetSync db connId RSStarted
+              setRatchetX3dhKeys db connId pks
+            pure msgId
+          lift $ enqueueRatchetKeyMsgs c cData' sqs msgId
          let cData'' = cData' {ratchetSyncState = RSStarted} :: ConnData
              conn' = DuplexConnection cData'' rqs sqs
          connectionStats c conn'
@@ -3404,6 +3407,9 @@ subscriber c@AgentClient {msgQ, subQ} = run $ forever $ do
    run a = a `catchOwn` \e -> notify $ CRITICAL True $ "Agent subscriber stopped: " <> show e
    notify err = atomically $ writeTBQueue subQ ("", "", AEvt SAEConn $ ERR err)

+maxRatchetKeyHashes :: Int
+maxRatchetKeyHashes = 100
+
 cleanupManager :: AgentClient -> AM' ()
 cleanupManager c@AgentClient {subQ} = do
  AgentConfig {initialCleanupDelay, cleanupInterval = int, storedMsgDataTTL = ttl, cleanupBatchSize = limit} <-
@@ -3413,7 +3419,7 @@ cleanupManager c@AgentClient {subQ} = do
    run ERR deleteConns
    run ERR $ withStore' c $ \db -> deleteRcvMsgHashesExpired db ttl limit
    run ERR $ withStore' c $ \db -> deleteSndMsgsExpired db ttl limit
-    run ERR $ withStore' c $ \db -> deleteRatchetKeyHashesExpired db ttl limit
+    run ERR $ withStore' c $ \db -> deleteRatchetKeyHashesExpired db ttl maxRatchetKeyHashes
    run ERR $ withStore' c (`deleteExpiredNtfTokensToDelete` ttl)
    run RFERR deleteRcvFilesExpired
    run RFERR deleteRcvFilesDeleted
@@ -3610,9 +3616,9 @@ processSMPTransmissions c@AgentClient {subQ} (tSess@(userId, srv, _), THandlePar
                      _ -> prohibited "handshake: incorrect state" >> ack
                  (Just e2eDh, Nothing) -> do
                    decryptClientMessage e2eDh clientMsg >>= \case
-                      (SMP.PHEmpty, AgentRatchetKey {agentVersion, e2eEncryption}) -> do
+                      (SMP.PHEmpty, AgentRatchetKey {agentVersion, e2eEncryption, info}) -> do
                        conn' <- updateConnVersion conn cData agentVersion
-                        qDuplex conn' "AgentRatchetKey" $ \a -> newRatchetKey e2eEncryption a >> ack
+                        qDuplex conn' "AgentRatchetKey" $ \a -> newRatchetKey e2eEncryption info a >> ack
                      (SMP.PHEmpty, AgentMsgEnvelope {agentVersion, encAgentMessage}) -> do
                        conn' <- updateConnVersion conn cData agentVersion
                        -- primary queue is set as Active in helloMsg, below is to set additional queues Active
@@ -3655,7 +3661,7 @@ processSMPTransmissions c@AgentClient {subQ} (tSess@(userId, srv, _), THandlePar
                              qDuplexAckDel conn'' name a = qDuplex conn'' name a >> ackDel msgId
                              resetRatchetSync :: AM (Connection c)
                              resetRatchetSync
-                                | rss `notElem` ([RSOk, RSStarted] :: [RatchetSyncState]) = do
+                                | rss `notElem` ([RSOk, RSStarted] :: [RatchetSyncState]) = ifM ((RSStarted ==) <$> getRatchetSyncState) (pure conn') $ do
                                    let cData'' = (toConnData conn') {ratchetSyncState = RSOk} :: ConnData
                                        conn'' = updateConnection cData'' conn'
                                    cStats <- connectionStats c conn''
@@ -3696,7 +3702,7 @@ processSMPTransmissions c@AgentClient {subQ} (tSess@(userId, srv, _), THandlePar
                              notifySync :: AM ()
                              notifySync = qDuplex conn' "AGENT A_CRYPTO error" $ \connDuplex -> do
                                let rss' = cryptoErrToSyncState e
-                                when (rss `elem` ([RSOk, RSAllowed, RSRequired] :: [RatchetSyncState])) $ do
+                                whenM ((`elem` ([RSOk, RSAllowed, RSRequired] :: [RatchetSyncState])) <$> getRatchetSyncState) $ do
                                  let cData'' = (toConnData conn') {ratchetSyncState = rss'} :: ConnData
                                      conn'' = updateConnection cData'' connDuplex
                                  cStats <- connectionStats c conn''
@@ -3792,6 +3798,9 @@ processSMPTransmissions c@AgentClient {subQ} (tSess@(userId, srv, _), THandlePar
          enqueueCmd :: InternalCommand -> AM ()
          enqueueCmd = enqueueCommand c "" connId (Just srv) . AInternalCommand

+          getRatchetSyncState :: AM RatchetSyncState
+          getRatchetSyncState = withStore c (`getConnRatchetSync` connId)
+
          unexpected :: BrokerMsg -> AM ()
          unexpected r = do
            logServer "<--" c srv rId $ "unexpected: " <> bshow r
@@ -4160,42 +4169,47 @@ processSMPTransmissions c@AgentClient {subQ} (tSess@(userId, srv, _), THandlePar
            DuplexConnection {} -> action conn'
            _ -> qError $ name <> ": message must be sent to duplex connection"

-          newRatchetKey :: CR.RcvE2ERatchetParams 'C.X448 -> Connection 'CDuplex -> AM ()
-          newRatchetKey e2eOtherPartyParams@(CR.E2ERatchetParams e2eVersion k1Rcv k2Rcv _) conn'@(DuplexConnection cData'@ConnData {lastExternalSndId, pqSupport} _ sqs) =
+          newRatchetKey :: CR.RcvE2ERatchetParams 'C.X448 -> ByteString -> Connection 'CDuplex -> AM ()
+          newRatchetKey e2eOtherPartyParams@(CR.E2ERatchetParams e2eVersion k1Rcv k2Rcv _) info conn'@(DuplexConnection cData'@ConnData {lastExternalSndId, pqSupport} _ sqs) =
            unlessM ratchetExists $ do
              AgentConfig {e2eEncryptVRange} <- asks config
              unless (e2eVersion `isCompatible` e2eEncryptVRange) (throwE $ AGENT A_VERSION)
-              keys <- getSendRatchetKeys
+              keys_ <- getSendRatchetKeys
              let rcVs = CR.RatchetVersions {current = e2eVersion, maxSupported = maxVersion e2eEncryptVRange}
-              initRatchet rcVs keys
-              notifyAgreed
+              case keys_ of
+                Just (keys, replyKey_) -> initRatchet rcVs keys replyKey_ >> notifyAgreed
+                Nothing -> withStore' c $ \db -> addProcessedRatchetKeyHash db connId rkHashRcv
            where
              rkHashRcv = rkHash k1Rcv k2Rcv
              rkHash k1 k2 = C.sha256Hash $ C.pubKeyBytes k1 <> C.pubKeyBytes k2
+              answeredKeyHash_ = case smpDecode info of
+                Right (AgentRatchetInfo RatchetInfo {answeredKeyHash}) -> answeredKeyHash
+                _ -> Nothing
              ratchetExists :: AM Bool
              ratchetExists = withStore' c $ \db -> do
                exists <- checkRatchetKeyHashExists db connId rkHashRcv
-                unless exists $ addProcessedRatchetKeyHash db connId rkHashRcv
                pure exists
-              getSendRatchetKeys :: AM (CR.RcvE2EPrivRatchetParams 'C.X448)
-              getSendRatchetKeys = case rss of
-                RSOk -> sendReplyKey -- receiving client
-                RSAllowed -> sendReplyKey
-                RSRequired -> sendReplyKey
-                RSStarted -> withStore c (`getRatchetX3dhKeys` connId) -- initiating client
-                RSAgreed -> do
-                  withStore' c $ \db -> setConnRatchetSync db connId RSRequired
+              getSendRatchetKeys :: AM (Maybe (CR.RcvE2EPrivRatchetParams 'C.X448, Maybe (CR.RcvE2ERatchetParams 'C.X448)))
+              getSendRatchetKeys = getRatchetSyncState >>= \rss' -> case (rss', answeredKeyHash_) of
+                (RSStarted, Nothing) -> Just . (,Nothing) <$> withStore c (`getRatchetX3dhKeys` connId) -- initiating client
+                (RSStarted, Just h) -> fmap (,Nothing) . answeredKeys h <$> withStore c (`getRatchetX3dhKeys` connId)
+                (_, Nothing) -> Just <$> sendReplyKey -- receiving client
+                (RSAgreed, Just _) -> do
+                  withStore' c $ \db -> addProcessedRatchetKeyHash db connId rkHashRcv >> setConnRatchetSync db connId RSRequired
                  notifyRatchetSyncError
                  -- can communicate for other client to reset to RSRequired
                  -- - need to add new AgentMsgEnvelope, AgentMessage, AgentMessageType
                  -- - need to deduplicate on receiving side
                  throwE $ AGENT (A_CRYPTO RATCHET_SYNC)
+                (_, Just _) -> pure Nothing
                where
+                  answeredKeys h keys@(pk1, pk2, _)
+                    | rkHash (C.publicKey pk1) (C.publicKey pk2) == h = Just keys
+                    | otherwise = Nothing
                  sendReplyKey = do
                    g <- asks random
                    (pks, e2eParams) <- liftIO $ CR.generateRcvE2EParams g e2eVersion pqSupport
-                    enqueueRatchetKeyMsgs c cData' sqs e2eParams
-                    pure pks
+                    pure (pks, Just e2eParams)
                  notifyRatchetSyncError = do
                    let cData'' = cData' {ratchetSyncState = RSRequired} :: ConnData
                        conn'' = updateConnection cData'' conn'
@@ -4207,23 +4221,33 @@ processSMPTransmissions c@AgentClient {subQ} (tSess@(userId, srv, _), THandlePar
                    conn'' = updateConnection cData'' conn'
                cStats <- connectionStats c conn''
                notify $ RSYNC RSAgreed Nothing cStats
-              recreateRatchet :: CR.Ratchet 'C.X448 -> AM ()
-              recreateRatchet rc = withStore' c $ \db -> do
-                setConnRatchetSync db connId RSAgreed
-                deleteRatchet db connId
-                createRatchet db connId rc
+              recreateRatchet :: Maybe (CR.RcvE2ERatchetParams 'C.X448) -> Maybe AMessage -> CR.Ratchet 'C.X448 -> AM ()
+              recreateRatchet replyKey_ eready_ rc = do
+                cfg@AgentConfig {smpAgentVRange} <- asks config
+                (msgId_, r_) <- withStore c $ \db -> runExceptT $ do
+                  msgId_ <- forM replyKey_ $ \e2eParams -> storeRatchetKey db (L.head sqs) (maxVersion smpAgentVRange) e2eParams (Just rkHashRcv)
+                  liftIO $ do
+                    addProcessedRatchetKeyHash db connId rkHashRcv
+                    setConnRatchetSync db connId RSAgreed
+                    deleteRatchet db connId
+                    createRatchet db connId rc
+                  r_ <- forM eready_ $ \msg -> liftIO $ storeSentMsg db cfg IM.empty (Right (Right (cData', sqs), Nothing, SMP.MsgFlags {notification = True}, vrValue msg)) >>= either E.throwIO pure . snd
+                  pure (msgId_, r_)
+                forM_ msgId_ $ lift . enqueueRatchetKeyMsgs c cData' sqs
+                forM_ r_ $ \r -> lift $ do
+                  r' <- submitSentMsg c r
+                  enqueueSavedMessageB c $ mapMaybe snd $ rights [r']
              -- compare public keys `k1` in AgentRatchetKey messages sent by self and other party
              -- to determine ratchet initilization ordering
-              initRatchet :: CR.RatchetVersions -> CR.RcvE2EPrivRatchetParams 'C.X448 -> AM ()
-              initRatchet rcVs (pk1, pk2, pKem)
+              initRatchet :: CR.RatchetVersions -> CR.RcvE2EPrivRatchetParams 'C.X448 -> Maybe (CR.RcvE2ERatchetParams 'C.X448) -> AM ()
+              initRatchet rcVs (pk1, pk2, pKem) replyKey_
                | rkHash (C.publicKey pk1) (C.publicKey pk2) <= rkHashRcv = do
                    rcParams <- liftError cryptoError $ CR.pqX3dhRcv (pk1, pk2, pKem) e2eOtherPartyParams
-                    recreateRatchet $ CR.initRcvRatchet rcVs pk2 rcParams pqSupport
+                    recreateRatchet replyKey_ Nothing $ CR.initRcvRatchet rcVs pk2 rcParams pqSupport
                | otherwise = do
                    (_, rcDHRs) <- atomically . C.generateKeyPair =<< asks random
                    rcParams <- liftEitherWith cryptoError $ CR.pqX3dhSnd (pk1, pk2, CR.APRKP CR.SRKSProposed <$> pKem) e2eOtherPartyParams
-                    recreateRatchet $ CR.initSndRatchet rcVs k2Rcv rcDHRs rcParams
-                    void . enqueueMessages' c cData' sqs SMP.MsgFlags {notification = True} $ EREADY lastExternalSndId
+                    recreateRatchet replyKey_ (Just $ EREADY lastExternalSndId) $ CR.initSndRatchet rcVs k2Rcv rcDHRs rcParams

          checkMsgIntegrity :: PrevExternalSndId -> ExternalSndId -> PrevRcvMsgHash -> ByteString -> MsgIntegrity
          checkMsgIntegrity prevExtSndId extSndId internalPrevMsgHash receivedPrevMsgHash
@@ -4336,32 +4360,25 @@ storeConfirmation c cData@ConnData {connId, pqSupport, connAgentVersion = v} sq
    liftIO $ createSndMsg db connId msgData
    liftIO $ createSndMsgDelivery db sq internalId

-enqueueRatchetKeyMsgs :: AgentClient -> ConnData -> NonEmpty SndQueue -> CR.RcvE2ERatchetParams 'C.X448 -> AM ()
-enqueueRatchetKeyMsgs c cData (sq :| sqs) e2eEncryption = do
-  msgId <- enqueueRatchetKey c sq e2eEncryption
-  mapM_ (lift . enqueueSavedMessage c msgId) $ filter (isActiveSndQ cData) sqs
+enqueueRatchetKeyMsgs :: AgentClient -> ConnData -> NonEmpty SndQueue -> InternalId -> AM' ()
+enqueueRatchetKeyMsgs c cData (sq :| sqs) (InternalId msgId) = do
+  submitPendingMsg c sq
+  mapM_ (enqueueSavedMessage c msgId) $ filter (isActiveSndQ cData) sqs

-enqueueRatchetKey :: AgentClient -> SndQueue -> CR.RcvE2ERatchetParams 'C.X448 -> AM AgentMsgId
-enqueueRatchetKey c sq@SndQueue {connId} e2eEncryption = do
-  aVRange <- asks $ smpAgentVRange . config
-  msgId <- storeRatchetKey $ maxVersion aVRange
-  lift $ submitPendingMsg c sq
-  pure $ unId msgId
-  where
-    storeRatchetKey :: VersionSMPA -> AM InternalId
-    storeRatchetKey agentVersion = withStore c $ \db -> runExceptT $ do
-      internalTs <- liftIO getCurrentTime
-      (internalId, internalSndId, prevMsgHash) <- ExceptT $ updateSndIds db connId
-      let agentMsg = AgentRatchetInfo ""
-          agentMsgStr = smpEncode agentMsg
-          internalHash = C.sha256Hash agentMsgStr
-      let msgBody = smpEncode $ AgentRatchetKey {agentVersion, e2eEncryption, info = agentMsgStr}
-          msgType = agentMessageType agentMsg
-          -- this message is e2e encrypted with queue key, not with double ratchet
-          msgData = SndMsgData {internalId, internalSndId, internalTs, msgType, msgBody, pqEncryption = PQEncOff, msgFlags = SMP.MsgFlags {notification = True}, internalHash, prevMsgHash, sndMsgPrepData_ = Nothing}
-      liftIO $ createSndMsg db connId msgData
-      liftIO $ createSndMsgDelivery db sq internalId
-      pure internalId
+storeRatchetKey :: DB.Connection -> SndQueue -> VersionSMPA -> CR.RcvE2ERatchetParams 'C.X448 -> Maybe ByteString -> ExceptT StoreError IO InternalId
+storeRatchetKey db sq@SndQueue {connId} agentVersion e2eEncryption answeredKeyHash_ = do
+  internalTs <- liftIO getCurrentTime
+  (internalId, internalSndId, prevMsgHash) <- ExceptT $ updateSndIds db connId
+  let agentMsg = AgentRatchetInfo RatchetInfo {answeredKeyHash = answeredKeyHash_}
+      agentMsgStr = smpEncode agentMsg
+      internalHash = C.sha256Hash agentMsgStr
+  let msgBody = smpEncode $ AgentRatchetKey {agentVersion, e2eEncryption, info = agentMsgStr}
+      msgType = agentMessageType agentMsg
+      -- this message is e2e encrypted with queue key, not with double ratchet
+      msgData = SndMsgData {internalId, internalSndId, internalTs, msgType, msgBody, pqEncryption = PQEncOff, msgFlags = SMP.MsgFlags {notification = True}, internalHash, prevMsgHash, sndMsgPrepData_ = Nothing}
+  liftIO $ createSndMsg db connId msgData
+  liftIO $ createSndMsgDelivery db sq internalId
+  pure internalId

 -- encoded AgentMessage -> encoded EncAgentMessage
 agentRatchetEncrypt :: DB.Connection -> ConnData -> ByteString -> (PQSupport -> Int) -> Maybe PQEncryption -> CR.VersionE2E -> ExceptT StoreError IO (ByteString, PQEncryption)
@@ -4386,10 +4403,11 @@ agentRatchetDecrypt g db connId encAgentMsg = do

 agentRatchetDecrypt' :: TVar ChaChaDRG -> DB.Connection -> ConnId -> CR.RatchetX448 -> ByteString -> ExceptT StoreError IO (ByteString, PQEncryption)
 agentRatchetDecrypt' g db connId rc encAgentMsg = do
-  skipped <- liftIO $ getSkippedMsgKeys db connId
+  skipped <- liftIO $ getSkippedMsgKeys db connId CR.maxSkippedMsgKeys
  (agentMsgBody_, rc', skippedDiff) <- withExceptT (SEAgentError . cryptoError) $ CR.rcDecrypt g rc skipped encAgentMsg
+  agentMsgBody <- liftEither $ first (SEAgentError . cryptoError) agentMsgBody_
  liftIO $ updateRatchet db connId rc' skippedDiff
-  liftEither $ bimap (SEAgentError . cryptoError) (,CR.rcRcvKEM rc') agentMsgBody_
+  pure (agentMsgBody, CR.rcRcvKEM rc')

 newSndQueue :: UserId -> ConnId -> Compatible SMPQueueInfo -> Maybe (C.APrivateAuthKey) -> AM' (NewSndQueue, C.PublicKeyX25519)
 newSndQueue userId connId (Compatible (SMPQueueInfo smpClientVersion SMPQueueAddress {smpServer, senderId, queueMode, dhPublicKey = rcvE2ePubDhKey})) sndKey_ = do
@@ -34,7 +34,6 @@ module Simplex.Messaging.Agent.Client
    withInvLock,
    withLockMap,
    withLocksMap,
-    getMapLock,
    ipAddressProtected,
    closeAgentClient,
    closeProtocolServerClients,
@@ -312,7 +311,7 @@ import Simplex.Messaging.Session
 import Simplex.Messaging.SystemTime
 import Simplex.Messaging.TMap (TMap)
 import qualified Simplex.Messaging.TMap as TM
-import Simplex.Messaging.Transport (HandshakeError (..), SMPServiceRole (..), SMPVersion, ServiceCredentials (..), SessionId, THClientService' (..), THandleAuth (..), THandleParams (sessionId, thAuth, thVersion, serverInfo), TransportError (..), TransportPeer (..), shortLinksSMPVersion, newNtfCredsSMPVersion)
+import Simplex.Messaging.Transport (HandshakeError (..), SMPServiceRole (..), SMPVersion, ServiceCredentials (..), SessionId, THClientService' (..), THandleAuth (..), THandleParams (sessionId, thAuth, thVersion, serverInfo), TransportError (..), newNtfCredsSMPVersion)
 import Simplex.Messaging.Transport.Client (TransportHost (..))
 import Simplex.Messaging.Transport.Credentials
 import Simplex.Messaging.Util
@@ -1261,9 +1260,12 @@ sendOrProxySMPCommand c nm userId destSrv@ProtocolServer {host = destHosts} conn
          Left e -> throwE e

 ipAddressProtected :: NetworkConfig -> ProtocolServer p -> Bool
-ipAddressProtected NetworkConfig {socksProxy, hostMode} (ProtocolServer _ hosts _ _) = do
-  isJust socksProxy || (hostMode == HMOnion && any isOnionHost hosts)
+ipAddressProtected NetworkConfig {socksProxy, socksMode, hostMode} (ProtocolServer _ hosts _ _)
+  | isJust socksProxy = socksMode == SMAlways || if hostMode == HMPublic then allOnion else anyOnion
+  | otherwise = hostMode == HMOnion && anyOnion
  where
+    anyOnion = any isOnionHost hosts
+    allOnion = all isOnionHost hosts
    isOnionHost = \case THOnionHost _ -> True; _ -> False

 withNtfClient :: AgentClient -> NetworkRequestMode -> NtfServer -> EntityId -> ByteString -> (NtfClient -> ExceptT NtfClientError IO a) -> AM a
@@ -1496,7 +1498,7 @@ newRcvQueue_ c nm userId connId (ProtoServerWithAuth srv auth) vRange cqrd enabl
  let sessServiceId = (\THClientService {serviceId = sId} -> sId) <$> (clientService =<< thAuth thParams')
  when (isJust serviceId && serviceId /= sessServiceId) $ logError "incorrect service ID in NEW response"
  liftIO . logServer "<--" c srv NoEntity $ B.unwords ["IDS", logSecret rcvId, logSecret sndId]
-  shortLink <- mkShortLinkCreds thParams' qik
+  shortLink <- mkShortLinkCreds qik
  let rq =
        RcvQueue
          { userId,
@@ -1537,8 +1539,8 @@ newRcvQueue_ c nm userId connId (ProtoServerWithAuth srv auth) vRange cqrd enabl
      (Just ((ntfPublicKey, ntfPrivateKey), dhpk), Just (ServerNtfCreds notifierId dhk')) ->
        Just ClientNtfCreds {ntfPublicKey, ntfPrivateKey, notifierId, rcvNtfDhSecret = C.dh' dhk' dhpk}
      _ -> Nothing
-    mkShortLinkCreds :: THandleParams SMPVersion 'TClient -> QueueIdsKeys -> AM (Maybe ShortLinkCreds)
-    mkShortLinkCreds thParams' QIK {sndId, queueMode, linkId} = case (cqrd, queueMode) of
+    mkShortLinkCreds :: QueueIdsKeys -> AM (Maybe ShortLinkCreds)
+    mkShortLinkCreds QIK {sndId, queueMode, linkId} = case (cqrd, queueMode) of
      (CQRMessaging ld, Just QMMessaging) ->
        withLinkData ld $ \lnkId CQRData {linkKey, privSigKey, srvReq = (sndId', d)} ->
          if sndId == sndId'
@@ -1552,11 +1554,9 @@ newRcvQueue_ c nm userId connId (ProtoServerWithAuth srv auth) vRange cqrd enabl
      (_, Nothing) -> newErr "unexpected link ID"
      _ -> newErr "unexpected queue mode"
      where
-        v = thVersion thParams'
        withLinkData :: Maybe d -> (SMP.LinkId -> d -> AM (Maybe ShortLinkCreds)) -> AM (Maybe ShortLinkCreds)
        withLinkData ld_ mkLink = case (ld_, linkId) of
          (Just ld, Just lnkId) -> mkLink lnkId ld
-          (Just _, Nothing) | v < shortLinksSMPVersion -> pure Nothing
          (Nothing, Nothing) -> pure Nothing
          _ -> newErr "unexpected or absent link ID"
        newErr :: String -> AM (Maybe ShortLinkCreds)
@@ -6,7 +6,6 @@ module Simplex.Messaging.Agent.Lock
    withLock',
    withGetLock,
    withGetLocks,
-    getPutLock,
  )
 where

@@ -79,6 +79,7 @@ module Simplex.Messaging.Agent.Protocol
    SMPConfirmation (..),
    AgentMsgEnvelope (..),
    AgentMessage (..),
+    RatchetInfo (..),
    RequestSignature (..),
    AgentMessageType (..),
    APrivHeader (..),
@@ -928,7 +929,7 @@ data AgentMessage
  | -- AgentConnInfoReply is used by accepting party in duplexHandshake mode (v2), allowing to include reply queue(s) in the initial confirmation.
    -- It made removed REPLY message unnecessary.
    AgentConnInfoReply (NonEmpty SMPQueueInfo) ConnInfo
-  | AgentRatchetInfo ByteString
+  | AgentRatchetInfo RatchetInfo
  | AgentMessage APrivHeader AMessage
  | AgentServiceRequest (NonEmpty SMPQueueInfo) (Maybe RequestSignature) MsgBody
  | AgentServiceResponse MsgBody
@@ -939,7 +940,7 @@ instance Encoding AgentMessage where
  smpEncode = \case
    AgentConnInfo cInfo -> smpEncode ('I', Tail cInfo)
    AgentConnInfoReply smpQueues cInfo -> smpEncode ('D', smpQueues, Tail cInfo) -- 'D' stands for "duplex"
-    AgentRatchetInfo info -> smpEncode ('R', Tail info)
+    AgentRatchetInfo info -> smpEncode ('R', info)
    AgentMessage hdr aMsg -> smpEncode ('M', hdr, aMsg)
    AgentServiceRequest qs sig_ body -> smpEncode ('A', qs, sig_, Tail body)
    AgentServiceResponse body -> smpEncode ('P', Tail body)
@@ -948,13 +949,23 @@ instance Encoding AgentMessage where
    smpP >>= \case
      'I' -> AgentConnInfo . unTail <$> smpP
      'D' -> AgentConnInfoReply <$> smpP <*> (unTail <$> smpP)
-      'R' -> AgentRatchetInfo . unTail <$> smpP
+      'R' -> AgentRatchetInfo <$> smpP
      'M' -> AgentMessage <$> smpP <*> smpP
      'A' -> AgentServiceRequest <$> smpP <*> smpP <*> (unTail <$> smpP)
      'P' -> AgentServiceResponse . unTail <$> smpP
      'J' -> AgentRejection . unTail <$> smpP
      _ -> fail "bad AgentMessage"

+data RatchetInfo = RatchetInfo {answeredKeyHash :: Maybe ByteString}
+  deriving (Eq, Show)
+
+instance Encoding RatchetInfo where
+  smpEncode RatchetInfo {answeredKeyHash} = smpEncode answeredKeyHash
+  smpP = do
+    answeredKeyHash <- smpP <|> pure Nothing
+    _ <- A.takeByteString
+    pure RatchetInfo {answeredKeyHash}
+
 -- internal type for storing message type in the database
 data AgentMessageType
  = AM_CONN_INFO
@@ -76,6 +76,7 @@ module Simplex.Messaging.Agent.Store.AgentStore
    getExpiredServiceConns,
    deleteExpiredServiceRequests,
    getDeletedWaitingDeliveryConnIds,
+    getConnRatchetSync,
    setConnRatchetSync,
    addProcessedRatchetKeyHash,
    checkRatchetKeyHashExists,
@@ -1594,12 +1595,16 @@ getRatchet_ q db connId =
  where
    ratchet = maybe (Left SERatchetNotFound) Right . fromOnly

-getSkippedMsgKeys :: DB.Connection -> ConnId -> IO SkippedMsgKeys
-getSkippedMsgKeys db connId =
-  skipped <$> DB.query db "SELECT header_key, msg_n, msg_key FROM skipped_messages WHERE conn_id = ?" (Only connId)
+getSkippedMsgKeys :: DB.Connection -> ConnId -> Int -> IO SkippedMsgKeys
+getSkippedMsgKeys db connId maxKeys = do
+  (keys, oldKeys) <- splitAt maxKeys <$> DB.query db "SELECT skipped_message_id, header_key, msg_n, msg_key FROM skipped_messages WHERE conn_id = ? ORDER BY skipped_message_id DESC LIMIT ?" (connId, maxKeys + 1)
+  case oldKeys of
+    (skippedMsgId :: Int64, _, _, _) : _ -> DB.execute db "DELETE FROM skipped_messages WHERE conn_id = ? AND skipped_message_id <= ?" (connId, skippedMsgId)
+    [] -> pure ()
+  pure $ skipped keys
  where
    skipped = foldl' addSkippedKey M.empty
-    addSkippedKey smks (hk, msgN, mk) = M.alter (Just . addMsgKey) hk smks
+    addSkippedKey smks (_, hk, msgN, mk) = M.alter (Just . addMsgKey) hk smks
      where
        addMsgKey = maybe (M.singleton msgN mk) (M.insert msgN mk)

@@ -2793,6 +2798,11 @@ getDeletedWaitingDeliveryConnIds :: DB.Connection -> IO [ConnId]
 getDeletedWaitingDeliveryConnIds db =
  map fromOnly <$> DB.query_ db "SELECT conn_id FROM connections WHERE deleted_at_wait_delivery IS NOT NULL"

+getConnRatchetSync :: DB.Connection -> ConnId -> IO (Either StoreError RatchetSyncState)
+getConnRatchetSync db connId =
+  firstRow fromOnly SEConnNotFound $
+    DB.query db "SELECT ratchet_sync_state FROM connections WHERE conn_id = ?" (Only connId)
+
 setConnRatchetSync :: DB.Connection -> ConnId -> RatchetSyncState -> IO ()
 setConnRatchetSync db connId ratchetSyncState =
  DB.execute db "UPDATE connections SET ratchet_sync_state = ? WHERE conn_id = ?" (ratchetSyncState, connId)
@@ -2814,21 +2824,35 @@ checkRatchetKeyHashExists db connId hash =
      (connId, Binary hash)

 deleteRatchetKeyHashesExpired :: DB.Connection -> NominalDiffTime -> Int -> IO ()
-deleteRatchetKeyHashesExpired db ttl limit = do
+deleteRatchetKeyHashesExpired db ttl maxConnHashes = do
  cutoffTs <- addUTCTime (-ttl) <$> getCurrentTime
+#if defined(dbPostgres)
  DB.execute
    db
-    [sql|
-      DELETE FROM processed_ratchet_key_hashes
-      WHERE processed_ratchet_key_hash_id IN (
-        SELECT processed_ratchet_key_hash_id
-        FROM processed_ratchet_key_hashes
-        WHERE created_at < ?
-        ORDER BY created_at ASC
-        LIMIT ?
-      )
-    |]
-    (cutoffTs, limit)
+    ("DELETE FROM processed_ratchet_key_hashes h USING (" <> maxExcessIdsQuery <> ") e WHERE h.conn_id = e.conn_id AND h.processed_ratchet_key_hash_id <= e.max_excess_id AND h.created_at < ?")
+    (maxConnHashes, maxConnHashes, cutoffTs)
+#else
+  maxExcessIds <- DB.query db maxExcessIdsQuery (maxConnHashes, maxConnHashes)
+  DB.executeMany
+    db
+    "DELETE FROM processed_ratchet_key_hashes WHERE conn_id = ? AND processed_ratchet_key_hash_id <= ? AND created_at < ?"
+    (map (\(connId :: ConnId, maxExcessId :: Int64) -> (connId, maxExcessId, cutoffTs)) maxExcessIds)
+#endif
+  where
+    maxExcessIdsQuery :: Query
+    maxExcessIdsQuery =
+      [sql|
+        SELECT conn_id, (
+          SELECT processed_ratchet_key_hash_id
+          FROM processed_ratchet_key_hashes
+          WHERE conn_id = c.conn_id
+          ORDER BY processed_ratchet_key_hash_id DESC
+          LIMIT 1 OFFSET ?
+        ) AS max_excess_id
+        FROM processed_ratchet_key_hashes c
+        GROUP BY conn_id
+        HAVING COUNT(*) > ?
+      |]

 -- | returns all connection queues, the first queue is the primary one
 getRcvQueuesByConnId_ :: DB.Connection -> ConnId -> IO (Maybe (NonEmpty RcvQueue))
@@ -16,6 +16,7 @@ import Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260411_service_certs
 import Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260712_address_dr_rpc
 import Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260823_snd_files_entitlement
 import Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260919_ratchet_verify_codes
+import Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260929_ratchet_indexes
 import Simplex.Messaging.Agent.Store.Shared (Migration (..))

 schemaMigrations :: [(String, Text, Maybe Text)]
@@ -31,7 +32,8 @@ schemaMigrations =
    ("20260411_service_certs", m20260411_service_certs, Just down_m20260411_service_certs),
    ("20260712_address_dr_rpc", m20260712_address_dr_rpc, Just down_m20260712_address_dr_rpc),
    ("20260823_snd_files_entitlement", m20260823_snd_files_entitlement, Just down_m20260823_snd_files_entitlement),
-    ("20260919_ratchet_verify_codes", m20260919_ratchet_verify_codes, Just down_m20260919_ratchet_verify_codes)
+    ("20260919_ratchet_verify_codes", m20260919_ratchet_verify_codes, Just down_m20260919_ratchet_verify_codes),
+    ("20260929_ratchet_indexes", m20260929_ratchet_indexes, Just down_m20260929_ratchet_indexes)
  ]

 -- | The list of migrations in ascending order by date
@@ -0,0 +1,34 @@
+{-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE QuasiQuotes #-}
+
+module Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260929_ratchet_indexes where
+
+import Data.Text (Text)
+import Text.RawString.QQ (r)
+
+m20260929_ratchet_indexes :: Text
+m20260929_ratchet_indexes =
+  [r|
+DROP INDEX idx_skipped_messages_conn_id;
+CREATE INDEX idx_skipped_messages_conn_id ON skipped_messages(conn_id, skipped_message_id);
+
+DELETE FROM processed_ratchet_key_hashes AS h
+WHERE EXISTS (
+  SELECT 1 FROM processed_ratchet_key_hashes d
+  WHERE d.conn_id = h.conn_id AND d.hash = h.hash AND d.processed_ratchet_key_hash_id < h.processed_ratchet_key_hash_id
+);
+
+DROP INDEX idx_processed_ratchet_key_hashes_hash;
+CREATE UNIQUE INDEX idx_processed_ratchet_key_hashes_hash ON processed_ratchet_key_hashes(conn_id, hash);
+CREATE INDEX idx_processed_ratchet_key_hashes_conn_id ON processed_ratchet_key_hashes(conn_id, processed_ratchet_key_hash_id);
+|]
+
+down_m20260929_ratchet_indexes :: Text
+down_m20260929_ratchet_indexes =
+  [r|
+DROP INDEX idx_processed_ratchet_key_hashes_conn_id;
+DROP INDEX idx_processed_ratchet_key_hashes_hash;
+CREATE INDEX idx_processed_ratchet_key_hashes_hash ON processed_ratchet_key_hashes(conn_id, hash);
+DROP INDEX idx_skipped_messages_conn_id;
+CREATE INDEX idx_skipped_messages_conn_id ON skipped_messages(conn_id);
+|]
@@ -1169,11 +1169,15 @@ CREATE INDEX idx_ntf_tokens_ntf_host_ntf_port ON smp_agent_test_protocol_schema.



+CREATE INDEX idx_processed_ratchet_key_hashes_conn_id ON smp_agent_test_protocol_schema.processed_ratchet_key_hashes USING btree (conn_id, processed_ratchet_key_hash_id);
+
+
+
 CREATE INDEX idx_processed_ratchet_key_hashes_created_at ON smp_agent_test_protocol_schema.processed_ratchet_key_hashes USING btree (created_at);



-CREATE INDEX idx_processed_ratchet_key_hashes_hash ON smp_agent_test_protocol_schema.processed_ratchet_key_hashes USING btree (conn_id, hash);
+CREATE UNIQUE INDEX idx_processed_ratchet_key_hashes_hash ON smp_agent_test_protocol_schema.processed_ratchet_key_hashes USING btree (conn_id, hash);



@@ -1241,7 +1245,7 @@ CREATE UNIQUE INDEX idx_server_certs_user_id_host_port ON smp_agent_test_protoco



-CREATE INDEX idx_skipped_messages_conn_id ON smp_agent_test_protocol_schema.skipped_messages USING btree (conn_id);
+CREATE INDEX idx_skipped_messages_conn_id ON smp_agent_test_protocol_schema.skipped_messages USING btree (conn_id, skipped_message_id);



@@ -52,6 +52,7 @@ import Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260411_service_certs
 import Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260712_address_dr_rpc
 import Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260823_snd_files_entitlement
 import Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260919_ratchet_verify_codes
+import Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260929_ratchet_indexes
 import Simplex.Messaging.Agent.Store.Shared (Migration (..))

 schemaMigrations :: [(String, Query, Maybe Query)]
@@ -103,7 +104,8 @@ schemaMigrations =
    ("m20260411_service_certs", m20260411_service_certs, Just down_m20260411_service_certs),
    ("m20260712_address_dr_rpc", m20260712_address_dr_rpc, Just down_m20260712_address_dr_rpc),
    ("m20260823_snd_files_entitlement", m20260823_snd_files_entitlement, Just down_m20260823_snd_files_entitlement),
-    ("m20260919_ratchet_verify_codes", m20260919_ratchet_verify_codes, Just down_m20260919_ratchet_verify_codes)
+    ("m20260919_ratchet_verify_codes", m20260919_ratchet_verify_codes, Just down_m20260919_ratchet_verify_codes),
+    ("m20260929_ratchet_indexes", m20260929_ratchet_indexes, Just down_m20260929_ratchet_indexes)
  ]

 -- | The list of migrations in ascending order by date
@@ -0,0 +1,29 @@
+{-# LANGUAGE QuasiQuotes #-}
+
+module Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260929_ratchet_indexes where
+
+import Database.SQLite.Simple (Query)
+import Database.SQLite.Simple.QQ (sql)
+
+-- idx_skipped_messages_conn_id is not changed: SQLite index entries include rowid (skipped_message_id)
+m20260929_ratchet_indexes :: Query
+m20260929_ratchet_indexes =
+  [sql|
+DELETE FROM processed_ratchet_key_hashes AS h
+WHERE EXISTS (
+  SELECT 1 FROM processed_ratchet_key_hashes d
+  WHERE d.conn_id = h.conn_id AND d.hash = h.hash AND d.processed_ratchet_key_hash_id < h.processed_ratchet_key_hash_id
+);
+
+DROP INDEX idx_processed_ratchet_key_hashes_hash;
+CREATE UNIQUE INDEX idx_processed_ratchet_key_hashes_hash ON processed_ratchet_key_hashes(conn_id, hash);
+CREATE INDEX idx_processed_ratchet_key_hashes_conn_id ON processed_ratchet_key_hashes(conn_id, processed_ratchet_key_hash_id);
+  |]
+
+down_m20260929_ratchet_indexes :: Query
+down_m20260929_ratchet_indexes =
+  [sql|
+DROP INDEX idx_processed_ratchet_key_hashes_conn_id;
+DROP INDEX idx_processed_ratchet_key_hashes_hash;
+CREATE INDEX idx_processed_ratchet_key_hashes_hash ON processed_ratchet_key_hashes(conn_id, hash);
+  |]
@@ -573,10 +573,6 @@ CREATE INDEX idx_encrypted_rcv_message_hashes_hash ON encrypted_rcv_message_hash
  conn_id,
  hash
 );
-CREATE INDEX idx_processed_ratchet_key_hashes_hash ON processed_ratchet_key_hashes(
-  conn_id,
-  hash
-);
 CREATE INDEX idx_snd_messages_rcpt_internal_id ON snd_messages(
  conn_id,
  rcpt_internal_id
@@ -641,6 +637,14 @@ CREATE INDEX idx_connections_deleted ON connections(deleted);
 CREATE INDEX idx_connections_service_request_expires_at ON connections(
  service_request_expires_at
 );
+CREATE UNIQUE INDEX idx_processed_ratchet_key_hashes_hash ON processed_ratchet_key_hashes(
+  conn_id,
+  hash
+);
+CREATE INDEX idx_processed_ratchet_key_hashes_conn_id ON processed_ratchet_key_hashes(
+  conn_id,
+  processed_ratchet_key_hash_id
+);
 CREATE TRIGGER tr_rcv_queue_insert
 AFTER INSERT ON rcv_queues
 FOR EACH ROW
@@ -195,6 +195,7 @@ data PClient v err msg = PClient
    transportHost :: TransportHost,
    tcpConnectTimeout :: NetworkTimeout,
    tcpTimeout :: NetworkTimeout,
+    proxiedRelayVRange :: VersionRange v,
    sendPings :: TVar Bool,
    lastReceived :: TVar UTCTime,
    timeoutErrorCount :: TVar Int,
@@ -240,6 +241,7 @@ smpClientStub g sessionId thVersion thAuth = do
              transportHost = "localhost",
              tcpConnectTimeout,
              tcpTimeout,
+              proxiedRelayVRange = supportedClientSMPRelayVRange,
              sendPings,
              lastReceived,
              timeoutErrorCount,
@@ -477,6 +479,7 @@ data ProtocolClientConfig v = ProtocolClientConfig
    serviceCredentials :: Maybe ServiceCredentials,
    -- | client-server protocol version range
    serverVRange :: VersionRange v,
+    proxiedRelayVRange :: VersionRange v,
    -- | agree shared session secret (used in SMP proxy for additional encryption layer)
    agreeSecret :: Bool,
    -- | Whether connecting client is a proxy server. See comment in ClientHandshake
@@ -495,6 +498,7 @@ defaultClientConfig clientALPN useSNI serverVRange =
      clientALPN,
      serviceCredentials = Nothing,
      serverVRange,
+      proxiedRelayVRange = serverVRange,
      agreeSecret = False,
      proxyServer = False,
      useSNI
@@ -505,6 +509,7 @@ defaultSMPClientConfig :: ProtocolClientConfig SMPVersion
 defaultSMPClientConfig =
  (defaultClientConfig (Just alpnSupportedSMPHandshakes) False supportedClientSMPRelayVRange)
    { defaultTransport = (show defaultSMPPort, transport @TLS),
+      proxiedRelayVRange = supportedClientSMPRelayVRange,
      agreeSecret = True
    }
 {-# INLINE defaultSMPClientConfig #-}
@@ -568,7 +573,7 @@ type SMPTransportSession = TransportSession BrokerMsg
 -- A single queue can be used for multiple 'SMPClient' instances,
 -- as 'SMPServerTransmission' includes server information.
 getProtocolClient :: forall v err msg. Protocol v err msg => TVar ChaChaDRG -> NetworkRequestMode -> TransportSession msg -> ProtocolClientConfig v -> [HostName] -> Maybe (TBQueue (ServerTransmissionBatch v err msg)) -> UTCTime -> (ProtocolClient v err msg -> IO ()) -> IO (Either (ProtocolClientError err) (ProtocolClient v err msg))
-getProtocolClient g nm transportSession@(_, srv, _) cfg@ProtocolClientConfig {qSize, networkConfig, clientALPN, serviceCredentials, serverVRange, agreeSecret, proxyServer, useSNI} presetDomains msgQ proxySessTs disconnected = do
+getProtocolClient g nm transportSession@(_, srv, _) cfg@ProtocolClientConfig {qSize, networkConfig, clientALPN, serviceCredentials, serverVRange, proxiedRelayVRange, agreeSecret, proxyServer, useSNI} presetDomains msgQ proxySessTs disconnected = do
  case chooseTransportHost networkConfig (host srv) of
    Right useHost ->
      (getCurrentTime >>= mkProtocolClient useHost >>= runClient useTransport useHost)
@@ -593,6 +598,7 @@ getProtocolClient g nm transportSession@(_, srv, _) cfg@ProtocolClientConfig {qS
            transportHost,
            tcpConnectTimeout,
            tcpTimeout,
+            proxiedRelayVRange,
            sendPings,
            lastReceived,
            timeoutErrorCount,
@@ -1115,10 +1121,10 @@ deleteSMPQueues = okSMPCommands DEL
 -- send PRXY :: SMPServer -> Maybe BasicAuth -> Command Sender
 -- receives PKEY :: SessionId -> X.CertificateChain -> X.SignedExact X.PubKey -> BrokerMsg
 connectSMPProxiedRelay :: SMPClient -> NetworkRequestMode -> SMPServer -> Maybe BasicAuth -> ExceptT SMPClientError IO ProxiedRelay
-connectSMPProxiedRelay c@ProtocolClient {client_ = PClient {tcpConnectTimeout, tcpTimeout}} nm relayServ@ProtocolServer {port = relayPort, keyHash = C.KeyHash kh} proxyAuth =
+connectSMPProxiedRelay c@ProtocolClient {client_ = PClient {tcpConnectTimeout, tcpTimeout, proxiedRelayVRange}} nm relayServ@ProtocolServer {port = relayPort, keyHash = C.KeyHash kh} proxyAuth =
  sendProtocolCommand_ c nm Nothing tOut Nothing NoEntity (Cmd SProxiedClient (PRXY relayServ proxyAuth)) >>= \case
    PKEY sId vr (CertChainPubKey chain key) ->
-      case supportedClientSMPRelayVRange `compatibleVersion` vr of
+      case proxiedRelayVRange `compatibleVersion` vr of
        Nothing -> throwE $ transportErr TEVersion
        Just (Compatible v) -> do
          relayKey <- liftEitherWith (const $ transportErr $ TEHandshake IDENTITY) =<< liftIO (runExceptT $ validateRelay chain key)
@@ -1169,7 +1175,7 @@ instance StrEncoding ProxyClientError where
 -- consider how to process slow responses - is it handled somehow locally or delegated to the caller
 -- this method is used in the client
 -- sends PFWD :: C.PublicKeyX25519 -> EncTransmission -> Command Sender
-- receives PRES :: EncResponse -> BrokerMsg -- proxy to client
+-- receives PRES :: Maybe C.CbNonce -> EncResponse -> BrokerMsg -- proxy to client

 -- When client sends message via proxy, there may be one successful scenario and 9 error scenarios
 -- as shown below (WTF stands for unexpected response, ??? for response that failed to parse).
@@ -1228,14 +1234,14 @@ proxySMPCommand c@ProtocolClient {thParams = proxyThParams, client_ = PClient {c
    TBError e _ : _ -> throwE $ PCETransportError e
    TBTransmission s _ : _ -> pure s
    TBTransmissions s _ _ : _ -> pure s
-  et <- liftEitherWith PCECryptoError $ EncTransmission <$> C.cbEncrypt cmdSecret nonce b paddedProxiedTLength
+  et <- liftEitherWith PCECryptoError $ EncTransmission <$> C.cbEncrypt cmdSecret (encTransmissionNonce v nonce) b paddedProxiedTLength
  -- proxy interaction errors are wrapped
  let tOut = Just $ 2 * netTimeoutInt tcpTimeout nm
  tryE (sendProtocolCommand_ c nm (Just nonce) tOut Nothing (EntityId sessionId) (Cmd SProxiedClient (PFWD v cmdPubKey et))) >>= \case
    Right r -> case r of
-      PRES (EncResponse er) -> do
+      PRES nonce_ (EncResponse er) -> do
        -- server interaction errors are thrown directly
-        t' <- liftEitherWith PCECryptoError $ C.cbDecrypt cmdSecret (C.reverseNonce nonce) er
+        t' <- liftEitherWith PCECryptoError $ C.cbDecrypt cmdSecret (fromMaybe (C.reverseNonce nonce) nonce_) er
        case tParse serverThParams t' of
          t'' :| [] -> case tDecodeClient serverThParams t'' of
            (_, _, cmd) -> case cmd of
@@ -1253,10 +1259,10 @@ proxySMPCommand c@ProtocolClient {thParams = proxyThParams, client_ = PClient {c

 -- this method is used in the proxy
 -- sends RFWD :: EncFwdTransmission -> Command Sender
-- receives RRES :: EncFwdResponse -> BrokerMsg
+-- receives RRES :: Maybe C.CbNonce -> EncFwdResponse -> BrokerMsg
 -- proxy should send PRES to the client with EncResponse
 -- Always uses background timeout mode
-forwardSMPTransmission :: SMPClient -> CorrId -> VersionSMP -> C.PublicKeyX25519 -> EncTransmission -> ExceptT SMPClientError IO EncResponse
+forwardSMPTransmission :: SMPClient -> CorrId -> VersionSMP -> C.PublicKeyX25519 -> EncTransmission -> ExceptT SMPClientError IO (Maybe C.CbNonce, EncResponse)
 forwardSMPTransmission c@ProtocolClient {thParams, client_ = PClient {clientCorrId = g}} fwdCorrId fwdVersion fwdKey fwdTransmission = do
  -- prepare params
  sessSecret <- case thAuth thParams of
@@ -1268,11 +1274,11 @@ forwardSMPTransmission c@ProtocolClient {thParams, client_ = PClient {clientCorr
      eft = EncFwdTransmission $ C.cbEncryptNoPad sessSecret nonce (smpEncode fwdT)
  -- send
  sendProtocolCommand_ c NRMBackground (Just nonce) Nothing Nothing NoEntity (Cmd SProxyService (RFWD eft)) >>= \case
-    RRES (EncFwdResponse efr) -> do
+    RRES nonce_ (EncFwdResponse efr) -> do
      -- unwrap
      r' <- liftEitherWith PCECryptoError $ C.cbDecryptNoPad sessSecret (C.reverseNonce nonce) efr
      FwdResponse {fwdCorrId = _, fwdResponse} <- liftEitherWith (const $ PCEResponseError BLOCK) $ smpDecode r'
-      pure fwdResponse
+      pure (nonce_, fwdResponse)
    r -> throwE $ unexpectedResponse r

 -- get queue information - always sent interactively
@@ -931,6 +931,8 @@ data CryptoError
    CERatchetEarlierMessage Word32
  | -- | duplicate message number
    CERatchetDuplicateMessage
+  | -- | KEM key generation failed, indicating a broken RNG
+    CryptoKEMKeyGenError
  deriving (Eq, Show, Exception)

 aesKeySize :: Int
@@ -1045,9 +1047,7 @@ md5Hash = BA.convert . (hash :: ByteString -> Digest MD5)

 -- | AEAD-GCM encryption with associated data.
 --
-- Used as part of double ratchet encryption.
-- This function requires 16 bytes IV, it transforms IV in cryptonite_aes_gcm_init here:
-- https://github.com/haskell-crypto/cryptonite/blob/master/cbits/cryptonite_aes.c
+-- Used as part of double ratchet encryption, with a 16-byte IV (see @initAEAD@).
 encryptAEAD :: Key -> IV -> Int -> ByteString -> ByteString -> ExceptT CryptoError IO (AuthTag, ByteString)
 encryptAEAD aesKey ivBytes paddedLen ad msg = do
  aead <- initAEAD @AES256 aesKey ivBytes
@@ -1067,10 +1067,7 @@ encryptAEADNoPad aesKey ivBytes ad msg = do

 -- | AEAD-GCM decryption with associated data.
 --
-- Used as part of double ratchet encryption.
-- This function requires 16 bytes IV, it transforms IV in cryptonite_aes_gcm_init here:
-- https://github.com/haskell-crypto/cryptonite/blob/master/cbits/cryptonite_aes.c
-- To make it compatible with WebCrypto we will need to start using initAEADGCM.
+-- Used as part of double ratchet encryption, with a 16-byte IV (see @initAEAD@).
 decryptAEAD :: Key -> IV -> ByteString -> ByteString -> AuthTag -> ExceptT CryptoError IO ByteString
 decryptAEAD aesKey ivBytes ad msg (AuthTag authTag) = do
  aead <- initAEAD @AES256 aesKey ivBytes
@@ -1148,9 +1145,9 @@ maxLength :: forall i. KnownNat i => Int
 maxLength = fromIntegral (natVal $ Proxy @i)
 {-# INLINE maxLength #-}

-- this function requires 16 bytes IV, it transforms IV in cryptonite_aes_gcm_init here:
-- https://github.com/haskell-crypto/cryptonite/blob/master/cbits/cryptonite_aes.c
-- This is used for double ratchet encryption, so to make it compatible with WebCrypto we will need to deprecate it and start using initAEADGCM
+-- The 16-byte double ratchet IV is intentionally not the 96-bit IV recommended by NIST SP 800-38D, so GCM derives J0 = GHASH(IV || 0^64 || [128]_64),
+-- as in crypton_aes_gcm_init: https://hackage.haskell.org/package/crypton-0.34/src/cbits/crypton_aes.c
+-- WebCrypto and other SP 800-38D implementations interoperate only when given all 16 IV bytes.
 initAEAD :: forall c. AES.BlockCipher c => Key -> IV -> ExceptT CryptoError IO (AES.AEAD c)
 initAEAD (Key aesKey) (IV ivBytes) = do
  iv <- makeIV @c ivBytes
@@ -72,6 +72,7 @@ module Simplex.Messaging.Crypto.Ratchet
    rcEncryptHeader,
    rcEncryptMsg,
    rcDecrypt,
+    maxSkippedMsgKeys,
    -- used in tests
    MsgHeader (..),
    RatchetInitParams (..),
@@ -119,7 +120,7 @@ import Simplex.Messaging.Crypto.SNTRUP761.Bindings
 import Simplex.Messaging.Encoding
 import Simplex.Messaging.Encoding.String
 import Simplex.Messaging.Parsers (defaultJSON, parseE, parseE')
-import Simplex.Messaging.Util (($>>=), (<$?>))
+import Simplex.Messaging.Util ((<$?>))
 import Simplex.Messaging.Version
 import Simplex.Messaging.Version.Internal
 import UnliftIO.STM
@@ -327,9 +328,13 @@ instance StrEncoding AnyE2ERatchetParamsUri where
            Nothing -> pure $ AnyE2ERatchetParamsUri SRKSProposed a $ E2ERatchetParamsUri vr k1 k2 Nothing
      _ -> fail "bad e2e params"
    where
-      kemP query =
-        queryParam_ "kem_key" query
-          $>>= \k -> Just . kemParams k <$> queryParam_ "kem_ct" query
+      kemP query = do
+        k_ <- queryParam_ "kem_key" query
+        ct_ <- queryParam_ "kem_ct" query
+        case (k_, ct_) of
+          (Just k, _) -> pure $ Just $ kemParams k ct_
+          (Nothing, Nothing) -> pure Nothing
+          (Nothing, Just _) -> fail "bad e2e params: kem_ct without kem_key"
      kemParams k = \case
        Nothing -> ARKP SRKSProposed $ RKParamsProposed k
        Just ct -> ARKP SRKSAccepted $ RKParamsAccepted ct k
@@ -958,6 +963,9 @@ type DecryptResult a = (Either CryptoError ByteString, Ratchet a, SkippedMsgDiff
 maxSkip :: Word32
 maxSkip = 512

+maxSkippedMsgKeys :: Int
+maxSkippedMsgKeys = 2000
+
 rcDecrypt ::
  forall a.
  (AlgorithmI a, DhAlgorithm a) =>
@@ -16,6 +16,8 @@ module Simplex.Messaging.Crypto.SNTRUP761.Bindings
  ) where

 import Control.Concurrent.STM
+import Control.Exception (throwIO)
+import Control.Monad (when)
 import Crypto.Random (ChaChaDRG)
 import Data.Aeson (FromJSON (..), ToJSON (..))
 import Data.Bifunctor (bimap)
@@ -23,6 +25,7 @@ import Data.ByteArray (ScrubbedBytes)
 import qualified Data.ByteArray as BA
 import Data.ByteString (ByteString)
 import Simplex.Messaging.Agent.Store.DB (FromField (..), ToField (..))
+import qualified Simplex.Messaging.Crypto as C
 import Simplex.Messaging.Crypto.SNTRUP761.Bindings.Defines
 import Simplex.Messaging.Crypto.SNTRUP761.Bindings.FFI
 import Simplex.Messaging.Crypto.SNTRUP761.Bindings.RNG (rngFuncPtr, withDRG)
@@ -55,14 +58,13 @@ pattern KEMSharedKey s <- KEMSharedKey_ s
 type KEMKeyPair = (KEMPublicKey, KEMSecretKey)

 sntrup761Keypair :: TVar ChaChaDRG -> IO KEMKeyPair
-sntrup761Keypair drg =
-  bimap KEMPublicKey_ KEMSecretKey
-    <$> BA.allocRet
-      c_SNTRUP761_SECRETKEY_SIZE
-      ( \skPtr ->
-          BA.alloc c_SNTRUP761_PUBLICKEY_SIZE $ \pkPtr ->
-            withDRG drg $ \cxtPtr -> c_sntrup761_keypair pkPtr skPtr cxtPtr rngFuncPtr
-      )
+sntrup761Keypair drg = do
+  ((r, pk), sk) <-
+    BA.allocRet c_SNTRUP761_SECRETKEY_SIZE $ \skPtr ->
+      BA.allocRet c_SNTRUP761_PUBLICKEY_SIZE $ \pkPtr ->
+        withDRG drg $ \cxtPtr -> c_sntrup761_keypair pkPtr skPtr cxtPtr rngFuncPtr
+  when (r /= 0) $ throwIO C.CryptoKEMKeyGenError
+  pure (KEMPublicKey_ pk, KEMSecretKey sk)

 sntrup761Enc :: TVar ChaChaDRG -> KEMPublicKey -> IO (KEMCiphertext, KEMSharedKey)
 sntrup761Enc drg (KEMPublicKey pk) =
@@ -10,9 +10,9 @@ import Foreign
 import Foreign.C
 import Simplex.Messaging.Crypto.SNTRUP761.Bindings.RNG (RNGContext, RNGFunc)

-- void sntrup761_keypair (uint8_t *pk, uint8_t *sk, void *random_ctx, sntrup761_random_func *random);
+-- int sntrup761_keypair (uint8_t *pk, uint8_t *sk, void *random_ctx, sntrup761_random_func *random);
 foreign import ccall "sntrup761_keypair"
-  c_sntrup761_keypair :: Ptr Word8 -> Ptr Word8 -> Ptr RNGContext -> FunPtr RNGFunc -> IO ()
+  c_sntrup761_keypair :: Ptr Word8 -> Ptr Word8 -> Ptr RNGContext -> FunPtr RNGFunc -> IO CInt

 -- void sntrup761_enc (uint8_t *c, uint8_t *k, const uint8_t *pk, void *random_ctx, sntrup761_random_func *random);
 foreign import ccall "sntrup761_enc"
@@ -168,6 +168,7 @@ module Simplex.Messaging.Protocol
    EncFwdTransmission (..),
    EncResponse (..),
    EncTransmission (..),
+    encTransmissionNonce,
    FwdResponse (..),
    FwdTransmission (..),
    NameRecord (..),
@@ -241,6 +242,7 @@ import Data.Attoparsec.ByteString.Char8 (Parser, (<?>))
 import qualified Data.Attoparsec.ByteString.Char8 as A
 import Data.Bifunctor (bimap, first)
 import Data.Bits (xor)
+import qualified Data.ByteArray as BA
 import qualified Data.ByteString as BS
 import qualified Data.ByteString.Base64 as B64
 import Data.ByteString.Char8 (ByteString)
@@ -279,7 +281,7 @@ import Simplex.Messaging.ServiceScheme
 import Simplex.Messaging.SimplexName (LabelHash, SimplexDomain (..), SimplexTLD (..), fullDomainName, labelHash)
 import Simplex.Messaging.Transport
 import Simplex.Messaging.Transport.Client (TransportHost, TransportHosts (..))
-import Simplex.Messaging.Util (bshow, eitherToMaybe, safeDecodeUtf8, (<$?>))
+import Simplex.Messaging.Util (bshow, eitherToMaybe, packZipWith, safeDecodeUtf8, (<$?>))
 import Simplex.Messaging.Version
 import Simplex.Messaging.Version.Internal

@@ -701,6 +703,11 @@ instance Encoding NewNtfCreds where
 newtype EncTransmission = EncTransmission ByteString
  deriving (Show)

+encTransmissionNonce :: VersionSMP -> C.CbNonce -> C.CbNonce
+encTransmissionNonce v nonce@(C.CbNonce s)
+  | v >= fwdNoncesSMPVersion = C.cbNonce $ packZipWith xor (smpEncode v) s <> BS.drop 2 s
+  | otherwise = nonce
+
 data FwdTransmission = FwdTransmission
  { fwdCorrId :: CorrId,
    fwdVersion :: VersionSMP,
@@ -736,8 +743,8 @@ data BrokerMsg where
  NMSG :: C.CbNonce -> EncNMsgMeta -> BrokerMsg
  -- Should include certificate chain
  PKEY :: SessionId -> VersionRangeSMP -> CertChainPubKey -> BrokerMsg -- TLS-signed server key for proxy shared secret and initial sender key
-  RRES :: EncFwdResponse -> BrokerMsg -- relay to proxy
-  PRES :: EncResponse -> BrokerMsg -- proxy to client
+  RRES :: Maybe C.CbNonce -> EncFwdResponse -> BrokerMsg -- relay to proxy
+  PRES :: Maybe C.CbNonce -> EncResponse -> BrokerMsg -- proxy to client
  END :: BrokerMsg
  ENDS :: Int64 -> IdsHash -> BrokerMsg
  DELD :: BrokerMsg
@@ -1323,7 +1330,10 @@ instance ProtocolTypeI p => FromJSON (ProtocolServer p) where
  parseJSON = strParseJSON "ProtocolServer"

 newtype BasicAuth = BasicAuth {unBasicAuth :: ByteString}
-  deriving (Eq, Ord, Show)
+  deriving (Ord, Show)
+
+instance Eq BasicAuth where
+  BasicAuth s == BasicAuth s' = BA.constEq s s'

 instance IsString BasicAuth where fromString = BasicAuth . B.pack

@@ -1814,8 +1824,7 @@ instance PartyI p => ProtocolEncoding SMPVersion ErrorType (Command p) where
  encodeProtocol v = \case
    NEW NewQueueReq {rcvAuthKey = rKey, rcvDhKey = dhKey, auth_, subMode, queueReqData, ntfCreds}
      | v >= newNtfCredsSMPVersion -> new <> e (subMode, queueReqData, ntfCreds)
-      | v >= shortLinksSMPVersion -> new <> e (subMode, queueReqData)
-      | otherwise -> new <> e (subMode, senderCanSecure (queueReqMode <$> queueReqData))
+      | otherwise -> new <> e (subMode, queueReqData)
      where
        new = e (NEW_, ' ', rKey, dhKey, auth_)
    SUB -> e SUB_
@@ -1902,20 +1911,18 @@ instance ProtocolEncoding SMPVersion ErrorType Cmd where
    CT SCreator NEW_ -> Cmd SCreator <$> newCmd
      where
        newCmd
-          | v >= newNtfCredsSMPVersion = new smpP smpP
-          | v >= shortLinksSMPVersion = new smpP nothing
-          | otherwise = new (qReq <$> smpP) nothing
+          | v >= newNtfCredsSMPVersion = new smpP
+          | otherwise = new nothing
          where
            nothing = pure Nothing
-            new p2 p3 = NEW <$> do
+            new p3 = NEW <$> do
              rcvAuthKey <- _smpP
              rcvDhKey <- smpP
              auth_ <- smpP
              subMode <- smpP
-              queueReqData <- p2
+              queueReqData <- smpP
              ntfCreds <- p3
              pure NewQueueReq {rcvAuthKey, rcvDhKey, auth_, subMode, queueReqData, ntfCreds}
-            qReq sndSecure = Just $ if sndSecure then QRMessaging Nothing else QRContact Nothing
    CT SRecipient tag ->
      Cmd SRecipient <$> case tag of
        SUB_ -> pure SUB
@@ -1966,8 +1973,7 @@ instance ProtocolEncoding SMPVersion ErrorType BrokerMsg where
    IDS QIK {rcvId, sndId, rcvPublicDhKey = srvDh, queueMode, linkId, serviceId, serverNtfCreds}
      | v >= newNtfCredsSMPVersion -> ids <> e (queueMode, linkId, serviceId, serverNtfCreds)
      | v >= serviceCertsSMPVersion -> ids <> e (queueMode, linkId, serviceId)
-      | v >= shortLinksSMPVersion -> ids <> e (queueMode, linkId)
-      | otherwise -> ids <> e (senderCanSecure queueMode)
+      | otherwise -> ids <> e (queueMode, linkId)
      where
        ids = e (IDS_, ' ', rcvId, sndId, srvDh)
    LNK sId d -> e (LNK_, ' ', sId, d)
@@ -1981,8 +1987,8 @@ instance ProtocolEncoding SMPVersion ErrorType BrokerMsg where
    NID nId srvNtfDh -> e (NID_, ' ', nId, srvNtfDh)
    NMSG nmsgNonce encNMsgMeta -> e (NMSG_, ' ', nmsgNonce, encNMsgMeta)
    PKEY sid vr certKey -> e (PKEY_, ' ', sid, vr, certKey)
-    RRES (EncFwdResponse encBlock) -> e (RRES_, ' ', Tail encBlock)
-    PRES (EncResponse encBlock) -> e (PRES_, ' ', Tail encBlock)
+    RRES nonce_ (EncFwdResponse encBlock) -> fwdResp RRES_ nonce_ encBlock
+    PRES nonce_ (EncResponse encBlock) -> fwdResp PRES_ nonce_ encBlock
    END -> e END_
    ENDS n idsHash -> serviceResp ENDS_ n idsHash
    DELD -> e DELD_
@@ -2006,6 +2012,9 @@ instance ProtocolEncoding SMPVersion ErrorType BrokerMsg where
      serviceResp tag n idsHash
        | v >= rcvServiceSMPVersion = e (tag, ' ', n, idsHash)
        | otherwise = e (tag, ' ', n)
+      fwdResp tag nonce_ encBlock
+        | v >= fwdNoncesSMPVersion = e (tag, ' ', nonce_, Tail encBlock)
+        | otherwise = e (tag, ' ', Tail encBlock)

  protocolP v = \case
    MSG_ -> do
@@ -2015,19 +2024,17 @@ instance ProtocolEncoding SMPVersion ErrorType BrokerMsg where
        bodyP = EncRcvMsgBody . unTail <$> smpP
    ALLS_ -> pure ALLS
    IDS_
-      | v >= newNtfCredsSMPVersion -> ids smpP smpP smpP smpP
-      | v >= serviceCertsSMPVersion -> ids smpP smpP smpP nothing
-      | v >= shortLinksSMPVersion -> ids smpP smpP nothing nothing
-      | otherwise -> ids (qm <$> smpP) nothing nothing nothing
+      | v >= newNtfCredsSMPVersion -> ids smpP smpP
+      | v >= serviceCertsSMPVersion -> ids smpP nothing
+      | otherwise -> ids nothing nothing
      where
-        qm sndSecure = Just $ if sndSecure then QMMessaging else QMContact
        nothing = pure Nothing
-        ids p1 p2 p3 p4 = do
+        ids p3 p4 = do
          rcvId <- _smpP
          sndId <- smpP
          rcvPublicDhKey <- smpP
-          queueMode <- p1
-          linkId <- p2
+          queueMode <- smpP
+          linkId <- smpP
          serviceId <- p3
          serverNtfCreds <- p4
          pure $ IDS QIK {rcvId, sndId, rcvPublicDhKey, queueMode, linkId, serviceId, serverNtfCreds}
@@ -2037,8 +2044,8 @@ instance ProtocolEncoding SMPVersion ErrorType BrokerMsg where
    NID_ -> NID <$> _smpP <*> smpP
    NMSG_ -> NMSG <$> _smpP <*> smpP
    PKEY_ -> PKEY <$> _smpP <*> smpP <*> smpP
-    RRES_ -> RRES <$> (EncFwdResponse . unTail <$> _smpP)
-    PRES_ -> PRES <$> (EncResponse . unTail <$> _smpP)
+    RRES_ -> fwdRespP RRES EncFwdResponse
+    PRES_ -> fwdRespP PRES EncResponse
    END_ -> pure END
    ENDS_ -> serviceRespP ENDS
    DELD_ -> pure DELD
@@ -2055,6 +2062,9 @@ instance ProtocolEncoding SMPVersion ErrorType BrokerMsg where
      serviceRespP resp
        | v >= rcvServiceSMPVersion = resp <$> _smpP <*> smpP
        | otherwise = resp <$> _smpP <*> pure mempty
+      fwdRespP :: (Maybe C.CbNonce -> a -> BrokerMsg) -> (ByteString -> a) -> Parser BrokerMsg
+      fwdRespP resp enc = resp <$> (A.space *> nonceP) <*> (enc <$> A.takeByteString)
+      nonceP = if v >= fwdNoncesSMPVersion then smpP else pure Nothing

  fromProtocolError = \case
    PECmdSyntax -> CMD SYNTAX
@@ -2071,7 +2081,7 @@ instance ProtocolEncoding SMPVersion ErrorType BrokerMsg where
    -- PONG response must not have queue ID
    PONG -> noEntityMsg
    PKEY {} -> noEntityMsg
-    RRES _ -> noEntityMsg
+    RRES {} -> noEntityMsg
    ALLS -> noEntityMsg
    RNAME {} -> noEntityMsg
    -- other broker responses must have queue ID
@@ -33,9 +33,6 @@ module Simplex.Messaging.Server
  ( runSMPServer,
    runSMPServerBlocking,
    controlPortAuth,
-    importMessages,
-    exportMessages,
-    printMessageStats,
    disconnectTransport,
    verifyCmdAuthorization,
    dummyVerifyCmd,
@@ -108,7 +105,6 @@ import Simplex.Messaging.Server.Control
 import Simplex.Messaging.Server.Env.STM as Env
 import Simplex.Messaging.Server.Expiration
 import Simplex.Messaging.Server.MsgStore
-import Simplex.Messaging.Server.MsgStore.Journal (JournalMsgStore, JournalQueue (..), getJournalQueueMessages)
 import Simplex.Messaging.Server.Names (NamesEnv, closeNamesEnv, resolveName)
 import Simplex.Messaging.Server.MsgStore.STM
 import Simplex.Messaging.Server.MsgStore.Types
@@ -116,6 +112,7 @@ import Simplex.Messaging.Server.NtfStore
 import Simplex.Messaging.Server.Prometheus
 import Simplex.Messaging.Server.QueueStore
 import Simplex.Messaging.Server.QueueStore.QueueInfo
+import Simplex.Messaging.Server.QueueStore.STM (withLoadedQueues)
 import Simplex.Messaging.Server.QueueStore.Types
 import Simplex.Messaging.Server.Stats
 import Simplex.Messaging.Server.StoreLog (foldLogLines)
@@ -145,7 +142,7 @@ import GHC.Conc.Sync (threadLabel)
 #endif

 #if defined(dbServerPostgres)
-import Simplex.Messaging.Server.MsgStore.Postgres (exportDbMessages, getDbMessageStats)
+import Simplex.Messaging.Server.MsgStore.Postgres (getDbMessageStats)
 #endif

 -- | Runs an SMP server using passed configuration.
@@ -710,7 +707,11 @@ smpServer started cfg@ServerConfig {transports, transportConfig = tCfg, startOpt
      (deliveredSubs, deliveredTimes) <- getDeliveredMetrics =<< getSystemSeconds
      smpSubs <- getSubscribersMetrics subscribers
      ntfSubs <- getSubscribersMetrics ntfSubscribers
-      loadedCounts <- loadedQueueCounts $ fromMsgStore ms
+      loadedCounts <- case ms of
+        StoreMemory st -> Just <$> loadedQueueCounts st
+#if defined(dbServerPostgres)
+        StoreDatabase _ -> pure Nothing
+#endif
      pure RealTimeMetrics {socketStats, threadsCount, clientsCount, deliveredSubs, deliveredTimes, smpSubs, ntfSubs, loadedCounts}
      where
        getSubscribersMetrics ServerSubscribers {queueSubscribers, serviceSubscribers, totalServiceSubs, subClients} = do
@@ -1092,6 +1093,7 @@ controlPortAuth h user admin role auth = do
  readTVarIO role >>= \case
    CPRNone -> do
      atomically $ writeTVar role $! newRole
+      when (newRole == CPRNone) $ logWarn "ControlPort: failed auth"
      hPutStrLn h $ currentRole newRole
    r -> hPutStrLn h $ currentRole r <> if r == newRole then "" else ", start new session to change."
  where
@@ -1318,11 +1320,6 @@ isContactQueue QueueRec {queueMode, senderKey} = case queueMode of
  Just QMContact -> True
  Nothing -> isNothing senderKey -- for backward compatibility with pre-SKEY contact addresses

-isSecuredMsgQueue :: QueueRec -> Bool
-isSecuredMsgQueue QueueRec {queueMode, senderKey} = case queueMode of
-  Just QMContact -> False
-  _ -> isJust senderKey
-
 -- Random correlation ID is used as a nonce in case crypto_box authenticator is used to authorize transmission
 verifyCmdAuthorization :: Maybe (THandleAuth 'TServer) -> Maybe TAuthorizations -> ByteString -> CorrId -> C.APublicAuthKey -> Bool
 verifyCmdAuthorization thAuth tAuth authorized corrId key = maybe False (verify key) tAuth
@@ -1461,7 +1458,7 @@ client
            inc own pRequests
            forkProxiedCmd $ do
              liftIO (runExceptT (forwardSMPTransmission smp corrId fwdV pubKey encBlock) `E.catches` clientHandlers)  >>= \case
-                Right r -> PRES r <$ inc own pSuccesses
+                Right (nonce_, r) -> PRES nonce_ r <$ inc own pSuccesses
                Left e -> ERR (smpProxyError e) <$ case e of
                  PCEProtocolError {} -> inc own pSuccesses
                  _ -> inc own pErrorsOther
@@ -2127,9 +2124,10 @@ client
          let proxyNonce = C.cbNonce $ bs corrId
          s' <- liftEitherWith (const CRYPTO) $ C.cbDecryptNoPad sessSecret proxyNonce s
          FwdTransmission {fwdCorrId, fwdVersion, fwdKey, fwdTransmission = EncTransmission et} <- liftEitherWith (const $ CMD SYNTAX) $ smpDecode s'
+          unless (fwdVersion `isCompatible` thServerVRange thParams') $ throwE $ transportErr TEVersion
          let clientSecret = C.dh' fwdKey serverPrivKey
              clientNonce = C.cbNonce $ bs fwdCorrId
-          b <- liftEitherWith (const CRYPTO) $ C.cbDecrypt clientSecret clientNonce et
+          b <- liftEitherWith (const CRYPTO) $ C.cbDecrypt clientSecret (encTransmissionNonce fwdVersion clientNonce) et
          let clntTHParams = smpTHParamsSetVersion fwdVersion thParams'
          -- only allowing single forwarded transactions
          t' <- case tParse clntTHParams b of
@@ -2142,9 +2140,13 @@ client
                  TBError _ _ : _ -> throwE BLOCK
                  TBTransmission b' _ : _ -> pure b'
                  TBTransmissions b' _ _ : _ -> pure b'
-                r2 <- liftEitherWith (const BLOCK) $ EncResponse <$> C.cbEncrypt clientSecret (C.reverseNonce clientNonce) r' paddedProxiedTLength
+                nonce_ <-
+                  if fwdVersion >= fwdNoncesSMPVersion
+                    then Just <$> (atomically . C.randomCbNonce =<< asks random)
+                    else pure Nothing
+                r2 <- liftEitherWith (const BLOCK) $ EncResponse <$> C.cbEncrypt clientSecret (fromMaybe (C.reverseNonce clientNonce) nonce_) r' paddedProxiedTLength
                let fr = FwdResponse {fwdCorrId, fwdResponse = r2}
-                pure $ RRES $ EncFwdResponse $ C.cbEncryptNoPad sessSecret (C.reverseNonce proxyNonce) (smpEncode fr)
+                pure $ RRES nonce_ $ EncFwdResponse $ C.cbEncryptNoPad sessSecret (C.reverseNonce proxyNonce) (smpEncode fr)
          -- the inner response, or Nothing if forked (RSLV).
          r_ <- lift (rejectOrVerify clntThAuth t') >>= \case
            -- rejectOrVerify filters allowed commands, no need to repeat it here.
@@ -2271,40 +2273,27 @@ randomId = fmap EntityId . randomId'

 saveServerMessages :: Bool -> MsgStore s -> IO ()
 saveServerMessages drainMsgs ms = case ms of
-  StoreMemory STMMsgStore {storeConfig = STMStoreConfig {storePath}} -> case storePath of
-    Just f -> exportMessages False ms f drainMsgs
+  StoreMemory ms'@STMMsgStore {storeConfig = STMStoreConfig {storePath}} -> case storePath of
+    Just f -> exportMessages ms' f drainMsgs
    Nothing -> logNote "undelivered messages are not saved"
-  StoreJournal _ -> logNote "closed journal message storage"
 #if defined(dbServerPostgres)
  StoreDatabase _ -> logNote "closed postgres message storage"
 #endif

-exportMessages :: forall s. MsgStoreClass s => Bool -> MsgStore s -> FilePath -> Bool -> IO ()
-exportMessages tty st f drainMsgs = do
+exportMessages :: STMMsgStore -> FilePath -> Bool -> IO ()
+exportMessages ms f drainMsgs = do
  logNote $ "saving messages to file " <> T.pack f
-  run $ case st of
-    StoreMemory ms -> exportMessages_ ms $ getMsgs ms
-    StoreJournal ms -> exportMessages_ ms $ getJournalMsgs ms
-#if defined(dbServerPostgres)
-    StoreDatabase ms -> exportDbMessages tty ms
-#endif
+  run $ fmap (\(Sum n) -> n) . withLoadedQueues (queueStore ms) . saveQueueMsgs
  where
-    exportMessages_ ms get = fmap (\(Sum n) -> n) . unsafeWithAllMsgQueues tty ms . saveQueueMsgs get
    run :: (Handle -> IO Int) -> IO ()
    run a = liftIO $ withFile f WriteMode $ tryAny . a >=> \case
      Right n -> logNote $ "messages saved: " <> tshow n
      Left e -> do
        logError $ "error exporting messages: " <> tshow e
        exitFailure
-    getJournalMsgs ms q =
-      readTVarIO (msgQueue' q) >>= \case
-        Just _ -> getMsgs ms q
-        Nothing -> getJournalQueueMessages ms q
-    getMsgs :: MsgStoreClass s' => s' -> StoreQueue s' -> IO [Message]
-    getMsgs ms q = unsafeRunStore q "saveQueueMsgs" $ getQueueMessages_ drainMsgs q =<< getMsgQueue ms q False
-    saveQueueMsgs :: (StoreQueue s -> IO [Message]) -> Handle -> StoreQueue s -> IO (Sum Int)
-    saveQueueMsgs get h q = do
-      msgs <- get q
+    saveQueueMsgs :: Handle -> STMQueue -> IO (Sum Int)
+    saveQueueMsgs h q = do
+      msgs <- getQueueMessages drainMsgs q
      unless (null msgs) $ BLD.hPutBuilder h $ encodeMessages (recipientId q) msgs
      pure $ Sum $ length msgs
    encodeMessages rId = mconcat . map (\msg -> BLD.byteString (strEncode $ MLRv3 rId msg) <> BLD.char8 '\n')
@@ -2316,37 +2305,16 @@ processServerMessages StartOptions {skipWarnings} = do
  asks msgStore_ >>= liftIO . processMessages old_ expire
    where
      processMessages :: Maybe Int64 -> Bool -> MsgStore s' -> IO (Maybe MessageStats)
+#if defined(dbServerPostgres)
      processMessages old_ expire = \case
+#else
+      processMessages old_ _ = \case
+#endif
        StoreMemory ms@STMMsgStore {storeConfig = STMStoreConfig {storePath}} -> case storePath of
-          Just f -> ifM (doesFileExist f) (Just <$> importMessages False ms f old_ skipWarnings) (pure Nothing)
+          Just f -> ifM (doesFileExist f) (Just <$> importMessages ms f old_ skipWarnings) (pure Nothing)
          Nothing -> pure Nothing
-        StoreJournal ms -> processJournalMessages old_ expire ms
 #if defined(dbServerPostgres)
        StoreDatabase ms -> processDbMessages old_ expire ms
-#endif
-      processJournalMessages :: forall s. Maybe Int64 -> Bool -> JournalMsgStore s -> IO (Maybe MessageStats)
-      processJournalMessages old_ expire ms
-        | expire = Just <$> case old_ of
-            Just old -> do
-              logNote "expiring journal store messages..."
-              run $ processExpireQueue old
-            Nothing -> do
-              logNote "validating journal store messages..."
-              run processValidateQueue
-        | otherwise = logWarn "skipping message expiration" $> Nothing
-        where
-          run a = unsafeWithAllMsgQueues False ms a `catchAny` \_ -> exitFailure
-          processExpireQueue :: Int64 -> JournalQueue s -> IO MessageStats
-          processExpireQueue old q = unsafeRunStore q "processExpireQueue" $ do
-            mq <- getMsgQueue ms q False
-            expiredMsgsCount <- deleteExpireMsgs_ old q mq
-            storedMsgsCount <- getQueueSize_ mq
-            pure MessageStats {storedMsgsCount, expiredMsgsCount, storedQueues = 1}
-          processValidateQueue :: JournalQueue s -> IO MessageStats
-          processValidateQueue q = unsafeRunStore q "processValidateQueue" $ do
-            storedMsgsCount <- getQueueSize_ =<< getMsgQueue ms q False
-            pure newMessageStats {storedMsgsCount, storedQueues = 1}
-#if defined(dbServerPostgres)
      processDbMessages old_ expire ms
        | expire = Just <$> case old_ of
            Just old -> do
@@ -2358,18 +2326,17 @@ processServerMessages StartOptions {skipWarnings} = do
        | otherwise = logWarn "skipping message expiration" $> Nothing
 #endif

-importMessages :: forall s. MsgStoreClass s => Bool -> s -> FilePath -> Maybe Int64 -> Bool -> IO MessageStats
-importMessages tty ms f old_ skipWarnings  = do
+importMessages :: STMMsgStore -> FilePath -> Maybe Int64 -> Bool -> IO MessageStats
+importMessages ms f old_ skipWarnings  = do
  logNote $ "restoring messages from file " <> T.pack f
  (_, (storedMsgsCount, expiredMsgsCount, overQuota)) <-
-    foldLogLines tty f restoreMsg (Nothing, (0, 0, M.empty))
+    foldLogLines False f restoreMsg (Nothing, (0, 0, M.empty))
  renameFile f $ f <> ".bak"
  mapM_ setOverQuota_ overQuota
-  logQueueStates ms
-  EntityCounts {queueCount} <- liftIO $ getEntityCounts @(StoreQueue s) $ queueStore ms
+  EntityCounts {queueCount} <- liftIO $ getEntityCounts @STMQueue $ queueStore ms
  pure MessageStats {storedMsgsCount, expiredMsgsCount, storedQueues = queueCount}
  where
-    restoreMsg :: (Maybe (RecipientId, StoreQueue s), (Int, Int, Map RecipientId (StoreQueue s))) -> Bool -> ByteString -> IO (Maybe (RecipientId, StoreQueue s), (Int, Int, Map RecipientId (StoreQueue s)))
+    restoreMsg :: (Maybe (RecipientId, STMQueue), (Int, Int, Map RecipientId STMQueue)) -> Bool -> ByteString -> IO (Maybe (RecipientId, STMQueue), (Int, Int, Map RecipientId STMQueue))
    restoreMsg (q_, counts@(!stored, !expired, !overQuota)) eof s = case strDecode s of
      Right (MLRv3 rId msg) -> runExceptT (addToMsgQueue rId msg) >>= either (exitErr . tshow) pure
      Left e
@@ -2377,7 +2344,6 @@ importMessages tty ms f old_ skipWarnings  = do
        | otherwise -> exitErr $ parsingErr e
      where
        exitErr e = do
-          when tty $ putStrLn ""
          logError $ "error restoring messages: " <> e
          liftIO exitFailure
        parsingErr :: String -> Text
@@ -2390,7 +2356,6 @@ importMessages tty ms f old_ skipWarnings  = do
          case qOrErr of
            Right q -> addToQueue_ q rId msg
            Left AUTH -> liftIO $ do
-              when tty $ putStrLn ""
              warnOrExit $ "queue " <> safeDecodeUtf8 (encode $ unEntityId rId) <> " does not exist"
              pure (Nothing, counts)
            Left e -> throwE e
@@ -2401,20 +2366,13 @@ importMessages tty ms f old_ skipWarnings  = do
                  writeMsg ms q False msg >>= \case
                    Just _ -> pure (stored + 1, expired, overQuota)
                    Nothing -> liftIO $ do
-                      when tty $ putStrLn ""
                      logError $ decodeLatin1 $ "message queue " <> strEncode rId <> " is full, message not restored: " <> strEncode (messageId msg)
                      pure counts
              | otherwise -> pure (stored, expired + 1, overQuota)
            MessageQuota {} ->
              -- queue was over quota at some point,
              -- it will be set as over quota once fully imported
-              mergeQuotaMsgs >> writeMsg ms q False msg $> (stored, expired, M.insert rId q overQuota)
-              where
-                -- if the first message in queue head is "quota", remove it.
-                mergeQuotaMsgs =
-                  withPeekMsgQueue ms q "mergeQuotaMsgs" $ maybe (pure ()) $ \case
-                    (mq, MessageQuota {}) -> tryDeleteMsg_ q mq False
-                    _ -> pure ()
+              deleteQuotaMsg q >> writeMsg ms q False msg $> (stored, expired, M.insert rId q overQuota)
        warnOrExit e
          | skipWarnings = logWarn e'
          | otherwise = do
@@ -42,7 +42,6 @@ module Simplex.Messaging.Server.Env.STM
    VerifiedTransmission,
    ResponseAndMessage,
    newEnv,
-    mkJournalStoreConfig,
    msgStore,
    fromMsgStore,
    newClient,
@@ -68,10 +67,6 @@ module Simplex.Messaging.Server.Env.STM
    defaultInactiveClientExpiration,
    defaultProxyClientConcurrency,
    defaultNameResolverConcurrency,
-    defaultMaxJournalMsgCount,
-    defaultMaxJournalStateLines,
-    defaultIdleQueueInterval,
-    journalMsgStoreDepth,
    readWriteQueueStore,
    noPostgresExitStr,
    noPostgresExit,
@@ -96,7 +91,7 @@ import Data.List.NonEmpty (NonEmpty)
 import Data.Map.Strict (Map)
 import Data.Maybe (isJust)
 import qualified Data.Text as T
-import Data.Time.Clock (getCurrentTime, nominalDay)
+import Data.Time.Clock (getCurrentTime)
 import Data.Time.Clock.System (SystemTime)
 import qualified Data.X509 as X
 import Data.X509.Validation (Fingerprint (..))
@@ -113,7 +108,6 @@ import qualified Simplex.Messaging.Crypto as C
 import Simplex.Messaging.Protocol
 import Simplex.Messaging.Server.Expiration
 import Simplex.Messaging.Server.Information
-import Simplex.Messaging.Server.MsgStore.Journal
 import Simplex.Messaging.Server.MsgStore.STM
 import Simplex.Messaging.Server.MsgStore.Types
 import Simplex.Messaging.Server.Names (NamesConfig (..), NamesEnv, newNamesEnv, pingEndpoint)
@@ -146,8 +140,6 @@ data ServerConfig s = ServerConfig
    smpHandshakeTimeout :: Int,
    tbqSize :: Natural,
    msgQueueQuota :: Int,
-    maxJournalMsgCount :: Int,
-    maxJournalStateLines :: Int,
    queueIdBytes :: Int,
    msgIdBytes :: Int,
    serverStoreCfg :: ServerStoreCfg s,
@@ -164,8 +156,6 @@ data ServerConfig s = ServerConfig
    messageExpiration :: Maybe ExpirationConfig,
    expireMessagesOnStart :: Bool,
    expireMessagesOnSend :: Bool,
-    -- | interval of inactivity after which journal queue is closed
-    idleQueueInterval :: Int64,
    -- | notification expiration interval (seconds)
    notificationExpiration :: ExpirationConfig,
    -- | time after which the socket with inactive client can be disconnected (without any messages or commands, incl. PING),
@@ -229,9 +219,6 @@ defaultMessageExpiration =
      checkInterval = 7200 -- seconds, 2 hours
    }

-defaultIdleQueueInterval :: Int64
-defaultIdleQueueInterval = 14400 -- seconds, 4 hours
-
 defNtfExpirationHours :: Int64
 defNtfExpirationHours = 24

@@ -255,21 +242,9 @@ defaultProxyClientConcurrency = 32
 defaultNameResolverConcurrency :: Int
 defaultNameResolverConcurrency = 1000

-journalMsgStoreDepth :: Int
-journalMsgStoreDepth = 5
-
-defaultMaxJournalStateLines :: Int
-defaultMaxJournalStateLines = 16
-
-defaultMaxJournalMsgCount :: Int
-defaultMaxJournalMsgCount = 256
-
 defaultMsgQueueQuota :: Int
 defaultMsgQueueQuota = 128

-defaultStateTailSize :: Int
-defaultStateTailSize = 512
-
 data Env s = Env
  { config :: ServerConfig s,
    serverActive :: TVar Bool,
@@ -295,7 +270,6 @@ msgStore = fromMsgStore . msgStore_
 fromMsgStore :: MsgStore s -> s
 fromMsgStore = \case
  StoreMemory s -> s
-  StoreJournal s -> s
 #if defined(dbServerPostgres)
  StoreDatabase s -> s
 #endif
@@ -303,12 +277,10 @@ fromMsgStore = \case

 type family SupportedStore (qs :: QSType) (ms :: MSType) :: Constraint where
  SupportedStore 'QSMemory 'MSMemory = ()
-  SupportedStore 'QSMemory 'MSJournal = ()
  SupportedStore 'QSMemory 'MSPostgres =
    (Int ~ Bool, TypeError ('TE.Text "Storing messages in Postgres DB with queues in memory is not supported"))
  SupportedStore 'QSPostgres 'MSMemory =
    (Int ~ Bool, TypeError ('TE.Text "Storing messages in memory with queues in Postgres DB is not supported"))
-  SupportedStore 'QSPostgres 'MSJournal = ()
 #if defined(dbServerPostgres)
  SupportedStore 'QSPostgres 'MSPostgres = ()
 #else
@@ -322,8 +294,6 @@ data AStoreType =

 data ServerStoreCfg s where
  SSCMemory :: Maybe StorePaths -> ServerStoreCfg STMMsgStore
-  SSCMemoryJournal :: {storeLogFile :: FilePath, storeMsgsPath :: FilePath} -> ServerStoreCfg (JournalMsgStore 'QSMemory)
-  SSCDatabaseJournal :: {storeCfg :: PostgresStoreCfg, storeMsgsPath' :: FilePath} -> ServerStoreCfg (JournalMsgStore 'QSPostgres)
 #if defined(dbServerPostgres)
  SSCDatabase :: PostgresStoreCfg -> ServerStoreCfg PostgresMsgStore
 #endif
@@ -331,8 +301,6 @@ data ServerStoreCfg s where
 dbStoreCfg :: ServerStoreCfg s -> Maybe PostgresStoreCfg
 dbStoreCfg = \case
  SSCMemory _ -> Nothing
-  SSCMemoryJournal {} -> Nothing
-  SSCDatabaseJournal {storeCfg} -> Just storeCfg
 #if defined(dbServerPostgres)
  SSCDatabase cfg -> Just cfg
 #endif
@@ -340,8 +308,6 @@ dbStoreCfg = \case
 storeLogFile' :: ServerStoreCfg s -> Maybe FilePath
 storeLogFile' = \case
  SSCMemory sp_ -> (\StorePaths {storeLogFile} -> storeLogFile) <$> sp_
-  SSCMemoryJournal {storeLogFile} -> Just storeLogFile
-  SSCDatabaseJournal {storeCfg = PostgresStoreCfg {dbStoreLogPath}} -> dbStoreLogPath
 #if defined(dbServerPostgres)
  SSCDatabase (PostgresStoreCfg {dbStoreLogPath}) -> dbStoreLogPath
 #endif
@@ -350,14 +316,12 @@ data StorePaths = StorePaths {storeLogFile :: FilePath, storeMsgsFile :: Maybe F

 type family MsgStoreType (qs :: QSType) (ms :: MSType) where
  MsgStoreType 'QSMemory 'MSMemory = STMMsgStore
-  MsgStoreType qs 'MSJournal = JournalMsgStore qs
 #if defined(dbServerPostgres)
  MsgStoreType 'QSPostgres 'MSPostgres = PostgresMsgStore
 #endif

 data MsgStore s where
  StoreMemory :: STMMsgStore -> MsgStore STMMsgStore
-  StoreJournal :: JournalMsgStore qs -> MsgStore (JournalMsgStore qs)
 #if defined(dbServerPostgres)
  StoreDatabase :: PostgresMsgStore -> MsgStore PostgresMsgStore
 #endif
@@ -571,39 +535,21 @@ newProhibitedSub = do
  return Sub {subThread = ProhibitSub, delivered}

 newEnv :: ServerConfig s -> IO (Env s)
-newEnv config@ServerConfig {smpCredentials, httpCredentials, serverStoreCfg, smpAgentCfg, information, messageExpiration, idleQueueInterval, msgQueueQuota, maxJournalMsgCount, maxJournalStateLines, namesConfig} = do
+newEnv config@ServerConfig {smpCredentials, httpCredentials, serverStoreCfg, smpAgentCfg, information, messageExpiration, msgQueueQuota, namesConfig} = do
  serverActive <- newTVarIO True
  server <- newServer
  msgStore_ <- case serverStoreCfg of
    SSCMemory storePaths_ -> do
      let storePath = storeMsgsFile =<< storePaths_
      ms <- newMsgStore STMStoreConfig {storePath, quota = msgQueueQuota}
-      forM_ storePaths_ $ \StorePaths {storeLogFile = f} -> loadStoreLog (mkQueue ms True) f $ queueStore ms
+      forM_ storePaths_ $ \StorePaths {storeLogFile = f} -> loadStoreLog (mkQueue ms) f $ queueStore ms
      pure $ StoreMemory ms
-    SSCMemoryJournal {storeLogFile, storeMsgsPath} -> do
-      logWarn $
-        "Journal message store is deprecated and will be removed soon.\n"
-          <> "Please migrate to in-memory storage using `journal export` command.\n"
-          <> "After that you can migrate to PostgreSQL using `database import` command."
-      let qsCfg = MQStoreCfg
-          cfg = mkJournalStoreConfig qsCfg storeMsgsPath msgQueueQuota maxJournalMsgCount maxJournalStateLines idleQueueInterval
-      ms <- newMsgStore cfg
-      loadStoreLog (mkQueue ms True) storeLogFile $ stmQueueStore ms
-      pure $ StoreJournal ms
 #if defined(dbServerPostgres)
-    SSCDatabaseJournal {storeCfg, storeMsgsPath'} -> do
-      let StartOptions {compactLog, confirmMigrations} = startOptions config
-          qsCfg = PQStoreCfg (storeCfg {confirmMigrations} :: PostgresStoreCfg)
-          cfg = mkJournalStoreConfig qsCfg storeMsgsPath' msgQueueQuota maxJournalMsgCount maxJournalStateLines idleQueueInterval
-      when compactLog $ compactDbStoreLog $ dbStoreLogPath storeCfg
-      StoreJournal <$> newMsgStore cfg
    SSCDatabase storeCfg -> do
      let StartOptions {compactLog, confirmMigrations} = startOptions config
          cfg = PostgresMsgStoreCfg storeCfg {confirmMigrations} msgQueueQuota
      when compactLog $ compactDbStoreLog $ dbStoreLogPath storeCfg
      StoreDatabase <$> newMsgStore cfg
-#else
-    SSCDatabaseJournal {} -> noPostgresExit
 #endif
  ntfStore <- NtfStore <$> TM.emptyIO
  random <- C.newRandom
@@ -655,8 +601,7 @@ newEnv config@ServerConfig {smpCredentials, httpCredentials, serverStoreCfg, smp
      Just f -> do
        logNote $ "compacting queues in file " <> T.pack f
        st <- newMsgStore STMStoreConfig {storePath = Nothing, quota = msgQueueQuota}
-        -- we don't need to have locks in the map
-        sl <- readWriteQueueStore False (mkQueue st False) f (queueStore st)
+        sl <- readWriteQueueStore False (mkQueue st) f (queueStore st)
        setStoreLog (queueStore st) sl
        closeMsgStore st
      Nothing -> do
@@ -703,7 +648,9 @@ newEnv config@ServerConfig {smpCredentials, httpCredentials, serverStoreCfg, smp
            Nothing -> SPMMemoryOnly
            Just StorePaths {storeMsgsFile = Just _} -> SPMMessages
            _ -> SPMQueues
-          _ -> SPMMessages
+#if defined(dbServerPostgres)
+          SSCDatabase _ -> SPMMessages
+#endif

 noPostgresExit :: IO a
 noPostgresExit = putStrLn noPostgresExitStr >> exitFailure
@@ -713,21 +660,6 @@ noPostgresExitStr =
  "Error: server binary is compiled without support for PostgreSQL database.\n"
    <> "Please download `smp-server-postgres` or re-compile with `cabal build -fserver_postgres`."

-mkJournalStoreConfig :: QStoreCfg s -> FilePath -> Int -> Int -> Int -> Int64 -> JournalStoreConfig s
-mkJournalStoreConfig queueStoreCfg storePath msgQueueQuota maxJournalMsgCount maxJournalStateLines idleQueueInterval =
-  JournalStoreConfig
-    { storePath,
-      quota = msgQueueQuota,
-      pathParts = journalMsgStoreDepth,
-      queueStoreCfg,
-      maxMsgCount = maxJournalMsgCount,
-      maxStateLines = maxJournalStateLines,
-      stateTailSize = defaultStateTailSize,
-      idleInterval = idleQueueInterval,
-      expireBackupsAfter = 14 * nominalDay,
-      keepMinBackups = 2
-    }
-
 newSMPProxyAgent :: SMPClientAgentConfig -> TVar ChaChaDRG -> IO ProxyAgent
 newSMPProxyAgent smpAgentCfg random = do
  smpAgent <- newSMPClientAgent SSender smpAgentCfg Nothing random
@@ -28,8 +28,6 @@ module Simplex.Messaging.Server.Main
    importMessagesToDatabase,
    exportDatabaseToStoreLog,
 #endif
-    newJournalMsgStore,
-    storeMsgsJournalDir',
    getServerSourceCode,
    simplexmqSource,
    serverPublicInfo,
@@ -61,7 +59,6 @@ import qualified Data.Text.IO as T
 import Options.Applicative
 import Simplex.Messaging.Agent.Protocol (ConnectionLink (..), ConnectionMode (..), connReqUriP')
 import Simplex.Messaging.Agent.Store.Postgres.Options (DBOpts (..))
-import Simplex.Messaging.Agent.Store.Shared (MigrationConfirmation (..))
 import Simplex.Messaging.Client (HostMode (..), NetworkConfig (..), ProtocolClientConfig (..), SMPWebPortServers (..), SocksMode (..), defaultNetworkConfig, textToHostMode)
 import Simplex.Messaging.Client.Agent (SMPClientAgentConfig (..), defaultSMPClientAgentConfig)
 import qualified Simplex.Messaging.Crypto as C
@@ -69,23 +66,20 @@ import Simplex.Messaging.Encoding.String
 import Simplex.Messaging.Parsers (parseAll)
 import Network.URI (URI (..), URIAuth (..), parseAbsoluteURI)
 import Simplex.Messaging.Protocol (BasicAuth (..), ProtoServerWithAuth (ProtoServerWithAuth), pattern SMPServer)
-import Simplex.Messaging.Server (AttachHTTP, exportMessages, importMessages, printMessageStats, runSMPServer)
+import Simplex.Messaging.Server (AttachHTTP, runSMPServer)
 import Simplex.Messaging.Server.CLI
 import Simplex.Messaging.Server.Env.STM
 import Simplex.Messaging.Server.Expiration
 import Simplex.Messaging.Server.Information
 import Simplex.Messaging.Server.Main.Init
 import Simplex.Messaging.Server.Web (EmbeddedWebParams (..), WebHttpsParams (..))
-import Simplex.Messaging.Server.MsgStore.Journal (JournalMsgStore (..), QStoreCfg (..), stmQueueStore)
-import Simplex.Messaging.Server.MsgStore.Types (MsgStoreClass (..), SQSType (..), SMSType (..), newMsgStore)
+import Simplex.Messaging.Server.MsgStore.Types (SQSType (..), SMSType (..))
 import Simplex.Messaging.Server.Names (NamesConfig (..), RpcAuth (..))
-import Simplex.Messaging.Server.QueueStore.Postgres.Config
-import Simplex.Messaging.Server.StoreLog.ReadWrite (readQueueStore)
 import Simplex.Messaging.Transport (supportedProxyClientSMPRelayVRange, alpnSupportedSMPHandshakes, supportedServerSMPRelayVRange)
 import Simplex.Messaging.Transport.Client (TransportHost (..), defaultSocksProxy)
 import Simplex.Messaging.Transport.HTTP2 (httpALPN)
 import Simplex.Messaging.Transport.Server (ServerCredentials (..), mkTransportServerConfig)
-import Simplex.Messaging.Util (eitherToMaybe, ifM, safeDecodeUtf8)
+import Simplex.Messaging.Util (eitherToMaybe, safeDecodeUtf8, whenM)
 import System.Directory (createDirectoryIfMissing, doesDirectoryExist, doesFileExist)
 import System.Exit (exitFailure)
 import System.FilePath (combine)
@@ -98,15 +92,16 @@ import Data.Int (Int64)
 import qualified Data.Map.Strict as M
 import Data.Semigroup (Sum (..))
 import Simplex.Messaging.Agent.Store.Postgres (checkSchemaExists)
-import Simplex.Messaging.Server.MsgStore.Journal (JournalQueue)
-import Simplex.Messaging.Server.MsgStore.Types (QSType (..))
-import Simplex.Messaging.Server.MsgStore.Journal (postgresQueueStore)
+import Simplex.Messaging.Agent.Store.Shared (MigrationConfirmation (..))
 import Simplex.Messaging.Server.MsgStore.Postgres
-import Simplex.Messaging.Server.QueueStore.Postgres (batchInsertQueues, batchInsertServices, foldQueueRecs, foldServiceRecs)
+import Simplex.Messaging.Server.MsgStore.STM (STMStoreConfig (..))
+import Simplex.Messaging.Server.MsgStore.Types (MsgStoreClass (..), newMsgStore)
+import Simplex.Messaging.Server.QueueStore.Postgres.Config
+import Simplex.Messaging.Server.QueueStore.Postgres (PostgresQueueStore, batchInsertQueues, batchInsertServices, foldQueueRecs, foldServiceRecs)
 import Simplex.Messaging.Server.QueueStore.STM (STMQueueStore (..))
 import Simplex.Messaging.Server.QueueStore.Types
 import Simplex.Messaging.Server.StoreLog (closeStoreLog, logNewService, logCreateQueue, openWriteStoreLog)
-import Simplex.Messaging.Util (unlessM)
+import Simplex.Messaging.Util (ifM)
 import System.Directory (renameFile)
 import System.IO (IOMode (..), withFile)
 #endif
@@ -136,79 +131,6 @@ smpServerCLI_ generateSite serveStaticFiles attachStaticFiles cfgPath logPath =
      deleteDirIfExists cfgPath
      deleteDirIfExists logPath
      putStrLn "Deleted configuration and log files"
-    Journal cmd -> withIniFile $ \ini -> do
-      msgsDirExists <- doesDirectoryExist storeMsgsJournalDir
-      msgsFileExists <- doesFileExist storeMsgsFilePath
-      storeLogFile <- getRequiredStoreLogFile ini
-      case cmd of
-        SCImport
-          | msgsFileExists && msgsDirExists -> exitConfigureMsgStorage
-          | msgsDirExists -> do
-              putStrLn $ storeMsgsJournalDir <> " directory already exists."
-              exitFailure
-          | not msgsFileExists -> do
-              putStrLn $ storeMsgsFilePath <> " file does not exist."
-              exitFailure
-          | otherwise -> do
-              confirmOrExit
-                ("WARNING: message log file " <> storeMsgsFilePath <> " will be imported to journal directory " <> storeMsgsJournalDir)
-                "Messages not imported"
-              ms <- newJournalMsgStore logPath MQStoreCfg
-              readQueueStore True (mkQueue ms False) storeLogFile $ stmQueueStore ms
-              msgStats <- importMessages True ms storeMsgsFilePath Nothing False -- no expiration
-              putStrLn "Import completed"
-              printMessageStats "Messages" msgStats
-              putStrLn $ case readStoreType ini of
-                Right (ASType SQSMemory SMSMemory) -> "store_messages set to `memory`, update it to `journal` in INI file"
-                Right (ASType SQSPostgres SMSPostgres) -> "store_messages set to `database`, update it to `journal` in INI file"
-                Right (ASType _ SMSJournal) -> "store_messages set to `journal`"
-                Left e -> e <> ", configure storage correctly"
-        SCExport
-          | msgsFileExists && msgsDirExists -> exitConfigureMsgStorage
-          | msgsFileExists -> do
-              putStrLn $ storeMsgsFilePath <> " file already exists."
-              exitFailure
-          | otherwise -> do
-              confirmOrExit
-                ("WARNING: journal directory " <> storeMsgsJournalDir <> " will be exported to message log file " <> storeMsgsFilePath)
-                "Journal not exported"
-              case readStoreType ini of
-                Right (ASType SQSMemory msType) -> do
-                  ms <- newJournalMsgStore logPath MQStoreCfg
-                  readQueueStore True (mkQueue ms False) storeLogFile $ stmQueueStore ms
-                  exportMessages True (StoreJournal ms) storeMsgsFilePath False
-                  putStrLn "Export completed"
-                  putStrLn $ case msType of
-                    SMSMemory -> "store_messages set to `memory`, start the server."
-                    SMSJournal -> "store_messages set to `journal`, update it to `memory` in INI file"
-#if defined(dbServerPostgres)
-                Right (ASType SQSPostgres SMSJournal) -> do
-                  let dbStoreLogPath = enableDbStoreLog' ini $> storeLogFilePath
-                      dbOpts@DBOpts {connstr, schema} = iniDBOptions ini defaultDBOpts
-                  unlessM (checkSchemaExists connstr schema) $ do
-                    putStrLn $ "Schema " <> B.unpack schema <> " does not exist in PostrgreSQL database: " <> B.unpack connstr
-                    exitFailure
-                  ms <- newJournalMsgStore logPath $ PQStoreCfg PostgresStoreCfg {dbOpts, dbStoreLogPath, confirmMigrations = MCYesUp, deletedTTL = iniDeletedTTL ini}
-                  exportMessages True (StoreJournal ms) storeMsgsFilePath False
-                  putStrLn "Export completed"
-                  putStrLn "store_messages set to `journal`, store_queues is set to `database`.\nExport queues to store log to use memory storage for messages (`smp-server database export`)."
-                Right (ASType SQSPostgres SMSPostgres) -> do
-                  putStrLn $ "Messages can be exported with `dabatase export --table messages`."
-                  exitFailure
-#else
-                Right (ASType SQSPostgres SMSJournal) -> noPostgresExit
-#endif
-                Left e -> putStrLn $ e <> ", configure storage correctly"
-        SCDelete
-          | not msgsDirExists -> do
-              putStrLn $ storeMsgsJournalDir <> " directory does not exists."
-              exitFailure
-          | otherwise -> do
-              confirmOrExit
-                ("WARNING: journal directory " <> storeMsgsJournalDir <> " will be permanently deleted.\nTHIS CANNOT BE UNDONE!")
-                "Messages NOT deleted"
-              deleteDirIfExists storeMsgsJournalDir
-              putStrLn $ "Deleted all messages in journal " <> storeMsgsJournalDir
 #if defined(dbServerPostgres)
    Database cmd tables dbOpts@DBOpts {connstr, schema} -> withIniFile $ \ini -> do
      schemaExists <- checkSchemaExists connstr schema
@@ -221,7 +143,7 @@ smpServerCLI_ generateSite serveStaticFiles attachStaticFiles cfgPath logPath =
              confirmOrExit
                ("WARNING: store log file " <> storeLogFile <> " and message log file " <> storeMsgsFilePath <> " will be imported to PostrgreSQL database: " <> B.unpack connstr <> ", schema: " <> B.unpack schema)
                "Store logs not imported"
-              (sCnt, qCnt) <- importStoreLogToDatabase logPath storeLogFile dbOpts
+              (sCnt, qCnt) <- importStoreLogToDatabase storeLogFile dbOpts
              putStrLn $ "Imported: " <> show sCnt <> " services, " <> show qCnt <> " queues"
              putStrLn "Importing messages..."
              mCnt <- importMessagesToDatabase storeMsgsFilePath dbOpts
@@ -248,11 +170,10 @@ smpServerCLI_ generateSite serveStaticFiles attachStaticFiles cfgPath logPath =
              confirmOrExit
                ("WARNING: store log file " <> storeLogFile <> " will be compacted and imported to PostrgreSQL database: " <> B.unpack connstr <> ", schema: " <> B.unpack schema)
                "Queue records not imported"
-              (sCnt, qCnt) <- importStoreLogToDatabase logPath storeLogFile dbOpts
+              (sCnt, qCnt) <- importStoreLogToDatabase storeLogFile dbOpts
              putStrLn $ "Import completed: " <> show sCnt <> " services, " <> show qCnt <> " queues"
              putStrLn $ case readStoreType ini of
-                Right (ASType SQSMemory SMSMemory) -> setToDbStr <> "\nstore_messages set to `memory`, import messages to journal to use PostgreSQL database for queues (`smp-server journal import`)"
-                Right (ASType SQSMemory SMSJournal) -> setToDbStr
+                Right (ASType SQSMemory SMSMemory) -> setToDbStr <> "\nstore_messages set to `memory`, update it to `database` and import messages (`smp-server database import --table messages`)"
                Right (ASType SQSPostgres _) -> "store_queues set to `database`, start the server."
                Left e -> e <> ", configure storage correctly"
          where
@@ -280,7 +201,7 @@ smpServerCLI_ generateSite serveStaticFiles attachStaticFiles cfgPath logPath =
              confirmOrExit
                ("WARNING: PostrgreSQL schema " <> B.unpack schema <> " (database: " <> B.unpack connstr <> ") will be exported to store log file " <> storeLogFilePath <> " and to message log file " <> storeMsgsFilePath)
                "Database store not exported"
-              (sCnt, qCnt) <- exportDatabaseToStoreLog logPath dbOpts storeLogFilePath
+              (sCnt, qCnt) <- exportDatabaseToStoreLog dbOpts storeLogFilePath
              putStrLn $ "Exported: " <> show sCnt <> " services, " <> show qCnt <> " queues"
              putStrLn "Exporting messages..."
              let storeCfg = PostgresStoreCfg {dbOpts, dbStoreLogPath = Nothing, confirmMigrations = MCConsole, deletedTTL = 86400 * defaultDeletedTTL}
@@ -306,7 +227,7 @@ smpServerCLI_ generateSite serveStaticFiles attachStaticFiles cfgPath logPath =
              confirmOrExit
                ("WARNING: PostrgreSQL schema " <> B.unpack schema <> " (database: " <> B.unpack connstr <> ") will be exported to store log file " <> storeLogFilePath)
                "Queue records not exported"
-              (sCnt, qCnt) <- exportDatabaseToStoreLog logPath dbOpts storeLogFilePath
+              (sCnt, qCnt) <- exportDatabaseToStoreLog dbOpts storeLogFilePath
              putStrLn $ "Export completed: " <> show sCnt <> " services, " <> show qCnt <> " queues"
              putStrLn $ case readStoreType ini of
                Right (ASType SQSPostgres _) -> "store_queues or store_messages set to `database`, update it to `memory` in INI file."
@@ -346,6 +267,7 @@ smpServerCLI_ generateSite serveStaticFiles attachStaticFiles cfgPath logPath =
      doesFileExist iniFile >>= \case
        True -> readIniFile iniFile >>= either exitError a
        _ -> exitError $ "Error: server is not initialized (" <> iniFile <> " does not exist).\nRun `" <> executableName <> " init`."
+#if defined(dbServerPostgres)
    getRequiredStoreLogFile ini = do
      case enableStoreLog' ini $> storeLogFilePath of
        Just storeLogFile -> do
@@ -354,20 +276,21 @@ smpServerCLI_ generateSite serveStaticFiles attachStaticFiles cfgPath logPath =
            (pure storeLogFile)
            (putStrLn ("Store log file " <> storeLogFile <> " not found") >> exitFailure)
        Nothing -> putStrLn "Store log disabled, see `[STORE_LOG] enable`" >> exitFailure
+#endif
    iniFile = combine cfgPath "smp-server.ini"
    serverVersion = "SMP server v" <> simplexmqVersionCommit
    executableName = "smp-server"
    storeLogFilePath = combine logPath "smp-server-store.log"
    storeMsgsFilePath = combine logPath "smp-server-messages.log"
-    storeMsgsJournalDir = storeMsgsJournalDir' logPath
+    storeMsgsJournalDir = combine logPath "messages"
    storeNtfsFilePath = combine logPath "smp-server-ntfs.log"
    readStoreType :: Ini -> Either String AStoreType
    readStoreType ini = case (iniStoreQueues, iniStoreMessage) of
      ("memory", "memory") -> Right $ ASType SQSMemory SMSMemory
-      ("memory", "journal") -> Right $ ASType SQSMemory SMSJournal
+      ("memory", "journal") -> Left journalRemovedStr
      ("memory", "database") -> Left "Database and memory storage are not compatible."
      ("database", "memory") -> Left "Database and memory storage are not compatible."
-      ("database", "journal") -> Right $ ASType SQSPostgres SMSJournal
+      ("database", "journal") -> Left journalRemovedStr
 #if defined(dbServerPostgres)
      ("database", "database") -> Right $ ASType SQSPostgres SMSPostgres
 #else
@@ -377,10 +300,12 @@ smpServerCLI_ generateSite serveStaticFiles attachStaticFiles cfgPath logPath =
      where
        iniStoreQueues = fromRight "memory" $ lookupValue "STORE_LOG" "store_queues" ini
        iniStoreMessage = fromRight "memory" $ lookupValue "STORE_LOG" "store_messages" ini
+#if defined(dbServerPostgres)
    iniDeletedTTL ini = readIniDefault (86400 * defaultDeletedTTL) "STORE_LOG" "db_deleted_ttl" ini
+    enableDbStoreLog' = settingIsOn "STORE_LOG" "db_store_log"
+#endif
    defaultStaticPath = combine logPath "www"
    enableStoreLog' = settingIsOn "STORE_LOG" "enable"
-    enableDbStoreLog' = settingIsOn "STORE_LOG" "db_store_log"
    initializeServer opts
      | scripted opts = initialize opts
      | otherwise = do
@@ -517,16 +442,13 @@ smpServerCLI_ generateSite serveStaticFiles attachStaticFiles cfgPath logPath =
        iniStoreType = either error id $! readStoreType ini
        iniStoreCfg :: SupportedStore qs ms => SQSType qs -> SMSType ms -> ServerStoreCfg (MsgStoreType qs ms)
        iniStoreCfg SQSMemory SMSMemory = SSCMemory $ enableStoreLog' ini $> StorePaths {storeLogFile = storeLogFilePath, storeMsgsFile = restoreMessagesFile storeMsgsFilePath}
-        iniStoreCfg SQSMemory SMSJournal = SSCMemoryJournal {storeLogFile = storeLogFilePath, storeMsgsPath = storeMsgsJournalDir}
-        iniStoreCfg SQSPostgres SMSJournal =
-          let dbStoreLogPath = enableDbStoreLog' ini $> storeLogFilePath
-              storeCfg = PostgresStoreCfg {dbOpts = iniDBOptions ini defaultDBOpts, dbStoreLogPath, confirmMigrations = MCYesUp, deletedTTL = iniDeletedTTL ini}
-           in SSCDatabaseJournal {storeCfg, storeMsgsPath' = storeMsgsJournalDir}
 #if defined(dbServerPostgres)
        iniStoreCfg SQSPostgres SMSPostgres =
          let dbStoreLogPath = enableDbStoreLog' ini $> storeLogFilePath
              storeCfg = PostgresStoreCfg {dbOpts = iniDBOptions ini defaultDBOpts, dbStoreLogPath, confirmMigrations = MCYesUp, deletedTTL = iniDeletedTTL ini}
           in SSCDatabase storeCfg
+#else
+        iniStoreCfg SQSPostgres _ = error noPostgresExitStr
 #endif
        serverConfig :: ServerStoreCfg s -> ServerConfig s
        serverConfig serverStoreCfg =
@@ -535,8 +457,6 @@ smpServerCLI_ generateSite serveStaticFiles attachStaticFiles cfgPath logPath =
              smpHandshakeTimeout = 120000000,
              tbqSize = 128,
              msgQueueQuota = defaultMsgQueueQuota,
-              maxJournalMsgCount = defaultMaxJournalMsgCount,
-              maxJournalStateLines = defaultMaxJournalStateLines,
              queueIdBytes = 24,
              msgIdBytes = 24, -- must be at least 24 bytes, it is used as 192-bit nonce for XSalsa20
              smpCredentials =
@@ -561,7 +481,6 @@ smpServerCLI_ generateSite serveStaticFiles attachStaticFiles cfgPath logPath =
                    },
              expireMessagesOnStart = fromMaybe True $ iniOnOff "STORE_LOG" "expire_messages_on_start" ini,
              expireMessagesOnSend = fromMaybe True $ iniOnOff "STORE_LOG" "expire_messages_on_send" ini,
-              idleQueueInterval = defaultIdleQueueInterval,
              notificationExpiration =
                defaultNtfExpiration
                  { ttl = 3600 * readIniDefault defNtfExpirationHours "STORE_LOG" "expire_ntfs_hours" ini
@@ -634,47 +553,26 @@ smpServerCLI_ generateSite serveStaticFiles attachStaticFiles cfgPath logPath =
        webStaticPath' = eitherToMaybe $ T.unpack <$> lookupValue "WEB" "static_path" ini

    checkMsgStoreMode :: Ini -> AStoreType -> IO ()
+#if defined(dbServerPostgres)
    checkMsgStoreMode ini mode = do
-      msgsDirExists <- doesDirectoryExist storeMsgsJournalDir
-      msgsFileExists <- doesFileExist storeMsgsFilePath
-      storeLogExists <- doesFileExist storeLogFilePath
+#else
+    checkMsgStoreMode _ mode = do
+#endif
+      whenM (doesDirectoryExist storeMsgsJournalDir) $ do
+        putStrLn $ "Error: " <> storeMsgsJournalDir <> " directory is present.\n" <> journalRemovedStr
+        exitFailure
      case mode of
 #if defined(dbServerPostgres)
-        ASType SQSPostgres SMSPostgres
-          | msgsFileExists || msgsDirExists -> do
-              putStrLn $ "Error: " <> storeMsgsFilePath <> " file or " <> storeMsgsJournalDir <> " directory are present."
-              putStrLn "Configure memory storage."
-              exitFailure
-          | otherwise -> checkDbStorage ini storeLogExists
-#endif
-        ASType qs SMSJournal
-          | msgsFileExists && msgsDirExists -> exitConfigureMsgStorage
-          | msgsFileExists -> do
-              putStrLn $ "Error: store_messages is `journal` with " <> storeMsgsFilePath <> " file present."
-              putStrLn "Set store_messages to `memory` or use `smp-server journal export` to migrate."
-              exitFailure
-          | not msgsDirExists ->
-              putStrLn $ "store_messages is `journal`, " <> storeMsgsJournalDir <> " directory will be created."
-          | otherwise -> case qs of
-              SQSMemory ->
-                unless (storeLogExists) $ putStrLn $ "store_queues is `memory`, " <> storeLogFilePath <> " file will be created."
-#if defined(dbServerPostgres)
-              SQSPostgres -> checkDbStorage ini storeLogExists
+        ASType SQSPostgres SMSPostgres -> do
+          whenM (doesFileExist storeMsgsFilePath) $ do
+            putStrLn $ "Error: " <> storeMsgsFilePath <> " file is present."
+            putStrLn "Configure memory storage."
+            exitFailure
+          checkDbStorage ini =<< doesFileExist storeLogFilePath
 #else
-              SQSPostgres -> noPostgresExit
+        ASType SQSPostgres _ -> noPostgresExit
 #endif
-        ASType SQSMemory SMSMemory
-          | msgsFileExists && msgsDirExists -> exitConfigureMsgStorage
-          | msgsDirExists -> do
-              putStrLn $ "Error: store_messages is `memory` with " <> storeMsgsJournalDir <> " directory present."
-              putStrLn "Set store_messages to `journal` or use `smp-server journal import` to migrate."
-              exitFailure
-          | otherwise -> pure ()
-
-    exitConfigureMsgStorage = do
-      putStrLn $ "Error: both " <> storeMsgsFilePath <> " file and " <> storeMsgsJournalDir <> " directory are present."
-      putStrLn "Configure memory storage."
-      exitFailure
+        ASType SQSMemory SMSMemory -> pure ()

 #if defined(dbServerPostgres)
    checkDbStorage ini storeLogExists = do
@@ -705,18 +603,18 @@ smpServerCLI_ generateSite serveStaticFiles attachStaticFiles cfgPath logPath =
      putStrLn "Configure queue storage."
      exitFailure

-importStoreLogToDatabase :: FilePath -> FilePath -> DBOpts -> IO (Int64, Int64)
-importStoreLogToDatabase logPath storeLogFile dbOpts = do
-  ms <- newJournalMsgStore logPath MQStoreCfg
-  let st = stmQueueStore ms
-  sl <- readWriteQueueStore True (mkQueue ms False) storeLogFile st
+importStoreLogToDatabase :: FilePath -> DBOpts -> IO (Int64, Int64)
+importStoreLogToDatabase storeLogFile dbOpts = do
+  ms <- newMsgStore STMStoreConfig {storePath = Nothing, quota = defaultMsgQueueQuota}
+  let st = queueStore ms
+  sl <- readWriteQueueStore True (mkQueue ms) storeLogFile st
  closeStoreLog sl
-  queues <- readTVarIO $ loadedQueues st
+  qs <- readTVarIO $ queues st
  services' <- M.elems <$> readTVarIO (services st)
  let storeCfg = PostgresStoreCfg {dbOpts = dbOpts {createSchema = True}, dbStoreLogPath = Nothing, confirmMigrations = MCConsole, deletedTTL = 86400 * defaultDeletedTTL}
-  ps <- newJournalMsgStore logPath $ PQStoreCfg storeCfg
-  sCnt <- batchInsertServices services' $ postgresQueueStore ps
-  qCnt <- batchInsertQueues @(JournalQueue 'QSMemory) True queues $ postgresQueueStore ps
+  ps :: PostgresQueueStore PostgresQueue <- newQueueStore @PostgresQueue storeCfg
+  sCnt <- batchInsertServices services' ps
+  qCnt <- batchInsertQueues True qs ps
  renameFile storeLogFile $ storeLogFile <> ".bak"
  pure (sCnt, qCnt)

@@ -735,24 +633,22 @@ importMessagesToDatabase msgsLogFile dbOpts = do
  renameFile msgsLogFile $ msgsLogFile <> ".bak"
  pure mCnt'

-exportDatabaseToStoreLog :: FilePath -> DBOpts -> FilePath -> IO (Int, Int)
-exportDatabaseToStoreLog logPath dbOpts storeLogFilePath = do
+exportDatabaseToStoreLog :: DBOpts -> FilePath -> IO (Int, Int)
+exportDatabaseToStoreLog dbOpts storeLogFilePath = do
  let storeCfg = PostgresStoreCfg {dbOpts, dbStoreLogPath = Nothing, confirmMigrations = MCConsole, deletedTTL = 86400 * defaultDeletedTTL}
-  ps <- newJournalMsgStore logPath $ PQStoreCfg storeCfg
+  ps :: PostgresQueueStore PostgresQueue <- newQueueStore @PostgresQueue storeCfg
  sl <- openWriteStoreLog False storeLogFilePath
-  Sum sCnt <- foldServiceRecs (postgresQueueStore ps) $ \sr -> logNewService sl sr $> Sum (1 :: Int)
-  Sum qCnt <- foldQueueRecs True True (postgresQueueStore ps) $ \(rId, qr) -> logCreateQueue sl rId qr $> Sum (1 :: Int)
+  Sum sCnt <- foldServiceRecs ps $ \sr -> logNewService sl sr $> Sum (1 :: Int)
+  Sum qCnt <- foldQueueRecs True True ps $ \(rId, qr) -> logCreateQueue sl rId qr $> Sum (1 :: Int)
  closeStoreLog sl
  pure (sCnt, qCnt)
 #endif

-newJournalMsgStore :: FilePath -> QStoreCfg s -> IO (JournalMsgStore s)
-newJournalMsgStore logPath qsCfg =
-  let cfg = mkJournalStoreConfig qsCfg (storeMsgsJournalDir' logPath) defaultMsgQueueQuota defaultMaxJournalMsgCount defaultMaxJournalStateLines $ checkInterval defaultMessageExpiration
-   in newMsgStore cfg
-
-storeMsgsJournalDir' :: FilePath -> FilePath
-storeMsgsJournalDir' logPath = combine logPath "messages"
+journalRemovedStr :: String
+journalRemovedStr =
+  "Journal message storage is removed.\n"
+    <> "Export messages with `smp-server journal export` using smp-server v7.1.0.10 or earlier,\n"
+    <> "then set store_messages to `memory`, or to `database` and run `smp-server database import --table messages`."

 getServerSourceCode :: IO (Maybe String)
 getServerSourceCode =
@@ -873,7 +769,6 @@ data CliCommand
  | OnlineCert CertOptions
  | Start StartOptions
  | Delete
-  | Journal StoreCmd
  | Database StoreCmd DatabaseTable DBOpts

 data StoreCmd = SCImport | SCExport | SCDelete
@@ -899,7 +794,6 @@ cliCommandP cfgPath logPath iniFile =
        <> command "cert" (info (OnlineCert <$> certOptionsP) (progDesc $ "Generate new online TLS server credentials (configuration: " <> iniFile <> ")"))
        <> command "start" (info (Start <$> startOptionsP) (progDesc $ "Start server (configuration: " <> iniFile <> ")"))
        <> command "delete" (info (pure Delete) (progDesc "Delete configuration and log files"))
-        <> command "journal" (info (Journal <$> journalCmdP) (progDesc "Import/export messages to/from journal storage"))
        <> command "database" (info (Database <$> databaseCmdP <*> dbTableP <*> dbOptsP defaultDBOpts) (progDesc "Import/export queues to/from PostgreSQL database storage"))
    )
  where
@@ -1046,7 +940,6 @@ cliCommandP cfgPath logPath iniFile =
            disableWeb,
            scripted
          }
-    journalCmdP = storeCmdP "message log file" "journal storage"
    databaseCmdP = storeCmdP "queue store log file" "PostgreSQL database schema"
    storeCmdP src dest =
      hsubparser
@@ -83,7 +83,7 @@ iniFileContent cfgPath logPath opts host basicAuth controlPortPwds =
    <> ("enable = " <> onOff enableStoreLog <> "\n\n")
    <> "# Queue storage mode: `memory` or `database` (to store queue records in PostgreSQL database).\n\
        \# `memory` - in-memory persistence, with optional append-only log (`enable = on`).\n\
-        \# `database`- PostgreSQL databass (requires `store_messages = journal`).\n\
+        \# `database`- PostgreSQL databass (requires `store_messages = database`).\n\
        \store_queues = memory\n\n\
        \# Database connection settings for PostgreSQL database (`store_queues = database`).\n"
    <> iniDbOpts dbOptions defaultDBOpts
@@ -91,7 +91,7 @@ iniFileContent cfgPath logPath opts host basicAuth controlPortPwds =
        \# db_store_log = off\n\n\
        \# Time to retain deleted queues in the database, days.\n"
    <> ("# db_deleted_ttl = " <> tshow defaultDeletedTTL <> "\n\n")
-    <> "# Message storage mode: `memory` or `journal`.\n\
+    <> "# Message storage mode: `memory` or `database` (requires `store_queues = database`).\n\
        \store_messages = memory\n\n\
        \# When store_messages is `memory`, undelivered messages are optionally saved and restored\n\
        \# when the server restarts, they are preserved in the .bak file until the next restart.\n"
@@ -1,44 +0,0 @@
-module Simplex.Messaging.Server.MsgStore.Journal.SharedLock
-  ( withLockWaitShared,
-    withLockMapWaitShared,
-    withSharedWaitLock,
-  )
-where
-
-import Control.Concurrent.STM
-import qualified Control.Exception as E
-import Control.Monad
-import Data.Text (Text)
-import Simplex.Messaging.Agent.Lock
-import Simplex.Messaging.Agent.Client (getMapLock)
-import Simplex.Messaging.Protocol (RecipientId)
-import Simplex.Messaging.TMap (TMap)
-import qualified Simplex.Messaging.TMap as TM
-import Simplex.Messaging.Util (($>>), ($>>=))
-
-- wait until shared lock with passed ID is released and take lock
-withLockWaitShared :: RecipientId -> Lock -> TMVar RecipientId -> Text -> IO a -> IO a
-withLockWaitShared rId lock shared name =
-  E.bracket_
-    (atomically $ waitShared rId shared >> putTMVar lock name)
-    (void $ atomically $ takeTMVar lock)
-
-- wait until shared lock with passed ID is released and take lock from Map for this ID
-withLockMapWaitShared :: RecipientId -> TMap RecipientId Lock -> TMVar RecipientId -> Text -> IO a -> IO a
-withLockMapWaitShared rId locks shared name a =
-  E.bracket
-    (atomically $ waitShared rId shared >> getPutLock (getMapLock locks) rId name)
-    (atomically . takeTMVar)
-    (const a)
-
-waitShared :: RecipientId -> TMVar RecipientId -> STM ()
-waitShared rId shared = tryReadTMVar shared >>= mapM_ (\rId' -> when (rId == rId') retry)
-
-- wait until lock with passed ID in Map is released and take shared lock for this ID
-withSharedWaitLock :: RecipientId -> TMap RecipientId Lock -> TMVar RecipientId -> IO a -> IO a
-withSharedWaitLock rId locks shared =
-  E.bracket_
-    (atomically $ waitLock >> putTMVar shared rId)
-    (atomically $ takeTMVar shared)
-  where
-    waitLock = TM.lookup rId locks $>>= tryReadTMVar $>> retry
@@ -29,7 +29,6 @@ where
 import Control.Concurrent.STM
 import qualified Control.Exception as E
 import Control.Monad
-import Control.Monad.Reader
 import Control.Monad.Trans.Except
 import qualified Data.ByteString as B
 import qualified Data.ByteString.Builder as BB
@@ -39,7 +38,6 @@ import Data.IORef
 import Data.Int (Int64)
 import Data.List (intersperse)
 import qualified Data.Map.Strict as M
-import Data.Text (Text)
 import Data.Time.Clock.System (SystemTime (..))
 import Database.PostgreSQL.Simple (Binary (..), In (..), Only (..), (:.) (..))
 import qualified Database.PostgreSQL.Simple as DB
@@ -56,7 +54,7 @@ import Simplex.Messaging.Server.QueueStore.Postgres
 import Simplex.Messaging.Server.QueueStore.Types
 import Simplex.Messaging.Server.StoreLog (foldLogLines)
 import Simplex.Messaging.Encoding.String
-import Simplex.Messaging.Util (maybeFirstRow, maybeFirstRow', (<$$>))
+import Simplex.Messaging.Util (maybeFirstRow, maybeFirstRow')
 import System.IO (Handle, hFlush, stdout)

 data PostgresMsgStore = PostgresMsgStore
@@ -81,32 +79,20 @@ instance StoreQueueClass PostgresQueue where
  {-# INLINE recipientId #-}
  queueRec = queueRec'
  {-# INLINE queueRec #-}
-  withQueueLock PostgresQueue {} _ = id -- TODO [messages] maybe it's just transaction?
-  {-# INLINE withQueueLock #-}
-
-newtype DBTransaction = DBTransaction {dbConn :: DB.Connection}
-
-type DBStoreIO a = ReaderT DBTransaction IO a

 instance MsgStoreClass PostgresMsgStore where
-  type StoreMonad PostgresMsgStore = ReaderT DBTransaction IO
-  type MsgQueue PostgresMsgStore = ()
  type QueueStore PostgresMsgStore = PostgresQueueStore'
  type StoreQueue PostgresMsgStore = PostgresQueue
  type MsgStoreConfig PostgresMsgStore = PostgresMsgStoreCfg

  newMsgStore :: PostgresMsgStoreCfg -> IO PostgresMsgStore
  newMsgStore config = do
-    queueStore_ <- newQueueStore @PostgresQueue (queueStoreCfg config, False)
+    queueStore_ <- newQueueStore @PostgresQueue (queueStoreCfg config)
    pure PostgresMsgStore {config, queueStore_}

  closeMsgStore :: PostgresMsgStore -> IO ()
  closeMsgStore = closeQueueStore @PostgresQueue . queueStore_

-  withActiveMsgQueues _ _ = error "withActiveMsgQueues not used"
-
-  unsafeWithAllMsgQueues _ _ _ = error "unsafeWithAllMsgQueues not used"
-
  expireOldMessages :: Bool -> PostgresMsgStore -> Int64 -> Int64 -> IO MessageStats
  expireOldMessages _tty ms now ttl =
    maybeFirstRow' newMessageStats toMessageStats $ withConnection st $ \db ->
@@ -114,7 +100,9 @@ instance MsgStoreClass PostgresMsgStore where
    where
      st = dbStore $ queueStore_ ms
      oldMsg = now - ttl
-      batchSize = 10000 :: Int
+      -- expired messages read per page in expire_old_messages, and the page is one
+      -- transaction: queues in it stay row-locked against SEND and ACK until it commits.
+      batchSize = 100 :: Int
      toMessageStats (expiredMsgsCount, storedMsgsCount, storedQueues) =
        MessageStats {expiredMsgsCount, storedMsgsCount, storedQueues}

@@ -148,35 +136,13 @@ instance MsgStoreClass PostgresMsgStore where
            msg_ = toMaybeMessage mRow
         in f a rId $ Right ((qr,) <$> msg_)

-  logQueueStates _ = error "logQueueStates not used"
-
-  logQueueState _ = error "logQueueState not used"
-
  queueStore = queueStore_
  {-# INLINE queueStore #-}

-  loadedQueueCounts :: PostgresMsgStore -> IO LoadedQueueCounts
-  loadedQueueCounts ms = do
-    loadedQueueCount <- M.size <$> readTVarIO queues
-    loadedNotifierCount <- M.size <$> readTVarIO notifiers
-    notifierLockCount <- M.size <$> readTVarIO notifierLocks
-    pure LoadedQueueCounts {loadedQueueCount, loadedNotifierCount, openJournalCount = 0, queueLockCount = 0, notifierLockCount}
-    where
-      PostgresQueueStore {queues, notifiers, notifierLocks} = queueStore_ ms
-
-  mkQueue :: PostgresMsgStore -> Bool -> RecipientId -> QueueRec -> IO PostgresQueue
-  mkQueue _ _keepLock rId qr = PostgresQueue rId <$> newTVarIO (Just qr)
+  mkQueue :: PostgresMsgStore -> RecipientId -> QueueRec -> IO PostgresQueue
+  mkQueue _ rId qr = PostgresQueue rId <$> newTVarIO (Just qr)
  {-# INLINE mkQueue #-}

-  getMsgQueue _ _ _ = pure ()
-  {-# INLINE getMsgQueue #-}
-
-  getPeekMsgQueue :: PostgresMsgStore -> PostgresQueue -> DBStoreIO (Maybe ((), Message))
-  getPeekMsgQueue _ q = ((),) <$$> tryPeekMsg_ q ()
-
-  withIdleMsgQueue :: Int64 -> PostgresMsgStore -> PostgresQueue -> (() -> DBStoreIO a) -> DBStoreIO (Maybe a, Int)
-  withIdleMsgQueue _ _ _ _ = error "withIdleMsgQueue not used"
-
  deleteQueue :: PostgresMsgStore -> PostgresQueue -> IO (Either ErrorType QueueRec)
  deleteQueue ms q = deleteStoreQueue (queueStore_ ms) q
  {-# INLINE deleteQueue #-}
@@ -187,8 +153,6 @@ instance MsgStoreClass PostgresMsgStore where
    qr <- ExceptT $ deleteStoreQueue (queueStore_ ms) q
    pure (qr, size)

-  getQueueMessages_ _ _ _ = error "getQueueMessages_ not used"
-
  writeMsg :: PostgresMsgStore -> PostgresQueue -> Bool -> Message -> ExceptT ErrorType IO (Maybe (Message, Bool))
  writeMsg ms q _ msg =
    uninterruptibleMask_ $
@@ -207,43 +171,26 @@ instance MsgStoreClass PostgresMsgStore where
        [] -> Nothing
      PostgresMsgStore {config = PostgresMsgStoreCfg {quota}} = ms

-  setOverQuota_ :: PostgresQueue -> IO () -- can ONLY be used while restoring messages, not while server running
-  setOverQuota_ _ = error "TODO setOverQuota_" -- TODO [messages]
-
-  getQueueSize_ :: () -> DBStoreIO Int
-  getQueueSize_ _ = error "getQueueSize_ not used"
-
  getQueueSize :: PostgresMsgStore -> PostgresQueue -> ExceptT ErrorType IO Int
  getQueueSize ms q =
    withDB' "getQueueSize" (queueStore_ ms) $ \db ->
      maybeFirstRow' 0 fromOnly $
        DB.query db "SELECT msg_queue_size FROM msg_queues WHERE recipient_id = ? AND deleted_at IS NULL" (Only (recipientId' q))

-  tryPeekMsg_ :: PostgresQueue -> () -> DBStoreIO (Maybe Message)
-  tryPeekMsg_ q _ = do
-    db <- asks dbConn
-    liftIO $ maybeFirstRow toMessage $
-      DB.query
-        db
-        [sql|
-          SELECT msg_id, msg_ts, msg_quota, msg_ntf_flag, msg_body
-          FROM messages
-          WHERE recipient_id = ?
-          ORDER BY message_id ASC LIMIT 1
-        |]
-        (Only (recipientId' q))
-
-  tryDeleteMsg_ :: PostgresQueue -> () -> Bool -> DBStoreIO ()
-  tryDeleteMsg_ _q _ _ = error "tryDeleteMsg_ not used" -- do
-
-  isolateQueue :: PostgresMsgStore -> PostgresQueue -> Text -> DBStoreIO a -> ExceptT ErrorType IO a
-  isolateQueue ms _q op a = uninterruptibleMask_ $ withDB' op (queueStore_ ms) $ runReaderT a . DBTransaction
-
-  unsafeRunStore _ _ _ = error "unsafeRunStore not used"
-
  tryPeekMsg :: PostgresMsgStore -> PostgresQueue -> ExceptT ErrorType IO (Maybe Message)
-  tryPeekMsg ms q = isolateQueue ms q "tryPeekMsg" $ tryPeekMsg_ q ()
-  {-# INLINE tryPeekMsg #-}
+  tryPeekMsg ms q =
+    uninterruptibleMask_ $
+      withDB' "tryPeekMsg" (queueStore_ ms) $ \db ->
+        maybeFirstRow toMessage $
+          DB.query
+            db
+            [sql|
+              SELECT msg_id, msg_ts, msg_quota, msg_ntf_flag, msg_body
+              FROM messages
+              WHERE recipient_id = ?
+              ORDER BY message_id ASC LIMIT 1
+            |]
+            (Only (recipientId' q))

  tryPeekMsgs :: PostgresMsgStore -> [PostgresQueue] -> ExceptT ErrorType IO (M.Map RecipientId Message)
  tryPeekMsgs _ms [] = pure M.empty
@@ -363,8 +310,8 @@ deleteAllMessages ms =
      db
      [sql|
        UPDATE msg_queues
-        SET msg_queue_size = 0, msg_can_write = TRUE, msg_queue_expire = FALSE
-        WHERE msg_queue_size != 0 OR msg_can_write = FALSE OR msg_queue_expire = TRUE
+        SET msg_queue_size = 0, msg_can_write = TRUE
+        WHERE msg_queue_size != 0 OR msg_can_write = FALSE
      |]

 updateQueueCounts :: PostgresMsgStore -> IO ()
@@ -384,16 +331,15 @@ updateQueueCounts ms =
      db
      [sql|
        UPDATE msg_queues
-        SET msg_queue_size = 0, msg_can_write = TRUE, msg_queue_expire = FALSE
-        WHERE msg_queue_size != 0 OR msg_can_write = FALSE OR msg_queue_expire = TRUE
+        SET msg_queue_size = 0, msg_can_write = TRUE
+        WHERE msg_queue_size != 0 OR msg_can_write = FALSE
      |]
    void $ DB.execute_
      db
      [sql|
        UPDATE msg_queues q
        SET msg_queue_size = s.size,
-            msg_can_write = s.quota_count = 0,
-            msg_queue_expire = s.size > s.quota_count
+            msg_can_write = s.quota_count = 0
        FROM queue_stats s
        WHERE q.recipient_id = s.recipient_id
      |]
@@ -15,6 +15,10 @@ module Simplex.Messaging.Server.MsgStore.STM
  ( STMMsgStore (..),
    STMStoreConfig (..),
    STMQueue,
+    loadedQueueCounts,
+    getQueueMessages,
+    setOverQuota_,
+    deleteQuotaMsg,
  )
 where

@@ -24,7 +28,8 @@ import Control.Monad.Trans.Except
 import Data.Functor (($>))
 import Data.Int (Int64)
 import qualified Data.Map.Strict as M
-import Data.Text (Text)
+import Data.Maybe (catMaybes)
+import Data.Time.Clock.System (SystemTime (systemSeconds))
 import Simplex.Messaging.Protocol
 import Simplex.Messaging.Server.MsgStore.Types
 import Simplex.Messaging.Server.QueueStore
@@ -61,12 +66,8 @@ instance StoreQueueClass STMQueue where
  {-# INLINE recipientId #-}
  queueRec = queueRec'
  {-# INLINE queueRec #-}
-  withQueueLock _ _ = id
-  {-# INLINE withQueueLock #-}

 instance MsgStoreClass STMMsgStore where
-  type StoreMonad STMMsgStore = STM
-  type MsgQueue STMMsgStore = STMMsgQueue
  type QueueStore STMMsgStore = STMQueueStore STMQueue
  type StoreQueue STMMsgStore = STMQueue
  type MsgStoreConfig STMMsgStore = STMStoreConfig
@@ -78,59 +79,22 @@ instance MsgStoreClass STMMsgStore where

  closeMsgStore = closeQueueStore @STMQueue . queueStore_
  {-# INLINE closeMsgStore #-}
-  withActiveMsgQueues = withLoadedQueues . queueStore_
-  {-# INLINE withActiveMsgQueues #-}
-  unsafeWithAllMsgQueues _ = withLoadedQueues . queueStore_
-  {-# INLINE unsafeWithAllMsgQueues #-}

  expireOldMessages :: Bool -> STMMsgStore -> Int64 -> Int64 -> IO MessageStats
  expireOldMessages _tty ms now ttl =
-    withLoadedQueues (queueStore_ ms) $ atomically . expireQueueMsgs ms now (now - ttl)
+    withLoadedQueues (queueStore_ ms) $ atomically . expireQueueMsgs (now - ttl)

  foldRcvServiceMessages :: STMMsgStore -> ServiceId -> (a -> RecipientId -> Either ErrorType (Maybe (QueueRec, Message)) -> IO a) -> a -> IO (Either ErrorType a)
  foldRcvServiceMessages ms serviceId f = fmap Right . foldRcvServiceQueues (queueStore_ ms) serviceId f'
    where
      f' a (q, qr) = runExceptT (tryPeekMsg ms q) >>= f a (recipientId q) . ((qr,) <$$>)

-  logQueueStates _ = pure ()
-  {-# INLINE logQueueStates #-}
-  logQueueState _ = pure ()
-  {-# INLINE logQueueState #-}
  queueStore = queueStore_
  {-# INLINE queueStore #-}

-  loadedQueueCounts :: STMMsgStore -> IO LoadedQueueCounts
-  loadedQueueCounts STMMsgStore {queueStore_ = st} = do
-    loadedQueueCount <- M.size <$> readTVarIO (queues st)
-    loadedNotifierCount <- M.size <$> readTVarIO (notifiers st)
-    pure LoadedQueueCounts {loadedQueueCount, loadedNotifierCount, openJournalCount = 0, queueLockCount = 0, notifierLockCount = 0}
-
-  mkQueue _ _ rId qr = STMQueue rId <$> newTVarIO (Just qr) <*> newTVarIO Nothing
+  mkQueue _ rId qr = STMQueue rId <$> newTVarIO (Just qr) <*> newTVarIO Nothing
  {-# INLINE mkQueue #-}

-  getMsgQueue :: STMMsgStore -> STMQueue -> Bool -> STM STMMsgQueue
-  getMsgQueue _ STMQueue {msgQueue'} _ = readTVar msgQueue' >>= maybe newQ pure
-    where
-      newQ = do
-        msgTQueue <- newTQueue
-        canWrite <- newTVar True
-        size <- newTVar 0
-        let q = STMMsgQueue {msgTQueue, canWrite, size}
-        writeTVar msgQueue' (Just q)
-        pure q
-
-  getPeekMsgQueue :: STMMsgStore -> STMQueue -> STM (Maybe (STMMsgQueue, Message))
-  getPeekMsgQueue _ q@STMQueue {msgQueue'} = readTVar msgQueue' $>>= \mq -> (mq,) <$$> tryPeekMsg_ q mq
-
-  -- does not create queue if it does not exist, does not delete it if it does (can't just close in-memory queue)
-  withIdleMsgQueue :: Int64 -> STMMsgStore -> STMQueue -> (STMMsgQueue -> STM a) -> STM (Maybe a, Int)
-  withIdleMsgQueue _ _ STMQueue {msgQueue'} action = readTVar msgQueue' >>= \case
-    Just q -> do
-      r <- action q
-      sz <- getQueueSize_ q
-      pure (Just r, sz)
-    Nothing -> pure (Nothing, 0)
-
  deleteQueue :: STMMsgStore -> STMQueue -> IO (Either ErrorType QueueRec)
  deleteQueue ms q = fst <$$> deleteQueue_ ms q

@@ -140,17 +104,9 @@ instance MsgStoreClass STMMsgStore where
    where
      getSize = maybe (pure 0) (\STMMsgQueue {size} -> readTVarIO size)

-  getQueueMessages_ :: Bool -> STMQueue -> STMMsgQueue -> STM [Message]
-  getQueueMessages_ drainMsgs _ = (if drainMsgs then flushTQueue else snapshotTQueue) . msgTQueue
-    where
-      snapshotTQueue q = do
-        msgs <- flushTQueue q
-        mapM_ (writeTQueue q) msgs
-        pure msgs
-
  writeMsg :: STMMsgStore -> STMQueue -> Bool -> Message -> ExceptT ErrorType IO (Maybe (Message, Bool))
  writeMsg ms q' _logState msg = liftIO $ atomically $ do
-    STMMsgQueue {msgTQueue = q, canWrite, size} <- getMsgQueue ms q' True
+    STMMsgQueue {msgTQueue = q, canWrite, size} <- getMsgQueue q'
    canWrt <- readTVar canWrite
    empty <- isEmptyTQueue q
    if canWrt || empty
@@ -166,29 +122,109 @@ instance MsgStoreClass STMMsgStore where
      STMMsgStore {storeConfig = STMStoreConfig {quota}} = ms
      msgQuota = MessageQuota {msgId = messageId msg, msgTs = messageTs msg}

-  setOverQuota_ :: STMQueue -> IO ()
-  setOverQuota_ q = readTVarIO (msgQueue' q) >>= mapM_ (\mq -> atomically $ writeTVar (canWrite mq) False)
+  tryPeekMsg :: STMMsgStore -> STMQueue -> ExceptT ErrorType IO (Maybe Message)
+  tryPeekMsg _ q = snd <$$> withPeekMsgQueue q pure
+  {-# INLINE tryPeekMsg #-}

-  getQueueSize_ :: STMMsgQueue -> STM Int
-  getQueueSize_ STMMsgQueue {size} = readTVar size
+  tryPeekMsgs :: STMMsgStore -> [STMQueue] -> ExceptT ErrorType IO (M.Map RecipientId Message)
+  tryPeekMsgs st qs = M.fromList . catMaybes <$> mapM (\q -> (recipientId q,) <$$> tryPeekMsg st q) qs

-  tryPeekMsg_ :: STMQueue -> STMMsgQueue -> STM (Maybe Message)
-  tryPeekMsg_ _ = tryPeekTQueue . msgTQueue
-  {-# INLINE tryPeekMsg_ #-}
+  tryDelMsg :: STMMsgStore -> STMQueue -> MsgId -> ExceptT ErrorType IO (Maybe Message)
+  tryDelMsg _ q msgId' =
+    withPeekMsgQueue q $
+      maybe (pure Nothing) $ \(mq, msg) ->
+        if messageId msg == msgId'
+          then tryDeleteMsg_ mq $> Just msg
+          else pure Nothing

-  tryDeleteMsg_ :: STMQueue -> STMMsgQueue -> Bool -> STM ()
-  tryDeleteMsg_ _ STMMsgQueue {msgTQueue = q, size} _logState =
-    tryReadTQueue q >>= \case
-      Just _ -> modifyTVar' size (subtract 1)
-      _ -> pure ()
+  tryDelPeekMsg :: STMMsgStore -> STMQueue -> MsgId -> ExceptT ErrorType IO (Maybe Message, Maybe Message)
+  tryDelPeekMsg _ q msgId' =
+    withPeekMsgQueue q $
+      maybe (pure (Nothing, Nothing)) $ \(mq, msg) ->
+        if messageId msg == msgId'
+          then (Just msg,) <$> (tryDeleteMsg_ mq >> tryPeekMsg_ mq)
+          else pure (Nothing, Just msg)

-  isolateQueue :: STMMsgStore -> STMQueue -> Text -> STM a -> ExceptT ErrorType IO a
-  isolateQueue _ _ _ = liftIO . atomically
-  {-# INLINE isolateQueue #-}
+  deleteExpiredMsgs :: STMMsgStore -> STMQueue -> Int64 -> ExceptT ErrorType IO Int
+  deleteExpiredMsgs _ q old = liftIO $ atomically $ getMsgQueue q >>= deleteExpireMsgs_ old

-  unsafeRunStore :: STMQueue -> Text -> STM a -> IO a
-  unsafeRunStore _ _ = atomically
-  {-# INLINE unsafeRunStore #-}
+  getQueueSize :: STMMsgStore -> STMQueue -> ExceptT ErrorType IO Int
+  getQueueSize _ q = withPeekMsgQueue q $ maybe (pure 0) (getQueueSize_ . fst)
+  {-# INLINE getQueueSize #-}
+
+loadedQueueCounts :: STMMsgStore -> IO LoadedQueueCounts
+loadedQueueCounts STMMsgStore {queueStore_ = st} = do
+  loadedQueueCount <- M.size <$> readTVarIO (queues st)
+  loadedNotifierCount <- M.size <$> readTVarIO (notifiers st)
+  pure LoadedQueueCounts {loadedQueueCount, loadedNotifierCount}
+
+getMsgQueue :: STMQueue -> STM STMMsgQueue
+getMsgQueue STMQueue {msgQueue'} = readTVar msgQueue' >>= maybe newQ pure
+  where
+    newQ = do
+      msgTQueue <- newTQueue
+      canWrite <- newTVar True
+      size <- newTVar 0
+      let q = STMMsgQueue {msgTQueue, canWrite, size}
+      writeTVar msgQueue' (Just q)
+      pure q
+
+-- The action is called with Nothing when it is known that the queue is empty
+withPeekMsgQueue :: STMQueue -> (Maybe (STMMsgQueue, Message) -> STM a) -> ExceptT ErrorType IO a
+withPeekMsgQueue STMQueue {msgQueue'} a =
+  liftIO $ atomically $ (readTVar msgQueue' $>>= \mq -> (mq,) <$$> tryPeekMsg_ mq) >>= a
+
+getQueueMessages :: Bool -> STMQueue -> IO [Message]
+getQueueMessages drainMsgs q = atomically $ (if drainMsgs then flushTQueue else snapshotTQueue) . msgTQueue =<< getMsgQueue q
+  where
+    snapshotTQueue mq = do
+      msgs <- flushTQueue mq
+      mapM_ (writeTQueue mq) msgs
+      pure msgs
+
+-- can ONLY be used while restoring messages, not while server running
+setOverQuota_ :: STMQueue -> IO ()
+setOverQuota_ q = readTVarIO (msgQueue' q) >>= mapM_ (\mq -> atomically $ writeTVar (canWrite mq) False)
+
+-- if the first message in queue head is "quota", remove it
+deleteQuotaMsg :: STMQueue -> ExceptT ErrorType IO ()
+deleteQuotaMsg q =
+  withPeekMsgQueue q $ \case
+    Just (mq, MessageQuota {}) -> tryDeleteMsg_ mq
+    _ -> pure ()
+
+expireQueueMsgs :: Int64 -> STMQueue -> STM MessageStats
+expireQueueMsgs old STMQueue {msgQueue'} =
+  readTVar msgQueue' >>= \case
+    Just mq -> do
+      expiredMsgsCount <- deleteExpireMsgs_ old mq
+      storedMsgsCount <- getQueueSize_ mq
+      pure MessageStats {storedMsgsCount, expiredMsgsCount, storedQueues = 1}
+    -- does not create queue if it does not exist
+    Nothing -> pure newMessageStats {storedQueues = 1}
+
+deleteExpireMsgs_ :: Int64 -> STMMsgQueue -> STM Int
+deleteExpireMsgs_ old mq = loop 0
+  where
+    loop dc =
+      tryPeekMsg_ mq >>= \case
+        Just Message {msgTs}
+          | systemSeconds msgTs < old ->
+              tryDeleteMsg_ mq >> loop (dc + 1)
+        _ -> pure dc
+
+getQueueSize_ :: STMMsgQueue -> STM Int
+getQueueSize_ STMMsgQueue {size} = readTVar size
+
+tryPeekMsg_ :: STMMsgQueue -> STM (Maybe Message)
+tryPeekMsg_ = tryPeekTQueue . msgTQueue
+{-# INLINE tryPeekMsg_ #-}
+
+tryDeleteMsg_ :: STMMsgQueue -> STM ()
+tryDeleteMsg_ STMMsgQueue {msgTQueue = q, size} =
+  tryReadTQueue q >>= \case
+    Just _ -> modifyTVar' size (subtract 1)
+    _ -> pure ()

 deleteQueue_ :: STMMsgStore -> STMQueue -> IO (Either ErrorType (QueueRec, Maybe STMMsgQueue))
 deleteQueue_ ms q = deleteStoreQueue (queueStore_ ms) q >>= mapM remove
@@ -4,17 +4,12 @@
 {-# LANGUAGE FlexibleContexts #-}
 {-# LANGUAGE GADTs #-}
 {-# LANGUAGE LambdaCase #-}
-{-# LANGUAGE MultiWayIf #-}
 {-# LANGUAGE NamedFieldPuns #-}
 {-# LANGUAGE OverloadedStrings #-}
 {-# LANGUAGE ScopedTypeVariables #-}
 {-# LANGUAGE TupleSections #-}
 {-# LANGUAGE TypeApplications #-}
 {-# LANGUAGE TypeFamilyDependencies #-}
-{-# OPTIONS_GHC -Wno-unrecognised-pragmas #-}
-
-{-# HLINT ignore "Redundant multi-way if" #-}
-
 module Simplex.Messaging.Server.MsgStore.Types
  ( MsgStoreClass (..),
    MSType (..),
@@ -30,107 +25,49 @@ module Simplex.Messaging.Server.MsgStore.Types
    getQueues,
    getQueueRecs,
    readQueueRec,
-    withPeekMsgQueue,
-    expireQueueMsgs,
-    deleteExpireMsgs_,
  ) where

 import Control.Concurrent.STM
 import Control.Monad
 import Control.Monad.Trans.Except
-import Data.Functor (($>))
 import Data.Int (Int64)
 import Data.Kind
 import Data.Map.Strict (Map)
-import qualified Data.Map.Strict as M
-import Data.Maybe (catMaybes, fromMaybe)
-import Data.Text (Text)
-import Data.Time.Clock.System (SystemTime (systemSeconds))
 import Simplex.Messaging.Protocol
 import Simplex.Messaging.Server.QueueStore
 import Simplex.Messaging.Server.QueueStore.Types
-import Simplex.Messaging.Util ((<$$>), ($>>=))
+import Simplex.Messaging.Util (($>>=))

-class (Monad (StoreMonad s), QueueStoreClass (StoreQueue s) (QueueStore s)) => MsgStoreClass s where
-  type StoreMonad s = (m :: Type -> Type) | m -> s
+class QueueStoreClass (StoreQueue s) (QueueStore s) => MsgStoreClass s where
  type MsgStoreConfig s = c | c -> s
-  type MsgQueue s = q | q -> s
  type StoreQueue s = q | q -> s
  type QueueStore s = qs | qs -> s
  newMsgStore :: MsgStoreConfig s -> IO s
  closeMsgStore :: s -> IO ()
-  withActiveMsgQueues :: Monoid a => s -> (StoreQueue s -> IO a) -> IO a
-  -- This function can only be used in server CLI commands or before server is started.
-  -- tty, store
-  unsafeWithAllMsgQueues :: Monoid a => Bool -> s -> (StoreQueue s -> IO a) -> IO a
  -- tty, store, now, ttl
  expireOldMessages :: Bool -> s -> Int64 -> Int64 -> IO MessageStats
  foldRcvServiceMessages :: s -> ServiceId -> (a -> RecipientId -> Either ErrorType (Maybe (QueueRec, Message)) -> IO a) -> a -> IO (Either ErrorType a)
-  logQueueStates :: s -> IO ()
-  logQueueState :: StoreQueue s -> StoreMonad s ()
  queueStore :: s -> QueueStore s
-  loadedQueueCounts :: s -> IO LoadedQueueCounts

  -- message store methods
-  mkQueue :: s -> Bool -> RecipientId -> QueueRec -> IO (StoreQueue s)
-  getMsgQueue :: s -> StoreQueue s -> Bool -> StoreMonad s (MsgQueue s)
-  getPeekMsgQueue :: s -> StoreQueue s -> StoreMonad s (Maybe (MsgQueue s, Message))
-
-  -- the journal queue will be closed after action if it was initially closed or idle longer than interval in config
-  withIdleMsgQueue :: Int64 -> s -> StoreQueue s -> (MsgQueue s -> StoreMonad s a) -> StoreMonad s (Maybe a, Int)
+  mkQueue :: s -> RecipientId -> QueueRec -> IO (StoreQueue s)
  deleteQueue :: s -> StoreQueue s -> IO (Either ErrorType QueueRec)
  deleteQueueSize :: s -> StoreQueue s -> IO (Either ErrorType (QueueRec, Int))
-  getQueueMessages_ :: Bool -> StoreQueue s -> MsgQueue s -> StoreMonad s [Message]
  writeMsg :: s -> StoreQueue s -> Bool -> Message -> ExceptT ErrorType IO (Maybe (Message, Bool))
-  setOverQuota_ :: StoreQueue s -> IO () -- can ONLY be used while restoring messages, not while server running
-  getQueueSize_ :: MsgQueue s -> StoreMonad s Int
-  tryPeekMsg_ :: StoreQueue s -> MsgQueue s -> StoreMonad s (Maybe Message)
-  tryDeleteMsg_ :: StoreQueue s -> MsgQueue s -> Bool -> StoreMonad s ()
-  isolateQueue :: s -> StoreQueue s -> Text -> StoreMonad s a -> ExceptT ErrorType IO a
-  unsafeRunStore :: StoreQueue s -> Text -> StoreMonad s a -> IO a
-
-  -- default implementations are overridden for PostgreSQL storage of messages
  tryPeekMsg :: s -> StoreQueue s -> ExceptT ErrorType IO (Maybe Message)
-  tryPeekMsg st q = snd <$$> withPeekMsgQueue st q "tryPeekMsg" pure
-  {-# INLINE tryPeekMsg #-}
-  
  tryPeekMsgs :: s -> [StoreQueue s] -> ExceptT ErrorType IO (Map RecipientId Message)
-  tryPeekMsgs st qs = M.fromList . catMaybes <$> mapM (\q -> (recipientId q,) <$$> tryPeekMsg st q) qs
-
  tryDelMsg :: s -> StoreQueue s -> MsgId -> ExceptT ErrorType IO (Maybe Message)
-  tryDelMsg st q msgId' =
-    withPeekMsgQueue st q "tryDelMsg" $
-      maybe (pure Nothing) $ \(mq, msg) ->
-        if
-          | messageId msg == msgId' ->
-              tryDeleteMsg_ q mq True $> Just msg
-          | otherwise -> pure Nothing
-
  -- atomic delete (== read) last and peek next message if available
  tryDelPeekMsg :: s -> StoreQueue s -> MsgId -> ExceptT ErrorType IO (Maybe Message, Maybe Message)
-  tryDelPeekMsg st q msgId' =
-    withPeekMsgQueue st q "tryDelPeekMsg" $
-      maybe (pure (Nothing, Nothing)) $ \(mq, msg) ->
-        if
-          | messageId msg == msgId' -> (Just msg,) <$> (tryDeleteMsg_ q mq True >> tryPeekMsg_ q mq)
-          | otherwise -> pure (Nothing, Just msg)
-
  deleteExpiredMsgs :: s -> StoreQueue s -> Int64 -> ExceptT ErrorType IO Int
-  deleteExpiredMsgs st q old =
-    isolateQueue st q "deleteExpiredMsgs" $
-      getMsgQueue st q False >>= deleteExpireMsgs_ old q
-
  getQueueSize :: s -> StoreQueue s -> ExceptT ErrorType IO Int
-  getQueueSize st q = withPeekMsgQueue st q "getQueueSize" $ maybe (pure 0) (getQueueSize_ . fst)
-  {-# INLINE getQueueSize #-}

-data MSType = MSMemory | MSJournal | MSPostgres
+data MSType = MSMemory | MSPostgres

 data QSType = QSMemory | QSPostgres

 data SMSType :: MSType -> Type where
  SMSMemory :: SMSType 'MSMemory
-  SMSJournal :: SMSType 'MSJournal
  SMSPostgres :: SMSType 'MSPostgres

 data SQSType :: QSType -> Type where
@@ -154,17 +91,14 @@ instance Semigroup MessageStats where

 data LoadedQueueCounts = LoadedQueueCounts
  { loadedQueueCount :: Int,
-    loadedNotifierCount :: Int,
-    openJournalCount :: Int,
-    queueLockCount :: Int,
-    notifierLockCount :: Int
+    loadedNotifierCount :: Int
  }

 newMessageStats :: MessageStats
 newMessageStats = MessageStats 0 0 0

 addQueue :: MsgStoreClass s => s -> RecipientId -> QueueRec -> IO (Either ErrorType (StoreQueue s))
-addQueue st = addQueue_ (queueStore st) (mkQueue st True)
+addQueue st = addQueue_ (queueStore st) (mkQueue st)
 {-# INLINE addQueue #-}

 getQueue :: (MsgStoreClass s, QueueParty p) => s -> SParty p -> QueueId -> IO (Either ErrorType (StoreQueue s))
@@ -184,28 +118,3 @@ getQueueRecs st party qIds = getQueues st party qIds >>= mapM (fmap join . mapM
 readQueueRec :: StoreQueueClass q => q -> IO (Either ErrorType (q, QueueRec))
 readQueueRec q = maybe (Left AUTH) (Right . (q,)) <$> readTVarIO (queueRec q)
 {-# INLINE readQueueRec #-}
-
-- The action is called with Nothing when it is known that the queue is empty
-withPeekMsgQueue :: MsgStoreClass s => s -> StoreQueue s -> Text -> (Maybe (MsgQueue s, Message) -> StoreMonad s a) -> ExceptT ErrorType IO a
-withPeekMsgQueue st q op a = isolateQueue st q op $ getPeekMsgQueue st q >>= a
-{-# INLINE withPeekMsgQueue #-}
-
-- not used with PostgreSQL message store
-expireQueueMsgs :: MsgStoreClass s => s -> Int64 -> Int64 -> StoreQueue s -> StoreMonad s MessageStats
-expireQueueMsgs st now old q = do
-  (expired_, stored) <- withIdleMsgQueue now st q $ deleteExpireMsgs_ old q
-  pure MessageStats {storedMsgsCount = stored, expiredMsgsCount = fromMaybe 0 expired_, storedQueues = 1}
-
-- not used with PostgreSQL message store
-deleteExpireMsgs_ :: MsgStoreClass s => Int64 -> StoreQueue s -> MsgQueue s -> StoreMonad s Int
-deleteExpireMsgs_ old q mq = do
-  n <- loop 0
-  logQueueState q
-  pure n
-  where
-    loop dc =
-      tryPeekMsg_ q mq >>= \case
-        Just Message {msgTs}
-          | systemSeconds msgTs < old ->
-              tryDeleteMsg_ q mq False >> loop (dc + 1)
-        _ -> pure dc
@@ -46,7 +46,7 @@ data RealTimeMetrics = RealTimeMetrics
    deliveredTimes :: TimeBuckets,
    smpSubs :: RTSubscriberMetrics,
    ntfSubs :: RTSubscriberMetrics,
-    loadedCounts :: LoadedQueueCounts
+    loadedCounts :: Maybe LoadedQueueCounts
  }

 data RTSubscriberMetrics = RTSubscriberMetrics
@@ -562,27 +562,17 @@ prometheusMetrics sm rtm ts =
      \\n\
      \# HELP simplex_smp_subscription_ntf_service_subs_total Total queues subscribed via NTF services\n\
      \# TYPE simplex_smp_subscription_ntf_service_subs_total gauge\n\
-      \simplex_smp_subscription_ntf_service_subs_total " <> mshow (subServiceSubsCount ntfSubs) <> "\n# ntf.subServiceSubsCount\n\
-      \\n\
-      \# HELP simplex_smp_loaded_queues_queue_count Total loaded queues count (all queues for memory/journal storage)\n\
+      \simplex_smp_subscription_ntf_service_subs_total " <> mshow (subServiceSubsCount ntfSubs) <> "\n# ntf.subServiceSubsCount\n"
+      <> maybe "" loadedQueues loadedCounts
+    loadedQueues LoadedQueueCounts {loadedQueueCount, loadedNotifierCount} =
+      "\n\
+      \# HELP simplex_smp_loaded_queues_queue_count Total loaded queues count (all queues for memory storage)\n\
      \# TYPE simplex_smp_loaded_queues_queue_count gauge\n\
-      \simplex_smp_loaded_queues_queue_count " <> mshow (loadedQueueCount loadedCounts) <> "\n# loadedCounts.loadedQueueCount\n\
+      \simplex_smp_loaded_queues_queue_count " <> mshow loadedQueueCount <> "\n# loadedCounts.loadedQueueCount\n\
      \\n\
-      \# HELP simplex_smp_loaded_queues_ntf_count Total loaded ntf credential references (all ntf credentials for memory/journal storage)\n\
+      \# HELP simplex_smp_loaded_queues_ntf_count Total loaded ntf credential references (all ntf credentials for memory storage)\n\
      \# TYPE simplex_smp_loaded_queues_ntf_count gauge\n\
-      \simplex_smp_loaded_queues_ntf_count " <> mshow (loadedNotifierCount loadedCounts) <> "\n# loadedCounts.loadedNotifierCount\n\
-      \\n\
-      \# HELP simplex_smp_loaded_queues_open_journal_count Total opened queue journals (0 for memory storage)\n\
-      \# TYPE simplex_smp_loaded_queues_open_journal_count gauge\n\
-      \simplex_smp_loaded_queues_open_journal_count " <> mshow (openJournalCount loadedCounts) <> "\n# loadedCounts.openJournalCount\n\
-      \\n\
-      \# HELP simplex_smp_loaded_queues_queue_lock_count Total queue locks (0 for memory storage)\n\
-      \# TYPE simplex_smp_loaded_queues_queue_lock_count gauge\n\
-      \simplex_smp_loaded_queues_queue_lock_count " <> mshow (queueLockCount loadedCounts) <> "\n# loadedCounts.queueLockCount\n\
-      \\n\
-      \# HELP simplex_smp_loaded_queues_ntf_lock_count Total notifier locks (0 for memory/journal storage)\n\
-      \# TYPE simplex_smp_loaded_queues_ntf_lock_count gauge\n\
-      \simplex_smp_loaded_queues_ntf_lock_count " <> mshow (notifierLockCount loadedCounts) <> "\n# loadedCounts.notifierLockCount\n"
+      \simplex_smp_loaded_queues_ntf_count " <> mshow loadedNotifierCount <> "\n# loadedCounts.loadedNotifierCount\n"

    showTimeBuckets :: Text -> IM.IntMap Int -> Text
    showTimeBuckets metric = T.concat . snd . mapAccumL accumBucket (0, 0) . IM.assocs
@@ -13,12 +13,14 @@ module Simplex.Messaging.Server.QueueStore
    ServiceRec (..),
    CertFingerprint,
    ServerEntityStatus (..),
+    isSecuredMsgQueue,
  ) where

 import Control.Applicative (optional, (<|>))
 import qualified Data.ByteString.Char8 as B
 import Data.Functor (($>))
 import Data.List.NonEmpty (NonEmpty)
+import Data.Maybe (isJust)
 import qualified Data.X509 as X
 import qualified Data.X509.Validation as XV
 import Simplex.Messaging.Encoding
@@ -48,6 +50,11 @@ data QueueRec = QueueRec
  }
  deriving (Show)

+isSecuredMsgQueue :: QueueRec -> Bool
+isSecuredMsgQueue QueueRec {queueMode, senderKey} = case queueMode of
+  Just QMContact -> False
+  _ -> isJust senderKey
+
 data NtfCreds = NtfCreds
  { notifierId :: NotifierId,
    notifierKey :: NtfPublicAuthKey,
@@ -24,9 +24,7 @@ module Simplex.Messaging.Server.QueueStore.Postgres
    batchInsertServices,
    batchInsertQueues,
    foldServiceRecs,
-    foldRcvServiceQueueRecs,
    foldQueueRecs,
-    foldRecentQueueRecs,
    handleDuplicate,
    rowToQueueRec,
    withLog_,
@@ -43,13 +41,12 @@ import Control.Monad
 import Control.Monad.Except
 import Control.Monad.IO.Class
 import Control.Monad.Trans.Except
-import Data.Bifunctor (first)
 import Data.ByteString.Builder (Builder)
 import qualified Data.ByteString.Builder as BB
 import Data.ByteString.Char8 (ByteString)
 import qualified Data.ByteString.Lazy as LB
 import Data.Bitraversable (bimapM)
-import Data.Either (fromRight, lefts)
+import Data.Either (fromRight)
 import Data.Functor (($>))
 import Data.Int (Int64)
 import Data.List (foldl', intersperse, partition)
@@ -91,7 +88,7 @@ import Simplex.Messaging.SystemTime
 import Simplex.Messaging.TMap (TMap)
 import qualified Simplex.Messaging.TMap as TM
 import Simplex.Messaging.Transport (SMPServiceRole (..))
-import Simplex.Messaging.Util (eitherToMaybe, firstRow, ifM, maybeFirstRow, maybeFirstRow', tshow, (<$$>), ($>>=))
+import Simplex.Messaging.Util (eitherToMaybe, firstRow, maybeFirstRow, maybeFirstRow', tshow, (<$$>), ($>>=))
 import System.Exit (exitFailure)
 import System.IO (IOMode (..), hFlush, stdout)
 import UnliftIO.STM
@@ -103,35 +100,19 @@ import Simplex.Messaging.Encoding.String
 data PostgresQueueStore q = PostgresQueueStore
  { dbStore :: DBStore,
    dbStoreLog :: Maybe (StoreLog 'WriteMode),
-    -- this map caches all created and opened queues
-    queues :: TMap RecipientId q,
-    -- this map only cashes the queues that were attempted to send messages to,
-    senders :: TMap SenderId RecipientId,
-    links :: TMap LinkId RecipientId,
-    -- this map only cashes the queues that were attempted to be subscribed to,
-    notifiers :: TMap NotifierId RecipientId,
-    notifierLocks :: TMap NotifierId Lock,
    serviceLocks :: TMap CertFingerprint Lock,
-    deletedTTL :: Int64,
-    useCache :: Bool
+    deletedTTL :: Int64
  }

-type UseQueueCache = Bool
-
 instance StoreQueueClass q => QueueStoreClass q (PostgresQueueStore q) where
-  type QueueStoreCfg (PostgresQueueStore q) = (PostgresStoreCfg, UseQueueCache)
+  type QueueStoreCfg (PostgresQueueStore q) = PostgresStoreCfg

-  newQueueStore :: (PostgresStoreCfg, UseQueueCache)  -> IO (PostgresQueueStore q)
-  newQueueStore (PostgresStoreCfg {dbOpts, dbStoreLogPath, confirmMigrations, deletedTTL}, useCache) = do
+  newQueueStore :: PostgresStoreCfg -> IO (PostgresQueueStore q)
+  newQueueStore PostgresStoreCfg {dbOpts, dbStoreLogPath, confirmMigrations, deletedTTL} = do
    dbStore <- either err pure =<< createDBStore dbOpts serverMigrations (MigrationConfig confirmMigrations Nothing)
    dbStoreLog <- mapM (openWriteStoreLog True) dbStoreLogPath
-    queues <- TM.emptyIO
-    senders <- TM.emptyIO
-    links <- TM.emptyIO
-    notifiers <- TM.emptyIO
-    notifierLocks <- TM.emptyIO
    serviceLocks <- TM.emptyIO
-    pure PostgresQueueStore {dbStore, dbStoreLog, queues, senders, links, notifiers, notifierLocks, serviceLocks, deletedTTL, useCache}
+    pure PostgresQueueStore {dbStore, dbStoreLog, serviceLocks, deletedTTL}
    where
      err e = do
        logError $ "STORE: newQueueStore, error opening PostgreSQL database, " <> tshow e
@@ -142,9 +123,6 @@ instance StoreQueueClass q => QueueStoreClass q (PostgresQueueStore q) where
    closeDBStore dbStore
    mapM_ closeStoreLog dbStoreLog

-  loadedQueues = queues
-  {-# INLINE loadedQueues #-}
-
  compactQueues :: PostgresQueueStore q -> IO Int64
  compactQueues st@PostgresQueueStore {deletedTTL} = do
    old <- subtract deletedTTL . systemSeconds <$> liftIO getSystemTime
@@ -170,136 +148,45 @@ instance StoreQueueClass q => QueueStoreClass q (PostgresQueueStore q) where
          (SRMessaging, SRNotifier)
      pure EntityCounts {queueCount, notifierCount, rcvServiceCount, ntfServiceCount, rcvServiceQueuesCount, ntfServiceQueuesCount}

-  -- this implementation assumes that the lock is already taken by addQueue
-  -- and relies on unique constraints in the database to prevent duplicate IDs.
+  -- this implementation relies on unique constraints in the database to prevent duplicate IDs.
  addQueue_ :: PostgresQueueStore q -> (RecipientId -> QueueRec -> IO q) -> RecipientId -> QueueRec -> IO (Either ErrorType q)
  addQueue_ st mkQ rId qr = do
    sq <- mkQ rId qr
-    withQueueLock sq "addQueue_" $ E.uninterruptibleMask_ $ runExceptT $ do
+    E.uninterruptibleMask_ $ runExceptT $ do
      void $ withDB "addQueue_" st $ \db ->
        E.try (DB.execute db insertQueueQuery $ queueRecToRow (rId, qr))
          >>= bimapM handleDuplicate pure
-      when useCache $ do
-        atomically $ TM.insert rId sq queues
-        atomically $ TM.insert (senderId qr) rId senders
-        forM_ (notifier qr) $ \NtfCreds {notifierId = nId} -> atomically $ TM.insert nId rId notifiers
-        forM_ (queueData qr) $ \(lnkId, _) -> atomically $ TM.insert lnkId rId links
      withLog "addStoreQueue" st $ \s -> logCreateQueue s rId qr
      pure sq
-    where
-      PostgresQueueStore {queues, senders, links, notifiers, useCache} = st
-      -- Not doing duplicate checks in maps as the probability of duplicates is very low.
-      -- It needs to be reconsidered when IDs are supplied by the users.
-      -- hasId = anyM [TM.memberIO rId queues, TM.memberIO senderId senders, hasNotifier]
-      -- hasNotifier = maybe (pure False) (\NtfCreds {notifierId} -> TM.memberIO notifierId notifiers) notifier

-  getQueue_ :: QueueParty p => PostgresQueueStore q -> (Bool -> RecipientId -> QueueRec -> IO q) -> SParty p -> QueueId -> IO (Either ErrorType q)
-  getQueue_ st mkQ party qId
-    | useCache = case party of
-        SRecipient -> getRcvQueue qId
-        SSender -> TM.lookupIO qId senders >>= maybe (mask loadSndQueue) getRcvQueue
-        SSenderLink -> TM.lookupIO qId links >>= maybe (mask loadLinkQueue) getRcvQueue
-        -- loaded queue is deleted from notifiers map to reduce cache size after queue was subscribed to by ntf server
-        SNotifier -> TM.lookupIO qId notifiers >>= maybe (mask loadNtfQueue) (getRcvQueue >=> (atomically (TM.delete qId notifiers) $>))
-    | otherwise = case party of
-        SRecipient -> loadQueueNoCache " WHERE recipient_id = ?"
-        SSender -> loadQueueNoCache " WHERE sender_id = ?"
-        SSenderLink -> loadQueueNoCache " WHERE link_id = ?"
-        SNotifier -> loadQueueNoCache " WHERE notifier_id = ?"
+  getQueue_ :: QueueParty p => PostgresQueueStore q -> (RecipientId -> QueueRec -> IO q) -> SParty p -> QueueId -> IO (Either ErrorType q)
+  getQueue_ st mkQ party qId = case party of
+    SRecipient -> loadQueue " WHERE recipient_id = ?"
+    SSender -> loadQueue " WHERE sender_id = ?"
+    SSenderLink -> loadQueue " WHERE link_id = ?"
+    SNotifier -> loadQueue " WHERE notifier_id = ?"
    where
-      PostgresQueueStore {queues, senders, links, notifiers, useCache} = st
-      getRcvQueue rId = TM.lookupIO rId queues >>= maybe (mask loadRcvQueue) (pure . Right)
-      loadRcvQueue = do
-        (rId, qRec) <- loadQueue " WHERE recipient_id = ?"
-        liftIO $ cacheQueue rId qRec $ \_ -> pure () -- recipient map already checked, not caching sender ref
-      loadSndQueue = loadSndQueue_ " WHERE sender_id = ?"
-      loadLinkQueue = loadSndQueue_ " WHERE link_id = ?"
-      loadNtfQueue = do
-        (rId, qRec) <- loadQueue " WHERE notifier_id = ?"
-        liftIO $
-          TM.lookupIO rId queues -- checking recipient map first, not creating lock in map, not caching queue
-            >>= maybe (mkQ False rId qRec) pure
-      loadSndQueue_ condition = do
-        (rId, qRec) <- loadQueue condition
-        liftIO $
-          TM.lookupIO rId queues -- checking recipient map first
-            >>= maybe (cacheQueue rId qRec cacheSender) (atomically (cacheSender rId) $>)
-      loadQueueNoCache cond = mask $ loadQueue cond >>= liftIO . uncurry (mkQ True)
-      mask = E.uninterruptibleMask_ . runExceptT
-      cacheSender rId = TM.insert qId rId senders
-      loadQueue condition =
-        withDB "getQueue_" st $ \db -> firstRow rowToQueueRec AUTH $
+      loadQueue condition = E.uninterruptibleMask_ $ runExceptT $ do
+        (rId, qRec) <- withDB "getQueue_" st $ \db -> firstRow rowToQueueRec AUTH $
          DB.query db (queueRecQuery <> condition <> " AND deleted_at IS NULL") (Only qId)
-      cacheQueue rId qRec insertRef = do
-        sq <- mkQ True rId qRec -- loaded queue
-        -- This lock prevents the scenario when the queue is added to cache,
-        -- while another thread is proccessing the same queue in withAllMsgQueues
-        -- without adding it to cache, possibly trying to open the same files twice.
-        -- Alse see comment in idleDeleteExpiredMsgs.
-        withQueueLock sq "getQueue_" $ atomically $
-          -- checking the cache again for concurrent reads,
-          -- use previously loaded queue if exists.
-          TM.lookup rId queues >>= \case
-            Just sq' -> pure sq'
-            Nothing -> do
-              insertRef rId
-              TM.insert rId sq queues
-              pure sq
+        liftIO $ mkQ rId qRec

-  getQueues_ :: forall p. BatchParty p => PostgresQueueStore q -> (Bool -> RecipientId -> QueueRec -> IO q) -> SParty p -> [QueueId] -> IO [Either ErrorType q]
+  getQueues_ :: forall p. BatchParty p => PostgresQueueStore q -> (RecipientId -> QueueRec -> IO q) -> SParty p -> [QueueId] -> IO [Either ErrorType q]
  getQueues_ st mkQ party qIds
    | null qIds = pure []
-    | useCache = case party of
-        SRecipient -> do
-          qs <- readTVarIO queues
-          let qs' = map (\qId -> get qs qId qId) qIds
-          E.uninterruptibleMask_ $ loadQueues qs' " WHERE recipient_id IN ?" cacheRcvQueue
-        SNotifier -> do
-          ns <- readTVarIO notifiers
-          qs <- readTVarIO queues
-          let qs' = map (\qId -> get ns qId qId >>= get qs qId) qIds
-          E.uninterruptibleMask_ $ loadQueues qs' " WHERE notifier_id IN ?" $ \(rId, qRec) ->
-            forM (notifier qRec) $ \NtfCreds {notifierId = nId} -> -- it is always Just with this query
-              (nId,) <$> maybe (mkQ False rId qRec) pure (M.lookup rId qs)
    | otherwise = E.uninterruptibleMask_ $ case party of
-        SRecipient -> loadQueuesNoCache " WHERE recipient_id IN ?" $ \(rId, qRec) ->
-          Just . (rId,) <$> mkQ False rId qRec
-        SNotifier -> loadQueuesNoCache " WHERE notifier_id IN ?" $ \(rId, qRec) ->
-          forM (notifier qRec) $ \NtfCreds {notifierId = nId} -> (nId,) <$> mkQ False rId qRec
+        SRecipient -> loadQueues " WHERE recipient_id IN ?" $ \(rId, qRec) ->
+          Just . (rId,) <$> mkQ rId qRec
+        SNotifier -> loadQueues " WHERE notifier_id IN ?" $ \(rId, qRec) ->
+          forM (notifier qRec) $ \NtfCreds {notifierId = nId} -> (nId,) <$> mkQ rId qRec
    where
-      PostgresQueueStore {queues, notifiers, useCache} = st
-      get :: M.Map QueueId a -> QueueId -> QueueId -> Either QueueId a
-      get m qId = maybe (Left qId) Right . (`M.lookup` m)
-      loadQueues :: [Either QueueId q] -> Query -> ((RecipientId, QueueRec) -> IO (Maybe (QueueId, q))) -> IO [Either ErrorType q]
-      loadQueues qs' cond mkCacheQueue = do
-        let qIds' = lefts qs'
-        if null qIds'
-          then pure $ map (first (const INTERNAL)) qs'
-          else do
-            qs_ <- dbLoadQueues qIds' cond mkCacheQueue
-            pure $ map (result qs_) qs'
-        where
-          result :: Either ErrorType (M.Map QueueId q) -> Either QueueId q -> Either ErrorType q
-          result _ (Right q) = Right q
-          result qs_ (Left qId) = maybe (Left AUTH) Right . M.lookup qId =<< qs_
-      dbLoadQueues qIds' cond mkQueue' =
-        runExceptT $ fmap M.fromList $
-          withDB' "getQueues_" st (\db -> DB.query db (queueRecQuery <> cond <> " AND deleted_at IS NULL") (Only (In qIds')))
-            >>= liftIO . fmap catMaybes . mapM (mkQueue' . rowToQueueRec)
-      cacheRcvQueue (rId, qRec) = do
-        sq <- mkQ True rId qRec
-        sq' <- withQueueLock sq "getQueue_" $ atomically $
-          -- checking the cache again for concurrent reads, use previously loaded queue if exists.
-          TM.lookup rId queues >>= \case
-            Just sq' -> pure sq'
-            Nothing -> sq <$ TM.insert rId sq queues
-        pure $ Just (rId, sq')
-      loadQueuesNoCache cond mkQueue' = do
-        qs_ <- dbLoadQueues qIds cond mkQueue'
-        pure $ map (result qs_) qIds
-        where
-          result :: Either ErrorType (M.Map QueueId q) -> QueueId -> Either ErrorType q
-          result qs_ qId = maybe (Left AUTH) Right . M.lookup qId =<< qs_
+      loadQueues :: Query -> ((RecipientId, QueueRec) -> IO (Maybe (QueueId, q))) -> IO [Either ErrorType q]
+      loadQueues cond mkQueue' = do
+        qs_ <-
+          runExceptT $ fmap M.fromList $
+            withDB' "getQueues_" st (\db -> DB.query db (queueRecQuery <> cond <> " AND deleted_at IS NULL") (Only (In qIds)))
+              >>= liftIO . fmap catMaybes . mapM (mkQueue' . rowToQueueRec)
+        pure $ map (\qId -> maybe (Left AUTH) Right . M.lookup qId =<< qs_) qIds

  getQueueLinkData :: PostgresQueueStore q -> q -> LinkId -> IO (Either ErrorType QueueLinkData)
  getQueueLinkData st sq lnkId = runExceptT $ do
@@ -312,11 +199,11 @@ instance StoreQueueClass q => QueueStoreClass q (PostgresQueueStore q) where

  addQueueLinkData :: PostgresQueueStore q -> q -> LinkId -> QueueLinkData -> IO (Either ErrorType ())
  addQueueLinkData st sq lnkId d =
-    withQueueRec sq "addQueueLinkData" $ \q -> case queueData q of
+    withQueueRec sq $ \q -> case queueData q of
      Nothing ->
-        addLink q $ \db -> DB.execute db qry (d :. (lnkId, rId))
+        addLink q $ \db -> DB.execute db qry (d :. (lnkId, rId, QMContact))
      Just (lnkId', _) | lnkId' == lnkId ->
-        addLink q $ \db -> DB.execute db (qry <> " AND (fixed_data IS NULL OR fixed_data = ?)") (d :. (lnkId, rId, fst d))
+        addLink q $ \db -> DB.execute db (qry <> " AND (fixed_data IS NULL OR fixed_data = ?)") (d :. (lnkId, rId, QMContact, fst d))
      _ -> throwE AUTH
    where
      rId = recipientId sq
@@ -324,11 +211,12 @@ instance StoreQueueClass q => QueueStoreClass q (PostgresQueueStore q) where
        assertUpdated $ withDB' "addQueueLinkData" st update
        atomically $ writeTVar (queueRec sq) $ Just q {queueData = Just (lnkId, d)}
        withLog "addQueueLinkData" st $ \s -> logCreateLink s rId lnkId d
-      qry = "UPDATE msg_queues SET fixed_data = ?, user_data = ?, link_id = ? WHERE recipient_id = ? AND deleted_at IS NULL"
+      -- the sender key condition is checked in SQL because without cache each command reads its own copy of the queue record
+      qry = "UPDATE msg_queues SET fixed_data = ?, user_data = ?, link_id = ? WHERE recipient_id = ? AND deleted_at IS NULL AND (sender_key IS NULL OR queue_mode = ?)"

  deleteQueueLinkData :: PostgresQueueStore q -> q -> IO (Either ErrorType ())
  deleteQueueLinkData st sq =
-    withQueueRec sq "deleteQueueLinkData" $ \q -> case queueData q of
+    withQueueRec sq $ \q -> case queueData q of
      Just _ -> do
        assertUpdated $ withDB' "deleteQueueLinkData" st $ \db ->
          DB.execute db "UPDATE msg_queues SET link_id = NULL, fixed_data = NULL, user_data = NULL WHERE recipient_id = ? AND deleted_at IS NULL" (Only rId)
@@ -340,7 +228,7 @@ instance StoreQueueClass q => QueueStoreClass q (PostgresQueueStore q) where

  secureQueue :: PostgresQueueStore q -> q -> SndPublicAuthKey -> IO (Either ErrorType ())
  secureQueue st sq sKey =
-    withQueueRec sq "secureQueue" $ \q -> do
+    withQueueRec sq $ \q -> do
      verify q
      assertUpdated $ withDB' "secureQueue" st $ \db ->
        DB.execute db "UPDATE msg_queues SET sender_key = ? WHERE recipient_id = ? AND deleted_at IS NULL AND (sender_key IS NULL OR sender_key = ?)" (sKey, rId, sKey)
@@ -354,7 +242,7 @@ instance StoreQueueClass q => QueueStoreClass q (PostgresQueueStore q) where

  updateKeys :: PostgresQueueStore q -> q -> NonEmpty RcvPublicAuthKey -> IO (Either ErrorType ())
  updateKeys st sq rKeys =
-    withQueueRec sq "updateKeys" $ \q -> do
+    withQueueRec sq $ \q -> do
      assertUpdated $ withDB' "updateKeys" st $ \db ->
        DB.execute db "UPDATE msg_queues SET recipient_keys = ? WHERE recipient_id = ? AND deleted_at IS NULL" (rKeys, rId)
      atomically $ writeTVar (queueRec sq) $ Just q {recipientKeys = rKeys}
@@ -364,24 +252,14 @@ instance StoreQueueClass q => QueueStoreClass q (PostgresQueueStore q) where

  addQueueNotifier :: PostgresQueueStore q -> q -> NtfCreds -> IO (Either ErrorType (Maybe NtfCreds))
  addQueueNotifier st sq ntfCreds@NtfCreds {notifierId = nId, notifierKey, rcvNtfDhSecret} =
-    withQueueRec sq "addQueueNotifier" $ \q ->
-      checkCachedNotifier $ do
-        assertUpdated $ withDB "addQueueNotifier" st $ \db ->
-          E.try (update db) >>= bimapM handleDuplicate pure
-        nc_ <- forM (notifier q) $ \nc@NtfCreds {notifierId} -> atomically (TM.delete notifierId notifiers) $> nc
-        let !q' = q {notifier = Just ntfCreds}
-        atomically $ writeTVar (queueRec sq) $ Just q'
-        when useCache $ do
-          atomically $ TM.insert nId rId notifiers
-        withLog "addQueueNotifier" st $ \s -> logAddNotifier s rId ntfCreds
-        pure nc_
+    withQueueRec sq $ \q -> do
+      assertUpdated $ withDB "addQueueNotifier" st $ \db ->
+        E.try (update db) >>= bimapM handleDuplicate pure
+      let !q' = q {notifier = Just ntfCreds}
+      atomically $ writeTVar (queueRec sq) $ Just q'
+      withLog "addQueueNotifier" st $ \s -> logAddNotifier s rId ntfCreds
+      pure $ notifier q
    where
-      checkCachedNotifier add
-        | useCache =
-            ExceptT $ withLockMap (notifierLocks st) nId "addQueueNotifier" $
-              ifM (TM.memberIO nId notifiers) (pure $ Left DUPLICATE_) $ runExceptT add
-        | otherwise = add
-      PostgresQueueStore {notifiers, useCache} = st
      rId = recipientId sq
      update db =
        DB.execute
@@ -395,18 +273,13 @@ instance StoreQueueClass q => QueueStoreClass q (PostgresQueueStore q) where

  deleteQueueNotifier :: PostgresQueueStore q -> q -> IO (Either ErrorType (Maybe NtfCreds))
  deleteQueueNotifier st sq =
-    withQueueRec sq "deleteQueueNotifier" $ \q ->
-      ExceptT $ fmap sequence $ forM (notifier q) $ \nc@NtfCreds {notifierId = nId} ->
-        withNotifierLock nId $ runExceptT $ do
-          assertUpdated $ withDB' "deleteQueueNotifier" st update
-          when (useCache st) $ atomically $ TM.delete nId $ notifiers st
-          atomically $ writeTVar (queueRec sq) $ Just q {notifier = Nothing}
-          withLog "deleteQueueNotifier" st (`logDeleteNotifier` rId)
-          pure nc
+    withQueueRec sq $ \q ->
+      forM (notifier q) $ \nc -> do
+        assertUpdated $ withDB' "deleteQueueNotifier" st update
+        atomically $ writeTVar (queueRec sq) $ Just q {notifier = Nothing}
+        withLog "deleteQueueNotifier" st (`logDeleteNotifier` rId)
+        pure nc
    where
-      withNotifierLock nId
-        | useCache st = withLockMap (notifierLocks st) nId "deleteQueueNotifier"
-        | otherwise = id
      rId = recipientId sq
      update db =
        DB.execute
@@ -435,7 +308,7 @@ instance StoreQueueClass q => QueueStoreClass q (PostgresQueueStore q) where

  updateQueueTime :: PostgresQueueStore q -> q -> SystemDate -> IO (Either ErrorType QueueRec)
  updateQueueTime st sq t =
-    withQueueRec sq "updateQueueTime" $ \q@QueueRec {updatedAt} ->
+    withQueueRec sq $ \q@QueueRec {updatedAt} ->
      if updatedAt == Just t
        then pure q
        else do
@@ -448,7 +321,6 @@ instance StoreQueueClass q => QueueStoreClass q (PostgresQueueStore q) where
    where
      rId = recipientId sq

-  -- this method is called from JournalMsgStore deleteQueue that already locks the queue
  deleteStoreQueue :: PostgresQueueStore q -> q -> IO (Either ErrorType QueueRec)
  deleteStoreQueue st sq = E.uninterruptibleMask_ $ runExceptT $ do
    q <- ExceptT $ readQueueRecIO qr
@@ -456,11 +328,6 @@ instance StoreQueueClass q => QueueStoreClass q (PostgresQueueStore q) where
    assertUpdated $ withDB' "deleteStoreQueue" st $ \db ->
      DB.execute db "UPDATE msg_queues SET deleted_at = ? WHERE recipient_id = ? AND deleted_at IS NULL" (ts, rId)
    atomically $ writeTVar qr Nothing
-    when (useCache st) $ do
-      atomically $ TM.delete (senderId q) $ senders st
-      forM_ (notifier q) $ \NtfCreds {notifierId} -> do
-        atomically $ TM.delete notifierId $ notifiers st
-        atomically $ TM.delete notifierId $ notifierLocks st
    withLog "deleteStoreQueue" st (`logDeleteQueue` rId)
    pure q
    where
@@ -483,7 +350,7 @@ instance StoreQueueClass q => QueueStoreClass q (PostgresQueueStore q) where
      pure serviceId

  setQueueService :: (PartyI p, ServiceParty p) => PostgresQueueStore q -> q -> SParty p -> Maybe ServiceId -> IO (Either ErrorType ())
-  setQueueService st sq party serviceId = withQueueRec sq "setQueueService" $ \q -> case party of
+  setQueueService st sq party serviceId = withQueueRec sq $ \q -> case party of
    SRecipientService
      | rcvServiceId q == serviceId -> pure ()
      | otherwise -> do
@@ -608,33 +475,8 @@ foldServiceRecs st f =
    DB.fold_ db "SELECT service_id, service_role, service_cert, service_cert_hash, created_at FROM services" mempty $
      \ !acc -> fmap (acc <>) . f . rowToServiceRec

-foldRcvServiceQueueRecs :: PostgresQueueStore q -> ServiceId -> (a -> (RecipientId, QueueRec) -> IO a) -> a -> IO (Either ErrorType a)
-foldRcvServiceQueueRecs st serviceId f acc =
-  runExceptT $ withDB' "foldRcvServiceQueueRecs" st $ \db ->
-    DB.fold db (queueRecQuery <> " WHERE rcv_service_id = ? AND deleted_at IS NULL") (Only serviceId) acc $ \a -> f a . rowToQueueRec
-
 foldQueueRecs :: Monoid a => Bool -> Bool -> PostgresQueueStore q -> ((RecipientId, QueueRec) -> IO a) -> IO a
-foldQueueRecs withData = foldQueueRecs_ foldRecs
-  where
-    foldRecs db acc f'
-      | withData = DB.fold_ db (queueRecQueryWithData <> cond) acc $ \acc' -> f' acc' . rowToQueueRecWithData
-      | otherwise = DB.fold_ db (queueRecQuery <> cond) acc $ \acc' -> f' acc' . rowToQueueRec
-    cond = " WHERE deleted_at IS NULL ORDER BY recipient_id ASC"
-
-foldRecentQueueRecs :: Monoid a => Int64 -> Bool -> PostgresQueueStore q -> ((RecipientId, QueueRec) -> IO a) -> IO a
-foldRecentQueueRecs old = foldQueueRecs_ foldRecs
-  where
-    foldRecs db acc f' = DB.fold db (queueRecQuery <> cond) (Only old) acc $ \acc' -> f' acc' . rowToQueueRec
-    cond = " WHERE deleted_at IS NULL AND updated_at > ? ORDER BY recipient_id ASC"
-
-foldQueueRecs_ ::
-  Monoid a =>
-  (DB.Connection -> (Int, a) -> ((Int, a) -> (RecipientId, QueueRec) -> IO (Int, a)) -> IO (Int, a)) ->
-  Bool ->
-  PostgresQueueStore q ->
-  ((RecipientId, QueueRec) -> IO a) ->
-  IO a
-foldQueueRecs_ foldRecs tty st f = do
+foldQueueRecs withData tty st f = do
  (n, r) <- withTransaction (dbStore st) $ \db ->
    foldRecs db (0 :: Int, mempty) $ \(i, acc) qr -> do
      r <- f qr
@@ -645,6 +487,10 @@ foldQueueRecs_ foldRecs tty st f = do
  when tty $ putStrLn $ progress n
  pure r
  where
+    foldRecs db acc f'
+      | withData = DB.fold_ db (queueRecQueryWithData <> cond) acc $ \acc' -> f' acc' . rowToQueueRecWithData
+      | otherwise = DB.fold_ db (queueRecQuery <> cond) acc $ \acc' -> f' acc' . rowToQueueRec
+    cond = " WHERE deleted_at IS NULL ORDER BY recipient_id ASC"
    progress i = "Processed: " <> show i <> " records"

 queueRecQuery :: Query
@@ -750,15 +596,15 @@ rowToServiceRec (serviceId, serviceRole, serviceCert, Binary fp, serviceCreatedA

 setStatusDB :: StoreQueueClass q => Text -> PostgresQueueStore q -> q -> ServerEntityStatus -> ExceptT ErrorType IO () -> IO (Either ErrorType ())
 setStatusDB op st sq status writeLog =
-  withQueueRec sq op $ \q -> do
+  withQueueRec sq $ \q -> do
    assertUpdated $ withDB' op st $ \db ->
      DB.execute db "UPDATE msg_queues SET status = ? WHERE recipient_id = ? AND deleted_at IS NULL" (status, recipientId sq)
    atomically $ writeTVar (queueRec sq) $ Just q {status}
    writeLog

-withQueueRec :: StoreQueueClass q => q -> Text -> (QueueRec -> ExceptT ErrorType IO a) -> IO (Either ErrorType a)
-withQueueRec sq op action =
-  withQueueLock sq op $ E.uninterruptibleMask_ $ runExceptT $ ExceptT (readQueueRecIO $ queueRec sq) >>= action
+withQueueRec :: StoreQueueClass q => q -> (QueueRec -> ExceptT ErrorType IO a) -> IO (Either ErrorType a)
+withQueueRec sq action =
+  E.uninterruptibleMask_ $ runExceptT $ ExceptT (readQueueRecIO $ queueRec sq) >>= action

 assertUpdated :: ExceptT ErrorType IO Int64 -> ExceptT ErrorType IO ()
 assertUpdated = (>>= \n -> when (n == 0) (throwE AUTH))
@@ -22,7 +22,8 @@ serverSchemaMigrations =
    ("20250903_store_messages", m20250903_store_messages, Just down_m20250903_store_messages),
    ("20250915_queue_ids_hash", m20250915_queue_ids_hash, Just down_m20250915_queue_ids_hash),
    ("20260916_prometheus_indexes", m20260916_prometheus_indexes, Just down_m20260916_prometheus_indexes),
-    ("20260917_msg_queues_hot", m20260917_msg_queues_hot, Just down_m20260917_msg_queues_hot)
+    ("20260917_msg_queues_hot", m20260917_msg_queues_hot, Just down_m20260917_msg_queues_hot),
+    ("20260918_expire_messages", m20260918_expire_messages, Just down_m20260918_expire_messages)
  ]

 -- | The list of migrations in ascending order by date
@@ -734,3 +735,504 @@ ALTER TABLE msg_queues RESET (fillfactor, autovacuum_vacuum_scale_factor, autova
 ALTER TABLE messages RESET (autovacuum_vacuum_scale_factor, autovacuum_analyze_scale_factor, toast.autovacuum_vacuum_scale_factor);
 ALTER TABLE services RESET (fillfactor, autovacuum_vacuum_threshold, autovacuum_vacuum_scale_factor);
    |]
+
+m20260918_expire_messages :: Text
+m20260918_expire_messages =
+  [r|
+CREATE INDEX idx_messages_expire ON messages (msg_ts, recipient_id) WHERE NOT msg_quota;
+
+DROP INDEX idx_messages_recipient_id_msg_ts;
+
+DROP INDEX idx_msg_queues_expire;
+
+DROP PROCEDURE expire_old_messages(bigint, integer);
+
+CREATE PROCEDURE expire_old_messages(IN p_old_ts bigint, IN batch_size integer, OUT r_expired_msgs_count bigint, OUT r_stored_msgs_count bigint, OUT r_stored_queues bigint)
+    LANGUAGE plpgsql
+    AS $$
+DECLARE
+  rids BYTEA[];
+  rid BYTEA;
+  last_ts BIGINT := -1;
+  last_rid BYTEA := '\x';
+  next_ts BIGINT;
+  next_rid BYTEA;
+  del_count BIGINT;
+  total_deleted BIGINT := 0;
+BEGIN
+  LOOP
+    -- The page is scanned in (msg_ts, recipient_id) order, so its last row is the next
+    -- keyset cursor. Advancing it past every row read, including queues left unexpired
+    -- because delete_expired_msgs skipped a locked row or raised, is what terminates the
+    -- loop; re-reading from the start instead would repeat those queues forever.
+    SELECT array_agg(DISTINCT recipient_id),
+           (array_agg(msg_ts ORDER BY msg_ts DESC, recipient_id DESC))[1],
+           (array_agg(recipient_id ORDER BY msg_ts DESC, recipient_id DESC))[1]
+    INTO rids, next_ts, next_rid
+    FROM (
+      SELECT msg_ts, recipient_id
+      FROM messages
+      WHERE NOT msg_quota
+        AND msg_ts < p_old_ts
+        AND (msg_ts, recipient_id) > (last_ts, last_rid)
+      ORDER BY msg_ts ASC, recipient_id ASC
+      LIMIT batch_size
+    ) m;
+
+    EXIT WHEN rids IS NULL;
+
+    FOREACH rid IN ARRAY rids
+    LOOP
+      BEGIN
+        del_count := delete_expired_msgs(rid, p_old_ts);
+        total_deleted := total_deleted + del_count;
+      EXCEPTION WHEN OTHERS THEN
+        RAISE WARNING 'STORE, expire_old_messages, error expiring queue %: %', encode(rid, 'base64'), SQLERRM;
+        CONTINUE;
+      END;
+    END LOOP;
+    COMMIT;
+
+    last_ts := next_ts;
+    last_rid := next_rid;
+  END LOOP;
+
+  r_expired_msgs_count := total_deleted;
+  r_stored_msgs_count := (SELECT COUNT(1) FROM messages);
+  r_stored_queues := (SELECT COUNT(1) FROM msg_queues WHERE deleted_at IS NULL);
+END;
+$$;
+
+CREATE OR REPLACE FUNCTION delete_expired_msgs(p_recipient_id bytea, p_old_ts bigint) RETURNS bigint
+    LANGUAGE plpgsql
+    AS $$
+DECLARE
+  q_size BIGINT;
+  keep_min_id BIGINT;
+  del_count BIGINT;
+BEGIN
+  SELECT msg_queue_size INTO q_size
+  FROM msg_queues
+  WHERE recipient_id = p_recipient_id AND deleted_at IS NULL
+  FOR UPDATE SKIP LOCKED;
+
+  IF NOT FOUND OR q_size = 0 THEN
+    RETURN 0;
+  END IF;
+
+  SELECT MIN(message_id) INTO keep_min_id
+  FROM messages WHERE recipient_id = p_recipient_id AND msg_ts >= p_old_ts AND msg_quota = FALSE;
+
+  IF keep_min_id IS NULL THEN
+    DELETE FROM messages WHERE recipient_id = p_recipient_id AND msg_quota = FALSE;
+  ELSE
+    DELETE FROM messages WHERE recipient_id = p_recipient_id AND message_id < keep_min_id AND msg_quota = FALSE;
+  END IF;
+
+  GET DIAGNOSTICS del_count = ROW_COUNT;
+  IF del_count > 0 THEN
+    UPDATE msg_queues
+    SET msg_can_write = msg_can_write OR msg_queue_size <= del_count,
+        msg_queue_size = GREATEST(msg_queue_size - del_count, 0)
+    WHERE recipient_id = p_recipient_id;
+  END IF;
+  RETURN del_count;
+END;
+$$;
+
+CREATE OR REPLACE FUNCTION write_message(p_recipient_id bytea, p_msg_id bytea, p_msg_ts bigint, p_msg_quota boolean, p_msg_ntf_flag boolean, p_msg_body bytea, p_quota integer) RETURNS TABLE(quota_written boolean, was_empty boolean)
+    LANGUAGE plpgsql
+    AS $$
+DECLARE
+  q_can_write BOOLEAN;
+  q_size BIGINT;
+BEGIN
+  SELECT msg_can_write, msg_queue_size INTO q_can_write, q_size
+  FROM msg_queues
+  WHERE recipient_id = p_recipient_id AND deleted_at IS NULL
+  FOR UPDATE;
+
+  IF q_can_write OR q_size = 0 THEN
+    quota_written := p_msg_quota OR q_size >= p_quota;
+    was_empty := q_size = 0;
+
+    INSERT INTO messages(recipient_id, msg_id, msg_ts, msg_quota, msg_ntf_flag, msg_body)
+    VALUES (p_recipient_id, p_msg_id, p_msg_ts, quota_written, p_msg_ntf_flag AND NOT quota_written, CASE WHEN quota_written THEN '' :: BYTEA ELSE p_msg_body END);
+
+    UPDATE msg_queues
+    SET msg_can_write = NOT quota_written,
+        msg_queue_size = msg_queue_size + 1
+    WHERE recipient_id = p_recipient_id;
+
+    RETURN QUERY VALUES (quota_written, was_empty);
+  END IF;
+END;
+$$;
+
+CREATE OR REPLACE FUNCTION try_del_msg(p_recipient_id bytea, p_msg_id bytea) RETURNS TABLE(r_msg_id bytea, r_msg_ts bigint, r_msg_quota boolean, r_msg_ntf_flag boolean, r_msg_body bytea)
+    LANGUAGE plpgsql
+    AS $$
+DECLARE
+  q_size BIGINT;
+  msg RECORD;
+BEGIN
+  SELECT msg_queue_size INTO q_size
+  FROM msg_queues
+  WHERE recipient_id = p_recipient_id AND deleted_at IS NULL
+  FOR UPDATE;
+
+  IF NOT FOUND THEN
+    RETURN;
+  END IF;
+
+  SELECT message_id, msg_id, msg_ts, msg_quota, msg_ntf_flag, msg_body
+  INTO msg
+  FROM messages
+  WHERE recipient_id = p_recipient_id
+  ORDER BY message_id ASC LIMIT 1;
+
+  IF NOT FOUND THEN
+    IF q_size != 0 THEN
+      UPDATE msg_queues
+      SET msg_can_write = TRUE,
+          msg_queue_size = 0
+      WHERE recipient_id = p_recipient_id;
+    END IF;
+    RETURN;
+  END IF;
+
+  IF msg.msg_id = p_msg_id THEN
+    DELETE FROM messages WHERE message_id = msg.message_id;
+    IF FOUND THEN
+      UPDATE msg_queues
+      SET msg_can_write = msg_can_write OR msg_queue_size <= 1,
+          msg_queue_size = GREATEST(msg_queue_size - 1, 0)
+      WHERE recipient_id = p_recipient_id;
+      RETURN QUERY VALUES (msg.msg_id, msg.msg_ts, msg.msg_quota, msg.msg_ntf_flag, msg.msg_body);
+    END IF;
+  END IF;
+END;
+$$;
+
+CREATE OR REPLACE FUNCTION try_del_peek_msg(p_recipient_id bytea, p_msg_id bytea) RETURNS TABLE(r_msg_id bytea, r_msg_ts bigint, r_msg_quota boolean, r_msg_ntf_flag boolean, r_msg_body bytea)
+    LANGUAGE plpgsql
+    AS $$
+DECLARE
+  q_size BIGINT;
+  msg RECORD;
+  msg_deleted BOOLEAN;
+BEGIN
+  SELECT msg_queue_size INTO q_size
+  FROM msg_queues
+  WHERE recipient_id = p_recipient_id AND deleted_at IS NULL
+  FOR UPDATE;
+
+  IF NOT FOUND THEN
+    RETURN;
+  END IF;
+
+  SELECT message_id, msg_id, msg_ts, msg_quota, msg_ntf_flag, msg_body
+  INTO msg
+  FROM messages
+  WHERE recipient_id = p_recipient_id
+  ORDER BY message_id ASC LIMIT 1;
+
+  IF NOT FOUND THEN
+    IF q_size != 0 THEN
+      UPDATE msg_queues
+      SET msg_can_write = TRUE,
+          msg_queue_size = 0
+      WHERE recipient_id = p_recipient_id;
+    END IF;
+    RETURN;
+  END IF;
+
+  IF msg.msg_id = p_msg_id THEN
+    DELETE FROM messages WHERE message_id = msg.message_id;
+
+    msg_deleted := FOUND;
+    IF msg_deleted THEN
+      RETURN QUERY VALUES (msg.msg_id, msg.msg_ts, msg.msg_quota, msg.msg_ntf_flag, msg.msg_body);
+    END IF;
+
+    SELECT msg_id, msg_ts, msg_quota, msg_ntf_flag, msg_body
+    INTO msg
+    FROM messages
+    WHERE recipient_id = p_recipient_id
+    ORDER BY message_id ASC LIMIT 1;
+
+    IF FOUND THEN
+      RETURN QUERY VALUES (msg.msg_id, msg.msg_ts, msg.msg_quota, msg.msg_ntf_flag, msg.msg_body);
+      IF msg_deleted THEN
+        UPDATE msg_queues
+        SET msg_can_write = msg_can_write OR msg_queue_size <= 1,
+            msg_queue_size = GREATEST(msg_queue_size - 1, 0)
+        WHERE recipient_id = p_recipient_id;
+      END IF;
+    ELSIF msg_deleted OR q_size != 0 THEN
+      UPDATE msg_queues
+      SET msg_can_write = TRUE,
+          msg_queue_size = 0
+      WHERE recipient_id = p_recipient_id;
+    END IF;
+  ELSE
+    RETURN QUERY VALUES (msg.msg_id, msg.msg_ts, msg.msg_quota, msg.msg_ntf_flag, msg.msg_body);
+  END IF;
+END;
+$$;
+
+ALTER TABLE msg_queues DROP COLUMN msg_queue_expire;
+    |]
+
+down_m20260918_expire_messages :: Text
+down_m20260918_expire_messages =
+  [r|
+ALTER TABLE msg_queues ADD COLUMN msg_queue_expire boolean NOT NULL DEFAULT FALSE;
+
+-- ADD COLUMN already defaulted every row to FALSE, so the backfill only needs the rows
+-- that become TRUE, which avoids rewriting the whole table. The flag is only read to pick
+-- queues for delete_expired_msgs, and that returns 0 without deleting when msg_queue_size
+-- is 0, so excluding empty queues here cannot change what expires.
+UPDATE msg_queues q
+SET msg_queue_expire = TRUE
+WHERE msg_queue_size > 0
+  AND EXISTS (SELECT 1 FROM messages m WHERE m.recipient_id = q.recipient_id AND NOT m.msg_quota);
+
+CREATE INDEX idx_msg_queues_expire ON msg_queues (recipient_id) WHERE deleted_at IS NULL AND msg_queue_expire;
+
+CREATE INDEX idx_messages_recipient_id_msg_ts ON messages (recipient_id, msg_ts);
+
+DROP INDEX idx_messages_expire;
+
+CREATE OR REPLACE FUNCTION delete_expired_msgs(p_recipient_id bytea, p_old_ts bigint) RETURNS bigint
+    LANGUAGE plpgsql
+    AS $$
+DECLARE
+  q_size BIGINT;
+  keep_min_id BIGINT;
+  del_count BIGINT;
+BEGIN
+  SELECT msg_queue_size INTO q_size
+  FROM msg_queues
+  WHERE recipient_id = p_recipient_id AND deleted_at IS NULL
+  FOR UPDATE SKIP LOCKED;
+
+  IF NOT FOUND OR q_size = 0 THEN
+    RETURN 0;
+  END IF;
+
+  SELECT MIN(message_id) INTO keep_min_id
+  FROM messages WHERE recipient_id = p_recipient_id AND msg_ts >= p_old_ts AND msg_quota = FALSE;
+
+  IF keep_min_id IS NULL THEN
+    DELETE FROM messages WHERE recipient_id = p_recipient_id AND msg_quota = FALSE;
+  ELSE
+    DELETE FROM messages WHERE recipient_id = p_recipient_id AND message_id < keep_min_id AND msg_quota = FALSE;
+  END IF;
+
+  GET DIAGNOSTICS del_count = ROW_COUNT;
+  IF del_count > 0 THEN
+    UPDATE msg_queues
+    SET msg_can_write = msg_can_write OR msg_queue_size <= del_count,
+        msg_queue_expire = msg_queue_size > del_count AND keep_min_id IS NOT NULL,
+        msg_queue_size = GREATEST(msg_queue_size - del_count, 0)
+    WHERE recipient_id = p_recipient_id;
+  END IF;
+  RETURN del_count;
+END;
+$$;
+
+CREATE OR REPLACE FUNCTION write_message(p_recipient_id bytea, p_msg_id bytea, p_msg_ts bigint, p_msg_quota boolean, p_msg_ntf_flag boolean, p_msg_body bytea, p_quota integer) RETURNS TABLE(quota_written boolean, was_empty boolean)
+    LANGUAGE plpgsql
+    AS $$
+DECLARE
+  q_can_write BOOLEAN;
+  q_size BIGINT;
+BEGIN
+  SELECT msg_can_write, msg_queue_size INTO q_can_write, q_size
+  FROM msg_queues
+  WHERE recipient_id = p_recipient_id AND deleted_at IS NULL
+  FOR UPDATE;
+
+  IF q_can_write OR q_size = 0 THEN
+    quota_written := p_msg_quota OR q_size >= p_quota;
+    was_empty := q_size = 0;
+
+    INSERT INTO messages(recipient_id, msg_id, msg_ts, msg_quota, msg_ntf_flag, msg_body)
+    VALUES (p_recipient_id, p_msg_id, p_msg_ts, quota_written, p_msg_ntf_flag AND NOT quota_written, CASE WHEN quota_written THEN '' :: BYTEA ELSE p_msg_body END);
+
+    UPDATE msg_queues
+    SET msg_can_write = NOT quota_written,
+        msg_queue_expire = TRUE,
+        msg_queue_size = msg_queue_size + 1
+    WHERE recipient_id = p_recipient_id;
+
+    RETURN QUERY VALUES (quota_written, was_empty);
+  END IF;
+END;
+$$;
+
+CREATE OR REPLACE FUNCTION try_del_msg(p_recipient_id bytea, p_msg_id bytea) RETURNS TABLE(r_msg_id bytea, r_msg_ts bigint, r_msg_quota boolean, r_msg_ntf_flag boolean, r_msg_body bytea)
+    LANGUAGE plpgsql
+    AS $$
+DECLARE
+  q_size BIGINT;
+  msg RECORD;
+BEGIN
+  SELECT msg_queue_size INTO q_size
+  FROM msg_queues
+  WHERE recipient_id = p_recipient_id AND deleted_at IS NULL
+  FOR UPDATE;
+
+  IF NOT FOUND THEN
+    RETURN;
+  END IF;
+
+  SELECT message_id, msg_id, msg_ts, msg_quota, msg_ntf_flag, msg_body
+  INTO msg
+  FROM messages
+  WHERE recipient_id = p_recipient_id
+  ORDER BY message_id ASC LIMIT 1;
+
+  IF NOT FOUND THEN
+    IF q_size != 0 THEN
+      UPDATE msg_queues
+      SET msg_can_write = TRUE,
+          msg_queue_expire = FALSE,
+          msg_queue_size = 0
+      WHERE recipient_id = p_recipient_id;
+    END IF;
+    RETURN;
+  END IF;
+
+  IF msg.msg_id = p_msg_id THEN
+    DELETE FROM messages WHERE message_id = msg.message_id;
+    IF FOUND THEN
+      UPDATE msg_queues
+      SET msg_can_write = msg_can_write OR msg_queue_size <= 1,
+          msg_queue_expire = msg_queue_size > 1,
+          msg_queue_size = GREATEST(msg_queue_size - 1, 0)
+      WHERE recipient_id = p_recipient_id;
+      RETURN QUERY VALUES (msg.msg_id, msg.msg_ts, msg.msg_quota, msg.msg_ntf_flag, msg.msg_body);
+    END IF;
+  END IF;
+END;
+$$;
+
+CREATE OR REPLACE FUNCTION try_del_peek_msg(p_recipient_id bytea, p_msg_id bytea) RETURNS TABLE(r_msg_id bytea, r_msg_ts bigint, r_msg_quota boolean, r_msg_ntf_flag boolean, r_msg_body bytea)
+    LANGUAGE plpgsql
+    AS $$
+DECLARE
+  q_size BIGINT;
+  msg RECORD;
+  msg_deleted BOOLEAN;
+BEGIN
+  SELECT msg_queue_size INTO q_size
+  FROM msg_queues
+  WHERE recipient_id = p_recipient_id AND deleted_at IS NULL
+  FOR UPDATE;
+
+  IF NOT FOUND THEN
+    RETURN;
+  END IF;
+
+  SELECT message_id, msg_id, msg_ts, msg_quota, msg_ntf_flag, msg_body
+  INTO msg
+  FROM messages
+  WHERE recipient_id = p_recipient_id
+  ORDER BY message_id ASC LIMIT 1;
+
+  IF NOT FOUND THEN
+    IF q_size != 0 THEN
+      UPDATE msg_queues
+      SET msg_can_write = TRUE,
+          msg_queue_expire = FALSE,
+          msg_queue_size = 0
+      WHERE recipient_id = p_recipient_id;
+    END IF;
+    RETURN;
+  END IF;
+
+  IF msg.msg_id = p_msg_id THEN
+    DELETE FROM messages WHERE message_id = msg.message_id;
+
+    msg_deleted := FOUND;
+    IF msg_deleted THEN
+      RETURN QUERY VALUES (msg.msg_id, msg.msg_ts, msg.msg_quota, msg.msg_ntf_flag, msg.msg_body);
+    END IF;
+
+    SELECT msg_id, msg_ts, msg_quota, msg_ntf_flag, msg_body
+    INTO msg
+    FROM messages
+    WHERE recipient_id = p_recipient_id
+    ORDER BY message_id ASC LIMIT 1;
+
+    IF FOUND THEN
+      RETURN QUERY VALUES (msg.msg_id, msg.msg_ts, msg.msg_quota, msg.msg_ntf_flag, msg.msg_body);
+      IF msg_deleted THEN
+        UPDATE msg_queues
+        SET msg_can_write = msg_can_write OR msg_queue_size <= 1,
+            msg_queue_expire = msg_queue_size > 1,
+            msg_queue_size = GREATEST(msg_queue_size - 1, 0)
+        WHERE recipient_id = p_recipient_id;
+      END IF;
+    ELSIF msg_deleted OR q_size != 0 THEN
+      UPDATE msg_queues
+      SET msg_can_write = TRUE,
+          msg_queue_expire = FALSE,
+          msg_queue_size = 0
+      WHERE recipient_id = p_recipient_id;
+    END IF;
+  ELSE
+    RETURN QUERY VALUES (msg.msg_id, msg.msg_ts, msg.msg_quota, msg.msg_ntf_flag, msg.msg_body);
+  END IF;
+END;
+$$;
+
+DROP PROCEDURE expire_old_messages(bigint, integer);
+
+CREATE PROCEDURE expire_old_messages(IN p_old_ts bigint, IN batch_size integer, OUT r_expired_msgs_count bigint, OUT r_stored_msgs_count bigint, OUT r_stored_queues bigint)
+    LANGUAGE plpgsql
+    AS $$
+DECLARE
+  rids BYTEA[];
+  rid BYTEA;
+  last_rid BYTEA := '\x';
+  del_count BIGINT;
+  total_deleted BIGINT := 0;
+BEGIN
+  LOOP
+    SELECT array_agg(recipient_id)
+    INTO rids
+    FROM (
+      SELECT recipient_id
+      FROM msg_queues
+      WHERE deleted_at IS NULL
+        AND msg_queue_expire = TRUE
+        AND recipient_id > last_rid
+      ORDER BY recipient_id ASC
+      LIMIT batch_size
+    ) qs;
+
+    EXIT WHEN rids IS NULL OR cardinality(rids) = 0;
+
+    FOREACH rid IN ARRAY rids
+    LOOP
+      BEGIN
+        del_count := delete_expired_msgs(rid, p_old_ts);
+        total_deleted := total_deleted + del_count;
+      EXCEPTION WHEN OTHERS THEN
+        RAISE WARNING 'STORE, expire_old_messages, error expiring queue %: %', encode(rid, 'base64'), SQLERRM;
+        CONTINUE;
+      END;
+      COMMIT;
+    END LOOP;
+    last_rid := rids[cardinality(rids)];
+  END LOOP;
+
+  r_expired_msgs_count := total_deleted;
+  r_stored_msgs_count := (SELECT COUNT(1) FROM messages);
+  r_stored_queues := (SELECT COUNT(1) FROM msg_queues WHERE deleted_at IS NULL);
+END;
+$$;
+    |]
@@ -47,7 +47,6 @@ BEGIN
  IF del_count > 0 THEN
    UPDATE msg_queues
    SET msg_can_write = msg_can_write OR msg_queue_size <= del_count,
-        msg_queue_expire = msg_queue_size > del_count AND keep_min_id IS NOT NULL,
        msg_queue_size = GREATEST(msg_queue_size - del_count, 0)
    WHERE recipient_id = p_recipient_id;
  END IF;
@@ -63,24 +62,33 @@ CREATE PROCEDURE smp_server.expire_old_messages(IN p_old_ts bigint, IN batch_siz
 DECLARE
  rids BYTEA[];
  rid BYTEA;
+  last_ts BIGINT := -1;
  last_rid BYTEA := '\x';
+  next_ts BIGINT;
+  next_rid BYTEA;
  del_count BIGINT;
  total_deleted BIGINT := 0;
 BEGIN
  LOOP
-    SELECT array_agg(recipient_id)
-    INTO rids
+    -- The page is scanned in (msg_ts, recipient_id) order, so its last row is the next
+    -- keyset cursor. Advancing it past every row read, including queues left unexpired
+    -- because delete_expired_msgs skipped a locked row or raised, is what terminates the
+    -- loop; re-reading from the start instead would repeat those queues forever.
+    SELECT array_agg(DISTINCT recipient_id),
+           (array_agg(msg_ts ORDER BY msg_ts DESC, recipient_id DESC))[1],
+           (array_agg(recipient_id ORDER BY msg_ts DESC, recipient_id DESC))[1]
+    INTO rids, next_ts, next_rid
    FROM (
-      SELECT recipient_id
-      FROM msg_queues
-      WHERE deleted_at IS NULL
-        AND msg_queue_expire = TRUE
-        AND recipient_id > last_rid
-      ORDER BY recipient_id ASC
+      SELECT msg_ts, recipient_id
+      FROM messages
+      WHERE NOT msg_quota
+        AND msg_ts < p_old_ts
+        AND (msg_ts, recipient_id) > (last_ts, last_rid)
+      ORDER BY msg_ts ASC, recipient_id ASC
      LIMIT batch_size
-    ) qs;
+    ) m;

-    EXIT WHEN rids IS NULL OR cardinality(rids) = 0;
+    EXIT WHEN rids IS NULL;

    FOREACH rid IN ARRAY rids
    LOOP
@@ -91,9 +99,11 @@ BEGIN
        RAISE WARNING 'STORE, expire_old_messages, error expiring queue %: %', encode(rid, 'base64'), SQLERRM;
        CONTINUE;
      END;
-      COMMIT;
    END LOOP;
-    last_rid := rids[cardinality(rids)];
+    COMMIT;
+
+    last_ts := next_ts;
+    last_rid := next_rid;
  END LOOP;

  r_expired_msgs_count := total_deleted;
@@ -195,7 +205,6 @@ BEGIN
    IF q_size != 0 THEN
      UPDATE msg_queues
      SET msg_can_write = TRUE,
-          msg_queue_expire = FALSE,
          msg_queue_size = 0
      WHERE recipient_id = p_recipient_id;
    END IF;
@@ -207,7 +216,6 @@ BEGIN
    IF FOUND THEN
      UPDATE msg_queues
      SET msg_can_write = msg_can_write OR msg_queue_size <= 1,
-          msg_queue_expire = msg_queue_size > 1,
          msg_queue_size = GREATEST(msg_queue_size - 1, 0)
      WHERE recipient_id = p_recipient_id;
      RETURN QUERY VALUES (msg.msg_id, msg.msg_ts, msg.msg_quota, msg.msg_ntf_flag, msg.msg_body);
@@ -245,7 +253,6 @@ BEGIN
    IF q_size != 0 THEN
      UPDATE msg_queues
      SET msg_can_write = TRUE,
-          msg_queue_expire = FALSE,
          msg_queue_size = 0
      WHERE recipient_id = p_recipient_id;
    END IF;
@@ -271,14 +278,12 @@ BEGIN
      IF msg_deleted THEN
        UPDATE msg_queues
        SET msg_can_write = msg_can_write OR msg_queue_size <= 1,
-            msg_queue_expire = msg_queue_size > 1,
            msg_queue_size = GREATEST(msg_queue_size - 1, 0)
        WHERE recipient_id = p_recipient_id;
      END IF;
    ELSIF msg_deleted OR q_size != 0 THEN
      UPDATE msg_queues
      SET msg_can_write = TRUE,
-          msg_queue_expire = FALSE,
          msg_queue_size = 0
      WHERE recipient_id = p_recipient_id;
    END IF;
@@ -348,7 +353,6 @@ BEGIN

    UPDATE msg_queues
    SET msg_can_write = NOT quota_written,
-        msg_queue_expire = TRUE,
        msg_queue_size = msg_queue_size + 1
    WHERE recipient_id = p_recipient_id;

@@ -440,7 +444,6 @@ CREATE TABLE smp_server.msg_queues (
    rcv_service_id bytea,
    ntf_service_id bytea,
    msg_can_write boolean DEFAULT true NOT NULL,
-    msg_queue_expire boolean DEFAULT false NOT NULL,
    msg_queue_size bigint DEFAULT 0 NOT NULL
 )
 WITH (fillfactor='80', autovacuum_vacuum_scale_factor='0.02', autovacuum_analyze_scale_factor='0.01', autovacuum_vacuum_cost_limit='1000');
@@ -485,6 +488,10 @@ ALTER TABLE ONLY smp_server.services



+CREATE INDEX idx_messages_expire ON smp_server.messages USING btree (msg_ts, recipient_id) WHERE (NOT msg_quota);
+
+
+
 CREATE INDEX idx_messages_recipient_id_message_id ON smp_server.messages USING btree (recipient_id, message_id);


@@ -493,14 +500,6 @@ CREATE INDEX idx_messages_recipient_id_msg_quota ON smp_server.messages USING bt



-CREATE INDEX idx_messages_recipient_id_msg_ts ON smp_server.messages USING btree (recipient_id, msg_ts);
-
-
-
-CREATE INDEX idx_msg_queues_expire ON smp_server.msg_queues USING btree (recipient_id) WHERE ((deleted_at IS NULL) AND msg_queue_expire);
-
-
-
 CREATE UNIQUE INDEX idx_msg_queues_link_id ON smp_server.msg_queues USING btree (link_id);


@@ -18,6 +18,7 @@ module Simplex.Messaging.Server.QueueStore.STM
  ( STMQueueStore (..),
    STMService (..),
    foldRcvServiceQueues,
+    withLoadedQueues,
    setStoreLog,
    withLog',
    readQueueRecIO,
@@ -91,8 +92,6 @@ instance StoreQueueClass q => QueueStoreClass q (STMQueueStore q) where
    atomically $ TM.clear senders
    atomically $ TM.clear notifiers

-  loadedQueues = queues
-  {-# INLINE loadedQueues #-}
  compactQueues _ = pure 0
  {-# INLINE compactQueues #-}

@@ -134,7 +133,7 @@ instance StoreQueueClass q => QueueStoreClass q (STMQueueStore q) where
      hasNotifier = maybe (pure False) (\NtfCreds {notifierId} -> TM.member notifierId notifiers) notifier
      hasLink = maybe (pure False) (\(lnkId, _) -> TM.member lnkId links) queueData

-  getQueue_ :: QueueParty p => STMQueueStore q -> (Bool -> RecipientId -> QueueRec -> IO q) -> SParty p -> QueueId -> IO (Either ErrorType q)
+  getQueue_ :: QueueParty p => STMQueueStore q -> (RecipientId -> QueueRec -> IO q) -> SParty p -> QueueId -> IO (Either ErrorType q)
  getQueue_ st _ party qId =
    maybe (Left AUTH) Right <$> case party of
      SRecipient -> TM.lookupIO qId queues
@@ -144,7 +143,7 @@ instance StoreQueueClass q => QueueStoreClass q (STMQueueStore q) where
    where
      STMQueueStore {queues, senders, notifiers, links} = st

-  getQueues_ :: BatchParty p => STMQueueStore q -> (Bool -> RecipientId -> QueueRec -> IO q) -> SParty p -> [QueueId] -> IO [Either ErrorType q]
+  getQueues_ :: BatchParty p => STMQueueStore q -> (RecipientId -> QueueRec -> IO q) -> SParty p -> [QueueId] -> IO [Either ErrorType q]
  getQueues_ st _ party qIds = case party of
    SRecipient -> do
      qs <- readTVarIO queues
@@ -173,6 +172,7 @@ instance StoreQueueClass q => QueueStoreClass q (STMQueueStore q) where
      rId = recipientId sq
      qr = queueRec sq
      add q = case queueData q of
+        _ | isSecuredMsgQueue q -> pure $ Left AUTH
        Nothing -> addLink
        Just (lnkId', d') | lnkId' == lnkId && fst d' == fst d -> addLink
        _ -> pure $ Left AUTH
@@ -277,9 +277,11 @@ instance StoreQueueClass q => QueueStoreClass q (STMQueueStore q) where
    where
      rId = recipientId sq
      qr = queueRec sq
-      delete q@QueueRec {senderId, rcvServiceId} = do
+      delete q@QueueRec {senderId, queueData, rcvServiceId} = do
        writeTVar qr Nothing
+        TM.delete rId $ queues st
        TM.delete senderId $ senders st
+        forM_ queueData $ \(lnkId, _) -> TM.delete lnkId $ links st
        mapM_ (removeServiceQueue st serviceRcvQueues rId) rcvServiceId
        mapM_ (removeNotifier st) $ notifier q
        pure q
@@ -378,6 +380,11 @@ foldRcvServiceQueues st serviceId f acc =
  where
    get rId = TM.lookupIO rId (queues st) $>>= \q -> (q,) <$$> readTVarIO (queueRec q)

+withLoadedQueues :: Monoid a => STMQueueStore q -> (q -> IO a) -> IO a
+withLoadedQueues st f = readTVarIO (queues st) >>= foldM run mempty
+  where
+    run !acc = fmap (acc <>) . f
+
 withQueueRec :: TVar (Maybe QueueRec) -> (QueueRec -> STM a) -> IO (Either ErrorType a)
 withQueueRec qr a = atomically $ readQueueRec qr >>= mapM a

@@ -1,5 +1,4 @@
 {-# LANGUAGE AllowAmbiguousTypes #-}
-{-# LANGUAGE BangPatterns #-}
 {-# LANGUAGE DataKinds #-}
 {-# LANGUAGE MultiParamTypeClasses #-}
 {-# LANGUAGE TypeFamilies #-}
@@ -9,35 +8,29 @@ module Simplex.Messaging.Server.QueueStore.Types
  ( StoreQueueClass (..),
    QueueStoreClass (..),
    EntityCounts (..),
-    withLoadedQueues,
  ) where

 import Control.Concurrent.STM
-import Control.Monad
 import Data.Int (Int64)
 import Data.List.NonEmpty (NonEmpty)
 import Data.Map.Strict (Map)
-import Data.Text (Text)
 import Simplex.Messaging.Protocol
 import Simplex.Messaging.Server.QueueStore
 import Simplex.Messaging.SystemTime
-import Simplex.Messaging.TMap (TMap)

 class StoreQueueClass q where
  recipientId :: q -> RecipientId
  queueRec :: q -> TVar (Maybe QueueRec)
-  withQueueLock :: q -> Text -> IO a -> IO a

 class StoreQueueClass q => QueueStoreClass q s where
  type QueueStoreCfg s
  newQueueStore :: QueueStoreCfg s -> IO s
  closeQueueStore :: s -> IO ()
  getEntityCounts :: s -> IO EntityCounts
-  loadedQueues :: s -> TMap RecipientId q
  compactQueues :: s -> IO Int64
  addQueue_ :: s -> (RecipientId -> QueueRec -> IO q) -> RecipientId -> QueueRec -> IO (Either ErrorType q)
-  getQueue_ :: QueueParty p => s -> (Bool -> RecipientId -> QueueRec -> IO q) -> SParty p -> QueueId -> IO (Either ErrorType q)
-  getQueues_ :: BatchParty p => s -> (Bool -> RecipientId -> QueueRec -> IO q) -> SParty p -> [QueueId] -> IO [Either ErrorType q]
+  getQueue_ :: QueueParty p => s -> (RecipientId -> QueueRec -> IO q) -> SParty p -> QueueId -> IO (Either ErrorType q)
+  getQueues_ :: BatchParty p => s -> (RecipientId -> QueueRec -> IO q) -> SParty p -> [QueueId] -> IO [Either ErrorType q]
  getQueueLinkData :: s -> q -> LinkId -> IO (Either ErrorType QueueLinkData)
  addQueueLinkData :: s -> q -> LinkId -> QueueLinkData -> IO (Either ErrorType ())
  deleteQueueLinkData :: s -> q -> IO (Either ErrorType ())
@@ -64,8 +57,3 @@ data EntityCounts = EntityCounts
    rcvServiceQueuesCount :: Int,
    ntfServiceQueuesCount :: Int
  }
-
-withLoadedQueues :: (Monoid a, QueueStoreClass q s) => s -> (q -> IO a) -> IO a
-withLoadedQueues st f = readTVarIO (loadedQueues st) >>= foldM run mempty
-  where
-    run !acc = fmap (acc <>) . f
@@ -14,19 +14,16 @@ module Simplex.Messaging.Server.StoreLog.ReadWrite

 import Control.Concurrent.STM
 import Control.Logger.Simple
-import Control.Monad
-import Control.Monad.IO.Class
-import Control.Monad.Trans.Except
 import qualified Data.ByteString.Char8 as B
 import qualified Data.Text as T
 import Data.Text.Encoding (decodeLatin1)
 import Simplex.Messaging.Encoding.String
 import Simplex.Messaging.Protocol (ASubscriberParty (..), ErrorType, RecipientId, SParty (..))
 import Simplex.Messaging.Server.QueueStore (QueueRec, ServiceRec (..))
-import Simplex.Messaging.Server.QueueStore.STM (STMQueueStore (..), STMService (..))
+import Simplex.Messaging.Server.QueueStore.STM (STMQueueStore (..), STMService (..), withLoadedQueues)
 import Simplex.Messaging.Server.QueueStore.Types
 import Simplex.Messaging.Server.StoreLog
-import Simplex.Messaging.Util (tshow)
+import Simplex.Messaging.Util (tshow, ($>>=))
 import System.IO

 writeQueueStore :: forall q. StoreQueueClass q => StoreLog 'WriteMode -> STMQueueStore q -> IO ()
@@ -68,13 +65,7 @@ readQueueStore tty mkQ f st = readLogLines tty f $ \_ -> processLine
        printError :: String -> IO ()
        printError e = B.putStrLn $ "Error parsing log: " <> B.pack e <> " - " <> s
        withQueue :: forall a. RecipientId -> T.Text -> (q -> IO (Either ErrorType a)) -> IO ()
-        withQueue qId op a = runExceptT go >>= qError qId op
-          where
-            go = do
-              q <- ExceptT $ getQueue_ st (\_ -> mkQ) SRecipient qId
-              liftIO (readTVarIO $ queueRec q) >>= \case
-                Nothing -> logWarn $ logPfx qId op <> "already deleted"
-                Just _ -> void $ ExceptT $ a q
+        withQueue qId op a = (getQueue_ st mkQ SRecipient qId $>>= a) >>= qError qId op
        qError qId op = \case
          Left e -> logError $ logPfx qId op <> tshow e
          Right _ -> pure ()
@@ -46,7 +46,6 @@ module Simplex.Messaging.Transport
    minServerSMPRelayVersion,
    currentClientSMPRelayVersion,
    currentServerSMPRelayVersion,
-    shortLinksSMPVersion,
    serviceCertsSMPVersion,
    newNtfCredsSMPVersion,
    clientNoticesSMPVersion,
@@ -54,6 +53,7 @@ module Simplex.Messaging.Transport
    namesSMPVersion,
    serverInfoSMPVersion,
    nameAvailSMPVersion,
+    fwdNoncesSMPVersion,
    simplexMQVersion,
    smpBlockSize,
    TransportConfig (..),
@@ -177,6 +177,7 @@ smpBlockSize = 16384
 -- 20 - public namespaces resolver, RSLV command (6/20/2026)
 -- 21 - server public information in handshake (7/5/2026)
 -- 22 - RNAME answers name availability as well as the record (7/25/2026)
+-- 23 - version in forwarded command nonce, random nonce in forwarded responses (10/2/2026)

 data SMPVersion

@@ -189,11 +190,8 @@ type VersionRangeSMP = VersionRange SMPVersion
 pattern VersionSMP :: Word16 -> VersionSMP
 pattern VersionSMP v = Version v

-_proxyServerHandshakeSMPVersion :: VersionSMP
-_proxyServerHandshakeSMPVersion = VersionSMP 14
-
-shortLinksSMPVersion :: VersionSMP
-shortLinksSMPVersion = VersionSMP 15
+_shortLinksSMPVersion :: VersionSMP
+_shortLinksSMPVersion = VersionSMP 15

 serviceCertsSMPVersion :: VersionSMP
 serviceCertsSMPVersion = VersionSMP 16
@@ -218,29 +216,32 @@ serverInfoSMPVersion = VersionSMP 21
 nameAvailSMPVersion :: VersionSMP
 nameAvailSMPVersion = VersionSMP 22

+fwdNoncesSMPVersion :: VersionSMP
+fwdNoncesSMPVersion = VersionSMP 23
+
 minClientSMPRelayVersion :: VersionSMP
-minClientSMPRelayVersion = VersionSMP 14
+minClientSMPRelayVersion = VersionSMP 15

 minServerSMPRelayVersion :: VersionSMP
-minServerSMPRelayVersion = VersionSMP 14
+minServerSMPRelayVersion = VersionSMP 15

 currentClientSMPRelayVersion :: VersionSMP
-currentClientSMPRelayVersion = VersionSMP 22
+currentClientSMPRelayVersion = VersionSMP 23

 currentServerSMPRelayVersion :: VersionSMP
-currentServerSMPRelayVersion = VersionSMP 22
+currentServerSMPRelayVersion = VersionSMP 23

 -- Max SMP protocol version to be used in e2e encrypted connection between
 -- client and server, as defined by SMP proxy. Normally set below the current
 -- version to prevent client version fingerprinting by the destination relays
-- when clients upgrade at different times. Pinned to the current version (22)
-- for this release because a proxied RSLV only carries availability from
-- nameAvailSMPVersion (22), so the one-version anti-fingerprinting buffer does
-- not apply yet; it reappears once the current version advances past 22.
+-- when clients upgrade at different times. Pinned to the current version (23)
+-- for this release because forwarded commands use the nonces from
+-- fwdNoncesSMPVersion (23), so the one-version anti-fingerprinting buffer does
+-- not apply yet; it reappears once the current version advances past 23.
 proxiedSMPRelayVersion :: VersionSMP
-proxiedSMPRelayVersion = VersionSMP 22
+proxiedSMPRelayVersion = VersionSMP 23

-- minimal supported protocol version is 14
+-- minimal supported protocol version is 15
 supportedClientSMPRelayVRange :: VersionRangeSMP
 supportedClientSMPRelayVRange = mkVersionRange minClientSMPRelayVersion currentClientSMPRelayVersion

@@ -865,7 +866,6 @@ forceCertChain :: CertChainPubKey -> CertChainPubKey
 forceCertChain cert@(CertChainPubKey (X.CertificateChain cc) signedKey) = length (show cc) `seq` show signedKey `seq` cert
 {-# INLINE forceCertChain #-}

-- This function is only used with v >= 8, so currently it's a simple record update.
 -- * Note: it requires updating version-based parameters, to be consistent with smpTHandle_.
 smpTHParamsSetVersion :: VersionSMP -> THandleParams SMPVersion p -> THandleParams SMPVersion p
 smpTHParamsSetVersion v params =
@@ -26,6 +26,7 @@ module Simplex.RemoteControl.Client
    -- for tests only
    sendRCPacket,
    receiveRCPacket,
+    findRCCtrlPairing,
  ) where

 import Control.Applicative ((<|>))
@@ -46,7 +47,7 @@ import Data.List.NonEmpty (NonEmpty (..))
 import qualified Data.List.NonEmpty as L
 import Data.Maybe (isNothing)
 import qualified Data.Text as T
-import Data.Time.Clock.System (getSystemTime)
+import Data.Time.Clock.System (SystemTime (..), getSystemTime)
 import Data.Tuple (swap)
 import Data.Word (Word16)
 import qualified Data.X509 as X
@@ -395,8 +396,10 @@ findRCCtrlPairing :: NonEmpty RCCtrlPairing -> RCEncInvitation -> ExceptT RCErro
 findRCCtrlPairing pairings RCEncInvitation {dhPubKey, nonce, encInvitation} = do
  (pairing, signedInvStr) <- liftEither $ decrypt (L.toList pairings)
  signedInv <- liftEitherWith RCESyntax $ strDecode signedInvStr
-  inv@(RCVerifiedInvitation RCInvitation {dh = invDh}) <- maybe (throwE RCEInvitation) pure $ verifySignedInvitation signedInv
+  inv@(RCVerifiedInvitation RCInvitation {dh = invDh, ts}) <- maybe (throwE RCEInvitation) pure $ verifySignedInvitation signedInv
  unless (invDh == dhPubKey) $ throwE RCEInvitation
+  now <- systemSeconds <$> liftIO getSystemTime
+  unless (now - 3660 <= systemSeconds ts && systemSeconds ts <= now + 3600) $ throwE RCEInvitation
  pure (pairing, inv)
  where
    decrypt :: [RCCtrlPairing] -> Either RCErrorType (RCCtrlPairing, ByteString)
@@ -122,18 +122,22 @@ closeListener subscribers sock =
 joinMulticast :: TMVar Int -> N.Socket -> N.HostAddress -> IO ()
 joinMulticast subscribers sock group = do
  now <- atomically $ takeTMVar subscribers
-  when (now == 0) $ do
-    setMembership sock group True >>= \case
-      Left e -> atomically (putTMVar subscribers now) >> logError ("setMembership failed " <> tshow e)
-      Right () -> atomically $ putTMVar subscribers (now + 1)
+  if now == 0
+    then
+      setMembership sock group True >>= \case
+        Left e -> atomically (putTMVar subscribers now) >> logError ("setMembership failed " <> tshow e)
+        Right () -> atomically $ putTMVar subscribers (now + 1)
+    else atomically $ putTMVar subscribers (now + 1)

 partMulticast :: TMVar Int -> N.Socket -> N.HostAddress -> IO ()
 partMulticast subscribers sock group = do
  now <- atomically $ takeTMVar subscribers
-  when (now == 1) $
-    setMembership sock group False >>= \case
-      Left e -> atomically (putTMVar subscribers now) >> logError ("setMembership failed " <> tshow e)
-      Right () -> atomically $ putTMVar subscribers (now - 1)
+  if now == 1
+    then
+      setMembership sock group False >>= \case
+        Left e -> atomically (putTMVar subscribers (now - 1)) >> logError ("setMembership failed " <> tshow e)
+        Right () -> atomically $ putTMVar subscribers (now - 1)
+    else atomically $ putTMVar subscribers (max 0 (now - 1))

 listenerHostAddr4 :: UDP.ListenSocket -> N.HostAddress
 listenerHostAddr4 sock = case UDP.mySockAddr sock of
@@ -20,6 +20,8 @@ module AgentTests.ConnectionRequestTests

 import AgentTests.EqInstances ()
 import Data.ByteString (ByteString)
+import qualified Data.ByteString.Char8 as B
+import Data.Either (isLeft)
 import Network.HTTP.Types (urlEncode)
 import Simplex.Messaging.Agent.Protocol
 import qualified Simplex.Messaging.Crypto as C
@@ -285,6 +287,9 @@ connectionRequestTests =
      contactAddressV6 #== ("https://simplex.chat/contact#/?v=1-2&smp=" <> url queueStr) -- adjusted to v6
      contactAddressV6 #== ("https://simplex.chat/contact#/?v=2-2&smp=" <> url queueStr)
      contactAddressClientData #==# ("simplex:/contact#/?v=6-8&smp=" <> url queueStr <> "&data=" <> url "{\"type\":\"group_link\", \"group_link_id\":\"abc\"}")
+    it "should reject KEM ciphertext without KEM key in e2e params" $
+      strDecode @(RcvE2ERatchetParamsUri 'C.X448) (strEncode testE2ERatchetParams <> "&kem_ct=" <> strEncode (B.replicate 1039 '\0'))
+        `shouldSatisfy` isLeft
    it "should serialize / parse queue address, connection invitations and contact addresses as binary" $ do
      smpEncodingTest queue
      smpEncodingTest queueNoQM -- this passes, no queue mode patch in SMPQueueUri encoding
@@ -357,6 +362,10 @@ connectionRequestTests =
      smpEncodingTest $ AgentServiceRequest [qInfo] Nothing "service request payload"
      smpEncodingTest $ AgentServiceResponse "service response payload"
      smpEncodingTest $ AgentRejection "rejected: not allowed"
+    it "should serialize and parse ratchet key info" $ do
+      smpDecode "R" `shouldBe` Right (AgentRatchetInfo RatchetInfo {answeredKeyHash = Nothing})
+      smpDecode "R1\3abcdef" `shouldBe` Right (AgentRatchetInfo RatchetInfo {answeredKeyHash = Just "abc"})
+      smpEncodingTest $ AgentRatchetInfo RatchetInfo {answeredKeyHash = Just "0123456789abcdef0123456789abcdef"}
  where
    smpEncodingTest :: (Encoding a, Eq a, Show a, HasCallStack) => a -> Expectation
    smpEncodingTest a = smpDecode (smpEncode a) `shouldBe` Right a
@@ -70,8 +70,8 @@ import Data.ByteString.Char8 (ByteString)
 import qualified Data.ByteString.Char8 as B
 import Data.Either (isRight)
 import Data.Int (Int64)
-import Data.List (find, isPrefixOf, isSuffixOf)
-import Data.List.NonEmpty (NonEmpty)
+import Data.List (find, isInfixOf, isPrefixOf, isSuffixOf)
+import Data.List.NonEmpty (NonEmpty (..))
 import qualified Data.Map as M
 import Data.Maybe (isJust, isNothing)
 import qualified Data.Set as S
@@ -87,12 +87,12 @@ import SMPAgentClient
 import SMPClient
 import Simplex.Messaging.Agent hiding (acceptContact, createConnection, deleteConnection, deleteConnections, getConnShortLink, joinConnection, sendMessage, setConnShortLink, subscribeConnection, suspendConnection)
 import qualified Simplex.Messaging.Agent as A
-import Simplex.Messaging.Agent.Client (ProtocolTestFailure (..), ProtocolTestStep (..), ServerQueueInfo (..), UserNetworkInfo (..), UserNetworkType (..), waitForUserNetwork)
+import Simplex.Messaging.Agent.Client (ProtocolTestFailure (..), ProtocolTestStep (..), ServerQueueInfo (..), UserNetworkInfo (..), UserNetworkType (..), sendAgentMessage, waitForUserNetwork)
 import Simplex.Messaging.Agent.Env.SQLite (AgentConfig (..), Env (..), InitialAgentServers (..), createAgentStore)
 import Simplex.Messaging.Agent.Protocol hiding (CON, CONF, INFO, REQ, SENT)
 import qualified Simplex.Messaging.Agent.Protocol as A
 import Simplex.Messaging.Agent.Store (Connection' (..), SomeConn' (..), StoredRcvQueue (..))
-import Simplex.Messaging.Agent.Store.AgentStore (getConn)
+import Simplex.Messaging.Agent.Store.AgentStore (deleteRatchetKeyHashesExpired, getConn, getRatchetX3dhKeys)
 import Simplex.Messaging.Agent.Store.Common (DBStore (..), withTransaction)
 import Simplex.Messaging.Agent.Store.Interface
 import qualified Simplex.Messaging.Agent.Store.DB as DB
@@ -112,7 +112,7 @@ import Simplex.Messaging.Server.Information (ServerPublicInfo (..))
 import Simplex.Messaging.Server.MsgStore.Types (SMSType (..), SQSType (..))
 import Simplex.Messaging.Server.QueueStore.QueueInfo
 import Simplex.Messaging.Server.StoreLog (StoreLogRecord (..))
-import Simplex.Messaging.Transport (ASrvTransport, SMPVersion, VersionSMP, currentServerSMPRelayVersion, minClientSMPRelayVersion, minServerSMPRelayVersion, alpnSupportedSMPHandshakes, supportedServerSMPRelayVRange)
+import Simplex.Messaging.Transport (ASrvTransport, SMPVersion, VersionSMP, currentServerSMPRelayVersion, minClientSMPRelayVersion, minServerSMPRelayVersion, alpnSupportedSMPHandshakes)
 import Simplex.Messaging.Transport.Server (TransportServerConfig (..))
 import Simplex.Messaging.Util (bshow, diffToMicroseconds)
 import Simplex.Messaging.Version (VersionRange (..))
@@ -133,9 +133,7 @@ import Simplex.Messaging.Agent.Store.AgentStore (deleteClientService, getSubscri
 import qualified Database.PostgreSQL.Simple as PSQL
 import qualified Simplex.Messaging.Agent.Store.Postgres as Postgres
 import qualified Simplex.Messaging.Agent.Store.Postgres.Common as Postgres
-import Simplex.Messaging.Server.MsgStore.Journal (JournalQueue)
 import Simplex.Messaging.Server.MsgStore.Postgres (PostgresQueue)
-import Simplex.Messaging.Server.MsgStore.Types (QSType (..))
 import Simplex.Messaging.Server.QueueStore.Postgres
 import Simplex.Messaging.Server.QueueStore.Postgres.Migrations
 import Simplex.Messaging.Server.QueueStore.Types (QueueStoreClass (..))
@@ -231,7 +229,11 @@ pattern Rcvd' :: AgentMsgId -> AgentMsgId -> AEvent 'AEConn
 pattern Rcvd' aMsgId rcvdMsgId <- RCVD MsgMeta {integrity = MsgOk, recipient = (aMsgId, _)} [MsgReceipt {agentMsgId = rcvdMsgId, msgRcptStatus = MROk}]

 smpCfgVPrev :: ProtocolClientConfig SMPVersion
-smpCfgVPrev = (smpCfg agentCfg) {serverVRange = prevRange $ serverVRange $ smpCfg agentCfg}
+smpCfgVPrev =
+  (smpCfg agentCfg)
+    { serverVRange = prevRange $ serverVRange $ smpCfg agentCfg,
+      proxiedRelayVRange = prevRange $ proxiedRelayVRange $ smpCfg agentCfg
+    }

 -- ntfCfgVPrev :: ProtocolClientConfig NTFVersion
 -- ntfCfgVPrev = (ntfCfg agentCfg) {clientALPN = Nothing, serverVRange = V.mkVersionRange (VersionNTF 1) (VersionNTF 1)}
@@ -411,7 +413,6 @@ functionalAPITests ps = do
    describe "should connect via 1-time short link with async join" $ testProxyMatrix ps testInvitationShortLinkAsync
    describe "should connect via contact short link" $ testProxyMatrix ps testContactShortLink
    describe "should add short link to existing contact and connect" $ testProxyMatrix ps testAddContactShortLink
-    xdescribe "try to create 1-time short link with prev versions" $ testProxyMatrixWithPrev ps testInvitationShortLinkPrev
    describe "server restart" $ do
      it "should get 1-time link data after restart" $ testInvitationShortLinkRestart ps
      it "should connect via contact short link after restart" $ testContactShortLinkRestart ps
@@ -459,6 +460,16 @@ functionalAPITests ps = do
        testRatchetSyncSuspendForeground ps
      it "should synchronize ratchets when clients start synchronization simultaneously" $
        testRatchetSyncSimultaneous ps
+      it "should ignore replayed ratchet key after expired hashes are deleted" $
+        testRatchetSyncReplayedKey ps
+      it "should synchronize ratchets when synchronization is forced again" $
+        testRatchetSyncRepeated ps
+      it "should not mark ratchet key as processed when ratchet recreation fails" $
+        testRatchetSyncFailedKeyNotProcessed ps
+      it "should not store reply ratchet key when ratchet recreation fails" $
+        testRatchetSyncFailedRecreationNoReply ps
+      it "should not store ratchet key when starting synchronization fails" $
+        testRatchetSyncStartFailedNoKey ps
 #endif
    describe "Subscription mode OnlyCreate" $ do
      it "messages delivered only when polled" $
@@ -663,19 +674,6 @@ testProxyMatrix ps runTest = do
  it "2 servers, directly" $ withSmpServers2 ps $ withAgentClientsServers2 (agentCfg, initAgentServers) (agentCfg, initAgentServers2) $ runTest False
  it "2 servers, via proxy" $ withSmpServersProxy2 ps $ withAgentClientsServers2 (agentCfg, initAgentServersProxy) (agentCfg, initAgentServersProxy2) $ runTest True

-testProxyMatrixWithPrev :: HasCallStack => (ASrvTransport, AStoreType) -> (Bool -> Bool -> AgentClient -> AgentClient -> IO ()) -> Spec
-testProxyMatrixWithPrev ps@(t, msType@(ASType qs _ms)) runTest = do
-  it "2 servers, directly, curr clients, prev servers" $ withSmpServers2Prev $ withAgentClientsServers2 (agentCfg, initAgentServers) (agentCfg, initAgentServers2) $ runTest False True
-  it "2 servers, via proxy, curr clients, prev servers" $ withSmpServersProxy2Prev $ withAgentClientsServers2 (agentCfg, initAgentServersProxy) (agentCfg, initAgentServersProxy2) $ runTest True True
-  it "2 servers, directly, prev clients, curr servers" $ withSmpServers2 ps $ withAgentClientsServers2 (agentCfgVPrevPQ, initAgentServers) (agentCfgVPrevPQ, initAgentServers2) $ runTest False False
-  it "2 servers, via proxy, prev clients, curr servers" $ withSmpServersProxy2 ps $ withAgentClientsServers2 (agentCfgVPrevPQ, initAgentServersProxy) (agentCfgVPrevPQ, initAgentServersProxy2) $ runTest True False
-  where
-    prev cfg' = updateCfg cfg' $ \cfg_ -> cfg_ {smpServerVRange = prevRange supportedServerSMPRelayVRange}
-    withSmpServers2Prev a = withServers2 (prev $ cfgMS msType) (prev $ cfgJ2QS qs) a
-    withSmpServersProxy2Prev a = withServers2 (prev $ proxyCfgMS msType) (prev $ proxyCfgJ2QS qs) a
-    withServers2 cfg1 cfg2 a =
-      withSmpServerConfigOn t cfg1 testPort $ \_ -> withSmpServerConfigOn t cfg2 testPort2 $ \_ -> a
-
 testPQMatrix2 :: HasCallStack => (ASrvTransport, AStoreType) -> (HasCallStack => (AgentClient, InitialKeys) -> (AgentClient, PQSupport) -> AgentMsgId -> IO ()) -> Spec
 testPQMatrix2 = pqMatrix2_ True

@@ -1554,7 +1552,7 @@ testAllowConnectionClientRestart ps@(t, ASType qsType _) = do
  bob <- getSMPAgentClient' 2 agentCfg initAgentServersSrv2 testDB2
  withSmpServerStoreLogOn ps testPort $ \_ -> do
    (aliceId, bobId, confId) <-
-      withSmpServerConfigOn t (cfgJ2QS qsType) testPort2 $ \_ -> do
+      withSmpServerConfigOn t (cfgS2QS qsType) testPort2 $ \_ -> do
        runRight $ do
          (bobId, qInfo) <- createConnection alice 1 True SCMInvitation Nothing SMSubscribe
          (aliceId, sqSecured) <- joinConnection bob 1 True qInfo "bob's connInfo" SMSubscribe
@@ -1576,7 +1574,7 @@ testAllowConnectionClientRestart ps@(t, ASType qsType _) = do
    alice2 <- getSMPAgentClient' 3 agentCfg initAgentServers testDB
    runRight_ $ subscribeConnection alice2 bobId
    threadDelay 500000
-    withSmpServerConfigOn t (cfgJ2QS qsType) testPort2 $ \_ -> do
+    withSmpServerConfigOn t (cfgS2QS qsType) testPort2 $ \_ -> do
      runRight $ do
        ("", "", UP _ _) <- nGet bob
        get alice2 ##> ("", bobId, CON)
@@ -1739,7 +1737,7 @@ withServer1 :: (ASrvTransport, AStoreType) -> IO a -> IO a
 withServer1 ps = withSmpServerStoreLogOn ps testPort . const

 withServer2 :: (ASrvTransport, AStoreType) -> IO a -> IO a
-withServer2 (t, ASType qsType _) = withSmpServerConfigOn t (cfgJ2QS qsType) testPort2 . const
+withServer2 (t, ASType qsType _) = withSmpServerConfigOn t (cfgS2QS qsType) testPort2 . const

 testInvitationShortLink :: HasCallStack => Bool -> AgentClient -> AgentClient -> IO ()
 testInvitationShortLink viaProxy a b =
@@ -1777,14 +1775,6 @@ testJoinConn_ viaProxy sndSecure a bId b connReq = do
  get b ##> ("", aId, CON)
  exchangeGreetingsViaProxy viaProxy a bId b aId

-testInvitationShortLinkPrev :: HasCallStack => Bool -> Bool -> AgentClient -> AgentClient -> IO ()
-testInvitationShortLinkPrev viaProxy sndSecure a b = runRight_ $ do
-  let userData = UserLinkData "some user data"
-      newLinkData = UserInvLinkData userData
-  -- can't create short link with previous version
-  (bId, CCLink connReq Nothing) <- A.createConnection a NRMInteractive 1 True True SCMInvitation (Just newLinkData) Nothing CR.IKPQOn False SMSubscribe
-  testJoinConn_ viaProxy sndSecure a bId b connReq
-
 testInvitationShortLinkAsync :: HasCallStack => Bool -> AgentClient -> AgentClient -> IO ()
 testInvitationShortLinkAsync viaProxy a b = do
  let userData = UserLinkData "some user data"
@@ -1987,18 +1977,11 @@ testOldContactQueueShortLink ps@(_, msType) = withAgentClients2 $ \a b -> do
 #endif
  () <- case testServerStoreConfig msType of
    ASSCfg _ _ (SSCMemory sp_) -> mapM_ (\StorePaths {storeLogFile} -> updateStoreLog storeLogFile) sp_
-    ASSCfg _ _ SSCMemoryJournal {storeLogFile} -> updateStoreLog storeLogFile
 #if defined(dbServerPostgres)
-    ASSCfg _ _ SSCDatabaseJournal {storeCfg} -> do
-      st :: PostgresQueueStore (JournalQueue 'QSPostgres) <- newQueueStore @(JournalQueue 'QSPostgres) (storeCfg, True)
-      updateDbStore st
-      closeQueueStore @(JournalQueue 'QSPostgres) st
    ASSCfg _ _ (SSCDatabase storeCfg) -> do
-      st :: PostgresQueueStore PostgresQueue <- newQueueStore @PostgresQueue (storeCfg, False)
+      st :: PostgresQueueStore PostgresQueue <- newQueueStore @PostgresQueue storeCfg
      updateDbStore st
      closeQueueStore @PostgresQueue st
-#else
-    ASSCfg _ _ SSCDatabaseJournal {} -> error "no dbServerPostgres flag"
 #endif

  withSmpServer ps $ do
@@ -2446,7 +2429,7 @@ testExpireMessageQuota (t, msType) = withSmpServerConfigOn t cfg' testPort $ \_
    ackMessage b' aId 4 Nothing
  disposeAgentClient a
  where
-    cfg' = updateCfg (cfgMS msType) $ \cfg_ -> cfg_ {msgQueueQuota = 1, maxJournalMsgCount = 2}
+    cfg' = updateCfg (cfgMS msType) $ \cfg_ -> cfg_ {msgQueueQuota = 1}

 testExpireManyMessagesQuota :: (ASrvTransport, AStoreType) -> IO ()
 testExpireManyMessagesQuota (t, msType) = withSmpServerConfigOn t cfg' testPort $ \_ -> do
@@ -2485,7 +2468,7 @@ testExpireManyMessagesQuota (t, msType) = withSmpServerConfigOn t cfg' testPort
    ackMessage b' aId 4 Nothing
  disposeAgentClient a
  where
-    cfg' = updateCfg (cfgMS msType) $ \cfg_ -> cfg_ {msgQueueQuota = 1, maxJournalMsgCount = 2}
+    cfg' = updateCfg (cfgMS msType) $ \cfg_ -> cfg_ {msgQueueQuota = 1}

 testJoinFullContactAsync :: HasCallStack => (ASrvTransport, AStoreType) -> IO ()
 testJoinFullContactAsync (t, msType) = withSmpServerConfigOn t cfg' testPort $ \_ -> do
@@ -2504,7 +2487,7 @@ testJoinFullContactAsync (t, msType) = withSmpServerConfigOn t cfg' testPort $ \
      ("", _, A.REQ _ _ _ "bob's connInfo" _ _) <- get alice
      pure ()
  where
-    cfg' = updateCfg (cfgMS msType) $ \cfg_ -> cfg_ {msgQueueQuota = 1, maxJournalMsgCount = 2}
+    cfg' = updateCfg (cfgMS msType) $ \cfg_ -> cfg_ {msgQueueQuota = 1}

 testJoinFullContactAsyncExpire :: HasCallStack => (ASrvTransport, AStoreType) -> IO ()
 testJoinFullContactAsyncExpire (t, msType) = withSmpServerConfigOn t cfg' testPort $ \_ -> do
@@ -2518,7 +2501,7 @@ testJoinFullContactAsyncExpire (t, msType) = withSmpServerConfigOn t cfg' testPo
    noMessages bob "joining should be retried until quota exceeded timeout"
    get bob =##> \case ("2", c, ERR (SMP _ QUOTA)) -> c == aliceId; _ -> False
  where
-    cfg' = updateCfg (cfgMS msType) $ \cfg_ -> cfg_ {msgQueueQuota = 1, maxJournalMsgCount = 2}
+    cfg' = updateCfg (cfgMS msType) $ \cfg_ -> cfg_ {msgQueueQuota = 1}

 fillContactAddress :: HasCallStack => IO (ConnId, ConnectionRequestUri 'CMContact)
 fillContactAddress = do
@@ -2744,6 +2727,117 @@ testRatchetSyncSimultaneous ps = do
  disposeAgentClient bob
  disposeAgentClient bob2

+testRatchetSyncReplayedKey :: HasCallStack => (ASrvTransport, AStoreType) -> IO ()
+testRatchetSyncReplayedKey ps = withAgentClients2 $ \alice bob -> do
+  (aliceId, bobId, bob2) <- withSmpServerStoreMsgLogOn ps testPort $ \_ ->
+    setupDesynchronizedRatchet alice bob
+  ("", "", DOWN _ _) <- nGet alice
+  ("", "", DOWN _ _) <- nGet bob2
+  _ <- runRight $ synchronizeRatchet bob2 aliceId PQSupportOn False
+  Right pks <- withTransaction (store $ agentEnv bob2) (`getRatchetX3dhKeys` aliceId)
+  Right (SomeConn _ (DuplexConnection _ _ (sq :| _))) <- withTransaction (store $ agentEnv bob2) (`getConn` aliceId)
+  withSmpServerStoreMsgLogOn ps testPort $ \_ -> do
+    concurrently_
+      (getInAnyOrder alice [ratchetSyncP' bobId RSAgreed, serverUpP])
+      (getInAnyOrder bob2 [ratchetSyncP' aliceId RSAgreed, serverUpP])
+    get alice =##> ratchetSyncP bobId RSOk
+    get bob2 =##> ratchetSyncP aliceId RSOk
+    withTransaction (store $ agentEnv alice) $ \db -> deleteRatchetKeyHashesExpired db 0 100
+    let keyMsg = AgentRatchetKey {agentVersion = currentSMPAgentVersion, e2eEncryption = CR.mkRcvE2ERatchetParams CR.currentE2EEncryptVersion pks, info = ""}
+    Right _ <- runReaderT (runExceptT $ sendAgentMessage bob2 sq SMP.noMsgFlags $ smpEncode keyMsg) (agentEnv bob2)
+    runRight_ $ exchangeGreetingsMsgIds alice bobId 10 bob2 aliceId 7
+  disposeAgentClient bob2
+
+testRatchetSyncRepeated :: HasCallStack => (ASrvTransport, AStoreType) -> IO ()
+testRatchetSyncRepeated ps = withAgentClients2 $ \alice bob -> do
+  (aliceId, bobId, bob2) <- startRatchetSyncOffline ps alice bob
+  ConnectionStats {ratchetSyncState = rss2} <- runRight $ synchronizeRatchet bob2 aliceId PQSupportOn True
+  rss2 `shouldBe` RSStarted
+
+  withSmpServerStoreMsgLogOn ps testPort $ \_ -> do
+    concurrently_
+      (getInAnyOrder alice [ratchetSyncP' bobId RSAgreed, serverUpP])
+      (getInAnyOrder bob2 [ratchetSyncP' aliceId RSAgreed, serverUpP])
+    runRight_ $ do
+      get alice =##> ratchetSyncP bobId RSAgreed
+      get alice =##> ratchetSyncP bobId RSOk
+      get bob2 =##> ratchetSyncP aliceId RSOk
+      msgId <- sendMessage alice bobId SMP.noMsgFlags "hello"
+      get alice ##> ("", bobId, SENT msgId)
+      get bob2 =##> \case ("", c, Msg "hello") -> c == aliceId; _ -> False
+      ackMessage bob2 aliceId 8 Nothing
+  map fst <$> processedRatchetKeyHashes bob2 `shouldReturn` [aliceId, aliceId]
+  disposeAgentClient bob2
+
+testRatchetSyncFailedKeyNotProcessed :: HasCallStack => (ASrvTransport, AStoreType) -> IO ()
+testRatchetSyncFailedKeyNotProcessed ps = withAgentClients2 $ \alice bob -> do
+  (aliceId, bobId, bob2) <- startRatchetSyncOffline ps alice bob
+  withTransaction (store $ agentEnv bob2) $ \db ->
+    DB.execute_ db "UPDATE ratchets SET x3dh_priv_key_1 = NULL"
+
+  withSmpServerStoreMsgLogOn ps testPort $ \_ ->
+    concurrently_
+      (getInAnyOrder alice [ratchetSyncP' bobId RSAgreed, serverUpP])
+      (getInAnyOrder bob2 [x3dhKeysNotFoundP aliceId, serverUpP])
+  map fst <$> processedRatchetKeyHashes alice `shouldReturn` [bobId]
+  processedRatchetKeyHashes bob2 `shouldReturn` []
+  disposeAgentClient bob2
+  where
+    x3dhKeysNotFoundP :: ConnId -> ATransmission -> Bool
+    x3dhKeysNotFoundP cId = \case
+      (_, cId', AEvt SAEConn (ERR (A.INTERNAL e))) -> cId' == cId && "SEX3dhKeysNotFound" `isPrefixOf` e
+      _ -> False
+
+testRatchetSyncFailedRecreationNoReply :: HasCallStack => (ASrvTransport, AStoreType) -> IO ()
+testRatchetSyncFailedRecreationNoReply ps = withAgentClients2 $ \alice bob -> do
+  (_, bobId, bob2) <- startRatchetSyncOffline ps alice bob
+  aliceSndMsgs <- sndMessages alice
+  withTransaction (store $ agentEnv alice) $ \db ->
+    DB.execute_ db "CREATE TRIGGER fail_ratchet_insert BEFORE INSERT ON ratchets BEGIN SELECT RAISE(ABORT, 'ratchet insert failed'); END"
+  withSmpServerStoreMsgLogOn ps testPort $ \_ ->
+    concurrently_
+      (getInAnyOrder alice [ratchetInsertFailedP bobId, serverUpP])
+      (getInAnyOrder bob2 [serverUpP])
+  processedRatchetKeyHashes alice `shouldReturn` []
+  sndMessages alice `shouldReturn` aliceSndMsgs
+  disposeAgentClient bob2
+  where
+    ratchetInsertFailedP :: ConnId -> ATransmission -> Bool
+    ratchetInsertFailedP cId = \case
+      (_, cId', AEvt SAEConn (ERR (A.INTERNAL e))) -> cId' == cId && "ratchet insert failed" `isInfixOf` e
+      _ -> False
+
+testRatchetSyncStartFailedNoKey :: HasCallStack => (ASrvTransport, AStoreType) -> IO ()
+testRatchetSyncStartFailedNoKey ps = withAgentClients2 $ \alice bob -> do
+  (aliceId, _, bob2) <- withSmpServerStoreMsgLogOn ps testPort $ \_ ->
+    setupDesynchronizedRatchet alice bob
+  bobSndMsgs <- sndMessages bob2
+  withTransaction (store $ agentEnv bob2) $ \db ->
+    DB.execute_ db "CREATE TRIGGER fail_ratchet_update BEFORE UPDATE ON ratchets BEGIN SELECT RAISE(ABORT, 'ratchet update failed'); END"
+  Left (A.INTERNAL e) <- runExceptT $ synchronizeRatchet bob2 aliceId PQSupportOff False
+  e `shouldContain` "ratchet update failed"
+  ConnectionStats {ratchetSyncState} <- runRight $ getConnectionServers bob2 aliceId
+  ratchetSyncState `shouldBe` RSRequired
+  withTransaction (store $ agentEnv bob2) (`DB.query_` "SELECT conn_id, pq_support FROM connections") `shouldReturn` [(aliceId, PQSupportOn)]
+  sndMessages bob2 `shouldReturn` bobSndMsgs
+  disposeAgentClient bob2
+
+startRatchetSyncOffline :: HasCallStack => (ASrvTransport, AStoreType) -> AgentClient -> AgentClient -> IO (ConnId, ConnId, AgentClient)
+startRatchetSyncOffline ps alice bob = do
+  (aliceId, bobId, bob2) <- withSmpServerStoreMsgLogOn ps testPort $ \_ ->
+    setupDesynchronizedRatchet alice bob
+  ("", "", DOWN _ _) <- nGet alice
+  ("", "", DOWN _ _) <- nGet bob2
+  ConnectionStats {ratchetSyncState} <- runRight $ synchronizeRatchet bob2 aliceId PQSupportOn False
+  ratchetSyncState `shouldBe` RSStarted
+  pure (aliceId, bobId, bob2)
+
+processedRatchetKeyHashes :: AgentClient -> IO [(ConnId, ByteString)]
+processedRatchetKeyHashes c = withTransaction (store $ agentEnv c) (`DB.query_` "SELECT conn_id, hash FROM processed_ratchet_key_hashes")
+
+sndMessages :: AgentClient -> IO [(ConnId, Int64)]
+sndMessages c = withTransaction (store $ agentEnv c) (`DB.query_` "SELECT conn_id, internal_id FROM snd_messages ORDER BY conn_id, internal_id")
+
 getMsg :: AgentClient -> ConnId -> ExceptT AgentErrorType IO a -> ExceptT AgentErrorType IO a
 getMsg c cId action = do
  liftIO $ noMessages c "nothing should be delivered before GET"
@@ -2985,7 +3079,7 @@ testBatchedSubscriptions nCreate nDel ps@(t, ASType qsType _) = do
    runServers :: ExceptT AgentErrorType IO a -> IO a
    runServers a = do
      withSmpServerStoreLogOn ps testPort $ \t1 -> do
-        res <- withSmpServerConfigOn t (cfgJ2QS qsType) testPort2 $ \t2 ->
+        res <- withSmpServerConfigOn t (cfgS2QS qsType) testPort2 $ \t2 ->
          runRight a `finally` killThread t2
        killThread t1
        pure res
@@ -3492,7 +3586,7 @@ testJoinConnectionAsyncReplyError ps@(t, ASType qsType _) = do
        ConnectionStats {rcvQueuesInfo = [], sndQueuesInfo = [SndQueueInfo {}]} <- getConnectionServers b aId
        pure (aId, bId)
      nGet a =##> \case ("", "", DOWN _ [c]) -> c == bId; _ -> False
-      withSmpServerConfigOn t (cfgJ2QS qsType) testPort2 $ \_ -> do
+      withSmpServerConfigOn t (cfgS2QS qsType) testPort2 $ \_ -> do
        confId <- withSmpServerStoreLogOn ps testPort $ \_ -> do
          -- both servers need to be online for connection to progress because of SKEY
          get b =##> \case ("2", c, JOINED sqSecured) -> c == aId && sqSecured; _ -> False
@@ -3625,7 +3719,7 @@ fastSwitchComplete a bId b aId = do
 testFastSwitchDeadOldServer :: HasCallStack => (ASrvTransport, AStoreType) -> IO ()
 testFastSwitchDeadOldServer ps@(t, ASType qsType _) = do
  let bServers = initAgentServers {smp = userServers [testSMPServer2]}
-  withSmpServerConfigOn t (cfgJ2QS qsType) testPort2 $ \_ ->
+  withSmpServerConfigOn t (cfgS2QS qsType) testPort2 $ \_ ->
    withAgent 1 agentCfg initAgentServers testDB $ \a ->
      withAgent 2 agentCfg bServers testDB2 $ \b -> do
        (aId, bId) <- withSmpServerStoreLogOn ps testPort $ \_ -> runRight $ do
@@ -4119,7 +4213,7 @@ testDeliveryReceiptsConcurrent (t, msType) =
      liftIO $ noMessages a "nothing else should be delivered to alice"
      liftIO $ noMessages b "nothing else should be delivered to bob"
  where
-    cfg' = updateCfg (cfgMS msType) $ \cfg_ -> cfg_ {msgQueueQuota = 256, maxJournalMsgCount = 512}
+    cfg' = updateCfg (cfgMS msType) $ \cfg_ -> cfg_ {msgQueueQuota = 256}
    runClient :: String -> AgentClient -> ConnId -> IO ()
    runClient _cName client connId = do
      concurrently_ send receive
@@ -869,7 +869,7 @@ testNotificationsSMPRestartBatch n ps@(t, ASType qsType _) apns =
    runServers :: ExceptT AgentErrorType IO a -> IO a
    runServers a = do
      withSmpServerStoreLogOn ps testPort $ \t1 -> do
-        res <- withSmpServerConfigOn t (cfgJ2QS qsType) testPort2 $ \t2 ->
+        res <- withSmpServerConfigOn t (cfgS2QS qsType) testPort2 $ \t2 ->
          runRight a `finally` killThread t2
        killThread t1
        pure res
@@ -20,12 +20,13 @@ import Control.Concurrent.Async (concurrently_)
 import Control.Concurrent.MVar
 import Control.Concurrent.STM
 import Control.Exception (SomeException)
-import Control.Monad (replicateM_)
+import Control.Monad (forM_, replicateM_)
 import Control.Monad.Trans.Except
 import Crypto.Random (ChaChaDRG)
 import Data.ByteArray (ScrubbedBytes)
 import Data.ByteString.Char8 (ByteString)
 import Data.List (isInfixOf)
+import qualified Data.Map.Strict as M
 import qualified Data.Text as T
 import Data.Text.Encoding (encodeUtf8)
 import Data.Time
@@ -51,6 +52,7 @@ import qualified Simplex.Messaging.Crypto as C
 import Simplex.Messaging.Crypto.File (CryptoFile (..))
 import Simplex.Messaging.Crypto.Ratchet (pattern IKPQOn)
 import qualified Simplex.Messaging.Crypto.Ratchet as CR
+import Simplex.Messaging.Encoding (Encoding (..))
 import Simplex.Messaging.Encoding.String (StrEncoding (..))
 import Simplex.Messaging.Protocol (EntityId (..), QueueMode (..), SubscriptionMode (..), pattern VersionSMPC)
 import qualified Simplex.Messaging.Protocol as SMP
@@ -135,6 +137,8 @@ storeTests = do
          testCreateRcvMsg
          testCreateSndMsg
          testCreateRcvAndSndMsgs
+      describe "deleteRatchetKeyHashesExpired" testDeleteRatchetKeyHashesExpired
+      it "should keep only the newest skipped message keys" testGetSkippedMsgKeys
      describe "Work items" $ do
        it "should getPendingQueueMsg" testGetPendingQueueMsg
        it "should getPendingServerCommand" testGetPendingServerCommand
@@ -599,6 +603,42 @@ testCreateRcvAndSndMsgs =
      testCreateSndMsg_ db "snd_hash_1" connId sq $ mkSndMsgData (InternalId 5) (InternalSndId 2) "snd_hash_2"
      testCreateSndMsg_ db "snd_hash_2" connId sq $ mkSndMsgData (InternalId 6) (InternalSndId 3) "snd_hash_3"

+testDeleteRatchetKeyHashesExpired :: SpecWith DBStore
+testDeleteRatchetKeyHashesExpired =
+  it "should delete expired ratchet key hashes except the newest in each connection" . withStoreTransaction $ \db -> do
+    g <- C.newRandom
+    Right connId <- createNewConn db g cData1 {connId = ""} SCMInvitation
+    Right connId' <- createNewConn db g cData1 {connId = ""} SCMContact
+    let hashes = ["h1", "h2", "h3", "h4", "h5", "h6"]
+    forM_ hashes $ addProcessedRatchetKeyHash db connId
+    forM_ (take 4 hashes) $ addProcessedRatchetKeyHash db connId'
+    deleteRatchetKeyHashesExpired db 86400 4
+    mapM (checkRatchetKeyHashExists db connId) hashes `shouldReturn` replicate 6 True
+    deleteRatchetKeyHashesExpired db 0 4
+    mapM (checkRatchetKeyHashExists db connId) hashes `shouldReturn` [False, False, True, True, True, True]
+    mapM (checkRatchetKeyHashExists db connId') (take 4 hashes) `shouldReturn` replicate 4 True
+
+testGetSkippedMsgKeys :: DBStore -> Expectation
+testGetSkippedMsgKeys st = do
+  g <- C.newRandom
+  withTransaction st $ \db -> do
+    Right connId <- createNewConn db g cData1 {connId = ""} SCMInvitation
+    Right connId' <- createNewConn db g cData1 {connId = ""} SCMInvitation
+    createSkippedKeys db connId'
+    createSkippedKeys db connId
+    M.map M.keys <$> getSkippedMsgKeys db connId 4
+      `shouldReturn` M.singleton (C.Key "header_key") [1, 2, 3, 4]
+    getMsgNs db connId `shouldReturn` [1 .. 4]
+    getMsgNs db connId' `shouldReturn` [1 .. 10]
+  where
+    createSkippedKeys :: DB.Connection -> ConnId -> IO ()
+    createSkippedKeys db connId = do
+      DB.execute db "INSERT INTO ratchets (conn_id) VALUES (?)" (Only connId)
+      forM_ ([10, 9 .. 1] :: [Int]) $ \msgN ->
+        DB.execute db "INSERT INTO skipped_messages (conn_id, header_key, msg_n, msg_key) VALUES (?, ?, ?, ?)" (connId, "header_key" :: ByteString, msgN, smpEncode ("key" :: ByteString, "iv" :: ByteString))
+    getMsgNs :: DB.Connection -> ConnId -> IO [Int]
+    getMsgNs db connId = map fromOnly <$> DB.query db "SELECT msg_n FROM skipped_messages WHERE conn_id = ? ORDER BY msg_n" (Only connId)
+
 testCloseReopenStore :: IO ()
 testCloseReopenStore = do
  st <- createStore'
@@ -48,6 +48,7 @@ schemaDumpTest = do
  it "verify strict tables" testVerifyStrict
  it "should NOT create user record for new database" testUsersMigrationNew
  it "should create user record for old database" testUsersMigrationOld
+  it "should remove duplicate ratchet key hashes before adding unique index" testRatchetKeyHashesUniqueMigration

 testVerifySchemaDump :: IO ()
 testVerifySchemaDump = do
@@ -114,12 +115,28 @@ testUsersMigrationOld = do
    `shouldReturn` ([Only (1 :: Int)])
  closeDBStore st'

+testRatchetKeyHashesUniqueMigration :: IO ()
+testRatchetKeyHashesUniqueMigration = do
+  let beforeUnique = takeWhile (("m20260929_ratchet_indexes" /=) . name) appMigrations
+  Right st <- createDBStore (DBOpts testDB [] "" False True TQOff) beforeUnique (MigrationConfig MCError Nothing)
+  withTransaction' st $ \db -> do
+    SQL.execute_ db "INSERT INTO users (user_id) VALUES (1)"
+    SQL.execute_ db "INSERT INTO connections (conn_id, conn_mode, user_id) VALUES (x'01', 'INV', 1), (x'02', 'INV', 1)"
+    SQL.execute_ db "INSERT INTO processed_ratchet_key_hashes (conn_id, hash) VALUES (x'01', x'aa'), (x'01', x'aa'), (x'01', x'bb'), (x'02', x'aa'), (x'01', x'aa')"
+  closeDBStore st
+  Right st' <- createDBStore (DBOpts testDB [] "" False True TQOff) appMigrations (MigrationConfig MCYesUp Nothing)
+  withTransaction' st' (`SQL.query_` "SELECT processed_ratchet_key_hash_id FROM processed_ratchet_key_hashes ORDER BY processed_ratchet_key_hash_id")
+    `shouldReturn` [Only (1 :: Int), Only 3, Only 4]
+  closeDBStore st'
+
 skipComparisonForDownMigrations :: [String]
 skipComparisonForDownMigrations =
  [ -- on down migration idx_messages_internal_snd_id_ts index moves down to the end of the file
    "m20230814_indexes",
    -- snd_secure and last_broker_ts columns swap order on down migration
-    "m20250322_short_links"
+    "m20250322_short_links",
+    -- on down migration idx_processed_ratchet_key_hashes_hash index moves down to the end of the file
+    "m20260929_ratchet_indexes"
  ]

 getSchema :: FilePath -> FilePath -> IO String
@@ -6,8 +6,9 @@

 module CoreTests.CryptoTests (cryptoTests) where

+import Control.Concurrent (forkIO, newEmptyMVar, putMVar, takeMVar)
 import Control.Concurrent.STM
-import Control.Exception (evaluate)
+import Control.Exception (bracket, evaluate)
 import Control.Monad.Except
 import qualified Data.Aeson as J
 import qualified Data.ByteString.Char8 as B
@@ -22,9 +23,12 @@ import Data.Time.Clock (UTCTime (..))
 import qualified Data.Text.Lazy as LT
 import qualified Data.Text.Lazy.Encoding as LE
 import Data.Type.Equality
+import Data.Word (Word8)
 import qualified Data.X509 as X
 import qualified Data.X509.CertificateStore as XS
 import qualified Data.X509.Validation as XV
+import Foreign (FunPtr, allocaBytes, fillBytes, freeHaskellFunPtr, nullPtr)
+import Foreign.C.Types (CInt, CSize (..))
 import qualified SMPClient
 import qualified Simplex.Messaging.Crypto as C
 import qualified Simplex.Messaging.Crypto.Lazy as LC
@@ -32,9 +36,12 @@ import Simplex.Messaging.Crypto.BBS
 import Simplex.Messaging.Crypto.Entitlement
 import Simplex.Messaging.Crypto.SNTRUP761.Bindings
 import Simplex.Messaging.Crypto.SNTRUP761.Bindings.Defines
+import Simplex.Messaging.Crypto.SNTRUP761.Bindings.FFI (c_sntrup761_keypair)
+import Simplex.Messaging.Crypto.SNTRUP761.Bindings.RNG (RNGFunc)
 import Simplex.Messaging.Encoding (Large (..), smpDecode, smpEncode)
 import Simplex.Messaging.Encoding.String (strDecode, strEncode)
 import Simplex.Messaging.Transport.Client
+import System.Timeout (timeout)
 import Test.Hspec hiding (fit, it)
 import Test.Hspec.QuickCheck (modifyMaxSuccess)
 import Test.QuickCheck hiding (Large)
@@ -113,6 +120,7 @@ cryptoTests = do
  describe "sntrup761" $ do
    it "should enc/dec key" testSNTRUP761
    it "should reject malformed KEM encodings" testSNTRUP761RejectsMalformedEncodings
+    it "should fail key generation with degenerate RNG" testSNTRUP761KeypairDegenerateRNG
  describe "BBS+" $ do
    it "should sign and verify" testBBSSignVerify
    it "should derive public key from secret key" testBBSPublicKeyDerivation
@@ -298,6 +306,26 @@ testSNTRUP761 = do
  KEMSharedKey k' <- sntrup761Dec c sk
  k' `shouldBe` k

+foreign import ccall "wrapper"
+  mkRNGFunc :: RNGFunc -> IO (FunPtr RNGFunc)
+
+testSNTRUP761KeypairDegenerateRNG :: IO ()
+testSNTRUP761KeypairDegenerateRNG = do
+  -- constant byte 0 draws invertible g = -(1 + x + ... + x^760), byte 0x20 draws g = 0
+  keypairWithConstantRNG 0 `shouldReturn` Just 0
+  keypairWithConstantRNG 0x20 `shouldReturn` Just (-1)
+  where
+    keypairWithConstantRNG :: Word8 -> IO (Maybe CInt)
+    keypairWithConstantRNG b = do
+      result <- newEmptyMVar
+      -- timeout cannot interrupt a foreign call, so the call runs in another thread
+      _ <- forkIO $
+        bracket (mkRNGFunc $ \_ sz buf -> fillBytes buf b (fromIntegral sz)) freeHaskellFunPtr $ \rng ->
+          allocaBytes c_SNTRUP761_PUBLICKEY_SIZE $ \pkPtr ->
+            allocaBytes c_SNTRUP761_SECRETKEY_SIZE $ \skPtr ->
+              c_sntrup761_keypair pkPtr skPtr nullPtr rng >>= putMVar result
+      timeout 10000000 $ takeMVar result
+
 testSNTRUP761RejectsMalformedEncodings :: IO ()
 testSNTRUP761RejectsMalformedEncodings = do
  smpDecode @KEMPublicKey (smpEncode $ Large shortPublicKey) `shouldSatisfy` isLeft
@@ -16,8 +16,7 @@

 module CoreTests.MsgStoreTests where

-import AgentTests.FunctionalAPITests (runRight, runRight_)
-import Control.Concurrent (threadDelay)
+import AgentTests.FunctionalAPITests (runRight_)
 import Control.Concurrent.STM
 import Control.Exception (bracket)
 import Control.Monad
@@ -25,81 +24,56 @@ import Control.Monad.IO.Class
 import Control.Monad.Trans.Except
 import Crypto.Random (ChaChaDRG)
 import Data.ByteString.Char8 (ByteString)
-import qualified Data.ByteString.Char8 as B
-import Data.Int (Int64)
-import Data.List (isPrefixOf, isSuffixOf)
-import Data.Maybe (fromJust)
-import Data.Time.Clock (addUTCTime)
-import Data.Time.Clock.System (SystemTime (..), getSystemTime)
-import SMPClient (testStoreLogFile, testStoreMsgsDir, testStoreMsgsDir2, testStoreMsgsFile, testStoreMsgsFile2)
+import qualified Data.Map.Strict as M
+import Data.Time.Clock.System (getSystemTime)
 import Simplex.Messaging.Crypto (pattern MaxLenBS)
 import qualified Simplex.Messaging.Crypto as C
-import Simplex.Messaging.Protocol (EntityId (..), ErrorType, LinkId, Message (..), QueueLinkData, RecipientId, SParty (..), noMsgFlags)
-import Simplex.Messaging.Server (exportMessages, importMessages, printMessageStats)
-import Simplex.Messaging.Server.Env.STM (MsgStore (..), journalMsgStoreDepth, readWriteQueueStore)
-import Simplex.Messaging.Server.Expiration (ExpirationConfig (..), expireBeforeEpoch)
-import Simplex.Messaging.Server.MsgStore.Journal
+import Simplex.Messaging.Protocol (EncDataBytes (..), EntityId (..), ErrorType (..), LinkId, Message (..), QueueLinkData, RecipientId, SParty (..), noMsgFlags)
 import Simplex.Messaging.Server.MsgStore.STM
 import Simplex.Messaging.Server.MsgStore.Types
 import Simplex.Messaging.Server.QueueStore
 import Simplex.Messaging.Server.QueueStore.QueueInfo
-import Simplex.Messaging.Server.StoreLog (closeStoreLog, logCreateQueue)
-import System.Directory (copyFile, createDirectoryIfMissing, listDirectory, removeFile, renameFile)
-import System.FilePath ((</>))
-import System.IO (IOMode (..), withFile)
+import Simplex.Messaging.Server.QueueStore.STM (STMQueueStore (..))
+import Simplex.Messaging.Server.QueueStore.Types
+import Simplex.Messaging.TMap (TMap)
 import Test.Hspec hiding (fit, it)
 import Util

 #if defined(dbServerPostgres)
+import AgentTests.FunctionalAPITests (runRight)
+import Control.Concurrent (threadDelay)
+import Data.Int (Int64)
+import Data.Time.Clock.System (SystemTime (..))
 import Database.PostgreSQL.Simple (Only (..))
 import qualified Database.PostgreSQL.Simple as DB
 import Simplex.Messaging.Agent.Store.Postgres.Common
 import Simplex.Messaging.Agent.Store.Shared (MigrationConfirmation (..))
 import Simplex.Messaging.Server.MsgStore.Postgres
 import Simplex.Messaging.Server.QueueStore.Postgres
-import Simplex.Messaging.Server.QueueStore.Types
 import SMPClient (postgressBracket, testServerDBConnectInfo, testStoreDBOpts)
 #endif

 msgStoreTests :: Spec
 msgStoreTests = do
-  around (withMsgStore testSMTStoreConfig) $ describe "STM message store" someMsgStoreTests
-  around (withMsgStore $ testJournalStoreCfg MQStoreCfg) $ describe "Journal message store" $ do
+  around (withMsgStore testSMTStoreConfig) $ describe "STM message store" $ do
    someMsgStoreTests
-    journalMsgStoreTests
-    it "should export and import journal store" testExportImportStore
+    it "should remove deleted queues from queue store maps" testDeleteQueueMaps
 #if defined(dbServerPostgres)
  around_ (postgressBracket testServerDBConnectInfo) $ do
-    around (withMsgStore $ testJournalStoreCfg $ PQStoreCfg testPostgresStoreCfg) $
-      describe "Postgres+journal message store" $ do
-        someMsgStoreTests
-        journalMsgStoreTests
    around (withMsgStore testPostgresStoreConfig) $
      describe "Postgres-only message store" $ do
        someMsgStoreTests
        it "should correctly update message counts and canWrite flag" testUpdateMessageCounts
        it "tryDelPeekMsg (ACK not from NSE) should reset message counts when queue is empty" testResetMessageCounts
+        it "should expire messages across commit batches" testExpireMessagesInBatches
 #endif
-  describe "Journal message store: queue state backup expiration" $ do
-    it "should remove old queue state backups" testRemoveQueueStateBackups
-    it "should expire messages in idle queues" testExpireIdleQueues
  where
-    journalMsgStoreTests :: SpecWith (JournalMsgStore s)
-    journalMsgStoreTests = do
-      describe "queue state" $ do
-        it "should restore queue state from the last line" testQueueState
-        it "should recover when message is written and state is not" testMessageState
-        it "should remove journal files when queue is empty" testRemoveJournals
-      describe "missing files" $ do
-        it "should create read file when missing" testReadFileMissing
-        it "should switch to write file when read file missing" testReadFileMissingSwitch
-        it "should create write file when missing" testWriteFileMissing
-        it "should create read file when read and write files are missing" testReadAndWriteFilesMissing
    someMsgStoreTests :: MsgStoreClass s => SpecWith s
    someMsgStoreTests = do
      it "should get queue and store/read messages" testGetQueue
      it "should write/ack messages" testWriteAckMessages
-      it "should not fail on EOF when changing read journal" testChangeReadJournal
+      it "should resolve sender ID equal to link ID of another queue" testLinkIdSenderIdCollision
+      it "should not add link data to secured messaging queue" testLinkDataSecuredQueue

 -- TODO constrain to STM stores?
 withMsgStore :: MsgStoreClass s => MsgStoreConfig s -> (s -> IO ()) -> IO ()
@@ -108,21 +82,6 @@ withMsgStore cfg = bracket (newMsgStore cfg) closeMsgStore
 testSMTStoreConfig :: STMStoreConfig
 testSMTStoreConfig = STMStoreConfig {storePath = Nothing, quota = 3}

-testJournalStoreCfg :: QStoreCfg s -> JournalStoreConfig s
-testJournalStoreCfg queueStoreCfg =
-  JournalStoreConfig
-    { storePath = testStoreMsgsDir,
-      pathParts = journalMsgStoreDepth,
-      queueStoreCfg,
-      quota = 3,
-      maxMsgCount = 4,
-      maxStateLines = 2,
-      stateTailSize = 256,
-      idleInterval = 21600,
-      expireBackupsAfter = 0,
-      keepMinBackups = 1
-    }
-
 #if defined(dbServerPostgres)
 testPostgresStoreConfig :: PostgresMsgStoreCfg
 testPostgresStoreConfig =
@@ -151,12 +110,6 @@ mkMessage body = liftIO $ do
 pattern Msg :: ByteString -> Maybe Message
 pattern Msg s <- Just Message {msgBody = MaxLenBS s}

-deriving instance Eq MsgQueueState
-
-deriving instance Eq (JournalState t)
-
-deriving instance Eq (SJournalType t)
-
 testNewQueueRec :: TVar ChaChaDRG -> QueueMode -> IO (RecipientId, QueueRec)
 testNewQueueRec g qm = testNewQueueRecData g qm Nothing

@@ -183,6 +136,18 @@ testNewQueueRecData g qm queueData = do
  where
    rndId = atomically $ EntityId <$> C.randomBytes 24 g

+testNtfCreds :: TVar ChaChaDRG -> IO NtfCreds
+testNtfCreds g = do
+  (notifierKey, _) <- atomically $ C.generateAuthKeyPair C.SX25519 g
+  (k, pk) <- atomically $ C.generateKeyPair @'C.X25519 g
+  pure
+    NtfCreds
+      { notifierId = EntityId "ijkl",
+        notifierKey,
+        rcvNtfDhSecret = C.dh' k pk,
+        ntfServiceId = Nothing
+      }
+
 testGetQueue :: MsgStoreClass s => s -> IO ()
 testGetQueue ms = do
  g <- C.newRandom
@@ -247,76 +212,97 @@ testWriteAckMessages ms = do
    void $ ExceptT $ deleteQueue ms q1
    void $ ExceptT $ deleteQueue ms q2

-testChangeReadJournal :: MsgStoreClass s => s -> IO ()
-testChangeReadJournal ms = do
+testLinkDataSecuredQueue :: MsgStoreClass s => s -> IO ()
+testLinkDataSecuredQueue ms = do
  g <- C.newRandom
+  (sKey, _) <- atomically $ C.generateAuthKeyPair C.SEd25519 g
+  let st = queueStore ms
+      ld = (EncDataBytes "fixed data", EncDataBytes "user data")
+      rndId = atomically $ EntityId <$> C.randomBytes 24 g
  (rId, qr) <- testNewQueueRec g QMMessaging
+  (cId, cqr) <- testNewQueueRec g QMContact
+  lnkId <- rndId
+  cLnkId <- rndId
  runRight_ $ do
    q <- ExceptT $ addQueue ms rId qr
-    let write s = writeMsg ms q True =<< mkMessage s
-    Just (Message {msgId = mId1}, True) <- write "message 1"
-    (Msg "message 1", Nothing) <- tryDelPeekMsg ms q mId1
-    Just (Message {msgId = mId2}, True) <- write "message 2"
-    (Msg "message 2", Nothing) <- tryDelPeekMsg ms q mId2
-    Just (Message {msgId = mId3}, True) <- write "message 3"
-    (Msg "message 3", Nothing) <- tryDelPeekMsg ms q mId3
-    Just (Message {msgId = mId4}, True) <- write "message 4"
-    (Msg "message 4", Nothing) <- tryDelPeekMsg ms q mId4
-    Just (Message {msgId = mId5}, True) <- write "message 5"
-    (Msg "message 5", Nothing) <- tryDelPeekMsg ms q mId5
+    -- the handle is read before SKEY, as in a command that raced with it
+    staleQ <- ExceptT $ getQueue ms SRecipient rId
+    ExceptT $ secureQueue st q sKey
+    liftIO $ addQueueLinkData st staleQ lnkId ld `shouldReturn` Left AUTH
+    freshQ <- ExceptT $ getQueue ms SRecipient rId
+    liftIO $ getQueueLinkData st freshQ lnkId `shouldReturn` Left AUTH
+    cq <- ExceptT $ addQueue ms cId cqr
+    ExceptT $ secureQueue st cq sKey
+    ExceptT $ addQueueLinkData st cq cLnkId ld
+    ld' <- ExceptT $ getQueueLinkData st cq cLnkId
+    liftIO $ ld' `shouldBe` ld
    void $ ExceptT $ deleteQueue ms q
+    void $ ExceptT $ deleteQueue ms cq

-testExportImportStore :: JournalMsgStore 'QSMemory -> IO ()
-testExportImportStore ms = do
+-- sizes of queues, senders, links and notifiers maps
+type QueueMapSizes = (Int, Int, Int, Int)
+
+queueMapSizes :: STMMsgStore -> IO QueueMapSizes
+queueMapSizes ms = (,,,) <$> size queues <*> size senders <*> size links <*> size notifiers
+  where
+    STMQueueStore {queues, senders, links, notifiers} = queueStore ms
+    size :: TMap k v -> IO Int
+    size = fmap M.size . readTVarIO
+
+testDeleteQueueMaps :: STMMsgStore -> IO ()
+testDeleteQueueMaps ms = do
  g <- C.newRandom
+  ntfCreds <- testNtfCreds g
+  let newLinkId = atomically $ EntityId <$> C.randomBytes 24 g
+  lnkId1 <- newLinkId
+  lnkId2 <- newLinkId
+  lnkId3 <- newLinkId
  (rId1, qr1) <- testNewQueueRec g QMMessaging
-  (rId2, qr2) <- testNewQueueRec g QMMessaging
-  sl <- readWriteQueueStore True (mkQueue ms True) testStoreLogFile $ stmQueueStore ms
+  (rId2, qr2) <- testNewQueueRecData g QMContact (Just (lnkId1, testLinkData))
+  (rId3, qr3) <- testNewQueueRec g QMMessaging
+  (rId4, qr4) <- testNewQueueRec g QMMessaging
+  let rIds = [rId1, rId2, rId3, rId4] :: [RecipientId]
+      sIds = map senderId [qr1, qr2, qr3, qr4]
+      lnkIds = [lnkId1, lnkId2, lnkId3] :: [LinkId]
+  queueMapSizes ms `shouldReturn` (0, 0, 0, 0)
  runRight_ $ do
-    let write q s = writeMsg ms q True =<< mkMessage s
-    q1 <- ExceptT $ addQueue ms rId1 qr1
-    liftIO $ logCreateQueue sl rId1 qr1
-    Just (Message {}, True) <- write q1 "message 1"
-    Just (Message {}, False) <- write q1 "message 2"
+    q1 <- ExceptT $ addQueue ms rId1 qr1 {notifier = Just ntfCreds}
    q2 <- ExceptT $ addQueue ms rId2 qr2
-    liftIO $ logCreateQueue sl rId2 qr2
-    Just (Message {msgId = mId3}, True) <- write q2 "message 3"
-    Just (Message {msgId = mId4}, False) <- write q2 "message 4"
-    (Msg "message 3", Msg "message 4") <- tryDelPeekMsg ms q2 mId3
-    (Msg "message 4", Nothing) <- tryDelPeekMsg ms q2 mId4
-    Just (Message {}, True) <- write q2 "message 5"
-    Just (Message {}, False) <- write q2 "message 6"
-    Just (Message {}, False) <- write q2 "message 7"
-    Nothing <- write q2 "message 8"
-    pure ()
-  length <$> listDirectory (msgQueueDirectory ms rId1) `shouldReturn` 2
-  length <$> listDirectory (msgQueueDirectory ms rId2) `shouldReturn` 3
-  exportMessages False (StoreJournal ms) testStoreMsgsFile False
-  closeMsgStore ms
-  closeStoreLog sl
-  -- export with closed queues and compare
-  ms2 <- newMsgStore $ testJournalStoreCfg MQStoreCfg
-  readWriteQueueStore True (mkQueue ms2 True) testStoreLogFile (stmQueueStore ms2) >>= closeStoreLog
-  exportMessages False (StoreJournal ms2) (testStoreMsgsFile <> ".copy") False
-  s <- B.readFile testStoreMsgsFile
-  B.readFile (testStoreMsgsFile <> ".copy") `shouldReturn` s
+    q3 <- ExceptT $ addQueue ms rId3 qr3
+    q4 <- ExceptT $ addQueue ms rId4 qr4
+    ExceptT $ addQueueLinkData (queueStore ms) q3 lnkId2 testLinkData
+    ExceptT $ addQueueLinkData (queueStore ms) q4 lnkId3 testLinkData
+    forM_ sIds $ void . ExceptT . getQueue ms SSender
+    forM_ lnkIds $ void . ExceptT . getQueue ms SSenderLink
+    liftIO $ queueMapSizes ms `shouldReturn` (4, 4, 3, 1)
+    ExceptT $ deleteQueueLinkData (queueStore ms) q3
+    liftIO $ queueMapSizes ms `shouldReturn` (4, 4, 2, 1)
+    forM_ ([q1, q2, q3, q4] :: [STMQueue]) $ void . ExceptT . deleteQueue ms
+  queueMapSizes ms `shouldReturn` (0, 0, 0, 0)
+  forM_ rIds $ \rId -> getQueue ms SRecipient rId >>= expectAuth
+  forM_ sIds $ \sId -> getQueue ms SSender sId >>= expectAuth
+  forM_ lnkIds $ \lnkId -> getQueue ms SSenderLink lnkId >>= expectAuth
+  queueMapSizes ms `shouldReturn` (0, 0, 0, 0)
+  where
+    expectAuth = either (`shouldBe` AUTH) (\_ -> expectationFailure "deleted queue is still found")

-  let cfg = (testJournalStoreCfg MQStoreCfg :: JournalStoreConfig 'QSMemory) {storePath = testStoreMsgsDir2}
-  ms' <- newMsgStore cfg
-  readWriteQueueStore True (mkQueue ms' True) testStoreLogFile (stmQueueStore ms') >>= closeStoreLog
-  stats@MessageStats {storedMsgsCount = 5, expiredMsgsCount = 0, storedQueues = 2} <-
-    importMessages False ms' testStoreMsgsFile Nothing False
-  printMessageStats "Messages" stats
-  length <$> listDirectory (msgQueueDirectory ms rId1) `shouldReturn` 2
-  length <$> listDirectory (msgQueueDirectory ms rId2) `shouldReturn` 3 -- 2 message files
-  exportMessages False (StoreJournal ms') testStoreMsgsFile2 False
-  (B.readFile testStoreMsgsFile2 `shouldReturn`) =<< B.readFile (testStoreMsgsFile <> ".bak")
-  stmStore <- newMsgStore testSMTStoreConfig
-  readWriteQueueStore True (mkQueue stmStore True) testStoreLogFile (queueStore stmStore) >>= closeStoreLog
-  MessageStats {storedMsgsCount = 5, expiredMsgsCount = 0, storedQueues = 2} <-
-    importMessages False stmStore testStoreMsgsFile2 Nothing False
-  exportMessages False (StoreMemory stmStore) testStoreMsgsFile False
-  (B.sort <$> B.readFile testStoreMsgsFile `shouldReturn`) =<< (B.sort <$> B.readFile (testStoreMsgsFile2 <> ".bak"))
+testLinkData :: QueueLinkData
+testLinkData = (EncDataBytes "fixed data", EncDataBytes "user data")
+
+testLinkIdSenderIdCollision :: MsgStoreClass s => s -> IO ()
+testLinkIdSenderIdCollision ms = do
+  g <- C.newRandom
+  (rIdV, qrV) <- testNewQueueRec g QMContact
+  (rIdA, qrA) <- testNewQueueRec g QMContact
+  let sIdV = senderId qrV
+  runRight_ $ do
+    void $ ExceptT $ addQueue ms rIdV qrV
+    qA <- ExceptT $ addQueue ms rIdA qrA
+    ExceptT $ addQueueLinkData (queueStore ms) qA sIdV testLinkData
+    qA' <- ExceptT $ getQueue ms SSenderLink sIdV
+    liftIO $ recipientId qA' `shouldBe` rIdA
+    qV <- ExceptT $ getQueue ms SSender sIdV
+    liftIO $ recipientId qV `shouldBe` rIdV

 #if defined(dbServerPostgres)
 testUpdateMessageCounts :: PostgresMsgStore -> IO ()
@@ -327,35 +313,34 @@ testUpdateMessageCounts ms = do
    q <- ExceptT $ addQueue ms rId qr
    let write s = writeMsg ms q True =<< mkMessage s
        hasSize = checkQueueSize ms
-    q `hasSize` (0, True, False)
+    q `hasSize` (0, True)
    Just (Message {msgId = mId1}, True) <- write "message 1"
-    q `hasSize` (1, True, True)
+    q `hasSize` (1, True)
    Just (Message {msgId = mId2}, False) <- write "message 2"
-    q `hasSize` (2, True, True)
+    q `hasSize` (2, True)
    Just (Message {msgId = mId3}, False) <- write "message 3"
-    q `hasSize` (3, True, True)
+    q `hasSize` (3, True)
    Nothing <- write "message 4"
-    q `hasSize` (4, False, True)
+    q `hasSize` (4, False)
    Msg "message 1" <- tryPeekMsg ms q
-    q `hasSize` (4, False, True)
+    q `hasSize` (4, False)
    Msg "message 1" <- tryDelMsg ms q mId1
-    q `hasSize` (3, False, True)
+    q `hasSize` (3, False)
    Msg "message 2" <- tryPeekMsg ms q
    (Msg "message 2", Msg "message 3") <- tryDelPeekMsg ms q mId2
-    q `hasSize` (2, False, True)
+    q `hasSize` (2, False)
    (Msg "message 3", Just MessageQuota {msgId = mId4}) <- tryDelPeekMsg ms q mId3
-    q `hasSize` (1, False, True)
+    q `hasSize` (1, False)
    (Just MessageQuota {}, Nothing) <- tryDelPeekMsg ms q mId4
-    q `hasSize` (0, True, False)
+    q `hasSize` (0, True)

-checkQueueSize :: PostgresMsgStore -> PostgresQueue -> (Int64, Bool, Bool) -> ExceptT ErrorType IO ()
-checkQueueSize ms q (size, canWrt, expire) = liftIO $ do
-  [(size', canWrt', expire')] <-
+checkQueueSize :: PostgresMsgStore -> PostgresQueue -> (Int64, Bool) -> ExceptT ErrorType IO ()
+checkQueueSize ms q (size, canWrt) = liftIO $ do
+  [(size', canWrt')] <-
    withTransaction (dbStore $ queueStore ms) $ \db ->
-      DB.query db "SELECT msg_queue_size, msg_can_write, msg_queue_expire FROM msg_queues WHERE recipient_id = ?" (Only (recipientId q))
+      DB.query db "SELECT msg_queue_size, msg_can_write FROM msg_queues WHERE recipient_id = ?" (Only (recipientId q))
  size' `shouldBe` size
  canWrt' `shouldBe` canWrt
-  expire' `shouldBe` expire

 testResetMessageCounts :: PostgresMsgStore -> IO ()
 testResetMessageCounts ms = do
@@ -369,341 +354,62 @@ testResetMessageCounts ms = do
    Just (Message {msgId = mId2}, False) <- write "message 2"
    Just (Message {msgId = mId3}, False) <- write "message 3"
    Nothing <- write "message 4"
-    q `hasSize` (4, False, True)
+    q `hasSize` (4, False)
    liftIO $ setIncorrectSize q (10, True)
    Nothing <- write "message 5"
-    q `hasSize` (11, False, True)
+    q `hasSize` (11, False)
    (Msg "message 1", Msg "message 2") <- tryDelPeekMsg ms q mId1
-    q `hasSize` (10, False, True)
+    q `hasSize` (10, False)
    (Msg "message 2", Msg "message 3") <- tryDelPeekMsg ms q mId2
-    q `hasSize` (9, False, True)
+    q `hasSize` (9, False)
    (Msg "message 3", Just MessageQuota {msgId = mId4}) <- tryDelPeekMsg ms q mId3
-    q `hasSize` (8, False, True)
+    q `hasSize` (8, False)
    (Just MessageQuota {}, Just MessageQuota {msgId = mId5}) <- tryDelPeekMsg ms q mId4
-    q `hasSize` (7, False, True)
+    q `hasSize` (7, False)
    (Just MessageQuota {}, Nothing) <- tryDelPeekMsg ms q mId5
-    q `hasSize` (0, True, False) -- reset
+    q `hasSize` (0, True) -- reset
  where
    setIncorrectSize :: PostgresQueue -> (Int64, Bool) -> IO ()
    setIncorrectSize q (size, canWrt) =
      void $ withTransaction (dbStore $ queueStore ms) $ \db ->
        DB.execute db "UPDATE msg_queues SET msg_queue_size = ?, msg_can_write = ? WHERE recipient_id = ?" (size, canWrt, recipientId q)
+
+testExpireMessagesInBatches :: PostgresMsgStore -> IO ()
+testExpireMessagesInBatches ms = do
+  g <- C.newRandom
+  emptiedQs <- replicateM emptiedCount $ newQueue g
+  partialQs <- replicateM partialCount $ newQueue g
+  overQuotaQs <- replicateM overQuotaCount $ newQueue g
+  quotaMsgs <- runRight $ do
+    forM_ (emptiedQs <> partialQs) $ \q -> void $ write q "old 1"
+    forM_ emptiedQs $ \q -> void $ write q "old 2"
+    mapM fillPastQuota overQuotaQs
+  -- msg_ts has second granularity, so the recent messages need a new second to be kept
+  threadDelay 1100000
+  boundary <- systemSeconds <$> getSystemTime
+  runRight_ $ forM_ partialQs $ \q -> void $ write q "recent"
+
+  MessageStats {expiredMsgsCount, storedMsgsCount, storedQueues} <- expireOldMessages False ms boundary 0
+  expiredMsgsCount `shouldBe` (emptiedCount * 2 + partialCount + sum quotaMsgs)
+  storedMsgsCount `shouldBe` (partialCount + overQuotaCount) -- recent messages and quota markers
+  storedQueues `shouldBe` (emptiedCount + partialCount + overQuotaCount)
+  runRight_ $ do
+    forM_ emptiedQs $ \q -> checkQueueSize ms q (0, True)
+    forM_ partialQs $ \q -> checkQueueSize ms q (1, True)
+    -- the quota marker is never expired, and the queue stays blocked until it is acked
+    forM_ overQuotaQs $ \q -> checkQueueSize ms q (1, False)
+  where
+    -- expire_old_messages pages through expired messages and commits per page, so these
+    -- counts put a page boundary inside each group of queues.
+    emptiedCount = 120 :: Int
+    partialCount = 40 :: Int
+    overQuotaCount = 10 :: Int
+    newQueue g = do
+      (rId, qr) <- testNewQueueRec g QMMessaging
+      runRight $ ExceptT $ addQueue ms rId qr
+    write q s = writeMsg ms q True =<< mkMessage s
+    fillPastQuota q = go 0
+      where
+        go n = write q "fill" >>= maybe (pure n) (const $ go (n + 1))
 #endif

-testQueueState :: JournalMsgStore s -> IO ()
-testQueueState ms = do
-  g <- C.newRandom
-  rId <- EntityId <$> atomically (C.randomBytes 24 g)
-  let dir = msgQueueDirectory ms rId
-      statePath = msgQueueStatePath dir rId
-  createDirectoryIfMissing True dir
-  state <- newMsgQueueState <$> newJournalId (random ms)
-  withFile statePath WriteMode (`appendState` state)
-  length . lines <$> readFile statePath `shouldReturn` 1
-  readQueueState ms statePath `shouldReturn` (Just state, False)
-  length <$> listDirectory dir `shouldReturn` 1 -- no backup
-  let state1 =
-        state
-          { size = 1,
-            readState = (readState state) {msgCount = 1, byteCount = 100},
-            writeState = (writeState state) {msgPos = 1, msgCount = 1, bytePos = 100, byteCount = 100}
-          }
-  withFile statePath AppendMode (`appendState` state1)
-  length . lines <$> readFile statePath `shouldReturn` 2
-  readQueueState ms statePath `shouldReturn` (Just state1, False)
-  length <$> listDirectory dir `shouldReturn` 1 -- no backup
-  let state2 =
-        state
-          { size = 2,
-            readState = (readState state) {msgCount = 2, byteCount = 200},
-            writeState = (writeState state) {msgPos = 2, msgCount = 2, bytePos = 200, byteCount = 200}
-          }
-  withFile statePath AppendMode (`appendState` state2)
-  length . lines <$> readFile statePath `shouldReturn` 3
-  copyFile statePath (statePath <> ".2")
-  readQueueState ms statePath `shouldReturn` (Just state2, True)
-  length <$> listDirectory dir `shouldReturn` 2 -- new state + copy
-  ls <- lines <$> readFile statePath
-  length ls `shouldBe` 3
-  -- mock compacting file
-  writeFile statePath $ last ls
-
-  -- corrupt the only line
-  corruptFile statePath
-  (Nothing, True) <- readQueueState ms statePath
-
-  -- corrupt the last line
-  renameFile (statePath <> ".2") statePath
-  removeOtherFiles dir statePath
-  length . lines <$> readFile statePath `shouldReturn` 3
-  corruptFile statePath
-  readQueueState ms statePath `shouldReturn` (Just state1, True)
-  length <$> listDirectory dir `shouldReturn` 1
-  length . lines <$> readFile statePath `shouldReturn` 3
-  where
-    corruptFile f = do
-      s <- readFile f
-      removeFile f
-      writeFile f $ take (length s - 4) s
-    removeOtherFiles dir keep = do
-      names <- listDirectory dir
-      forM_ names $ \name ->
-        let f = dir </> name
-         in unless (f == keep) $ removeFile f
-
-testMessageState :: JournalMsgStore s -> IO ()
-testMessageState ms = do
-  g <- C.newRandom
-  (rId, qr) <- testNewQueueRec g QMMessaging
-  let dir = msgQueueDirectory ms rId
-      statePath = msgQueueStatePath dir rId
-      write q s = writeMsg ms q True =<< mkMessage s
-
-  mId1 <- runRight $ do
-    q <- ExceptT $ addQueue ms rId qr
-    Just (Message {msgId = mId1}, True) <- write q "message 1"
-    Just (Message {}, False) <- write q "message 2"
-    liftIO $ closeMsgQueue ms q
-    pure mId1
-
-  ls <- B.lines <$> B.readFile statePath
-  B.writeFile statePath $ B.unlines $ take (length ls - 1) ls
-
-  runRight_ $ do
-    q <- ExceptT $ getQueue ms SRecipient rId
-    Just (Message {msgId = mId3}, False) <- write q "message 3"
-    (Msg "message 1", Msg "message 3") <- tryDelPeekMsg ms q mId1
-    (Msg "message 3", Nothing) <- tryDelPeekMsg ms q mId3
-    liftIO $ closeMsgQueue ms q
-
-testRemoveJournals :: JournalMsgStore s -> IO ()
-testRemoveJournals ms = do
-  g <- C.newRandom
-  (rId, qr) <- testNewQueueRec g QMMessaging
-  let dir = msgQueueDirectory ms rId
-      statePath = msgQueueStatePath dir rId
-      write q s = writeMsg ms q True =<< mkMessage s
-
-  runRight $ do
-    q <- ExceptT $ addQueue ms rId qr
-    Just (Message {msgId = mId1}, True) <- write q "message 1"
-    Just (Message {msgId = mId2}, False) <- write q "message 2"
-    (Msg "message 1", Msg "message 2") <- tryDelPeekMsg ms q mId1
-    (Msg "message 2", Nothing) <- tryDelPeekMsg ms q mId2
-    liftIO $ closeMsgQueue ms q
-
-  ls <- B.lines <$> B.readFile statePath
-  length ls `shouldBe` 4
-  journalFilesCount dir `shouldReturn` 1
-  stateBackupCount dir `shouldReturn` 0
-
-  runRight $ do
-    q <- ExceptT $ getQueue ms SRecipient rId
-    -- not removed yet
-    liftIO $ journalFilesCount dir `shouldReturn` 1
-    liftIO $ stateBackupCount dir `shouldReturn` 0
-    Nothing <- tryPeekMsg ms q
-    -- still not removed, queue is empty and not opened
-    liftIO $ journalFilesCount dir `shouldReturn` 1
-    _mq <- isolateQueue ms q "test" $ getMsgQueue ms q False
-    -- journal is removed
-    liftIO $ journalFilesCount dir `shouldReturn` 0
-    liftIO $ stateBackupCount dir `shouldReturn` 1
-    Just (Message {msgId = mId3}, True) <- write q "message 3"
-    -- journal is created
-    liftIO $ journalFilesCount dir `shouldReturn` 1
-    Just (Message {msgId = mId4}, False) <- write q "message 4"
-    (Msg "message 3", Msg "message 4") <- tryDelPeekMsg ms q mId3
-    (Msg "message 4", Nothing) <- tryDelPeekMsg ms q mId4
-    Just (Message {msgId = mId5}, True) <- write q "message 5"
-    Just (Message {msgId = mId6}, False) <- write q "message 6"
-    liftIO $ journalFilesCount dir `shouldReturn` 1
-    Just (Message {msgId = mId7}, False) <- write q "message 7"
-    -- separate write journal is created
-    liftIO $ journalFilesCount dir `shouldReturn` 2
-    Nothing <- write q "message 8"
-    (Msg "message 5", Msg "message 6") <- tryDelPeekMsg ms q mId5
-    liftIO $ journalFilesCount dir `shouldReturn` 2
-    (Msg "message 6", Msg "message 7") <- tryDelPeekMsg ms q mId6
-    -- read journal is removed
-    liftIO $ journalFilesCount dir `shouldReturn` 1
-    (Msg "message 7", Just MessageQuota {msgId = mId8}) <- tryDelPeekMsg ms q mId7
-    (Just MessageQuota {}, Nothing) <- tryDelPeekMsg ms q mId8
-    liftIO $ closeMsgQueue ms q
-
-  journalFilesCount dir `shouldReturn` 1
-  runRight $ do
-    q <- ExceptT $ getQueue ms SRecipient rId
-    Just (Message {}, True) <- write q "message 8"
-    liftIO $ journalFilesCount dir `shouldReturn` 1
-    liftIO $ stateBackupCount dir `shouldReturn` 2
-    liftIO $ closeMsgQueue ms q
-  where
-    journalFilesCount dir = length . filter ("messages." `isPrefixOf`) <$> listDirectory dir
-    stateBackupCount dir = length . filter (".bak" `isSuffixOf`) <$> listDirectory dir
-
-testRemoveQueueStateBackups :: IO ()
-testRemoveQueueStateBackups = do
-  g <- C.newRandom
-  (rId, qr) <- testNewQueueRec g QMMessaging
-
-  ms' <- newMsgStore (testJournalStoreCfg MQStoreCfg) {maxStateLines = 1, expireBackupsAfter = 0, keepMinBackups = 0}
-  -- set expiration time 1 second ahead
-  let ms = ms' {expireBackupsBefore = addUTCTime 1 $ expireBackupsBefore ms'}
-
-  let dir = msgQueueDirectory ms rId
-      write q s = writeMsg ms q True =<< mkMessage s
-
-  runRight $ do
-    q <- ExceptT $ addQueue ms rId qr
-    Just (Message {msgId = mId1}, True) <- write q "message 1"
-    Just (Message {msgId = mId2}, False) <- write q "message 2"
-    (Msg "message 1", Msg "message 2") <- tryDelPeekMsg ms q mId1
-    (Msg "message 2", Nothing) <- tryDelPeekMsg ms q mId2
-    liftIO $ closeMsgQueue ms q
-    liftIO $ stateBackupCount dir `shouldReturn` 0
-
-    q1 <- ExceptT $ getQueue ms SRecipient rId
-    Just (Message {}, True) <- write q1 "message 3"
-    Just (Message {}, False) <- write q1 "message 4"
-    liftIO $ closeMsgQueue ms q1
-    liftIO $ stateBackupCount dir `shouldReturn` 0
-
-    liftIO $ threadDelay 1000000
-    q2 <- ExceptT $ getQueue ms SRecipient rId
-    Just (Message {}, False) <- write q2 "message 5"
-    Nothing <- write q2 "message 5"
-    liftIO $ closeMsgQueue ms q2
-    liftIO $ stateBackupCount dir `shouldReturn` 1
-  where
-    stateBackupCount dir = length . filter (".bak" `isSuffixOf`) <$> listDirectory dir
-
-testExpireIdleQueues :: IO ()
-testExpireIdleQueues = do
-  g <- C.newRandom
-  (rId, qr) <- testNewQueueRec g QMMessaging
-
-  ms <- newMsgStore (testJournalStoreCfg MQStoreCfg) {idleInterval = 0}
-
-  let dir = msgQueueDirectory ms rId
-      statePath = msgQueueStatePath dir rId
-      write q s = writeMsg ms q True =<< mkMessage s
-
-  q <- runRight $ do
-    q <- ExceptT $ addQueue ms rId qr
-    Just (Message {msgId = mId1}, True) <- write q "message 1"
-    Just (Message {msgId = mId2}, False) <- write q "message 2"
-    (Msg "message 1", Msg "message 2") <- tryDelPeekMsg ms q mId1
-    (Msg "message 2", Nothing) <- tryDelPeekMsg ms q mId2
-    liftIO $ closeMsgQueue ms q
-    pure q
-
-  (Just MsgQueueState {size = 0, readState = rs, writeState = ws}, True) <- readQueueState ms statePath
-  msgCount rs `shouldBe` 2
-  msgCount ws `shouldBe` 2
-
-  old <- expireBeforeEpoch ExpirationConfig {ttl = 1, checkInterval = 1} -- no old messages
-  now <- systemSeconds <$> getSystemTime
-
-  (expired_, stored) <- runRight $ isolateQueue ms q "" $ withIdleMsgQueue now ms q $ deleteExpireMsgs_ old q
-  expired_ `shouldBe` Just 0
-  stored `shouldBe` 0
-  (Nothing, False) <- readQueueState ms statePath
-  pure ()
-
-testReadFileMissing :: JournalMsgStore s -> IO ()
-testReadFileMissing ms = do
-  g <- C.newRandom
-  (rId, qr) <- testNewQueueRec g QMMessaging
-  let write q s = writeMsg ms q True =<< mkMessage s
-  q <- runRight $ do
-    q <- ExceptT $ addQueue ms rId qr
-    Just (Message {}, True) <- write q "message 1"
-    Msg "message 1" <- tryPeekMsg ms q
-    pure q
-
-  mq <- fromJust <$> readTVarIO (msgQueue' q)
-  MsgQueueState {readState = rs} <- readTVarIO $ state mq
-  closeMsgQueue ms q
-  let path = journalFilePath (queueDirectory $ queue mq) $ journalId rs
-  removeFile path
-
-  runRight_ $ do
-    q' <- ExceptT $ getQueue ms SRecipient rId
-    Nothing <- tryPeekMsg ms q'
-    Just (Message {}, True) <- write q' "message 2"
-    Msg "message 2" <- tryPeekMsg ms q'
-    pure ()
-
-testReadFileMissingSwitch :: JournalMsgStore s -> IO ()
-testReadFileMissingSwitch ms = do
-  g <- C.newRandom
-  (rId, qr) <- testNewQueueRec g QMMessaging
-  q <- writeMessages ms rId qr
-
-  mq <- fromJust <$> readTVarIO (msgQueue' q)
-  MsgQueueState {readState = rs} <- readTVarIO $ state mq
-  closeMsgQueue ms q
-  let path = journalFilePath (queueDirectory $ queue mq) $ journalId rs
-  removeFile path
-
-  runRight_ $ do
-    q' <- ExceptT $ getQueue ms SRecipient rId
-    Just (Message {}, False) <- writeMsg ms q' True =<< mkMessage "message 6"
-    Msg "message 5" <- tryPeekMsg ms q'
-    pure ()
-
-testWriteFileMissing :: JournalMsgStore s -> IO ()
-testWriteFileMissing ms = do
-  g <- C.newRandom
-  (rId, qr) <- testNewQueueRec g QMMessaging
-  q <- writeMessages ms rId qr
-
-  mq <- fromJust <$> readTVarIO (msgQueue' q)
-  MsgQueueState {writeState = ws} <- readTVarIO $ state mq
-  closeMsgQueue ms q
-  let path = journalFilePath (queueDirectory $ queue mq) $ journalId ws
-  print path
-  removeFile path
-
-  runRight_ $ do
-    q' <- ExceptT $ getQueue ms SRecipient rId
-    Just Message {msgId = mId3} <- tryPeekMsg ms q'
-    (Msg "message 3", Msg "message 4") <- tryDelPeekMsg ms q' mId3
-    Just Message {msgId = mId4} <- tryPeekMsg ms q'
-    (Msg "message 4", Nothing) <- tryDelPeekMsg ms q' mId4
-    Just (Message {}, True) <- writeMsg ms q' True =<< mkMessage "message 6"
-    Msg "message 6" <- tryPeekMsg ms q'
-    pure ()
-
-testReadAndWriteFilesMissing :: JournalMsgStore s -> IO ()
-testReadAndWriteFilesMissing ms = do
-  g <- C.newRandom
-  (rId, qr) <- testNewQueueRec g QMMessaging
-  q <- writeMessages ms rId qr
-
-  mq <- fromJust <$> readTVarIO (msgQueue' q)
-  MsgQueueState {readState = rs, writeState = ws} <- readTVarIO $ state mq
-  closeMsgQueue ms q
-  removeFile $ journalFilePath (queueDirectory $ queue mq) $ journalId rs
-  removeFile $ journalFilePath (queueDirectory $ queue mq) $ journalId ws
-
-  runRight_ $ do
-    q' <- ExceptT $ getQueue ms SRecipient rId
-    Nothing <- tryPeekMsg ms q'
-    Just (Message {}, True) <- writeMsg ms q' True =<< mkMessage "message 6"
-    Msg "message 6" <- tryPeekMsg ms q'
-    pure ()
-
-writeMessages :: JournalMsgStore s -> RecipientId -> QueueRec -> IO (JournalQueue s)
-writeMessages ms rId qr = runRight $ do
-  q <- ExceptT $ addQueue ms rId qr
-  let write s = writeMsg ms q True =<< mkMessage s
-  Just (Message {msgId = mId1}, True) <- write "message 1"
-  Just (Message {msgId = mId2}, False) <- write "message 2"
-  Just (Message {}, False) <- write "message 3"
-  (Msg "message 1", Msg "message 2") <- tryDelPeekMsg ms q mId1
-  (Msg "message 2", Msg "message 3") <- tryDelPeekMsg ms q mId2
-  Just (Message {}, False) <- write "message 4"
-  Just (Message {}, False) <- write "message 5"
-  pure q
@@ -2,15 +2,18 @@
 {-# LANGUAGE NamedFieldPuns #-}
 {-# LANGUAGE OverloadedLists #-}
 {-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE PatternSynonyms #-}
 {-# LANGUAGE TypeApplications #-}
 {-# OPTIONS_GHC -fno-warn-ambiguous-fields #-}

 module CoreTests.SOCKSSettings where

 import Network.Socket (SockAddr (..), tupleToHostAddress)
+import Simplex.Messaging.Agent.Client (ipAddressProtected)
 import Simplex.Messaging.Client
+import qualified Simplex.Messaging.Crypto as C
 import Simplex.Messaging.Encoding.String
-import Simplex.Messaging.Protocol (ErrorType)
+import Simplex.Messaging.Protocol (ErrorType, pattern SMPServer)
 import Simplex.Messaging.Transport.Client
 import Test.Hspec hiding (fit, it)
 import Util
@@ -19,6 +22,7 @@ socksSettingsTests :: Spec
 socksSettingsTests = do
  describe "hostMode and requiredHostMode settings" testHostMode
  describe "socksMode setting, independent of hostMode setting" testSocksMode
+  describe "ipAddressProtected, consistent with chosen host and socksMode" testIPAddressProtected
  describe "socks proxy address encoding" testSocksProxyEncoding

 testPublicHost :: TransportHost
@@ -94,6 +98,19 @@ testSocksMode = do
      let TransportClientConfig {socksProxy} = transportClientConfig cfg NRMInteractive host False Nothing
       in socksProxy

+testIPAddressProtected :: Spec
+testIPAddressProtected = do
+  it "should be protected if SOCKS proxy is used for the chosen host" $ do
+    protected SMAlways HMOnionViaSocks [testPublicHost] `shouldBe` True
+    protected SMOnion HMOnionViaSocks [testPublicHost, testOnionHost] `shouldBe` True
+    protected SMOnion HMPublic [testOnionHost] `shouldBe` True
+  it "should not be protected if SOCKS proxy is not used for the chosen host" $ do
+    protected SMOnion HMOnionViaSocks [testPublicHost] `shouldBe` False
+    protected SMOnion HMPublic [testPublicHost, testOnionHost] `shouldBe` False
+  where
+    protected socksMode hostMode hosts =
+      ipAddressProtected defaultNetworkConfig {socksProxy = Just defaultSocksProxyWithAuth, socksMode, hostMode} (SMPServer hosts "" (C.KeyHash ""))
+
 testSocksProxyEncoding :: Spec
 testSocksProxyEncoding = do
  it "should decode SOCKS proxy with isolate-by-auth mode" $ do
@@ -22,13 +22,13 @@ import qualified Data.X509.Validation as XV
 import SMPClient
 import Simplex.FileTransfer.Protocol (FileInfo (..))
 import Simplex.FileTransfer.Server.Store (FileRec (..), FileRecipient (..), FileStoreClass (..), RoundedFileTime, STMFileStore (..))
-import Simplex.FileTransfer.Server.StoreLog (FileStoreLogRecord (..), readWriteFileStore)
+import Simplex.FileTransfer.Server.StoreLog (FileStoreLogRecord (..), readWriteFileStore, serverFileName)
 import qualified Simplex.Messaging.Crypto as C
 import Simplex.Messaging.Encoding.String
 import Simplex.Messaging.Protocol
 import Simplex.Messaging.Protocol.Types (ClientNotice (..))
 import Simplex.Messaging.Server.Env.STM (readWriteQueueStore)
-import Simplex.Messaging.Server.MsgStore.Journal
+import Simplex.Messaging.Server.MsgStore.STM (STMMsgStore (..), STMStoreConfig (..))
 import Simplex.Messaging.Server.MsgStore.Types
 import Simplex.Messaging.Server.QueueStore
 import Simplex.Messaging.Server.QueueStore.STM (STMQueueStore (..))
@@ -37,6 +37,7 @@ import Simplex.Messaging.Server.StoreLog
 import Simplex.Messaging.SystemTime
 import Simplex.Messaging.Transport (SMPServiceRole (..))
 import Simplex.Messaging.Transport.Credentials (genCredentials)
+import System.FilePath ((</>))
 import Test.Hspec hiding (fit, it)
 import Util

@@ -47,18 +48,6 @@ import Simplex.Messaging.Server.Main
 testPublicAuthKey :: C.APublicAuthKey
 testPublicAuthKey = C.APublicAuthKey C.SEd25519 (C.publicKey "MC4CAQAwBQYDK2VwBCIEIDfEfevydXXfKajz3sRkcQ7RPvfWUPoq6pu1TYHV1DEe")

-testNtfCreds :: TVar ChaChaDRG -> IO NtfCreds
-testNtfCreds g = do
-  (notifierKey, _) <- atomically $ C.generateAuthKeyPair C.SX25519 g
-  (k, pk) <- atomically $ C.generateKeyPair @'C.X25519 g
-  pure
-    NtfCreds
-      { notifierId = EntityId "ijkl",
-        notifierKey,
-        rcvNtfDhSecret = C.dh' k pk,
-        ntfServiceId = Nothing
-      }
-
 data StoreLogTestCase r s = SLTC {name :: String, saved :: [r], state :: s, compacted :: [r]}

 type SMPStoreLogTestCase = StoreLogTestCase StoreLogRecord (M.Map RecipientId QueueRec)
@@ -176,10 +165,10 @@ testSMPStoreLog testSuite tests =
    closeStoreLog l
    replicateM_ 3 $ testReadWrite t
 #if defined(dbServerPostgres)
-    (sCnt, qCnt) <- importStoreLogToDatabase "tests/tmp/" testStoreLogFile testStoreDBOpts
+    (sCnt, qCnt) <- importStoreLogToDatabase testStoreLogFile testStoreDBOpts
    fromIntegral (sCnt + qCnt) `shouldBe` length (compacted t)
    imported <- B.readFile $ testStoreLogFile <> ".bak"
-    (sCnt', qCnt') <- exportDatabaseToStoreLog "tests/tmp/" testStoreDBOpts testStoreLogFile
+    (sCnt', qCnt') <- exportDatabaseToStoreLog testStoreDBOpts testStoreLogFile
    sCnt' `shouldBe` fromIntegral sCnt
    qCnt' `shouldBe` fromIntegral qCnt
    exported <- B.readFile testStoreLogFile
@@ -187,14 +176,14 @@ testSMPStoreLog testSuite tests =
 #endif
  where
    testReadWrite SLTC {compacted, state} = do
-      st <- newMsgStore $ testJournalStoreCfg MQStoreCfg
-      l <- readWriteQueueStore True (mkQueue st True) testStoreLogFile $ stmQueueStore st
+      st <- newMsgStore STMStoreConfig {storePath = Nothing, quota = 3}
+      l <- readWriteQueueStore True (mkQueue st) testStoreLogFile $ queueStore st
      storeState st `shouldReturn` state
      closeStoreLog l
      ([], compacted') <- partitionEithers . map strDecode . B.lines <$> B.readFile testStoreLogFile
      compacted' `shouldBe` compacted
-    storeState :: JournalMsgStore 'QSMemory -> IO (M.Map RecipientId QueueRec)
-    storeState st = M.mapMaybe id <$> (readTVarIO (queues $ stmQueueStore st) >>= mapM (readTVarIO . queueRec))
+    storeState :: STMMsgStore -> IO (M.Map RecipientId QueueRec)
+    storeState st = M.mapMaybe id <$> (readTVarIO (queues $ queueStore st) >>= mapM (readTVarIO . queueRec))

 type FileRecState = (FileInfo, RoundedFileTime, Maybe RoundedFileTime, ServerEntityStatus)

@@ -269,6 +258,18 @@ fileStoreLogTests = do
          saved = [AddFile sId file createdAt Nothing EntityActive, BlockFile sId blockedWithNotice],
          compacted = [AddFile sId file createdAt Nothing (EntityBlocked blockedWithNotice)],
          state = M.fromList [(sId, (file, createdAt, Nothing, EntityBlocked blockedWithNotice))]
+        },
+      SLTC
+        { name = "put file with wrong file name",
+          saved = [AddFile sId file createdAt (Just expiresAt) EntityActive, PutFile sId "/etc/passwd"],
+          compacted = [AddFile sId file createdAt (Just expiresAt) EntityActive],
+          state = M.fromList [(sId, (file, createdAt, Just expiresAt, EntityActive))]
+        },
+      SLTC
+        { name = "put file in another directory",
+          saved = [AddFile sId file createdAt (Just expiresAt) EntityActive, PutFile sId ("/srv/xftp" </> serverFileName sId)],
+          compacted = [AddFile sId file createdAt (Just expiresAt) EntityActive, PutFile sId ("/srv/xftp" </> serverFileName sId)],
+          state = M.fromList [(sId, (file, createdAt, Just expiresAt, EntityActive))]
        }
    ]

@@ -9,7 +9,7 @@ import Data.Word (Word32)
 import Simplex.FileTransfer.Protocol (FileInfo (..), SFileParty (..))
 import Simplex.FileTransfer.Server.Store
 import Simplex.FileTransfer.Server.Store.Postgres (PostgresFileStore, importFileStore, exportFileStore)
-import Simplex.FileTransfer.Server.StoreLog (closeStoreLog, readWriteFileStore, writeFileStore)
+import Simplex.FileTransfer.Server.StoreLog (closeStoreLog, readWriteFileStore, serverFileName, writeFileStore)
 import Simplex.FileTransfer.Transport (XFTPErrorType (..))
 import qualified Simplex.Messaging.Crypto as C
 import Simplex.Messaging.Protocol (BlockingInfo (..), BlockingReason (..), EntityId (..))
@@ -17,6 +17,7 @@ import Simplex.Messaging.Server.QueueStore (ServerEntityStatus (..))
 import Simplex.Messaging.Server.StoreLog (openWriteStoreLog)
 import Simplex.Messaging.SystemTime (RoundedSystemTime (..))
 import System.Directory (doesFileExist, removeFile)
+import System.FilePath ((</>))
 import Test.Hspec hiding (fit, it)
 import UnliftIO.STM
 import Util
@@ -265,7 +266,7 @@ testMigrationRoundTrip = do
      sId2 = EntityId "migration_file_2"
      rId1 = EntityId "migration_rcp_1_"
  addFile stmStore sId1 fileInfo1 testCreatedAt (Just testExpiresAt) EntityActive `shouldReturn` Right ()
-  void $ setFilePath stmStore sId1 "/tmp/file1"
+  void $ setFilePath stmStore sId1 ("/tmp" </> serverFileName sId1)
  addRecipient stmStore sId1 (FileRecipient rId1 rcpKey1) `shouldReturn` Right ()
  let testBlockInfo = BlockingInfo {reason = BRSpam, notice = Nothing}
  addFile stmStore sId2 fileInfo2 testCreatedAt Nothing (EntityBlocked testBlockInfo) `shouldReturn` Right ()
@@ -289,7 +290,7 @@ testMigrationRoundTrip = do
  case result1 of
    Right (FileRec {fileInfo = fi, filePath, expiresAt, fileStatus}, _) -> do
      size fi `shouldBe` 128000
-      readTVarIO filePath `shouldReturn` Just "/tmp/file1"
+      readTVarIO filePath `shouldReturn` Just ("/tmp" </> serverFileName sId1)
      expiresAt `shouldBe` Just testExpiresAt
      readTVarIO fileStatus `shouldReturn` EntityActive
    Left e -> expectationFailure $ "getFile sId1 failed: " <> show e
@@ -9,13 +9,15 @@ module RemoteControl where

 import AgentTests.FunctionalAPITests (runRight)
 import Control.Logger.Simple
+import Control.Monad (void)
+import Control.Monad.Trans.Except (runExceptT)
 import Crypto.Random (ChaChaDRG)
 import qualified Data.Aeson as J
 import qualified Data.ByteString.Char8 as B
 import qualified Data.ByteString.Lazy.Char8 as LB
 import Data.List (stripPrefix)
 import Data.List.NonEmpty (NonEmpty (..))
-import Data.Time.Clock.System (SystemTime (..))
+import Data.Time.Clock.System (SystemTime (..), getSystemTime)
 import qualified Simplex.Messaging.Crypto as C
 import Simplex.Messaging.Encoding.String (StrEncoding (..))
 import Simplex.Messaging.Transport (TSbChainKeys (..))
@@ -24,8 +26,10 @@ import qualified Simplex.RemoteControl.Client as HC (RCHostClient (action))
 import qualified Simplex.RemoteControl.Client as RC
 import Simplex.RemoteControl.Discovery (mkLastLocalHost, preferAddress)
 import Simplex.RemoteControl.Invitation
-  ( RCInvitation (..),
+  ( RCEncInvitation (..),
+    RCInvitation (..),
    RCSignedInvitation,
+    signInvitation,
    verifySignedInvitation,
  )
 import Simplex.RemoteControl.Types
@@ -45,6 +49,7 @@ remoteControlTests = do
    it "should connect to existing pairing" testExistingPairing
  describe "Multicast discovery" $ do
    it "should find paired host and connect" testMulticast
+    it "should accept announcement only within timestamp window" testAnnouncementTimestamp

 testPreferAddress :: Spec
 testPreferAddress = do
@@ -244,6 +249,26 @@ testMulticast = do
        Nothing -> fail "timeout"
        Just _ -> pure ()

+testAnnouncementTimestamp :: IO ()
+testAnnouncementTimestamp = do
+  drg <- C.newRandom
+  RCHostPairing {caKey, caCert, idPrivKey} <- RC.newRCHostPairing drg
+  (hostDhPubKey, dhPrivKey) <- atomically $ C.generateKeyPair @'C.X25519 drg
+  (skey, sessPrivKey) <- atomically $ C.generateKeyPair @'C.Ed25519 drg
+  (dh, ctrlDhPrivKey) <- atomically $ C.generateKeyPair @'C.X25519 drg
+  nonce <- atomically $ C.randomCbNonce drg
+  now <- systemSeconds <$> getSystemTime
+  let pairing = RCCtrlPairing {caKey, caCert, ctrlFingerprint = C.KeyHash "test-ca", idPubKey = C.publicKey idPrivKey, dhPrivKey, prevDhPrivKey = Nothing}
+      announce offset = do
+        let inv = RCInvitation {ca = C.KeyHash "test-ca", host = "127.0.0.1", port = 5223, v = supportedRCPVRange, app = J.String "app", ts = MkSystemTime (now + offset) 0, skey, idkey = C.publicKey idPrivKey, dh}
+        encInvitation <- either (fail . show) pure $ C.cbEncrypt (C.dh' hostDhPubKey ctrlDhPrivKey) nonce (strEncode $ signInvitation sessPrivKey idPrivKey inv) 900
+        runExceptT . void $ RC.findRCCtrlPairing (pairing :| []) RCEncInvitation {dhPubKey = dh, nonce, encInvitation}
+  announce 0 `shouldReturn` Right ()
+  announce (-3600) `shouldReturn` Right ()
+  announce 3500 `shouldReturn` Right ()
+  announce (-3700) `shouldReturn` Left RCEInvitation
+  announce 3700 `shouldReturn` Left RCEInvitation
+
 runCtrl :: TVar ChaChaDRG -> Bool -> RCHostPairing -> MVar RCSignedInvitation -> IO (Async RCHostPairing)
 runCtrl drg multicast hp invVar = async . runRight $ do
  (_found, inv, hc, r) <- RC.connectRCHost drg hp (J.String "app") multicast Nothing Nothing
@@ -33,7 +33,6 @@ import Simplex.Messaging.Protocol
 import Simplex.Messaging.Server (runSMPServerBlocking)
 import Simplex.Messaging.Server.Env.STM
 import Simplex.Messaging.Server.MsgStore.Types (MsgStoreClass (..), SMSType (..), SQSType (..))
-import Simplex.Messaging.Server.QueueStore.Postgres.Config (PostgresStoreCfg (..))
 import Simplex.Messaging.Transport
 import Simplex.Messaging.Transport.Client
 import Simplex.Messaging.Transport.Server
@@ -51,6 +50,7 @@ import Util

 #if defined(dbServerPostgres)
 import Database.PostgreSQL.Simple (defaultConnectInfo)
+import Simplex.Messaging.Server.QueueStore.Postgres.Config (PostgresStoreCfg (..))
 #endif

 #if defined(dbPostgres) || defined(dbServerPostgres)
@@ -123,12 +123,6 @@ testStoreMsgsFile = "tests/tmp/smp-server-messages.log"
 testStoreMsgsFile2 :: FilePath
 testStoreMsgsFile2 = "tests/tmp/smp-server-messages.log.2"

-testStoreMsgsDir :: FilePath
-testStoreMsgsDir = "tests/tmp/messages"
-
-testStoreMsgsDir2 :: FilePath
-testStoreMsgsDir2 = "tests/tmp/messages.2"
-
 testStoreNtfsFile :: FilePath
 testStoreNtfsFile = "tests/tmp/smp-server-ntfs.log"

@@ -212,24 +206,35 @@ ntfTestServerCredentials =
    }

 cfg :: AServerConfig
-cfg = cfgMS (ASType SQSMemory SMSJournal)
+cfg = cfgMS (ASType SQSMemory SMSMemory)

-cfgJ2 :: AServerConfig
-cfgJ2 = journalCfg cfg testStoreLogFile2 testStoreMsgsDir2
+cfgS2 :: AServerConfig
+cfgS2 = memoryCfg cfg testStoreLogFile2 testStoreMsgsFile2

-cfgJ2QS :: SQSType s -> AServerConfig
-cfgJ2QS = \case
-  SQSMemory -> journalCfg (cfgMS $ ASType SQSMemory SMSJournal) testStoreLogFile2 testStoreMsgsDir2
-  SQSPostgres -> journalCfgDB (cfgMS $ ASType SQSPostgres SMSJournal) testStoreDBOpts2 testStoreMsgsDir2
+cfgS2QS :: SQSType s -> AServerConfig
+cfgS2QS = \case
+  SQSMemory -> memoryCfg (cfgMS $ ASType SQSMemory SMSMemory) testStoreLogFile2 testStoreMsgsFile2
+  SQSPostgres -> databaseCfg (cfgMS postgresStoreType) testStoreDBOpts2

-journalCfg :: AServerConfig -> FilePath -> FilePath -> AServerConfig
-journalCfg (ASrvCfg _ _ cfg') storeLogFile storeMsgsPath =
-  ASrvCfg SQSMemory SMSJournal cfg' {serverStoreCfg = SSCMemoryJournal {storeLogFile, storeMsgsPath}}
+memoryCfg :: AServerConfig -> FilePath -> FilePath -> AServerConfig
+memoryCfg (ASrvCfg _ _ cfg') storeLogFile storeMsgsFile =
+  ASrvCfg SQSMemory SMSMemory cfg' {serverStoreCfg = SSCMemory $ Just StorePaths {storeLogFile, storeMsgsFile = Just storeMsgsFile}}

-journalCfgDB :: AServerConfig -> DBOpts -> FilePath -> AServerConfig
-journalCfgDB (ASrvCfg _ _ cfg') dbOpts storeMsgsPath' =
+databaseCfg :: AServerConfig -> DBOpts -> AServerConfig
+#if defined(dbServerPostgres)
+databaseCfg (ASrvCfg _ _ cfg') dbOpts =
  let storeCfg = PostgresStoreCfg {dbOpts, dbStoreLogPath = Nothing, confirmMigrations = MCYesUp, deletedTTL = 86400}
-   in ASrvCfg SQSPostgres SMSJournal cfg' {serverStoreCfg = SSCDatabaseJournal {storeCfg, storeMsgsPath'}}
+   in ASrvCfg SQSPostgres SMSPostgres cfg' {serverStoreCfg = SSCDatabase storeCfg}
+#else
+databaseCfg _ _ = error "no dbServerPostgres flag"
+#endif
+
+postgresStoreType :: AStoreType
+#if defined(dbServerPostgres)
+postgresStoreType = ASType SQSPostgres SMSPostgres
+#else
+postgresStoreType = error "no dbServerPostgres flag"
+#endif

 cfgMS :: AStoreType -> AServerConfig
 cfgMS msType = withStoreCfg (testServerStoreConfig msType) $ \serverStoreCfg ->
@@ -238,8 +243,6 @@ cfgMS msType = withStoreCfg (testServerStoreConfig msType) $ \serverStoreCfg ->
      smpHandshakeTimeout = 60000000,
      tbqSize = 4,
      msgQueueQuota = 4,
-      maxJournalMsgCount = 5,
-      maxJournalStateLines = 2,
      queueIdBytes = 24,
      msgIdBytes = 24,
      serverStoreCfg,
@@ -252,7 +255,6 @@ cfgMS msType = withStoreCfg (testServerStoreConfig msType) $ \serverStoreCfg ->
      messageExpiration = Just defaultMessageExpiration,
      expireMessagesOnStart = True,
      expireMessagesOnSend = False,
-      idleQueueInterval = defaultIdleQueueInterval,
      notificationExpiration = defaultNtfExpiration,
      inactiveClientExpiration = Just defaultInactiveClientExpiration,
      logStatsInterval = Nothing,
@@ -292,20 +294,21 @@ testServerStoreConfig :: AStoreType -> AServerStoreCfg
 testServerStoreConfig = serverStoreConfig_ False

 serverStoreConfig_ :: Bool -> AStoreType -> AServerStoreCfg
+#if defined(dbServerPostgres)
 serverStoreConfig_ useDbStoreLog = \case
+#else
+serverStoreConfig_ _ = \case
+#endif
  ASType SQSMemory SMSMemory ->
    ASSCfg SQSMemory SMSMemory $ SSCMemory $ Just StorePaths {storeLogFile = testStoreLogFile, storeMsgsFile = Just testStoreMsgsFile}
-  ASType SQSMemory SMSJournal ->
-    ASSCfg SQSMemory SMSJournal $ SSCMemoryJournal {storeLogFile = testStoreLogFile, storeMsgsPath = testStoreMsgsDir}
-  ASType SQSPostgres SMSJournal ->
-    ASSCfg SQSPostgres SMSJournal SSCDatabaseJournal {storeCfg, storeMsgsPath' = testStoreMsgsDir}
 #if defined(dbServerPostgres)
  ASType SQSPostgres SMSPostgres ->
-    ASSCfg SQSPostgres SMSPostgres $ SSCDatabase storeCfg
+    let dbStoreLogPath = if useDbStoreLog then Just testStoreLogFile else Nothing
+        storeCfg = PostgresStoreCfg {dbOpts = testStoreDBOpts, dbStoreLogPath, confirmMigrations = MCYesUp, deletedTTL = 86400}
+     in ASSCfg SQSPostgres SMSPostgres $ SSCDatabase storeCfg
+#else
+  ASType SQSPostgres _ -> error "no dbServerPostgres flag"
 #endif
-  where
-    dbStoreLogPath = if useDbStoreLog then Just testStoreLogFile else Nothing
-    storeCfg = PostgresStoreCfg {dbOpts = testStoreDBOpts, dbStoreLogPath, confirmMigrations = MCYesUp, deletedTTL = 86400}

 cfgVPrev :: AStoreType -> AServerConfig
 cfgVPrev msType = updateCfg (cfgMS msType) $ \cfg' -> cfg' {smpServerVRange = prevRange $ smpServerVRange cfg'}
@@ -316,8 +319,11 @@ prevRange vr = vr {maxVersion = max (minVersion vr) (prevVersion $ maxVersion vr
 prevVersion :: Version v -> Version v
 prevVersion (Version v) = Version (v - 1)

+nextVersion :: Version v -> Version v
+nextVersion (Version v) = Version (v + 1)
+
 proxyCfg :: AServerConfig
-proxyCfg = proxyCfgMS (ASType SQSMemory SMSJournal)
+proxyCfg = proxyCfgMS (ASType SQSMemory SMSMemory)

 proxyCfgMS :: AStoreType -> AServerConfig
 proxyCfgMS msType =
@@ -328,13 +334,13 @@ proxyCfgMS msType =
            smpAgentCfg = smpAgentCfg' {smpCfg = (smpCfg smpAgentCfg') {agreeSecret = True, proxyServer = True, serverVRange = supportedProxyClientSMPRelayVRange}}
          }

-proxyCfgJ2 :: AServerConfig
-proxyCfgJ2 = journalCfg proxyCfg testStoreLogFile2 testStoreMsgsDir2
+proxyCfgS2 :: AServerConfig
+proxyCfgS2 = memoryCfg proxyCfg testStoreLogFile2 testStoreMsgsFile2

-proxyCfgJ2QS :: SQSType qs -> AServerConfig
-proxyCfgJ2QS = \case
-  SQSMemory -> journalCfg (proxyCfgMS $ ASType SQSMemory SMSJournal) testStoreLogFile2 testStoreMsgsDir2
-  SQSPostgres -> journalCfgDB (proxyCfgMS $ ASType SQSPostgres SMSJournal) testStoreDBOpts2 testStoreMsgsDir2
+proxyCfgS2QS :: SQSType qs -> AServerConfig
+proxyCfgS2QS = \case
+  SQSMemory -> memoryCfg (proxyCfgMS $ ASType SQSMemory SMSMemory) testStoreLogFile2 testStoreMsgsFile2
+  SQSPostgres -> databaseCfg (proxyCfgMS postgresStoreType) testStoreDBOpts2

 -- Proxy config with a short relay-connection timeout, to bound how long a failing
 -- proxy->relay connection attempt blocks in the relay reconnection tests.
@@ -346,6 +352,16 @@ proxyCfgShortTimeout =
        nt = NetworkTimeout {backgroundTimeout = 4_000000, interactiveTimeout = 4_000000}
     in cfg' {smpAgentCfg = aCfg {smpCfg = cCfg {networkConfig = (networkConfig cCfg) {tcpConnectTimeout = nt}}}}

+proxyCfgVPrev :: AStoreType -> AServerConfig
+proxyCfgVPrev msType =
+  updateCfg (proxyCfgMS msType) $ \cfg' ->
+    let aCfg = smpAgentCfg cfg'
+        cCfg = smpCfg aCfg
+     in cfg'
+          { smpServerVRange = prevRange $ smpServerVRange cfg',
+            smpAgentCfg = aCfg {smpCfg = cCfg {serverVRange = prevRange $ serverVRange cCfg}}
+          }
+
 withSmpServerStoreMsgLogOn :: HasCallStack => (ASrvTransport, AStoreType) -> ServiceName -> (HasCallStack => ThreadId -> IO a) -> IO a
 withSmpServerStoreMsgLogOn (t, msType) =
  withSmpServerConfigOn t $ updateCfg (cfgMS msType) $ \cfg' -> cfg' {storeNtfsFile = Just testStoreNtfsFile, serverStatsBackupFile = Just testServerStatsBackupFile}
@@ -406,10 +422,10 @@ withSmpServerProxy :: HasCallStack => (ASrvTransport, AStoreType) -> IO a -> IO
 withSmpServerProxy (t, msType) = withSmpServerConfigOn t (proxyCfgMS msType) testPort . const

 withSmpServers2 :: HasCallStack => (ASrvTransport, AStoreType) -> IO a -> IO a
-withSmpServers2 ps@(t, ASType qs _ms) = withSmpServer ps . withSmpServerConfigOn t (cfgJ2QS qs) testPort2 . const
+withSmpServers2 ps@(t, ASType qs _ms) = withSmpServer ps . withSmpServerConfigOn t (cfgS2QS qs) testPort2 . const

 withSmpServersProxy2 :: HasCallStack => (ASrvTransport, AStoreType) -> IO a -> IO a
-withSmpServersProxy2 ps@(t, ASType qs _ms) = withSmpServerProxy ps . withSmpServerConfigOn t (proxyCfgJ2QS qs) testPort2 . const
+withSmpServersProxy2 ps@(t, ASType qs _ms) = withSmpServerProxy ps . withSmpServerConfigOn t (proxyCfgS2QS qs) testPort2 . const

 runSmpTest :: forall c a. (HasCallStack, Transport c) => AStoreType -> (HasCallStack => THandleSMP c 'TClient -> IO a) -> IO a
 runSmpTest msType test = withSmpServerConfigOn (transport @c) (cfgMS msType) testPort $ \_ -> testSMPClient test
@@ -430,7 +446,7 @@ smpServerTest ::
  TProxy c 'TServer ->
  (Maybe TAuthorizations, ByteString, ByteString, smp) ->
  IO (Maybe TAuthorizations, ByteString, ByteString, BrokerMsg)
-smpServerTest _ t = runSmpTest (ASType SQSMemory SMSJournal) $ \h -> tPut' h t >> tGet' h
+smpServerTest _ t = runSmpTest (ASType SQSMemory SMSMemory) $ \h -> tPut' h t >> tGet' h
  where
    tPut' :: THandleSMP c 'TClient -> (Maybe TAuthorizations, ByteString, ByteString, smp) -> IO ()
    tPut' h@THandle {params = THandleParams {sessionId, implySessId}} (sig, corrId, queueId, smp) = do
@@ -73,6 +73,8 @@ smpProxyTests = do
      xit "no SMP service at host/port" todo
      xit "bad SMP fingerprint" todo
    xit "batching proxy requests" todo
+    it "relay rejects forwarded command with changed version" $ \_ ->
+      testChangedFwdVersion
  describe "deliver message via SMP proxy" $ do
    let srv1 = SMPServer testHost testPort testKeyHash
        srv2 = SMPServer testHost2 testPort2 testKeyHash
@@ -95,6 +97,11 @@ smpProxyTests = do
          deliverMessageViaProxy proxyServ relayServ C.SEd25519 msg1 msg2
        it "max message size, X25519 keys" . twoServersFirstProxy $
          deliverMessageViaProxy proxyServ relayServ C.SX25519 msg1 msg2
+      describe "version compatibility" $ do
+        let deliver clientVR = deliverMessagesViaProxyVR clientVR srv1 srv2 C.SEd448 ["hello 1"] ["hello 2"]
+        it "prev client" . twoServersFirstProxy $ deliver (prevRange supportedClientSMPRelayVRange)
+        it "prev proxy" . twoServersPrevProxy $ deliver supportedClientSMPRelayVRange
+        it "prev relay" . twoServersPrevRelay $ deliver supportedClientSMPRelayVRange
      describe "stress test 1k" $ do
        let deliver n = deliverMessagesViaProxy srv1 srv2 C.SEd448 [] (map bshow [1 :: Int .. n])
        it "1x1000" . twoServersFirstProxy $ deliver 1000
@@ -150,28 +157,34 @@ smpProxyTests = do
        let deliver nAgents nMsgs = agentDeliverMessagesViaProxyConc (replicate nAgents [srv1]) (map bshow [1 :: Int .. nMsgs])
        it "25 agents, 300 pairs, 17 messages" . oneServer . withNumCapabilities 4 $ deliver 25 17
  where
-    oneServer test msType = withSmpServerConfigOn (transport @TLS) (updateCfg (proxyCfgMS msType) $ \cfg_ -> cfg_ {msgQueueQuota = 128, maxJournalMsgCount = 256}) testPort $ const test
+    oneServer test msType = withSmpServerConfigOn (transport @TLS) (updateCfg (proxyCfgMS msType) $ \cfg_ -> cfg_ {msgQueueQuota = 128}) testPort $ const test
    twoServers test msType = twoServers_ (proxyCfgMS msType) (proxyCfgMS msType) test msType
-    twoServersFirstProxy test msType = twoServers_ (proxyCfgMS msType) (updateCfg (cfgMS msType) $ \cfg_ -> cfg_ {msgQueueQuota = 128, maxJournalMsgCount = 256}) test msType
-    twoServersMoreConc test msType = twoServers_ (updateCfg (proxyCfgMS msType) $ \cfg_ -> cfg_ {serverClientConcurrency = 128}) (updateCfg (cfgMS msType) $ \cfg_ -> cfg_ {msgQueueQuota = 128, maxJournalMsgCount = 256}) test msType
-    twoServersNoConc test msType = twoServers_ (updateCfg (proxyCfgMS msType) $ \cfg_ -> cfg_ {serverClientConcurrency = 1}) (updateCfg (cfgMS msType) $ \cfg_ -> cfg_ {msgQueueQuota = 128, maxJournalMsgCount = 256}) test msType
+    twoServersFirstProxy test msType = twoServers_ (proxyCfgMS msType) (updateCfg (cfgMS msType) $ \cfg_ -> cfg_ {msgQueueQuota = 128}) test msType
+    twoServersMoreConc test msType = twoServers_ (updateCfg (proxyCfgMS msType) $ \cfg_ -> cfg_ {serverClientConcurrency = 128}) (updateCfg (cfgMS msType) $ \cfg_ -> cfg_ {msgQueueQuota = 128}) test msType
+    twoServersNoConc test msType = twoServers_ (updateCfg (proxyCfgMS msType) $ \cfg_ -> cfg_ {serverClientConcurrency = 1}) (updateCfg (cfgMS msType) $ \cfg_ -> cfg_ {msgQueueQuota = 128}) test msType
+    twoServersPrevProxy test msType = twoServers_ (proxyCfgVPrev msType) (cfgMS msType) test msType
+    twoServersPrevRelay test msType = twoServers_ (proxyCfgMS msType) (prevServerVRange $ cfgMS msType) test msType
+    prevServerVRange cfg' = updateCfg cfg' $ \cfg_ -> cfg_ {smpServerVRange = prevRange $ smpServerVRange cfg_}
    twoServers_ :: AServerConfig -> AServerConfig -> IO () -> AStoreType -> IO ()
    twoServers_ cfg1 cfg2 runTest (ASType qsType _) =
      withSmpServerConfigOn (transport @TLS) cfg1 testPort $ \_ ->
        let cfg2' = case qsType of
-              SQSMemory -> journalCfg cfg2 testStoreLogFile2 testStoreMsgsDir2
-              SQSPostgres -> journalCfgDB cfg2 testStoreDBOpts2 testStoreMsgsDir2
+              SQSMemory -> memoryCfg cfg2 testStoreLogFile2 testStoreMsgsFile2
+              SQSPostgres -> databaseCfg cfg2 testStoreDBOpts2
         in withSmpServerConfigOn (transport @TLS) cfg2' testPort2 $ const runTest

 deliverMessageViaProxy :: (C.AlgorithmI a, C.AuthAlgorithm a) => SMPServer -> SMPServer -> C.SAlgorithm a -> ByteString -> ByteString -> IO ()
 deliverMessageViaProxy proxyServ relayServ alg msg msg' = deliverMessagesViaProxy proxyServ relayServ alg [msg] [msg']

 deliverMessagesViaProxy :: (C.AlgorithmI a, C.AuthAlgorithm a) => SMPServer -> SMPServer -> C.SAlgorithm a -> [ByteString] -> [ByteString] -> IO ()
-deliverMessagesViaProxy proxyServ relayServ alg unsecuredMsgs securedMsgs = do
+deliverMessagesViaProxy = deliverMessagesViaProxyVR $ mkVersionRange minServerSMPRelayVersion currentClientSMPRelayVersion
+
+deliverMessagesViaProxyVR :: (C.AlgorithmI a, C.AuthAlgorithm a) => VersionRangeSMP -> SMPServer -> SMPServer -> C.SAlgorithm a -> [ByteString] -> [ByteString] -> IO ()
+deliverMessagesViaProxyVR clientVR proxyServ relayServ alg unsecuredMsgs securedMsgs = do
  g <- C.newRandom
  -- set up proxy
  ts <- getCurrentTime
-  pc' <- getProtocolClient g NRMInteractive (1, proxyServ, Nothing) defaultSMPClientConfig {serverVRange = mkVersionRange minServerSMPRelayVersion currentClientSMPRelayVersion} [] Nothing ts (\_ -> pure ())
+  pc' <- getProtocolClient g NRMInteractive (1, proxyServ, Nothing) defaultSMPClientConfig {serverVRange = clientVR, proxiedRelayVRange = clientVR} [] Nothing ts (\_ -> pure ())
  pc <- either (fail . show) pure pc'
  THAuthClient {} <- maybe (fail "getProtocolClient returned no thAuth") pure $ thAuth $ thParams pc
  -- set up relay
@@ -191,6 +204,8 @@ deliverMessagesViaProxy proxyServ relayServ alg unsecuredMsgs securedMsgs = do
  forM_ unsecuredMsgs $ \msg -> do
    runExceptT' (proxySMPMessage pc NRMInteractive sess Nothing sndId noMsgFlags msg) `shouldReturn` Right ()
    runExceptT' (proxySMPMessage pc NRMInteractive sess {prSessionId = "bad session"} Nothing sndId noMsgFlags msg) `shouldReturn` Left (ProxyProtocolError $ SMP.PROXY SMP.NO_SESSION)
+    forM_ ([prevVersion minServerSMPRelayVersion, nextVersion currentServerSMPRelayVersion] :: [VersionSMP]) $ \v ->
+      runExceptT' (proxySMPMessage pc NRMInteractive sess {prVersion = v} Nothing sndId noMsgFlags msg) `shouldReturn` Left (ProxyProtocolError $ SMP.PROXY $ SMP.PROTOCOL $ SMP.PROXY $ SMP.BROKER $ SMP.TRANSPORT TEVersion)
    -- receive 1
    (_tSess, _, [(_entId, STEvent (Right (SMP.MSG RcvMessage {msgId, msgBody = EncRcvMsgBody encBody})))]) <- atomically $ readTBQueue msgQ
    dec msgId encBody `shouldBe` Right msg
@@ -386,15 +401,15 @@ agentViaProxyRetryOffline = do
            ackMessage alice bobId (baseId + 4) Nothing
  where
    withServer :: (ThreadId -> IO a) -> IO a
-    withServer = withServer_ testStoreLogFile testStoreMsgsDir testStoreNtfsFile testPort
+    withServer = withServer_ testStoreLogFile testStoreMsgsFile testStoreNtfsFile testPort
    -- TODO [postgres]
-    -- withServer = withServer_ testStoreDBOpts testStoreMsgsDir testStoreNtfsFile testPort
+    -- withServer = withServer_ testStoreDBOpts testStoreNtfsFile testPort
    withServer2 :: (ThreadId -> IO a) -> IO a
-    withServer2 = withServer_ testStoreLogFile2 testStoreMsgsDir2 testStoreNtfsFile2 testPort2
+    withServer2 = withServer_ testStoreLogFile2 testStoreMsgsFile2 testStoreNtfsFile2 testPort2
    -- TODO [postgres]
-    -- withServer2 = withServer_ testStoreDBOpts2 testStoreMsgsDir2 testStoreNtfsFile2 testPort2
+    -- withServer2 = withServer_ testStoreDBOpts2 testStoreNtfsFile2 testPort2
    withServer_ storeLog storeMsgs storeNtfs =
-      let cfg' = updateCfg (journalCfg proxyCfg storeLog storeMsgs) $ \cfg_ -> cfg_ {storeNtfsFile = Just storeNtfs}
+      let cfg' = updateCfg (memoryCfg proxyCfg storeLog storeMsgs) $ \cfg_ -> cfg_ {storeNtfsFile = Just storeNtfs}
       in withSmpServerConfigOn (transport @TLS) cfg'
    a `up` cId = nGet a =##> \case ("", "", UP _ [c]) -> c == cId; _ -> False
    a `down` cId = nGet a =##> \case ("", "", DOWN _ [c]) -> c == cId; _ -> False
@@ -420,7 +435,7 @@ agentViaProxyRetryNoSession = do
          _ <- runRight $ makeConnection b a
          pure ()
  where
-    withServer2 = withSmpServerConfigOn (transport @TLS) proxyCfgJ2 testPort2
+    withServer2 = withSmpServerConfigOn (transport @TLS) proxyCfgS2 testPort2
    servers srv = initAgentServersProxy {smp = userServers [srv]}

 testNoProxy :: AStoreType -> IO ()
@@ -446,12 +461,27 @@ requestRelaySession =
  testSMPClient_ "localhost" testPort supportedServerSMPRelayVRange Nothing $ \(th :: THandleSMP TLS 'TClient) ->
    (\(_, _, reply) -> reply) <$> sendRecv th (Nothing, "1", NoEntity, SMP.PRXY testSMPServer2 Nothing)

+testChangedFwdVersion :: IO ()
+testChangedFwdVersion =
+  withSmpServerConfigOn (transport @TLS) cfg testPort $ \_ -> do
+    g <- C.newRandom
+    ts <- getCurrentTime
+    rc <- either (fail . show) pure =<< getProtocolClient g NRMInteractive (1, testSMPServer, Nothing) defaultSMPClientConfig [] Nothing ts (\_ -> pure ())
+    THAuthClient {peerServerPubKey} <- maybe (fail "getProtocolClient returned no thAuth") pure $ thAuth $ thParams rc
+    (cmdPubKey, cmdPrivKey) <- atomically $ C.generateKeyPair g
+    nonce@(C.CbNonce corrId) <- atomically $ C.randomCbNonce g
+    let v = currentClientSMPRelayVersion
+    et <- either (fail . show) (pure . SMP.EncTransmission) $ C.cbEncrypt (C.dh' peerServerPubKey cmdPrivKey) (SMP.encTransmissionNonce v nonce) "" SMP.paddedProxiedTLength
+    let forward fwdVersion = forwardSMPTransmission rc (SMP.CorrId corrId) fwdVersion cmdPubKey et
+    _ <- runExceptT' $ forward v
+    runExceptT (forward $ prevVersion v) `shouldReturn` Left (PCEProtocolError SMP.CRYPTO)
+
 -- Shared "phase 2" of the reconnection tests: start a healthy relay, confirm it is reachable
 -- directly (PING, not via the proxy) so a proxy failure can only mean the proxy didn't reconnect,
 -- let any stored connection error expire, then require the proxy to establish the session (PKEY).
 requireProxyReconnect :: IO ()
 requireProxyReconnect =
-  withSmpServerConfigOn (transport @TLS) proxyCfgJ2 testPort2 $ \_ -> do
+  withSmpServerConfigOn (transport @TLS) proxyCfgS2 testPort2 $ \_ -> do
    testSMPClient_ "127.0.0.1" testPort2 supportedServerSMPRelayVRange Nothing $ \(th :: THandleSMP TLS 'TClient) -> do
      (_, _, reply) <- sendRecv th (Nothing, "0", NoEntity, SMP.PING)
      reply `shouldBe` Right SMP.PONG
@@ -507,7 +537,7 @@ testAgentClientReconnectAfterCancel =
      t <- async $ runExceptT $ A.createConnection a NRMInteractive 1 True True SCMInvitation Nothing Nothing CR.IKPQOn False SMSubscribe
      threadDelay 1000000 -- let the connect to the stalling relay start, then kill it mid-flight
      cancel t
-    withSmpServerConfigOn (transport @TLS) cfgJ2 testPort2 $ \_ -> do
+    withSmpServerConfigOn (transport @TLS) cfgS2 testPort2 $ \_ -> do
      testSMPClient_ "127.0.0.1" testPort2 supportedServerSMPRelayVRange Nothing $ \(th :: THandleSMP TLS 'TClient) -> do
        (_, _, reply) <- sendRecv th (Nothing, "0", NoEntity, SMP.PING)
        reply `shouldBe` Right SMP.PONG -- the relay is up and reachable, so a timeout can only be the poisoned var
@@ -24,7 +24,6 @@ import Control.Concurrent.STM
 import Control.Exception (SomeException, throwIO, try)
 import Control.Monad
 import Control.Monad.IO.Class
-import CoreTests.MsgStoreTests (testJournalStoreCfg)
 import Data.Bifunctor (first)
 import qualified Data.ByteString.Base64 as B64
 import Data.ByteString.Char8 (ByteString)
@@ -46,17 +45,14 @@ import Simplex.Messaging.Encoding
 import Simplex.Messaging.Encoding.String
 import Simplex.Messaging.Parsers (parseAll, parseString)
 import Simplex.Messaging.Protocol
-import Simplex.Messaging.Server (exportMessages)
-import Simplex.Messaging.Server.Env.STM (AStoreType (..), MsgStore (..), ServerConfig (..), ServerStoreCfg (..), readWriteQueueStore)
+import Simplex.Messaging.Server.Env.STM (AStoreType (..), ServerConfig (..), ServerStoreCfg (..))
 import Simplex.Messaging.Server.Expiration
-import Simplex.Messaging.Server.MsgStore.Journal (JournalStoreConfig (..), QStoreCfg (..), stmQueueStore)
-import Simplex.Messaging.Server.MsgStore.Types (MsgStoreClass (..), QSType (..), SMSType (..), SQSType (..), newMsgStore)
+import Simplex.Messaging.Server.MsgStore.Types (SMSType (..), SQSType (..))
 import Simplex.Messaging.Server.Stats (PeriodStatsData (..), ServerStatsData (..))
-import Simplex.Messaging.Server.StoreLog (StoreLogRecord (..), closeStoreLog)
+import Simplex.Messaging.Server.StoreLog (StoreLogRecord (..))
 import Simplex.Messaging.Transport
 import Simplex.Messaging.Transport.Credentials
-import Simplex.Messaging.Util (whenM)
-import System.Directory (doesDirectoryExist, doesFileExist, removeDirectoryRecursive, removeFile)
+import System.Directory (doesFileExist, removeFile)
 import System.IO (IOMode (..), withFile)
 import System.TimeIt (timeItT)
 import System.Timeout
@@ -66,7 +62,11 @@ import Util

 #if defined(dbServerPostgres)
 import CoreTests.MsgStoreTests (testPostgresStoreConfig)
+import Simplex.Messaging.Server.Env.STM (readWriteQueueStore)
 import Simplex.Messaging.Server.MsgStore.Postgres (PostgresMsgStoreCfg (..), exportDbMessages)
+import Simplex.Messaging.Server.MsgStore.STM (STMStoreConfig (..))
+import Simplex.Messaging.Server.MsgStore.Types (MsgStoreClass (..), newMsgStore)
+import Simplex.Messaging.Server.StoreLog (closeStoreLog)
 #endif

 serverTests :: SpecWith (ASrvTransport, AStoreType)
@@ -1068,7 +1068,6 @@ testRestoreMessages =
  it "should store messages on exit and restore on start" $ \(at@(ATransport t), msType) -> do
    removeFileIfExists testStoreLogFile
    removeFileIfExists testStoreMsgsFile
-    whenM (doesDirectoryExist testStoreMsgsDir) $ removeDirectoryRecursive testStoreMsgsDir
    removeFileIfExists testServerStatsBackupFile

    g <- C.newRandom
@@ -1138,7 +1137,6 @@ testRestoreMessages =
    Right stats3 <- strDecode <$> B.readFile testServerStatsBackupFile
    checkStats stats3 [rId] 5 5
    removeFileIfExists testStoreMsgsFile
-    whenM (doesDirectoryExist testStoreMsgsDir) $ removeDirectoryRecursive testStoreMsgsDir
    removeFile testServerStatsBackupFile
  where
    runTest :: Transport c => TProxy c 'TServer -> (THandleSMP c 'TClient -> IO ()) -> ThreadId -> Expectation
@@ -1218,28 +1216,21 @@ testRestoreExpireMessages =
  where
    exportStoreMessages :: AStoreType -> IO ()
    exportStoreMessages = \case
-      ASType _ SMSJournal -> export
      ASType _ SMSPostgres -> exportDB
      ASType _ SMSMemory -> pure ()
      where
-        export = do
-          ms <- readWriteQueues
-          exportMessages False (StoreJournal ms) testStoreMsgsFile False
-          closeMsgStore ms
 #if defined(dbServerPostgres)
        exportDB = do
-          readWriteQueues >>= closeMsgStore
+          ms <- newMsgStore STMStoreConfig {storePath = Nothing, quota = 4}
+          readWriteQueueStore True (mkQueue ms) testStoreLogFile (queueStore ms) >>= closeStoreLog
+          removeFileIfExists testStoreMsgsFile
+          closeMsgStore ms
          ms' <- newMsgStore (testPostgresStoreConfig {quota = 4} :: PostgresMsgStoreCfg)
          _n <- withFile testStoreMsgsFile WriteMode $ exportDbMessages False ms'
          closeMsgStore ms'
 #else
        exportDB = error "compiled without server_postgres flag"
 #endif
-        readWriteQueues = do
-          ms <- newMsgStore ((testJournalStoreCfg MQStoreCfg) {quota = 4} :: JournalStoreConfig 'QSMemory)
-          readWriteQueueStore True (mkQueue ms True) testStoreLogFile (stmQueueStore ms) >>= closeStoreLog
-          removeFileIfExists testStoreMsgsFile
-          pure ms
    runTest :: Transport c => TProxy c 'TServer -> (THandleSMP c 'TClient -> IO ()) -> ThreadId -> Expectation
    runTest _ test' server = do
      testSMPClient test' `shouldReturn` ()
@@ -1549,7 +1540,7 @@ testMsgExpireOnInterval =
  it "should expire messages that are not received before messageTTL after expiry interval" $ \(ATransport (t :: TProxy c 'TServer), msType) -> do
    g <- C.newRandom
    (sPub, sKey) <- atomically $ C.generateAuthKeyPair C.SEd25519 g
-    let cfg' = updateCfg (cfgMS msType) $ \cfg_ -> cfg_ {messageExpiration = Just ExpirationConfig {ttl = 1, checkInterval = 1}, idleQueueInterval = 1}
+    let cfg' = updateCfg (cfgMS msType) $ \cfg_ -> cfg_ {messageExpiration = Just ExpirationConfig {ttl = 1, checkInterval = 1}}
    withSmpServerConfigOn (ATransport t) cfg' testPort $ \_ ->
      testSMPClient @c $ \sh -> do
        (sId, rId, rKey, _) <- testSMPClient @c $ \rh -> createAndSecureQueue rh sPub
@@ -109,24 +109,21 @@ main = do
          describe "SMP server schema dump" $
            postgresSchemaDumpTest
              serverMigrations
-              [ "20250320_short_links" -- snd_secure moves to the bottom on down migration
+              [ "20250320_short_links", -- snd_secure moves to the bottom on down migration
+                "20260918_expire_messages" -- msg_queue_expire moves to the bottom on down migration
              ] -- skipComparisonForDownMigrations
              testStoreDBOpts
              "src/Simplex/Messaging/Server/QueueStore/Postgres/server_schema.sql"
        around_ (postgressBracket testServerDBConnectInfo) $ do
-          -- xdescribe "SMP server via TLS, postgres+jornal message store" $
-          --   before (pure (transport @TLS, ASType SQSPostgres SMSJournal)) serverTests
          describe "SMP server via TLS, postgres-only message store" $
            before (pure (transport @TLS, ASType SQSPostgres SMSPostgres)) serverTests
 #endif
-        describe "SMP server via TLS, jornal message store" $ do
+        describe "SMP server via TLS, memory message store" $ do
          describe "SMP syntax" $ serverSyntaxTests (transport @TLS)
-          before (pure (transport @TLS, ASType SQSMemory SMSJournal)) serverTests
-        describe "SMP server via TLS, memory message store" $
          before (pure (transport @TLS, ASType SQSMemory SMSMemory)) serverTests
        -- xdescribe "SMP server via WebSockets" $ do
        --   describe "SMP syntax" $ serverSyntaxTests (transport @WS)
-        --   before (pure (transport @WS, ASType SQSMemory SMSJournal)) serverTests
+        --   before (pure (transport @WS, ASType SQSMemory SMSMemory)) serverTests
 #if defined(dbServerPostgres)
        around_ (postgressBracket ntfTestServerDBConnectInfo) $
          describe "Ntf server schema dump" $
@@ -139,22 +136,16 @@ main = do
          describe "Notifications server (SMP server: memory store)" $
            ntfServerTests (transport @TLS, ASType SQSMemory SMSMemory)
          around_ (postgressBracket testServerDBConnectInfo) $ do
-            -- xdescribe "Notifications server (SMP server: postgres+jornal store)" $
-            --   ntfServerTests (transport @TLS, ASType SQSPostgres SMSJournal)
            describe "Notifications server (SMP server: postgres-only store)" $
              ntfServerTests (transport @TLS, ASType SQSPostgres SMSPostgres)
        around_ (postgressBracket testServerDBConnectInfo) $ do
-          -- xdescribe "SMP client agent, postgres+jornal message store" $ agentTests (transport @TLS, ASType SQSPostgres SMSJournal)
          describe "SMP client agent, server postgres-only message store" $ agentTests (transport @TLS, ASType SQSPostgres SMSPostgres)
-          -- xdescribe "SMP proxy, postgres+jornal message store" $
-          --   before (pure $ ASType SQSPostgres SMSJournal) smpProxyTests
          describe "SMP proxy, postgres-only message store" $
            before (pure $ ASType SQSPostgres SMSPostgres) smpProxyTests
 #endif
-        -- xdescribe "SMP client agent, server jornal message store" $ agentTests (transport @TLS, ASType SQSMemory SMSJournal)
        describe "SMP client agent, server memory message store" $ agentTests (transport @TLS, ASType SQSMemory SMSMemory)
-        describe "SMP proxy, jornal message store" $
-          before (pure $ ASType SQSMemory SMSJournal) smpProxyTests
+        describe "SMP proxy, memory message store" $
+          before (pure $ ASType SQSMemory SMSMemory) smpProxyTests
        describe "XFTP" $ do
          describe "XFTP server" $
            before (pure $ AFSType SFSMemory) xftpServerTests
Author	SHA1	Message	Date
shum	8106178dae	Merge remote-tracking branch 'origin/master' into sh/queue-map-leak # Conflicts: # tests/AgentTests/FunctionalAPITests.hs	2026-10-06 07:32:52 +00:00
EvgenyandEvgeny @ SimpleX Chat	4ab5376a6e	smp: reduce min supported version to 15 (short links support, 3/30/2025) (#1931 ) Co-authored-by: Evgeny @ SimpleX Chat <[email protected]>	2026-10-05 11:31:57 +01:00
shum	5c6eb768cf	Merge remote-tracking branch 'origin/master' into sh/queue-map-leak # Conflicts: # tests/CoreTests/MsgStoreTests.hs # tests/SMPProxyTests.hs	2026-10-05 07:57:57 +00:00
shum	2c3641f97a	smp-server: remove journal message store	2026-10-05 07:44:32 +00:00
Evgeny	aaffb5b07d	Merge pull request #1922 from simplex-chat/ratchet Improve double ratchet	2026-10-05 08:07:04 +01:00
EvgenyandEvgeny @ SimpleX Chat	65893a359a	agent: resolve potential race conditions in double ratchet (#1923 ) * agent: resolve potential race conditions in double ratchet * update * improve * diff * diff * diff * simpler * more races --------- Co-authored-by: Evgeny @ SimpleX Chat <[email protected]>	2026-10-04 22:28:50 +01:00
EvgenyandEvgeny @ SimpleX Chat	fcc5670ae2	agent: keep ratchet hashes for connection lifetime, up to some cap, to prevent replay of expired ratchets (#1893 ) * agent: keep ratchet hashes for connection lifetime, to prevent replay of expired ratchets * cap * simpler * simpler2 * split * one query * simplify * performance * migration * fix schema * comment * improve --------- Co-authored-by: Evgeny @ SimpleX Chat <[email protected]>	2026-10-04 14:43:21 +01:00
Evgenyandshum	18440c0d17	smp-server: check queue is not secured in link store (#1917 ) Co-authored-by: shum <[email protected]>	2026-10-04 13:41:48 +01:00
shum	f929911fa0	smp-server: remove queue lock when queue is not added	2026-10-04 11:36:05 +00:00
shum	8eade7ef1b	smp-server: fix queue cache collisions and leaks	2026-10-04 11:20:48 +00:00
EvgenyandEvgeny @ SimpleX Chat	64c5d19fc4	agent: aggregate cap on skipped message keys (#1903 ) * agent: aggregate cap on skipped message keys * migration * update migration --------- Co-authored-by: Evgeny @ SimpleX Chat <[email protected]>	2026-10-04 10:18:41 +01:00
EvgenyandEvgeny @ SimpleX Chat	e11dfb985d	consider IP address protected only if it is used for the chosen host (#1895 ) * consider IP address protected only if it is used for the chosen host * simplify * simplify * simplify --------- Co-authored-by: Evgeny @ SimpleX Chat <[email protected]>	2026-10-03 22:45:44 +01:00
Evgenyandshum	76248dd767	sntrup761: bound KeyGen retries on R3 inversion (#1918 ) Co-authored-by: shum <[email protected]>	2026-10-03 17:01:36 +01:00
brenzi	9c641fed91	reject e2e params with kem_ct and without kem_key (#1901 )	2026-10-03 15:10:42 +01:00
sh	91e070cb5e	crypto: document 16-byte GCM IV in double ratchet (#1914 )	2026-10-03 15:00:18 +01:00
sh	9fe3745401	docs: use role names for key exchange parties (#1912 )	2026-10-03 14:59:22 +01:00
EvgenyandEvgeny @ SimpleX Chat	24036368f8	xrcp: limit multicast validity time window (#1899 ) Co-authored-by: Evgeny @ SimpleX Chat <[email protected]>	2026-10-03 14:51:48 +01:00
sh	acd5c0f530	docs: add cryptographic primitive registry (#1913 )	2026-10-03 14:21:21 +01:00
EvgenyandEvgeny @ SimpleX Chat	ead8f10f26	smp protocol: bind agreed version of forwarded command (#1915 ) * smp protocol: bind agreed version of forwarded command * refactor * pass proxied relay version range via config * simplify * refactor * packZipWith --------- Co-authored-by: Evgeny @ SimpleX Chat <[email protected]>	2026-10-03 08:56:19 +01:00
shum	83eb9a55fd	smp-server: remove deleted queues from store maps	2026-10-02 11:03:32 +00:00
sh	053e83b704	resolver: multicall reads, keep-alive, workers and structured logs (#1883 ) * resolver: queue up to 128 pending connections * resolver: reuse connections to the node * resolver: read each round in one multicall * resolver: run worker processes, log durations * resolver: keep smp-server connections alive * resolver: test with pytest, lock dependencies * resolver: structured logs and real client addresses * resolver: refuse requests with a body * resolver: harden pool, health and odd answers * resolver: pin images, rotate logs * resolver: stricter body check, redact RPC URL * resolver: ship .env as .env.example * resolver: pass settings from .env * resolver: name the image snrc-resolve:local	2026-10-01 09:10:42 +01:00
sh	234b5cd856	smp-server: rework message expiration (#1873 ) * smp server: rework message expiration * smp server: paginate message expiration sweep	2026-10-01 09:08:16 +01:00
Evgeny Poberezkin	38fdc0c544	7.1.0.10	2026-09-30 13:03:16 +01:00
EvgenyandEvgeny @ SimpleX Chat	b4c288c597	docs: clarify security model (#1902 ) Co-authored-by: Evgeny @ SimpleX Chat <[email protected]>	2026-09-30 10:18:52 +01:00
EvgenyandEvgeny @ SimpleX Chat	0e1ecdcff6	xftp server: validate paths in store log (#1894 ) Co-authored-by: Evgeny @ SimpleX Chat <[email protected]>	2026-09-30 09:36:45 +01:00
EvgenyandEvgeny @ SimpleX Chat	c1776dc5a4	server: improve control port auth (#1898 ) Co-authored-by: Evgeny @ SimpleX Chat <[email protected]>	2026-09-30 09:04:28 +01:00
EvgenyandEvgeny @ SimpleX Chat	e2df2e1b7c	docs: update SMP protocol (#1900 ) Co-authored-by: Evgeny @ SimpleX Chat <[email protected]>	2026-09-30 08:10:35 +01:00
EvgenyandEvgeny @ SimpleX Chat	267e2e0727	agent: fix ratchet - dont save if body auth failed (#1892 ) Co-authored-by: Evgeny @ SimpleX Chat <[email protected]>	2026-09-29 21:32:21 +01:00
EvgenyandEvgeny @ SimpleX Chat	551a79d99b	smp server: validate client version in forwarded command (#1890 ) Co-authored-by: Evgeny @ SimpleX Chat <[email protected]>	2026-09-29 19:52:15 +01:00
EvgenyandEvgeny @ SimpleX Chat	972e50e768	docs: update XRCP doc to make code verification optional for known connections (#1891 ) Co-authored-by: Evgeny @ SimpleX Chat <[email protected]>	2026-09-29 18:08:54 +01:00