database field conversion instance (#1940)

Co-authored-by: Evgeny @ SimpleX Chat <[email protected]>

.github/workflows/build.yml

@@ -9,6 +9,10 @@ on:
       - "v*"
   pull_request:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ !startsWith(github.ref, 'refs/tags/v') }}
+
 jobs:
 
 # =============================
@@ -62,8 +66,19 @@ jobs:
   build:
     name: "ubuntu-${{ matrix.os }}-${{ matrix.arch }}, GHC: ${{ matrix.ghc }}"
     needs: maybe-release
+    permissions:
+      contents: write
+      packages: write
     env:
       apps: "smp-server xftp-server ntf-server xftp"
+      opt: ${{ github.event_name == 'pull_request' && 'O0' || 'O2' }}
+      ghc_options: ${{ github.event_name == 'pull_request' && '--ghc-options=-O0' || '' }}
+      release: ${{ startsWith(github.ref, 'refs/tags/v') }}
+      postgres_dist: ${{ startsWith(github.ref, 'refs/tags/v') && 'dist-newstyle' || 'dist-postgres' }}
+      branch: ${{ github.base_ref || github.ref_name }}
+      cache_o0: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/stable') }}
+      image_cache: ghcr.io/simplex-chat/simplexmq-build-cache:${{ matrix.os }}-${{ matrix.arch }}-ghc${{ matrix.ghc }}
+      source_hashes: dist-newstyle/sources.sha256
     runs-on: ${{ matrix.runner }}
     services:
       postgres:
@@ -142,6 +157,18 @@ jobs:
           sudo apt update
           sudo apt -y install postgresql-client-15
 
+      - name: Log in to GitHub Container Registry
+        if: matrix.should_run == true && env.release != 'true'
+        shell: bash
+        env:
+          GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: echo "$GHCR_TOKEN" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin
+
+      - name: Compute image cache week
+        id: week
+        if: matrix.should_run == true && env.release != 'true'
+        run: echo "week=$(date -u +%G-W%V)" >> "$GITHUB_OUTPUT"
+
       - name: Build and cache Docker image
         if: matrix.should_run == true
         uses: simplex-chat/docker-build-push-action@v6
@@ -153,20 +180,14 @@ jobs:
           build-args: |
             TAG=${{ matrix.os }}
             GHC=${{ matrix.ghc }}
-
-      - name: Cache dependencies
-        if: matrix.should_run == true
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cabal/store
-            dist-newstyle
-          key: ubuntu-${{ matrix.os }}-${{ matrix.arch }}-ghc${{ matrix.ghc }}-${{ hashFiles('cabal.project', 'simplexmq.cabal') }}
+          cache-from: ${{ env.release != 'true' && format('type=registry,ref={0}-{1}', env.image_cache, steps.week.outputs.week) || '' }}
+          cache-to: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && format('type=registry,ref={0}-{1},mode=max,ignore-error=true', env.image_cache, steps.week.outputs.week) || '' }}
 
       - name: Start container
         if: matrix.should_run == true
         shell: bash
         run: |
+          mkdir -p ~/.cabal
           docker run -t -d \
             --device /dev/fuse \
             --cap-add SYS_ADMIN \
@@ -177,21 +198,66 @@ jobs:
             -v ${{ github.workspace }}:/project \
             build/${{ matrix.os }}:latest
 
+      - name: Compute cache key prefixes
+        id: keys
+        if: matrix.should_run == true && env.release != 'true'
+        run: |
+          toolchain=$(docker exec builder dpkg-query -W -f '${db:Status-Status} ${binary:Package} ${Version}\n' 'binutils*' 'cpp*' 'gcc*' '*-dev' | grep '^installed ' | sha256sum | cut -c1-16)
+          target=${{ matrix.os }}-${{ matrix.arch }}-ghc${{ matrix.ghc }}-$toolchain
+          echo "deps=deps-$target" >> "$GITHUB_OUTPUT"
+          echo "dist=dist-$target" >> "$GITHUB_OUTPUT"
+
+      - name: Restore dependencies from cache
+        id: restore-deps
+        if: matrix.should_run == true && env.release != 'true'
+        uses: actions/cache/restore@v4
+        with:
+          path: ~/.cabal/store
+          key: ${{ steps.keys.outputs.deps }}-${{ env.branch }}-${{ github.sha }}
+          restore-keys: |
+            ${{ steps.keys.outputs.deps }}-${{ env.branch }}-
+            ${{ steps.keys.outputs.deps }}-
+
+      - name: Warn if dependencies not in cache
+        if: matrix.should_run == true && env.release != 'true' && steps.restore-deps.outputs.cache-matched-key == ''
+        run: echo "::warning title=Cache miss::${{ steps.keys.outputs.deps }}-${{ env.branch }}"
+
+      - name: Restore build directories from cache
+        id: restore-build
+        if: matrix.should_run == true && env.release != 'true'
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            dist-newstyle
+            dist-postgres
+          key: ${{ steps.keys.outputs.dist }}-${{ env.opt }}-${{ env.branch }}-${{ github.sha }}
+          restore-keys: |
+            ${{ steps.keys.outputs.dist }}-${{ env.opt }}-${{ env.branch }}-
+            ${{ steps.keys.outputs.dist }}-${{ env.opt }}-
+
+      - name: Warn if build directories not in cache
+        if: matrix.should_run == true && env.release != 'true' && steps.restore-build.outputs.cache-matched-key == ''
+        run: echo "::warning title=Cache miss::${{ steps.keys.outputs.dist }}-${{ env.opt }}-${{ env.branch }}"
+
+      - name: Set old timestamps on unchanged sources (GHC 8.10.7)
+        if: matrix.ghc == '8.10.7' && steps.restore-build.outputs.cache-matched-key != '' && hashFiles(env.source_hashes) != ''
+        run: scripts/ci/source-timestamps.sh restore ${{ env.source_hashes }}
+
       - name: Build smp-server, xftp-server (postgresql) and tests
         if: matrix.should_run == true
         shell: docker exec -t builder sh -eu {0}
         run: |
-          chmod -fR 777 ~/.cabal ./dist-newstyle || :; git config --global --add safe.directory '*'
-          cabal clean
+          chmod -fR 777 ~/.cabal ./dist-newstyle ./dist-postgres || :; git config --global --add safe.directory '*'
+          scripts/ci/prune-old-builds.sh dist-newstyle dist-postgres
           cabal update
-          cabal build --jobs=$(nproc) --enable-tests -fserver_postgres
+          cabal build --builddir=${{ env.postgres_dist }} --jobs=$(nproc) --enable-tests -fserver_postgres ${{ env.ghc_options }}
           mkdir -p /out
-          for i in smp-server xftp-server simplexmq-test; do
-            bin=$(find /project/dist-newstyle -name "$i" -type f -executable)
+          for i in smp-server xftp-server ntf-server simplexmq-test; do
+            bin=$(find /project/${{ env.postgres_dist }} -name "$i" -type f -executable)
             chmod +x "$bin"
-            mv "$bin" /out/
+            cp "$bin" /out/
           done
-          strip /out/smp-server /out/xftp-server
+          strip /out/smp-server /out/xftp-server /out/ntf-server
 
       - name: Copy simplexmq-test from container
         if: matrix.should_run == true
@@ -224,16 +290,17 @@ jobs:
           cat hashes.output >> "$GITHUB_OUTPUT"
 
       - name: Build everything else (standard)
+        id: build-standard
         if: matrix.should_run == true
         shell: docker exec -t builder sh -eu {0}
         run: |
-          cabal build --jobs=$(nproc)
+          cabal build --jobs=$(nproc) ${{ env.ghc_options }}
           mkdir -p /out
-          for i in ${{ env.apps }}; do
+          for i in smp-server xftp-server xftp; do
             bin=$(find /project/dist-newstyle -name "$i" -type f -executable)
             strip "$bin"
             chmod +x "$bin"
-            mv "$bin" /out/
+            cp "$bin" /out/
           done
 
       - name: Copy binaries from container and prepare them
@@ -278,6 +345,33 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Record source hashes (GHC 8.10.7)
+        if: matrix.should_run == true && matrix.ghc == '8.10.7'
+        shell: docker exec -t builder sh -eu {0}
+        run: scripts/ci/source-timestamps.sh record ${{ env.source_hashes }}
+
+      - name: Compute dependency cache key
+        id: hash-deps
+        if: matrix.should_run == true && env.release != 'true'
+        shell: bash
+        run: echo "hash=$(ls ~/.cabal/store/ghc-*/ | sha256sum | cut -c1-16)" >> "$GITHUB_OUTPUT"
+
+      - name: Save dependencies to cache
+        if: matrix.should_run == true && env.release != 'true' && steps.restore-deps.outputs.cache-matched-key != format('{0}-{1}-{2}', steps.keys.outputs.deps, env.branch, steps.hash-deps.outputs.hash)
+        uses: actions/cache/save@v4
+        with:
+          path: ~/.cabal/store
+          key: ${{ steps.keys.outputs.deps }}-${{ env.branch }}-${{ steps.hash-deps.outputs.hash }}
+
+      - name: Save build directories to cache
+        if: matrix.should_run == true && env.release != 'true' && steps.restore-build.outputs.cache-hit != 'true'
+        uses: actions/cache/save@v4
+        with:
+          path: |
+            dist-newstyle
+            dist-postgres
+          key: ${{ steps.keys.outputs.dist }}-${{ env.opt }}-${{ env.branch }}-${{ github.sha }}
+
       - name: Test
         if: matrix.should_run == true && matrix.arch == 'x86-64'
         timeout-minutes: 120
@@ -301,3 +395,75 @@ jobs:
             echo "All "$attempts" attempts failed."
             exit 1
           fi
+
+      - name: Remove -O2 build directories
+        id: remove-o2
+        if: env.cache_o0 == 'true' && !cancelled() && steps.build-standard.outcome == 'success'
+        shell: docker exec -t builder sh -eu {0}
+        run: rm -rf dist-newstyle dist-postgres
+
+      - name: Restore -O0 build directories from cache
+        id: restore-build-o0
+        if: ${{ !cancelled() && steps.remove-o2.outcome == 'success' }}
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            dist-newstyle
+            dist-postgres
+          key: ${{ steps.keys.outputs.dist }}-O0-${{ env.branch }}-${{ github.sha }}
+          restore-keys: |
+            ${{ steps.keys.outputs.dist }}-O0-${{ env.branch }}-
+            ${{ steps.keys.outputs.dist }}-O0-
+
+      - name: Set old timestamps on unchanged sources for -O0 build (GHC 8.10.7)
+        if: ${{ !cancelled() && matrix.ghc == '8.10.7' && steps.restore-build-o0.outputs.cache-matched-key != '' && hashFiles(env.source_hashes) != '' }}
+        run: scripts/ci/source-timestamps.sh restore ${{ env.source_hashes }}
+
+      - name: Build at -O0 for pull request caches
+        id: build-o0
+        if: ${{ !cancelled() && steps.remove-o2.outcome == 'success' }}
+        shell: docker exec -t builder sh -eu {0}
+        run: |
+          scripts/ci/prune-old-builds.sh dist-newstyle dist-postgres
+          cabal build --builddir=dist-postgres --jobs=$(nproc) --enable-tests -fserver_postgres --ghc-options=-O0
+          cabal build --jobs=$(nproc) --ghc-options=-O0
+
+      - name: Record source hashes for -O0 build (GHC 8.10.7)
+        if: ${{ !cancelled() && matrix.ghc == '8.10.7' && steps.build-o0.outcome == 'success' }}
+        shell: docker exec -t builder sh -eu {0}
+        run: scripts/ci/source-timestamps.sh record ${{ env.source_hashes }}
+
+      - name: Save -O0 build directories to cache
+        if: ${{ !cancelled() && steps.build-o0.outcome == 'success' && steps.restore-build-o0.outputs.cache-hit != 'true' }}
+        uses: actions/cache/save@v4
+        with:
+          path: |
+            dist-newstyle
+            dist-postgres
+          key: ${{ steps.keys.outputs.dist }}-O0-${{ env.branch }}-${{ github.sha }}
+
+# =============================
+#      Resolver test job
+# =============================
+
+# The SNRC resolver is Python, so this job needs none of the Haskell toolchain
+# above and runs independently of it.
+
+  resolver-test:
+    name: "resolver (python)"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Clone project
+        uses: actions/checkout@v3
+
+      - name: Set up uv
+        # uv takes Python from .python-version, the version of the Dockerfile's runtime stage.
+        uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
+
+      - name: Lint
+        working-directory: scripts/resolver/service
+        run: uv run --frozen ruff check .
+
+      - name: Test
+        working-directory: scripts/resolver/service
+        run: uv run --frozen pytest -v

.gitignore

@@ -1,4 +1,8 @@
 *.lock
+# the resolver's Docker image and CI install exactly these
+!/scripts/resolver/service/uv.lock
+# local overrides of scripts/resolver/.env.example
+/scripts/resolver/.env
 *.db
 *.db.bak
 *.session.sql
@@ -12,3 +16,4 @@ cabal.project.local~
 *.tix
 .coverage
 
+__pycache__/

.gitmodules

@@ -4,3 +4,6 @@
 [submodule "cbits/blst"]
 	path = cbits/blst
 	url = https://github.com/supranational/blst.git
+[submodule "cbits/libsecp256k1"]
+	path = cbits/libsecp256k1
+	url = https://github.com/bitcoin-core/secp256k1.git

apps/xftp-server/XFTPWeb.hs

@@ -58,7 +58,7 @@ xftpSubsts XFTPServerConfig {fileExpiration, logStatsInterval, allowNewFiles, ne
   [("smpConfig", Nothing), ("xftpConfig", Just "y")] <> substConfig <> serverInfoSubsts simplexmqSource information <> [("onionHost", strEncode <$> onionHost), ("iniFileName", Just "file-server.ini")]
   where
     substConfig =
-      [ ("fileExpiration", Just $ maybe "Never" (fromString . timedTTLText . ttl) fileExpiration),
+      [ ("fileExpiration", Just . fromString . timedTTLText . ttl $ fileExpiration),
         ("statsEnabled", Just . yesNo $ isJust logStatsInterval),
         ("newUploadsAllowed", Just . yesNo $ allowNewFiles),
         ("basicAuthEnabled", Just . yesNo $ isJust newFileBasicAuth)

cabal.project

@@ -31,7 +31,7 @@ source-repository-package
 source-repository-package
     type: git
     location: https://github.com/simplex-chat/direct-sqlcipher.git
-    tag: f814ee68b16a9447fbb467ccc8f29bdd3546bfd9
+    tag: 2330df3dc8c4b660674d3c02d76bb86f66c2abee
 
 source-repository-package
     type: git

cbits/sntrup761.c

@@ -719,23 +719,32 @@ Small_random (small * out, void *random_ctx, sntrup761_random_func * random)
 
 /* ----- Streamlined NTRU Prime Core */
 
+/* x^p-x-1 has a degree-19 factor mod 3, so a random g is not invertible in R3
+   with probability about 3^-19; KeyGen_attempts failures in a row mean a broken RNG */
+#define KeyGen_attempts 10
+
 /* h,(f,ginv) = KeyGen() */
-static void
+/* returns 0 if KeyGen succeeded; else -1 */
+static int
 KeyGen (Fq * h, small * f, small * ginv, void *random_ctx,
         sntrup761_random_func * random)
 {
   small g[p];
   Fq finv[p];
+  int i;
 
-  for (;;)
+  for (i = 0; i < KeyGen_attempts; ++i)
     {
       Small_random (g, random_ctx, random);
       if (R3_recip (ginv, g) == 0)
         break;
     }
+  if (i == KeyGen_attempts)
+    return -1;
   Short_random (f, random_ctx, random);
   Rq_recip3 (finv, f);          /* always works */
   Rq_mult_small (h, finv, g);
+  return 0;
 }
 
 /* c = Encrypt(r,h) */
@@ -884,18 +893,21 @@ typedef small Inputs[p];        /* passed by reference */
 #define PublicKeys_bytes Rq_bytes
 
 /* pk,sk = ZKeyGen() */
-static void
+/* returns 0 if KeyGen succeeded; else -1 */
+static int
 ZKeyGen (unsigned char *pk, unsigned char *sk, void *random_ctx,
          sntrup761_random_func * random)
 {
   Fq h[p];
   small f[p], v[p];
 
-  KeyGen (h, f, v, random_ctx, random);
+  if (KeyGen (h, f, v, random_ctx, random) != 0)
+    return -1;
   Rq_encode (pk, h);
   Small_encode (sk, f);
   sk += Small_bytes;
   Small_encode (sk, v);
+  return 0;
 }
 
 /* C = ZEncrypt(r,pk) */
@@ -960,19 +972,21 @@ HashSession (unsigned char *k, int b, const unsigned char *y,
 /* ----- Streamlined NTRU Prime */
 
 /* pk,sk = KEM_KeyGen() */
-void
+int
 sntrup761_keypair (unsigned char *pk, unsigned char *sk, void *random_ctx,
                    sntrup761_random_func * random)
 {
   int i;
 
-  ZKeyGen (pk, sk, random_ctx, random);
+  if (ZKeyGen (pk, sk, random_ctx, random) != 0)
+    return -1;
   sk += SecretKeys_bytes;
   for (i = 0; i < PublicKeys_bytes; ++i)
     *sk++ = pk[i];
   random (random_ctx, Inputs_bytes, sk);
   sk += Inputs_bytes;
   Hash_prefix (sk, 4, pk, PublicKeys_bytes);
+  return 0;
 }
 
 /* c,r_enc = Hide(r,pk,cache); cache is Hash4(pk) */

cbits/sntrup761.h

@@ -19,7 +19,8 @@
 
 typedef void sntrup761_random_func (void *ctx, size_t length, uint8_t *dst);
 
-void
+/* returns 0 on success, -1 if the RNG never produced an invertible polynomial */
+int
 sntrup761_keypair (uint8_t *pk, uint8_t *sk,
                    void *random_ctx, sntrup761_random_func *random);
 

contributing/CODE.md

@@ -7,6 +7,7 @@ This file provides guidance on coding style and approaches and on building the c
 When designing code and planning implementations:
 - Apply adversarial thinking, and consider what may happen if one of the communicating parties is malicious.
 - Formulate an explicit threat model for each change - who can do which undesirable things and under which circumstances.
+- Use the default cryptographic primitive for each purpose from the [primitive registry](../protocol/crypto-registry.md); any exception requires review, and the registry must be updated with every new primitive use or domain-separation string.
 
 ## Code Quality Standards
 

docs/smp-server-db-load.md

@@ -0,0 +1,200 @@
+# SMP server database load: root causes and fixes
+
+## Summary
+
+The recurring `pMsgFwdsOwn_pErrorsOther` spikes are a symptom of chronic disk saturation of the
+`smp_server` PostgreSQL database, not a proxy or forwarding bug. The baseline read load is the
+Prometheus scrape running whole-table `COUNT` scans of `msg_queues` (48.5M rows) every 30 seconds,
+compounded by the message-expiration sweep. The write load comes from indexing `updated_at`:
+`updateQueueTime` stamps it on the first send to each active queue per UTC day, and these non-HOT
+updates bloat the indexes to 104 GB and amplify WAL to about 425 GB per day (37% full-page images,
+~31% from `updateQueueTime`). Because the stamp is day-granular, this write burst clusters at 00:00 UTC,
+tipping the saturated disk into the observed spike and loading the pgBackRest backup repository.
+
+Code fixes replace the largest metric scan with an estimate and index the sweep and notifier counts.
+Reclaiming the index bloat, which also clears the remaining metric scan, and keeping it down are
+operator actions, because PostgreSQL never shrinks indexes automatically.
+
+## Environment
+
+The affected deployment runs the pure PostgreSQL backend (`store_queues: database`,
+`store_messages: database`), schema `smp_server`, `db_pool_size 20`, `prometheus_interval 30`,
+`[NAMES]` enabled, own-server proxying enabled, and forwards routed over a SOCKS proxy.
+
+`msg_queues`: 48.5M live rows, 12.3% dead tuples, 14 GB heap, 104 GB indexes, 118 GB total.
+
+## Evidence
+
+`pg_stat_statements` over a 5 minute window, ordered by disk reads (`track_io_timing` was off, so
+`shared_blks_read` is the disk proxy; the two `EXPLAIN ANALYZE` rows are manual and excluded):
+
+| query | calls | mean ms | reads | share of all reads |
+| --- | --- | --- | --- | --- |
+| `getEntityCounts` (the six-`COUNT` metric query) | 30 | 111,456 | 990 GB | 70.7% |
+| queue-record load (`SELECT recipient_id, …`) | 47,027 | 121 | 89 GB | 6.3% |
+| expiration sweep batch (`array_agg` in `expire_old_messages`) | 13 | 233,143 | 62 GB | 4.5% |
+| message peek (`DISTINCT ON (recipient_id)`) | 46,932 | 76 | 43 GB | 3.0% |
+| `updateQueueTime` (`UPDATE msg_queues SET updated_at`) | 172,885 | 4 | 7.7 GB | 0.5% |
+| `write_message` | 110,911 | 5 | 7.0 GB | 0.5% |
+
+`EXPLAIN (ANALYZE, BUFFERS)` of each `msg_queues` count inside `getEntityCounts`:
+
+| count | plan | time |
+| --- | --- | --- |
+| `queue_count` (`deleted_at IS NULL`) | Parallel Seq Scan | 30 s |
+| `notifier_count` (`deleted_at IS NULL AND notifier_id IS NOT NULL`) | Parallel Seq Scan | 32.6 s |
+| `ntf_service_queues_count` (`ntf_service_id IS NOT NULL AND deleted_at IS NULL`) | Parallel Seq Scan | 30.2 s |
+| `rcv_service_queues_count` (`rcv_service_id IS NOT NULL AND deleted_at IS NULL`) | Index Only Scan | 1.4 ms |
+
+The expiration sweep batch: 229 s for one 10,000-row batch, walking `msg_queues_pkey` and discarding
+809,729 rows by filter to find 10,000 expirable ones.
+
+`pg_stat_wal` over the measurement window: 27 TB of WAL accumulated, 37.2% of records full-page
+images, averaging about 425 GB per day; recent days reach ~470 GB (from the `archive-push` log) as
+the queue count grows.
+
+Per-statement WAL from `pg_stat_statements` (`wal_bytes`), leaf statements only (this instance runs
+`pg_stat_statements.track = all`, so the `write_message` wrapper double-counts the `INSERT` and
+`UPDATE` it runs, and is excluded):
+
+| category | share of WAL | main statements |
+| --- | --- | --- |
+| `msg_queues` updates | ~42% | `updateQueueTime` ~31%, `msg_can_write` flags ~9%, insert/delete ~2% |
+| `messages` insert and delete | ~34% | message `INSERT`, `DELETE` by `message_id` and `recipient_id` |
+| read path (hint bits, FPIs on SELECT) | ~24% | message peek ~15%, `msg_queue_size`, `getEntityCounts` |
+
+## Root causes
+
+1. Prometheus scrape. `getEntityCounts` (`QueueStore/Postgres.hs:154`) runs six `COUNT(1)` subqueries
+   every `prometheus_interval` (30 s), called from the metrics path (`Server.hs:698`). Four count
+   `msg_queues`; three of those seq-scan the 48.5M-row heap. At 70.7% of all reads it is the dominant
+   disk consumer, and it runs continuously (mean 111 s per scrape exceeds the 30 s interval).
+
+2. Message-expiration sweep. `expireMessagesThread` (`Server.hs:477`) calls the `expire_old_messages`
+   procedure (`server_schema.sql`), whose inner batch query selects expirable queues ordered by
+   `recipient_id`. With `oldQueue = 0` the `updated_at` filter matches everything, so the planner
+   walks the primary key and filters `msg_queue_expire`, discarding ~99% of rows per batch.
+
+3. Index bloat. 104 GB of indexes on a 14 GB table. `updated_at` is part of
+   `idx_msg_queues_updated_at_recipient_id` (`server_schema.sql:526`) and `updateQueueTime`
+   (`QueueStore/Postgres.hs:443`) updates it on the day's first send per queue (172,885 times in the
+   window above). Updating an indexed column prevents HOT updates, so every update writes a new row
+   version plus new entries in every index on the table, and the old index entries accumulate as bloat.
+
+4. WAL amplification. Only 15.6% of `msg_queues` updates are HOT, because its composite index covers
+   `updated_at` and `msg_queue_expire`, both changed on the hot path (cause 3); the rest rewrite every
+   index on the table. With the 104 GB of bloated indexes and frequent checkpoints (`max_wal_size 4GB`,
+   `checkpoint_timeout 5min`, `wal_compression off`), 37% of WAL records are full-page images, and the
+   server generates about 425 GB of WAL per day (27 TB accumulated; recent days near 470 GB).
+   `updateQueueTime`, stamping `updated_at` on the ~5.3M daily-active queues, is the single largest
+   statement at ~31% of WAL (see Evidence). pgBackRest `archive-push` ships all of it, loading the
+   primary disk and the backup repository. The load peaks
+   at 00:00 UTC because `getSystemDate` rounds `updated_at` to the UTC day, so the first send to each
+   active queue after midnight runs `updateQueueTime`, clustering these writes and the `archive-push`
+   volume in the 00h hour.
+
+## Index bloat and autovacuum
+
+Autovacuum is running (60 runs on this table) and is not misconfigured, but two limits apply.
+
+Autovacuum reclaims heap dead tuples and refreshes planner statistics; it does not shrink indexes.
+Only `REINDEX` or `pg_repack` rebuilds a bloated B-tree. Index bloat therefore has no automatic
+remedy.
+
+Default autovacuum pacing assumes moderate churn. `autovacuum_vacuum_scale_factor = 0.2` waits until
+20% of the table is dead (about 9.7M rows here) before vacuuming, and cost-based throttling caps its
+I/O, so a hot table stays around 12% dead. Per-table tuning makes it run sooner and faster.
+
+The amplifier is indexing `updated_at`, a column updated on every send, which the fixes and operator
+actions below address.
+
+## Implemented fixes (batch 1)
+
+Branch `sh/leaks-batch-1`, two commits.
+
+| commit | change | effect |
+| --- | --- | --- |
+| `88673eee` | migration `20260916_prometheus_indexes` adds partial indexes `idx_msg_queues_expire (recipient_id) WHERE deleted_at IS NULL AND msg_queue_expire` and `idx_msg_queues_notifier_active (notifier_id) WHERE deleted_at IS NULL AND notifier_id IS NOT NULL` | the sweep batch and `notifier_count` become index scans |
+| `2912be1e` | `getEntityCounts.queue_count` uses `pg_class.reltuples` (`QueueStore/Postgres.hs:154`) | removes the 30 s, 990 GB whole-table `COUNT` on every scrape |
+
+Verified: both indexes are created and used as index-only scans, the schema-dump test passes, and
+`smp-server` builds. `queueCount` is consumed only by metrics, logs, and display
+(`Server.hs:528,698,815,2369`), so an estimate is safe.
+
+Not fixed by new indexes:
+
+- `ntf_service_queues_count` seq-scans only because of index bloat; `idx_msg_queues_ntf_service_id`
+  already covers it. `REINDEX` restores the index-only scan, as proven by `rcv_service_queues_count`.
+- `rcv_service_queues_count` and the two `services` counts are already cheap.
+
+## Implemented fixes (batch 2)
+
+Migration `20260917_msg_queues_hot` and `MsgStore/Postgres.hs`.
+
+| change | effect |
+| --- | --- |
+| drop `idx_msg_queues_updated_at_recipient_id`, set `msg_queues` `fillfactor = 80` | `updateQueueTime` updates become HOT, so they rewrite no index entries; removes its index bloat and cuts its WAL |
+| remove the dead `updated_at > p_old_queue` filter and its parameter from `expire_old_messages`, matching the `CALL` (`MsgStore/Postgres.hs:113`) | the sweep query runs as an index-only scan on `idx_msg_queues_expire`; `updated_at` is no longer read on the hot path |
+| autovacuum reloptions on `msg_queues`, `messages` (and its TOAST), and `services` | keeps dead tuples and bloat low per table (see the table below) |
+
+Verified on PostgreSQL 16 with the daily-active pattern (~11% of queues updated per day): `updateQueueTime`
+goes from 0% HOT (indexed `updated_at`) to 100% HOT at `fillfactor = 80` (96.7% at 90), and per-update
+WAL drops about 4x. The HOT ratio depends on how many rows per page change before autovacuum reclaims
+space, so it is workload-dependent; `fillfactor = 80` reached 100% for this fraction, and updating a much
+larger share of queues at once would need a lower value. `fillfactor` applies to pages rewritten after
+the change, so the ratio ramps up as the heap turns over, or immediately after `pg_repack`.
+
+Dropping the index regresses the deprecated journal message store's expiration (`foldRecentQueueRecs`,
+`Journal.hs:434`) to a sequential scan; this is accepted because that store is being retired. The
+Postgres message store does not read `updated_at`.
+
+Autovacuum reloptions applied by the migration:
+
+| table | reloptions | reason |
+| --- | --- | --- |
+| `msg_queues` | `fillfactor 80`, `autovacuum_vacuum_scale_factor 0.02`, `autovacuum_analyze_scale_factor 0.01`, `autovacuum_vacuum_cost_limit 1000` | HOT headroom; vacuum/analyze at ~2%/1% churn instead of 20%/10%; faster vacuum |
+| `messages` (and TOAST) | `autovacuum_vacuum_scale_factor 0.02`, `autovacuum_analyze_scale_factor 0.01`, `toast.autovacuum_vacuum_scale_factor 0.02` | high insert and delete churn; 152 GB, mostly TOASTed message bodies |
+| `services` | `fillfactor 70`, `autovacuum_vacuum_threshold 1000`, `autovacuum_vacuum_scale_factor 0` | tiny, very hot table was autovacuumed tens of thousands of times; make updates HOT and vacuum it far less |
+
+## Operator actions
+
+1. Reclaim index bloat (online, no lock; needs free disk near the index size; run largest first):
+   ```
+   REINDEX INDEX CONCURRENTLY smp_server.idx_msg_queues_updated_at_recipient_id;
+   REINDEX INDEX CONCURRENTLY smp_server.idx_msg_queues_notifier_id;
+   REINDEX INDEX CONCURRENTLY smp_server.idx_msg_queues_ntf_service_id;
+   REINDEX INDEX CONCURRENTLY smp_server.idx_msg_queues_rcv_service_id;
+   REINDEX INDEX CONCURRENTLY smp_server.idx_msg_queues_sender_id;
+   REINDEX INDEX CONCURRENTLY smp_server.idx_msg_queues_link_id;
+   ANALYZE smp_server.msg_queues;
+   ```
+   Expect `pg_indexes_size('smp_server.msg_queues')` to drop from 104 GB to roughly 12 GB (measured
+   after reindexing). `ANALYZE` restores the index-only scan for `ntf_service_queues_count`. With the
+   batch-2 `fillfactor` the index bloat no longer rebuilds, so this is a one-time reclaim.
+
+2. Autovacuum reloptions are applied by the batch-2 migration (`msg_queues`, `messages` and its TOAST,
+   `services`), so no manual `ALTER TABLE` is needed. Reclaim the current `messages` bloat once with
+   `pg_repack smp_server.messages` (152 GB heap and TOAST, roughly 30-50 GB reclaimable); running
+   `pg_repack` on `msg_queues` also makes `fillfactor = 80` effective immediately rather than as pages
+   turn over. With `fillfactor = 80` the `msg_queues` index bloat no longer rebuilds, so scheduled
+   reindexing is rarely needed. To automate it anyway, run `reindexdb --concurrently` or `pg_repack`
+   from a scheduler; `pg_cron` runs each job inside a transaction and cannot execute
+   `REINDEX ... CONCURRENTLY`, so use `pg_cron` only for `VACUUM`/`ANALYZE` or a non-concurrent
+   maintenance-window `REINDEX`.
+
+3. Cut WAL and checkpoint pressure (all reloadable, no restart):
+   ```
+   ALTER SYSTEM SET wal_compression = 'lz4';
+   ALTER SYSTEM SET max_wal_size = '32GB';
+   ALTER SYSTEM SET checkpoint_timeout = '30min';
+   SELECT pg_reload_conf();
+   ```
+   Fewer checkpoints produce fewer full-page images; `wal_compression` shrinks those that remain.
+   Together these reduce the WAL written on the primary and shipped by `archive-push`. The message
+   insert and delete volume itself is fixed by traffic, but most of its WAL cost is amplification
+   (full-page images and hint-bit writes on bloated pages), which these settings and `REINDEX` reduce.
+   This is separate from pgBackRest repo-side compression, which does not reduce WAL on the primary.
+   Verify with `pg_stat_reset_shared('wal')`, wait, then re-check `wal_fpi` and `wal_bytes` in
+   `pg_stat_wal`.
+
+4. Enable `track_io_timing = on` so disk wait time is measurable in `pg_stat_statements`.

plans/2026-08-05-eth-crypto-bindings.md

@@ -0,0 +1,301 @@
+# Ethereum crypto primitives for simplexmq
+
+Client-side crypto for SimpleX names: enough to derive an Ethereum key and its
+address from a recovery phrase. General-purpose: these modules contain nothing
+specific to names, registrars or relayers.
+
+This is the first part of Workstream B of the SimpleX names v2 plan. The design
+it serves: names are owned by a plain EOA derived per name from one BIP-39 seed
+held by the device.
+
+## What is deliberately absent
+
+- **No EIP-7702 authorizations.** RLP is implemented only as far as EIP-1559
+  transactions need it, for registering names; the `RSLV` resolver path in
+  this repo stays strictly read-only.
+- **No stealth addresses and no EIP-712.** Signing covers recoverable ECDSA
+  over a 32-byte digest and EIP-1559 transactions, which name registration
+  requires. EIP-712 typed-data hashing and stealth addresses are added with
+  the first code that uses them.
+- **No BIP-32 public derivation.** We always hold the seed, so CKDpub, xpub
+  serialization and fingerprints are not implemented. Non-hardened *private*
+  derivation is, because BIP-44 paths end in non-hardened components.
+- **English wordlist only, and no full NFKD.** Every English BIP-39 word is
+  ASCII, so NFKD normalization, which BIP-39 mandates, does not change a valid
+  phrase's words. NFKD maps Unicode space characters such as U+00A0 and U+3000
+  to a space; `parsePhrase` splits on any character `Data.Char.isSpace`
+  accepts and lower-cases with `Data.Text.toLower`, and a word that is then not
+  in the ASCII wordlist, such as one in fullwidth letters, is rejected. So no
+  normalization dependency is needed. A passphrase is bytes the caller
+  normalizes.
+
+## Modules
+
+```
+Simplex.Messaging.Crypto.Secp256k1      FFI to libsecp256k1
+Simplex.Messaging.Crypto.BIP39          mnemonics
+Simplex.Messaging.Crypto.BIP39.English  embedded upstream 2048-word list
+Simplex.Messaging.Crypto.BIP32          HD derivation, WalletMaster
+Simplex.Messaging.Crypto.BIP44          BIP-44 paths, AccountIndex
+Simplex.Messaging.Eth.Address           addresses, EIP-55
+Simplex.Messaging.Eth.RLP               Recursive Length Prefix encoding
+Simplex.Messaging.Eth.Transaction       signed EIP-1559 transactions
+```
+
+`keccak256` is added to `Simplex.Messaging.Crypto`, next to `sha3_256`.
+
+## Types
+
+```haskell
+newtype Secp256k1PrivateKey         -- 32 bytes, validated in [1, n-1]
+newtype Secp256k1PublicKey          -- libsecp256k1's opaque 64-byte form
+newtype Secp256k1Context            -- randomised libsecp256k1 context, internal to simplexmq
+data PubKeyFormat = Compressed | Uncompressed
+
+newtype WalletEntropy               -- 16, 20, 24, 28 or 32 bytes
+data EntropyStrength = ES128 | ES160 | ES192 | ES224 | ES256
+
+data ExtendedKey = ExtendedKey {xkKey :: Secp256k1PrivateKey, xkChainCode :: ScrubbedBytes}
+data WalletMaster                   -- entropy with the master key it derives
+
+data CoinType = Ethereum            -- SLIP-44 coin types
+newtype AccountIndex                -- below 2^31
+
+newtype Address                     -- 20 bytes; StrEncoding is EIP-55, JSON via it
+```
+
+Private keys, chain codes, BIP-39 entropy and seeds are `ScrubbedBytes`:
+constant-time `Eq`, no readable `Show`, and zeroed when freed. A chain code is
+secret too: with the parent public key and one non-hardened child private key,
+it yields the parent private key.
+
+## Functions
+
+```haskell
+-- Secp256k1
+mkPrivateKey        :: ScrubbedBytes -> Either String Secp256k1PrivateKey  -- 32 bytes in [1, n-1]
+unPrivateKey        :: Secp256k1PrivateKey -> ScrubbedBytes
+secp256k1PublicKey  :: TVar ChaChaDRG -> Secp256k1PrivateKey -> IO Secp256k1PublicKey  -- total: key is validated
+serializePublicKey  :: PubKeyFormat -> Secp256k1PublicKey -> IO ByteString
+privateKeyTweakAdd  :: Secp256k1PrivateKey -> ScrubbedBytes -> IO (Maybe Secp256k1PrivateKey)
+
+-- BIP39
+mkEntropy           :: ScrubbedBytes -> Either String WalletEntropy
+unEntropy           :: WalletEntropy -> ScrubbedBytes
+randomEntropy       :: EntropyStrength -> TVar ChaChaDRG -> STM WalletEntropy
+parsePhrase         :: Text -> Either String WalletEntropy   -- word count, wordlist, checksum
+entropyPhrase       :: WalletEntropy -> ByteString           -- canonical lowercase phrase
+entropySeed         :: WalletEntropy -> ByteString -> ScrubbedBytes  -- PBKDF2 with the passphrase
+
+-- BIP32
+masterKey           :: ScrubbedBytes -> Either String ExtendedKey
+derivePath          :: TVar ChaChaDRG -> ExtendedKey -> [Word32] -> IO (Either String ExtendedKey)
+renderPath          :: [Word32] -> ByteString
+hardened            :: Word32 -> Word32
+isHardened          :: Word32 -> Bool
+mkWalletMaster      :: WalletEntropy -> Either String WalletMaster      -- empty passphrase
+parseWalletMaster   :: ScrubbedBytes -> ScrubbedBytes -> Either String WalletMaster  -- entropy and stored master
+masterEntropy       :: WalletMaster -> WalletEntropy
+walletMasterKey     :: WalletMaster -> ExtendedKey
+masterBytes         :: WalletMaster -> ScrubbedBytes         -- key then chain code, the storage form
+
+-- BIP44
+mkAccountIndex      :: Word32 -> Either String AccountIndex
+unAccountIndex      :: AccountIndex -> Word32
+bip44Path           :: CoinType -> AccountIndex -> [Word32]  -- m/44'/coin'/account'/0/0
+
+-- Crypto
+keccak256           :: ByteString -> ByteString
+
+-- Eth
+addressFromPublicKey  :: Secp256k1PublicKey -> IO Address
+addressFromPrivateKey :: TVar ChaChaDRG -> Secp256k1PrivateKey -> IO Address
+deriveAddress         :: TVar ChaChaDRG -> ExtendedKey -> [Word32] -> IO (Either String (ExtendedKey, Address))
+```
+
+Internal to simplexmq, for sharing one context:
+
+```haskell
+withContext         :: TVar ChaChaDRG -> (Secp256k1Context -> IO a) -> IO a
+secp256k1PublicKey_ :: Secp256k1Context -> Secp256k1PrivateKey -> IO Secp256k1PublicKey
+derivePath_         :: Secp256k1Context -> ExtendedKey -> [Word32] -> IO (Either String ExtendedKey)
+```
+
+`Address` has a `StrEncoding` instance: `strEncode` is the EIP-55 checksummed
+form and `strP` accepts bare or `0x`-prefixed hex, rejecting a bad mixed-case
+checksum.
+
+Like `Simplex.Messaging.Crypto.randomBytes`, `randomEntropy` takes a
+`TVar ChaChaDRG` and runs in `STM`. Every exported function that computes a
+public key takes the same generator for the context blinding seed, so this code
+never reads system entropy itself.
+
+A value of one of these types is valid by construction: `WalletEntropy` has one
+of the five sizes, `AccountIndex` is below 2^31, and `WalletMaster` holds a
+master key derived from its entropy, so every function from them is total. The
+fallible steps are the boundaries: `parsePhrase` for typed text,
+`parseWalletMaster` for a stored row and `mkAccountIndex` for a number.
+
+BIP-32 declares a master or child key invalid when its IL is 0 or at least n,
+one case in 2^128, and says to use the next index. This code instead applies
+SLIP-0010's rule, which trezor-crypto also implements: recompute the HMAC over
+`I` for a master key, or over `0x01 || IR || ser32(i)` for a child. Every key
+BIP-32 produces is produced unchanged; only an index BIP-32 would skip gets a
+key. The recomputation is bounded to three attempts, after which `masterKey`,
+`mkWalletMaster`, `derivePath` and `deriveAddress` return
+`Left "derivation failed after 3 attempts"`, an error no input is expected to
+produce, which consumers report as an internal error rather than a wallet
+condition.
+
+Because `parsePhrase` lower-cases each word, a recovery phrase with a
+capitalised word is accepted. This does not change the derived seed:
+`entropyPhrase` always rebuilds the canonical lowercase sentence from the
+wordlist, and that is what `entropySeed` hashes.
+
+## How applications use it
+
+An application defines the derivation path. For SimpleX names, one seed per
+device and one account per name, as in `Simplex.Chat.Wallet` in simplex-chat:
+
+```haskell
+ent        <- either fail pure $ parsePhrase phrase  -- phrase :: Text
+master     <- either fail pure $ mkWalletMaster ent
+n          <- either fail pure $ mkAccountIndex account
+(xk, addr) <- either fail pure =<< deriveAddress g (walletMasterKey master) (bip44Path Ethereum n)
+```
+
+## libsecp256k1 C API mapping
+
+```c
+secp256k1_context_create(SECP256K1_CONTEXT_NONE)   /* per withContext, then _randomize */
+secp256k1_context_destroy(ctx)
+secp256k1_ec_pubkey_create(ctx, pubkey, seckey)
+secp256k1_ec_pubkey_serialize(secp256k1_context_static, output, outputlen, pubkey, flags)
+secp256k1_ec_seckey_tweak_add(secp256k1_context_static, seckey, tweak)
+```
+
+Context randomisation blinds only the multiplication of a secret scalar by the
+generator, which only `secp256k1_ec_pubkey_create` performs here. `withContext`
+creates a context, randomises it with 32 bytes from the caller's
+`TVar ChaChaDRG`, passes it to the action and destroys it after. Each exported
+function that computes a public key creates one context: `deriveAddress` uses
+it for the derivation and the address of its result. Functions that end in `_`
+take the context and are defined for sharing it inside simplexmq.
+`secp256k1_ec_pubkey_serialize` and `secp256k1_ec_seckey_tweak_add` run on
+`secp256k1_context_static`, so `serializePublicKey`, `addressFromPublicKey`
+and `privateKeyTweakAdd` take no generator. Every function that calls
+libsecp256k1 runs in `IO`; `unsafePerformIO` is used only to read
+`secp256k1_context_static` once. The functions that take a `Secp256k1Context`
+are therefore not evaluated after the context is destroyed; the context is not
+retained past `withContext`.
+
+`Secp256k1PublicKey` is produced only by libsecp256k1 and has no encoding: its
+64-byte form is not portable between libsecp256k1 versions, and
+`serializePublicKey` returns the SEC1 bytes.
+
+`secp256k1_ec_seckey_tweak_add` returns 0 exactly when BIP-32 declares the
+child invalid (tweak out of range, or a zero result), which is why
+`privateKeyTweakAdd` returns `Maybe`; `deriveChild` then recomputes as above.
+
+libsecp256k1 never reads OS entropy: the caller supplies the context blinding
+seed. So unlike libbbs it causes no `getentropy` / ITMS-90338 issue on iOS, and
+requires no equivalent of the `commoncrypto` flag.
+
+## Build
+
+Submodule in `cbits/`, same pattern as blst and libbbs:
+`cbits/libsecp256k1`, https://github.com/bitcoin-core/secp256k1, pinned to
+**v0.8.0**.
+
+```
+c-sources:    cbits/libsecp256k1/src/{secp256k1,precomputed_ecmult,precomputed_ecmult_gen}.c
+```
+
+No `include-dirs` are needed: every libsecp256k1 include is quoted and relative
+to the including file. The embedded wordlist makes `file-embed` a library
+dependency; before this change only the `smp-server` and `xftp-server`
+executables and the test suite used it. `wide-word` provides `Word128` and
+`Word256` for transaction fields, each as narrow as both geth and reth accept.
+
+Built with one `-D` of our own, `ENABLE_MODULE_RECOVERY=1`, for recoverable
+signatures. The table-size settings (`ECMULT_WINDOW_SIZE`, `COMB_BLOCKS`,
+`COMB_TEETH`) have `#ifndef` defaults in the headers, and the checked-in
+precomputed tables are generated for those defaults; every other option, such
+as the other optional modules and x86-64 assembly, is off unless a `-D`
+enables it. `secp256k1.c` defines `SECP256K1_BUILD` itself.
+
+32-bit targets (armv7a-android, i686 musl) are covered by libsecp256k1's own
+fallback: `src/util.h` selects `SECP256K1_WIDEMUL_INT64`, and with it the 10x26
+field and 8x32 scalar backends, when the target has neither a native 128-bit
+integer nor 64-bit pointers.
+
+No `flake.nix` change is needed in simplex-chat: the per-platform overrides
+there only force `packages.simplexmq.components.library.libs` (external
+libraries, i.e. openssl for `extra-libraries: crypto`) and flags. Vendored
+`c-sources` require no nix entry, which is why blst and libbbs have none either.
+
+### Cross-compilation status
+
+Verified by building simplex-chat through its flake, before the wordlist in
+`Simplex.Messaging.Crypto.BIP39.English` was embedded with Template Haskell
+(`embedFile`), the construct the armv7a build fails on below. These builds have
+to be rerun:
+
+| Target | Result |
+|---|---|
+| `x86_64-linux` (native, nix) | compiles and links |
+| `aarch64-android` | **compiles and links** into the final shared object |
+| `armv7a-android` | libsecp256k1 compiles; final link not reached (see below) |
+| `x86_64-windows` (mingw) | blocked before our code, see below |
+| `aarch64-darwin-ios` | not yet run (requires a darwin host) |
+
+`aarch64-android` is the meaningful pass: it proves the C both cross-compiles
+and links into the shared object included in the app.
+
+The `armv7a-android` build compiles libsecp256k1, which proves that its
+`SECP256K1_WIDEMUL_INT64` fallback builds under the NDK, but the build then
+fails in simplex-chat's own `Simplex.Chat.Operators`, on the
+`$(embedFile "PRIVACY.md")` splice. Cross-compiled Template Haskell runs the
+splice on the target via `iserv-proxy` under `qemu-arm`, and that interpreter
+fails to resolve `realpath` from `libHSdirectory` and segfaults. So 32-bit
+*linking* remains unproven.
+
+`x86_64-windows` fails while bootstrapping the mingw cross-GHC, before any
+simplexmq code is compiled: haskell.nix applies
+`ghc-9.6-fix-code-symbol-jumps.patch` to `rts/linker/PEi386.c` twice from the
+same store path, and the second application aborts. That is a duplicate entry in
+the patch list of the pinned haskell.nix branch
+(`github:input-output-hk/haskell.nix/armv7a`), not something this change can
+influence.
+
+Both gaps can be closed without GHC by compiling the three C files with the
+cross toolchain directly and linking a test program against them, which verifies
+the C code independently of the Haskell build.
+
+## Tests
+
+`tests/CoreTests/EthCryptoTests.hs`, 104 examples, with published vectors read
+from vendored upstream files in `tests/fixtures`, each pinned by a sha256 test:
+
+- **BIP-39**: all 24 English vectors of `trezor/python-mnemonic/vectors.json`:
+  entropy to phrase, phrase to entropy, seed with the `TREZOR` passphrase, and
+  the master key against the vector's `xprv`.
+- **BIP-32**: vectors 1 to 4 of `bip-0032.mediawiki`, every chain, against its
+  `xprv`; the two vector 5 keys outside `[1, n-1]` drive the `mkPrivateKey`
+  rejections. The test decodes base58check itself; the library does not parse
+  `xprv`, so vector 5's malformed serializations do not apply.
+- **EIP-55**: the eight addresses from the EIP-55 spec, round-tripped.
+- **BIP-44**: the well-known `0x9858EfFD232B4033E47d90003D41EC34EcaEda94` for
+  the `abandon ... about` mnemonic at `m/44'/60'/0'/0/0`, plus accounts 1 and 2,
+  all through `deriveAddress`, whose key is checked against `derivePath`.
+- Keccak-256 of the empty string and of `abc`.
+- Recovery phrases with capitalised words, extra whitespace, and non-breaking or
+  ideographic spaces between words.
+- Negative cases: zero, short and out-of-range private keys, a tweak that makes
+  the key zero or is not 32 bytes, bad BIP-39 checksums and word counts,
+  out-of-range seeds, a stored master that does not match its entropy, account
+  indexes at or above 2^31, and bad EIP-55 checksums.
+
+The BIP-44 expectations were additionally reproduced by an independent
+pure-Python secp256k1 reference written for the purpose, so they were not
+computed only by this implementation.

plans/2026-08-22-xftp-file-storage-time.md

@@ -0,0 +1,168 @@
+# Implementation plan: XFTP variable file storage time
+
+Proposal: `../rfcs/2026-08-22-xftp-file-storage-time.md`.
+
+## simplexmq: entitlement crypto
+
+New module `Simplex.Messaging.Crypto.Entitlement`, over `Simplex.Messaging.Crypto.BBS`:
+
+Types:
+
+```
+newtype MasterKey = MasterKey ByteString
+
+data Entitlement = Entitlement
+  { expiresAt :: UTCTime,
+    entitlementName :: Text,
+    extraInfo :: Text
+  }
+
+data EntitlementCredential = EntitlementCredential
+  { issuerKeyIdx :: Word16,
+    masterKey :: MasterKey,
+    entitlement :: Entitlement,
+    issuerSignature :: BBSSignature
+  }
+
+data EntitlementProof = EntitlementProof
+  { issuerKeyIdx :: Word16,
+    entitlement :: Entitlement,
+    entProof :: BBSProof
+  }
+```
+
+Functions and constants:
+
+- the disclosed-message encoding: the master key is message 0 and stays undisclosed; `expiresAt`, `entitlementName`, and `extraInfo` are messages 1 to 3 and are disclosed. The protocol encoding of `Entitlement` and its field order follow the same order
+- the BBS header string `"SimpleX badges v1"` (shared with chat's badges, which sign under it), the message count, and the disclosed indexes
+- `generateEntitlementProof :: Map Word16 BBSPublicKey -> EntitlementCredential -> BBSPresHeader -> IO (Either String EntitlementProof)` (the issuer key is looked up by the credential's index; an absent index is `Left`)
+- `verifyEntitlement :: Map Word16 BBSPublicKey -> BBSPresHeader -> EntitlementProof -> IO EntitlementVerification`, where `data EntitlementVerification = EVValid | EVInvalid | EVUnknownIssuer` (the caller supplies the presentation header; the server reconstructs it, the proof never includes it)
+- the issuer public keys constant `Map Word16 BBSPublicKey`
+
+## simplexmq: protocol, new XFTP version
+
+In `Simplex.FileTransfer.Transport`:
+
+- add the next `VersionXFTP` and set `currentXFTPVersion` to 4
+- add `entitlementProof :: Maybe EntitlementProof` to `XFTPClientHandshake`, encoded before the `Tail` and only from this version
+- the presentation header is the session id alone
+
+In `Simplex.FileTransfer.Protocol`:
+
+- add `GrantedStorageTime` and its encoding; retain the one-character sum prefix for future variants:
+
+```
+data GrantedStorageTime = GSTExpires {epochSeconds :: Int64}
+```
+
+- add the storage time (`Maybe Word32`: `Nothing` and `Just 0` request the server maximum, a value above zero requests that number of hours) to `FNEW`
+- add the granted storage to `FRSndIds` as `Maybe GrantedStorageTime` (`Nothing` when decoding a response from a server below this version)
+
+## simplexmq: server configuration
+
+In `Simplex.FileTransfer.Server.Env` and `Simplex.FileTransfer.Server.Main`:
+
+- make `fileExpiration` non-optional (`ExpirationConfig`, no longer `Maybe`); the server always expires files, so the server maximum is always a concrete number of seconds
+- read a maximum storage time (a number of hours) for each entitlement name from the `[STORE_LOG]` INI section, from the keys `expire_files_hours_for_supporter` and `expire_files_hours_for_legend`, into `fileStorageEntitlements :: Map Text EntitlementConfig`, where `newtype EntitlementConfig = EntitlementConfig {storageTime :: Int64}` holds seconds; an absent key is skipped (that name gets the default), a present but malformed value fails startup
+- exit at startup if any name's maximum is below the default file expiration
+- add `entitlementKeys :: Map Word16 BBSPublicKey` to the server config (default = the shared constant, set from `Main`); the handshake verifies the proof against it, so the trusted keys never come from the sender
+
+## simplexmq: server session
+
+In `Simplex.FileTransfer.Server`:
+
+- `processClientHandshake` verifies the proof from the handshake, once per session, and resolves the maximum storage time for the entitlement name. `HandshakeSent` carries `EntitlementChecked`, so the first handshake verifies and every later one on that connection reuses the result, including after `processHello` returns the session to `HandshakeSent`
+- verify only when the answer can change: the name is configured (startup rejects a maximum below the default), and the entitlement expired less than 24 hours ago. A proof that fails these checks gets no verification; a proof that fails to verify is logged. In both cases the session gets the default maximum
+- a verified proof becomes `peerEntitlement :: Maybe SessionEntitlement` in `THAuthServer`, where `data SessionEntitlement = SessionEntitlement {expiresAt :: SystemSeconds, entConfig :: EntitlementConfig}`; `processXFTPRequest` takes it from there, so no proof is verified while a command is processed
+- `createFile` caps the requested storage time by the session maximum, which is the entitlement's storage time when the entitlement is still valid, and the default otherwise
+
+## simplexmq: server store and expiration
+
+The `files` table gets a nullable `expires_at`. Every new file stores a concrete `expires_at`. It is NULL only for pre-feature rows, which the migration must not re-date (it has no access to the operator's configured TTL); those are expired at query time as `created_at + ttl`.
+
+Common to both stores, in `Simplex.FileTransfer.Server.Store`:
+
+- add `expiresAt :: Maybe RoundedFileTime` to `FileRec`
+- in `createFile`, cap the requested hours at the session maximum, round the expiry up to the hour, store it, and return that same value as the granted storage
+- `expiredFiles` receives `now` and `old` (= `now - ttl`). A stored expiry is deleted when `expires_at < now` (no grace — it is already rounded up); a legacy row (no `expires_at`) is deleted when `created_at + fileTimePrecision < old` (the grace covers `created_at` being floored to the hour)
+- retain `created_at` for statistics, export, and the legacy fallback
+
+STM store:
+
+- in `expiredFiles`, expire a new file when `roundedSeconds expiresAt < now`, and a legacy file (no `expiresAt`) when `created_at + fileTimePrecision < old`
+
+PostgreSQL store, in `Simplex.FileTransfer.Server.Store.Postgres` and its migrations:
+
+- add the nullable column `expires_at BIGINT` (no backfill)
+- add one composite index `idx_files_expiry ON files (expires_at, created_at)`
+- `expiredFiles` query: `(SELECT ... WHERE expires_at < ? LIMIT ?) UNION ALL (SELECT ... WHERE expires_at IS NULL AND created_at < ? LIMIT ?)` with `(now, limit, old - fileTimePrecision, limit)`. The first arm deletes stored (already rounded-up) expiries; the second drains legacy rows, with the grace folded into `old - fileTimePrecision` so the columns stay bare and sargable. Each arm is one range over the composite index and stops at its own limit; a single `OR` predicate builds a bitmap of every match before the limit applies. A `COALESCE(expires_at, created_at + ttl)` predicate is avoided (not sargable, would force a sequential scan). No `ORDER BY` — the batch loop deletes all expired rows regardless of order. New files always store an expiry, so the second arm drains permanently once the legacy rows expire, and is then removed.
+
+Store log, in `Simplex.FileTransfer.Server.StoreLog`:
+
+- add the optional expiration to the `AddFile` record; a record without it parses to `Nothing` (the configured default), never a hardcoded value
+
+## simplexmq: agent
+
+The credential belongs to the user, so the agent holds it the way it holds the user's servers: in memory, supplied when the agent is created and replaced through an API. It is not stored by the agent.
+
+Per-user state in `Simplex.Messaging.Agent.Env.SQLite` and `Simplex.Messaging.Agent.Client`:
+
+- add `entitlements :: Map UserId EntitlementCredential` to `InitialAgentServers`, beside the servers
+- add `userEntitlements :: TMap UserId EntitlementCredential` to `AgentClient`, filled from it by `newAgentClient`
+- add `entitlementKeys :: Map Word16 BBSPublicKey` to `AgentConfig` (default = the shared constant), for the issuer key that proof generation needs
+
+Public API in `Simplex.Messaging.Agent`:
+
+- add storage time (`Maybe Word32` hours) to `xftpSendFile`
+- add `setUserEntitlement :: AgentClient -> UserId -> Maybe EntitlementCredential -> IO ()`, in the shape of `setProtocolServers`: it replaces the entry, and when the credential changed it closes that user's XFTP clients, so the next upload presents the new credential
+
+Store, in both the SQLite and PostgreSQL agent stores:
+
+- add a nullable storage time column (integer hours; NULL means the server maximum) to `snd_files`
+- add the migration to both stores
+- in `createSndFile`, store the storage time
+
+Upload, in `Simplex.Messaging.Agent.Client` and `Simplex.FileTransfer.Client`:
+
+- `getXFTPClient` takes a proof for the session as a parameter, `SessionId -> IO (Maybe EntitlementProof)`, beside the callback it already takes for a closed client. The client config holds no credential and no keys
+- `getXFTPServerClient` passes a function that generates the proof over the session id from the configured issuer keys, and only for a server of this user, matched by the key hash that TLS pins, so a file description of the sender cannot direct the entitlement to another server. A server that is not the user's, a missing credential, or a failure to generate gives `Nothing`, with the failure logged
+- `xftpClientHandshakeV1` calls it with the session id from the connection, and sends the result in the handshake
+- `agentXFTPNewChunk` reads the storage time from the send record and sends FNEW with it
+- `createXFTPChunk` returns the granted expiry (epoch seconds); `agentXFTPNewChunk` stores it on `NewSndChunkReplica`
+
+Completion:
+
+- `createXFTPChunk` returns the granted expiry as `Maybe GrantedStorageTime`; `SndFileChunkReplica` and `NewSndChunkReplica` carry `expiresAt :: Maybe GrantedStorageTime`
+- persist it in a nullable `replica_expires_at` column on `snd_file_chunk_replicas` (added to the entitlement migration): `createSndFileReplica` stores `epochSeconds`, `getSndFile` and `getNextSndChunkToUpload` read it back into `GSTExpires`
+- on `SFDONE`, report the file expiry: a chunk expires when its last replica expires (`max` over replicas, absent replicas ignored, `Nothing` only if none report); the file expires when its first chunk expires (`min` over chunks, `Nothing` if any chunk is unknown). `GrantedStorageTime` derives `Ord`
+- `SFDONE` gains a trailing `Maybe GrantedStorageTime` (not str-encoded); chat consumes it (wired later)
+
+Testing:
+
+- e2e test in `tests/XFTPAgent.hs`: generate a BBS keypair, sign a supporter credential (issuer key index 1), run the server with `entitlementKeys = {1: testPk}` and a supporter maximum above the default, run the sender agent with the same `entitlementKeys` and the credential for the user, send a file requesting a number of hours above the default and below that maximum, and assert `SFDONE`'s granted expiry rounds up `now + requested` (proof of the entitlement raising the max above the default)
+- the same upload without the credential is capped at the default maximum
+- the expiry is written before the status, because `BlockingInfo`'s notice parser is terminal (`A.takeByteString`), so any field after the blocking info makes the record unparseable and the file is dropped on restart. Every field added to `AddFile` must go before the status
+- store log round trip in `tests/CoreTests/StoreLogTests.hs`, in the shape of the SMP store log test: a file record survives a write, a read into the store, and compaction, with and without the expiry, including a file blocked with a notice in both cases
+
+## simplex-chat
+
+- remove lifetime badges: make `badgeExpiry` a `UTCTime`, drop the `"lifetime"` encoding, and remove the lifetime option from the UI and the CLI
+- map `BadgeInfo` to `Entitlement` (`entitlementName = textEncode badgeType`, `expiresAt = badgeExpiry`, `extraInfo = badgeExtra`) when calling the agent
+- pass the user's credential to the agent when it is created, and through `setUserEntitlement` when the badge changes, in the same places that pass and update the user's servers
+- pass the storage time to `xftpSendFile`
+- retain the `maxXFTPFileSize` size limit
+- reuse `verifyEntitlement` for peer-badge verification
+- import the issuer public keys from the shared simplexmq constant
+
+## State
+
+Steps 2 to 5 are implemented in simplexmq. Step 1 and step 6 belong to the chat branch that carries badges.
+
+## Order
+
+1. Add the entitlement crypto module; move chat's badge verification onto it and remove lifetime badges.
+2. Add the new XFTP version, the FNEW storage time, and the response.
+3. Change the server configuration, store, expiration, and store log.
+4. Move the proof to the handshake: the handshake field, the session state on the server, and the proof for the session on the client.
+5. Hold the credential per user in the agent, and add the API to replace it.
+6. Wire chat to pass the credential and the storage time.

plans/2026-10-08-fast-rotation-old-queue-drain.md

@@ -0,0 +1,62 @@
+# Fast rotation: draining the old receive queue
+
+Recipient only. The sender is unchanged.
+
+## Decisions
+
+- New `RcvSwitchStatus` value:
+
+  | constructor      | encoding        |
+  |------------------|-----------------|
+  | `RSReceivedQEND` | `received_qend` |
+
+  - Stored in `rcv_queues.switch_status`.
+  - `canAbortRcvSwitch` returns `False` for it.
+- `qEndMsg`, by the queue that delivered `QEND`:
+  - the queue named in `QEND`: the named queue is marked deleted, `ICDeleteRcvQueue` is enqueued.
+  - the new queue: the named queue is set to `RSReceivedQEND` and stays subscribed.
+  - `NSCCreate` and `SWITCH QDRcv SPCompleted`: sent when the named queue had no `RSReceivedQEND`.
+- Deletion of an `RSReceivedQEND` queue:
+  - trigger: `OK` response to an `ACK` on that queue, without a `GET` lock on it.
+  - action: the queue is marked deleted, `ICDeleteRcvQueue` is enqueued.
+- Bound, a `cleanupManager` step, for each connection with an `RSReceivedQEND` queue:
+  - condition: another receive queue of the connection has an active subscription.
+  - `delete_errors + 1 < deleteErrorCount`: `delete_errors` is incremented.
+  - otherwise: the queue is deleted as above; the first `DEL` error of `ICDeleteRcvQueue` removes it locally.
+- `ackSMPMessage` result, `type QueueDrained = Bool`:
+  - `OK`: `True`.
+  - `MSG`: `False`; the message is written to `msgQ`.
+- Apps: `received_qend` is added to `RcvSwitchStatus`.
+- Migration `20261008_rcv_switch_status`, SQLite and Postgres:
+  - up: index `idx_rcv_queues_switch_status` on `rcv_queues(switch_status)`.
+  - down: `received_qend` is set to `NULL`, the index is dropped.
+
+## Mechanics
+
+- `Agent/Protocol.hs`: `RSReceivedQEND`.
+- `Agent/Store.hs`: `canAbortRcvSwitch`.
+- `Client.hs`: `QueueDrained`; `ackSMPMessage` returns it.
+- `Agent/Client.hs`: `sendAck` returns `QueueDrained`.
+- `Agent/Store/AgentStore.hs`: `getEndedRcvQueueConnIds`:
+
+  ```sql
+  SELECT conn_id FROM rcv_queues WHERE switch_status = ? AND deleted = 0
+  ```
+
+- `Agent.hs`:
+  - `deleteRcvQueueAsync`: `setRcvQueueDeleted`, then `ICDeleteRcvQueue`.
+  - `qEndMsg`.
+  - `ackQueueMessage`.
+  - `cleanupManager`: `expireEndedRcvQueues`.
+- Apps:
+  - `apps/ios/SimpleXChat/APITypes.swift`
+  - `apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/model/SimpleXAPI.kt`
+
+## Tests
+
+- `QEND` on the new queue while a message waits on the old queue:
+  - the message is delivered;
+  - the old queue is deleted after the `ACK` that returns `OK`.
+- The same with a recipient restart before the `ACK`.
+- Old server stopped: the old queue is deleted by the bound.
+- Downgrade: `received_qend` is cleared, other switch statuses are kept.

plans/2026-10-09-auth-error-delay.md

@@ -0,0 +1,52 @@
+# Delayed AUTH error responses
+
+## Decisions
+
+- Responses that include `ERR AUTH` are sent at the next multiple of the delay after the start time.
+- Delayed response sets:
+
+  | Source | Responses sent together | Start time |
+  |---|---|---|
+  | `receive`, failed verification | verification and parsing errors of one received block | block receipt |
+  | `client`, command processing | responses and messages of one batch | batch read from `rcvQ` |
+  | `RFWD`, forwarded command | the `RRES` of that command | start of forwarded command processing |
+
+- A thread created by `forkClient` writes the responses to the send queue.
+- Delay 0: the responses are written to the send queue directly.
+- `authDelayExceeded`: incremented when processing took longer than the delay.
+  - Prometheus counter `simplex_smp_auth_delay_exceeded`.
+  - Not saved in the stats backup.
+  - Not in the daily stats log.
+- Configuration:
+
+  | Item | Value |
+  |---|---|
+  | INI section | `TRANSPORT` |
+  | INI key | `auth_error_delay_ms` |
+  | default | 50 |
+  | `ServerConfig` field | `authErrorDelay`, microseconds |
+  | test servers | 20 ms |
+
+- AUTH timing test: elapsed time of 5 requests, test servers' delay.
+- Out of scope: NTF and XFTP servers.
+
+## Mechanics
+
+- `Server.hs`:
+  - `isAuthError`: response is `ERR AUTH`.
+  - `sendResponses`: responses with `ERR AUTH` to `sendAfterDelay`, others to the send queue.
+  - `sendAfterDelay`: writes responses to the send queue at the next multiple of the delay.
+  - `receive`: block receipt time from the `rcvActiveAt` reading.
+  - `client`: batch time.
+  - `processForwardedCommand`: start time.
+- `Server/Env/STM.hs`: `authErrorDelay` field, `defaultAuthErrorDelayMs`.
+- `Server/Stats.hs`: `authDelayExceeded` in `ServerStats` and `ServerStatsData`.
+- `Server/Prometheus.hs`: `simplex_smp_auth_delay_exceeded`.
+- `Server/Main.hs`: INI value.
+- `Server/Main/Init.hs`: INI template line.
+- `simplexmq.cabal`: `timeit` removed from the test suite.
+- `tests/SMPClient.hs`: `authErrorDelay = 20000`.
+- `tests/ServerTests.hs`:
+  - delayed AUTH for wrong key, absent queue, absent link and suspended queue; immediate `SOK`.
+  - AUTH timing test measures elapsed time.
+- `tests/SMPProxyTests.hs`: delayed AUTH to forwarded `SEND` and `LGET`.

protocol/agent-protocol.md

@@ -85,7 +85,7 @@ SMP agent protocol has 2 main parts:
 
 ![Duplex connection procedure](./diagrams/duplex-messaging/duplex-creating.svg)
 
-The procedure of establishing a duplex connection is explained on the example of Alice and Bob creating a bi-directional connection consisting of two unidirectional (simplex) queues, using SMP agents (A and B) to facilitate it, and two different SMP routers (which could be the same router). It is shown on the diagram above and has these steps:
+The procedure of establishing a duplex connection is explained on the example of Alice (the initiating party) and Bob (the joining party) creating a bi-directional connection consisting of two unidirectional (simplex) queues, using SMP agents (A and B) to facilitate it, and two different SMP routers (which could be the same router). It is shown on the diagram above and has these steps:
 
 1. Alice requests the new connection from the SMP agent A using agent `createConnection` api function.
 2. Agent A creates an SMP queue on the router (using [SMP protocol](./simplex-messaging.md) `NEW` command) and responds to Alice with the invitation that contains queue information and the encryption keys Bob's agent B should use. The invitation format is described in [Connection link](connection-link-1-time-invitation-and-contact-address).
@@ -208,7 +208,7 @@ This syntax of decrypted SMP client message body is defined by `decryptedAgentMe
 Decrypted SMP message client body can be one of 4 types:
 - `agentConnInfo` - used by the initiating party when confirming reply queue - sent in `agentConfirmation` envelope.
 - `agentConnInfoReply` - used by accepting party, includes reply queue(s) in the initial confirmation - sent in `agentConfirmation` envelope.
-- `agentRatchetInfo` - used to pass additional information when renegotiating double ratchet encryption - sent in `agentRatchetKey` envelope.
+- `agentRatchetInfo` - used to pass additional information when renegotiating double ratchet encryption - sent in `agentRatchetKey` envelope. A key sent in reply to another key includes the hash of that key; agents do not reply to such keys.
 - `agentMessage` - all other agent messages.
 
 `agentMessage` contains these parts:
@@ -233,7 +233,8 @@ connInfo = *OCTET
 agentConnInfoReply = %s"D" smpQueues connInfo
 smpQueues = length 1*newQueueInfo ; NonEmpty list of reply queues
 agentRatchetInfo = %s"R" ratchetInfo
-ratchetInfo = *OCTET
+ratchetInfo = [answeredKeyHash *OCTET] ; bytes after answeredKeyHash are ignored
+answeredKeyHash = %s"0" / (%s"1" shortString) ; "0" in a key that starts renegotiation, otherwise SHA-256 of the two raw public keys of the answered key
 
 agentMessage = %s"M" agentMsgHeader aMessage
 agentMsgHeader = agentMsgId prevMsgHash

protocol/crypto-registry.md

@@ -0,0 +1,189 @@
+Revision 1, 2026-10-01
+
+# SimpleX Network: cryptographic primitive registry
+
+The cryptographic primitives, constructions and domain-separation strings used by this repository, and which of them new code may use. For the threat model see [Security](./security.md).
+
+Module aliases follow the codebase: `C` = [Simplex.Messaging.Crypto](../src/Simplex/Messaging/Crypto.hs), `LC` = [Crypto.Lazy](../src/Simplex/Messaging/Crypto/Lazy.hs), `CR` = [Crypto.Ratchet](../src/Simplex/Messaging/Crypto/Ratchet.hs), `SL` = [Crypto.ShortLink](../src/Simplex/Messaging/Crypto/ShortLink.hs), `KEM` = [Crypto.SNTRUP761](../src/Simplex/Messaging/Crypto/SNTRUP761.hs) and [its bindings](../src/Simplex/Messaging/Crypto/SNTRUP761/Bindings.hs), `BBS` = [Crypto.BBS](../src/Simplex/Messaging/Crypto/BBS.hs). Lengths are in bytes.
+
+## Table of contents
+
+- [Policy](#policy)
+- [Defaults for new code](#defaults-for-new-code)
+- [Primitives](#primitives)
+- [SimpleX box constructions](#simplex-box-constructions)
+- [Domain separation and KDF inputs](#domain-separation-and-kdf-inputs)
+- [Nonces and IVs](#nonces-and-ivs)
+- [Implementation sources](#implementation-sources)
+- [xftp-web (TypeScript)](#xftp-web-typescript)
+- [Tests](#tests)
+
+## Policy
+
+- New code uses the default primitive for its purpose from [Defaults for new code](#defaults-for-new-code).
+- Using any other primitive, or a primitive marked *restricted* outside its listed purpose, requires a written justification in the pull request and review by a maintainer responsible for cryptography.
+- *Legacy-only* primitives stay only for the existing wire formats and stored data listed below. Do not add call sites.
+- Every new HKDF info string or hash prefix must be unique, start with `SimpleX`, and be added to [Domain separation and KDF inputs](#domain-separation-and-kdf-inputs).
+- Changing a primitive, key length, KDF input, info string or nonce construction changes the wire format. It requires a protocol version and an update of this registry and of the affected `protocol/` specification.
+
+Status values used below:
+
+| Status | Meaning |
+|---|---|
+| Default | Approved and preferred for new code for this purpose |
+| Approved | Approved for new code for this purpose when the default does not fit |
+| Restricted | Approved only for the listed purpose (external interoperability or a single protocol) |
+| Legacy-only | Kept for existing wire formats or stored data; no new call sites |
+
+## Defaults for new code
+
+| Purpose | Default | Functions |
+|---|---|---|
+| Signature | Ed25519 | `C.sign'`, `C.verify'` with `C.SEd25519` |
+| Key agreement | X25519 | `C.generateKeyPair`, `C.dh'` |
+| Post-quantum KEM | sntrup761 | `KEM.sntrup761Keypair`, `sntrup761Enc`, `sntrup761Dec` |
+| Hybrid DH + KEM secret | HKDF-SHA512 over `dh \|\| kemSecret` with a new info string, as in `CR.rootKdf` | `C.hkdf` |
+| Authenticated encryption | XSalsa20-Poly1305 [SimpleX secretbox](#simplex-box-constructions) | `C.sbEncrypt`/`C.sbDecrypt`, `LC.sbEncryptTailTag` for streams |
+| Authenticated encryption with a DH secret | XSalsa20-Poly1305 [SimpleX crypto_box](#simplex-box-constructions) | `C.cbEncrypt`/`C.cbDecrypt` |
+| Forward-secret symmetric session | HKDF-SHA512 key chain | `C.sbcInit`, `C.sbcHkdf` |
+| KDF | HKDF-SHA512 | `C.hkdf` |
+| Hash commitment, derived identifier | SHA3-256 | `C.sha3_256` |
+| Certificate fingerprint | SHA-256 X.509 fingerprint | `C.signedFingerprint`, `C.certificateFingerprint` |
+| Randomness | ChaChaDRG seeded from system entropy | `C.newRandom`, `C.randomBytes` and the `random*` helpers |
+
+## Primitives
+
+| Primitive | Status | Purpose and protocol | Implementation | Lengths |
+|---|---|---|---|---|
+| Ed25519 | Default | SMP/NTF/XFTP queue and file keys (agent default `rcvAuthAlg`, `sndAuthAlg` in [Agent/Env/SQLite.hs](../src/Simplex/Messaging/Agent/Env/SQLite.hs)), short-link signatures and owner auth (`SL.encodeSign`, `SL.newOwnerAuth`), XRCP identity and session keys ([RemoteControl/Invitation.hs](../src/Simplex/RemoteControl/Invitation.hs)), agent service request signatures, client/service TLS certificates ([Transport/Credentials.hs](../src/Simplex/Messaging/Transport/Credentials.hs)) | crypton `Crypto.PubKey.Ed25519` via `C.sign'`/`C.verify'` | pub 32, priv 32, sig 64; X.509 SPKI 44 |
+| Ed448 | Approved | Server CA and online certificates (default `signAlgorithm = ED448` in [Server/CLI.hs](../src/Simplex/Messaging/Server/CLI.hs)); accepted for SMP command signatures and TLS | crypton `Crypto.PubKey.Ed448`; server key generation by the `openssl genpkey` CLI | pub 57, priv 57, sig 114; SPKI 69 |
+| X25519 | Default | Per-queue e2e and server-to-recipient `crypto_box` keys, SMP session DH, command authenticator (SMP v7+), proxy (PRXY/PFWD), notification tokens, XFTP chunk transport, XRCP hello and announcements | crypton `Crypto.PubKey.Curve25519` via `C.dh'` | pub 32, priv 32, secret 32; SPKI 44 |
+| X448 | Restricted | E2E double ratchet only: X3DH and DH ratchet (`CR.RatchetX448`, E2E v3) | crypton `Crypto.PubKey.Curve448` | pub 56, priv 56, secret 56 |
+| sntrup761 | Default | PQ KEM in double ratchet (E2E v3, agent v5+) and XRCP v1 hello | Vendored C [cbits/sntrup761.c](../cbits/sntrup761.c) via FFI; randomness from ChaChaDRG through `haskell_rng_func` ([Bindings/RNG.hs](../src/Simplex/Messaging/Crypto/SNTRUP761/Bindings/RNG.hs)) | pk 1158, sk 1763, ct 1039, shared 32 |
+| XSalsa20-Poly1305, SimpleX crypto_box | Default | SMP message bodies (server to recipient), per-queue e2e envelope, confirmations, proxy forwarding, notifications, XRCP hello, XFTP chunk transport | crypton `Crypto.Cipher.XSalsa`, `Crypto.MAC.Poly1305`; `C.cryptoBox`, `LC.cbInit` | key 32 (DH secret), nonce 24, tag 16 |
+| XSalsa20-Poly1305, SimpleX secretbox | Default | SMP transport block encryption (SMP v11+), short-link data (SMP v15+), XFTP file encryption, local file encryption ([Crypto/File.hs](../src/Simplex/Messaging/Crypto/File.hs)), XRCP session | same; `C.sbEncrypt`, `LC.sbEncryptTailTag`, `LC.sbInit` | key 32, nonce 24, tag 16 |
+| crypto_box authenticator | Approved | Deniable sender command authorization with X25519 queue keys (SMP v7+); agent currently creates Ed25519 keys | `C.cbAuthenticate`, `C.cbVerify`: `crypto_box(sha512(msg))` | 80 = 64 + 16 |
+| AES-256-GCM, 16-byte IV | Legacy-only | Double ratchet header and body (E2E v3, [pqdr.md](./pqdr.md)) | crypton `Crypto.Cipher.AES`, `Crypto.Cipher.Types`; `C.encryptAEAD`, `C.decryptAEAD`, `C.initAEAD`; J0 = GHASH(IV) as NIST SP 800-38D defines for non-96-bit IVs | key 32, IV 16, tag 16; padded header 88 (PQ off) or 2310 (PQ on), `CR.paddedHeaderLen` |
+| AES-256-GCM, 12-byte IV | Restricted | WebRTC frame encryption in simplex-chat (`Simplex.Chat.Mobile.WebRTC`), for WebCrypto interoperability | `C.encryptAESNoPad`, `C.decryptAESNoPad`, `C.initAEADGCM`, `C.GCMIV` | key 32, IV 12, tag 16 |
+| HKDF-SHA512 | Default | All KDFs listed in [Domain separation](#domain-separation-and-kdf-inputs) | crypton `Crypto.KDF.HKDF` with `SHA512`; `C.hkdf` (extract + expand) | output per use, at most 255 * 64 |
+| SHA-256 | Default for X.509 fingerprints, otherwise Restricted | X.509 fingerprints (`C.KeyHash`, server identity, XRCP CA, service certificate hash, SMP v16+), XFTP chunk digests, agent message hashes, `requestCode`, ratchet key dedup hash, security code `codeAD = sha256(rcAD)` | crypton; `C.sha256Hash`, `LC.sha256Hash`, `getFingerprint ... HashSHA256` | 32 |
+| SHA-512 | Restricted | XFTP file digests and file description hash, input of the crypto_box authenticator | crypton; `C.sha512Hash`, `LC.sha512Hash` | 64 |
+| SHA-512 (inside sntrup761) | Restricted | sntrup761 internal hash | [cbits/sha512.c](../cbits/sha512.c) calls OpenSSL `SHA512()` from the system `libcrypto` (`extra-libraries: crypto`) | 64 |
+| SHA3-256 | Default | Short-link key `sha3_256(fixedData)` (SMP v15+), XRCP hybrid secret, service request binding (agent v8) | crypton; `C.sha3_256`, `KEM.kemHybridSecret` | 32 |
+| SHA3-384 | Restricted | Client-supplied sender ID: first 24 bytes of `sha3_384(corrId)`, computed by the agent (`prepareConnectionLink'`, `newRcvConnSrv`) and the server (`createQueue`) | crypton; `C.sha3_384` | 48, truncated to 24 |
+| Keccak-256 | Restricted | Label hash for SMP name queries (`NameQuery NQHash`), must match the on-chain registry | crypton `Keccak_256`; `labelHash` in [SimplexName.hs](../src/Simplex/Messaging/SimplexName.hs) | 32 |
+| MD5 | Legacy-only, non-security | XOR-aggregated queue ID hash `IdsHash` for service subscription reconciliation (SUBS, NSUBS, SOKS, ENDS) | crypton (`C.md5Hash`, `Protocol.queueIdHash`); SQLite UDF `simplex_xor_md5_combine` ([Agent/Store/SQLite.hs](../src/Simplex/Messaging/Agent/Store/SQLite.hs)); PostgreSQL pgcrypto `digest(..., 'md5')` in server and NTF schemas | 16 |
+| ChaChaDRG | Default | Keys, nonces, correlation IDs, random IDs, sntrup761 randomness | crypton `Crypto.Random`; `C.newRandom` seeds with `drgNew` from system entropy | n/a |
+| BBS+ over BLS12-381, SHA-256 suite | Restricted | Badge entitlement credentials and proofs ([Crypto/Entitlement.hs](../src/Simplex/Messaging/Crypto/Entitlement.hs)), XFTP storage-time extension | Vendored submodules libbbs and blst (C and assembly) via FFI; randomness from libbbs `getentropy`, `SecRandomCopyBytes` with the `commoncrypto` flag, `BCryptGenRandom` on Windows ([cbits/getentropy_win.c](../cbits/getentropy_win.c)) | sk 32, pk 96, sig 80, proof 272 + 32 per undisclosed message |
+| ECDSA with SHA-256 (ES256) | Restricted | APNS provider JWT ([Push/APNS.hs](../src/Simplex/Messaging/Notifications/Server/Push/APNS.hs)) | crypton `Crypto.PubKey.ECC.ECDSA.sign`; key read by cryptostore `Crypto.Store.PKCS8`; curve taken from the key file | signature DER `SEQUENCE {r, s}`, base64url |
+| TLS 1.3 / 1.2 | Restricted | SMP, XFTP, NTF, XRCP transport: `TLS_CHACHA20_POLY1305_SHA256` (1.3), `ECDHE_ECDSA_CHACHA20POLY1305_SHA256` (1.2), groups X448 and X25519, Ed448 and Ed25519 signatures (`defaultSupportedParams`) | `tls` 1.9 | n/a |
+| TLS for browsers | Restricted | XFTP server with HTTPS credentials (`defaultSupportedParamsHTTPS`: `ciphersuite_strong`, FFDHE, P-521, ECDSA and RSA signatures); SMP server HTTPS requires RSA-4096 (`checkHTTPSCredentials`) | `tls`, `warp-tls` | n/a |
+| X.509 | Restricted | Certificate chains, fingerprints, signed session keys (`C.signX509`, `C.verifyX509`) | `crypton-x509`, `crypton-x509-validation`, `cryptostore` | n/a |
+| SQLCipher | Restricted | Agent SQLite database at rest (`PRAGMA key`); cipher parameters are SQLCipher defaults, not set in this repository | `direct-sqlcipher` (git dependency, [cabal.project](../cabal.project)) | n/a |
+
+## SimpleX box constructions
+
+`C.cryptoBox` and `LC.sbInit_` compute, for a 32-byte key `k` and 24-byte nonce `n`:
+
+```
+k1 = HSalsa20(k, 0^16)
+subkey = HSalsa20(k1, n[0..16])
+stream = Salsa20(subkey, n[16..24])
+polyKey = stream[0..32]
+ct = msg XOR stream[32..]
+tag = Poly1305(polyKey, ct)
+```
+
+| Construction | Key `k` | Equivalent libsodium call | Layout |
+|---|---|---|---|
+| SimpleX crypto_box (`C.cbEncrypt`, `LC.cbInit`) | X25519 shared secret | `crypto_box_easy` (`crypto_box_beforenm` is `HSalsa20(dh, 0^16)`) | strict: `tag \|\| ct`; lazy tail-tag: `ct \|\| tag` |
+| SimpleX secretbox (`C.sbEncrypt`, `LC.sbInit`, `KEM.kcb*`) | 32-byte symmetric key | `crypto_secretbox_easy` with key `crypto_core_hsalsa20(0^16, k)`, not with `k` | same |
+
+Padding before encryption: `C.pad` prefixes a 2-byte big-endian length and fills with `#` (message at most 65533 bytes); `LC.pad` prefixes an 8-byte length. `*NoPad` variants skip padding.
+
+## Domain separation and KDF inputs
+
+HKDF is `C.hkdf salt ikm info len` (HKDF-SHA512).
+
+| Info string | Salt | IKM | Output (split) | Function | Use |
+|---|---|---|---|---|---|
+| `"SimpleXX3DH"` | 64 zero bytes | `dh1 \|\| dh2 \|\| dh3 [\|\| kemShared]` | 96: `hk`, `nhk`, root key (32 each) | `CR.pqX3dh` | Ratchet initialization, added in E2E v2; KEM secret from E2E v3 (current minimum) |
+| `"SimpleXVerifyCode"` | 64 zero bytes | same as `SimpleXX3DH` | 32: `rcVCPQ` | `CR.pqX3dh` | Verification code covering all handshake keys, new ratchets |
+| `"SimpleXRootRatchet"` | root key | `dh [\|\| kemShared]` | 96: root key, chain key, next header key | `CR.rootKdf` | DH/PQ ratchet step |
+| `"SimpleXChainRatchet"` | empty | chain key | 96: chain key, message key (32 each), message IV (16), header IV (16) | `CR.chainKdf` | Per-message keys |
+| `"SimpleXSbChainInit"` | SMP session ID | X25519 session secret | 64: two 32-byte chain keys | `C.sbcInit` from `Transport.blockEncryption` | SMP transport block encryption, SMP v11+ |
+| `"SimpleXSbChainInit"` | empty | XRCP hybrid secret (below) | 64: two 32-byte chain keys | `C.sbcInit` in [RemoteControl/Client.hs](../src/Simplex/RemoteControl/Client.hs) | XRCP v1 session |
+| `"SimpleXSbChain"` | empty | chain key | 88: chain key (32), secretbox key (32), nonce (24) | `C.sbcHkdf` | Each SMP block and XRCP message |
+| `"SimpleXContactLink"` | empty | link key (32) | 56: link ID (24), secretbox key (32) | `SL.contactShortLinkKdf` | Contact short links, SMP v15+ |
+| `"SimpleXInvLink"` | empty | link key (32) | 32: secretbox key | `SL.invShortLinkKdf` | Invitation short links, SMP v15+ |
+
+Other derivations:
+
+| Value | Construction | Function | Use |
+|---|---|---|---|
+| Service request binding | `sha3_256("SimpleXService" \|\| rcAD)` | `serviceReqBinding` in [Agent.hs](../src/Simplex/Messaging/Agent.hs) | Agent RPC, agent v8 |
+| BBS header | `"SimpleX badges v1"` | `entitlementBBSHeader` | Badge credentials |
+| XRCP hybrid secret | `sha3_256(x25519Dh \|\| kemShared)` | `KEM.kemHybridSecret` | XRCP v1 |
+| Short-link key | `sha3_256(smpEncode FixedLinkData)` | `SL.encodeSignFixedData`, checked in `SL.decryptLinkData` | SMP v15+ |
+| Sender ID | `take 24 (sha3_384 corrId)` | `prepareConnectionLink'`, `newRcvConnSrv` in Agent.hs; `createQueue` in [Server.hs](../src/Simplex/Messaging/Server.hs) | Client-supplied sender ID with link data, SMP v15+; the server rejects a mismatch to prevent an ID oracle |
+| Ratchet associated data | `pubKeyBytes sk1 \|\| pubKeyBytes rk1` (raw X448, 112) | `CR.pqX3dh` | AD of every ratchet AEAD |
+| Security code | `sha256(rcAD)` | `ratchetVerifyCodes` in [AgentStore.hs](../src/Simplex/Messaging/Agent/Store/AgentStore.hs) | Connection verification |
+| Contact request code | `sha256(smpEncode (k1, k2, kem, sndId))` | `requestCode` in Agent.hs | Contact request binding |
+| Command authenticator | `crypto_box(sha512(authorized))` | `C.cbAuthenticate` | SMP v7+ |
+| Queue IDs hash | `xor` of `md5(queueId)` | `Protocol.queueIdsHash` | Service subscriptions |
+
+`"SimpleXSbChainInit"` and `"SimpleXSbChain"` are shared by SMP block encryption and XRCP. Their outputs are separated only by the IKM (and the salt for `sbcInit`).
+
+## Nonces and IVs
+
+| Use | Construction |
+|---|---|
+| SMP command authenticator, proxied command | `C.cbNonce corrId`, where `corrId` is a random 24-byte nonce (`C.randomCbNonce`) |
+| SMP proxied responses | `C.reverseNonce` of the request nonce |
+| Server-to-recipient message body | `C.cbNonce msgId`; `msgIdBytes = 24` in [Server/Main.hs](../src/Simplex/Messaging/Server/Main.hs) |
+| Per-queue e2e envelope, short-link data, notifications, XRCP hello, XFTP chunk download | Random 24-byte nonce, sent with the ciphertext |
+| XFTP file, local encrypted file | Random key and nonce per file (`C.randomSbKey`, `C.randomCbNonce`) |
+| SMP block, XRCP session messages | Key and nonce from `C.sbcHkdf`, one per block |
+| Ratchet header, body | 16-byte IVs from `CR.chainKdf`; header IV is sent, body IV is not |
+| WebRTC frames | 12-byte `C.GCMIV` supplied by the caller in simplex-chat |
+
+## Implementation sources
+
+| Source | Version or pin | Provides |
+|---|---|---|
+| crypton | 0.34 | Ed25519, Ed448, X25519, X448, XSalsa20, Poly1305, AES-GCM, HKDF, SHA-2, SHA-3, Keccak, MD5, ChaChaDRG, ECDSA |
+| tls, crypton-x509, crypton-x509-validation, cryptostore | 1.9.0, 1.7.6, 1.6.12, 0.3.0.1 | TLS, X.509, PKCS#8 |
+| [cbits/sntrup761.c](../cbits/sntrup761.c) | Copy of draft-josefsson-ntruprime-streamlined-00 ([cbits/README.md](../cbits/README.md)) | sntrup761 |
+| [cbits/sha512.c](../cbits/sha512.c) and system OpenSSL `libcrypto` | system | SHA-512 for sntrup761 |
+| `cbits/libbbs` submodule ([simplex-chat/libbbs](https://github.com/simplex-chat/libbbs)) | `59a0f4bf` | BBS+ (`bbs_sha256_ciphersuite`), SHA-256, SHAKE256 |
+| `cbits/blst` submodule ([supranational/blst](https://github.com/supranational/blst)) | `db3defd0` | BLS12-381 (C and `build/assembly.S`, `-D__BLST_PORTABLE__`) |
+| direct-sqlcipher | git `f814ee68` | SQLCipher |
+| `openssl` CLI | system | Server CA and certificate key generation (`createServerX509_`) |
+
+## xftp-web (TypeScript)
+
+[xftp-web](../xftp-web/) reimplements a subset for the browser XFTP client. It has no AES-GCM, HKDF, SHA-3, MD5, X448 or sntrup761 code.
+
+| Primitive | Implementation | Haskell counterpart |
+|---|---|---|
+| Ed25519 sign, verify, keys | libsodium-wrappers-sumo (`crypto_sign_*`), [src/crypto/keys.ts](../xftp-web/src/crypto/keys.ts) | `C.sign'`, `C.verify'` |
+| Ed448 verify | `@noble/curves/ed448`, keys.ts `verifyEd448` | `C.verify'` |
+| X25519 | libsodium `crypto_scalarmult`, `crypto_box_keypair` | `C.dh'` |
+| SimpleX crypto_box, secretbox, tail-tag streaming | Salsa20 block function written in TypeScript, libsodium `crypto_core_hsalsa20` and `crypto_onetimeauth_*`, [src/crypto/secretbox.ts](../xftp-web/src/crypto/secretbox.ts) | `C.cryptoBox`, `LC.sbInit_` |
+| crypto_box authenticator | [src/protocol/client.ts](../xftp-web/src/protocol/client.ts) `cbAuthenticate`, `cbVerify` | `C.cbAuthenticate`, `C.cbVerify` |
+| SHA-256, SHA-512 | libsodium `crypto_hash_sha256`, `crypto_hash_sha512*`, [src/crypto/digest.ts](../xftp-web/src/crypto/digest.ts) | `C.sha256Hash`, `C.sha512Hash` |
+| Random | WebCrypto `crypto.getRandomValues`; libsodium for key generation | `C.randomBytes` |
+
+## Tests
+
+None of the tests below compares against published reference vectors (NIST, RFC 8032, RFC 7748, RFC 5869, NaCl, the sntrup761 draft or the BBS draft). Interoperability is tested between this repository's own implementations.
+
+| Area | Test module (hspec group) | Kind |
+|---|---|---|
+| Ed25519, Ed448, X25519 crypto_box, secretbox, lazy and tail-tag secretbox, AES-GCM 12-byte IV, X.509 key encoding, X.509 chains, sntrup761, BBS+, entitlements, padding | [tests/CoreTests/CryptoTests.hs](../tests/CoreTests/CryptoTests.hs) | Round-trip; fixed lengths for BBS+; fixed bytes for padding |
+| Double ratchet (X25519 and X448), PQ KEM agreement, AES-GCM 16-byte IV | [tests/AgentTests/DoubleRatchetTests.hs](../tests/AgentTests/DoubleRatchetTests.hs) | Round-trip; decoding of a stored v2 ratchet JSON |
+| Short links (HKDF info strings, SHA3-256 link key) | [tests/AgentTests/ShortLinkTests.hs](../tests/AgentTests/ShortLinkTests.hs) | Round-trip, tamper rejection |
+| File encryption | [tests/CoreTests/CryptoFileTests.hs](../tests/CoreTests/CryptoFileTests.hs) | Round-trip |
+| XRCP session (SHA3-256 hybrid, sb chain) | [tests/RemoteControl.hs](../tests/RemoteControl.hs) | End-to-end |
+| SMP authenticators, block encryption, `IdsHash` (MD5) | [tests/ServerTests.hs](../tests/ServerTests.hs) | End-to-end; with the PostgreSQL queue store this checks Haskell MD5 against pgcrypto |
+| Haskell and xftp-web: SHA-256/512, Ed25519, Ed448 identity proof, X25519, DER keys, crypto_box, secretbox, tail-tag, authenticator, file encryption | [tests/XFTPWebTests.hs](../tests/XFTPWebTests.hs) (`XFTP Web Client`) | Byte-identical cross-language output |

protocol/diagrams/xrcp/session.mmd

@@ -17,11 +17,11 @@ note over CC, HC: TLS handshake
 par
     note over CC: validate client X509 credentials
     CC->>CI: session code from tlsUnique
-    CI-->>CC: user confirmation
+    CI-->>CC: user confirmation (optional if known)
 and
     note over HC: validate server X509 credentials
     HC->>HI: session code from tlsUnique
-    HI-->>HC: user confirmation
+    HI-->>HC: user confirmation (optional if known)
 end
 
 note over CI, HI: 3. Session verification and protocol negotiation

protocol/pqdr.md

@@ -62,14 +62,16 @@ It is possible to reduce size overhead by using only one KEM agreement and makin
 
 ## Double ratchet with encrypted headers augmented with double PQ KEM
 
-Algorithm below assumes that in addition to shared secret from the initial key agreement, there will be an encapsulation key available from the party that published its keys (Bob).
+Algorithm below assumes that in addition to shared secret from the initial key agreement, there will be an encapsulation key available from the initiating party, that sent its keys in the connection invitation to the joining party (see [agent protocol](./agent-protocol.md)). Following the double ratchet specification, the pseudo-code below names the joining party Alice and the initiating party Bob.
+
+Unlike X3DH prekey bundles, these keys are not reusable keys published for any party to use. The initiating party generates two X448 key pairs and, optionally, a sntrup761 KEM key pair for each connection, sends the public keys in the invitation, and deletes the stored private keys once the ratchet is initialized; only the second X448 private key and the agreed KEM key pair remain in the ratchet state as its initial ratchet keys. The joining party generates its keys when joining and keeps only its KEM key pair, in the ratchet state. The exception is a contact address that publishes ratchet keys in its link data: all requesters use these keys until the address owner rotates them, and the owner keeps the private keys of a few recent generations.
 
 ### Initialization
 
 The double ratchet initialization is defined in pseudo-code. This pseudo-code is identical to Signal algorithm specification except for that parts that add post-quantum key agreement.
 
 ```
-// Alice obtained Bob's keys and initializes ratchet first
+// Alice (joining party) received Bob's keys in the invitation and initializes ratchet first
 def RatchetInitAlicePQ2HE(state, SK, bob_dh_public_key, shared_hka, shared_nhkb, bob_pq_kem_encapsulation_key):
     state.DHRs = GENERATE_DH()
     state.DHRr = bob_dh_public_key
@@ -89,7 +91,7 @@ def RatchetInitAlicePQ2HE(state, SK, bob_dh_public_key, shared_hka, shared_nhkb,
     state.HKr = None
     state.NHKr = shared_nhkb
 
-// Bob initializes ratchet second, having received Alice's connection request
+// Bob (initiating party) initializes ratchet second, having received Alice's first message
 def RatchetInitBobPQ2HE(state, SK, bob_dh_key_pair, shared_hka, shared_nhkb, bob_pq_kem_key_pair):
     state.DHRs = bob_dh_key_pair
     state.DHRr = None
@@ -210,6 +212,8 @@ The outer envelope contains the encrypted header (used as associated data for bo
 
 The message body is encrypted with AES-256-GCM using the message key derived from the sending chain key (`KDF_CK`). The associated data for body encryption is the concatenation of the ratchet associated data and the encoded encrypted header.
 
+`KDF_CK(CK)` is HKDF-SHA512 with empty salt, `CK` as input key material and info `"SimpleXChainRatchet"`, producing 96 bytes split into the next chain key (32 bytes), the message key (32 bytes), the message body IV (16 bytes, not transmitted) and `headerIV` (16 bytes). Both IVs are used as 16-byte AES-256-GCM IVs, not the 12-byte IVs recommended by NIST SP 800-38D, so the initial counter block is J0 = GHASH(IV || 0^64 || [128]_64) as defined there for non-96-bit IVs. WebCrypto and other conforming implementations compute it when given the full 16-byte IV; truncating the IV to 12 bytes produces different ciphertext.
+
 ```abnf
 encRatchetMessage = versionedLength encMessageHeader msgAuthTag encMsgBody
 ; encMessageHeader is used as associated data for body decryption: AD = rcAD || encMessageHeader
@@ -274,7 +278,7 @@ As SimpleX Messaging Protocol pads messages to a fixed size, using 16kb transpor
 
 Sharing the initial keys in case of SimpleX Chat it is equivalent to sharing the invitation link. As encapsulation key is large, it may be inconvenient to share it in the link in some contexts, e.g. when QR codes are used.
 
-It is possible to postpone sharing the encapsulation key until the first message from Alice (confirmation message in SMP protocol), the party sending connection request. The upside here is that the invitation link size would not increase. The downside is that the user profile shared in this confirmation will not be encrypted with PQ-resistant algorithm.
+It is possible to postpone sharing the encapsulation key until the first message from the joining party (confirmation message in SMP protocol). The upside here is that the invitation link size would not increase. The downside is that the user profile shared in this confirmation will not be encrypted with PQ-resistant algorithm.
 
 Another consideration is pairwise ratchets in groups. Key generation in sntrup761 is quite slow - on slow devices it can be as slow as 10-20 keys per second, so using this primitive in groups larger than 10-20 members would result in slow performance.
 

protocol/security.md

@@ -15,7 +15,7 @@ This document describes the cryptographic primitives and threat model for the Si
   - [SimpleX Messaging Protocol router that proxies the messages to another SMP router](#simplex-messaging-protocol-router-that-proxies-the-messages-to-another-smp-router)
   - [An attacker who obtained Alice's (decrypted) chat database](#an-attacker-who-obtained-alices-decrypted-chat-database)
   - [A user's contact](#a-users-contact)
-  - [An attacker who observes Alice showing an introduction message to Bob](#an-attacker-who-observes-alice-showing-an-introduction-message-to-bob)
+  - [An attacker who observes the initiating party showing an introduction message to the joining party](#an-attacker-who-observes-the-initiating-party-showing-an-introduction-message-to-the-joining-party)
   - [An attacker with Internet access](#an-attacker-with-internet-access)
 
 
@@ -37,6 +37,8 @@ This document describes the cryptographic primitives and threat model for the Si
   - AES-GCM AEAD cipher,
   - SHA512-based HKDF for key derivation.
 
+All primitives in use, their lengths, domain-separation strings and the policy for new code are listed in the [cryptographic primitive registry](./crypto-registry.md).
+
 
 ## Threat Model
 
@@ -168,7 +170,7 @@ This document describes the cryptographic primitives and threat model for the Si
 
 *cannot:*
 
-- impersonate a sender and send messages to the user whose database was stolen. Doing so requires also compromising the router (to place the message in the queue, that is possible until the Double-Ratchet advances forward) or the user's device at a subsequent time (to place the message in the database).
+- impersonate a sender and send messages to the user whose database was stolen, unless the database contains a new rotation queue that is not secured yet. Doing so requires also compromising the router (to place the message in the queue, that is possible until the Double-Ratchet advances forward) or the user's device at a subsequent time (to place the message in the database).
 
 - undetectably communicate at the same time as Alice with her contacts. Doing so would result in the contact getting different messages with repeated IDs.
 
@@ -190,15 +192,15 @@ This document describes the cryptographic primitives and threat model for the Si
 
 - cannot collaborate with another of the user's contacts to confirm they are communicating with the same user.
 
-### An attacker who observes Alice showing an introduction message to Bob
+### An attacker who observes the initiating party showing an introduction message to the joining party
 
 *can:*
 
-- Impersonate Bob to Alice.
+- Impersonate the joining party to the initiating party.
 
 *cannot:*
 
-- Impersonate Alice to Bob.
+- Impersonate the initiating party to the joining party.
 
 ### An attacker with Internet access
 

protocol/simplex-messaging.md

@@ -86,7 +86,7 @@ It's designed with the focus on communication security and integrity, under the
 
 It is designed as a low level protocol for other application protocols to solve the problem of secure and private message transmission, making [MITM attack][1] very difficult at any part of the message transmission system.
 
-This document describes SMP protocol version 20. Versions 1-5 are discontinued. The version history:
+This document describes SMP protocol version 23. Versions 1-5 are discontinued. The version history:
 
 - v1: binary protocol encoding
 - v2: message flags (used to control notifications)
@@ -108,6 +108,8 @@ This document describes SMP protocol version 20. Versions 1-5 are discontinued.
 - v19: service subscriptions to messages (SUBS, NSUBS, SOKS, ENDS, ALLS commands)
 - v20: public namespaces resolver (RSLV command, RNAME response) — direct or forwarded via PFWD
 - v21: server public information in handshake
+- v22: `RNAME` says whether a name can be registered, not only what it resolves to
+- v23: version in nonces of forwarded commands, random nonces for forwarded responses
 
 ## Introduction
 
@@ -1119,13 +1121,15 @@ When the client receives PKEY response it MUST validate that:
 - the fingerprint of the received certificate matches fingerprint in the router address - it mitigates MITM attack by proxy.
 - the router session key is correctly signed with the received certificate.
 
+`smpVersionRange` in `PKEY` is not signed.
+
 The proxy router may respond with error response in case the destination router is not available or in case it has an earlier version that does not support proxied commands.
 
 #### Send command via proxy
 
 Sender can send `SKEY` and `SEND` commands via proxy after obtaining the session ID with `PRXY` command (see [Request proxied session](#request-proxied-session)).
 
-Transmission sent to proxy router should use session ID as entity ID and use a random correlation ID of 24 bytes as a nonce for crypto_box encryption of transmission to the destination router. The random ephemeral X25519 key to encrypt transmission should be unique per command, and it should be combined with the key sent by the router in the handshake header to proxy and to the client in `PKEY` command.
+Transmission sent to proxy router should use session ID as entity ID and use a random correlation ID of 24 bytes as a nonce for crypto_box encryption of transmission to the destination router. When `smpVersion` in `PFWD` is 23 or higher, the first 2 bytes of this nonce are XOR-ed with `smpVersion`. The random ephemeral X25519 key to encrypt transmission should be unique per command, and it should be combined with the key sent by the router in the handshake header to proxy and to the client in `PKEY` command.
 
 Encrypted transmission should use the received session ID from the connection between proxy router and destination router in the authorized body.
 
@@ -1137,10 +1141,11 @@ commandKey = length x509encoded
 
 The proxy router will forward the encrypted transmission in `RFWD` command (see below).
 
-Having received the `RRES` response from the destination router, proxy router will forward `PRES` response to the client. `PRES` response should use the same correlation ID as `PFWD` command. The destination router will use this correlation ID increased by 1 as a nonce for encryption of the response.
+Having received the `RRES` response from the destination router, proxy router will forward `PRES` response to the client. `PRES` response should use the same correlation ID as `PFWD` command. The destination router will use this correlation ID increased by 1 as a nonce for encryption of the response. When `smpVersion` in `PFWD` is 23 or higher, the destination router uses a random nonce instead, and sends it before the encrypted response.
 
 ```abnf
-proxyResponse = %s"PRES" SP <encrypted padded(forwardedResponse, 16226)>
+proxyResponse = %s"PRES" SP [responseNonce] <encrypted padded(forwardedResponse, 16226)>
+responseNonce = %s"0" / (%s"1" 24*24 OCTET) ; from v23
 forwardedResponse = *OCTET ; client-encrypted SMP response, decrypted by client using per-command DH secret
 ```
 
@@ -1167,7 +1172,7 @@ The shared secret for encrypting transmission bodies between proxy router and de
 
 
 ```abnf
-relayResponse = %s"RRES" SP <encrypted(responseTransmission)>
+relayResponse = %s"RRES" SP [responseNonce] <encrypted(responseTransmission)>
 responseTransmission = fwdCorrId forwardedResponse
   ; fwdCorrId and forwardedResponse defined above in RFWD section
 ```
@@ -1451,53 +1456,150 @@ reads `NameRecord` from. The reference implementation forwards each RSLV to a
 companion REST resolver process (`scripts/resolver/snrc-resolve.py`) that
 queries the SNRC contract on Ethereum; alternative backings (different chains,
 DHT, etc.) are valid as long as they expose the documented HTTP shape (`GET
-/resolve/<name>` returning a `NameRecord` on 200, 404 / 400 for unknown names
-or TLDs, 502 for upstream RPC failures) or substitute a different transport
-while still returning a `NameRecord` matching the encoding below.
+/v2/resolve/<query>` returning a `NameRegistration` on 200 for every
+registration shape, 400 for unknown TLDs, 502 for upstream failures) or
+substitute a different transport returning the same JSON. The resolver API is
+versioned separately from this protocol: `/v1/resolve/<name>` returns a bare
+`NameRecord` and is what relays before v22 call as `/resolve/<name>`.
 
 #### Resolve name command
 
-The `RSLV` command carries the canonical fully-qualified name directly as the
-payload (not JSON):
+The `RSLV` command carries the query as text, not JSON. A client sends the
+hashed form only from v22, and the name itself below it:
 
 ```abnf
-rslv = %s"RSLV" SP domain   ; domain = canonical name as non-space bytes, consuming the remainder of the transmission
+rslv   = %s"RSLV" SP query
+query  = domain / hashed        ; hashed only from v22
+domain = 1*253 OCTET            ; the name as text
+hashed = "[" 64HEXDIG "]" tld   ; keccak-256 of the second-level label
+tld    = %s".simplex" / %s".testing"
 ```
 
 `domain` is the UTF-8 canonical fully-qualified name with the TLD always
 explicit (e.g. `privacy.simplex`, `test.testing`, `example.com`), bounded to
 253 bytes.
 
+**Hashed labels.** The query is a name, or the keccak-256 of a second-level
+label in ENS's bracketed form. A label can only be letters, digits and hyphens,
+so `[` tells the two apart and no tag is needed.
+
+Only a second-level name may be hashed. A name with subnames is sent as text: it
+is resolved rather than priced, and its record names it anyway. A web TLD has no
+registry to key a hash on.
+
+From v22 a client MUST send the hash. Older routers can only read the name, so a
+client on an older session sends the name. A router answering a hashed query
+does not know the label's length, so it cannot check a minimum-length policy
+either: the client does that, from the pricing it is sent.
+
+The same form reaches the backing resolver, which is what its HTTP API takes, so
+the query is one string end to end.
+
+A hashed query still answers with the name. The registrar records the plaintext
+label when a name is registered, keyed by the hash of that label, so a router can
+look up what the hash stands for without ever being told. The router is not
+trusted for it: a client MUST check that the record names the name it asked
+about, and reject the answer otherwise. A registry that does not record the
+label cannot answer a hashed query at all, and the router answers `ERR NAME
+RESOLVER` rather than a record it knows the client will reject. An unregistered
+name has no recorded label, so it cannot be looked up either.
+
 **Server-side validation.** The names router parses `domain` as a
 fully-qualified name (TLD required — bare labels are rejected) and forwards it
 to the configured backing resolver, which is the source of truth for which
 on-chain registry maps to each TLD.
 
-The names router responds with either an `RNAME` response carrying the resolved
-record, or an `ERR NAME` error whose subcode a client iterating across several
-configured servers can act on distinctly:
+The names router responds with either an `RNAME` response saying what it knows
+about the name, or an `ERR NAME` error whose subcode a client iterating across
+several configured servers can act on distinctly:
 
 | Response | Condition | Client action |
 |---|---|---|
-| `RNAME` | record resolved | use it |
-| `ERR NAME NOT_FOUND` | name not registered, unknown TLD, or malformed name | authoritative "no such name" — stop |
+| `RNAME` | the router read the registry | use it |
+| `ERR NAME NOT_FOUND` | below v22 only: every name that does not resolve. From v22 a router never sends it | stop, and do not read it as registrable |
 | `ERR NAME NO_RESOLVER` | this router has no resolver (names role not enabled) | skip this server, try the next |
-| `ERR NAME RESOLVER <detail>` | transient failure: backing resolver error (upstream 5xx, transport, timeout, decode) | transient — retry or surface, do not treat as "not found" |
+| `ERR NAME RESOLVER <detail>` | the router cannot state an answer completely: no registrar or price oracle for the TLD, an unreachable chain, a transport failure, a timeout, a registration it could not date or resolve | surface `<detail>`; retry only if it reads as transient |
 
 A client SHOULD NOT broadcast a `name` to further servers after a name-capable
 router has answered (`NOT_FOUND` or `RESOLVER`), since that router has already
 seen the lookup key; `NO_RESOLVER` discloses nothing about the name beyond the
 fact that this router cannot resolve, so iterating past it is safe.
 
-#### Name record response
+#### Name response
 
-The `RNAME` response carries a JSON-encoded record as the payload:
+`RNAME` answers both what a name resolves to and whether it can be registered.
 
 ```abnf
-rname = %s"RNAME" SP json-bytes   ; json-bytes consumes the remainder of the transmission
+rname = %s"RNAME" SP nameResponse
 ```
 
-`json-bytes` MUST be a UTF-8 JSON object with the following schema:
+`nameResponse` is a UTF-8 JSON object consuming the remainder of the
+transmission: the `registration` below, and `lastBlockTs`, the block timestamp the
+registry was read at. Money is US cents, times are seconds since the Unix epoch, and lengths are
+characters.
+
+| Field | JSON type | Constraints |
+|---|---|---|
+| `lastBlockTs` | number | block timestamp the answer was read at. Absent only from a v20/v21 router, which sent the record alone |
+| `registration` | object | below |
+
+A router reads the registry through a node of its own, which can lag. A name
+registered or changed after `lastBlockTs` still reads as it did before, so a client
+MUST NOT treat an answer as current without checking it.
+
+`registration`'s `type` selects which of the three answers it is.
+
+| `type` | Meaning |
+|---|---|
+| `registered` | held by someone until `expires`, renewable by its owner alone until `graceUntil`. It always carries `nameRecord`: where the owner set none, every field is unset and the resolver address is zero |
+| `available` | held by nobody and registrable now, at `pricing` |
+| `reserved` | held back by the registry and not registered. It carries no price, and a router MUST NOT quote one |
+
+| Field | On | JSON type | Constraints |
+|---|---|---|---|
+| `expires` | `registered` | number | absent only from a v20/v21 router, which sent the record alone |
+| `graceUntil` | `registered` | number | after `expires`; until here only the owner may renew. Absent on the same condition as `expires` |
+| `reservedReason_` | `registered` | string | a reason word, present only when the name is held back as well |
+| `nameRecord` | `registered` | object | the record, schema below |
+| `pricing` | `available` | object | `registrationPrices`, `basePrice` and `minLabelLength`, below |
+| `reservedReason` | `reserved` | string | a reason word |
+
+| `pricing` field | JSON type | Constraints |
+|---|---|---|
+| `registrationPrices` | object | label length, as a decimal string, to US cents per year, for the lengths the registry prices specially |
+| `basePrice` | number | US cents per year for every other length |
+| `minLabelLength` | number | characters; the registry refuses shorter labels |
+
+A reason word is `internal`, `trademark`, `community`, or a word a later version
+reserves under, at most 32 printable ASCII characters. A router truncates an
+unknown word to that and otherwise passes it through unchanged. A client MUST
+read a word it does not know as unknown and still treat the name as reserved.
+
+**Computing the price.** In US cents, for a duration in seconds:
+
+```
+price len duration = tier len * duration / 31536000
+tier len = the entry for len in registrationPrices, or basePrice when there is none
+```
+
+The registry's minimum registration is 730 days, a contract constant, so it is
+specified here rather than sent. `registrationPrices` omits any length below
+`minLabelLength`. A client MUST NOT show a quote for a label the registry
+refuses: a hashed query carries no length, so only the client can check it.
+
+Below v22, `RNAME` carries the bare record and nothing else, and every answer
+without one is `ERR NAME NOT_FOUND`. A v22 client reads such an answer as
+`registered` with no expiry, grace or reservation.
+
+From v22 a client MUST NOT read `ERR NAME NOT_FOUND` as "registrable": only
+`available` says that.
+
+A router that cannot state an answer completely MUST send `ERR NAME RESOLVER
+<detail>` rather than answer partially or guess. That covers a TLD with no
+registrar or price oracle, an unreachable chain, a timeout, a registration it
+could not date or resolve, and any status word it does not recognise.
+
+`nameRecord` MUST be a UTF-8 JSON object with the following schema:
 
 | Field | JSON type | Constraints |
 |---|---|---|
@@ -1514,34 +1616,16 @@ rname = %s"RNAME" SP json-bytes   ; json-bytes consumes the remainder of the tra
 | `owner` | string | `"0x"` followed by 40 lowercase hex characters (20 raw bytes) |
 | `resolver` | string | `"0x"` followed by 40 lowercase hex characters; the resolver contract address that produced the record |
 
-Text fields (`nickname`, `website`, `location`) use the empty string `""` as
-the "unset" sentinel: a backing resolver with no value for the field MUST emit
-an empty string, not JSON `null` and not an absent key. Link fields
-(`simplexContact`, `simplexChannel`) are arrays, primary link first, and use the
-empty array `[]` when unset. Coin fields (`eth`, `btc`, `xmr`, `dot`) use JSON
-`null` as the "unset" sentinel and MAY also be absent from the object entirely.
+Testnet-vs-mainnet status is derived from the queried TLD, not from the record.
 
-The backing resolver filters records that are expired or otherwise unavailable
-(the names router then returns `ERR NAME NOT_FOUND` to the client), so the wire
-format carries no expiry field. Testnet-vs-mainnet status is derived from the
-queried TLD rather than an in-record flag.
+Receivers MUST tolerate extra unknown fields; adding a required field is a
+breaking change requiring an SMP version bump. Receivers parse by key name, so
+peers MUST NOT rely on a byte-canonical form.
 
-Receivers MUST tolerate extra unknown fields (forward-compatibility for future
-field additions). Adding a required field is a breaking change requiring an
-SMP version bump.
-
-**Field order is not significant.** Receivers parse JSON by key name, so object
-key order, insignificant whitespace, and number formatting carry no meaning;
-records are interpreted by decoded value, never compared byte-for-byte. Peers
-MUST NOT rely on a byte-canonical form — a different resolver or server may emit
-the same record with different key order or spacing. This order-independence is
-what makes the format forward-compatible (see the unknown-field rule above).
-
-**Wire-size budget.** The names router caps the resolver response it will
-accept (`resolver_max_response_bytes`, ≤ 16000 bytes, the default) so the
-re-encoded `RNAME` stays within the SMP proxied transmission budget of 16224
-bytes; a response over the cap is rejected as `ERR NAME RESOLVER`. The link
-arrays are bounded by this overall budget rather than a fixed per-field count.
+The names router caps the resolver response it will accept
+(`resolver_max_response_bytes`, at most 16000 bytes) so the re-encoded `RNAME`
+stays within the SMP proxied transmission budget of 16224 bytes; a response over
+the cap is `ERR NAME RESOLVER`.
 
 ## Transport connection with the SMP router
 

protocol/xftp.md

@@ -1,4 +1,4 @@
-Version 3, 2025-01-24
+Version 4, 2026-08-08
 
 # SimpleX File Transfer Protocol
 
@@ -50,11 +50,12 @@ The objective of SimpleX File Transfer Protocol (XFTP) is to facilitate the secu
 
 XFTP is implemented as an application level protocol on top of HTTP2 and TLS.
 
-This document describes XFTP protocol version 3. The version history:
+This document describes XFTP protocol version 4. The version history:
 
 - v1: initial version
 - v2: authenticated commands - added basic auth support for commands
 - v3: blocked files - added BLOCKED error type for policy violations
+- v4: server public information and entitlement proof in handshake, file storage time in FNEW and SIDS
 
 The protocol describes the set of commands that senders and recipients can send to XFTP routers to create, upload, download and delete data packets of several pre-defined sizes. XFTP routers SHOULD support packets of 4 sizes: 64KB, 256KB, 1MB and 4MB (1KB = 1024 bytes, 1MB = 1024KB).
 
@@ -329,7 +330,7 @@ Once TLS handshake is complete, client and router will exchange blocks of fixed
 
 ```abnf
 paddedRouterHello = <padded(routerHello, 16384)>
-routerHello = xftpVersionRange sessionIdentifier routerCerts signedRouterKey ignoredPart
+routerHello = xftpVersionRange sessionIdentifier routerCerts signedRouterKey webIdentityProof serverInfo ignoredPart
 xftpVersionRange = minXftpVersion maxXftpVersion
 minXftpVersion = xftpVersion
 maxXftpVersion = xftpVersion
@@ -338,15 +339,27 @@ sessionIdentifier = shortString
 routerCerts = length 1*routerCert ; NonEmpty list of certificates in chain
 routerCert = originalLength <x509encoded>
 signedRouterKey = originalLength <x509encoded> ; signed by router certificate
+webIdentityProof = shortString ; signature over the web client challenge and sessionIdentifier, empty when the client sent no challenge
+serverInfo = %s"0" / (%s"1" largeString) ; JSON server public information, sent when maxXftpVersion is 4 or above
 
 paddedClientHello = <padded(clientHello, 16384)>
-clientHello = xftpVersion keyHash ignoredPart
+clientHello = xftpVersion keyHash entitlementProof ignoredPart
 ; chosen XFTP protocol version - must be the maximum supported version
 ; within the range offered by the router
 
+entitlementProof = %s"0" / (%s"1" issuerKeyIndex bbsProof entitlement)
+; proof of the user entitlement, encoded from v4, bound to sessionIdentifier
+issuerKeyIndex = 2*2OCTET ; Word16 index of the issuer public key
+bbsProof = largeString
+entitlement = entExpires entName entExtra
+entExpires = shortString ; expiration as an ISO8601 UTC timestamp
+entName = shortString ; e.g. "supporter", "legend"
+entExtra = largeString ; opaque to the router
+
 xftpVersion = 2*2OCTET ; Word16 version number
 keyHash = shortString
 shortString = length length*OCTET
+largeString = originalLength *OCTET
 length = 1*1OCTET
 originalLength = 2*2OCTET
 ignoredPart = *OCTET
@@ -432,7 +445,7 @@ Routers SHOULD support basic auth with this command, to allow only router owners
 The syntax is:
 
 ```abnf
-register = %s"FNEW " fileInfo rcvPublicAuthKeys basicAuth
+register = %s"FNEW " fileInfo rcvPublicAuthKeys basicAuth fileStorageTime
 fileInfo = sndKey size digest
 sndKey = length x509encoded
 size = 4*4 OCTET ; Word32 big-endian
@@ -440,19 +453,27 @@ digest = length *OCTET
 rcvPublicAuthKeys = length 1*rcvPublicAuthKey
 rcvPublicAuthKey = length x509encoded
 basicAuth = "0" / "1" length *OCTET
+fileStorageTime = %s"0" / (%s"1" storageHours)
+; encoded from v4; absent or zero requests the maximum the router allows
+storageHours = 4*4 OCTET ; Word32 big-endian
 
 x509encoded = <binary X509 key encoding>
 
 length = 1*1 OCTET
 ```
 
+`fileStorageTime` requests how long the router stores the data packet. The router grants the smaller of the request and the maximum it allows for the entitlement presented in the handshake, and returns the granted expiration in `sndIds`.
+
 If the data packet is registered successfully, the router must send `sndIds` response with the sender's and recipients' data packet IDs:
 
 ```abnf
-sndIds = %s"SIDS " senderId recipientIds
+sndIds = %s"SIDS " senderId recipientIds grantedStorageTime
 senderId = length *OCTET
 recipientIds = length 1*recipientId
 recipientId = length *OCTET
+grantedStorageTime = %s"0" / (%s"1" grantedExpires) ; encoded from v4
+grantedExpires = %s"T" expiresAt
+expiresAt = 8*8 OCTET ; Int64 big-endian, seconds since epoch
 ```
 
 #### Add data packet recipients

protocol/xrcp.md

@@ -71,7 +71,7 @@ The session invitation contains this data:
 
 Host application decrypts (except the first session) and validates the invitation:
 - Session signature is valid.
-- Timestamp is within some window from the current time.
+- Timestamp of a multicast announcement is not earlier than 3660 seconds before and not later than 3600 seconds after the current time of the host. The host ignores announcements outside of this interval and continues listening.
 - Long-term key signature is valid.
 - Long-term CA and signature key are the same as in the first session.
 - Some version in the offered range is supported.
@@ -121,9 +121,15 @@ During TLS handshake, parties validate certificate chains against previously kno
 
 ### Session verification and protocol negotiation
 
-Once TLS session is established, both the host and controller devices present a "session security code" to the user who must match them (e.g., visually or via QR code scan) and confirm on the host device. The session security code must be a digest of tlsunique channel binding. As it is computed as a digest of the TLS handshake for both the controller and the host, it will validate that the same TLS certificates are used on both sides, and that the same TLS session is established, mitigating the possibility of MITM attack in the connection.
+Once TLS session is established, both the host and controller devices present a "session security code" to the user. The session security code must be a digest of tlsunique channel binding. As it is computed as a digest of the TLS handshake for both the controller and the host, it will validate that the same TLS certificates are used on both sides, and that the same TLS session is established, mitigating the possibility of MITM attack in the connection.
 
-Once the session is confirmed by the user, the host sends HELLO block to the controller.
+To verify the session, the user matches the codes (e.g., visually or via QR code scan) and confirms the session on the host device.
+
+The user MUST verify the first session with a new controller. The host stores the long-term identity of the controller (CA fingerprint and Ed25519 public key) only after this session is verified. A controller with the stored long-term identity is a known controller.
+
+The user MAY verify the subsequent sessions with a known controller. In a session that the host confirms without user action, the controller is authenticated only by its long-term identity stored in the first session (see [Threat model](#threat-model)). SimpleX Chat mobile apps require user verification of sessions with known controllers only when the user enables the "Verify connections" option.
+
+Once the session is confirmed, the host sends HELLO block to the controller.
 
 XRCP blocks inside TLS are padded to 16384 bytes.
 
@@ -305,7 +311,7 @@ to receive: (rcvKey', sk, nonce) = HKDF(rcvKey, "SimpleXSbChain", 88)
 - prevent host and controller devices from establishing the session
 
 *cannot:*
-- same as passive adversary, provided that user visually verified session code out-of-band.
+- same as passive adversary, provided that user visually verified session code out-of-band in the first session with the controller.
 
 #### An active adversary with the access to the network:
 
@@ -324,6 +330,15 @@ to receive: (rcvKey', sk, nonce) = HKDF(rcvKey, "SimpleXSbChain", 88)
 *cannot:*
 - connect to the host or make host connect to itself.
 
+#### An active adversary with the access to the network who also obtained the long-term private keys of a known controller (e.g., from the controller device):
+
+*can:*
+- make host connect to itself instead of the controller.
+- access any data of the controlled host application, within the capabilities of the provided API, if the host confirms the session without user action.
+
+*cannot:*
+- access host application data, provided that user visually verified session code out-of-band.
+
 #### Compromised controller device:
 
 *can:*

rfcs/2026-08-22-xftp-file-storage-time.md

@@ -0,0 +1,109 @@
+# XFTP variable file storage time
+
+## Summary
+
+The server stores a storage time for each file. The sender sets it in the FNEW command. The client may present a proof of an entitlement in the handshake to raise the maximum storage time the server allows. The proof is bound to the TLS session, so it cannot be reused for another session.
+
+An entitlement belongs to the user, not to a file: the client presents it once per connection, and the server applies it to everything the client does in that session. The server can also vary other limits, such as throttling, by the entitlement.
+
+## Entitlement
+
+An entitlement is a name, an expiration, and an extra string. It is the disclosed content of a BBS proof: the holder's secret remains undisclosed, and the three fields are revealed. The server reads `entName` to select a maximum storage time, checks `entExpires`, and ignores `entExtra`; the interpretation of `entExtra` is out of scope here. The protocol references only the entitlement, never a badge; chat maps its own badge to an entitlement before it asks the agent to send.
+
+The proof discloses the entitlement and includes the issuer key index and the BBS proof. The holder's secret and the BBS signature remain with the sender and are never transmitted. The origin of the sender's signed entitlement, from the entitlement service, is out of scope here.
+
+```
+entitlement = entExpires entName entExtra
+entExpires = shortString     ; expiration as a UTCTime ISO8601 string
+entName = shortString        ; e.g. "supporter", "legend"
+entExtra = largeString       ; opaque, interpretation out of scope
+
+entitlementProof = issuerKeyIndex bbsProof entitlement
+issuerKeyIndex = 2*2 OCTET   ; Word16, network byte order
+bbsProof = largeString       ; BBS proof bytes
+```
+
+The presentation header that the BBS proof is generated over is not transmitted; the server takes it from the session (see [Binding](#binding)), which is what binds the proof.
+
+### Issuer keys
+
+Client apps and servers share one list of issuer public keys, indexed by `issuerKeyIndex`. The secret key for the current index is held by the service that issues entitlements, on conditions that are out of scope here, such as payment.
+
+The list holds eight keys so that the issuing service can rotate its current key without an app or server release: the next index is already known to every app and server. Each rotation consumes one index, so a new key has to be added to client apps and servers eventually, and released before the list is exhausted.
+
+## Storage time
+
+```
+fileStorageTime = %s"0" / (%s"1" storageHours)
+storageHours = 4*4 OCTET     ; Word32, network byte order
+```
+
+The storage time is an optional number of hours. Absent (`%s"0"`), or present as zero hours, requests the maximum the server allows for the presented entitlement, or the default maximum when no proof is present. A value above zero requests that number of hours, and the server grants the smaller of it and the maximum.
+
+## Handshake, new XFTP version
+
+The client handshake carries the entitlement proof.
+
+```
+clientHandshake = xftpVersion keyHash optEntitlementProof
+optEntitlementProof = %s"0" / (%s"1" entitlementProof)
+```
+
+`xftpVersion` and `keyHash` are defined by the current XFTP protocol. Version 3 and earlier encode no proof.
+
+The server verifies the proof when it accepts the handshake and keeps the result for the session; further handshakes on the same connection keep that result and verify nothing, so a session costs one verification however many handshakes it makes. A proof that names an entitlement the server does not configure, or one whose expiration passed 24 hours ago or more, is ignored without verification; a proof that fails to verify is logged. In each case the session gets the default maximum. The response is the same in every case, but only a configured, unlapsed name costs a verification, so the handshake latency tells the client which names the server configures.
+
+## Commands
+
+The new protocol version extends FNEW with the storage time.
+
+```
+fnew = %s"FNEW " fileInfo rcvKeys optBasicAuth fileStorageTime
+```
+
+`fileInfo`, `rcvKeys`, and `optBasicAuth` are defined by the current XFTP protocol. Version 3 and earlier encode no `fileStorageTime`, and the server applies the default storage time.
+
+## Responses
+
+FNEW extends the SIDS response with the granted storage.
+
+```
+sndIds = %s"SIDS " senderId rcvIds optGrantedStorageTime
+optGrantedStorageTime = %s"0" / (%s"1" grantedStorageTime)
+grantedStorageTime = grantedExpires
+grantedExpires = %s"T" expiresAt
+expiresAt = 8*8 OCTET        ; Int64, seconds since epoch (absolute UTC instant), network byte order
+```
+
+`grantedExpires` returns the absolute expiration — the same value stored for the file. The sum encoding retains a one-character prefix so further variants can be added. Version 3 and earlier omit `optGrantedStorageTime` entirely; a client decoding such a response reads it as absent. `senderId` and `rcvIds` are defined by the current XFTP protocol.
+
+## Binding
+
+The presentation header binds the proof to the TLS session, so a proof presented on any other session fails to verify.
+
+```
+presHeader = sessionId
+```
+
+`sessionId` is the TLS session identifier, the TLS unique channel binding. Both sides take it from the connection: the client has it once TLS is established, and the client checks that the identifier the server sends in its handshake matches.
+
+Binding to the session is what stops a proof being replayed by another client. A proof is not bound to a file, because the entitlement belongs to the user and authorises everything the client does in that session.
+
+BBS proofs of one credential are unlinkable, but every proof discloses the same `entExpires`, `entName` and `entExtra`, so a server can link the sessions of one holder by that triple, and two servers can correlate them. The issuing service decides how identifying it is: `entExpires` set to the same instant for everyone who buys in the same period, and an `entExtra` that carries nothing per-holder, make the holders of that period indistinguishable.
+
+## Maximum storage time
+
+The server configures a maximum storage time for each entitlement name, and a default maximum for requests with no proof. Each maximum is a number of hours. The server exits at startup if any name's maximum is below the default, so a proof never reduces the allowed time. The server honours an entitlement for 24 hours after its expiration; past that grace it is treated as no proof.
+
+If the requested time exceeds the maximum, the server stores the file for the maximum and does not reject the request. The expiration is rounded up to the hour, stored, and returned as `grantedExpires`.
+
+## Encoding primitives
+
+```
+shortString = length *OCTET   ; 0-255 bytes
+largeString = length2 *OCTET
+length = 1*1 OCTET
+length2 = 2*2 OCTET           ; Word16, network byte order
+```
+
+`senderId`, `rcvIds`, `fileInfo`, `sndKey`, `digest`, `rcvKeys`, `optBasicAuth`, and `sessionId` are defined by the current XFTP and SMP protocols.

scripts/ci/prune-old-builds.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+set -eu
+
+version=$(awk '/^version:/ {print $2}' simplexmq.cabal)
+for dir in "$@"; do
+  if [ -d "$dir/build" ]; then
+    find "$dir/build" -mindepth 3 -maxdepth 3 -type d -name 'simplexmq-*' ! -name "simplexmq-$version" -exec rm -rf {} +
+  fi
+done

scripts/ci/source-timestamps.sh

@@ -0,0 +1,27 @@
+#!/bin/sh
+set -eu
+
+command=$1
+hashes=$2
+
+sources() {
+  git ls-files -z '*.hs' '*.hsc'
+}
+
+case "$command" in
+  record)
+    sources | xargs -0 sha256sum > "$hashes"
+    ;;
+  restore)
+    sources | xargs -0 touch
+    while read -r hash file; do
+      if [ -f "$file" ] && [ "$(sha256sum "$file" | cut -d' ' -f1)" = "$hash" ]; then
+        touch -d 2000-01-01 "$file"
+      fi
+    done < "$hashes"
+    ;;
+  *)
+    echo "usage: $0 record|restore <hashes file>" >&2
+    exit 1
+    ;;
+esac

scripts/resolver/.env.example

@@ -1,5 +1,8 @@
+# Copy to .env (next to docker-compose.yml) only to change a setting; without it
+# docker-compose.yml uses the values shown here.
+
 # ============================================================================
-# Required settings — the stack will not start without these.
+# Chain settings.
 # ============================================================================
 
 # Ethereum network: mainnet (the SNRC `.testing` contracts live on mainnet)
@@ -14,9 +17,16 @@ NETWORK=mainnet
 TRUSTED_NODE_URL=https://mainnet-checkpoint-sync.attestant.io
 
 # ============================================================================
-# Optional overrides — sensible defaults are baked into docker-compose.yml,
-# so leave these commented unless you need to change them.
+# Optional overrides — leave commented unless you need to change them.
 # ============================================================================
 
 # Nimbus NAT (default: any). For a stable public node set an explicit IP:
 #   NAT=extip:1.2.3.4        # your public IPv4: curl -s ifconfig.me
+
+# Resolver (defaults in the README's "Load and scaling" and "Logs" sections)
+#   SNRC_WORKERS=4                          # worker processes; default CPU count, at most 4
+#   SNRC_RPC_TIMEOUT=5                      # seconds per request to reth
+#   SNRC_LOG_FORMAT=json                    # default text (key=value)
+#   SNRC_LOG_COLOR=always                   # coloured `docker compose logs`
+#   SNRC_LOG_LEVEL=debug                    # also log health checks and idle connections
+#   SNRC_TRUSTED_PROXIES=172.16.0.0/12      # behind a reverse proxy on the host; see README

scripts/resolver/README.md

@@ -19,15 +19,15 @@ against **Ethereum mainnet** (where the `.testing` contracts live):
 
 ## 1. Configure
 
-Edit `.env` — the defaults work as-is; override only if needed:
+The defaults in `docker-compose.yml` work as-is:
 
 ```sh
 NETWORK=mainnet                                               # default
 TRUSTED_NODE_URL=https://mainnet-checkpoint-sync.attestant.io # default
 ```
 
-Everything else (NAT) has a working default baked into `docker-compose.yml`;
-uncomment the hints in `.env` only to override.
+To override these or NAT, copy `.env.example` to `.env` and edit it; `.env` is
+not committed.
 
 ## 2. Run
 
@@ -62,15 +62,26 @@ curl -s -X POST http://127.0.0.1:8545 \
 **2. resolver is healthy:**
 ```sh
 curl -s http://127.0.0.1:8000/health | jq
-# → {"ok": true, "rpc": "http://reth:8545", "registries": {"testing": "0x…", "simplex": ""}}
+# → {"ok": true, "rpc": "http://reth:8545", "registries": {"testing": "0x…", "simplex": ""},
+#    "blockNumber": 23400000, "chainLagSeconds": 12}
 ```
 
+`chainLagSeconds` is how far the node is behind the wall clock. A resolver that
+is reachable and answering can still be hours behind, and every name it reports
+is that stale. `null` means the node could not be reached.
+
 **3. resolver resolves a live name** (`foobar.testing` is a populated test name):
 ```sh
 curl -s http://127.0.0.1:8000/resolve/foobar.testing | jq
 # → {"name":"foobar.testing","nickname":"Foo","simplexContact":["https://smp16.simplex.im/a#…"], … }
 ```
 
+**4. the route your router will call** (check 3 passes on an older resolver too):
+```sh
+curl -s http://127.0.0.1:8000/v2/resolve/foobar.testing | jq
+# → {"type":"registered","expires":1780000000,"graceUntil":…,"nameRecord":{…}}
+```
+
 **Wire your smp-server:** in its `[NAMES]` section set
 `resolver_endpoint: http://127.0.0.1:8000` (no auth needed for loopback).
 
@@ -82,7 +93,7 @@ curl -s http://127.0.0.1:8000/resolve/foobar.testing | jq
 | reth p2p | `:30303` tcp/udp | Ethereum sync (open on firewall) |
 | nimbus p2p | `:9000` tcp/udp | beacon sync (open on firewall) |
 | nimbus REST | `127.0.0.1:5052` | beacon API |
-| **resolver** | `127.0.0.1:8000` | SNRC REST (`/resolve`, `/health`) |
+| **resolver** | `127.0.0.1:8000` | SNRC REST (`/v2/resolve`, `/resolve`, `/health`) |
 
 ## Caveats
 
@@ -110,7 +121,66 @@ standalone for local dev (no Docker), via [`uv`](https://docs.astral.sh/uv/):
 uv run scripts/resolver/service/snrc-resolve.py  # defaults to local reth + mainnet .testing
 ```
 
-### Response shape
+Tests and lint run from `scripts/resolver/service`:
+
+```sh
+uv run pytest
+uv run ruff check .
+```
+
+Dependencies are locked in `uv.lock`; the Docker image installs exactly those.
+
+Three routes, versioned separately from the protocol so each only changes when
+its own shape does:
+
+| Route | Called by | Answers |
+|---|---|---|
+| `/v2/resolve/<query>` | routers from SMP v22 | a `NameRegistration` |
+| `/resolve/<name>` | routers before SMP v22 | a name record, flat |
+| `/health` | anyone | readiness |
+
+`/v1/resolve/<name>` is an alias for `/resolve/<name>`.
+
+### v2: `/v2/resolve/<query>`
+
+The body is the SMP protocol's `NameResponse`, which the router decodes as is
+and forwards; translating the registry's model to it is this resolver's job. It
+is the registration below, plus `lastBlockTs`, the timestamp of the block it was read
+at: a node that lags answers with names it has not seen registered yet.
+Its `type` is `registered`, `available` or `reserved`, and the fields each one
+carries are specified once, in the **Name response** section of
+[`protocol/simplex-messaging.md`](../../protocol/simplex-messaging.md). It is
+the wire format, so it is documented with the wire.
+
+Two things follow from that and are worth stating here. Expiry and grace belong
+to the registration, not to the record: `expires` and `graceUntil` sit beside
+`nameRecord`, not inside it. And a name nobody holds is not an error: it
+answers 200 with `type: available` and its price, so no status code from this
+route means "not registered".
+
+| Status | Meaning |
+|---|---|
+| 200 | a registration: `registered`, `available` or `reserved` |
+| 400 | `tldNotConfigured`, `notFullyQualified` |
+| 502 | `noPriceOracle`, `labelNotRecorded`, an unreadable status, or `upstreamError` |
+
+Error bodies carry `name` and a fixed `error` code to branch on. Only
+`upstreamError` adds a `message`; the v1 route always adds one.
+
+`labelNotRecorded` means the registrar holds the name but never recorded its
+label, so a hashed query cannot be answered with a name. See
+[Querying by labelhash](#querying-by-labelhash).
+
+A subname reports the expiry and grace of the 2LD above it, since that is what
+bounds its lifetime. A subname nobody created reports as not registered.
+
+### v1: `/resolve/<name>`
+
+What routers before SMP v22 call. Its shape is unrelated to v2's: the record is
+flat, and `status`, `expires`, `graceEnds`, `reasonCode` and `reason` sit
+alongside its fields.
+
+#### v1 response shape
 
 ```jsonc
 {
@@ -119,28 +189,281 @@ uv run scripts/resolver/service/snrc-resolve.py  # defaults to local reth + main
   "simplexContact": ["https://smp16.simplex.im/a#…", "https://smp11…"],  // primary first, fallbacks after
   "simplexChannel": [],
   "eth": null, "btc": "bc1q…", "xmr": "4ANz…", "dot": "139G…",
-  "owner": "0xd83b…", "resolver": "0x80fa…"
+  "owner": "0xd83b…", "resolver": "0x80fa…",
+  "status": "registered",      // registered | grace | expired | unregistered | unknown
+  "expires": 1780000000,       // Unix seconds; when the registration ends
+  "graceEnds": 1787776000,     // expires + GRACE_PERIOD; last moment the owner can renew
+  "reasonCode": null,          // set when the name is held back as well
+  "reason": null               // set when the name is held back as well
 }
 ```
 
 `simplexContact`/`simplexChannel` are arrays (a name can advertise multiple SMP
-servers; clients try them in order). On-chain they're a single comma-separated
+servers; clients try them in order). On-chain they're a single `;`-separated
 text record; the resolver splits/trims/drops-empties. Address encodings are
 canonical per chain (EIP-55 / bech32 / SS58 / Monero-base58). Subnames work
 identically (`bar.foobar.testing`).
 
-### Status codes
+#### v1 registration status and expiry
+
+A response carries `status`, `expires` and `graceEnds` whenever the resolver
+read them, a successful resolve included, so a client that has just resolved a
+name already knows when it expires. Both timestamps are Unix seconds, and
+`null` when they could not be read.
+
+| `status` | Meaning |
+|---|---|
+| `registered` | live; `expires` is when that ends |
+| `grace` | lapsed, but only the previous owner may renew it, until `graceEnds` |
+| `expired` | lapsed and past grace; anyone may register it |
+| `unregistered` | never registered, and free to take |
+| `unknown` | no `SNRC_REGISTRAR_<TLD>` configured, so status could not be read |
+
+A reservation is orthogonal to the status: a name held back by the registry
+carries `reasonCode` and `reason` whether or not it is registered.
+
+`grace` and `expired` are told apart by the registrar's own `available(id)`
+rule, `expires + GRACE_PERIOD < now`. `GRACE_PERIOD` is read from the contract
+rather than assumed, and `now` is the latest block's timestamp rather than the
+host clock, which the registrar compares against too, so a machine with a wrong
+clock cannot misreport a registration. That rule alone is not enough: it also
+holds for a name nobody ever registered (`0 + GRACE_PERIOD < now`), so a zero
+expiry is what separates *never registered* from *registered and since
+released*.
+
+A subname reports the status of the 2LD above it, which is only as good as the
+name it sits under. A subname nobody created answers 404 `unregistered`.
+
+#### v1 errors
+
+Every non-2xx body carries two fields: `error` is a fixed code to branch on,
+and `message` is a sentence for a human. Match on `error`, never on `message`,
+which is free to change.
+
+```jsonc
+{"name": "nope.testing", "error": "unregistered",
+ "message": "this name has never been registered",
+ "status": "unregistered", "expires": null, "graceEnds": null}
+```
+
+The codes are `tldNotConfigured`, `notFullyQualified`, `unregistered`,
+`expired`, `noSuchRoute` and `upstreamError`. When the registration is what went
+wrong, `error` and `status` hold the same value, so one field is enough to read.
+
+`upstreamError` says only which exception type the RPC call raised. The text
+goes to the resolver's log instead, because `SNRC_RPC` can carry a provider key
+and the exception can carry the URL it failed on. It is also the answer
+when a registrar, controller or oracle address has no contract behind it: the
+empty reply is refused rather than read as zero, which would make every name
+look free.
+
+#### v1 status codes
 
 | Status | Meaning |
 |---|---|
-| 200 | resolved |
+| 200 | resolved (`status` is `registered` or `grace`, or `unknown` when no registrar is configured) |
 | 400 | TLD not configured, or not a fully-qualified name |
-| 404 | name has no resolver set on the registry |
+| 404 | `unregistered` |
+| 410 | `expired`: lapsed and past grace, so anyone may take it |
 | 502 | upstream RPC error / reth not synced |
 
-### Configuring registries
+### Querying by labelhash
 
-Defaults to mainnet `.testing` (`0x03f438…`); `.simplex` is unset until
-deployed. Override per TLD via env on the `resolver` service in
-`docker-compose.yml` (`SNRC_REGISTRY_TESTING` / `SNRC_REGISTRY_SIMPLEX`), or as
-env vars for the standalone script.
+A client asking whether a name is free is usually about to register it, and
+whoever runs the resolver could register it first. To avoid that, send the
+keccak hash of the label in ENS's `[<64 hex>]` form instead of the label:
+
+```sh
+# instead of /resolve/acme.testing
+curl -s "http://127.0.0.1:8000/resolve/[$(printf acme | keccak-256sum | cut -d' ' -f1)].testing"
+```
+
+namehash is `keccak(parent || keccak(label))`, so this reaches the same node and
+returns the same record. The registrar keys `nameExpires` and `reservedNames` on
+the labelhash too, so the status fields do not need the label either. The
+resolver learns the name only by guessing the label and hashing it.
+
+Only the second-level label is a registry key, and `status` decodes a bracket
+there at any depth. The record does not: a bracket is decoded only in a
+two-label name, so `sub.[<hash>].testing` is not a supported query. Subname
+labels stay text; a bracket label left of the 2LD is an ordinary label. Routers
+from v22 send every 2LD this way, so a registrable name normally never reaches
+this service.
+
+On v2, read `type`: only `available` means the name is free. On v1, read
+`status`: a name is free on `unregistered` (404) and on `expired` (410), every
+other status means somebody holds it, and a `reasonCode` means the registry will
+refuse it whatever the status says.
+
+The hash must be keccak-256. `openssl dgst -sha3-256` and `sha3sum` compute
+SHA3-256, a different function that returns 64 valid-looking hex characters
+pointing at the wrong node.
+
+The resolver lowercases the query before matching, so uppercase hex works too.
+Clients that refuse raw brackets in a path can percent-encode them as `%5B` and
+`%5D`.
+
+Brackets cannot collide with a real name: they are invalid in a normalised ENS
+name, and a `[<64 hex>]` label is 66 bytes against the registrar's
+`maxLabelLength` of 63. A plain `0x…` label is not treated as a hash, since that
+is an ordinary, registrable name.
+
+Only 2LDs can be queried this way, as only a 2LD can be raced for: subnames are
+created by the 2LD's owner. A bracket label in a subname is hashed as written,
+so it points at a node nobody can own. ENS tooling accepts the bracketed form at
+any depth; this resolver does not, on purpose.
+
+This hides interest in a name and nothing else: the registration itself is
+public, and commit-reveal covers that step. A short or well-known label is easy
+to guess by hashing candidates, and the reveal publishes the labelhash, so an
+operator who logged the query can match it to the name afterwards.
+
+### What a name costs
+
+The controller's `prices()` names the price oracle, so no extra configuration is
+needed beyond `SNRC_CONTROLLER_<TLD>`. A `SimplexPriceOracle` exposes its curve
+through `prices()`, in US cents per year, which is the unit this API carries.
+
+An ENS-shaped oracle exposes only `price1Letter()`..`price6Letter()`, in attoUSD
+per second, and charges a premium on a lapsed name that it does not expose. A
+quote from one is therefore only safe for a name that was never registered: an
+`expired` name gets no price rather than one below what the registrar charges.
+
+**Set `SNRC_CONTROLLER_<TLD>` wherever `SNRC_REGISTRAR_<TLD>` is.** Without a
+controller there is no oracle, so no name can be priced.
+
+**Upgrade this service before the routers that query it.** Routers from SMP v22
+call `/v2/resolve`, which an older resolver does not serve. Every name then
+answers `ERR NAME RESOLVER "HTTP 404"` until this service is upgraded, while
+`/health` still reports it as ready.
+
+### Why a name is reserved
+
+v2 carries the controller's reason as `reservedReason`. v1 carries the same word
+as `reasonCode`, plus `reason`, an English sentence for a human reading this API.
+Clients should branch on the word and phrase it themselves, in the user's
+language.
+
+| `reasonCode` | Meaning |
+|---|---|
+| `internal` | reserved for SimpleX |
+| `trademark` | reserved to protect a trademark |
+| `community` | reserved for the community |
+| `unknown` | a reason added to the contract after this resolver; still reserved |
+
+These are `SimplexController.Reason`, where 0 means not reserved. A controller
+from before the enum stores a boolean, whose `true` decodes as 1, which is why
+1 reads as `internal`, so nothing needs migrating.
+
+### Configuring addresses
+
+The resolver reads three contracts, each configured per TLD.
+
+The **registry** answers who owns a node, and `/resolve` reads the records from
+it. The **registrar** (ERC-721) holds `nameExpires` and `GRACE_PERIOD`, which
+is where every expiry field comes from, and `labelOf`, which is how a hashed
+query is answered with a name. A name registered without recording its label
+cannot answer one, and `/v2/resolve` refuses it rather than answer with a name
+the client will reject. With no registrar for a TLD, `/resolve` still works and
+reports `"status": "unknown"`. The **controller** holds
+`reservedNames`, which is where `reasonCode` comes from. With no controller a
+held-back name reads as not reserved, and no name can be priced.
+
+All three default to the mainnet `.testing` deployment. `.simplex` is unset
+until it is deployed.
+
+The controller default is the **proxy**, not `SimplexControllerImpl`. Storage
+lives in the proxy, so the implementation address answers nothing. The two
+deployment files use different names for that proxy:
+`deployments.mainnet.testing.json` records it under the ENS role name
+`ETHRegistrarController`, and `verification.mainnet.testing.json` calls it
+`SimplexControllerProxy`. Both are the same address, and it is the one used
+here.
+
+To override any of them, set `SNRC_REGISTRY_<TLD>`, `SNRC_REGISTRAR_<TLD>` or
+`SNRC_CONTROLLER_<TLD>` on the `resolver` service in `docker-compose.yml`, or
+as env vars when you run the script directly.
+
+### Load and scaling
+
+A lookup reads the chain in at most three rounds: the name's status and
+registry entries, then its record from its resolver, then prices for a name
+that is free. The node runs the calls of a JSON-RPC batch one after another,
+so the contract reads of a round go to the chain as one `eth_call` to
+[Multicall3](https://github.com/mds1/multicall). A node that is slow for a
+moment therefore delays a lookup a few times, not once per read. Connections to
+the node are kept open and reused.
+
+The resolver answers HTTP/1.1, so each smp-server keeps its connections to it
+open instead of connecting for every lookup. An idle connection is closed after
+60 s; the smp-server drops its own idle ones after 30 s, so it closes them first.
+
+Each access log line ends with how long the request took, so slow requests
+show up in `docker compose logs resolver`.
+
+| Variable | Default | Meaning |
+|---|---|---|
+| `SNRC_WORKERS` | CPU count, at most 4 | processes sharing the port. If one exits, the others stop, so the container restarts |
+| `SNRC_RPC_TIMEOUT` | `5` | seconds to wait for each request to the node; the smp-server gives up after 3 |
+| `SNRC_MULTICALL` | `0xcA11bde05977b3631167028862bE2a173976CA11` | Multicall3 address. If it does not answer, each round is sent as a plain batch and the log says so once |
+
+### Logs
+
+Each event is one line: UTC time, level, event name, then `key=value` fields.
+
+```
+2026-09-26T08:06:19.992Z INFO  request  client=203.0.113.7 worker=12 method=GET path=/v2/resolve/[4fdd…].testing status=200 bytes=701 ms=35
+2026-09-26T08:06:20.051Z WARN  upstream_error  name=foobar.testing error=ConnectionRefusedError message="[Errno 111] Connection refused"
+```
+
+| Event | Level | Meaning |
+|---|---|---|
+| `listening` | info | started: address, workers, RPC endpoint, registries, trusted proxies |
+| `request` | info | one answered request; `/health` only at debug |
+| `upstream_error` | warn | a read from the node failed; the caller got 502 |
+| `multicall_unavailable` | warn | Multicall3 did not answer; rounds are sent as plain batches |
+| `client_gone` | warn | the caller hung up before the answer, usually an smp-server past its timeout |
+| `http_error` | warn | a malformed request; idle keep-alive timeouts only at debug |
+| `request_failed` | error | a request failed unexpectedly, with its traceback |
+| `worker_exited`, `worker_failed` | error | a worker stopped; the others stop too, so the container restarts |
+| `stopping` | info | stopped by a signal |
+
+| Variable | Default | Meaning |
+|---|---|---|
+| `SNRC_LOG_FORMAT` | `text` | `text` (`key=value`) or `json`, one object per line |
+| `SNRC_LOG_COLOR` | `auto` | `auto` colours only a terminal, `always`, `never`. Docker output is no terminal, so set `always` for coloured `docker compose logs` |
+| `SNRC_LOG_LEVEL` | `info` | `debug` also logs health checks and idle connections closing |
+| `SNRC_TRUSTED_PROXIES` | none | addresses or CIDRs of reverse proxies whose `X-Forwarded-For` names the client |
+
+With docker compose, set these, and those in "Load and scaling", in `.env`
+(`.env.example` lists them).
+
+Behind a reverse proxy such as Caddy on the host, the resolver sees the Docker
+network's gateway rather than the client. Trust that gateway, and the logged
+client is the last `X-Forwarded-For` address no trusted proxy added, so a caller
+through the proxy cannot claim another address. Everything else on the host that
+connects to `127.0.0.1:8000`, the smp-servers included, arrives through the same
+gateway and could set the logged address too; run the proxy in the compose network
+and trust only its address to avoid that.
+
+The gateway is
+`docker network inspect resolver_default --format '{{(index .IPAM.Config 0).Gateway}}'`,
+with the network named after the compose project. It changes when the network is
+recreated, as `docker compose down` does, so trust one of, in `.env`:
+
+- `SNRC_TRUSTED_PROXIES=172.16.0.0/12`: Docker's default pools for compose
+  networks, whatever subnet it picks. This also trusts every other container on the
+  host. Add the gateway too if Docker ever gives the network a `192.168.x.x` one.
+- a pinned subnet, so the gateway stays fixed, and only that gateway:
+
+  ```yaml
+  networks:
+    default:
+      ipam:
+        config:
+          - subnet: 172.30.0.0/24
+            gateway: 172.30.0.1
+  ```
+
+  in `docker-compose.yml`, with `SNRC_TRUSTED_PROXIES=172.30.0.1`, on a subnet no
+  other network on the host uses.

scripts/resolver/docker-compose.yml

@@ -38,14 +38,14 @@ services:
       - sh
       - -c
       - |
-        if [ -d /home/user/nimbus-eth2/build/data/${NETWORK}/db ]; then
+        if [ -d /home/user/nimbus-eth2/build/data/${NETWORK:-mainnet}/db ]; then
           echo "Nimbus data dir already initialised — skipping checkpoint sync";
           exit 0;
         fi;
         /home/user/nimbus-eth2/build/nimbus_beacon_node trustedNodeSync \
-          --network=${NETWORK} \
-          --data-dir=/home/user/nimbus-eth2/build/data/${NETWORK} \
-          --trusted-node-url=${TRUSTED_NODE_URL} \
+          --network=${NETWORK:-mainnet} \
+          --data-dir=/home/user/nimbus-eth2/build/data/${NETWORK:-mainnet} \
+          --trusted-node-url=${TRUSTED_NODE_URL:-https://mainnet-checkpoint-sync.attestant.io} \
           --backfill=false
     restart: "no"
 
@@ -69,8 +69,8 @@ services:
           echo "Reth data already initialised — skipping snapshot download";
           exit 0;
         fi;
-        echo "Downloading Reth ${NETWORK} --minimal snapshot...";
-        reth download --datadir /data --chain ${NETWORK} --minimal && \
+        echo "Downloading Reth ${NETWORK:-mainnet} --minimal snapshot...";
+        reth download --datadir /data --chain ${NETWORK:-mainnet} --minimal && \
         touch /data/.snapshot-done && \
         echo "Snapshot download complete"
     restart: "no"
@@ -92,7 +92,7 @@ services:
     command: >
       node
       --datadir /data
-      --chain ${NETWORK}
+      --chain ${NETWORK:-mainnet}
       --minimal
       --authrpc.jwtsecret /jwt/jwt.hex
       --authrpc.addr 0.0.0.0 --authrpc.port 8551
@@ -117,8 +117,8 @@ services:
       - "9000:9000/udp"
       - "127.0.0.1:5052:5052"
     command: >
-      --network=${NETWORK}
-      --data-dir=/home/user/nimbus-eth2/build/data/${NETWORK}
+      --network=${NETWORK:-mainnet}
+      --data-dir=/home/user/nimbus-eth2/build/data/${NETWORK:-mainnet}
       --el=http://reth:8551
       --jwt-secret=/jwt/jwt.hex
       --non-interactive
@@ -131,6 +131,7 @@ services:
   # smp-server points its [NAMES] resolver_endpoint at this URL.
   # To change the host port, edit the LEFT side of the port mapping below.
   resolver:
+    image: snrc-resolve:local
     build:
       context: ./service
       dockerfile: Dockerfile
@@ -150,8 +151,24 @@ services:
       # only if you're deploying against a different network or contract.
       # SNRC_REGISTRY_TESTING: 0x...
       # SNRC_REGISTRY_SIMPLEX: 0x...
+      # Registrar and controller, same cascade. Without the registrar `status`
+      # is "unknown"; without the controller a reserved name is "unregistered".
+      # SNRC_REGISTRAR_TESTING: 0x...
+      # SNRC_REGISTRAR_SIMPLEX: 0x...
+      # SNRC_CONTROLLER_TESTING: 0x...
+      # SNRC_CONTROLLER_SIMPLEX: 0x...
+      # Set in .env (see .env.example); unset or empty means the resolver's default.
+      SNRC_WORKERS: ${SNRC_WORKERS:-}
+      SNRC_RPC_TIMEOUT: ${SNRC_RPC_TIMEOUT:-}
+      SNRC_LOG_FORMAT: ${SNRC_LOG_FORMAT:-}
+      SNRC_LOG_COLOR: ${SNRC_LOG_COLOR:-}
+      SNRC_LOG_LEVEL: ${SNRC_LOG_LEVEL:-}
+      SNRC_TRUSTED_PROXIES: ${SNRC_TRUSTED_PROXIES:-}
     ports:
       - "127.0.0.1:8000:8000"
+    # a line per lookup; the local driver keeps 5 rotated files of 20 MB
+    logging:
+      driver: local
     restart: unless-stopped
 
 volumes:

scripts/resolver/service/.dockerignore

@@ -0,0 +1,5 @@
+.venv
+**/__pycache__
+.pytest_cache
+.ruff_cache
+test_snrc_resolve.py

scripts/resolver/service/.python-version

@@ -0,0 +1 @@
+3.13

scripts/resolver/service/Dockerfile

@@ -1,9 +1,10 @@
 # syntax=docker/dockerfile:1.7
 # ---------- builder ----------
-# Use the official uv image (Astral) on top of a slim Python base.
-# uv resolves and installs the lockfile-free pyproject.toml in seconds and
-# produces a portable .venv we can copy into the runtime stage.
-FROM ghcr.io/astral-sh/uv:python3.13-bookworm-slim AS builder
+# The official uv image (Astral) on top of a slim Python base installs the
+# locked dependencies into a portable .venv for the runtime stage.
+# Pinned to the uv that wrote uv.lock (its bookworm images stopped at 0.9.30), on the
+# runtime's Debian release, so the .venv's Python path exists there.
+FROM ghcr.io/astral-sh/uv:0.12.16-python3.13-trixie-slim AS builder
 
 ENV UV_LINK_MODE=copy \
     UV_COMPILE_BYTECODE=1 \
@@ -12,17 +13,17 @@ ENV UV_LINK_MODE=copy \
 
 WORKDIR /app
 
-# Install deps first (separate layer) — script edits won't bust this cache.
-COPY pyproject.toml ./
+# Dependencies first (separate layer), exactly as locked: script edits don't bust this cache.
+COPY pyproject.toml uv.lock .python-version ./
 RUN --mount=type=cache,target=/root/.cache/uv \
-    uv sync --no-dev --no-install-project
+    uv sync --frozen --no-dev --no-install-project
 
 # Script is added after the dep layer for cache friendliness.
 COPY snrc-resolve.py ./
 
 # ---------- runtime ----------
 # Slim runtime — only the venv + script. No uv, no apt.
-FROM python:3.13-slim AS runtime
+FROM python:3.13-slim-trixie AS runtime
 
 ENV PYTHONUNBUFFERED=1 \
     PYTHONDONTWRITEBYTECODE=1 \
@@ -33,15 +34,15 @@ RUN groupadd --system --gid 10001 snrc && \
     useradd  --system --uid 10001 --gid snrc --no-create-home --shell /usr/sbin/nologin snrc
 
 WORKDIR /app
-COPY --from=builder --chown=snrc:snrc /app /app
+# owned by root, so the service cannot rewrite its own code
+COPY --from=builder /app /app
 
 USER snrc:snrc
 
 EXPOSE 8000
 
-# Liveness check hits the script's own /health route. ThreadingHTTPServer is
-# fast enough that 3s is generous for a localhost probe; restart if it stops
-# responding entirely.
+# Liveness check hits the script's own /health route. Three failures mark the
+# container unhealthy; compose does not restart it for that, but `docker ps` shows it.
 HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
     CMD ["python", "-c", "import urllib.request, sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=3).status == 200 else 1)"]
 

scripts/resolver/service/pyproject.toml

@@ -9,5 +9,14 @@ dependencies = [
   "eth-hash[pycryptodome]>=0.7",
 ]
 
+[dependency-groups]
+dev = [
+  "pytest>=8",
+  "ruff>=0.6",
+]
+
 [tool.uv]
 package = false
+
+[tool.ruff.lint]
+select = ["E9", "F", "B"]

scripts/resolver/service/uv.lock

@@ -0,0 +1,156 @@
+version = 1
+revision = 3
+requires-python = ">=3.11"
+
+[[package]]
+name = "colorama"
+version = "0.4.6"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" },
+]
+
+[[package]]
+name = "eth-hash"
+version = "0.8.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/3c/f5/c67fc24f2f676aa9b7ab29679d44f113f314c817207cd4319353356f62da/eth_hash-0.8.0.tar.gz", hash = "sha256:b009752b620da2e9c7668014849d1f5fadbe4f138603f1871cc5d4ca706896b1", size = 12225, upload-time = "2026-03-25T16:36:55.099Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/87/87/b36792150ca0b28e4df683a34be15a61461ca0e349e5b5cf3ec8f694edb9/eth_hash-0.8.0-py3-none-any.whl", hash = "sha256:523718a51b369ab89866b929a5c93c52978cd866ea309192ad980dd8271f9fac", size = 7965, upload-time = "2026-03-25T16:36:54.205Z" },
+]
+
+[package.optional-dependencies]
+pycryptodome = [
+    { name = "pycryptodome" },
+]
+
+[[package]]
+name = "iniconfig"
+version = "2.3.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" },
+]
+
+[[package]]
+name = "packaging"
+version = "26.3"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" },
+]
+
+[[package]]
+name = "pluggy"
+version = "1.6.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" },
+]
+
+[[package]]
+name = "pycryptodome"
+version = "3.23.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/8e/a6/8452177684d5e906854776276ddd34eca30d1b1e15aa1ee9cefc289a33f5/pycryptodome-3.23.0.tar.gz", hash = "sha256:447700a657182d60338bab09fdb27518f8856aecd80ae4c6bdddb67ff5da44ef", size = 4921276, upload-time = "2025-05-17T17:21:45.242Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/04/5d/bdb09489b63cd34a976cc9e2a8d938114f7a53a74d3dd4f125ffa49dce82/pycryptodome-3.23.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:0011f7f00cdb74879142011f95133274741778abba114ceca229adbf8e62c3e4", size = 2495152, upload-time = "2025-05-17T17:20:20.833Z" },
+    { url = "https://files.pythonhosted.org/packages/a7/ce/7840250ed4cc0039c433cd41715536f926d6e86ce84e904068eb3244b6a6/pycryptodome-3.23.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:90460fc9e088ce095f9ee8356722d4f10f86e5be06e2354230a9880b9c549aae", size = 1639348, upload-time = "2025-05-17T17:20:23.171Z" },
+    { url = "https://files.pythonhosted.org/packages/ee/f0/991da24c55c1f688d6a3b5a11940567353f74590734ee4a64294834ae472/pycryptodome-3.23.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4764e64b269fc83b00f682c47443c2e6e85b18273712b98aa43bcb77f8570477", size = 2184033, upload-time = "2025-05-17T17:20:25.424Z" },
+    { url = "https://files.pythonhosted.org/packages/54/16/0e11882deddf00f68b68dd4e8e442ddc30641f31afeb2bc25588124ac8de/pycryptodome-3.23.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eb8f24adb74984aa0e5d07a2368ad95276cf38051fe2dc6605cbcf482e04f2a7", size = 2270142, upload-time = "2025-05-17T17:20:27.808Z" },
+    { url = "https://files.pythonhosted.org/packages/d5/fc/4347fea23a3f95ffb931f383ff28b3f7b1fe868739182cb76718c0da86a1/pycryptodome-3.23.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d97618c9c6684a97ef7637ba43bdf6663a2e2e77efe0f863cce97a76af396446", size = 2309384, upload-time = "2025-05-17T17:20:30.765Z" },
+    { url = "https://files.pythonhosted.org/packages/6e/d9/c5261780b69ce66d8cfab25d2797bd6e82ba0241804694cd48be41add5eb/pycryptodome-3.23.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9a53a4fe5cb075075d515797d6ce2f56772ea7e6a1e5e4b96cf78a14bac3d265", size = 2183237, upload-time = "2025-05-17T17:20:33.736Z" },
+    { url = "https://files.pythonhosted.org/packages/5a/6f/3af2ffedd5cfa08c631f89452c6648c4d779e7772dfc388c77c920ca6bbf/pycryptodome-3.23.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:763d1d74f56f031788e5d307029caef067febf890cd1f8bf61183ae142f1a77b", size = 2343898, upload-time = "2025-05-17T17:20:36.086Z" },
+    { url = "https://files.pythonhosted.org/packages/9a/dc/9060d807039ee5de6e2f260f72f3d70ac213993a804f5e67e0a73a56dd2f/pycryptodome-3.23.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:954af0e2bd7cea83ce72243b14e4fb518b18f0c1649b576d114973e2073b273d", size = 2269197, upload-time = "2025-05-17T17:20:38.414Z" },
+    { url = "https://files.pythonhosted.org/packages/f9/34/e6c8ca177cb29dcc4967fef73f5de445912f93bd0343c9c33c8e5bf8cde8/pycryptodome-3.23.0-cp313-cp313t-win32.whl", hash = "sha256:257bb3572c63ad8ba40b89f6fc9d63a2a628e9f9708d31ee26560925ebe0210a", size = 1768600, upload-time = "2025-05-17T17:20:40.688Z" },
+    { url = "https://files.pythonhosted.org/packages/e4/1d/89756b8d7ff623ad0160f4539da571d1f594d21ee6d68be130a6eccb39a4/pycryptodome-3.23.0-cp313-cp313t-win_amd64.whl", hash = "sha256:6501790c5b62a29fcb227bd6b62012181d886a767ce9ed03b303d1f22eb5c625", size = 1799740, upload-time = "2025-05-17T17:20:42.413Z" },
+    { url = "https://files.pythonhosted.org/packages/5d/61/35a64f0feaea9fd07f0d91209e7be91726eb48c0f1bfc6720647194071e4/pycryptodome-3.23.0-cp313-cp313t-win_arm64.whl", hash = "sha256:9a77627a330ab23ca43b48b130e202582e91cc69619947840ea4d2d1be21eb39", size = 1703685, upload-time = "2025-05-17T17:20:44.388Z" },
+    { url = "https://files.pythonhosted.org/packages/db/6c/a1f71542c969912bb0e106f64f60a56cc1f0fabecf9396f45accbe63fa68/pycryptodome-3.23.0-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:187058ab80b3281b1de11c2e6842a357a1f71b42cb1e15bce373f3d238135c27", size = 2495627, upload-time = "2025-05-17T17:20:47.139Z" },
+    { url = "https://files.pythonhosted.org/packages/6e/4e/a066527e079fc5002390c8acdd3aca431e6ea0a50ffd7201551175b47323/pycryptodome-3.23.0-cp37-abi3-macosx_10_9_x86_64.whl", hash = "sha256:cfb5cd445280c5b0a4e6187a7ce8de5a07b5f3f897f235caa11f1f435f182843", size = 1640362, upload-time = "2025-05-17T17:20:50.392Z" },
+    { url = "https://files.pythonhosted.org/packages/50/52/adaf4c8c100a8c49d2bd058e5b551f73dfd8cb89eb4911e25a0c469b6b4e/pycryptodome-3.23.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:67bd81fcbe34f43ad9422ee8fd4843c8e7198dd88dd3d40e6de42ee65fbe1490", size = 2182625, upload-time = "2025-05-17T17:20:52.866Z" },
+    { url = "https://files.pythonhosted.org/packages/5f/e9/a09476d436d0ff1402ac3867d933c61805ec2326c6ea557aeeac3825604e/pycryptodome-3.23.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c8987bd3307a39bc03df5c8e0e3d8be0c4c3518b7f044b0f4c15d1aa78f52575", size = 2268954, upload-time = "2025-05-17T17:20:55.027Z" },
+    { url = "https://files.pythonhosted.org/packages/f9/c5/ffe6474e0c551d54cab931918127c46d70cab8f114e0c2b5a3c071c2f484/pycryptodome-3.23.0-cp37-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aa0698f65e5b570426fc31b8162ed4603b0c2841cbb9088e2b01641e3065915b", size = 2308534, upload-time = "2025-05-17T17:20:57.279Z" },
+    { url = "https://files.pythonhosted.org/packages/18/28/e199677fc15ecf43010f2463fde4c1a53015d1fe95fb03bca2890836603a/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:53ecbafc2b55353edcebd64bf5da94a2a2cdf5090a6915bcca6eca6cc452585a", size = 2181853, upload-time = "2025-05-17T17:20:59.322Z" },
+    { url = "https://files.pythonhosted.org/packages/ce/ea/4fdb09f2165ce1365c9eaefef36625583371ee514db58dc9b65d3a255c4c/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_i686.whl", hash = "sha256:156df9667ad9f2ad26255926524e1c136d6664b741547deb0a86a9acf5ea631f", size = 2342465, upload-time = "2025-05-17T17:21:03.83Z" },
+    { url = "https://files.pythonhosted.org/packages/22/82/6edc3fc42fe9284aead511394bac167693fb2b0e0395b28b8bedaa07ef04/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:dea827b4d55ee390dc89b2afe5927d4308a8b538ae91d9c6f7a5090f397af1aa", size = 2267414, upload-time = "2025-05-17T17:21:06.72Z" },
+    { url = "https://files.pythonhosted.org/packages/59/fe/aae679b64363eb78326c7fdc9d06ec3de18bac68be4b612fc1fe8902693c/pycryptodome-3.23.0-cp37-abi3-win32.whl", hash = "sha256:507dbead45474b62b2bbe318eb1c4c8ee641077532067fec9c1aa82c31f84886", size = 1768484, upload-time = "2025-05-17T17:21:08.535Z" },
+    { url = "https://files.pythonhosted.org/packages/54/2f/e97a1b8294db0daaa87012c24a7bb714147c7ade7656973fd6c736b484ff/pycryptodome-3.23.0-cp37-abi3-win_amd64.whl", hash = "sha256:c75b52aacc6c0c260f204cbdd834f76edc9fb0d8e0da9fbf8352ef58202564e2", size = 1799636, upload-time = "2025-05-17T17:21:10.393Z" },
+    { url = "https://files.pythonhosted.org/packages/18/3d/f9441a0d798bf2b1e645adc3265e55706aead1255ccdad3856dbdcffec14/pycryptodome-3.23.0-cp37-abi3-win_arm64.whl", hash = "sha256:11eeeb6917903876f134b56ba11abe95c0b0fd5e3330def218083c7d98bbcb3c", size = 1703675, upload-time = "2025-05-17T17:21:13.146Z" },
+]
+
+[[package]]
+name = "pygments"
+version = "2.21.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz", hash = "sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", size = 5005329, upload-time = "2026-08-17T08:02:48.824Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" },
+]
+
+[[package]]
+name = "pytest"
+version = "9.1.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "colorama", marker = "sys_platform == 'win32'" },
+    { name = "iniconfig" },
+    { name = "packaging" },
+    { name = "pluggy" },
+    { name = "pygments" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" },
+]
+
+[[package]]
+name = "ruff"
+version = "0.16.9"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/96/bf/c935ca98e73fe8ce65b87ef08a280c0c1e85295d569228c15e87d8fdfaf1/ruff-0.16.9.tar.gz", hash = "sha256:12b625c6cfba78d285d9f48eda5f053374f1e53cb10ef17342a383750db99161", size = 4948764, upload-time = "2026-09-24T20:37:49.416Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/0d/26/df51322b52ee1ada7eff2d071ea09d11d5c2d1dcc9f02594f5785c4d1635/ruff-0.16.9-py3-none-linux_armv6l.whl", hash = "sha256:95e6f022090368ab3b824c36276839c53b2adf1a3f4c09fefc33dfc400f6da96", size = 10082922, upload-time = "2026-09-24T20:37:13.045Z" },
+    { url = "https://files.pythonhosted.org/packages/a5/27/7bf51f5a7aa375e9f339280a303aab44ca75dc1525f1cdc5991761685b0f/ruff-0.16.9-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:a5f27be168556594a86d2f415db0cf43f5291917849318f873c7e2791f7a8c67", size = 10236360, upload-time = "2026-09-24T20:37:16.049Z" },
+    { url = "https://files.pythonhosted.org/packages/b6/63/09659283f92f02dff45809da194a70da2f688d87c55d8875c4fae3536072/ruff-0.16.9-py3-none-macosx_11_0_arm64.whl", hash = "sha256:1632eb1d6197f33bd00b1acbc5b71009e89a8895c158e2d2b03a834fac964ab6", size = 9892940, upload-time = "2026-09-24T20:37:17.957Z" },
+    { url = "https://files.pythonhosted.org/packages/24/58/98de1b72ec172f5f8f1731236fe21585b3998bf7dfe9fcc44ae9ba626012/ruff-0.16.9-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b3f951b14d865d5952c89d40a5ca07e87abe24fa5453299878411e127748fb1c", size = 10032114, upload-time = "2026-09-24T20:37:19.942Z" },
+    { url = "https://files.pythonhosted.org/packages/c8/7d/f1e17c54ab59d4bad1dce8ee3e22a7a1d0ef4745240decacdcf3832b5bb2/ruff-0.16.9-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:447fc07e1573afff7cb02803462b12b6c8ece7cf10e2cd78565fa6d7a1c0bf8d", size = 9910227, upload-time = "2026-09-24T20:37:21.872Z" },
+    { url = "https://files.pythonhosted.org/packages/34/19/436f647a65075bbd3bab2668b3bdaa5120559b294694018cdcefabbbf30b/ruff-0.16.9-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8a3e039a6a40ed976c491722b60e0ae4a4aa1a86057f540ee7a37a5d19ae9120", size = 10547484, upload-time = "2026-09-24T20:37:24.229Z" },
+    { url = "https://files.pythonhosted.org/packages/03/59/38430a6bc2f6d8095447ac39625cf8b6e9344a47e6c26225c8ba1bff3ffb/ruff-0.16.9-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4684dded7db60aa57cb118fa158630f5feade4af5782903b6053484bdf9bd129", size = 11412367, upload-time = "2026-09-24T20:37:26.307Z" },
+    { url = "https://files.pythonhosted.org/packages/c8/bd/bbb6d7fc7f208c8b8c50dd5dc8206e4cfdb1e7a8fb852606a3adf370c880/ruff-0.16.9-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d29c934357e45642fda2f34c0b1f4025b4a6c01e15e4bf0016879d60078a142c", size = 10869787, upload-time = "2026-09-24T20:37:28.35Z" },
+    { url = "https://files.pythonhosted.org/packages/bc/b8/9c543074918061abbefc3bd139bee22de35abedb00dde0f2d27288838962/ruff-0.16.9-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a21713e629d3e5bdb2f5c2def1cc7f04f47fa8e1a7eb0571b4a28e1da64bc728", size = 10406494, upload-time = "2026-09-24T20:37:30.624Z" },
+    { url = "https://files.pythonhosted.org/packages/35/7a/5a8851bd146e7ccf8fd4b003f6c75c11f8fbdb0e60673b45097986b6bf41/ruff-0.16.9-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:7baa24ef5fc8e77aa93879e1d3f43754a01ae488e869f1ae30cf431afd4d2452", size = 10590083, upload-time = "2026-09-24T20:37:32.439Z" },
+    { url = "https://files.pythonhosted.org/packages/87/f0/4c3467188f23f806960b46fa76575a7cd0514c9ba90562650b71efc96980/ruff-0.16.9-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:a41aac6230aadfaa133bdfa1614488531ffa3e0837567ae04c0da2058a9c0f9e", size = 10119151, upload-time = "2026-09-24T20:37:34.581Z" },
+    { url = "https://files.pythonhosted.org/packages/15/34/5a4def5adea572ce6aea0bb64f21f928ee317b80e0db747d7979b01d7261/ruff-0.16.9-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:c2529fb5896d49115b0e9aa8f887490b34bbe76baf879ec2264ac59406869ce7", size = 9911544, upload-time = "2026-09-24T20:37:36.796Z" },
+    { url = "https://files.pythonhosted.org/packages/62/5d/d15ebea7499eef9373318c0ee6ca127832927c6529731f6d48e18dce7ca9/ruff-0.16.9-py3-none-musllinux_1_2_i686.whl", hash = "sha256:41e3870277694177429b56406d65dfbdb2c2802c52b715edaf6a0b829c69d4ee", size = 10269884, upload-time = "2026-09-24T20:37:38.857Z" },
+    { url = "https://files.pythonhosted.org/packages/d1/56/c5d3cd119ded7a3c7aba0e961b69cb3df701c91662c98ad694467d060ce1/ruff-0.16.9-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:8adbe4e58af167f767d7b2ba5e83c42e878350796cf78c2f5e14ab9903a92588", size = 10749366, upload-time = "2026-09-24T20:37:41.042Z" },
+    { url = "https://files.pythonhosted.org/packages/ac/fe/734ec7527029ac757ecf821f143c9f3fcf69149c21f53a044a899b430f5a/ruff-0.16.9-py3-none-win32.whl", hash = "sha256:0e1dbc2073624dee6618d41d0098690a7244654af746704b64759e12b6b6b385", size = 10152355, upload-time = "2026-09-24T20:37:43.025Z" },
+    { url = "https://files.pythonhosted.org/packages/14/21/26e4643629b3ebb44f0a06f9c9a53058d63d989415f63a9a3c28e2ee7f22/ruff-0.16.9-py3-none-win_amd64.whl", hash = "sha256:6bd40fec8cd4c8a3d4dd589bd8ad4e6320c13c29234159bfd959a40d529d597b", size = 10592965, upload-time = "2026-09-24T20:37:44.944Z" },
+    { url = "https://files.pythonhosted.org/packages/51/60/5fb1a39dbb5ae314d5f59bc7348a63c1d5c20f3cd83914c4b5cb0be31d2d/ruff-0.16.9-py3-none-win_arm64.whl", hash = "sha256:ed1a252039200f57a59eebc063b54beabea67bfbaaca0eeaa7f54b5fbcda2284", size = 10458649, upload-time = "2026-09-24T20:37:46.882Z" },
+]
+
+[[package]]
+name = "snrc-resolve"
+version = "0.1.0"
+source = { virtual = "." }
+dependencies = [
+    { name = "eth-hash", extra = ["pycryptodome"] },
+]
+
+[package.dev-dependencies]
+dev = [
+    { name = "pytest" },
+    { name = "ruff" },
+]
+
+[package.metadata]
+requires-dist = [{ name = "eth-hash", extras = ["pycryptodome"], specifier = ">=0.7" }]
+
+[package.metadata.requires-dev]
+dev = [
+    { name = "pytest", specifier = ">=8" },
+    { name = "ruff", specifier = ">=0.6" },
+]

simplexmq.cabal

@@ -1,7 +1,7 @@
 cabal-version: 3.0
 
 name:           simplexmq
-version:        7.1.0.4
+version:        7.1.0.10
 synopsis:       SimpleXMQ message broker
 description:    This package includes <./docs/Simplex-Messaging-Server.html server>,
                 <./docs/Simplex-Messaging-Client.html client> and
@@ -31,6 +31,11 @@ extra-source-files:
     cbits/blst/**/*.asm
     cbits/libbbs/**/*.c
     cbits/libbbs/**/*.h
+    cbits/libsecp256k1/**/*.c
+    cbits/libsecp256k1/**/*.h
+    src/Simplex/Messaging/Crypto/BIP39/english.txt
+    tests/fixtures/bip39-vectors.json
+    tests/fixtures/bip-0032.mediawiki
     apps/common/Web/static/index.html
     apps/common/Web/static/link.html
     apps/common/Web/static/media/apk_icon.png
@@ -135,6 +140,12 @@ library
       Simplex.Messaging.Crypto.Lazy
       Simplex.Messaging.Crypto.Ratchet
       Simplex.Messaging.Crypto.BBS
+      Simplex.Messaging.Crypto.BIP32
+      Simplex.Messaging.Crypto.BIP39
+      Simplex.Messaging.Crypto.BIP39.English
+      Simplex.Messaging.Crypto.BIP44
+      Simplex.Messaging.Crypto.Entitlement
+      Simplex.Messaging.Crypto.Secp256k1
       Simplex.Messaging.Crypto.SNTRUP761
       Simplex.Messaging.Crypto.SNTRUP761.Bindings
       Simplex.Messaging.Crypto.SNTRUP761.Bindings.Defines
@@ -143,6 +154,9 @@ library
       Simplex.Messaging.Crypto.ShortLink
       Simplex.Messaging.Encoding
       Simplex.Messaging.Encoding.String
+      Simplex.Messaging.Eth.Address
+      Simplex.Messaging.Eth.RLP
+      Simplex.Messaging.Eth.Transaction
       Simplex.Messaging.Names.Record
       Simplex.Messaging.Notifications.Client
       Simplex.Messaging.Notifications.Protocol
@@ -192,6 +206,10 @@ library
           Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260410_receive_attempts
           Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260411_service_certs
           Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260712_address_dr_rpc
+          Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260823_snd_files_entitlement
+          Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260919_ratchet_verify_codes
+          Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260929_ratchet_indexes
+          Simplex.Messaging.Agent.Store.Postgres.Migrations.M20261008_rcv_switch_status
   else
       exposed-modules:
           Simplex.Messaging.Agent.Store.SQLite
@@ -245,6 +263,11 @@ library
           Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260410_receive_attempts
           Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260411_service_certs
           Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260712_address_dr_rpc
+          Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260823_snd_files_entitlement
+          Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260919_ratchet_verify_codes
+          Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260929_ratchet_indexes
+          Simplex.Messaging.Agent.Store.SQLite.Migrations.M20261005_commands_autoincrement
+          Simplex.Messaging.Agent.Store.SQLite.Migrations.M20261008_rcv_switch_status
           Simplex.Messaging.Agent.Store.SQLite.Util
   if flag(client_postgres) || flag(server_postgres)
       exposed-modules:
@@ -323,7 +346,8 @@ library
       cbits/blst/src
       cbits/libbbs/include
       cbits/libbbs/src
-  cc-options: -D__BLST_PORTABLE__
+  -- libsecp256k1's only -D flag enables its recovery module: every other setting has an #ifndef default that matches the checked-in tables
+  cc-options: -D__BLST_PORTABLE__ -DENABLE_MODULE_RECOVERY=1
   if flag(commoncrypto)
     cc-options: -DBBS_CRYPTO_CC
     frameworks: Security
@@ -337,6 +361,9 @@ library
       cbits/libbbs/src/compat-string.c
       cbits/libbbs/src/sha256.c
       cbits/libbbs/src/shake256.c
+      cbits/libsecp256k1/src/secp256k1.c
+      cbits/libsecp256k1/src/precomputed_ecmult.c
+      cbits/libsecp256k1/src/precomputed_ecmult_gen.c
   asm-sources:
       cbits/blst/build/assembly.S
   extra-libraries:
@@ -364,6 +391,7 @@ library
     , cryptostore ==0.3.*
     , data-default ==0.7.*
     , directory ==1.3.*
+    , file-embed >=0.0.10 && <0.1
     , filepath ==1.4.*
     , hourglass ==0.2.*
     , http-types ==0.12.*
@@ -387,6 +415,7 @@ library
     , transformers ==0.6.*
     , unliftio ==0.2.*
     , unliftio-core ==0.2.*
+    , wide-word ==0.1.6.*
     , yaml ==0.11.*
     , zstd ==0.1.3.*
   default-language: Haskell2010
@@ -541,6 +570,7 @@ test-suite simplexmq-test
       CoreTests.CryptoFileTests
       CoreTests.CryptoTests
       CoreTests.EncodingTests
+      CoreTests.EthCryptoTests
       CoreTests.MsgStoreTests
       CoreTests.RetryIntervalTests
       CoreTests.SOCKSSettings
@@ -630,7 +660,6 @@ test-suite simplexmq-test
     , stm
     , text
     , time
-    , timeit ==2.0.*
     , tls >=1.9.0 && <1.10
     , transformers
     , unliftio

src/Simplex/FileTransfer/Agent.hs

@@ -17,11 +17,16 @@ module Simplex.FileTransfer.Agent
     toFSFilePath,
     -- Receiving files
     xftpReceiveFile',
+    xftpPrepareReceiveFile',
+    xftpStartReceiveFile',
     xftpDeleteRcvFile',
     xftpDeleteRcvFiles',
     -- Sending files
     xftpSendFile',
     xftpSendDescription',
+    xftpPrepareSendFile',
+    xftpPrepareSendDescription',
+    xftpStartSendFile',
     deleteSndFileInternal,
     deleteSndFilesInternal,
     deleteSndFileRemote,
@@ -50,11 +55,12 @@ import qualified Data.Set as S
 import Data.Text (Text, pack)
 import Data.Time.Clock (getCurrentTime)
 import Data.Time.Format (defaultTimeLocale, formatTime)
+import Data.Word (Word32)
 import Simplex.FileTransfer.Chunks (toKB)
 import Simplex.FileTransfer.Client (XFTPChunkSpec (..), getChunkDigest, prepareChunkSizes, prepareChunkSpecs, singleChunkSize)
 import Simplex.FileTransfer.Crypto
 import Simplex.FileTransfer.Description
-import Simplex.FileTransfer.Protocol (FileParty (..), SFileParty (..))
+import Simplex.FileTransfer.Protocol (FileParty (..), GrantedStorageTime, SFileParty (..))
 import Simplex.FileTransfer.Transport (XFTPRcvChunkSpec (..))
 import qualified Simplex.FileTransfer.Transport as XFTP
 import Simplex.FileTransfer.Types
@@ -126,7 +132,12 @@ closeXFTPAgent a = do
     stopWorkers workers = atomically (swapTVar workers M.empty) >>= mapM_ (liftIO . cancelWorker)
 
 xftpReceiveFile' :: AgentClient -> UserId -> ValidFileDescription 'FRecipient -> Maybe CryptoFileArgs -> Bool -> AM RcvFileId
-xftpReceiveFile' c userId (ValidFileDescription fd@FileDescription {chunks, redirect}) cfArgs approvedRelays = do
+xftpReceiveFile' c userId vfd cfArgs approvedRelays = do
+  fId <- xftpPrepareReceiveFile' c userId vfd cfArgs approvedRelays
+  fId <$ xftpStartReceiveFile' c fId
+
+xftpPrepareReceiveFile' :: AgentClient -> UserId -> ValidFileDescription 'FRecipient -> Maybe CryptoFileArgs -> Bool -> AM RcvFileId
+xftpPrepareReceiveFile' c userId (ValidFileDescription fd@FileDescription {redirect}) cfArgs approvedRelays = do
   g <- asks random
   prefixPath <- lift $ getPrefixPath "rcv.xftp"
   createDirectory prefixPath
@@ -136,7 +147,7 @@ xftpReceiveFile' c userId (ValidFileDescription fd@FileDescription {chunks, redi
   lift $ createDirectory =<< toFSFilePath relTmpPath
   lift $ createEmptyFile =<< toFSFilePath relSavePath
   let saveFile = CryptoFile relSavePath cfArgs
-  fId <- case redirect of
+  case redirect of
     Nothing -> withStore c $ \db -> createRcvFile db g userId fd relPrefixPath relTmpPath saveFile approvedRelays
     Just _ -> do
       -- prepare description paths
@@ -148,8 +159,11 @@ xftpReceiveFile' c userId (ValidFileDescription fd@FileDescription {chunks, redi
       let saveFileRedirect = CryptoFile relSavePathRedirect $ Just cfArgsRedirect
       -- create download tasks
       withStore c $ \db -> createRcvFileRedirect db g userId fd relPrefixPath relTmpPathRedirect saveFileRedirect relTmpPath saveFile approvedRelays
-  forM_ chunks (downloadChunk c)
-  pure fId
+
+xftpStartReceiveFile' :: AgentClient -> RcvFileId -> AM ()
+xftpStartReceiveFile' c rcvFileEntityId = do
+  srvs <- withStore c (`startPreparedRcvFile` rcvFileEntityId)
+  lift $ forM_ srvs $ void . getXFTPRcvWorker True c . Just
 
 downloadChunk :: AgentClient -> FileChunk -> AM ()
 downloadChunk c FileChunk {replicas = (FileChunkReplica {server} : _)} = do
@@ -350,8 +364,13 @@ xftpDeleteRcvFiles' c rcvFileEntityIds = do
 notify :: forall m e. (MonadIO m, AEntityI e) => AgentClient -> AEntityId -> AEvent e -> m ()
 notify c entId cmd = atomically $ writeTBQueue (subQ c) ("", entId, AEvt (sAEntity @e) cmd)
 
-xftpSendFile' :: AgentClient -> UserId -> CryptoFile -> Int -> AM SndFileId
-xftpSendFile' c userId file numRecipients = do
+xftpSendFile' :: AgentClient -> UserId -> CryptoFile -> Int -> Maybe Word32 -> AM SndFileId
+xftpSendFile' c userId file numRecipients storageHours = do
+  fId <- xftpPrepareSendFile' c userId file numRecipients storageHours
+  fId <$ xftpStartSendFile' c fId
+
+xftpPrepareSendFile' :: AgentClient -> UserId -> CryptoFile -> Int -> Maybe Word32 -> AM SndFileId
+xftpPrepareSendFile' c userId file numRecipients storageHours = do
   g <- asks random
   prefixPath <- lift $ getPrefixPath "snd.xftp"
   createDirectory prefixPath
@@ -359,12 +378,15 @@ xftpSendFile' c userId file numRecipients = do
   key <- atomically $ C.randomSbKey g
   nonce <- atomically $ C.randomCbNonce g
   -- saving absolute filePath will not allow to restore file encryption after app update, but it's a short window
-  fId <- withStore c $ \db -> createSndFile db g userId file numRecipients relPrefixPath key nonce Nothing
-  lift . void $ getXFTPSndWorker True c Nothing
-  pure fId
+  withStore c $ \db -> createSndFile db g userId file numRecipients relPrefixPath key nonce Nothing storageHours
 
 xftpSendDescription' :: AgentClient -> UserId -> ValidFileDescription 'FRecipient -> Int -> AM SndFileId
-xftpSendDescription' c userId (ValidFileDescription fdDirect@FileDescription {size, digest}) numRecipients = do
+xftpSendDescription' c userId vfd numRecipients = do
+  fId <- xftpPrepareSendDescription' c userId vfd numRecipients
+  fId <$ xftpStartSendFile' c fId
+
+xftpPrepareSendDescription' :: AgentClient -> UserId -> ValidFileDescription 'FRecipient -> Int -> AM SndFileId
+xftpPrepareSendDescription' c userId (ValidFileDescription fdDirect@FileDescription {size, digest}) numRecipients = do
   g <- asks random
   prefixPath <- lift $ getPrefixPath "snd.xftp"
   createDirectory prefixPath
@@ -375,9 +397,12 @@ xftpSendDescription' c userId (ValidFileDescription fdDirect@FileDescription {si
   liftError (FILE . FILE_IO . show) $ CF.writeFile file (LB.fromStrict $ strEncode fdDirect)
   key <- atomically $ C.randomSbKey g
   nonce <- atomically $ C.randomCbNonce g
-  fId <- withStore c $ \db -> createSndFile db g userId file numRecipients relPrefixPath key nonce $ Just RedirectFileInfo {size, digest}
+  withStore c $ \db -> createSndFile db g userId file numRecipients relPrefixPath key nonce (Just RedirectFileInfo {size, digest}) Nothing
+
+xftpStartSendFile' :: AgentClient -> SndFileId -> AM ()
+xftpStartSendFile' c sndFileEntityId = do
+  withStore c (`startPreparedSndFile` sndFileEntityId)
   lift . void $ getXFTPSndWorker True c Nothing
-  pure fId
 
 resumeXFTPSndWork :: AgentClient -> Maybe XFTPServer -> AM' ()
 resumeXFTPSndWork = void .: getXFTPSndWorker False
@@ -405,7 +430,7 @@ runXFTPSndPrepareWorker c Worker {doWork} = do
     prepareFile _ SndFile {prefixPath = Nothing} =
       throwE $ INTERNAL "no prefix path"
     prepareFile cfg sndFile@SndFile {sndFileId, sndFileEntityId, userId, prefixPath = Just ppath, status} = do
-      SndFile {numRecipients, chunks} <-
+      SndFile {numRecipients, chunks, storageHours} <-
         if status /= SFSEncrypted -- status is SFSNew or SFSEncrypting
           then do
             fsEncPath <- lift . toFSFilePath $ sndFileEncPath ppath
@@ -424,7 +449,7 @@ runXFTPSndPrepareWorker c Worker {doWork} = do
       let (pendingChunks, preparedSrvs) = partitionEithers $ map srvOrPendingChunk chunks
       -- concurrently?
       -- separate worker to create chunks? record retries and delay on snd_file_chunks?
-      srvs <- forM pendingChunks $ createChunk numRecipients'
+      srvs <- forM pendingChunks $ createChunk numRecipients' storageHours
       let allSrvs = S.fromList $ preparedSrvs <> srvs
       lift $ forM_ allSrvs $ \srv -> getXFTPSndWorker True c (Just srv)
       withStore' c $ \db -> updateSndFileStatus db sndFileId SFSUploading
@@ -454,8 +479,8 @@ runXFTPSndPrepareWorker c Worker {doWork} = do
         srvOrPendingChunk ch@SndFileChunk {replicas} = case replicas of
           [] -> Left ch
           SndFileChunkReplica {server} : _ -> Right server
-        createChunk :: Int -> SndFileChunk -> AM (ProtocolServer 'PXFTP)
-        createChunk numRecipients' ch = do
+        createChunk :: Int -> Maybe Word32 -> SndFileChunk -> AM (ProtocolServer 'PXFTP)
+        createChunk numRecipients' storageHours ch = do
           liftIO $ assertAgentForeground c
           (replica, ProtoServerWithAuth srv _) <- tryCreate
           withStore' c $ \db -> createSndFileReplica db ch replica
@@ -482,7 +507,7 @@ runXFTPSndPrepareWorker c Worker {doWork} = do
               deleted <- withStore' c $ \db -> getSndFileDeleted db sndFileId
               when deleted $ throwE $ FILE NO_FILE
               withNextSrv c userId storageSrvs triedHosts [] $ \srvAuth -> do
-                replica <- agentXFTPNewChunk c ch numRecipients' srvAuth
+                replica <- agentXFTPNewChunk c ch numRecipients' srvAuth storageHours
                 pure (replica, srvAuth)
 
 sndWorkerInternalError :: AgentClient -> DBSndFileId -> SndFileId -> Maybe FilePath -> AgentErrorType -> AM ()
@@ -543,9 +568,10 @@ runXFTPSndWorker c srv Worker {doWork} = do
       notify c sndFileEntityId $ SFPROG uploaded total
       when complete $ do
         (sndDescr, rcvDescrs) <- sndFileToDescrs sf
-        notify c sndFileEntityId $ SFDONE sndDescr rcvDescrs
-        lift . forM_ prefixPath $ removePath <=< toFSFilePath
-        withStore' c $ \db -> updateSndFileComplete db sndFileId
+        flip catchAllErrors (\e -> logError $ "XFTP snd worker error: " <> tshow e) $ do
+          lift . forM_ prefixPath $ removePath <=< toFSFilePath
+          withStore' c $ \db -> updateSndFileComplete db sndFileId
+        notify c sndFileEntityId $ SFDONE sndDescr rcvDescrs (sndFileExpiresAt chunks)
       where
         addRecipients :: SndFileChunk -> SndFileChunkReplica -> AM SndFileChunkReplica
         addRecipients ch@SndFileChunk {numRecipients} cr@SndFileChunkReplica {sndChunkReplicaId, rcvIdsKeys}
@@ -577,6 +603,10 @@ runXFTPSndWorker c srv Worker {doWork} = do
           let chunkSize = FileSize $ sndChunkSize ch
               replicas = [FileChunkReplica {server, replicaId, replicaKey}]
           pure FileChunk {chunkNo, digest = chDigest, chunkSize, replicas}
+        sndFileExpiresAt :: [SndFileChunk] -> Maybe GrantedStorageTime
+        sndFileExpiresAt chunks' = fmap minimum $ L.nonEmpty =<< mapM chunkExpiresAt chunks'
+          where
+            chunkExpiresAt SndFileChunk {replicas} = maximum <$> L.nonEmpty (mapMaybe (\SndFileChunkReplica {expiresAt} -> expiresAt) replicas)
         createRcvFileDescriptions :: FileDescription 'FRecipient -> [SndFileChunk] -> [FileDescription 'FRecipient]
         createRcvFileDescriptions fd sndChunks = map (\chunks -> (fd :: (FileDescription 'FRecipient)) {chunks}) rcvChunks
           where

src/Simplex/FileTransfer/Client.hs

@@ -36,8 +36,10 @@ import qualified Control.Exception as E
 import Control.Logger.Simple
 import Control.Monad
 import Control.Monad.Except
+import Control.Monad.IO.Class (liftIO)
 import Control.Monad.Trans.Except
 import Crypto.Random (ChaChaDRG)
+import qualified Data.Aeson as J
 import Data.Bifunctor (first)
 import Data.ByteString.Builder (Builder, byteString)
 import Data.ByteString.Char8 (ByteString)
@@ -57,6 +59,7 @@ import Network.Socket (HostName)
 import Simplex.FileTransfer.Chunks
 import Simplex.FileTransfer.Protocol
 import Simplex.FileTransfer.Transport
+import Simplex.Messaging.Crypto.Entitlement (EntitlementProof)
 import Simplex.Messaging.Client
   ( NetworkConfig (..),
     NetworkRequestMode (..),
@@ -84,7 +87,7 @@ import Simplex.Messaging.Protocol
     SenderId,
     pattern NoEntity,
   )
-import Simplex.Messaging.Transport (ALPN, CertChainPubKey (..), HandshakeError (..), THandleAuth (..), THandleParams (..), TransportError (..), TransportPeer (..), defaultSupportedParams)
+import Simplex.Messaging.Transport (ALPN, CertChainPubKey (..), HandshakeError (..), SessionId, THandleAuth (..), THandleParams (..), TransportError (..), TransportPeer (..), defaultSupportedParams)
 import Simplex.Messaging.Transport.Client (TransportClientConfig (..), TransportHost)
 import Simplex.Messaging.Transport.HTTP2
 import Simplex.Messaging.Transport.HTTP2.Client
@@ -126,8 +129,8 @@ defaultXFTPClientConfig =
       clientALPN = Just alpnSupportedXFTPhandshakes
     }
 
-getXFTPClient :: TransportSession FileResponse -> XFTPClientConfig -> [HostName] -> UTCTime -> (XFTPClient -> IO ()) -> IO (Either XFTPClientError XFTPClient)
-getXFTPClient transportSession@(_, srv, _) config@XFTPClientConfig {clientALPN, xftpNetworkConfig, serverVRange} presetDomains proxySessTs disconnected = runExceptT $ do
+getXFTPClient :: TransportSession FileResponse -> XFTPClientConfig -> [HostName] -> UTCTime -> (SessionId -> IO (Maybe EntitlementProof)) -> (XFTPClient -> IO ()) -> IO (Either XFTPClientError XFTPClient)
+getXFTPClient transportSession@(_, srv, _) config@XFTPClientConfig {clientALPN, xftpNetworkConfig, serverVRange} presetDomains proxySessTs mkEntitlementProof disconnected = runExceptT $ do
   let socksCreds = clientSocksCredentials xftpNetworkConfig proxySessTs transportSession
       ProtocolServer _ host port keyHash = srv
       useALPN = if useWebPort xftpNetworkConfig presetDomains srv then Just [httpALPN11] else clientALPN
@@ -146,21 +149,23 @@ getXFTPClient transportSession@(_, srv, _) config@XFTPClientConfig {clientALPN,
   thParams@THandleParams {thVersion} <- case sessionALPN of
     Just alpn
       | alpn == xftpALPNv1 || alpn == httpALPN11 ->
-          xftpClientHandshakeV1 serverVRange keyHash http2Client thParams0
+          xftpClientHandshakeV1 serverVRange keyHash http2Client mkEntitlementProof thParams0
     _ -> pure thParams0
   logDebug $ "Client negotiated protocol: " <> tshow thVersion
   let c = XFTPClient {http2Client, thParams, transportSession, config}
   atomically $ writeTVar clientVar $ Just c
   pure c
 
-xftpClientHandshakeV1 :: VersionRangeXFTP -> C.KeyHash -> HTTP2Client -> THandleParamsXFTP 'TClient -> ExceptT XFTPClientError IO (THandleParamsXFTP 'TClient)
-xftpClientHandshakeV1 serverVRange keyHash@(C.KeyHash kh) c@HTTP2Client {sessionId, serverKey} thParams0 = do
-  shs@XFTPServerHandshake {authPubKey = ck} <- getServerHandshake
+xftpClientHandshakeV1 :: VersionRangeXFTP -> C.KeyHash -> HTTP2Client -> (SessionId -> IO (Maybe EntitlementProof)) -> THandleParamsXFTP 'TClient -> ExceptT XFTPClientError IO (THandleParamsXFTP 'TClient)
+xftpClientHandshakeV1 serverVRange keyHash@(C.KeyHash kh) c@HTTP2Client {sessionId, serverKey} mkEntitlementProof thParams0 = do
+  shs@XFTPServerHandshake {authPubKey = ck, serverInfoBytes} <- getServerHandshake
   (vr, sk) <- processServerHandshake shs
   let v = maxVersion vr
-  sendClientHandshake XFTPClientHandshake {xftpVersion = v, keyHash}
+  ep <- if v >= fileStorageTimeXFTPVersion then liftIO (mkEntitlementProof sessionId) else pure Nothing
+  sendClientHandshake XFTPClientHandshake {xftpVersion = v, keyHash, entitlementProof = ep}
   let thAuth = Just THAuthClient {peerServerPubKey = sk, peerServerCertKey = ck, clientService = Nothing, sessSecret = Nothing}
-  pure thParams0 {thAuth, thVersion = v, thServerVRange = vr}
+      serverInfo = J.eitherDecodeStrict' <$> serverInfoBytes
+  pure thParams0 {thAuth, thVersion = v, thServerVRange = vr, serverInfo}
   where
     getServerHandshake :: ExceptT XFTPClientError IO XFTPServerHandshake
     getServerHandshake = do
@@ -253,10 +258,11 @@ createXFTPChunk ::
   FileInfo ->
   NonEmpty C.APublicAuthKey ->
   Maybe BasicAuth ->
-  ExceptT XFTPClientError IO (SenderId, NonEmpty RecipientId)
-createXFTPChunk c spKey file rcps auth_ =
-  sendXFTPCommand c spKey NoEntity (FNEW file rcps auth_) Nothing >>= \case
-    (FRSndIds sId rIds, body) -> noFile body (sId, rIds)
+  Maybe Word32 ->
+  ExceptT XFTPClientError IO (SenderId, NonEmpty RecipientId, Maybe GrantedStorageTime)
+createXFTPChunk c spKey file rcps auth_ storageHours =
+  sendXFTPCommand c spKey NoEntity (FNEW file rcps auth_ storageHours) Nothing >>= \case
+    (FRSndIds sId rIds gs, body) -> noFile body (sId, rIds, gs)
     (r, _) -> throwE $ unexpectedResponse r
 
 addXFTPRecipients :: XFTPClient -> C.APrivateAuthKey -> XFTPFileId -> NonEmpty C.APublicAuthKey -> ExceptT XFTPClientError IO (NonEmpty RecipientId)

src/Simplex/FileTransfer/Client/Agent.hs

@@ -81,7 +81,7 @@ getXFTPServerClient XFTPClientAgent {xftpClients, startedAt, config} srv = do
     connectClient =
       ExceptT $
         first (XFTPClientAgentError srv)
-          <$> getXFTPClient (1, srv, Nothing) (xftpConfig config) [] startedAt clientDisconnected
+          <$> getXFTPClient (1, srv, Nothing) (xftpConfig config) [] startedAt (\_ -> pure Nothing) clientDisconnected
 
     clientDisconnected :: XFTPClient -> IO ()
     clientDisconnected _ = do

src/Simplex/FileTransfer/Client/Main.hs

@@ -328,7 +328,7 @@ cliSendFileOpts SendOptions {filePath, outputDir, numRecipients, xftpServers, re
           digest <- liftIO $ getChunkDigest chunkSpec
           let ch = FileInfo {sndKey, size = chunkSize, digest}
           c <- withRetry retryCount $ getXFTPServerClient a xftpServer
-          (sndId, rIds) <- withRetry retryCount $ createXFTPChunk c spKey ch (L.map fst rKeys) auth
+          (sndId, rIds, _) <- withRetry retryCount $ createXFTPChunk c spKey ch (L.map fst rKeys) auth Nothing
           withReconnect a xftpServer retryCount $ \c' -> uploadXFTPChunk c' spKey sndId chunkSpec
           logDebug $ "uploaded chunk " <> tshow chunkNo
           uploaded <- atomically . stateTVar uploadedChunks $ \cs ->
@@ -441,7 +441,6 @@ cliReceiveFile ReceiveOptions {fileDescription, filePath, retryCount, tempPath,
           | depth > 0 -> do
               CryptoFile tmpFile _ <- withExceptT cliCryptoError $ decryptChunks encSize chunkPaths key nonce $ \_ ->
                 fmap CF.plain $ uniqueCombine encPath "redirect.yaml"
-              forM_ chunks $ acknowledgeFileChunk a
               yaml <- liftIO $ B.readFile tmpFile
               whenM (doesPathExist encPath) $ removeDirectoryRecursive encPath
               innerVfd <- either (throwE . CLIError . ("Redirect: invalid file description: " <>)) pure $ strDecode yaml
@@ -450,7 +449,6 @@ cliReceiveFile ReceiveOptions {fileDescription, filePath, retryCount, tempPath,
         Nothing -> do
           liftIO $ printNoNewLine "Decrypting file..."
           CryptoFile path _ <- withExceptT cliCryptoError $ decryptChunks encSize chunkPaths key nonce $ fmap CF.plain . getFilePath
-          forM_ chunks $ acknowledgeFileChunk a
           whenM (doesPathExist encPath) $ removeDirectoryRecursive encPath
           liftIO $ do
             printNoNewLine $ "File downloaded: " <> path
@@ -479,12 +477,6 @@ cliReceiveFile ReceiveOptions {fileDescription, filePath, retryCount, tempPath,
       _ -> (`uniqueCombine` name') . (</> "Downloads") =<< getHomeDirectory
       where
         name' = T.unpack name
-    acknowledgeFileChunk :: XFTPClientAgent -> FileChunk -> ExceptT CLIError IO ()
-    acknowledgeFileChunk a FileChunk {replicas = replica : _} = do
-      let FileChunkReplica {server, replicaId, replicaKey} = replica
-      c <- withRetry retryCount $ getXFTPServerClient a server
-      withRetry retryCount $ ackXFTPChunk c replicaKey (unChunkReplicaId replicaId)
-    acknowledgeFileChunk _ _ = throwE $ CLIError "chunk has no replicas"
 
 printProgress :: String -> Int64 -> Int64 -> IO ()
 printProgress s part total = printNoNewLine $ s <> " " <> show ((part * 100) `div` total) <> "%"

src/Simplex/FileTransfer/Description.hs

@@ -29,6 +29,7 @@ module Simplex.FileTransfer.Description
     YAMLFileDescription (..), -- for tests
     YAMLServerReplicas (..), -- for tests
     validateFileDescription,
+    sharedDescriptionHash,
     groupReplicasByServer,
     fdSeparator,
     kb,
@@ -72,6 +73,7 @@ import Simplex.FileTransfer.Protocol
 import Simplex.Messaging.Agent.QueryString
 import Simplex.Messaging.Agent.Store.DB (Binary (..), FromField (..), ToField (..))
 import qualified Simplex.Messaging.Crypto as C
+import Simplex.Messaging.Encoding (Encoding (..))
 import Simplex.Messaging.Encoding.String
 import Simplex.Messaging.Parsers (defaultJSON, parseAll)
 import Simplex.Messaging.Protocol (XFTPServer)
@@ -227,6 +229,13 @@ validateFileDescription fd@FileDescription {size, chunks}
     chunkNos = map (\FileChunk {chunkNo} -> chunkNo) chunks
     chunksSize = foldl' (\(s :: Int64) FileChunk {chunkSize} -> s + fromIntegral (unFileSize chunkSize)) 0
 
+sharedDescriptionHash :: FileDescription p -> ByteString
+sharedDescriptionHash FileDescription {size, digest, key, nonce, chunkSize, chunks} =
+  C.sha512Hash $ smpEncode (unFileSize size, unFileDigest digest, C.unSbKey key, nonce, unFileSize chunkSize) <> foldMap chunkFields chunks
+  where
+    chunkFields FileChunk {chunkNo, chunkSize = cSize, digest = cDigest} =
+      smpEncode (fromIntegral chunkNo :: Int64, unFileSize cSize, unFileDigest cDigest)
+
 encodeFileDescription :: FileDescription p -> YAMLFileDescription
 encodeFileDescription FileDescription {party, size, digest, key, nonce, chunkSize, chunks, redirect} =
   YAMLFileDescription

src/Simplex/FileTransfer/Protocol.hs

@@ -22,6 +22,7 @@ module Simplex.FileTransfer.Protocol
     FileCommand (..),
     FileCmd (..),
     FileInfo (..),
+    GrantedStorageTime (..),
     XFTPFileId,
     FileResponse (..),
     xftpBlockSize,
@@ -38,12 +39,13 @@ import qualified Data.Aeson.TH as J
 import Data.Bifunctor (first)
 import Data.ByteString.Char8 (ByteString)
 import qualified Data.ByteString.Char8 as B
+import Data.Int (Int64)
 import Data.Kind (Type)
 import Data.List.NonEmpty (NonEmpty (..))
 import Data.Maybe (isNothing)
 import Data.Type.Equality
 import Data.Word (Word32)
-import Simplex.FileTransfer.Transport (XFTPErrorType (..), XFTPVersion, blockedFilesXFTPVersion, xftpClientHandshakeStub)
+import Simplex.FileTransfer.Transport (XFTPErrorType (..), XFTPVersion, blockedFilesXFTPVersion, fileStorageTimeXFTPVersion, xftpClientHandshakeStub)
 import Simplex.Messaging.Client (authTransmission)
 import qualified Simplex.Messaging.Crypto as C
 import Simplex.Messaging.Encoding
@@ -78,7 +80,7 @@ import Simplex.Messaging.Protocol
     tParse,
   )
 import Simplex.Messaging.Transport (THandleParams (..), TransportError (..), TransportPeer (..))
-import Simplex.Messaging.Util ((<$?>))
+import Simplex.Messaging.Util (eitherToMaybe, (<$?>))
 
 xftpBlockSize :: Int
 xftpBlockSize = 16384
@@ -175,7 +177,7 @@ instance Protocol XFTPVersion XFTPErrorType FileResponse where
   {-# INLINE protocolError #-}
 
 data FileCommand (p :: FileParty) where
-  FNEW :: FileInfo -> NonEmpty RcvPublicAuthKey -> Maybe BasicAuth -> FileCommand FSender
+  FNEW :: FileInfo -> NonEmpty RcvPublicAuthKey -> Maybe BasicAuth -> Maybe Word32 -> FileCommand FSender
   FADD :: NonEmpty RcvPublicAuthKey -> FileCommand FSender
   FPUT :: FileCommand FSender
   FDEL :: FileCommand FSender
@@ -196,12 +198,27 @@ data FileInfo = FileInfo
   }
   deriving (Show)
 
+data GrantedStorageTime = GSTExpires {epochSeconds :: Int64}
+  deriving (Eq, Ord, Show)
+
+instance Encoding GrantedStorageTime where
+  smpEncode = \case
+    GSTExpires t -> smpEncode ('T', t)
+  smpP =
+    smpP >>= \case
+      'T' -> GSTExpires <$> smpP
+      _ -> fail "bad GrantedStorageTime"
+
 type XFTPFileId = EntityId
 
 instance FilePartyI p => ProtocolEncoding XFTPVersion XFTPErrorType (FileCommand p) where
   type Tag (FileCommand p) = FileCommandTag p
-  encodeProtocol _v = \case
-    FNEW file rKeys auth_ -> e (FNEW_, ' ', file, rKeys, auth_)
+  encodeProtocol v = \case
+    FNEW file rKeys auth_ st
+      | v >= fileStorageTimeXFTPVersion -> fnew <> e st
+      | otherwise -> fnew
+      where
+        fnew = e (FNEW_, ' ', file, rKeys, auth_)
     FADD rKeys -> e (FADD_, ' ', rKeys)
     FPUT -> e FPUT_
     FDEL -> e FDEL_
@@ -235,10 +252,14 @@ instance ProtocolEncoding XFTPVersion XFTPErrorType FileCmd where
   type Tag FileCmd = FileCmdTag
   encodeProtocol _v (FileCmd _ c) = encodeProtocol _v c
 
-  protocolP _v = \case
+  protocolP v = \case
     FCT SFSender tag ->
       FileCmd SFSender <$> case tag of
-        FNEW_ -> FNEW <$> _smpP <*> smpP <*> smpP
+        FNEW_
+          | v >= fileStorageTimeXFTPVersion -> fnewP smpP
+          | otherwise -> fnewP (pure Nothing)
+          where
+            fnewP stP = FNEW <$> _smpP <*> smpP <*> smpP <*> stP
         FADD_ -> FADD <$> _smpP
         FPUT_ -> pure FPUT
         FDEL_ -> pure FDEL
@@ -292,7 +313,7 @@ instance ProtocolMsgTag FileResponseTag where
     _ -> Nothing
 
 data FileResponse
-  = FRSndIds SenderId (NonEmpty RecipientId)
+  = FRSndIds SenderId (NonEmpty RecipientId) (Maybe GrantedStorageTime)
   | FRRcvIds (NonEmpty RecipientId)
   | FRFile RcvPublicDhKey C.CbNonce
   | FROk
@@ -303,7 +324,9 @@ data FileResponse
 instance ProtocolEncoding XFTPVersion XFTPErrorType FileResponse where
   type Tag FileResponse = FileResponseTag
   encodeProtocol v = \case
-    FRSndIds fId rIds -> e (FRSndIds_, ' ', fId, rIds)
+    FRSndIds fId rIds gs
+      | v >= fileStorageTimeXFTPVersion -> e (FRSndIds_, ' ', fId, rIds, gs)
+      | otherwise -> e (FRSndIds_, ' ', fId, rIds)
     FRRcvIds rIds -> e (FRRcvIds_, ' ', rIds)
     FRFile rDhKey nonce -> e (FRFile_, ' ', rDhKey, nonce)
     FROk -> e FROk_
@@ -315,8 +338,10 @@ instance ProtocolEncoding XFTPVersion XFTPErrorType FileResponse where
       e :: Encoding a => a -> ByteString
       e = smpEncode
 
-  protocolP _v = \case
-    FRSndIds_ -> FRSndIds <$> _smpP <*> smpP
+  protocolP v = \case
+    FRSndIds_
+      | v >= fileStorageTimeXFTPVersion -> FRSndIds <$> _smpP <*> smpP <*> smpP
+      | otherwise -> FRSndIds <$> _smpP <*> smpP <*> pure Nothing
     FRRcvIds_ -> FRRcvIds <$> _smpP
     FRFile_ -> FRFile <$> _smpP <*> smpP
     FROk_ -> pure FROk
@@ -358,7 +383,7 @@ checkParty' c = case testEquality (sFileParty @p) (sFileParty @p') of
 xftpEncodeAuthTransmission :: ProtocolEncoding XFTPVersion XFTPErrorType c => THandleParams XFTPVersion 'TClient -> C.APrivateAuthKey -> Transmission c -> Either TransportError ByteString
 xftpEncodeAuthTransmission thParams@THandleParams {thAuth} pKey t@(corrId, _, _) = do
   let TransmissionForAuth {tForAuth, tToSend} = encodeTransmissionForAuth thParams t
-  xftpEncodeBatch1 . (,tToSend) =<< authTransmission thAuth False (Just pKey) (C.cbNonce $ bs corrId) tForAuth
+  xftpEncodeBatch1 . (,tToSend) =<< authTransmission thAuth False (Just pKey) (eitherToMaybe $ C.cbNonce $ bs corrId) tForAuth
 
 xftpEncodeTransmission :: ProtocolEncoding XFTPVersion XFTPErrorType c => THandleParams XFTPVersion p -> Transmission c -> Either TransportError ByteString
 xftpEncodeTransmission thParams t = xftpEncodeBatch1 (Nothing, encodeTransmission thParams t)

src/Simplex/FileTransfer/Server.hs

@@ -23,18 +23,21 @@ import Control.Monad
 import Control.Monad.Except
 import Control.Monad.Reader
 import Control.Monad.Trans.Except
+import qualified Data.Aeson as J
 import Data.Bifunctor (first)
-import qualified Data.ByteString.Base64.URL as B64
 import Data.ByteString.Builder (Builder, byteString)
 import Data.ByteString.Char8 (ByteString)
 import qualified Data.ByteString.Char8 as B
+import qualified Data.ByteString.Lazy.Char8 as LB
 import Data.Int (Int64)
 import Data.List.NonEmpty (NonEmpty)
 import qualified Data.List.NonEmpty as L
+import qualified Data.Map.Strict as M
 import Data.Maybe (fromMaybe, isJust)
 import qualified Data.Text as T
 import qualified Data.Text.IO as T
 import Data.Time.Clock (UTCTime (..), diffTimeToPicoseconds, getCurrentTime)
+import Data.Time.Clock.System (systemSeconds, utcToSystemTime)
 import Data.Time.Format.ISO8601 (iso8601Show)
 import Data.Word (Word32)
 import qualified Data.X509 as X
@@ -54,10 +57,12 @@ import Simplex.FileTransfer.Server.Store
 import Simplex.FileTransfer.Server.StoreLog
 import Simplex.FileTransfer.Transport
 import qualified Simplex.Messaging.Crypto as C
+import Simplex.Messaging.Crypto.BBS (BBSPresHeader (..))
+import Simplex.Messaging.Crypto.Entitlement (Entitlement (..), EntitlementProof (..), EntitlementVerification (..), verifyEntitlement)
 import qualified Simplex.Messaging.Crypto.Lazy as LC
 import Simplex.Messaging.Encoding
 import Simplex.Messaging.Encoding.String
-import Simplex.Messaging.Protocol (BlockingInfo, EntityId (..), RcvPublicAuthKey, RcvPublicDhKey, RecipientId, SignedTransmission, pattern NoEntity)
+import Simplex.Messaging.Protocol (BlockingInfo, CommandError (..), EntityId (..), RcvPublicAuthKey, RcvPublicDhKey, RecipientId, SignedTransmission, pattern NoEntity)
 import Simplex.Messaging.Server (controlPortAuth, dummyVerifyCmd, verifyCmdAuthorization)
 import Simplex.Messaging.Server.Control (CPClientRole (..))
 import Simplex.Messaging.Server.Expiration
@@ -66,7 +71,7 @@ import Simplex.Messaging.Server.Stats
 import Simplex.Messaging.SystemTime
 import Simplex.Messaging.TMap (TMap)
 import qualified Simplex.Messaging.TMap as TM
-import Simplex.Messaging.Transport (CertChainPubKey (..), SessionId, THandleAuth (..), THandleParams (..), TransportPeer (..), defaultSupportedParams, defaultSupportedParamsHTTPS)
+import Simplex.Messaging.Transport (CertChainPubKey (..), EntitlementConfig (..), SessionEntitlement (..), SessionId, THandleAuth (..), THandleParams (..), TransportPeer (..), defaultSupportedParams, defaultSupportedParamsHTTPS)
 import Simplex.Messaging.Transport.Buffer (trimCR)
 import Simplex.Messaging.Transport.HTTP2
 import Simplex.Messaging.Transport.HTTP2.File (fileBlockSize)
@@ -120,17 +125,20 @@ runXFTPServerBlocking :: FileStoreClass s => TMVar Bool -> XFTPServerConfig s ->
 runXFTPServerBlocking started cfg = newXFTPServerEnv cfg >>= runReaderT (xftpServer cfg started)
 
 data Handshake
-  = HandshakeSent C.PrivateKeyX25519
+  = HandshakeSent C.PrivateKeyX25519 EntitlementChecked
   | HandshakeAccepted (THandleParams XFTPVersion 'TServer)
 
+-- | the entitlement of the session is resolved once, however many handshakes it has
+data EntitlementChecked = EntNotChecked | EntChecked (Maybe SessionEntitlement)
+
 xftpServer :: forall s. FileStoreClass s => XFTPServerConfig s -> TMVar Bool -> M s ()
-xftpServer cfg@XFTPServerConfig {xftpPort, transportConfig, inactiveClientExpiration, fileExpiration, xftpServerVRange} started = do
-  mapM_ (expireServerFiles Nothing) fileExpiration
+xftpServer cfg@XFTPServerConfig {xftpPort, transportConfig, inactiveClientExpiration, fileExpiration, xftpServerVRange, information} started = do
+  expireServerFiles Nothing fileExpiration
   restoreServerStats
   raceAny_
     ( runServer
-        : expireFilesThread_ cfg
-          <> serverStatsThread_ cfg
+        : expireFiles fileExpiration
+        : serverStatsThread_ cfg
           <> prometheusMetricsThread_ cfg
           <> controlPortThread_ cfg
     )
@@ -174,12 +182,12 @@ xftpServer cfg@XFTPServerConfig {xftpPort, transportConfig, inactiveClientExpira
         Nothing
           | sniUsed && not webHello -> throwE SESSION
           | otherwise -> processHello Nothing
-        Just (HandshakeSent pk)
+        Just (HandshakeSent pk ent_)
           | webHello -> processHello (Just pk)
-          | otherwise -> processClientHandshake pk
+          | otherwise -> processClientHandshake pk ent_
         Just (HandshakeAccepted thParams)
           | webHello -> processHello (serverPrivKey <$> thAuth thParams)
-          | webHandshake, Just auth <- thAuth thParams -> processClientHandshake (serverPrivKey auth)
+          | webHandshake, Just auth <- thAuth thParams -> processClientHandshake (serverPrivKey auth) (EntChecked $ peerEntitlement auth)
           | otherwise -> pure $ Just thParams
       either sendError pure r
       where
@@ -196,28 +204,35 @@ xftpServer cfg@XFTPServerConfig {xftpPort, transportConfig, inactiveClientExpira
               | otherwise -> throwE HANDSHAKE
           rng <- asks random
           k <- atomically $ TM.lookup sessionId sessions >>= \case
-            Just (HandshakeSent pk') -> pure $ C.publicKey pk'
-            _ -> do
+            Just (HandshakeSent pk' _) -> pure $ C.publicKey pk'
+            s' -> do
               kp <- maybe (C.generateKeyPair rng) (\p -> pure (C.publicKey p, p)) pk_
-              fst kp <$ TM.insert sessionId (HandshakeSent $ snd kp) sessions
+              let ent_ = case s' of
+                    Just (HandshakeAccepted thParams) -> EntChecked $ peerEntitlement =<< thAuth thParams
+                    _ -> EntNotChecked
+              fst kp <$ TM.insert sessionId (HandshakeSent (snd kp) ent_) sessions
           let authPubKey = CertChainPubKey chain (C.signX509 serverSignKey $ C.publicToX509 k)
               webIdentityProof = C.sign serverSignKey . (<> sessionId) <$> challenge_
-          let hs = XFTPServerHandshake {xftpVersionRange = xftpServerVRange, sessionId, authPubKey, webIdentityProof}
+              serverInfoBytes = LB.toStrict . J.encode <$> information
+          let hs = XFTPServerHandshake {xftpVersionRange = xftpServerVRange, sessionId, authPubKey, webIdentityProof, serverInfoBytes}
           shs <- encodeXftp hs
 #ifdef slow_servers
           lift randomDelay
 #endif
           liftIO . sendResponse $ H.responseBuilder N.ok200 (corsHeaders addCORS) shs
           pure Nothing
-        processClientHandshake pk = do
+        processClientHandshake pk ent_ = do
           unless (B.length bodyHead == xftpBlockSize) $ throwE HANDSHAKE
           body <- liftHS $ C.unPad bodyHead
-          XFTPClientHandshake {xftpVersion = v, keyHash} <- liftHS $ smpDecode body
+          XFTPClientHandshake {xftpVersion = v, keyHash, entitlementProof} <- liftHS $ smpDecode body
           kh <- asks serverIdentity
           unless (keyHash == kh) $ throwE HANDSHAKE
           case compatibleVRange' xftpServerVRange v of
             Just (Compatible vr) -> do
-              let auth = THAuthServer {serverPrivKey = pk, peerClientService = Nothing, sessSecret' = Nothing}
+              ent <- case ent_ of
+                EntChecked ent -> pure ent
+                EntNotChecked -> lift $ verifiedEntitlement entitlementProof
+              let auth = THAuthServer {serverPrivKey = pk, peerClientService = Nothing, peerEntitlement = ent, sessSecret' = Nothing}
                   thParams = thParams0 {thAuth = Just auth, thVersion = v, thServerVRange = vr}
               atomically $ TM.insert sessionId (HandshakeAccepted thParams) sessions
 #ifdef slow_servers
@@ -226,6 +241,19 @@ xftpServer cfg@XFTPServerConfig {xftpPort, transportConfig, inactiveClientExpira
               liftIO . sendResponse $ H.responseNoBody N.ok200 (corsHeaders addCORS)
               pure Nothing
             Nothing -> throwE HANDSHAKE
+        verifiedEntitlement :: Maybe EntitlementProof -> M s (Maybe SessionEntitlement)
+        verifiedEntitlement ep =
+          pure ep $>>= \proof@EntitlementProof {entitlement = Entitlement {entitlementName, expiresAt = expiresAtTs}} -> do
+            entCfg <- asks $ M.lookup entitlementName . fileStorageEntitlements . config
+            now <- liftIO getSystemSeconds
+            let expiresAt = RoundedSystemTime $ systemSeconds $ utcToSystemTime expiresAtTs
+            case entCfg of
+              Just cfg | entitlementValid now expiresAt -> do
+                keys <- asks $ entitlementKeys . config
+                liftIO (verifyEntitlement keys (BBSPresHeader sessionId) proof) >>= \case
+                  EVValid -> pure $ Just SessionEntitlement {expiresAt, entConfig = cfg}
+                  r -> Nothing <$ logError ("entitlement not verified: " <> tshow r)
+              _ -> pure Nothing
         sendError :: XFTPErrorType -> M s (Maybe (THandleParams XFTPVersion 'TServer))
         sendError err = do
           runExceptT (encodeXftp err) >>= \case
@@ -243,10 +271,6 @@ xftpServer cfg@XFTPServerConfig {xftpPort, transportConfig, inactiveClientExpira
       saveServerStats
       logNote "Server stopped"
 
-    expireFilesThread_ :: XFTPServerConfig s -> [M s ()]
-    expireFilesThread_ XFTPServerConfig {fileExpiration = Just fileExp} = [expireFiles fileExp]
-    expireFilesThread_ _ = []
-
     expireFiles :: ExpirationConfig -> M s ()
     expireFiles expCfg = do
       let interval = checkInterval expCfg * 1000000
@@ -402,8 +426,9 @@ processRequest XFTPTransportRequest {thParams, reqBody = body@HTTP2Body {bodyHea
       case xftpDecodeTServer thParams bodyHead of
         Right (Right t@(_, _, (corrId, fId, _))) -> do
           let THandleParams {thAuth} = thParams
+              ent = peerEntitlement =<< thAuth
           verifyXFTPTransmission thAuth t >>= \case
-            VRVerified req -> uncurry send =<< processXFTPRequest body req
+            VRVerified req -> uncurry send =<< processXFTPRequest ent body req
             VRFailed e -> send (FRErr e) Nothing
           where
             send resp = sendXFTPResponse (corrId, fId, resp)
@@ -443,7 +468,7 @@ data VerificationResult = VRVerified XFTPRequest | VRFailed XFTPErrorType
 verifyXFTPTransmission :: forall s. FileStoreClass s => Maybe (THandleAuth 'TServer) -> SignedTransmission FileCmd -> M s VerificationResult
 verifyXFTPTransmission thAuth (tAuth, authorized, (corrId, fId, cmd)) =
   case cmd of
-    FileCmd SFSender (FNEW file rcps auth') -> pure $ XFTPReqNew file rcps auth' `verifyWith` sndKey file
+    FileCmd SFSender (FNEW file rcps auth' st) -> pure $ XFTPReqNew file rcps auth' st `verifyWith` sndKey file
     FileCmd SFRecipient PING -> pure $ VRVerified XFTPReqPing
     FileCmd party _ -> verifyCmd party
   where
@@ -464,9 +489,9 @@ verifyXFTPTransmission thAuth (tAuth, authorized, (corrId, fId, cmd)) =
     -- TODO verify with DH authorization
     req `verifyWith` k = if verifyCmdAuthorization thAuth tAuth authorized corrId k then VRVerified req else VRFailed AUTH
 
-processXFTPRequest :: forall s. FileStoreClass s => HTTP2Body -> XFTPRequest -> M s (FileResponse, Maybe ServerFile)
-processXFTPRequest HTTP2Body {bodyPart} = \case
-  XFTPReqNew file rks auth -> noFile =<< ifM allowNew (createFile file rks) (pure $ FRErr AUTH)
+processXFTPRequest :: forall s. FileStoreClass s => Maybe SessionEntitlement -> HTTP2Body -> XFTPRequest -> M s (FileResponse, Maybe ServerFile)
+processXFTPRequest ent HTTP2Body {bodyPart} = \case
+  XFTPReqNew file rks auth storageHours -> noFile =<< ifM allowNew (createFile file rks storageHours) (pure $ FRErr AUTH)
     where
       allowNew = do
         XFTPServerConfig {allowNewFiles, newFileBasicAuth} <- asks config
@@ -483,29 +508,42 @@ processXFTPRequest HTTP2Body {bodyPart} = \case
   XFTPReqPing -> noFile FRPong
   where
     noFile resp = pure (resp, Nothing)
-    createFile :: FileInfo -> NonEmpty RcvPublicAuthKey -> M s FileResponse
-    createFile file rks = do
+    createFile :: FileInfo -> NonEmpty RcvPublicAuthKey -> Maybe Word32 -> M s FileResponse
+    createFile file rks storageHours = do
       st <- asks fileStore
       r <- runExceptT $ do
         sizes <- asks $ allowedChunkSizes . config
         unless (size file `elem` sizes) $ throwE SIZE
-        ts <- liftIO getFileTime
+        now <- liftIO getSystemSeconds
+        maxSeconds <- lift $ storageMaxSeconds now
+        let nowSeconds = roundedSeconds now
+            ts = RoundedSystemTime $ (nowSeconds `div` fileTimePrecision) * fileTimePrecision
+            secs = case storageHours of
+              Just hours | hours > 0 -> min maxSeconds (fromIntegral hours * 3600)
+              _ -> maxSeconds
+            fileExpiresAt = RoundedSystemTime $ ((nowSeconds + secs + fileTimePrecision - 1) `div` fileTimePrecision) * fileTimePrecision
         -- TODO validate body empty
-        sId <- ExceptT $ addFileRetry st file 3 ts
+        sId <- ExceptT $ addFileRetry st file 3 ts (Just fileExpiresAt)
         rcps <- mapM (ExceptT . addRecipientRetry st 3 sId) rks
         lift $ withFileLog $ \sl -> do
-          logAddFile sl sId file ts EntityActive
+          logAddFile sl sId file ts (Just fileExpiresAt) EntityActive
           logAddRecipients sl sId rcps
         stats <- asks serverStats
         lift $ incFileStat filesCreated
         liftIO $ atomicModifyIORef'_ (fileRecipients stats) (+ length rks)
         let rIds = L.map (\(FileRecipient rId _) -> rId) rcps
-        pure $ FRSndIds sId rIds
+        pure $ FRSndIds sId rIds (Just (GSTExpires (roundedSeconds fileExpiresAt)))
       pure $ either FRErr id r
-    addFileRetry :: s -> FileInfo -> Int -> RoundedFileTime -> M s (Either XFTPErrorType XFTPFileId)
-    addFileRetry st file n ts =
+    storageMaxSeconds :: SystemSeconds -> M s Int64
+    storageMaxSeconds now = do
+      defaultMax <- asks $ ttl . fileExpiration . config
+      pure $ case ent of
+        Just SessionEntitlement {expiresAt, entConfig} | entitlementValid now expiresAt -> max (storageTime entConfig) defaultMax
+        _ -> defaultMax
+    addFileRetry :: s -> FileInfo -> Int -> RoundedFileTime -> Maybe RoundedFileTime -> M s (Either XFTPErrorType XFTPFileId)
+    addFileRetry st file n ts expiresAt =
       retryAdd n $ \sId -> runExceptT $ do
-        ExceptT $ addFile st sId file ts EntityActive
+        ExceptT $ addFile st sId file ts expiresAt EntityActive
         pure sId
     addRecipientRetry :: s -> Int -> XFTPFileId -> RcvPublicAuthKey -> M s (Either XFTPErrorType FileRecipient)
     addRecipientRetry st n sId rpk =
@@ -555,7 +593,7 @@ processXFTPRequest HTTP2Body {bodyPart} = \case
               \used -> let used' = used + fromIntegral size in if used' <= quota then (True, used') else (False, used)
           receive = do
             path <- asks $ filesPath . config
-            let fPath = path </> B.unpack (B64.encode $ unEntityId senderId)
+            let fPath = path </> serverFileName senderId
             receiveChunk (XFTPRcvChunkSpec fPath size digest) >>= \case
               Right () -> do
                 stats <- asks serverStats
@@ -607,11 +645,12 @@ processXFTPRequest HTTP2Body {bodyPart} = \case
 
     ackFileReception :: RecipientId -> FileRec -> M s FileResponse
     ackFileReception rId fr = do
-      withFileLog (`logAckFile` rId)
-      st <- asks fileStore
-      liftIO $ deleteRecipient st rId fr
-      incFileStat fileDownloadAcks
-      pure FROk
+      pure $ FRErr $ CMD PROHIBITED
+      -- withFileLog (`logAckFile` rId)
+      -- st <- asks fileStore
+      -- liftIO $ deleteRecipient st rId fr
+      -- incFileStat fileDownloadAcks
+      -- pure FROk
 
 deleteServerFile_ :: FileStoreClass s => FileRec -> M s (Either XFTPErrorType ())
 deleteServerFile_ fr@FileRec {senderId} = do
@@ -640,24 +679,25 @@ deleteOrBlockServerFile_ FileRec {filePath, fileInfo} stat storeAction = runExce
       liftIO $ atomicModifyIORef'_ (filesCount stats) (subtract 1)
       liftIO $ atomicModifyIORef'_ (filesSize stats) (subtract $ fromIntegral $ size fileInfo)
 
-getFileTime :: IO RoundedFileTime
-getFileTime = getRoundedSystemTime
+entitlementValid :: SystemSeconds -> SystemSeconds -> Bool
+entitlementValid now expiresAt = roundedSeconds expiresAt + 86400 > roundedSeconds now
 
 expireServerFiles :: FileStoreClass s => Maybe Int -> ExpirationConfig -> M s ()
 expireServerFiles itemDelay expCfg = do
   st <- asks fileStore
   us <- asks usedStorage
   usedStart <- readTVarIO us
+  now <- liftIO getSystemSeconds
   old <- liftIO $ expireBeforeEpoch expCfg
   filesCount <- liftIO $ getFileCount st
   logNote $ "Expiration check: " <> tshow filesCount <> " files"
-  expireLoop st us old
+  expireLoop st us now old
   usedEnd <- readTVarIO us
   logNote $ "Used " <> mbs usedStart <> " -> " <> mbs usedEnd <> ", " <> mbs (usedStart - usedEnd) <> " reclaimed."
   where
     mbs bs = tshow (bs `div` 1048576) <> "mb"
-    expireLoop st us old = do
-      expired <- liftIO $ expiredFiles st old 10000
+    expireLoop st us now old = do
+      expired <- liftIO $ expiredFiles st now old 10000
       forM_ expired $ \(sId, filePath_, fileSize) -> do
         mapM_ threadDelay itemDelay
         forM_ filePath_ $ \fp ->
@@ -670,7 +710,7 @@ expireServerFiles itemDelay expCfg = do
       unless (null sIds) $ do
         withFileLog $ \sl -> mapM_ (logDeleteFile sl) sIds
         liftIO $ deleteFiles st sIds
-        expireLoop st us old
+        expireLoop st us now old
 
 randomId :: Int -> M s ByteString
 randomId n = atomically . C.randomBytes n =<< asks random

src/Simplex/FileTransfer/Server/Env.hs

@@ -37,8 +37,11 @@ import Control.Monad
 import Crypto.Random
 import Data.Int (Int64)
 import Data.List.NonEmpty (NonEmpty)
+import Data.Map.Strict (Map)
+import qualified Data.Map.Strict as M
+import Data.Text (Text)
 import Data.Time.Clock (getCurrentTime)
-import Data.Word (Word32)
+import Data.Word (Word16, Word32)
 import Data.X509.Validation (Fingerprint (..))
 import Network.Socket
 import qualified Network.TLS as T
@@ -62,8 +65,11 @@ import System.Directory (doesFileExist)
 import Simplex.FileTransfer.Server.StoreLog
 import Simplex.FileTransfer.Transport (VersionRangeXFTP)
 import qualified Simplex.Messaging.Crypto as C
+import Simplex.Messaging.Crypto.BBS (BBSPublicKey)
 import Simplex.Messaging.Protocol (BasicAuth, RcvPublicAuthKey)
 import Simplex.Messaging.Server.Expiration
+import Simplex.Messaging.Server.Information (ServerPublicInfo)
+import Simplex.Messaging.Transport (EntitlementConfig (..))
 import Simplex.Messaging.Transport.Server (ServerCredentials (..), TransportServerConfig (..), loadFingerprint, loadServerCredential)
 import Simplex.Messaging.Util (tshow)
 import System.IO (IOMode (..))
@@ -88,7 +94,10 @@ data XFTPServerConfig s = XFTPServerConfig
     controlPortUserAuth :: Maybe BasicAuth,
     controlPortAdminAuth :: Maybe BasicAuth,
     -- | time after which the files can be removed and check interval, seconds
-    fileExpiration :: Maybe ExpirationConfig,
+    fileExpiration :: ExpirationConfig,
+    -- | what each entitlement name grants
+    fileStorageEntitlements :: Map Text EntitlementConfig,
+    entitlementKeys :: Map Word16 BBSPublicKey,
     -- | timeout to receive file
     fileTimeout :: Int,
     -- | time after which inactive clients can be disconnected and check interval, seconds
@@ -97,6 +106,8 @@ data XFTPServerConfig s = XFTPServerConfig
     httpCredentials :: Maybe ServerCredentials,
     -- | XFTP client-server protocol version range
     xftpServerVRange :: VersionRangeXFTP,
+    -- | server public information sent in handshake and used to generate static mini-site
+    information :: Maybe ServerPublicInfo,
     -- stats config - see SMP server config
     logStatsInterval :: Maybe Int64,
     logStatsStartTime :: Int64,
@@ -171,7 +182,13 @@ defaultFileExpiration =
     }
 
 newXFTPServerEnv :: FileStoreClass s => XFTPServerConfig s -> IO (XFTPEnv s)
-newXFTPServerEnv config@XFTPServerConfig {serverStoreCfg, fileSizeQuota, xftpCredentials, httpCredentials} = do
+newXFTPServerEnv config@XFTPServerConfig {serverStoreCfg, fileSizeQuota, fileExpiration, fileStorageEntitlements, xftpCredentials, httpCredentials} = do
+  let defaultMax = ttl fileExpiration
+      belowDefault = M.filter ((< defaultMax) . storageTime) fileStorageEntitlements
+  unless (M.null belowDefault) $ do
+    forM_ (M.assocs belowDefault) $ \(name, EntitlementConfig {storageTime}) ->
+      logError $ "expire_files_hours_for_" <> name <> " is " <> tshow (storageTime `div` 3600) <> " hours, below expire_files_hours " <> tshow (defaultMax `div` 3600) <> " hours, server not started"
+    exitFailure
   random <- C.newRandom
   (store, storeLog) <- case serverStoreCfg of
     XSCMemory storeLogPath -> do
@@ -196,7 +213,7 @@ newXFTPServerEnv config@XFTPServerConfig {serverStoreCfg, fileSizeQuota, xftpCre
   pure XFTPEnv {config, store, usedStorage, storeLog, random, tlsServerCreds, httpServerCreds, serverIdentity = C.KeyHash fp, serverStats}
 
 data XFTPRequest
-  = XFTPReqNew FileInfo (NonEmpty RcvPublicAuthKey) (Maybe BasicAuth)
+  = XFTPReqNew FileInfo (NonEmpty RcvPublicAuthKey) (Maybe BasicAuth) (Maybe Word32)
   | XFTPReqCmd XFTPFileId FileRec FileCmd
   | XFTPReqPing
 

src/Simplex/FileTransfer/Server/Main.hs

@@ -6,6 +6,7 @@
 {-# LANGUAGE OverloadedStrings #-}
 {-# LANGUAGE PatternSynonyms #-}
 {-# LANGUAGE RankNTypes #-}
+{-# LANGUAGE TupleSections #-}
 {-# LANGUAGE TypeApplications #-}
 
 module Simplex.FileTransfer.Server.Main
@@ -13,14 +14,16 @@ module Simplex.FileTransfer.Server.Main
     xftpServerCLI_,
   ) where
 
-import Control.Monad (unless, when)
+import Control.Monad (forM_, unless, when)
 import Data.Either (fromRight)
 import Data.Functor (($>))
-import Data.Ini (lookupValue, readIniFile)
+import Data.Ini (Ini, lookupValue, readIniFile)
 import Data.Int (Int64)
 import Data.List (find)
 import qualified Data.List.NonEmpty as L
-import Data.Maybe (fromMaybe, isJust)
+import Data.Map.Strict (Map)
+import qualified Data.Map.Strict as M
+import Data.Maybe (fromMaybe, isJust, mapMaybe)
 import Data.Text.Encoding (encodeUtf8)
 import qualified Data.Text as T
 import qualified Data.Text.IO as T
@@ -32,6 +35,7 @@ import Simplex.FileTransfer.Server (runXFTPServer)
 import Simplex.FileTransfer.Server.Env (XFTPServerConfig (..), XFTPStoreConfig, AFStoreType (..), defFileExpirationHours, defaultFileExpiration, defaultInactiveClientExpiration, readFileStoreType, runWithStoreConfig, checkFileStoreMode, importToDatabase, exportFromDatabase)
 import Simplex.FileTransfer.Transport (alpnSupportedXFTPhandshakes, supportedFileServerVRange)
 import qualified Simplex.Messaging.Crypto as C
+import Simplex.Messaging.Crypto.Entitlement (entitlementIssuerKeys)
 import Simplex.Messaging.Encoding.String
 import Simplex.Messaging.Protocol (ProtoServerWithAuth (..), pattern XFTPServer)
 import Simplex.Messaging.Agent.Store.Shared (MigrationConfirmation (..))
@@ -40,6 +44,7 @@ import Simplex.Messaging.Server.Expiration
 import Simplex.Messaging.Server.Information (ServerPublicInfo (..))
 import Simplex.Messaging.Server.Main (serverPublicInfo, printSourceCode)
 import Simplex.Messaging.Server.Web (EmbeddedWebParams (..), WebHttpsParams (..))
+import Simplex.Messaging.Transport (EntitlementConfig (..))
 import Simplex.Messaging.Transport.Client (TransportHost (..))
 import Simplex.Messaging.Transport.HTTP2 (httpALPN)
 import Simplex.Messaging.Transport.Server (ServerCredentials (..), TransportServerConfig (..), mkTransportServerConfig)
@@ -155,8 +160,14 @@ xftpServerCLI_ generateSite serveStaticFiles cfgPath logPath = do
                \# db_pool_size = 10\n\n\
                \# Write database changes to store log file\n\
                \# db_store_log = off\n\n"
-            <> "# Expire files after the specified number of hours.\n"
+            <> "# Expire files after the specified number of hours.\n\
+               \# The change only affects new files.\n"
             <> ("expire_files_hours = " <> tshow defFileExpirationHours <> "\n\n")
+            <> "# Expire files after the specified number of hours for the senders that present\n\
+               \# a proof of the entitlement. Must not be below expire_files_hours.\n\
+               \# The change only affects new files.\n\
+               \# expire_files_hours_for_supporter = 168\n\
+               \# expire_files_hours_for_legend = 504\n\n"
             <> "log_stats = off\n\
                \\n\
                \# Log interval for real-time Prometheus metrics\n\
@@ -235,13 +246,13 @@ xftpServerCLI_ generateSite serveStaticFiles cfgPath logPath = do
         enableStoreLog = settingIsOn "STORE_LOG" "enable" ini
         logStats = settingIsOn "STORE_LOG" "log_stats" ini
         c = combine cfgPath . ($ defaultX509Config)
-        printXFTPConfig XFTPServerConfig {allowNewFiles, newFileBasicAuth, xftpPort, storeLogFile, fileExpiration, inactiveClientExpiration} = do
+        printXFTPConfig XFTPServerConfig {allowNewFiles, newFileBasicAuth, xftpPort, storeLogFile, fileExpiration, fileStorageEntitlements, inactiveClientExpiration} = do
           putStrLn $ case storeLogFile of
             Just f -> "Store log: " <> f
             _ -> "Store log disabled."
-          putStrLn $ case fileExpiration of
-            Just ExpirationConfig {ttl} -> "expiring files after " <> showTTL ttl
-            _ -> "not expiring files"
+          putStrLn $ "expiring files after " <> showTTL (ttl fileExpiration)
+          forM_ (M.assocs fileStorageEntitlements) $ \(name, EntitlementConfig {storageTime}) ->
+            putStrLn $ "expiring files of " <> T.unpack name <> " after " <> showTTL storageTime
           putStrLn $ case inactiveClientExpiration of
             Just ExpirationConfig {ttl, checkInterval} -> "expiring clients inactive for " <> show ttl <> " seconds every " <> show checkInterval <> " seconds"
             _ -> "not expiring inactive clients"
@@ -287,10 +298,11 @@ xftpServerCLI_ generateSite serveStaticFiles cfgPath logPath = do
               controlPortAdminAuth = either error id <$> strDecodeIni "AUTH" "control_port_admin_password" ini,
               controlPortUserAuth = either error id <$> strDecodeIni "AUTH" "control_port_user_password" ini,
               fileExpiration =
-                Just
-                  defaultFileExpiration
-                    { ttl = 3600 * readIniDefault defFileExpirationHours "STORE_LOG" "expire_files_hours" ini
-                    },
+                defaultFileExpiration
+                  { ttl = 3600 * readIniDefault defFileExpirationHours "STORE_LOG" "expire_files_hours" ini
+                  },
+              fileStorageEntitlements = iniEntitlements ini,
+              entitlementKeys = entitlementIssuerKeys,
               fileTimeout = 5 * 60 * 1000000, -- 5 mins to send 4mb chunk
               inactiveClientExpiration =
                 settingIsOn "INACTIVE_CLIENTS" "disconnect" ini
@@ -306,6 +318,7 @@ xftpServerCLI_ generateSite serveStaticFiles cfgPath logPath = do
                   },
               httpCredentials = httpCredentials_,
               xftpServerVRange = supportedFileServerVRange,
+              information = serverPublicInfo ini,
               logStatsInterval = logStats $> 86400, -- seconds
               logStatsStartTime = 0, -- seconds from 00:00 UTC
               serverStatsLogFile = combine logPath "file-server-stats.daily.log",
@@ -437,3 +450,10 @@ cliCommandP cfgPath logPath iniFile =
         ( command "import" (info (pure SCImport) (progDesc "Import store log file into PostgreSQL database"))
             <> command "export" (info (pure SCExport) (progDesc "Export PostgreSQL database to store log file"))
         )
+
+iniEntitlements :: Ini -> Map T.Text EntitlementConfig
+iniEntitlements ini =
+  M.fromList $ mapMaybe readEntitlement [("supporter", "expire_files_hours_for_supporter"), ("legend", "expire_files_hours_for_legend")]
+  where
+    readEntitlement (name, key) = (name,) . EntitlementConfig . parseHours key <$> eitherToMaybe (lookupValue "STORE_LOG" key ini)
+    parseHours key t = maybe (error $ "Error: invalid " <> T.unpack key <> " value: " <> T.unpack t) (3600 *) (readMaybe (T.unpack (T.strip t)) :: Maybe Int64)

src/Simplex/FileTransfer/Server/Store.hs

@@ -55,6 +55,7 @@ data FileRec = FileRec
     filePath :: TVar (Maybe FilePath),
     recipientIds :: TVar (Set RecipientId),
     createdAt :: RoundedFileTime,
+    expiresAt :: Maybe RoundedFileTime,
     fileStatus :: TVar ServerEntityStatus
   }
 
@@ -74,7 +75,7 @@ class FileStoreClass s where
   type FileStoreConfig s
   newFileStore :: FileStoreConfig s -> IO s
   closeFileStore :: s -> IO ()
-  addFile :: s -> SenderId -> FileInfo -> RoundedFileTime -> ServerEntityStatus -> IO (Either XFTPErrorType ())
+  addFile :: s -> SenderId -> FileInfo -> RoundedFileTime -> Maybe RoundedFileTime -> ServerEntityStatus -> IO (Either XFTPErrorType ())
   setFilePath :: s -> SenderId -> FilePath -> IO (Either XFTPErrorType ())
   addRecipient :: s -> SenderId -> FileRecipient -> IO (Either XFTPErrorType ())
   deleteFile :: s -> SenderId -> IO (Either XFTPErrorType ())
@@ -84,7 +85,7 @@ class FileStoreClass s where
   deleteRecipient :: s -> RecipientId -> FileRec -> IO ()
   getFile :: s -> SFileParty p -> XFTPFileId -> IO (Either XFTPErrorType (FileRec, C.APublicAuthKey))
   ackFile :: s -> RecipientId -> IO (Either XFTPErrorType ())
-  expiredFiles :: s -> Int64 -> Int -> IO [(SenderId, Maybe FilePath, Word32)]
+  expiredFiles :: s -> SystemSeconds -> Int64 -> Int -> IO [(SenderId, Maybe FilePath, Word32)]
   getUsedStorage :: s -> IO Int64
   getFileCount :: s -> IO Int
 
@@ -107,9 +108,9 @@ instance FileStoreClass STMFileStore where
 
   closeFileStore STMFileStore {stmStoreLog} = readTVarIO stmStoreLog >>= mapM_ closeStoreLog
 
-  addFile STMFileStore {files} sId fileInfo createdAt status = atomically $
+  addFile STMFileStore {files} sId fileInfo createdAt expiresAt status = atomically $
     ifM (TM.member sId files) (pure $ Left DUPLICATE_) $ do
-      f <- newFileRec sId fileInfo createdAt status
+      f <- newFileRec sId fileInfo createdAt expiresAt status
       TM.insert sId f files
       pure $ Right ()
 
@@ -166,14 +167,17 @@ instance FileStoreClass STMFileStore where
           pure $ Right ()
       _ -> pure $ Left AUTH
 
-  expiredFiles STMFileStore {files} old _limit = do
+  expiredFiles STMFileStore {files} now old _limit = do
     fs <- readTVarIO files
-    fmap catMaybes . forM (M.toList fs) $ \(sId, FileRec {fileInfo = FileInfo {size}, filePath, createdAt = RoundedSystemTime createdAt}) ->
-      if createdAt + fileTimePrecision < old
-        then do
-          path <- readTVarIO filePath
-          pure $ Just (sId, path, size)
-        else pure Nothing
+    fmap catMaybes . forM (M.toList fs) $ \(sId, FileRec {fileInfo = FileInfo {size}, filePath, createdAt = RoundedSystemTime createdAt, expiresAt}) ->
+      let expired = case expiresAt of
+            Just e -> roundedSeconds e < roundedSeconds now
+            Nothing -> createdAt + fileTimePrecision < old
+       in if expired
+            then do
+              path <- readTVarIO filePath
+              pure $ Just (sId, path, size)
+            else pure Nothing
 
   getUsedStorage STMFileStore {files} = foldM addSize 0 =<< readTVarIO files
     where
@@ -184,12 +188,12 @@ instance FileStoreClass STMFileStore where
 
 -- Internal STM helpers
 
-newFileRec :: SenderId -> FileInfo -> RoundedFileTime -> ServerEntityStatus -> STM FileRec
-newFileRec senderId fileInfo createdAt status = do
+newFileRec :: SenderId -> FileInfo -> RoundedFileTime -> Maybe RoundedFileTime -> ServerEntityStatus -> STM FileRec
+newFileRec senderId fileInfo createdAt expiresAt status = do
   recipientIds <- newTVar S.empty
   filePath <- newTVar Nothing
   fileStatus <- newTVar status
-  pure FileRec {senderId, fileInfo, filePath, recipientIds, createdAt, fileStatus}
+  pure FileRec {senderId, fileInfo, filePath, recipientIds, createdAt, expiresAt, fileStatus}
 
 withFile :: STMFileStore -> SenderId -> (FileRec -> STM (Either XFTPErrorType a)) -> STM (Either XFTPErrorType a)
 withFile STMFileStore {files} sId a =

src/Simplex/FileTransfer/Server/Store/Postgres.hs

@@ -55,6 +55,7 @@ import Simplex.Messaging.Transport (EntityId (..))
 import Simplex.Messaging.Server.QueueStore (ServerEntityStatus (..))
 import Simplex.Messaging.Server.QueueStore.Postgres ()
 import Simplex.Messaging.Server.StoreLog (openWriteStoreLog)
+import Simplex.Messaging.SystemTime (roundedSeconds)
 import Simplex.Messaging.Util (firstRow, tshow)
 import System.Directory (renameFile)
 import System.Exit (exitFailure)
@@ -82,17 +83,17 @@ instance FileStoreClass PostgresFileStore where
     closeDBStore dbStore
     mapM_ closeStoreLog dbStoreLog
 
-  addFile st sId fileInfo@FileInfo {sndKey, size, digest} createdAt status =
+  addFile st sId fileInfo@FileInfo {sndKey, size, digest} createdAt expiresAt status =
     E.uninterruptibleMask_ $ runExceptT $ do
       void $ withDB "addFile" st $ \db ->
         E.try
           ( DB.execute
               db
-              "INSERT INTO files (sender_id, file_size, file_digest, sender_key, created_at, status) VALUES (?,?,?,?,?,?)"
-              (sId, (fromIntegral size :: Int32), Binary digest, Binary (C.encodePubKey sndKey), createdAt, status)
+              "INSERT INTO files (sender_id, file_size, file_digest, sender_key, created_at, expires_at, status) VALUES (?,?,?,?,?,?,?)"
+              (sId, (fromIntegral size :: Int32), Binary digest, Binary (C.encodePubKey sndKey), createdAt, expiresAt, status)
           )
           >>= either handleDuplicate (pure . Right)
-      withLog "addFile" st $ \s -> logAddFile s sId fileInfo createdAt status
+      withLog "addFile" st $ \s -> logAddFile s sId fileInfo createdAt expiresAt status
 
   setFilePath st sId fPath = E.uninterruptibleMask_ $ runExceptT $ do
     assertUpdated $ withDB' "setFilePath" st $ \db ->
@@ -131,13 +132,13 @@ instance FileStoreClass PostgresFileStore where
 
   getFile st party fId = runExceptT $ case party of
     SFSender -> do
-      row <- loadFileRow "SELECT sender_id, file_size, file_digest, sender_key, file_path, created_at, status FROM files WHERE sender_id = ?"
+      row <- loadFileRow "SELECT sender_id, file_size, file_digest, sender_key, file_path, created_at, expires_at, status FROM files WHERE sender_id = ?"
       fr <- ExceptT $ rowToFileRec row
       pure (fr, sndKey (fileInfo fr))
     SFRecipient -> do
       row :. Only rcpKeyBs <-
         loadFileRow
-          "SELECT f.sender_id, f.file_size, f.file_digest, f.sender_key, f.file_path, f.created_at, f.status, r.recipient_key FROM files f JOIN recipients r ON r.sender_id = f.sender_id WHERE r.recipient_id = ?"
+          "SELECT f.sender_id, f.file_size, f.file_digest, f.sender_key, f.file_path, f.created_at, f.expires_at, f.status, r.recipient_key FROM files f JOIN recipients r ON r.sender_id = f.sender_id WHERE r.recipient_id = ?"
       fr <- ExceptT $ rowToFileRec row
       rcpKey <- either (const $ throwE INTERNAL) pure $ C.decodePubKey rcpKeyBs
       pure (fr, rcpKey)
@@ -152,12 +153,12 @@ instance FileStoreClass PostgresFileStore where
       DB.execute db "DELETE FROM recipients WHERE recipient_id = ?" (Only rId)
     withLog "ackFile" st $ \s -> logAckFile s rId
 
-  expiredFiles st old limit =
+  expiredFiles st now old limit =
     fmap toResult $ withTransaction (dbStore st) $ \db ->
       DB.query
         db
-        "SELECT sender_id, file_path, file_size FROM files WHERE created_at + ? < ? ORDER BY created_at LIMIT ?"
-        (fileTimePrecision, old, limit)
+        "(SELECT sender_id, file_path, file_size FROM files WHERE expires_at < ? LIMIT ?) UNION ALL (SELECT sender_id, file_path, file_size FROM files WHERE expires_at IS NULL AND created_at < ? LIMIT ?)"
+        (roundedSeconds now, limit, old - fileTimePrecision, limit)
     where
       toResult :: [(SenderId, Maybe FilePath, Int32)] -> [(SenderId, Maybe FilePath, Word32)]
       toResult = map (\(sId, path, size) -> (sId, path, fromIntegral size))
@@ -174,21 +175,21 @@ instance FileStoreClass PostgresFileStore where
 
 -- Internal helpers
 
-mkFileRec :: SenderId -> FileInfo -> Maybe FilePath -> RoundedFileTime -> ServerEntityStatus -> IO FileRec
-mkFileRec senderId fileInfo path createdAt status = do
+mkFileRec :: SenderId -> FileInfo -> Maybe FilePath -> RoundedFileTime -> Maybe RoundedFileTime -> ServerEntityStatus -> IO FileRec
+mkFileRec senderId fileInfo path createdAt expiresAt status = do
   filePath <- newTVarIO path
   recipientIds <- newTVarIO S.empty
   fileStatus <- newTVarIO status
-  pure FileRec {senderId, fileInfo, filePath, recipientIds, createdAt, fileStatus}
+  pure FileRec {senderId, fileInfo, filePath, recipientIds, createdAt, expiresAt, fileStatus}
 
-type FileRecRow = (SenderId, Int32, ByteString, ByteString, Maybe FilePath, RoundedFileTime, ServerEntityStatus)
+type FileRecRow = (SenderId, Int32, ByteString, ByteString, Maybe FilePath, RoundedFileTime, Maybe RoundedFileTime, ServerEntityStatus)
 
 rowToFileRec :: FileRecRow -> IO (Either XFTPErrorType FileRec)
-rowToFileRec (sId, size, digest, sndKeyBs, path, createdAt, status) =
+rowToFileRec (sId, size, digest, sndKeyBs, path, createdAt, expiresAt, status) =
   case C.decodePubKey sndKeyBs of
     Right sndKey -> do
       let fileInfo = FileInfo {sndKey, size = fromIntegral size, digest}
-      Right <$> mkFileRec sId fileInfo path createdAt status
+      Right <$> mkFileRec sId fileInfo path createdAt expiresAt status
     Left _ -> pure $ Left INTERNAL
 
 -- DB helpers
@@ -243,7 +244,7 @@ importFileStore storeLogFilePath dbCfg = do
   fCnt <- withTransaction (dbStore pgStore) $ \db -> do
     DB.copy_
       db
-      "COPY files (sender_id, file_size, file_digest, sender_key, file_path, created_at, status) FROM STDIN WITH (FORMAT csv)"
+      "COPY files (sender_id, file_size, file_digest, sender_key, file_path, created_at, expires_at, status) FROM STDIN WITH (FORMAT csv)"
     iforM_ (M.toList allFiles) $ \i (sId, fr) -> do
       DB.putCopyData db =<< fileRecToCSV sId fr
       when (i > 0 && i `mod` 10000 == 0) $ putStr ("  " <> show i <> " files\r") >> hFlush stdout
@@ -282,13 +283,13 @@ exportFileStore storeLogFilePath dbCfg = do
   !fCnt <- withTransaction (dbStore pgStore) $ \db ->
     DB.fold_
       db
-      "SELECT sender_id, file_size, file_digest, sender_key, file_path, created_at, status FROM files ORDER BY created_at"
+      "SELECT sender_id, file_size, file_digest, sender_key, file_path, created_at, expires_at, status FROM files ORDER BY created_at"
       (0 :: Int)
-      ( \(!fc) (sId, size :: Int32, digest :: ByteString, sndKeyBs :: ByteString, path :: Maybe String, createdAt, status) ->
+      ( \(!fc) (sId, size :: Int32, digest :: ByteString, sndKeyBs :: ByteString, path :: Maybe String, createdAt, expiresAt, status) ->
           case C.decodePubKey sndKeyBs of
             Right sndKey -> do
               let fileInfo = FileInfo {sndKey, size = fromIntegral size, digest}
-              logAddFile sl sId fileInfo createdAt status
+              logAddFile sl sId fileInfo createdAt expiresAt status
               forM_ path $ logPutFile sl sId
               pure (fc + 1)
             Left _ -> do
@@ -326,7 +327,7 @@ iforM_ :: Monad m => [a] -> (Int -> a -> m ()) -> m ()
 iforM_ xs f = zipWithM_ f [0 ..] xs
 
 fileRecToCSV :: SenderId -> FileRec -> IO ByteString
-fileRecToCSV sId FileRec {fileInfo = FileInfo {sndKey, size, digest}, filePath, createdAt, fileStatus} = do
+fileRecToCSV sId FileRec {fileInfo = FileInfo {sndKey, size, digest}, filePath, createdAt, expiresAt, fileStatus} = do
   path <- readTVarIO filePath
   status <- readTVarIO fileStatus
   pure $ LB.toStrict $ BB.toLazyByteString $ mconcat (BB.char7 ',' `intersperse` fields path status) <> BB.char7 '\n'
@@ -338,6 +339,7 @@ fileRecToCSV sId FileRec {fileInfo = FileInfo {sndKey, size, digest}, filePath,
         renderField (toField (Binary (C.encodePubKey sndKey))),
         nullable (toField <$> path),
         renderField (toField createdAt),
+        nullable (toField <$> expiresAt),
         quotedField (toField status)
       ]
 

src/Simplex/FileTransfer/Server/Store/Postgres/Migrations.hs

@@ -14,7 +14,8 @@ import Text.RawString.QQ (r)
 
 xftpSchemaMigrations :: [(String, Text, Maybe Text)]
 xftpSchemaMigrations =
-  [ ("20260325_initial", m20260325_initial, Nothing)
+  [ ("20260325_initial", m20260325_initial, Nothing),
+    ("20260823_file_expiration", m20260823_file_expiration, Just down_m20260823_file_expiration)
   ]
 
 -- | The list of migrations in ascending order by date
@@ -45,3 +46,17 @@ CREATE TABLE recipients (
 CREATE INDEX idx_recipients_sender_id ON recipients (sender_id);
 CREATE INDEX idx_files_created_at ON files (created_at);
 |]
+
+m20260823_file_expiration :: Text
+m20260823_file_expiration =
+  [r|
+ALTER TABLE files ADD COLUMN expires_at BIGINT;
+CREATE INDEX idx_files_expiry ON files (expires_at, created_at);
+|]
+
+down_m20260823_file_expiration :: Text
+down_m20260823_file_expiration =
+  [r|
+DROP INDEX idx_files_expiry;
+ALTER TABLE files DROP COLUMN expires_at;
+|]

src/Simplex/FileTransfer/Server/StoreLog.hs

@@ -17,16 +17,18 @@ module Simplex.FileTransfer.Server.StoreLog
     logDeleteFile,
     logBlockFile,
     logAckFile,
+    serverFileName,
   )
 where
 
-import Control.Applicative ((<|>))
+import Control.Applicative (optional, (<|>))
 import Control.Concurrent.STM
 import Control.Monad.Except
 import qualified Data.Attoparsec.ByteString.Char8 as A
+import qualified Data.ByteString.Base64.URL as B64
 import qualified Data.ByteString.Char8 as B
 import qualified Data.ByteString.Lazy.Char8 as LB
-import Data.Composition ((.:), (.::))
+import Data.Composition ((.:), (.::.))
 import Data.List.NonEmpty (NonEmpty)
 import qualified Data.List.NonEmpty as L
 import Data.Map.Strict (Map)
@@ -35,14 +37,15 @@ import Simplex.FileTransfer.Protocol (FileInfo (..))
 import Simplex.FileTransfer.Server.Store
 import Simplex.FileTransfer.Transport (XFTPErrorType (..))
 import Simplex.Messaging.Encoding.String
-import Simplex.Messaging.Protocol (BlockingInfo, RcvPublicAuthKey, RecipientId, SenderId)
+import Simplex.Messaging.Protocol (BlockingInfo, EntityId (..), RcvPublicAuthKey, RecipientId, SenderId)
 import Simplex.Messaging.Server.QueueStore (ServerEntityStatus (..))
 import Simplex.Messaging.Server.StoreLog
 import Simplex.Messaging.Util (bshow)
+import System.FilePath (takeFileName)
 import System.IO
 
 data FileStoreLogRecord
-  = AddFile SenderId FileInfo RoundedFileTime ServerEntityStatus
+  = AddFile SenderId FileInfo RoundedFileTime (Maybe RoundedFileTime) ServerEntityStatus
   | PutFile SenderId FilePath
   | AddRecipients SenderId (NonEmpty FileRecipient)
   | DeleteFile SenderId
@@ -52,27 +55,37 @@ data FileStoreLogRecord
 
 instance StrEncoding FileStoreLogRecord where
   strEncode = \case
-    AddFile sId file createdAt status -> strEncode (Str "FNEW", sId, file, createdAt, status)
+    AddFile sId file createdAt expiresAt status -> B.concat [strEncode (Str "FNEW", sId, file, createdAt), expE expiresAt, " ", strEncode status]
     PutFile sId path -> strEncode (Str "FPUT", sId, path)
     AddRecipients sId rcps -> strEncode (Str "FADD", sId, rcps)
     DeleteFile sId -> strEncode (Str "FDEL", sId)
     BlockFile sId info -> strEncode (Str "FBLK", sId, info)
     AckFile rId -> strEncode (Str "FACK", rId)
+    where
+      expE = maybe "" ((" " <>) . strEncode)
   strP =
     A.choice
-      [ "FNEW " *> (AddFile <$> strP_ <*> strP_ <*> strP <*> (_strP <|> pure EntityActive)),
+      [ "FNEW " *> addFileP,
         "FPUT " *> (PutFile <$> strP_ <*> strP),
         "FADD " *> (AddRecipients <$> strP_ <*> strP),
         "FDEL " *> (DeleteFile <$> strP),
         "FBLK " *> (BlockFile <$> strP_ <*> strP),
         "FACK " *> (AckFile <$> strP)
       ]
+    where
+      addFileP = do
+        sId <- strP_
+        file <- strP_
+        createdAt <- strP
+        expiresAt <- optional _strP
+        status <- _strP <|> pure EntityActive
+        pure $ AddFile sId file createdAt expiresAt status
 
 logFileStoreRecord :: StoreLog 'WriteMode -> FileStoreLogRecord -> IO ()
 logFileStoreRecord = writeStoreLogRecord
 
-logAddFile :: StoreLog 'WriteMode -> SenderId -> FileInfo -> RoundedFileTime -> ServerEntityStatus -> IO ()
-logAddFile s = logFileStoreRecord s .:: AddFile
+logAddFile :: StoreLog 'WriteMode -> SenderId -> FileInfo -> RoundedFileTime -> Maybe RoundedFileTime -> ServerEntityStatus -> IO ()
+logAddFile s = logFileStoreRecord s .::. AddFile
 
 logPutFile :: StoreLog 'WriteMode -> SenderId -> FilePath -> IO ()
 logPutFile s = logFileStoreRecord s .: PutFile
@@ -89,6 +102,9 @@ logBlockFile s fId = logFileStoreRecord s . BlockFile fId
 logAckFile :: StoreLog 'WriteMode -> RecipientId -> IO ()
 logAckFile s = logFileStoreRecord s . AckFile
 
+serverFileName :: SenderId -> FilePath
+serverFileName = B.unpack . B64.encode . unEntityId
+
 readWriteFileStore :: FilePath -> STMFileStore -> IO (StoreLog 'WriteMode)
 readWriteFileStore = readWriteStoreLog readFileStore writeFileStore
 
@@ -102,10 +118,12 @@ readFileStore f st = mapM_ (addFileLogRecord . LB.toStrict) . LB.lines =<< LB.re
           Left e -> B.putStrLn $ "Log processing error (" <> bshow e <> "): " <> B.take 100 s
           _ -> pure ()
     addToStore = \case
-      AddFile sId file createdAt status
-        | size file > 0 -> addFile st sId file createdAt status
+      AddFile sId file createdAt expiresAt status
+        | size file > 0 -> addFile st sId file createdAt expiresAt status
         | otherwise -> pure $ Left SIZE
-      PutFile qId path -> setFilePath st qId path
+      PutFile qId path
+        | takeFileName path == serverFileName qId -> setFilePath st qId path
+        | otherwise -> pure $ Left AUTH
       AddRecipients sId rcps -> runExceptT $ addRecipients sId rcps
       DeleteFile sId -> deleteFile st sId
       BlockFile sId info -> blockFile st sId info True
@@ -118,9 +136,9 @@ writeFileStore s STMFileStore {files, recipients} = do
   readTVarIO files >>= mapM_ (logFile allRcps)
   where
     logFile :: Map RecipientId (SenderId, RcvPublicAuthKey) -> FileRec -> IO ()
-    logFile allRcps FileRec {senderId, fileInfo, filePath, recipientIds, createdAt, fileStatus} = do
+    logFile allRcps FileRec {senderId, fileInfo, filePath, recipientIds, createdAt, expiresAt, fileStatus} = do
       status <- readTVarIO fileStatus
-      logAddFile s senderId fileInfo createdAt status
+      logAddFile s senderId fileInfo createdAt expiresAt status
       (rcpErrs, rcps) <- M.mapEither getRcp . M.fromSet id <$> readTVarIO recipientIds
       mapM_ (logAddRecipients s senderId) $ L.nonEmpty $ M.elems rcps
       mapM_ (B.putStrLn . ("Error storing log: " <>)) rcpErrs

src/Simplex/FileTransfer/Transport.hs

@@ -12,6 +12,7 @@ module Simplex.FileTransfer.Transport
   ( supportedFileServerVRange,
     authCmdsXFTPVersion,
     blockedFilesXFTPVersion,
+    fileStorageTimeXFTPVersion,
     xftpClientHandshakeStub,
     alpnSupportedXFTPhandshakes,
     xftpALPNv1,
@@ -36,7 +37,6 @@ module Simplex.FileTransfer.Transport
   )
 where
 
-import Control.Applicative (optional)
 import qualified Control.Exception as E
 import Control.Logger.Simple
 import Control.Monad
@@ -56,13 +56,14 @@ import Data.Word (Word16, Word32)
 import Network.HTTP2.Client (HTTP2Error)
 import qualified Simplex.Messaging.Crypto as C
 import qualified Simplex.Messaging.Crypto.Lazy as LC
+import Simplex.Messaging.Crypto.Entitlement (EntitlementProof)
 import Simplex.Messaging.Encoding
 import Simplex.Messaging.Encoding.String
 import Simplex.Messaging.Parsers
 import Simplex.Messaging.Protocol (BlockingInfo, CommandError)
 import Simplex.Messaging.Transport (ALPN, CertChainPubKey, ServiceCredentials, SessionId, THandle (..), THandleParams (..), TransportError (..), TransportPeer (..))
 import Simplex.Messaging.Transport.HTTP2.File
-import Simplex.Messaging.Util (bshow, tshow, (<$?>))
+import Simplex.Messaging.Util (bshow, tshow, (<$?>), (<$$>))
 import Simplex.Messaging.Version
 import Simplex.Messaging.Version.Internal
 import System.IO (Handle, IOMode (..), withFile)
@@ -97,8 +98,11 @@ authCmdsXFTPVersion = VersionXFTP 2
 blockedFilesXFTPVersion :: VersionXFTP
 blockedFilesXFTPVersion = VersionXFTP 3
 
+fileStorageTimeXFTPVersion :: VersionXFTP
+fileStorageTimeXFTPVersion = VersionXFTP 4
+
 currentXFTPVersion :: VersionXFTP
-currentXFTPVersion = VersionXFTP 3
+currentXFTPVersion = VersionXFTP 4
 
 supportedFileServerVRange :: VersionRangeXFTP
 supportedFileServerVRange = mkVersionRange initialXFTPVersion currentXFTPVersion
@@ -124,14 +128,18 @@ data XFTPServerHandshake = XFTPServerHandshake
     -- | pub key to agree shared secrets for command authorization and entity ID encryption.
     authPubKey :: CertChainPubKey,
     -- | signed identity challenge from  XFTPClientHello
-    webIdentityProof :: Maybe C.ASignature
+    webIdentityProof :: Maybe C.ASignature,
+    -- | optional server public information (JSON-encoded ServerPublicInfo), sent when version >= fileStorageTimeXFTPVersion
+    serverInfoBytes :: Maybe ByteString
   }
 
 data XFTPClientHandshake = XFTPClientHandshake
   { -- | agreed XFTP server protocol version
     xftpVersion :: VersionXFTP,
     -- | server identity - CA certificate fingerprint
-    keyHash :: C.KeyHash
+    keyHash :: C.KeyHash,
+    -- | proof of the user entitlement bound to the session
+    entitlementProof :: Maybe EntitlementProof
   }
 
 instance Encoding XFTPClientHello where
@@ -143,21 +151,33 @@ instance Encoding XFTPClientHello where
     pure XFTPClientHello {webChallenge}
 
 instance Encoding XFTPClientHandshake where
-  smpEncode XFTPClientHandshake {xftpVersion, keyHash} =
-    smpEncode (xftpVersion, keyHash)
+  smpEncode XFTPClientHandshake {xftpVersion = v, keyHash, entitlementProof} =
+    smpEncode (v, keyHash) <> ifHasEntitlement v (smpEncode entitlementProof) ""
   smpP = do
-    (xftpVersion, keyHash) <- smpP
+    (v, keyHash) <- smpP
+    entitlementProof <- ifHasEntitlement v smpP (pure Nothing)
     Tail _compat <- smpP
-    pure XFTPClientHandshake {xftpVersion, keyHash}
+    pure XFTPClientHandshake {xftpVersion = v, keyHash, entitlementProof}
+
+ifHasEntitlement :: VersionXFTP -> a -> a -> a
+ifHasEntitlement v a b = if v >= fileStorageTimeXFTPVersion then a else b
 
 instance Encoding XFTPServerHandshake where
-  smpEncode XFTPServerHandshake {xftpVersionRange, sessionId, authPubKey, webIdentityProof} =
-    smpEncode (xftpVersionRange, sessionId, authPubKey, C.signatureBytes webIdentityProof)
+  smpEncode XFTPServerHandshake {xftpVersionRange, sessionId, authPubKey, webIdentityProof, serverInfoBytes} =
+    smpEncode (xftpVersionRange, sessionId, authPubKey, C.signatureBytes webIdentityProof) <> info
+    where
+      info = ifHasServerInfo (maxVersion xftpVersionRange) (smpEncode (Large <$> serverInfoBytes)) ""
   smpP = do
     (xftpVersionRange, sessionId, authPubKey) <- smpP
-    webIdentityProof <- optional $ C.decodeSignature <$?> smpP
+    -- decode the (length-prefixed) signature bytes deterministically: empty bytes decode to Nothing.
+    -- (Must not use `optional`, which would backtrack and leave the bytes for the parsers that follow.)
+    webIdentityProof <- C.decodeSignature <$?> smpP
+    serverInfoBytes <- ifHasServerInfo (maxVersion xftpVersionRange) (unLarge <$$> smpP) (pure Nothing)
     Tail _compat <- smpP
-    pure XFTPServerHandshake {xftpVersionRange, sessionId, authPubKey, webIdentityProof}
+    pure XFTPServerHandshake {xftpVersionRange, sessionId, authPubKey, webIdentityProof, serverInfoBytes}
+
+ifHasServerInfo :: VersionXFTP -> a -> a -> a
+ifHasServerInfo v a b = if v >= fileStorageTimeXFTPVersion then a else b
 
 sendEncFile :: Handle -> (Builder -> IO ()) -> LC.SbState -> Word32 -> IO ()
 sendEncFile h send = go

src/Simplex/FileTransfer/Types.hs

@@ -39,6 +39,7 @@ import Data.Text.Encoding (encodeUtf8)
 import Data.Word (Word32)
 import Simplex.FileTransfer.Client (XFTPChunkSpec (..))
 import Simplex.FileTransfer.Description
+import Simplex.FileTransfer.Protocol (GrantedStorageTime (..))
 import Simplex.Messaging.Agent.Store.DB (FromField (..), ToField (..), fromTextField_)
 import qualified Simplex.Messaging.Crypto as C
 import Simplex.Messaging.Crypto.File (CryptoFile (..))
@@ -92,7 +93,8 @@ data RcvFile = RcvFile
   deriving (Show)
 
 data RcvFileStatus
-  = RFSReceiving
+  = RFSPrepared
+  | RFSReceiving
   | RFSReceived
   | RFSDecrypting
   | RFSComplete
@@ -105,6 +107,7 @@ instance ToField RcvFileStatus where toField = toField . textEncode
 
 instance TextEncoding RcvFileStatus where
   textDecode = \case
+    "prepared" -> Just RFSPrepared
     "receiving" -> Just RFSReceiving
     "received" -> Just RFSReceived
     "decrypting" -> Just RFSDecrypting
@@ -112,6 +115,7 @@ instance TextEncoding RcvFileStatus where
     "error" -> Just RFSError
     _ -> Nothing
   textEncode = \case
+    RFSPrepared -> "prepared"
     RFSReceiving -> "receiving"
     RFSReceived -> "received"
     RFSDecrypting -> "decrypting"
@@ -167,7 +171,8 @@ data SndFile = SndFile
     prefixPath :: Maybe FilePath,
     status :: SndFileStatus,
     deleted :: Bool,
-    redirect :: Maybe RedirectFileInfo
+    redirect :: Maybe RedirectFileInfo,
+    storageHours :: Maybe Word32
   }
   deriving (Show)
 
@@ -175,7 +180,8 @@ sndFileEncPath :: FilePath -> FilePath
 sndFileEncPath prefixPath = prefixPath </> "xftp.encrypted"
 
 data SndFileStatus
-  = SFSNew -- db record created
+  = SFSPrepared
+  | SFSNew -- db record created
   | SFSEncrypting -- encryption started
   | SFSEncrypted -- encryption complete
   | SFSUploading -- all chunk replicas are created on servers
@@ -189,6 +195,7 @@ instance ToField SndFileStatus where toField = toField . textEncode
 
 instance TextEncoding SndFileStatus where
   textDecode = \case
+    "prepared" -> Just SFSPrepared
     "new" -> Just SFSNew
     "encrypting" -> Just SFSEncrypting
     "encrypted" -> Just SFSEncrypted
@@ -197,6 +204,7 @@ instance TextEncoding SndFileStatus where
     "error" -> Just SFSError
     _ -> Nothing
   textEncode = \case
+    SFSPrepared -> "prepared"
     SFSNew -> "new"
     SFSEncrypting -> "encrypting"
     SFSEncrypted -> "encrypted"
@@ -225,7 +233,8 @@ data NewSndChunkReplica = NewSndChunkReplica
   { server :: XFTPServer,
     replicaId :: ChunkReplicaId,
     replicaKey :: C.APrivateAuthKey,
-    rcvIdsKeys :: [(ChunkReplicaId, C.APrivateAuthKey)]
+    rcvIdsKeys :: [(ChunkReplicaId, C.APrivateAuthKey)],
+    expiresAt :: Maybe GrantedStorageTime
   }
   deriving (Show)
 
@@ -237,7 +246,8 @@ data SndFileChunkReplica = SndFileChunkReplica
     rcvIdsKeys :: [(ChunkReplicaId, C.APrivateAuthKey)],
     replicaStatus :: SndFileReplicaStatus,
     delay :: Maybe Int64,
-    retries :: Int
+    retries :: Int,
+    expiresAt :: Maybe GrantedStorageTime
   }
   deriving (Show)
 

src/Simplex/Messaging/Agent/Client.hs

@@ -41,6 +41,7 @@ module Simplex.Messaging.Agent.Client
     reconnectServerClients,
     reconnectSMPServer,
     closeXFTPServerClient,
+    closeUserXFTPClients,
     runSMPServerTest,
     runXFTPServerTest,
     runNTFServerTest,
@@ -227,7 +228,7 @@ import Data.Text (Text)
 import Data.Text.Encoding
 import Data.Time (UTCTime, addUTCTime, defaultTimeLocale, formatTime, getCurrentTime)
 import Data.Time.Clock.System (getSystemTime)
-import Data.Word (Word16)
+import Data.Word (Word16, Word32)
 import qualified Data.X509.Validation as XV
 import Network.Socket (HostName)
 import Simplex.FileTransfer.Client (XFTPChunkSpec (..), XFTPClient, XFTPClientConfig (..), XFTPClientError)
@@ -253,6 +254,8 @@ import Simplex.Messaging.Agent.TSessionSubs (TSessionSubs)
 import qualified Simplex.Messaging.Agent.TSessionSubs as SS
 import Simplex.Messaging.Client
 import qualified Simplex.Messaging.Crypto as C
+import Simplex.Messaging.Crypto.BBS (BBSPresHeader (..), BBSPublicKey)
+import Simplex.Messaging.Crypto.Entitlement (EntitlementCredential, EntitlementProof, generateEntitlementProof)
 import Simplex.Messaging.Encoding
 import Simplex.Messaging.Encoding.String
 import Simplex.Messaging.Notifications.Client
@@ -269,7 +272,7 @@ import Simplex.Messaging.Protocol
     NetworkError (..),
     MsgFlags (..),
     MsgId,
-    NameRecord,
+    NameResponse,
     NtfServer,
     NtfServerWithAuth,
     ProtoServer,
@@ -309,7 +312,7 @@ import Simplex.Messaging.Session
 import Simplex.Messaging.SystemTime
 import Simplex.Messaging.TMap (TMap)
 import qualified Simplex.Messaging.TMap as TM
-import Simplex.Messaging.Transport (HandshakeError (..), SMPServiceRole (..), SMPVersion, ServiceCredentials (..), SessionId, THClientService' (..), THandleAuth (..), THandleParams (sessionId, thAuth, thVersion, serverInfo), TransportError (..), TransportPeer (..), shortLinksSMPVersion, newNtfCredsSMPVersion)
+import Simplex.Messaging.Transport (HandshakeError (..), SMPServiceRole (..), SMPVersion, ServiceCredentials (..), SessionId, THClientService' (..), THandleAuth (..), THandleParams (sessionId, thAuth, thVersion, serverInfo), TransportError (..), newNtfCredsSMPVersion)
 import Simplex.Messaging.Transport.Client (TransportHost (..))
 import Simplex.Messaging.Transport.Credentials
 import Simplex.Messaging.Util
@@ -354,6 +357,7 @@ data AgentClient = AgentClient
     ntfClients :: TMap NtfTransportSession NtfClientVar,
     xftpServers :: TMap UserId (UserServers 'PXFTP),
     xftpClients :: TMap XFTPTransportSession XFTPClientVar,
+    userEntitlements :: TMap UserId EntitlementCredential,
     useNetworkConfig :: TVar (NetworkConfig, NetworkConfig), -- (slow, fast) networks
     presetDomains :: [HostName],
     presetServers :: [SMPServer],
@@ -511,7 +515,7 @@ data UserNetworkType = UNNone | UNCellular | UNWifi | UNEthernet | UNOther
 
 -- | Creates an SMP agent client instance that receives commands and sends responses via 'TBQueue's.
 newAgentClient :: Int -> InitialAgentServers -> UTCTime -> Map (Maybe SMPServer) (Maybe SystemSeconds) -> Env -> IO AgentClient
-newAgentClient clientId InitialAgentServers {smp, ntf, xftp, netCfg, useServices, presetDomains, presetServers} currentTs notices agentEnv = do
+newAgentClient clientId InitialAgentServers {smp, ntf, xftp, entitlements, netCfg, useServices, presetDomains, presetServers} currentTs notices agentEnv = do
   let cfg = config agentEnv
       qSize = tbqSize cfg
   proxySessTs <- newTVarIO =<< getCurrentTime
@@ -527,6 +531,7 @@ newAgentClient clientId InitialAgentServers {smp, ntf, xftp, netCfg, useServices
   ntfClients <- TM.emptyIO
   xftpServers <- newTVarIO $ M.map mkUserServers xftp
   xftpClients <- TM.emptyIO
+  userEntitlements <- newTVarIO entitlements
   useNetworkConfig <- newTVarIO (slowNetworkConfig netCfg, netCfg)
   userNetworkInfo <- newTVarIO $ UserNetworkInfo UNOther True
   userNetworkUpdated <- newTVarIO Nothing
@@ -568,6 +573,7 @@ newAgentClient clientId InitialAgentServers {smp, ntf, xftp, netCfg, useServices
         ntfClients,
         xftpServers,
         xftpClients,
+        userEntitlements,
         useNetworkConfig,
         presetDomains,
         presetServers,
@@ -878,7 +884,7 @@ getNtfServerClient c@AgentClient {active, ntfClients, workerSeq, proxySessTs, pr
       logInfo . decodeUtf8 $ "Agent disconnected from " <> showServer srv
 
 getXFTPServerClient :: AgentClient -> XFTPTransportSession -> AM XFTPClient
-getXFTPServerClient c@AgentClient {active, xftpClients, workerSeq, proxySessTs, presetDomains} tSess@(_, srv, _) = do
+getXFTPServerClient c@AgentClient {active, xftpClients, userEntitlements, workerSeq, proxySessTs, presetDomains} tSess@(userId, srv, _) = do
   unlessM (readTVarIO active) $ throwE INACTIVE
   ts <- liftIO getCurrentTime
   withGetSessVar workerSeq tSess xftpClients ts (newProtocolClient c tSess xftpClients connectClient) (waitForProtocolClient c NRMBackground tSess xftpClients)
@@ -886,12 +892,28 @@ getXFTPServerClient c@AgentClient {active, xftpClients, workerSeq, proxySessTs,
     connectClient :: XFTPClientVar -> AM XFTPClient
     connectClient v = do
       cfg <- asks $ xftpCfg . config
+      keys <- asks $ entitlementKeys . config
       xftpNetworkConfig <- getNetworkConfig c
       ts <- readTVarIO proxySessTs
       liftError' (protocolClientError XFTP $ B.unpack $ strEncode srv) $
-        X.getXFTPClient tSess cfg {xftpNetworkConfig} presetDomains ts $
+        X.getXFTPClient tSess cfg {xftpNetworkConfig} presetDomains ts (mkEntitlementProof keys) $
           clientDisconnected v
 
+    mkEntitlementProof :: Map Word16 BBSPublicKey -> SessionId -> IO (Maybe EntitlementProof)
+    mkEntitlementProof keys sessId =
+      ifM knownServer proof (pure Nothing)
+      where
+        -- the entitlement is presented only to the servers of this user, matched by key hash that TLS pins,
+        -- so a file description of the sender cannot direct it to another server
+        knownServer = maybe False (any (sameKeyHash . snd) . storageSrvs) <$> TM.lookupIO userId (xftpServers c)
+        sameKeyHash (ProtoServerWithAuth srv' _) = srvKeyHash srv' == srvKeyHash srv
+        srvKeyHash (ProtocolServer _ _ _ kh) = kh
+        proof =
+          TM.lookupIO userId userEntitlements $>>= \cred ->
+            generateEntitlementProof keys cred (BBSPresHeader sessId) >>= \case
+              Right p -> pure $ Just p
+              Left e -> Nothing <$ logError ("entitlement proof error: " <> tshow e)
+
     clientDisconnected :: XFTPClientVar -> XFTPClient -> IO ()
     clientDisconnected v client = do
       atomically $ removeSessVar v tSess xftpClients
@@ -1037,6 +1059,13 @@ reconnectSMPServer c userId srv = do
       | userId == userId' && srv == srv' = (v :)
       | otherwise = id
 
+closeUserXFTPClients :: AgentClient -> UserId -> IO ()
+closeUserXFTPClients c userId = do
+  vs <- atomically $ stateTVar (xftpClients c) $ \cs ->
+    let (userCs, cs') = M.partitionWithKey (\(userId', _, _) _ -> userId == userId') cs
+     in (M.elems userCs, cs')
+  mapM_ (forkIO . closeClient_ c) vs
+
 closeClient :: ProtocolServerClient v err msg => AgentClient -> (AgentClient -> TMap (TransportSession msg) (ClientVar msg)) -> TransportSession msg -> IO ()
 closeClient c clientSel tSess =
   atomically (TM.lookupDelete tSess $ clientSel c) >>= mapM_ (closeClient_ c)
@@ -1232,9 +1261,12 @@ sendOrProxySMPCommand c nm userId destSrv@ProtocolServer {host = destHosts} conn
           Left e -> throwE e
 
 ipAddressProtected :: NetworkConfig -> ProtocolServer p -> Bool
-ipAddressProtected NetworkConfig {socksProxy, hostMode} (ProtocolServer _ hosts _ _) = do
-  isJust socksProxy || (hostMode == HMOnion && any isOnionHost hosts)
+ipAddressProtected NetworkConfig {socksProxy, socksMode, hostMode} (ProtocolServer _ hosts _ _)
+  | isJust socksProxy = socksMode == SMAlways || if hostMode == HMPublic then allOnion else anyOnion
+  | otherwise = hostMode == HMOnion && anyOnion
   where
+    anyOnion = any isOnionHost hosts
+    allOnion = all isOnionHost hosts
     isOnionHost = \case THOnionHost _ -> True; _ -> False
 
 withNtfClient :: AgentClient -> NetworkRequestMode -> NtfServer -> EntityId -> ByteString -> (NtfClient -> ExceptT NtfClientError IO a) -> AM a
@@ -1326,7 +1358,7 @@ runSMPServerTest c@AgentClient {presetDomains} nm userId (ProtoServerWithAuth sr
     testErr :: ProtocolTestStep -> SMPClientError -> ProtocolTestFailure
     testErr step = ProtocolTestFailure step . protocolClientError SMP addr
 
-runXFTPServerTest :: AgentClient -> NetworkRequestMode -> UserId -> XFTPServerWithAuth -> AM' (Maybe ProtocolTestFailure)
+runXFTPServerTest :: AgentClient -> NetworkRequestMode -> UserId -> XFTPServerWithAuth -> AM' (Either ProtocolTestFailure (Maybe (Either String ServerPublicInfo)))
 runXFTPServerTest c@AgentClient {presetDomains} nm userId (ProtoServerWithAuth srv auth) = do
   cfg <- asks $ xftpCfg . config
   g <- asks random
@@ -1337,7 +1369,7 @@ runXFTPServerTest c@AgentClient {presetDomains} nm userId (ProtoServerWithAuth s
   liftIO $ do
     let tSess = (userId, srv, Nothing)
     ts <- readTVarIO $ proxySessTs c
-    X.getXFTPClient tSess cfg {xftpNetworkConfig} presetDomains ts (\_ -> pure ()) >>= \case
+    X.getXFTPClient tSess cfg {xftpNetworkConfig} presetDomains ts (\_ -> pure Nothing) (\_ -> pure ()) >>= \case
       Right xftp -> withTestChunk filePath $ do
         (sndKey, spKey) <- atomically $ C.generateAuthKeyPair C.SEd25519 g
         (rcvKey, rpKey) <- atomically $ C.generateAuthKeyPair C.SEd25519 g
@@ -1345,15 +1377,15 @@ runXFTPServerTest c@AgentClient {presetDomains} nm userId (ProtoServerWithAuth s
         let file = FileInfo {sndKey, size = chSize, digest}
             chunkSpec = X.XFTPChunkSpec {filePath, chunkOffset = 0, chunkSize = chSize}
         r <- runExceptT $ do
-          (sId, [rId]) <- liftError (testErr TSCreateFile) $ X.createXFTPChunk xftp spKey file [rcvKey] auth
+          (sId, [rId], _) <- liftError (testErr TSCreateFile) $ X.createXFTPChunk xftp spKey file [rcvKey] auth Nothing
           liftError (testErr TSUploadFile) $ X.uploadXFTPChunk xftp spKey sId chunkSpec
           liftError (testErr TSDownloadFile) $ X.downloadXFTPChunk g xftp rpKey rId $ XFTPRcvChunkSpec rcvPath chSize digest
           rcvDigest <- liftIO $ C.sha256Hash <$> B.readFile rcvPath
           unless (digest == rcvDigest) $ throwE $ ProtocolTestFailure TSCompareFile $ XFTP (B.unpack $ strEncode srv) DIGEST
           liftError (testErr TSDeleteFile) $ X.deleteXFTPChunk xftp spKey sId
         ok <- netTimeoutInt (tcpTimeout xftpNetworkConfig) nm `timeout` X.closeXFTPClient xftp
-        pure $ either Just (const Nothing) r <|> maybe (Just (ProtocolTestFailure TSDisconnect $ BROKER addr TIMEOUT)) (const Nothing) ok
-      Left e -> pure (Just $ testErr TSConnect e)
+        pure $ r >> maybe (Left (ProtocolTestFailure TSDisconnect $ BROKER addr TIMEOUT)) (const $ Right $ serverInfo (X.thParams xftp)) ok
+      Left e -> pure $ Left (testErr TSConnect e)
   where
     addr = B.unpack $ strEncode srv
     testErr :: ProtocolTestStep -> XFTPClientError -> ProtocolTestFailure
@@ -1467,7 +1499,7 @@ newRcvQueue_ c nm userId connId (ProtoServerWithAuth srv auth) vRange cqrd enabl
   let sessServiceId = (\THClientService {serviceId = sId} -> sId) <$> (clientService =<< thAuth thParams')
   when (isJust serviceId && serviceId /= sessServiceId) $ logError "incorrect service ID in NEW response"
   liftIO . logServer "<--" c srv NoEntity $ B.unwords ["IDS", logSecret rcvId, logSecret sndId]
-  shortLink <- mkShortLinkCreds thParams' qik
+  shortLink <- mkShortLinkCreds qik
   let rq =
         RcvQueue
           { userId,
@@ -1508,8 +1540,8 @@ newRcvQueue_ c nm userId connId (ProtoServerWithAuth srv auth) vRange cqrd enabl
       (Just ((ntfPublicKey, ntfPrivateKey), dhpk), Just (ServerNtfCreds notifierId dhk')) ->
         Just ClientNtfCreds {ntfPublicKey, ntfPrivateKey, notifierId, rcvNtfDhSecret = C.dh' dhk' dhpk}
       _ -> Nothing
-    mkShortLinkCreds :: THandleParams SMPVersion 'TClient -> QueueIdsKeys -> AM (Maybe ShortLinkCreds)
-    mkShortLinkCreds thParams' QIK {sndId, queueMode, linkId} = case (cqrd, queueMode) of
+    mkShortLinkCreds :: QueueIdsKeys -> AM (Maybe ShortLinkCreds)
+    mkShortLinkCreds QIK {sndId, queueMode, linkId} = case (cqrd, queueMode) of
       (CQRMessaging ld, Just QMMessaging) ->
         withLinkData ld $ \lnkId CQRData {linkKey, privSigKey, srvReq = (sndId', d)} ->
           if sndId == sndId'
@@ -1523,11 +1555,9 @@ newRcvQueue_ c nm userId connId (ProtoServerWithAuth srv auth) vRange cqrd enabl
       (_, Nothing) -> newErr "unexpected link ID"
       _ -> newErr "unexpected queue mode"
       where
-        v = thVersion thParams'
         withLinkData :: Maybe d -> (SMP.LinkId -> d -> AM (Maybe ShortLinkCreds)) -> AM (Maybe ShortLinkCreds)
         withLinkData ld_ mkLink = case (ld_, linkId) of
           (Just ld, Just lnkId) -> mkLink lnkId ld
-          (Just _, Nothing) | v < shortLinksSMPVersion -> pure Nothing
           (Nothing, Nothing) -> pure Nothing
           _ -> newErr "unexpected or absent link ID"
         newErr :: String -> AM (Maybe ShortLinkCreds)
@@ -1952,7 +1982,7 @@ decryptSMPMessage :: RcvQueue -> SMP.RcvMessage -> AM SMP.ClientRcvMsgBody
 decryptSMPMessage rq SMP.RcvMessage {msgId, msgBody = SMP.EncRcvMsgBody body} =
   liftEither $ parse SMP.clientRcvMsgBodyP (AGENT $ A_MESSAGE "decrypt message") =<< decrypt body
   where
-    decrypt = agentCbDecrypt (rcvDhSecret rq) (C.cbNonce msgId)
+    decrypt = agentCbDecrypt (rcvDhSecret rq) msgId
 
 secureQueue :: AgentClient -> NetworkRequestMode -> RcvQueue -> SndPublicAuthKey -> AM ()
 secureQueue c nm rq@RcvQueue {rcvId, rcvPrivateKey} senderKey =
@@ -1993,7 +2023,7 @@ getQueueLink c nm userId server lnkId =
 -- resolver) and falls back to a direct send when the proxy is unavailable
 -- (faster but exposes the client IP). Mode selection is delegated to
 -- `sendOrProxySMPCommand`, which honours the network config (SPMNever etc.).
-resolveName :: AgentClient -> NetworkRequestMode -> UserId -> SMPServer -> SimplexDomain -> AM NameRecord
+resolveName :: AgentClient -> NetworkRequestMode -> UserId -> SMPServer -> SimplexDomain -> AM NameResponse
 resolveName c nm userId server domain =
   snd <$> sendOrProxySMPCommand c nm userId server "" "RSLV" NoEntity resolveViaProxy resolveDirectly
   where
@@ -2050,9 +2080,9 @@ disableQueuesNtfs c = sendTSessionBatches "NDEL" (mkSMPTSession . snd) disableQu
     queueCreds :: DisableQueueNtfReq -> (SMP.RecipientId, SMP.RcvPrivateAuthKey)
     queueCreds (_, RcvQueue {rcvPrivateKey, rcvId}) = (rcvId, rcvPrivateKey)
 
-sendAck :: AgentClient -> RcvQueue -> MsgId -> AM ()
+sendAck :: AgentClient -> RcvQueue -> MsgId -> AM QueueDrained
 sendAck c rq@RcvQueue {rcvId, rcvPrivateKey} msgId =
-  withSMPClient c NRMBackground rq ("ACK:" <> logSecret' msgId) $ \smp ->
+  withSMPClient c NRMBackground rq ("ACK:" <> logSecret' (SMP.unMsgId msgId)) $ \smp ->
     ackSMPMessage smp rcvPrivateKey rcvId msgId
 
 hasGetLock :: SomeRcvQueue q => AgentClient -> q -> IO Bool
@@ -2186,16 +2216,17 @@ agentXFTPDownloadChunk c userId (FileDigest chunkDigest) RcvFileChunkReplica {se
   g <- asks random
   withXFTPClient c (userId, server, chunkDigest) "FGET" $ \xftp -> X.downloadXFTPChunk g xftp replicaKey fId chunkSpec
 
-agentXFTPNewChunk :: AgentClient -> SndFileChunk -> Int -> XFTPServerWithAuth -> AM NewSndChunkReplica
-agentXFTPNewChunk c SndFileChunk {userId, chunkSpec = XFTPChunkSpec {chunkSize}, digest = FileDigest chunkDigest} n (ProtoServerWithAuth srv auth) = do
+agentXFTPNewChunk :: AgentClient -> SndFileChunk -> Int -> XFTPServerWithAuth -> Maybe Word32 -> AM NewSndChunkReplica
+agentXFTPNewChunk c SndFileChunk {userId, chunkSpec = XFTPChunkSpec {chunkSize}, digest = FileDigest chunkDigest} n (ProtoServerWithAuth srv auth) storageHours = do
   rKeys <- xftpRcvKeys n
   (sndKey, replicaKey) <- atomically . C.generateAuthKeyPair C.SEd25519 =<< asks random
   let fileInfo = FileInfo {sndKey, size = chunkSize, digest = chunkDigest}
   logServer "-->" c srv NoEntity "FNEW"
   tSess <- mkTransportSession c userId srv chunkDigest
-  (sndId, rIds) <- withClient c NRMBackground tSess $ \xftp -> X.createXFTPChunk xftp replicaKey fileInfo (L.map fst rKeys) auth
+  (sndId, rIds, expiresAt) <- withClient c NRMBackground tSess $ \xftp ->
+    X.createXFTPChunk xftp replicaKey fileInfo (L.map fst rKeys) auth storageHours
   logServer "<--" c srv NoEntity $ B.unwords ["SIDS", logSecret sndId]
-  pure NewSndChunkReplica {server = srv, replicaId = ChunkReplicaId sndId, replicaKey, rcvIdsKeys = L.toList $ xftpRcvIdsKeys rIds rKeys}
+  pure NewSndChunkReplica {server = srv, replicaId = ChunkReplicaId sndId, replicaKey, rcvIdsKeys = L.toList $ xftpRcvIdsKeys rIds rKeys, expiresAt}
 
 agentXFTPUploadChunk :: AgentClient -> UserId -> FileDigest -> SndFileChunkReplica -> XFTPChunkSpec -> AM ()
 agentXFTPUploadChunk c userId (FileDigest chunkDigest) SndFileChunkReplica {server, replicaId = ChunkReplicaId fId, replicaKey} chunkSpec =
@@ -2246,7 +2277,7 @@ agentCbEncryptOnce clientVersion dhRcvPubKey msg = do
 
 -- | NaCl crypto-box decrypt - both for messages received from the server
 -- and per-queue E2E encrypted messages from the sender that were inside.
-agentCbDecrypt :: C.DhSecretX25519 -> C.CbNonce -> ByteString -> Either AgentErrorType ByteString
+agentCbDecrypt :: C.CbNonceI n => C.DhSecretX25519 -> n -> ByteString -> Either AgentErrorType ByteString
 agentCbDecrypt dhSecret nonce msg =
   first cryptoError $
     C.cbDecrypt dhSecret nonce msg

src/Simplex/Messaging/Agent/Env/SQLite.hs

@@ -67,6 +67,8 @@ import Simplex.Messaging.Agent.Store.Interface (DBOpts)
 import Simplex.Messaging.Agent.Store.Shared (MigrationConfig (..), MigrationError (..))
 import Simplex.Messaging.Client
 import qualified Simplex.Messaging.Crypto as C
+import Simplex.Messaging.Crypto.BBS (BBSPublicKey)
+import Simplex.Messaging.Crypto.Entitlement (EntitlementCredential, entitlementIssuerKeys)
 import Simplex.Messaging.Crypto.Ratchet (VersionRangeE2E, supportedE2EEncryptVRange)
 import Simplex.Messaging.Notifications.Client (defaultNTFClientConfig)
 import Simplex.Messaging.Notifications.Transport (NTFVersion)
@@ -89,6 +91,7 @@ data InitialAgentServers = InitialAgentServers
   { smp :: Map UserId (NonEmpty (ServerCfg 'PSMP)),
     ntf :: [NtfServer],
     xftp :: Map UserId (NonEmpty (ServerCfg 'PXFTP)),
+    entitlements :: Map UserId EntitlementCredential,
     netCfg :: NetworkConfig,
     useServices :: Map UserId Bool,
     presetDomains :: [HostName],
@@ -148,6 +151,7 @@ data AgentConfig = AgentConfig
     smpCfg :: ProtocolClientConfig SMPVersion,
     ntfCfg :: ProtocolClientConfig NTFVersion,
     xftpCfg :: XFTPClientConfig,
+    entitlementKeys :: Map Word16 BBSPublicKey,
     reconnectInterval :: RetryInterval,
     messageRetryInterval :: RetryInterval2,
     userNetworkInterval :: Int,
@@ -158,6 +162,7 @@ data AgentConfig = AgentConfig
     connDeleteDeliveryTimeout :: NominalDiffTime,
     helloTimeout :: NominalDiffTime,
     quotaExceededTimeout :: NominalDiffTime,
+    commandQuotaRetryInterval :: RetryInterval,
     persistErrorInterval :: NominalDiffTime,
     initialCleanupDelay :: Int64,
     cleanupInterval :: Int64,
@@ -226,6 +231,7 @@ defaultAgentConfig =
       smpCfg = defaultSMPClientConfig,
       ntfCfg = defaultNTFClientConfig,
       xftpCfg = defaultXFTPClientConfig,
+      entitlementKeys = entitlementIssuerKeys,
       reconnectInterval = defaultReconnectInterval,
       messageRetryInterval = defaultMessageRetryInterval,
       userNetworkInterval = 1800_000000, -- 30 minutes, should be less than Int32 max value
@@ -236,6 +242,7 @@ defaultAgentConfig =
       connDeleteDeliveryTimeout = 2 * nominalDay,
       helloTimeout = 2 * nominalDay,
       quotaExceededTimeout = 7 * nominalDay,
+      commandQuotaRetryInterval = RetryInterval {initialInterval = 30_000000, increaseAfter = 0, maxInterval = 3600_000000},
       persistErrorInterval = 3, -- seconds
       initialCleanupDelay = 30 * 1000000, -- 30 seconds
       cleanupInterval = 5 * 60 * 1000000, -- 5 minutes

src/Simplex/Messaging/Agent/Protocol.hs

@@ -79,6 +79,7 @@ module Simplex.Messaging.Agent.Protocol
     SMPConfirmation (..),
     AgentMsgEnvelope (..),
     AgentMessage (..),
+    RatchetInfo (..),
     RequestSignature (..),
     AgentMessageType (..),
     APrivHeader (..),
@@ -166,6 +167,8 @@ module Simplex.Messaging.Agent.Protocol
     ConnId,
     ConfirmationId,
     InvitationId,
+    ConnVerifyCodes (..),
+    ContactRequestBinding (..),
     MsgIntegrity (..),
     MsgErrorType (..),
     QueueStatus (..),
@@ -228,7 +231,7 @@ import Data.Type.Equality
 import Data.Typeable (Typeable)
 import Data.Word (Word16, Word32)
 import Simplex.FileTransfer.Description
-import Simplex.FileTransfer.Protocol (FileParty (..))
+import Simplex.FileTransfer.Protocol (FileParty (..), GrantedStorageTime)
 import Simplex.FileTransfer.Transport (XFTPErrorType)
 import Simplex.FileTransfer.Types (FileErrorType)
 import Simplex.Messaging.Agent.QueryString
@@ -243,7 +246,6 @@ import Simplex.Messaging.Crypto.Ratchet
     RcvE2ERatchetParams,
     RcvE2ERatchetParamsUri,
     SndE2ERatchetParams,
-    RcvE2EPrivRatchetParams,
     pattern PQSupportOff,
     pattern PQSupportOn,
   )
@@ -404,7 +406,7 @@ data AEvent (e :: AEntity) where
   LINK :: ConnShortLink 'CMContact -> UserConnLinkData 'CMContact -> AEvent AEConn
   LDATA :: FixedLinkData 'CMContact -> ConnLinkData 'CMContact -> ConnectionRequestUri 'CMContact -> AEvent AEConn
   CONF :: ConfirmationId -> PQSupport -> [SMPServer] -> ConnInfo -> AEvent AEConn -- ConnInfo is from sender, [SMPServer] will be empty only in v1 handshake
-  REQ :: InvitationId -> PQSupport -> NonEmpty SMPServer -> ConnInfo -> Bool -> AEvent AEConn -- ConnInfo is from sender; Bool - rejection reason can be sent
+  REQ :: InvitationId -> PQSupport -> NonEmpty SMPServer -> ConnInfo -> ContactRequestBinding -> Bool -> AEvent AEConn -- ConnInfo is from sender; Bool - rejection reason can be sent
   SREQ :: InvitationId -> Maybe C.PublicKeyEd25519 -> MsgBody -> AEvent AEConn
   SSENT :: AgentMsgId -> Maybe SMPServer -> AEvent AEConn
   RJCT :: ConnInfo -> AEvent AEConn
@@ -444,7 +446,7 @@ data AEvent (e :: AEntity) where
   RFERR :: AgentErrorType -> AEvent AERcvFile
   RFWARN :: AgentErrorType -> AEvent AERcvFile
   SFPROG :: Int64 -> Int64 -> AEvent AESndFile
-  SFDONE :: ValidFileDescription 'FSender -> [ValidFileDescription 'FRecipient] -> AEvent AESndFile
+  SFDONE :: ValidFileDescription 'FSender -> [ValidFileDescription 'FRecipient] -> Maybe GrantedStorageTime -> AEvent AESndFile
   SFERR :: AgentErrorType -> AEvent AESndFile
   SFWARN :: AgentErrorType -> AEvent AESndFile
 
@@ -609,6 +611,7 @@ data RcvSwitchStatus
   | RSSendingQADD
   | RSSendingQUSE
   | RSReceivedMessage
+  | RSReceivedQEND
   deriving (Eq, Show)
 
 instance StrEncoding RcvSwitchStatus where
@@ -617,12 +620,14 @@ instance StrEncoding RcvSwitchStatus where
     RSSendingQADD -> "sending_qadd"
     RSSendingQUSE -> "sending_quse"
     RSReceivedMessage -> "received_message"
+    RSReceivedQEND -> "received_qend"
   strP =
     A.takeTill (== ' ') >>= \case
       "switch_started" -> pure RSSwitchStarted
       "sending_qadd" -> pure RSSendingQADD
       "sending_quse" -> pure RSSendingQUSE
       "received_message" -> pure RSReceivedMessage
+      "received_qend" -> pure RSReceivedQEND
       _ -> fail "bad RcvSwitchStatus"
 
 instance ToField RcvSwitchStatus where toField = toField . decodeLatin1 . strEncode
@@ -926,7 +931,7 @@ data AgentMessage
   | -- AgentConnInfoReply is used by accepting party in duplexHandshake mode (v2), allowing to include reply queue(s) in the initial confirmation.
     -- It made removed REPLY message unnecessary.
     AgentConnInfoReply (NonEmpty SMPQueueInfo) ConnInfo
-  | AgentRatchetInfo ByteString
+  | AgentRatchetInfo RatchetInfo
   | AgentMessage APrivHeader AMessage
   | AgentServiceRequest (NonEmpty SMPQueueInfo) (Maybe RequestSignature) MsgBody
   | AgentServiceResponse MsgBody
@@ -937,7 +942,7 @@ instance Encoding AgentMessage where
   smpEncode = \case
     AgentConnInfo cInfo -> smpEncode ('I', Tail cInfo)
     AgentConnInfoReply smpQueues cInfo -> smpEncode ('D', smpQueues, Tail cInfo) -- 'D' stands for "duplex"
-    AgentRatchetInfo info -> smpEncode ('R', Tail info)
+    AgentRatchetInfo info -> smpEncode ('R', info)
     AgentMessage hdr aMsg -> smpEncode ('M', hdr, aMsg)
     AgentServiceRequest qs sig_ body -> smpEncode ('A', qs, sig_, Tail body)
     AgentServiceResponse body -> smpEncode ('P', Tail body)
@@ -946,13 +951,23 @@ instance Encoding AgentMessage where
     smpP >>= \case
       'I' -> AgentConnInfo . unTail <$> smpP
       'D' -> AgentConnInfoReply <$> smpP <*> (unTail <$> smpP)
-      'R' -> AgentRatchetInfo . unTail <$> smpP
+      'R' -> AgentRatchetInfo <$> smpP
       'M' -> AgentMessage <$> smpP <*> smpP
       'A' -> AgentServiceRequest <$> smpP <*> smpP <*> (unTail <$> smpP)
       'P' -> AgentServiceResponse . unTail <$> smpP
       'J' -> AgentRejection . unTail <$> smpP
       _ -> fail "bad AgentMessage"
 
+data RatchetInfo = RatchetInfo {answeredKeyHash :: Maybe ByteString}
+  deriving (Eq, Show)
+
+instance Encoding RatchetInfo where
+  smpEncode RatchetInfo {answeredKeyHash} = smpEncode answeredKeyHash
+  smpP = do
+    answeredKeyHash <- smpP <|> pure Nothing
+    _ <- A.takeByteString
+    pure RatchetInfo {answeredKeyHash}
+
 -- internal type for storing message type in the database
 data AgentMessageType
   = AM_CONN_INFO
@@ -1372,6 +1387,15 @@ type ConfirmationId = ByteString
 
 type InvitationId = ByteString
 
+data ConnVerifyCodes = ConnVerifyCodes
+  { codeAD :: ByteString,
+    codePQ :: Maybe ByteString
+  }
+  deriving (Eq, Show)
+
+data ContactRequestBinding = CRBRatchet ConnVerifyCodes | CRBRequest ByteString
+  deriving (Eq, Show)
+
 extraSMPServerHosts :: Map TransportHost TransportHost
 extraSMPServerHosts =
   M.fromList
@@ -1602,7 +1626,7 @@ newtype LinkKey = LinkKey ByteString -- sha3-256(fixed_data)
 instance ToField LinkKey where toField (LinkKey s) = toField $ Binary s
 
 -- | Parameters for creating a connection with a prepared link.
-data PreparedLinkParams = PreparedLinkParams
+data PreparedLinkParams k = PreparedLinkParams
   { -- | Correlation ID / determines sender ID
     plpNonce :: C.CbNonce,
     -- | Queue E2EE DH key pair
@@ -1617,8 +1641,8 @@ data PreparedLinkParams = PreparedLinkParams
     plpSrvWithAuth :: SMPServerWithAuth,
     -- | Initial PQ keys
     plpInitKeys :: InitialKeys,
-    -- | Contact address double ratchet keys
-    plpAddressKeys :: Maybe (RatchetKeyId, RcvE2EPrivRatchetParams 'C.X448)
+    -- | Contact address double ratchet keys or invitation X3DH keys
+    plpRatchetKeys :: k
   }
 
 instance ConnectionModeI c => ToField (ConnectionLink c) where toField = toField . Binary . strEncode

src/Simplex/Messaging/Agent/RetryInterval.hs

@@ -10,6 +10,7 @@ module Simplex.Messaging.Agent.RetryInterval
     withRetryInterval,
     withRetryIntervalCount,
     withRetryForeground,
+    withRetryInterval2,
     withRetryLock2,
     updateRetryInterval2,
     nextRetryDelay,
@@ -86,9 +87,24 @@ withRetryForeground ri isForeground isOnline action = callAction 0 $ initialInte
                 | otherwise = (elapsed + delay, nextRetryDelay elapsed' delay ri)
           callAction elapsed' delay'
 
+withRetryInterval2 :: forall m. MonadIO m => RetryInterval2 -> (RI2State -> (RetryIntervalMode -> m ()) -> m ()) -> m ()
+withRetryInterval2 = withRetryWait2 $ liftIO . threadDelay'
+
 -- This function allows action to toggle between slow and fast retry intervals.
 withRetryLock2 :: forall m. MonadIO m => RetryInterval2 -> TMVar () -> (RI2State -> (RetryIntervalMode -> m ()) -> m ()) -> m ()
-withRetryLock2 RetryInterval2 {riSlow, riFast} lock action =
+withRetryLock2 ri lock = withRetryWait2 wait ri
+  where
+    wait delay = do
+      waiting <- newTVarIO True
+      _ <- liftIO . forkIO $ do
+        threadDelay' delay
+        atomically $ whenM (readTVar waiting) $ void $ tryPutTMVar lock ()
+      atomically $ do
+        takeTMVar lock
+        writeTVar waiting False
+
+withRetryWait2 :: forall m. Monad m => (Int64 -> m ()) -> RetryInterval2 -> (RI2State -> (RetryIntervalMode -> m ()) -> m ()) -> m ()
+withRetryWait2 wait RetryInterval2 {riSlow, riFast} action =
   callAction (0, initialInterval riSlow) (0, initialInterval riFast)
   where
     callAction :: (Int64, Int64) -> (Int64, Int64) -> m ()
@@ -102,14 +118,6 @@ withRetryLock2 RetryInterval2 {riSlow, riFast} lock action =
           let elapsed' = elapsed + delay
               delay' = nextRetryDelay elapsed' delay ri
           call (elapsed', delay')
-        wait delay = do
-          waiting <- newTVarIO True
-          _ <- liftIO . forkIO $ do
-            threadDelay' delay
-            atomically $ whenM (readTVar waiting) $ void $ tryPutTMVar lock ()
-          atomically $ do
-            takeTMVar lock
-            writeTVar waiting False
 
 nextRetryDelay :: Int64 -> Int64 -> RetryInterval -> Int64
 nextRetryDelay elapsed delay RetryInterval {increaseAfter, maxInterval} =

src/Simplex/Messaging/Agent/Store.hs

@@ -219,6 +219,7 @@ canAbortRcvSwitch ConnData {connAgentVersion} = maybe False canAbort . rcvSwchSt
       -- if switch is in RSReceivedMessage status, aborting switch (deleting new queue)
       -- will break the connection because the sender would have original queue deleted
       RSReceivedMessage -> False
+      RSReceivedQEND -> False
 
 data ClientNtfCreds = ClientNtfCreds
   { -- | key pair to be used by the notification server to authorize transmissions
@@ -489,7 +490,8 @@ data PendingCommand = PendingCommand
     corrId :: ACorrId,
     userId :: UserId,
     connId :: ConnId,
-    command :: AgentCommand
+    command :: AgentCommand,
+    createdAt :: UTCTime
   }
 
 data AgentCmdType = ACClient | ACInternal

src/Simplex/Messaging/Agent/Store/AgentStore.hs

@@ -76,6 +76,7 @@ module Simplex.Messaging.Agent.Store.AgentStore
     getExpiredServiceConns,
     deleteExpiredServiceRequests,
     getDeletedWaitingDeliveryConnIds,
+    getConnRatchetSync,
     setConnRatchetSync,
     addProcessedRatchetKeyHash,
     checkRatchetKeyHashExists,
@@ -99,6 +100,7 @@ module Simplex.Messaging.Agent.Store.AgentStore
     setSndQueuePrimary,
     deleteConnRcvQueue,
     incRcvDeleteErrors,
+    getEndedRcvQueueConnIds,
     deleteConnSndQueue,
     getPrimaryRcvQueue,
     getRcvQueue,
@@ -165,6 +167,8 @@ module Simplex.Messaging.Agent.Store.AgentStore
     createRatchet,
     deleteRatchet,
     getRatchet,
+    getRatchetVerifyCodes,
+    ratchetVerifyCodes,
     getRatchetForUpdate,
     getSkippedMsgKeys,
     updateRatchet,
@@ -213,6 +217,7 @@ module Simplex.Messaging.Agent.Store.AgentStore
     -- Rcv files
     createRcvFile,
     createRcvFileRedirect,
+    startPreparedRcvFile,
     lockRcvFileForUpdate,
     getRcvFile,
     getRcvFileByEntityId,
@@ -234,6 +239,7 @@ module Simplex.Messaging.Agent.Store.AgentStore
     getRcvFilesExpired,
     -- Snd files
     createSndFile,
+    startPreparedSndFile,
     lockSndFileForUpdate,
     getSndFile,
     getSndFileByEntityId,
@@ -291,6 +297,7 @@ import Data.ByteString (ByteString)
 import qualified Data.ByteString.Base64.URL as U
 import qualified Data.ByteString.Char8 as B
 import qualified Data.ByteString.Lazy as LB
+import Data.Either (partitionEithers)
 import Data.Functor (($>))
 import Data.Int (Int64)
 import Data.List (foldl', sortBy)
@@ -309,7 +316,7 @@ import Network.Socket (ServiceName)
 import qualified Network.TLS as TLS
 import Simplex.FileTransfer.Client (XFTPChunkSpec (..))
 import Simplex.FileTransfer.Description
-import Simplex.FileTransfer.Protocol (FileParty (..), SFileParty (..))
+import Simplex.FileTransfer.Protocol (FileParty (..), GrantedStorageTime (..), SFileParty (..))
 import Simplex.FileTransfer.Types
 import Simplex.Messaging.Agent.Protocol
 import Simplex.Messaging.Agent.RetryInterval (RI2State (..))
@@ -757,6 +764,10 @@ incRcvDeleteErrors :: DB.Connection -> RcvQueue -> IO ()
 incRcvDeleteErrors db RcvQueue {connId, dbQueueId} =
   DB.execute db "UPDATE rcv_queues SET delete_errors = delete_errors + 1 WHERE conn_id = ? AND rcv_queue_id = ?" (connId, dbQueueId)
 
+getEndedRcvQueueConnIds :: DB.Connection -> IO [ConnId]
+getEndedRcvQueueConnIds db =
+  map fromOnly <$> DB.query db "SELECT conn_id FROM rcv_queues WHERE switch_status = ? AND deleted = 0" (Only RSReceivedQEND)
+
 deleteConnRcvQueue :: DB.Connection -> RcvQueue -> IO ()
 deleteConnRcvQueue db RcvQueue {connId, dbQueueId} =
   DB.execute db "DELETE FROM rcv_queues WHERE conn_id = ? AND rcv_queue_id = ?" (connId, dbQueueId)
@@ -1249,7 +1260,7 @@ getLastMsg db connId msgId =
         LEFT JOIN snd_messages s ON s.conn_id = r.conn_id AND s.rcpt_internal_id = r.internal_id
         WHERE r.conn_id = ? AND r.broker_id = ?
       |]
-      (connId, Binary msgId)
+      (connId, msgId)
 
 toRcvMsg :: (Int64, InternalTs, BrokerId, BrokerTs) :. (AgentMsgId, MsgIntegrity, MsgHash, AgentMessageType, MsgBody, PQEncryption, Maybe AgentMsgId, Maybe MsgReceiptStatus, BoolInt) -> RcvMsg
 toRcvMsg ((agentMsgId, internalTs, brokerId, brokerTs) :. (sndMsgId, integrity, internalHash, msgType, msgBody, pqEncryption, rcptInternalId_, rcptStatus_, BI userAck)) =
@@ -1281,7 +1292,7 @@ checkRcvMsgHashExists db connId hash =
 getRcvMsgBrokerTs :: DB.Connection -> ConnId -> SMP.MsgId -> IO (Either StoreError BrokerTs)
 getRcvMsgBrokerTs db connId msgId =
   firstRow fromOnly (SEMsgNotFound "getRcvMsgBrokerTs") $
-    DB.query db "SELECT broker_ts FROM rcv_messages WHERE conn_id = ? AND broker_id = ?" (connId, Binary msgId)
+    DB.query db "SELECT broker_ts FROM rcv_messages WHERE conn_id = ? AND broker_id = ?" (connId, msgId)
 
 deleteMsg :: DB.Connection -> ConnId -> InternalId -> IO ()
 deleteMsg db connId msgId =
@@ -1474,9 +1485,11 @@ createSndRatchet db connId ratchetState (CR.AE2ERatchetParams s (CR.E2ERatchetPa
     db
     [sql|
       INSERT INTO ratchets
-        (conn_id, ratchet_state, x3dh_pub_key_1, x3dh_pub_key_2, pq_pub_kem) VALUES (?, ?, ?, ?, ?)
+        (conn_id, ratchet_state, rc_verify_code_ad, rc_verify_code_pq, x3dh_pub_key_1, x3dh_pub_key_2, pq_pub_kem) VALUES (?, ?, ?, ?, ?, ?, ?)
       ON CONFLICT (conn_id) DO UPDATE SET
         ratchet_state = EXCLUDED.ratchet_state,
+        rc_verify_code_ad = EXCLUDED.rc_verify_code_ad,
+        rc_verify_code_pq = EXCLUDED.rc_verify_code_pq,
         x3dh_priv_key_1 = NULL,
         x3dh_priv_key_2 = NULL,
         x3dh_pub_key_1 = EXCLUDED.x3dh_pub_key_1,
@@ -1484,7 +1497,13 @@ createSndRatchet db connId ratchetState (CR.AE2ERatchetParams s (CR.E2ERatchetPa
         pq_priv_kem = NULL,
         pq_pub_kem = EXCLUDED.pq_pub_kem
     |]
-    (connId, ratchetState, x3dhPubKey1, x3dhPubKey2, CR.ARKP s <$> pqPubKem)
+    ((connId, ratchetState) :. verifyCodesRow (ratchetVerifyCodes ratchetState) :. (x3dhPubKey1, x3dhPubKey2, CR.ARKP s <$> pqPubKem))
+
+ratchetVerifyCodes :: RatchetX448 -> ConnVerifyCodes
+ratchetVerifyCodes CR.Ratchet {rcAD, rcVCPQ} = ConnVerifyCodes {codeAD = C.sha256Hash $ unStr rcAD, codePQ = unStr <$> rcVCPQ}
+
+verifyCodesRow :: ConnVerifyCodes -> (Binary ByteString, Maybe (Binary ByteString))
+verifyCodesRow ConnVerifyCodes {codeAD, codePQ} = (Binary codeAD, Binary <$> codePQ)
 
 getSndRatchet :: DB.Connection -> ConnId -> CR.VersionE2E -> IO (Either StoreError (RatchetX448, CR.AE2ERatchetParams 'C.X448))
 getSndRatchet db connId v =
@@ -1505,10 +1524,12 @@ createRatchet db connId rc =
   DB.execute
     db
     [sql|
-      INSERT INTO ratchets (conn_id, ratchet_state)
-      VALUES (?, ?)
+      INSERT INTO ratchets (conn_id, ratchet_state, rc_verify_code_ad, rc_verify_code_pq)
+      VALUES (?, ?, ?, ?)
       ON CONFLICT (conn_id) DO UPDATE SET
-        ratchet_state = ?,
+        ratchet_state = EXCLUDED.ratchet_state,
+        rc_verify_code_ad = EXCLUDED.rc_verify_code_ad,
+        rc_verify_code_pq = EXCLUDED.rc_verify_code_pq,
         x3dh_priv_key_1 = NULL,
         x3dh_priv_key_2 = NULL,
         x3dh_pub_key_1 = NULL,
@@ -1516,7 +1537,42 @@ createRatchet db connId rc =
         pq_priv_kem = NULL,
         pq_pub_kem = NULL
     |]
-    (connId, rc, rc)
+    ((connId, rc) :. verifyCodesRow (ratchetVerifyCodes rc))
+
+getRatchetVerifyCodes :: DB.Connection -> [ConnId] -> IO (Map ConnId ConnVerifyCodes)
+getRatchetVerifyCodes db connIds = do
+  (saved, computed) <- partitionEithers . mapMaybe rowCodes <$> selectCodes
+  stored <- saveCodes computed
+  pure $ M.fromList $ saved <> stored
+  where
+    rowCodes (connId, codeAD_, codePQ_, rc_) = case codeAD_ of
+      Just codeAD -> Just $ Left $ toCodes (connId, codeAD, codePQ_)
+      Nothing -> Right . (connId,) . ratchetVerifyCodes <$> rc_
+    toCodes (connId, Binary codeAD, codePQ_) = (connId, ConnVerifyCodes {codeAD, codePQ = fromBinary <$> codePQ_})
+    codesRow (connId, cs) = verifyCodesRow cs :. Only connId
+    codesQuery :: Query
+    codesQuery = "SELECT conn_id, rc_verify_code_ad, rc_verify_code_pq, CASE WHEN rc_verify_code_ad IS NULL THEN ratchet_state END FROM ratchets"
+    selectCodes :: IO [(ConnId, Maybe (Binary ByteString), Maybe (Binary ByteString), Maybe RatchetX448)]
+    saveCodes :: [(ConnId, ConnVerifyCodes)] -> IO [(ConnId, ConnVerifyCodes)]
+#if defined(dbPostgres)
+    selectCodes = DB.query db (codesQuery <> " WHERE conn_id IN ?") (Only (In connIds))
+    saveCodes cs =
+      map toCodes
+        <$> DB.returning
+          db
+          [sql|
+            UPDATE ratchets r
+            SET rc_verify_code_ad = COALESCE(r.rc_verify_code_ad, (upd.rc_verify_code_ad :: BYTEA)),
+              rc_verify_code_pq = CASE WHEN r.rc_verify_code_ad IS NULL THEN (upd.rc_verify_code_pq :: BYTEA) ELSE r.rc_verify_code_pq END
+            FROM (VALUES(?, ?, ?)) AS upd(rc_verify_code_ad, rc_verify_code_pq, conn_id)
+            WHERE r.conn_id = (upd.conn_id :: BYTEA)
+            RETURNING r.conn_id, r.rc_verify_code_ad, r.rc_verify_code_pq
+          |]
+          (map codesRow cs)
+#else
+    selectCodes = concat <$> mapM (DB.query db (codesQuery <> " WHERE conn_id = ?") . Only) connIds
+    saveCodes cs = cs <$ unless (null cs) (DB.executeMany db "UPDATE ratchets SET rc_verify_code_ad = ?, rc_verify_code_pq = ? WHERE conn_id = ?" $ map codesRow cs)
+#endif
 
 deleteRatchet :: DB.Connection -> ConnId -> IO ()
 deleteRatchet db connId =
@@ -1544,12 +1600,16 @@ getRatchet_ q db connId =
   where
     ratchet = maybe (Left SERatchetNotFound) Right . fromOnly
 
-getSkippedMsgKeys :: DB.Connection -> ConnId -> IO SkippedMsgKeys
-getSkippedMsgKeys db connId =
-  skipped <$> DB.query db "SELECT header_key, msg_n, msg_key FROM skipped_messages WHERE conn_id = ?" (Only connId)
+getSkippedMsgKeys :: DB.Connection -> ConnId -> Int -> IO SkippedMsgKeys
+getSkippedMsgKeys db connId maxKeys = do
+  (keys, oldKeys) <- splitAt maxKeys <$> DB.query db "SELECT skipped_message_id, header_key, msg_n, msg_key FROM skipped_messages WHERE conn_id = ? ORDER BY skipped_message_id DESC LIMIT ?" (connId, maxKeys + 1)
+  case oldKeys of
+    (skippedMsgId :: Int64, _, _, _) : _ -> DB.execute db "DELETE FROM skipped_messages WHERE conn_id = ? AND skipped_message_id <= ?" (connId, skippedMsgId)
+    [] -> pure ()
+  pure $ skipped keys
   where
     skipped = foldl' addSkippedKey M.empty
-    addSkippedKey smks (hk, msgN, mk) = M.alter (Just . addMsgKey) hk smks
+    addSkippedKey smks (_, hk, msgN, mk) = M.alter (Just . addMsgKey) hk smks
       where
         addMsgKey = maybe (M.singleton msgN mk) (M.insert msgN mk)
 
@@ -1668,7 +1728,7 @@ getPendingServerCommand db connId srv_ = getWorkItem "command" getCmdId getComma
         DB.query
           db
           [sql|
-            SELECT c.corr_id, cs.user_id, c.command
+            SELECT c.corr_id, cs.user_id, c.command, c.created_at
             FROM commands c
             JOIN connections cs USING (conn_id)
             WHERE c.command_id = ?
@@ -1676,7 +1736,7 @@ getPendingServerCommand db connId srv_ = getWorkItem "command" getCmdId getComma
           (Only cmdId)
       where
         err = SEInternal $ "command  " <> bshow cmdId <> " returned []"
-        pendingCommand (corrId, userId, command) = PendingCommand {cmdId, corrId, userId, connId, command}
+        pendingCommand (corrId, userId, command, createdAt) = PendingCommand {cmdId, corrId, userId, connId, command, createdAt}
     markCommandFailed cmdId = DB.execute db "UPDATE commands SET failed = 1 WHERE command_id = ?" (Only cmdId)
 
 updateCommandServer :: DB.Connection -> AsyncCmdId -> SMPServer -> IO (Either StoreError ())
@@ -2450,12 +2510,12 @@ getSubscriptionServers db onlyNeeded =
 
 -- TODO [certs rcv] check index for getting queues with service present
 getUserServerRcvQueueSubs :: DB.Connection -> UserId -> SMPServer -> Bool -> ServiceAssoc -> Int -> Maybe SMP.RecipientId -> IO [RcvQueueSub]
-getUserServerRcvQueueSubs db userId (SMPServer h p kh) onlyNeeded hasService limit cursor_ =
-  map toRcvQueueSub <$> case cursor_ of
+getUserServerRcvQueueSubs db userId srv@(SMPServer h p kh) onlyNeeded hasService limit cursor_ =
+  map (toServerRcvQueueSub srv) <$> case cursor_ of
     Nothing -> DB.query db (q <> orderLimit) (userId, h, p, kh, limit)
     Just cursor -> DB.query db (q <> " AND q.rcv_id > ? " <> orderLimit) (userId, h, p, kh, cursor, limit)
   where
-    q = rcvQueueSubQuery <> toSubscribe <> " c.deleted = 0 AND q.deleted = 0 AND c.user_id = ? AND q.host = ? AND q.port = ? AND COALESCE(q.server_key_hash, s.key_hash) = ?" <> serviceCond
+    q = serverRcvQueueSubQuery <> toSubscribe <> " c.deleted = 0 AND q.deleted = 0 AND c.user_id = ? AND q.host = ? AND q.port = ? AND COALESCE(q.server_key_hash, s.key_hash) = ?" <> serviceCond
     orderLimit = " ORDER BY q.rcv_id LIMIT ?"
     toSubscribe
       | onlyNeeded = " WHERE q.to_subscribe = 1 AND "
@@ -2468,7 +2528,7 @@ unassocUserServerRcvQueueSubs :: DB.Connection -> UserId -> SMPServer -> IO [Rcv
 unassocUserServerRcvQueueSubs db userId srv@(SMPServer h p kh) = do
   deleteClientService db userId srv
 #if defined(dbPostgres)
-  map toRcvQueueSub
+  map (toServerRcvQueueSub srv)
     <$> DB.query
       db
       (removeRcvAssocsQuery <> " " <> returningColumns)
@@ -2476,15 +2536,15 @@ unassocUserServerRcvQueueSubs db userId srv@(SMPServer h p kh) = do
   where
     returningColumns =
       [sql|
-        RETURNING c.user_id, rcv_queues.conn_id, rcv_queues.host, rcv_queues.port, COALESCE(rcv_queues.server_key_hash, s.key_hash),
+        RETURNING c.user_id, rcv_queues.conn_id,
           rcv_queues.rcv_id, rcv_queues.rcv_private_key, rcv_queues.status, c.enable_ntfs, rcv_queues.client_notice_id,
           rcv_queues.rcv_queue_id, rcv_queues.rcv_primary, rcv_queues.replace_rcv_queue_id
       |]
 #else
-  qs <- map toRcvQueueSub
+  qs <- map (toServerRcvQueueSub srv)
     <$> DB.query
       db
-      (rcvQueueSubQuery <> " WHERE c.user_id = ? AND q.host = ? AND q.port = ? AND COALESCE(q.server_key_hash, s.key_hash) = ? AND q.rcv_service_assoc = 1")
+      (serverRcvQueueSubQuery <> " WHERE c.user_id = ? AND q.host = ? AND q.port = ? AND COALESCE(q.server_key_hash, s.key_hash) = ? AND q.rcv_service_assoc = 1")
       (userId, h, p, kh)
   DB.execute db removeRcvAssocsQuery (h, p, userId, kh)
   pure qs
@@ -2743,6 +2803,11 @@ getDeletedWaitingDeliveryConnIds :: DB.Connection -> IO [ConnId]
 getDeletedWaitingDeliveryConnIds db =
   map fromOnly <$> DB.query_ db "SELECT conn_id FROM connections WHERE deleted_at_wait_delivery IS NOT NULL"
 
+getConnRatchetSync :: DB.Connection -> ConnId -> IO (Either StoreError RatchetSyncState)
+getConnRatchetSync db connId =
+  firstRow fromOnly SEConnNotFound $
+    DB.query db "SELECT ratchet_sync_state FROM connections WHERE conn_id = ?" (Only connId)
+
 setConnRatchetSync :: DB.Connection -> ConnId -> RatchetSyncState -> IO ()
 setConnRatchetSync db connId ratchetSyncState =
   DB.execute db "UPDATE connections SET ratchet_sync_state = ? WHERE conn_id = ?" (ratchetSyncState, connId)
@@ -2764,21 +2829,35 @@ checkRatchetKeyHashExists db connId hash =
       (connId, Binary hash)
 
 deleteRatchetKeyHashesExpired :: DB.Connection -> NominalDiffTime -> Int -> IO ()
-deleteRatchetKeyHashesExpired db ttl limit = do
+deleteRatchetKeyHashesExpired db ttl maxConnHashes = do
   cutoffTs <- addUTCTime (-ttl) <$> getCurrentTime
+#if defined(dbPostgres)
   DB.execute
     db
-    [sql|
-      DELETE FROM processed_ratchet_key_hashes
-      WHERE processed_ratchet_key_hash_id IN (
-        SELECT processed_ratchet_key_hash_id
-        FROM processed_ratchet_key_hashes
-        WHERE created_at < ?
-        ORDER BY created_at ASC
-        LIMIT ?
-      )
-    |]
-    (cutoffTs, limit)
+    ("DELETE FROM processed_ratchet_key_hashes h USING (" <> maxExcessIdsQuery <> ") e WHERE h.conn_id = e.conn_id AND h.processed_ratchet_key_hash_id <= e.max_excess_id AND h.created_at < ?")
+    (maxConnHashes, maxConnHashes, cutoffTs)
+#else
+  maxExcessIds <- DB.query db maxExcessIdsQuery (maxConnHashes, maxConnHashes)
+  DB.executeMany
+    db
+    "DELETE FROM processed_ratchet_key_hashes WHERE conn_id = ? AND processed_ratchet_key_hash_id <= ? AND created_at < ?"
+    (map (\(connId :: ConnId, maxExcessId :: Int64) -> (connId, maxExcessId, cutoffTs)) maxExcessIds)
+#endif
+  where
+    maxExcessIdsQuery :: Query
+    maxExcessIdsQuery =
+      [sql|
+        SELECT conn_id, (
+          SELECT processed_ratchet_key_hash_id
+          FROM processed_ratchet_key_hashes
+          WHERE conn_id = c.conn_id
+          ORDER BY processed_ratchet_key_hash_id DESC
+          LIMIT 1 OFFSET ?
+        ) AS max_excess_id
+        FROM processed_ratchet_key_hashes c
+        GROUP BY conn_id
+        HAVING COUNT(*) > ?
+      |]
 
 -- | returns all connection queues, the first queue is the primary one
 getRcvQueuesByConnId_ :: DB.Connection -> ConnId -> IO (Maybe (NonEmpty RcvQueue))
@@ -2835,17 +2914,32 @@ getRcvQueueSubsByConnId_ db connId =
 rcvQueueSubQuery :: Query
 rcvQueueSubQuery =
   [sql|
-    SELECT c.user_id, q.conn_id, q.host, q.port, COALESCE(q.server_key_hash, s.key_hash), q.rcv_id, q.rcv_private_key, q.status, c.enable_ntfs, q.client_notice_id,
+    SELECT q.host, q.port, COALESCE(q.server_key_hash, s.key_hash), c.user_id, q.conn_id, q.rcv_id, q.rcv_private_key, q.status, c.enable_ntfs, q.client_notice_id,
       q.rcv_queue_id, q.rcv_primary, q.replace_rcv_queue_id
     FROM rcv_queues q
     JOIN servers s ON q.host = s.host AND q.port = s.port
     JOIN connections c ON q.conn_id = c.conn_id
   |]
 
-toRcvQueueSub :: (UserId, ConnId, NonEmpty TransportHost, ServiceName, C.KeyHash, SMP.RecipientId, SMP.RcvPrivateAuthKey) :. (QueueStatus, Maybe BoolInt, Maybe NoticeId, Int64, BoolInt, Maybe Int64) -> RcvQueueSub
-toRcvQueueSub ((userId, connId, host, port, keyHash, rcvId, rcvPrivateKey) :. (status, enableNtfs_, clientNoticeId, dbQueueId, BI primary, dbReplaceQueueId)) =
+serverRcvQueueSubQuery :: Query
+serverRcvQueueSubQuery =
+  [sql|
+    SELECT c.user_id, q.conn_id, q.rcv_id, q.rcv_private_key, q.status, c.enable_ntfs, q.client_notice_id,
+      q.rcv_queue_id, q.rcv_primary, q.replace_rcv_queue_id
+    FROM rcv_queues q
+    JOIN servers s ON q.host = s.host AND q.port = s.port
+    JOIN connections c ON q.conn_id = c.conn_id
+  |]
+
+type RcvQueueSubRow = (UserId, ConnId, SMP.RecipientId, SMP.RcvPrivateAuthKey) :. (QueueStatus, Maybe BoolInt, Maybe NoticeId, Int64, BoolInt, Maybe Int64)
+
+toRcvQueueSub :: (NonEmpty TransportHost, ServiceName, C.KeyHash) :. RcvQueueSubRow -> RcvQueueSub
+toRcvQueueSub ((host, port, keyHash) :. r) = toServerRcvQueueSub (SMPServer host port keyHash) r
+
+toServerRcvQueueSub :: SMPServer -> RcvQueueSubRow -> RcvQueueSub
+toServerRcvQueueSub server ((userId, connId, rcvId, rcvPrivateKey) :. (status, enableNtfs_, clientNoticeId, dbQueueId, BI primary, dbReplaceQueueId)) =
   let enableNtfs = maybe True unBI enableNtfs_
-   in RcvQueueSub {userId, connId, server = SMPServer host port keyHash, rcvId, rcvPrivateKey, status, enableNtfs, clientNoticeId, dbQueueId, primary, dbReplaceQueueId}
+   in RcvQueueSub {userId, connId, server, rcvId, rcvPrivateKey, status, enableNtfs, clientNoticeId, dbQueueId, primary, dbReplaceQueueId}
 
 getRcvQueueById :: DB.Connection -> ConnId -> Int64 -> IO (Either StoreError RcvQueue)
 getRcvQueueById db connId dbRcvId =
@@ -2946,7 +3040,7 @@ insertRcvMsgDetails_ db connId RcvQueue {dbQueueId} RcvMsgData {msgMeta, interna
       VALUES
         (?,?,?,?,?,?,?,?,?,?)
     |]
-    (connId, dbQueueId, internalRcvId, fst recipient, sndMsgId, Binary (fst broker), snd broker, Binary internalHash, Binary externalPrevSndHash, integrity)
+    (connId, dbQueueId, internalRcvId, fst recipient, sndMsgId, fst broker, snd broker, Binary internalHash, Binary externalPrevSndHash, integrity)
   DB.execute db "INSERT INTO encrypted_rcv_message_hashes (conn_id, hash) VALUES (?,?)" (connId, Binary encryptedMsgHash)
 
 updateRcvMsgHash :: DB.Connection -> ConnId -> AgentMsgId -> InternalRcvId -> MsgHash -> IO ()
@@ -3094,7 +3188,8 @@ createRcvFile db gVar userId fd@FileDescription {chunks} prefixPath tmpPath file
 createRcvFileRedirect :: DB.Connection -> TVar ChaChaDRG -> UserId -> FileDescription 'FRecipient -> FilePath -> FilePath -> CryptoFile -> FilePath -> CryptoFile -> Bool -> IO (Either StoreError RcvFileId)
 createRcvFileRedirect _ _ _ FileDescription {redirect = Nothing} _ _ _ _ _ _ = pure $ Left $ SEInternal "createRcvFileRedirect called without redirect"
 createRcvFileRedirect db gVar userId redirectFd@FileDescription {chunks = redirectChunks, redirect = Just RedirectFileInfo {size, digest}} prefixPath redirectPath redirectFile dstPath dstFile approvedRelays = runExceptT $ do
-  (dstEntityId, dstId) <- ExceptT $ insertRcvFile db gVar userId dummyDst prefixPath dstPath dstFile Nothing Nothing approvedRelays
+  nonce <- atomically $ C.randomCbNonce gVar
+  (dstEntityId, dstId) <- ExceptT $ insertRcvFile db gVar userId (dummyDst nonce) prefixPath dstPath dstFile Nothing Nothing approvedRelays
   (_, redirectId) <- ExceptT $ insertRcvFile db gVar userId redirectFd prefixPath redirectPath redirectFile (Just dstId) (Just dstEntityId) approvedRelays
   liftIO $
     forM_ redirectChunks $ \fc@FileChunk {replicas} -> do
@@ -3102,7 +3197,7 @@ createRcvFileRedirect db gVar userId redirectFd@FileDescription {chunks = redire
       forM_ (zip [1 ..] replicas) $ \(rno, replica) -> insertRcvFileChunkReplica db rno replica chunkId
   pure dstEntityId
   where
-    dummyDst =
+    dummyDst nonce =
       FileDescription
         { party = SFRecipient,
           size,
@@ -3110,11 +3205,34 @@ createRcvFileRedirect db gVar userId redirectFd@FileDescription {chunks = redire
           redirect = Nothing,
           -- updated later with updateRcvFileRedirect
           key = C.unsafeSbKey $ B.replicate 32 '#',
-          nonce = C.cbNonce "",
+          nonce,
           chunkSize = FileSize 0,
           chunks = []
         }
 
+startPreparedRcvFile :: DB.Connection -> RcvFileId -> IO (Either StoreError [XFTPServer])
+startPreparedRcvFile db rcvFileEntityId = runExceptT $ do
+  rcvFileId <- ExceptT $ getRcvFileIdByEntityId_ db rcvFileEntityId
+  liftIO $ do
+    updatedAt <- getCurrentTime
+    DB.execute
+      db
+      "UPDATE rcv_files SET status = ?, updated_at = ? WHERE (rcv_file_id = ? OR redirect_id = ?) AND status = ?"
+      (RFSReceiving, updatedAt, rcvFileId, rcvFileId, RFSPrepared)
+    map toXFTPServer
+      <$> DB.query
+        db
+        [sql|
+          SELECT DISTINCT
+            s.xftp_host, s.xftp_port, s.xftp_key_hash
+          FROM rcv_file_chunk_replicas r
+          JOIN xftp_servers s ON s.xftp_server_id = r.xftp_server_id
+          JOIN rcv_file_chunks c ON c.rcv_file_chunk_id = r.rcv_file_chunk_id
+          JOIN rcv_files f ON f.rcv_file_id = c.rcv_file_id
+          WHERE (f.rcv_file_id = ? OR f.redirect_id = ?) AND r.replica_number = 1
+        |]
+        (rcvFileId, rcvFileId)
+
 insertRcvFile :: DB.Connection -> TVar ChaChaDRG -> UserId -> FileDescription 'FRecipient -> FilePath -> FilePath -> CryptoFile -> Maybe DBRcvFileId -> Maybe RcvFileId -> Bool -> IO (Either StoreError (RcvFileId, DBRcvFileId))
 insertRcvFile db gVar userId FileDescription {size, digest, key, nonce, chunkSize, redirect} prefixPath tmpPath (CryptoFile savePath cfArgs) redirectId_ redirectEntityId_ approvedRelays = runExceptT $ do
   let (redirectDigest_, redirectSize_) = case redirect of
@@ -3125,7 +3243,7 @@ insertRcvFile db gVar userId FileDescription {size, digest, key, nonce, chunkSiz
       DB.execute
         db
         "INSERT INTO rcv_files (rcv_file_entity_id, user_id, size, digest, key, nonce, chunk_size, prefix_path, tmp_path, save_path, save_file_key, save_file_nonce, status, redirect_id, redirect_entity_id, redirect_digest, redirect_size, approved_relays) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
-        ((Binary rcvFileEntityId, userId, size, digest, key, nonce, chunkSize, prefixPath, tmpPath) :. (savePath, fileKey <$> cfArgs, fileNonce <$> cfArgs, RFSReceiving, redirectId_, Binary <$> redirectEntityId_, redirectDigest_, redirectSize_, BI approvedRelays))
+        ((Binary rcvFileEntityId, userId, size, digest, key, nonce, chunkSize, prefixPath, tmpPath) :. (savePath, fileKey <$> cfArgs, fileNonce <$> cfArgs, RFSPrepared, redirectId_, Binary <$> redirectEntityId_, redirectDigest_, redirectSize_, BI approvedRelays))
   rcvFileId <- liftIO $ insertedRowId db
   pure (rcvFileEntityId, rcvFileId)
 
@@ -3424,19 +3542,29 @@ getRcvFilesExpired db ttl = do
     |]
     (Only cutoffTs)
 
-createSndFile :: DB.Connection -> TVar ChaChaDRG -> UserId -> CryptoFile -> Int -> FilePath -> C.SbKey -> C.CbNonce -> Maybe RedirectFileInfo -> IO (Either StoreError SndFileId)
-createSndFile db gVar userId (CryptoFile path cfArgs) numRecipients prefixPath key nonce redirect_ =
+createSndFile :: DB.Connection -> TVar ChaChaDRG -> UserId -> CryptoFile -> Int -> FilePath -> C.SbKey -> C.CbNonce -> Maybe RedirectFileInfo -> Maybe Word32 -> IO (Either StoreError SndFileId)
+createSndFile db gVar userId (CryptoFile path cfArgs) numRecipients prefixPath key nonce redirect_ storageHours =
   createWithRandomId db gVar $ \sndFileEntityId ->
     DB.execute
       db
-      "INSERT INTO snd_files (snd_file_entity_id, user_id, path, src_file_key, src_file_nonce, num_recipients, prefix_path, key, nonce, status, redirect_size, redirect_digest) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)"
-      ((Binary sndFileEntityId, userId, path, fileKey <$> cfArgs, fileNonce <$> cfArgs, numRecipients) :. (prefixPath, key, nonce, SFSNew, redirectSize_, redirectDigest_))
+      "INSERT INTO snd_files (snd_file_entity_id, user_id, path, src_file_key, src_file_nonce, num_recipients, prefix_path, key, nonce, status, redirect_size, redirect_digest, storage_time) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)"
+      ((Binary sndFileEntityId, userId, path, fileKey <$> cfArgs, fileNonce <$> cfArgs, numRecipients) :. (prefixPath, key, nonce, SFSPrepared, redirectSize_, redirectDigest_, storageHours))
   where
     (redirectSize_, redirectDigest_) =
       case redirect_ of
         Nothing -> (Nothing, Nothing)
         Just RedirectFileInfo {size, digest} -> (Just size, Just digest)
 
+startPreparedSndFile :: DB.Connection -> SndFileId -> IO (Either StoreError ())
+startPreparedSndFile db sndFileEntityId = runExceptT $ do
+  sndFileId <- ExceptT $ getSndFileIdByEntityId_ db sndFileEntityId
+  liftIO $ do
+    updatedAt <- getCurrentTime
+    DB.execute
+      db
+      "UPDATE snd_files SET status = ?, updated_at = ? WHERE snd_file_id = ? AND status = ?"
+      (SFSNew, updatedAt, sndFileId, SFSPrepared)
+
 getSndFileByEntityId :: DB.Connection -> SndFileId -> IO (Either StoreError SndFile)
 getSndFileByEntityId db sndFileEntityId = runExceptT $ do
   sndFileId <- ExceptT $ getSndFileIdByEntityId_ db sndFileEntityId
@@ -3466,7 +3594,7 @@ getSndFile db sndFileId = runExceptT $ do
         DB.query
           db
           ( [sql|
-              SELECT snd_file_entity_id, user_id, path, src_file_key, src_file_nonce, num_recipients, digest, prefix_path, key, nonce, status, deleted, redirect_size, redirect_digest
+              SELECT snd_file_entity_id, user_id, path, src_file_key, src_file_nonce, num_recipients, digest, prefix_path, key, nonce, status, deleted, redirect_size, redirect_digest, storage_time
               FROM snd_files
               WHERE snd_file_id = ?
             |]
@@ -3476,12 +3604,12 @@ getSndFile db sndFileId = runExceptT $ do
           )
           (Only sndFileId)
       where
-        toFile :: (SndFileId, UserId, FilePath, Maybe C.SbKey, Maybe C.CbNonce, Int, Maybe FileDigest, Maybe FilePath, C.SbKey, C.CbNonce) :. (SndFileStatus, BoolInt, Maybe (FileSize Int64), Maybe FileDigest) -> SndFile
-        toFile ((sndFileEntityId, userId, srcPath, srcKey_, srcNonce_, numRecipients, digest, prefixPath, key, nonce) :. (status, BI deleted, redirectSize_, redirectDigest_)) =
+        toFile :: (SndFileId, UserId, FilePath, Maybe C.SbKey, Maybe C.CbNonce, Int, Maybe FileDigest, Maybe FilePath, C.SbKey, C.CbNonce) :. (SndFileStatus, BoolInt, Maybe (FileSize Int64), Maybe FileDigest, Maybe Word32) -> SndFile
+        toFile ((sndFileEntityId, userId, srcPath, srcKey_, srcNonce_, numRecipients, digest, prefixPath, key, nonce) :. (status, BI deleted, redirectSize_, redirectDigest_, storageHours)) =
           let cfArgs = CFArgs <$> srcKey_ <*> srcNonce_
               srcFile = CryptoFile srcPath cfArgs
               redirect = RedirectFileInfo <$> redirectSize_ <*> redirectDigest_
-           in SndFile {sndFileId, sndFileEntityId, userId, srcFile, numRecipients, digest, prefixPath, key, nonce, status, deleted, redirect, chunks = []}
+           in SndFile {sndFileId, sndFileEntityId, userId, srcFile, numRecipients, digest, prefixPath, key, nonce, status, deleted, redirect, storageHours, chunks = []}
     getChunks :: SndFileId -> UserId -> Int -> FilePath -> IO [SndFileChunk]
     getChunks sndFileEntityId userId numRecipients filePrefixPath = do
       chunks <-
@@ -3510,7 +3638,7 @@ getSndFile db sndFileId = runExceptT $ do
             db
             [sql|
               SELECT
-                r.snd_file_chunk_replica_id, r.replica_id, r.replica_key, r.replica_status, r.delay, r.retries,
+                r.snd_file_chunk_replica_id, r.replica_id, r.replica_key, r.replica_status, r.delay, r.retries, r.replica_expires_at,
                 s.xftp_host, s.xftp_port, s.xftp_key_hash
               FROM snd_file_chunk_replicas r
               JOIN xftp_servers s ON s.xftp_server_id = r.xftp_server_id
@@ -3521,10 +3649,10 @@ getSndFile db sndFileId = runExceptT $ do
         rcvIdsKeys <- getChunkReplicaRecipients_ db sndChunkReplicaId
         pure (replica :: SndFileChunkReplica) {rcvIdsKeys}
       where
-        toReplica :: (Int64, ChunkReplicaId, C.APrivateAuthKey, SndFileReplicaStatus, Maybe Int64, Int, NonEmpty TransportHost, ServiceName, C.KeyHash) -> SndFileChunkReplica
-        toReplica (sndChunkReplicaId, replicaId, replicaKey, replicaStatus, delay, retries, host, port, keyHash) =
+        toReplica :: (Int64, ChunkReplicaId, C.APrivateAuthKey, SndFileReplicaStatus, Maybe Int64, Int, Maybe Int64, NonEmpty TransportHost, ServiceName, C.KeyHash) -> SndFileChunkReplica
+        toReplica (sndChunkReplicaId, replicaId, replicaKey, replicaStatus, delay, retries, expiresAtSec, host, port, keyHash) =
           let server = XFTPServer host port keyHash
-           in SndFileChunkReplica {sndChunkReplicaId, server, replicaId, replicaKey, replicaStatus, delay, retries, rcvIdsKeys = []}
+           in SndFileChunkReplica {sndChunkReplicaId, server, replicaId, replicaKey, replicaStatus, delay, retries, expiresAt = GSTExpires <$> expiresAtSec, rcvIdsKeys = []}
 
 getChunkReplicaRecipients_ :: DB.Connection -> Int64 -> IO [(ChunkReplicaId, C.APrivateAuthKey)]
 getChunkReplicaRecipients_ db replicaId =
@@ -3605,16 +3733,16 @@ createSndFileReplica :: DB.Connection -> SndFileChunk -> NewSndChunkReplica -> I
 createSndFileReplica db SndFileChunk {sndChunkId} = createSndFileReplica_ db sndChunkId
 
 createSndFileReplica_ :: DB.Connection -> Int64 -> NewSndChunkReplica -> IO ()
-createSndFileReplica_ db sndChunkId NewSndChunkReplica {server, replicaId, replicaKey, rcvIdsKeys} = do
+createSndFileReplica_ db sndChunkId NewSndChunkReplica {server, replicaId, replicaKey, rcvIdsKeys, expiresAt} = do
   srvId <- createXFTPServer_ db server
   DB.execute
     db
     [sql|
       INSERT INTO snd_file_chunk_replicas
-        (snd_file_chunk_id, replica_number, xftp_server_id, replica_id, replica_key, replica_status)
-      VALUES (?,?,?,?,?,?)
+        (snd_file_chunk_id, replica_number, xftp_server_id, replica_id, replica_key, replica_status, replica_expires_at)
+      VALUES (?,?,?,?,?,?,?)
     |]
-    (sndChunkId, 1 :: Int, srvId, replicaId, replicaKey, SFRSCreated)
+    (sndChunkId, 1 :: Int, srvId, replicaId, replicaKey, SFRSCreated, epochSeconds <$> expiresAt)
   rId <- insertedRowId db
   forM_ rcvIdsKeys $ \(rcvId, rcvKey) -> do
     DB.execute
@@ -3660,7 +3788,7 @@ getNextSndChunkToUpload db server@ProtocolServer {host, port, keyHash} ttl = do
               SELECT
                 f.snd_file_id, f.snd_file_entity_id, f.user_id, f.num_recipients, f.prefix_path,
                 c.snd_file_chunk_id, c.chunk_no, c.chunk_offset, c.chunk_size, c.digest,
-                r.snd_file_chunk_replica_id, r.replica_id, r.replica_key, r.replica_status, r.delay, r.retries
+                r.snd_file_chunk_replica_id, r.replica_id, r.replica_key, r.replica_status, r.delay, r.retries, r.replica_expires_at
               FROM snd_file_chunk_replicas r
               JOIN xftp_servers s ON s.xftp_server_id = r.xftp_server_id
               JOIN snd_file_chunks c ON c.snd_file_chunk_id = r.snd_file_chunk_id
@@ -3674,8 +3802,8 @@ getNextSndChunkToUpload db server@ProtocolServer {host, port, keyHash} ttl = do
           pure (replica :: SndFileChunkReplica) {rcvIdsKeys}
         pure (chunk {replicas = replicas'} :: SndFileChunk)
       where
-        toChunk :: ((DBSndFileId, SndFileId, UserId, Int, FilePath) :. (Int64, Int, Int64, Word32, FileDigest) :. (Int64, ChunkReplicaId, C.APrivateAuthKey, SndFileReplicaStatus, Maybe Int64, Int)) -> SndFileChunk
-        toChunk ((sndFileId, sndFileEntityId, userId, numRecipients, filePrefixPath) :. (sndChunkId, chunkNo, chunkOffset, chunkSize, digest) :. (sndChunkReplicaId, replicaId, replicaKey, replicaStatus, delay, retries)) =
+        toChunk :: ((DBSndFileId, SndFileId, UserId, Int, FilePath) :. (Int64, Int, Int64, Word32, FileDigest) :. (Int64, ChunkReplicaId, C.APrivateAuthKey, SndFileReplicaStatus, Maybe Int64, Int, Maybe Int64)) -> SndFileChunk
+        toChunk ((sndFileId, sndFileEntityId, userId, numRecipients, filePrefixPath) :. (sndChunkId, chunkNo, chunkOffset, chunkSize, digest) :. (sndChunkReplicaId, replicaId, replicaKey, replicaStatus, delay, retries, expiresAtSec)) =
           let chunkSpec = XFTPChunkSpec {filePath = sndFileEncPath filePrefixPath, chunkOffset, chunkSize}
            in SndFileChunk
                 { sndFileId,
@@ -3687,7 +3815,7 @@ getNextSndChunkToUpload db server@ProtocolServer {host, port, keyHash} ttl = do
                   chunkSpec,
                   digest,
                   filePrefixPath,
-                  replicas = [SndFileChunkReplica {sndChunkReplicaId, server, replicaId, replicaKey, replicaStatus, delay, retries, rcvIdsKeys = []}]
+                  replicas = [SndFileChunkReplica {sndChunkReplicaId, server, replicaId, replicaKey, replicaStatus, delay, retries, expiresAt = GSTExpires <$> expiresAtSec, rcvIdsKeys = []}]
                 }
 
 updateSndChunkReplicaDelay :: DB.Connection -> Int64 -> Int64 -> IO ()

src/Simplex/Messaging/Agent/Store/Postgres/DB.hs

@@ -13,6 +13,7 @@ module Simplex.Messaging.Agent.Store.Postgres.DB
     execute,
     execute_,
     executeMany,
+    returning,
     query,
     query_,
     blobFieldDecoder,
@@ -60,6 +61,10 @@ executeMany :: ToRow q => PSQL.Connection -> Query -> [q] -> IO ()
 executeMany db q qs = void $ PSQL.executeMany db q qs `E.catch` addSql q
 {-# INLINE executeMany #-}
 
+returning :: (ToRow q, FromRow r) => PSQL.Connection -> Query -> [q] -> IO [r]
+returning db q qs = PSQL.returning db q qs `E.catch` addSql q
+{-# INLINE returning #-}
+
 query :: (ToRow q, FromRow r) => PSQL.Connection -> Query -> q -> IO [r]
 query db q qs = PSQL.query db q qs `E.catch` addSql q
 {-# INLINE query #-}

src/Simplex/Messaging/Agent/Store/Postgres/Migrations/App.hs

@@ -14,6 +14,10 @@ import Simplex.Messaging.Agent.Store.Postgres.Migrations.M20251230_strict_tables
 import Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260410_receive_attempts
 import Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260411_service_certs
 import Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260712_address_dr_rpc
+import Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260823_snd_files_entitlement
+import Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260919_ratchet_verify_codes
+import Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260929_ratchet_indexes
+import Simplex.Messaging.Agent.Store.Postgres.Migrations.M20261008_rcv_switch_status
 import Simplex.Messaging.Agent.Store.Shared (Migration (..))
 
 schemaMigrations :: [(String, Text, Maybe Text)]
@@ -27,7 +31,11 @@ schemaMigrations =
     ("20251230_strict_tables", m20251230_strict_tables, Just down_m20251230_strict_tables),
     ("20260410_receive_attempts", m20260410_receive_attempts, Just down_m20260410_receive_attempts),
     ("20260411_service_certs", m20260411_service_certs, Just down_m20260411_service_certs),
-    ("20260712_address_dr_rpc", m20260712_address_dr_rpc, Just down_m20260712_address_dr_rpc)
+    ("20260712_address_dr_rpc", m20260712_address_dr_rpc, Just down_m20260712_address_dr_rpc),
+    ("20260823_snd_files_entitlement", m20260823_snd_files_entitlement, Just down_m20260823_snd_files_entitlement),
+    ("20260919_ratchet_verify_codes", m20260919_ratchet_verify_codes, Just down_m20260919_ratchet_verify_codes),
+    ("20260929_ratchet_indexes", m20260929_ratchet_indexes, Just down_m20260929_ratchet_indexes),
+    ("20261008_rcv_switch_status", m20261008_rcv_switch_status, Just down_m20261008_rcv_switch_status)
   ]
 
 -- | The list of migrations in ascending order by date

src/Simplex/Messaging/Agent/Store/Postgres/Migrations/M20260823_snd_files_entitlement.hs

@@ -0,0 +1,21 @@
+{-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE QuasiQuotes #-}
+
+module Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260823_snd_files_entitlement where
+
+import Data.Text (Text)
+import Text.RawString.QQ (r)
+
+m20260823_snd_files_entitlement :: Text
+m20260823_snd_files_entitlement =
+  [r|
+ALTER TABLE snd_files ADD COLUMN storage_time BIGINT;
+ALTER TABLE snd_file_chunk_replicas ADD COLUMN replica_expires_at BIGINT;
+|]
+
+down_m20260823_snd_files_entitlement :: Text
+down_m20260823_snd_files_entitlement =
+  [r|
+ALTER TABLE snd_file_chunk_replicas DROP COLUMN replica_expires_at;
+ALTER TABLE snd_files DROP COLUMN storage_time;
+|]

src/Simplex/Messaging/Agent/Store/Postgres/Migrations/M20260919_ratchet_verify_codes.hs

@@ -0,0 +1,21 @@
+{-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE QuasiQuotes #-}
+
+module Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260919_ratchet_verify_codes where
+
+import Data.Text (Text)
+import Text.RawString.QQ (r)
+
+m20260919_ratchet_verify_codes :: Text
+m20260919_ratchet_verify_codes =
+  [r|
+ALTER TABLE ratchets ADD COLUMN rc_verify_code_ad BYTEA;
+ALTER TABLE ratchets ADD COLUMN rc_verify_code_pq BYTEA;
+|]
+
+down_m20260919_ratchet_verify_codes :: Text
+down_m20260919_ratchet_verify_codes =
+  [r|
+ALTER TABLE ratchets DROP COLUMN rc_verify_code_pq;
+ALTER TABLE ratchets DROP COLUMN rc_verify_code_ad;
+|]

src/Simplex/Messaging/Agent/Store/Postgres/Migrations/M20260929_ratchet_indexes.hs

@@ -0,0 +1,34 @@
+{-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE QuasiQuotes #-}
+
+module Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260929_ratchet_indexes where
+
+import Data.Text (Text)
+import Text.RawString.QQ (r)
+
+m20260929_ratchet_indexes :: Text
+m20260929_ratchet_indexes =
+  [r|
+DROP INDEX idx_skipped_messages_conn_id;
+CREATE INDEX idx_skipped_messages_conn_id ON skipped_messages(conn_id, skipped_message_id);
+
+DELETE FROM processed_ratchet_key_hashes AS h
+WHERE EXISTS (
+  SELECT 1 FROM processed_ratchet_key_hashes d
+  WHERE d.conn_id = h.conn_id AND d.hash = h.hash AND d.processed_ratchet_key_hash_id < h.processed_ratchet_key_hash_id
+);
+
+DROP INDEX idx_processed_ratchet_key_hashes_hash;
+CREATE UNIQUE INDEX idx_processed_ratchet_key_hashes_hash ON processed_ratchet_key_hashes(conn_id, hash);
+CREATE INDEX idx_processed_ratchet_key_hashes_conn_id ON processed_ratchet_key_hashes(conn_id, processed_ratchet_key_hash_id);
+|]
+
+down_m20260929_ratchet_indexes :: Text
+down_m20260929_ratchet_indexes =
+  [r|
+DROP INDEX idx_processed_ratchet_key_hashes_conn_id;
+DROP INDEX idx_processed_ratchet_key_hashes_hash;
+CREATE INDEX idx_processed_ratchet_key_hashes_hash ON processed_ratchet_key_hashes(conn_id, hash);
+DROP INDEX idx_skipped_messages_conn_id;
+CREATE INDEX idx_skipped_messages_conn_id ON skipped_messages(conn_id);
+|]

src/Simplex/Messaging/Agent/Store/Postgres/Migrations/M20261008_rcv_switch_status.hs

@@ -0,0 +1,20 @@
+{-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE QuasiQuotes #-}
+
+module Simplex.Messaging.Agent.Store.Postgres.Migrations.M20261008_rcv_switch_status where
+
+import Data.Text (Text)
+import Text.RawString.QQ (r)
+
+m20261008_rcv_switch_status :: Text
+m20261008_rcv_switch_status =
+  [r|
+CREATE INDEX idx_rcv_queues_switch_status ON rcv_queues(switch_status);
+|]
+
+down_m20261008_rcv_switch_status :: Text
+down_m20261008_rcv_switch_status =
+  [r|
+UPDATE rcv_queues SET switch_status = NULL WHERE switch_status = 'received_qend';
+DROP INDEX idx_rcv_queues_switch_status;
+|]

src/Simplex/Messaging/Agent/Store/Postgres/Migrations/agent_postgres_schema.sql

@@ -448,7 +448,9 @@ CREATE TABLE smp_agent_test_protocol_schema.ratchets (
     x3dh_pub_key_1 bytea,
     x3dh_pub_key_2 bytea,
     pq_priv_kem bytea,
-    pq_pub_kem bytea
+    pq_pub_kem bytea,
+    rc_verify_code_ad bytea,
+    rc_verify_code_pq bytea
 );
 
 
@@ -682,7 +684,8 @@ CREATE TABLE smp_agent_test_protocol_schema.snd_file_chunk_replicas (
     delay bigint,
     retries bigint DEFAULT 0 NOT NULL,
     created_at timestamp with time zone DEFAULT now() NOT NULL,
-    updated_at timestamp with time zone DEFAULT now() NOT NULL
+    updated_at timestamp with time zone DEFAULT now() NOT NULL,
+    replica_expires_at bigint
 );
 
 
@@ -741,7 +744,8 @@ CREATE TABLE smp_agent_test_protocol_schema.snd_files (
     src_file_nonce bytea,
     failed smallint DEFAULT 0,
     redirect_size bigint,
-    redirect_digest bytea
+    redirect_digest bytea,
+    storage_time bigint
 );
 
 
@@ -1165,11 +1169,15 @@ CREATE INDEX idx_ntf_tokens_ntf_host_ntf_port ON smp_agent_test_protocol_schema.
 
 
 
+CREATE INDEX idx_processed_ratchet_key_hashes_conn_id ON smp_agent_test_protocol_schema.processed_ratchet_key_hashes USING btree (conn_id, processed_ratchet_key_hash_id);
+
+
+
 CREATE INDEX idx_processed_ratchet_key_hashes_created_at ON smp_agent_test_protocol_schema.processed_ratchet_key_hashes USING btree (created_at);
 
 
 
-CREATE INDEX idx_processed_ratchet_key_hashes_hash ON smp_agent_test_protocol_schema.processed_ratchet_key_hashes USING btree (conn_id, hash);
+CREATE UNIQUE INDEX idx_processed_ratchet_key_hashes_hash ON smp_agent_test_protocol_schema.processed_ratchet_key_hashes USING btree (conn_id, hash);
 
 
 
@@ -1225,6 +1233,10 @@ CREATE UNIQUE INDEX idx_rcv_queues_ntf ON smp_agent_test_protocol_schema.rcv_que
 
 
 
+CREATE INDEX idx_rcv_queues_switch_status ON smp_agent_test_protocol_schema.rcv_queues USING btree (switch_status);
+
+
+
 CREATE INDEX idx_rcv_queues_to_subscribe ON smp_agent_test_protocol_schema.rcv_queues USING btree (to_subscribe);
 
 
@@ -1237,7 +1249,7 @@ CREATE UNIQUE INDEX idx_server_certs_user_id_host_port ON smp_agent_test_protoco
 
 
 
-CREATE INDEX idx_skipped_messages_conn_id ON smp_agent_test_protocol_schema.skipped_messages USING btree (conn_id);
+CREATE INDEX idx_skipped_messages_conn_id ON smp_agent_test_protocol_schema.skipped_messages USING btree (conn_id, skipped_message_id);
 
 
 

src/Simplex/Messaging/Agent/Store/SQLite/Common.hs

@@ -100,7 +100,8 @@ withTransaction st = withTransactionPriority st False
 {-# INLINE withTransaction #-}
 
 withTransactionPriority :: DBStore -> Bool -> (DB.Connection -> IO a) -> IO a
-withTransactionPriority st priority action = withConnectionPriority st priority $ dbBusyLoop . transaction
+withTransactionPriority st@DBStore {dbClosed} priority action = withConnectionPriority st priority $ \db ->
+  ifM (readTVarIO dbClosed) (fail "withTransaction: database closed") (dbBusyLoop $ transaction db)
   where
     transaction db@DB.Connection {conn} = SQL.withImmediateTransaction conn $ action db
 

src/Simplex/Messaging/Agent/Store/SQLite/Migrations/App.hs

@@ -50,6 +50,11 @@ import Simplex.Messaging.Agent.Store.SQLite.Migrations.M20251230_strict_tables
 import Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260410_receive_attempts
 import Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260411_service_certs
 import Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260712_address_dr_rpc
+import Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260823_snd_files_entitlement
+import Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260919_ratchet_verify_codes
+import Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260929_ratchet_indexes
+import Simplex.Messaging.Agent.Store.SQLite.Migrations.M20261005_commands_autoincrement
+import Simplex.Messaging.Agent.Store.SQLite.Migrations.M20261008_rcv_switch_status
 import Simplex.Messaging.Agent.Store.Shared (Migration (..))
 
 schemaMigrations :: [(String, Query, Maybe Query)]
@@ -99,7 +104,12 @@ schemaMigrations =
     ("m20251230_strict_tables", m20251230_strict_tables, Just down_m20251230_strict_tables),
     ("m20260410_receive_attempts", m20260410_receive_attempts, Just down_m20260410_receive_attempts),
     ("m20260411_service_certs", m20260411_service_certs, Just down_m20260411_service_certs),
-    ("m20260712_address_dr_rpc", m20260712_address_dr_rpc, Just down_m20260712_address_dr_rpc)
+    ("m20260712_address_dr_rpc", m20260712_address_dr_rpc, Just down_m20260712_address_dr_rpc),
+    ("m20260823_snd_files_entitlement", m20260823_snd_files_entitlement, Just down_m20260823_snd_files_entitlement),
+    ("m20260919_ratchet_verify_codes", m20260919_ratchet_verify_codes, Just down_m20260919_ratchet_verify_codes),
+    ("m20260929_ratchet_indexes", m20260929_ratchet_indexes, Just down_m20260929_ratchet_indexes),
+    ("m20261005_commands_autoincrement", m20261005_commands_autoincrement, Just down_m20261005_commands_autoincrement),
+    ("m20261008_rcv_switch_status", m20261008_rcv_switch_status, Just down_m20261008_rcv_switch_status)
   ]
 
 -- | The list of migrations in ascending order by date

src/Simplex/Messaging/Agent/Store/SQLite/Migrations/M20260823_snd_files_entitlement.hs

@@ -0,0 +1,20 @@
+{-# LANGUAGE QuasiQuotes #-}
+
+module Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260823_snd_files_entitlement where
+
+import Database.SQLite.Simple (Query)
+import Database.SQLite.Simple.QQ (sql)
+
+m20260823_snd_files_entitlement :: Query
+m20260823_snd_files_entitlement =
+  [sql|
+ALTER TABLE snd_files ADD COLUMN storage_time INTEGER;
+ALTER TABLE snd_file_chunk_replicas ADD COLUMN replica_expires_at INTEGER;
+  |]
+
+down_m20260823_snd_files_entitlement :: Query
+down_m20260823_snd_files_entitlement =
+  [sql|
+ALTER TABLE snd_file_chunk_replicas DROP COLUMN replica_expires_at;
+ALTER TABLE snd_files DROP COLUMN storage_time;
+  |]

src/Simplex/Messaging/Agent/Store/SQLite/Migrations/M20260919_ratchet_verify_codes.hs

@@ -0,0 +1,20 @@
+{-# LANGUAGE QuasiQuotes #-}
+
+module Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260919_ratchet_verify_codes where
+
+import Database.SQLite.Simple (Query)
+import Database.SQLite.Simple.QQ (sql)
+
+m20260919_ratchet_verify_codes :: Query
+m20260919_ratchet_verify_codes =
+  [sql|
+ALTER TABLE ratchets ADD COLUMN rc_verify_code_ad BLOB;
+ALTER TABLE ratchets ADD COLUMN rc_verify_code_pq BLOB;
+  |]
+
+down_m20260919_ratchet_verify_codes :: Query
+down_m20260919_ratchet_verify_codes =
+  [sql|
+ALTER TABLE ratchets DROP COLUMN rc_verify_code_pq;
+ALTER TABLE ratchets DROP COLUMN rc_verify_code_ad;
+  |]

src/Simplex/Messaging/Agent/Store/SQLite/Migrations/M20260929_ratchet_indexes.hs

@@ -0,0 +1,29 @@
+{-# LANGUAGE QuasiQuotes #-}
+
+module Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260929_ratchet_indexes where
+
+import Database.SQLite.Simple (Query)
+import Database.SQLite.Simple.QQ (sql)
+
+-- idx_skipped_messages_conn_id is not changed: SQLite index entries include rowid (skipped_message_id)
+m20260929_ratchet_indexes :: Query
+m20260929_ratchet_indexes =
+  [sql|
+DELETE FROM processed_ratchet_key_hashes AS h
+WHERE EXISTS (
+  SELECT 1 FROM processed_ratchet_key_hashes d
+  WHERE d.conn_id = h.conn_id AND d.hash = h.hash AND d.processed_ratchet_key_hash_id < h.processed_ratchet_key_hash_id
+);
+
+DROP INDEX idx_processed_ratchet_key_hashes_hash;
+CREATE UNIQUE INDEX idx_processed_ratchet_key_hashes_hash ON processed_ratchet_key_hashes(conn_id, hash);
+CREATE INDEX idx_processed_ratchet_key_hashes_conn_id ON processed_ratchet_key_hashes(conn_id, processed_ratchet_key_hash_id);
+  |]
+
+down_m20260929_ratchet_indexes :: Query
+down_m20260929_ratchet_indexes =
+  [sql|
+DROP INDEX idx_processed_ratchet_key_hashes_conn_id;
+DROP INDEX idx_processed_ratchet_key_hashes_hash;
+CREATE INDEX idx_processed_ratchet_key_hashes_hash ON processed_ratchet_key_hashes(conn_id, hash);
+  |]

src/Simplex/Messaging/Agent/Store/SQLite/Migrations/M20261005_commands_autoincrement.hs

@@ -0,0 +1,33 @@
+{-# LANGUAGE QuasiQuotes #-}
+
+module Simplex.Messaging.Agent.Store.SQLite.Migrations.M20261005_commands_autoincrement where
+
+import Database.SQLite.Simple (Query)
+import Database.SQLite.Simple.QQ (sql)
+
+m20261005_commands_autoincrement :: Query
+m20261005_commands_autoincrement =
+  [sql|
+INSERT INTO sqlite_sequence (name, seq)
+SELECT 'commands', MAX(ROWID) FROM commands;
+
+PRAGMA writable_schema=1;
+
+UPDATE sqlite_master SET sql = replace(sql, 'command_id INTEGER PRIMARY KEY,', 'command_id INTEGER PRIMARY KEY AUTOINCREMENT,')
+WHERE name = 'commands' AND type = 'table';
+
+PRAGMA writable_schema=RESET;
+  |]
+
+down_m20261005_commands_autoincrement :: Query
+down_m20261005_commands_autoincrement =
+  [sql|
+DELETE FROM sqlite_sequence WHERE name = 'commands';
+
+PRAGMA writable_schema=1;
+
+UPDATE sqlite_master SET sql = replace(sql, 'command_id INTEGER PRIMARY KEY AUTOINCREMENT,', 'command_id INTEGER PRIMARY KEY,')
+WHERE name = 'commands' AND type = 'table';
+
+PRAGMA writable_schema=RESET;
+  |]

src/Simplex/Messaging/Agent/Store/SQLite/Migrations/M20261008_rcv_switch_status.hs

@@ -0,0 +1,19 @@
+{-# LANGUAGE QuasiQuotes #-}
+
+module Simplex.Messaging.Agent.Store.SQLite.Migrations.M20261008_rcv_switch_status where
+
+import Database.SQLite.Simple (Query)
+import Database.SQLite.Simple.QQ (sql)
+
+m20261008_rcv_switch_status :: Query
+m20261008_rcv_switch_status =
+  [sql|
+CREATE INDEX idx_rcv_queues_switch_status ON rcv_queues(switch_status);
+|]
+
+down_m20261008_rcv_switch_status :: Query
+down_m20261008_rcv_switch_status =
+  [sql|
+UPDATE rcv_queues SET switch_status = NULL WHERE switch_status = 'received_qend';
+DROP INDEX idx_rcv_queues_switch_status;
+|]

src/Simplex/Messaging/Agent/Store/SQLite/Migrations/agent_schema.sql

@@ -182,7 +182,9 @@ CREATE TABLE ratchets(
   x3dh_pub_key_1 BLOB,
   x3dh_pub_key_2 BLOB,
   pq_priv_kem BLOB,
-  pq_pub_kem BLOB
+  pq_pub_kem BLOB,
+  rc_verify_code_ad BLOB,
+  rc_verify_code_pq BLOB
 ) WITHOUT ROWID, STRICT;
 CREATE TABLE skipped_messages(
   skipped_message_id INTEGER PRIMARY KEY,
@@ -245,7 +247,7 @@ CREATE TABLE ntf_subscriptions(
   ON DELETE RESTRICT ON UPDATE CASCADE
 ) WITHOUT ROWID, STRICT;
 CREATE TABLE commands(
-  command_id INTEGER PRIMARY KEY,
+  command_id INTEGER PRIMARY KEY AUTOINCREMENT,
   conn_id BLOB NOT NULL REFERENCES connections ON DELETE CASCADE,
   host TEXT,
   port TEXT,
@@ -259,6 +261,7 @@ CREATE TABLE commands(
   FOREIGN KEY(host, port) REFERENCES servers
   ON DELETE RESTRICT ON UPDATE CASCADE
 ) STRICT;
+CREATE TABLE sqlite_sequence(name,seq);
 CREATE TABLE snd_message_deliveries(
   snd_message_delivery_id INTEGER PRIMARY KEY AUTOINCREMENT,
   conn_id BLOB NOT NULL REFERENCES connections ON DELETE CASCADE,
@@ -267,7 +270,6 @@ CREATE TABLE snd_message_deliveries(
   failed INTEGER DEFAULT 0,
   FOREIGN KEY(conn_id, internal_id) REFERENCES messages ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED
 ) STRICT;
-CREATE TABLE sqlite_sequence(name,seq);
 CREATE TABLE users(
   user_id INTEGER PRIMARY KEY AUTOINCREMENT
   ,
@@ -352,7 +354,8 @@ CREATE TABLE snd_files(
   src_file_nonce BLOB,
   failed INTEGER DEFAULT 0,
   redirect_size INTEGER,
-  redirect_digest BLOB
+  redirect_digest BLOB,
+  storage_time INTEGER
 ) STRICT;
 CREATE TABLE snd_file_chunks(
   snd_file_chunk_id INTEGER PRIMARY KEY,
@@ -376,6 +379,8 @@ CREATE TABLE snd_file_chunk_replicas(
   retries INTEGER NOT NULL DEFAULT 0,
   created_at TEXT NOT NULL DEFAULT(datetime('now')),
   updated_at TEXT NOT NULL DEFAULT(datetime('now'))
+  ,
+  replica_expires_at INTEGER
 ) STRICT;
 CREATE TABLE snd_file_chunk_replica_recipients(
   snd_file_chunk_replica_recipient_id INTEGER PRIMARY KEY,
@@ -568,10 +573,6 @@ CREATE INDEX idx_encrypted_rcv_message_hashes_hash ON encrypted_rcv_message_hash
   conn_id,
   hash
 );
-CREATE INDEX idx_processed_ratchet_key_hashes_hash ON processed_ratchet_key_hashes(
-  conn_id,
-  hash
-);
 CREATE INDEX idx_snd_messages_rcpt_internal_id ON snd_messages(
   conn_id,
   rcpt_internal_id
@@ -636,6 +637,15 @@ CREATE INDEX idx_connections_deleted ON connections(deleted);
 CREATE INDEX idx_connections_service_request_expires_at ON connections(
   service_request_expires_at
 );
+CREATE UNIQUE INDEX idx_processed_ratchet_key_hashes_hash ON processed_ratchet_key_hashes(
+  conn_id,
+  hash
+);
+CREATE INDEX idx_processed_ratchet_key_hashes_conn_id ON processed_ratchet_key_hashes(
+  conn_id,
+  processed_ratchet_key_hash_id
+);
+CREATE INDEX idx_rcv_queues_switch_status ON rcv_queues(switch_status);
 CREATE TRIGGER tr_rcv_queue_insert
 AFTER INSERT ON rcv_queues
 FOR EACH ROW

src/Simplex/Messaging/Client.hs

@@ -67,6 +67,7 @@ module Simplex.Messaging.Client
     enableSMPQueuesNtfs,
     disableSMPQueuesNtfs,
     sendSMPMessage,
+    QueueDrained,
     ackSMPMessage,
     suspendSMPQueue,
     deleteSMPQueue,
@@ -166,7 +167,7 @@ import Simplex.Messaging.Parsers (defaultJSON, dropPrefix, enumJSON, sumTypeJSON
 import Simplex.Messaging.Protocol
 import Simplex.Messaging.Protocol.Types
 import Simplex.Messaging.Server.QueueStore.QueueInfo
-import Simplex.Messaging.SimplexName (SimplexDomain)
+import Simplex.Messaging.SimplexName (SimplexDomain, fullDomainName)
 import Simplex.Messaging.TMap (TMap)
 import qualified Simplex.Messaging.TMap as TM
 import Simplex.Messaging.Transport
@@ -195,6 +196,7 @@ data PClient v err msg = PClient
     transportHost :: TransportHost,
     tcpConnectTimeout :: NetworkTimeout,
     tcpTimeout :: NetworkTimeout,
+    proxiedRelayVRange :: VersionRange v,
     sendPings :: TVar Bool,
     lastReceived :: TVar UTCTime,
     timeoutErrorCount :: TVar Int,
@@ -240,6 +242,7 @@ smpClientStub g sessionId thVersion thAuth = do
               transportHost = "localhost",
               tcpConnectTimeout,
               tcpTimeout,
+              proxiedRelayVRange = supportedClientSMPRelayVRange,
               sendPings,
               lastReceived,
               timeoutErrorCount,
@@ -477,6 +480,7 @@ data ProtocolClientConfig v = ProtocolClientConfig
     serviceCredentials :: Maybe ServiceCredentials,
     -- | client-server protocol version range
     serverVRange :: VersionRange v,
+    proxiedRelayVRange :: VersionRange v,
     -- | agree shared session secret (used in SMP proxy for additional encryption layer)
     agreeSecret :: Bool,
     -- | Whether connecting client is a proxy server. See comment in ClientHandshake
@@ -495,6 +499,7 @@ defaultClientConfig clientALPN useSNI serverVRange =
       clientALPN,
       serviceCredentials = Nothing,
       serverVRange,
+      proxiedRelayVRange = serverVRange,
       agreeSecret = False,
       proxyServer = False,
       useSNI
@@ -505,6 +510,7 @@ defaultSMPClientConfig :: ProtocolClientConfig SMPVersion
 defaultSMPClientConfig =
   (defaultClientConfig (Just alpnSupportedSMPHandshakes) False supportedClientSMPRelayVRange)
     { defaultTransport = (show defaultSMPPort, transport @TLS),
+      proxiedRelayVRange = supportedClientSMPRelayVRange,
       agreeSecret = True
     }
 {-# INLINE defaultSMPClientConfig #-}
@@ -568,7 +574,7 @@ type SMPTransportSession = TransportSession BrokerMsg
 -- A single queue can be used for multiple 'SMPClient' instances,
 -- as 'SMPServerTransmission' includes server information.
 getProtocolClient :: forall v err msg. Protocol v err msg => TVar ChaChaDRG -> NetworkRequestMode -> TransportSession msg -> ProtocolClientConfig v -> [HostName] -> Maybe (TBQueue (ServerTransmissionBatch v err msg)) -> UTCTime -> (ProtocolClient v err msg -> IO ()) -> IO (Either (ProtocolClientError err) (ProtocolClient v err msg))
-getProtocolClient g nm transportSession@(_, srv, _) cfg@ProtocolClientConfig {qSize, networkConfig, clientALPN, serviceCredentials, serverVRange, agreeSecret, proxyServer, useSNI} presetDomains msgQ proxySessTs disconnected = do
+getProtocolClient g nm transportSession@(_, srv, _) cfg@ProtocolClientConfig {qSize, networkConfig, clientALPN, serviceCredentials, serverVRange, proxiedRelayVRange, agreeSecret, proxyServer, useSNI} presetDomains msgQ proxySessTs disconnected = do
   case chooseTransportHost networkConfig (host srv) of
     Right useHost ->
       (getCurrentTime >>= mkProtocolClient useHost >>= runClient useTransport useHost)
@@ -593,6 +599,7 @@ getProtocolClient g nm transportSession@(_, srv, _) cfg@ProtocolClientConfig {qS
             transportHost,
             tcpConnectTimeout,
             tcpTimeout,
+            proxiedRelayVRange,
             sendPings,
             lastReceived,
             timeoutErrorCount,
@@ -1054,11 +1061,11 @@ proxySMPMessage c nm proxiedRelay spKey sId flags msg = proxyOKSMPCommand c nm p
 -- through `proxySMPCommand` and pattern-matches the expected RNAME response.
 -- Version-gated on the destination relay (mirrors `connectSMPProxiedRelay`):
 -- the client never sends RSLV to a relay that predates names support.
-proxyResolveName :: SMPClient -> NetworkRequestMode -> ProxiedRelay -> SimplexDomain -> ExceptT SMPClientError IO (Either ProxyClientError NameRecord)
+proxyResolveName :: SMPClient -> NetworkRequestMode -> ProxiedRelay -> SimplexDomain -> ExceptT SMPClientError IO (Either ProxyClientError NameResponse)
 proxyResolveName c nm proxiedRelay name
   | prVersion proxiedRelay >= namesSMPVersion =
-      proxySMPCommand c nm proxiedRelay Nothing NoEntity (RSLV name) >>= \case
-        Right (RNAME nr) -> pure $ Right nr
+      proxySMPCommand c nm proxiedRelay Nothing NoEntity (RSLV (NQDomain name)) >>= \case
+        Right (RNAME reg) | resolvedNameOrNotFound name reg -> pure $ Right reg
         Right r -> throwE $ unexpectedResponse r
         Left e -> pure $ Left e
   | otherwise = throwE $ PCETransportError TEVersion
@@ -1066,25 +1073,32 @@ proxyResolveName c nm proxiedRelay name
 -- | Direct (non-PFWD) name resolution. Exposes the client IP to the resolver;
 -- callers that want anonymity should use `proxyResolveName` via the standard
 -- proxy fallback in the agent. RSLV requires no entity ID or authorization
--- (see `noAuthCmd` in Protocol.hs). Version-gated on the session here, not the
--- encoder, so an old server never receives RSLV.
-directResolveName :: SMPClient -> NetworkRequestMode -> SimplexDomain -> ExceptT SMPClientError IO NameRecord
+-- (see `noAuthCmd` in Protocol.hs). Gated on the session version, below which
+-- the server has no RSLV at all; the encoder gates the query format separately.
+directResolveName :: SMPClient -> NetworkRequestMode -> SimplexDomain -> ExceptT SMPClientError IO NameResponse
 directResolveName c nm name
   | thVersion (thParams c) >= namesSMPVersion =
-      sendProtocolCommand c nm Nothing NoEntity (Cmd SResolver (RSLV name)) >>= \case
-        RNAME nr -> pure nr
+      sendProtocolCommand c nm Nothing NoEntity (Cmd SResolver (RSLV (NQDomain name))) >>= \case
+        RNAME reg | resolvedNameOrNotFound name reg -> pure reg
         r -> throwE $ unexpectedResponse r
   | otherwise = throwE $ PCETransportError TEVersion
 
+resolvedNameOrNotFound :: SimplexDomain -> NameResponse -> Bool
+resolvedNameOrNotFound d NameResponse {registration} = case registration of
+  NRRegistered {nameRecord} -> T.toLower (nrName nameRecord) == fullDomainName d
+  _ -> True
+
+type QueueDrained = Bool
+
 -- | Acknowledge message delivery (server deletes the message).
 --
 -- https://github.com/simplex-chat/simplexmq/blob/master/protocol/simplex-messaging.md#acknowledge-message-delivery
 -- This command is always sent in background request mode
-ackSMPMessage :: SMPClient -> RcvPrivateAuthKey -> QueueId -> MsgId -> ExceptT SMPClientError IO ()
+ackSMPMessage :: SMPClient -> RcvPrivateAuthKey -> QueueId -> MsgId -> ExceptT SMPClientError IO QueueDrained
 ackSMPMessage c rpKey rId msgId =
   sendSMPCommand c NRMBackground (Just rpKey) rId (ACK msgId) >>= \case
-    OK -> return ()
-    cmd@MSG {} -> liftIO $ writeSMPMessage c rId cmd
+    OK -> pure True
+    cmd@MSG {} -> liftIO (writeSMPMessage c rId cmd) $> False
     r -> throwE $ unexpectedResponse r
 
 -- | Irreversibly suspend SMP queue.
@@ -1110,10 +1124,10 @@ deleteSMPQueues = okSMPCommands DEL
 -- send PRXY :: SMPServer -> Maybe BasicAuth -> Command Sender
 -- receives PKEY :: SessionId -> X.CertificateChain -> X.SignedExact X.PubKey -> BrokerMsg
 connectSMPProxiedRelay :: SMPClient -> NetworkRequestMode -> SMPServer -> Maybe BasicAuth -> ExceptT SMPClientError IO ProxiedRelay
-connectSMPProxiedRelay c@ProtocolClient {client_ = PClient {tcpConnectTimeout, tcpTimeout}} nm relayServ@ProtocolServer {port = relayPort, keyHash = C.KeyHash kh} proxyAuth =
+connectSMPProxiedRelay c@ProtocolClient {client_ = PClient {tcpConnectTimeout, tcpTimeout, proxiedRelayVRange}} nm relayServ@ProtocolServer {port = relayPort, keyHash = C.KeyHash kh} proxyAuth =
   sendProtocolCommand_ c nm Nothing tOut Nothing NoEntity (Cmd SProxiedClient (PRXY relayServ proxyAuth)) >>= \case
     PKEY sId vr (CertChainPubKey chain key) ->
-      case supportedClientSMPRelayVRange `compatibleVersion` vr of
+      case proxiedRelayVRange `compatibleVersion` vr of
         Nothing -> throwE $ transportErr TEVersion
         Just (Compatible v) -> do
           relayKey <- liftEitherWith (const $ transportErr $ TEHandshake IDENTITY) =<< liftIO (runExceptT $ validateRelay chain key)
@@ -1164,7 +1178,7 @@ instance StrEncoding ProxyClientError where
 -- consider how to process slow responses - is it handled somehow locally or delegated to the caller
 -- this method is used in the client
 -- sends PFWD :: C.PublicKeyX25519 -> EncTransmission -> Command Sender
--- receives PRES :: EncResponse -> BrokerMsg -- proxy to client
+-- receives PRES :: Maybe C.CbNonce -> EncResponse -> BrokerMsg -- proxy to client
 
 -- When client sends message via proxy, there may be one successful scenario and 9 error scenarios
 -- as shown below (WTF stands for unexpected response, ??? for response that failed to parse).
@@ -1217,20 +1231,20 @@ proxySMPCommand c@ProtocolClient {thParams = proxyThParams, client_ = PClient {c
   -- encode
   let TransmissionForAuth {tForAuth, tToSend} = encodeTransmissionForAuth serverThParams (CorrId corrId, sId, Cmd (sParty @p) command)
   -- serviceAuth is False here – proxied commands are not used with service certificates
-  auth <- liftEitherWith PCETransportError $ authTransmission serverThAuth False spKey nonce tForAuth
+  auth <- liftEitherWith PCETransportError $ authTransmission serverThAuth False spKey (Just nonce) tForAuth
   b <- case batchTransmissions serverThParams [Right (auth, tToSend)] of
     [] -> throwE $ PCETransportError TELargeMsg
     TBError e _ : _ -> throwE $ PCETransportError e
     TBTransmission s _ : _ -> pure s
     TBTransmissions s _ _ : _ -> pure s
-  et <- liftEitherWith PCECryptoError $ EncTransmission <$> C.cbEncrypt cmdSecret nonce b paddedProxiedTLength
+  et <- liftEitherWith PCECryptoError $ EncTransmission <$> C.cbEncrypt cmdSecret (encTransmissionNonce v nonce) b paddedProxiedTLength
   -- proxy interaction errors are wrapped
   let tOut = Just $ 2 * netTimeoutInt tcpTimeout nm
   tryE (sendProtocolCommand_ c nm (Just nonce) tOut Nothing (EntityId sessionId) (Cmd SProxiedClient (PFWD v cmdPubKey et))) >>= \case
     Right r -> case r of
-      PRES (EncResponse er) -> do
+      PRES nonce_ (EncResponse er) -> do
         -- server interaction errors are thrown directly
-        t' <- liftEitherWith PCECryptoError $ C.cbDecrypt cmdSecret (C.reverseNonce nonce) er
+        t' <- liftEitherWith PCECryptoError $ C.cbDecrypt cmdSecret (fromMaybe (C.reverseNonce nonce) nonce_) er
         case tParse serverThParams t' of
           t'' :| [] -> case tDecodeClient serverThParams t'' of
             (_, _, cmd) -> case cmd of
@@ -1248,10 +1262,10 @@ proxySMPCommand c@ProtocolClient {thParams = proxyThParams, client_ = PClient {c
 
 -- this method is used in the proxy
 -- sends RFWD :: EncFwdTransmission -> Command Sender
--- receives RRES :: EncFwdResponse -> BrokerMsg
+-- receives RRES :: Maybe C.CbNonce -> EncFwdResponse -> BrokerMsg
 -- proxy should send PRES to the client with EncResponse
 -- Always uses background timeout mode
-forwardSMPTransmission :: SMPClient -> CorrId -> VersionSMP -> C.PublicKeyX25519 -> EncTransmission -> ExceptT SMPClientError IO EncResponse
+forwardSMPTransmission :: SMPClient -> C.CorrCbNonce -> VersionSMP -> C.PublicKeyX25519 -> EncTransmission -> ExceptT SMPClientError IO (Maybe C.CbNonce, EncResponse)
 forwardSMPTransmission c@ProtocolClient {thParams, client_ = PClient {clientCorrId = g}} fwdCorrId fwdVersion fwdKey fwdTransmission = do
   -- prepare params
   sessSecret <- case thAuth thParams of
@@ -1263,11 +1277,11 @@ forwardSMPTransmission c@ProtocolClient {thParams, client_ = PClient {clientCorr
       eft = EncFwdTransmission $ C.cbEncryptNoPad sessSecret nonce (smpEncode fwdT)
   -- send
   sendProtocolCommand_ c NRMBackground (Just nonce) Nothing Nothing NoEntity (Cmd SProxyService (RFWD eft)) >>= \case
-    RRES (EncFwdResponse efr) -> do
+    RRES nonce_ (EncFwdResponse efr) -> do
       -- unwrap
       r' <- liftEitherWith PCECryptoError $ C.cbDecryptNoPad sessSecret (C.reverseNonce nonce) efr
       FwdResponse {fwdCorrId = _, fwdResponse} <- liftEitherWith (const $ PCEResponseError BLOCK) $ smpDecode r'
-      pure fwdResponse
+      pure (nonce_, fwdResponse)
     r -> throwE $ unexpectedResponse r
 
 -- get queue information - always sent interactively
@@ -1389,7 +1403,7 @@ mkTransmission_ :: forall v err msg. Protocol v err msg => ProtocolClient v err
 mkTransmission_ ProtocolClient {thParams, client_ = PClient {clientCorrId, sentCommands}} nonce_ (entityId, pKey_, command) = do
   nonce@(C.CbNonce corrId) <- maybe (atomically $ C.randomCbNonce clientCorrId) pure nonce_
   let TransmissionForAuth {tForAuth, tToSend} = encodeTransmissionForAuth thParams (CorrId corrId, entityId, command)
-      auth = authTransmission (thAuth thParams) (useServiceAuth command) pKey_ nonce tForAuth
+      auth = authTransmission (thAuth thParams) (useServiceAuth command) pKey_ (Just nonce) tForAuth
   r <- mkRequest (CorrId corrId)
   pure ((,tToSend) <$> auth, r)
   where
@@ -1408,14 +1422,14 @@ mkTransmission_ ProtocolClient {thParams, client_ = PClient {clientCorrId, sentC
       atomically $ TM.insert corrId r sentCommands
       pure r
 
-authTransmission :: Maybe (THandleAuth 'TClient) -> Bool -> Maybe C.APrivateAuthKey -> C.CbNonce -> ByteString -> Either TransportError (Maybe TAuthorizations)
-authTransmission thAuth serviceAuth pKey_ nonce t = traverse authenticate pKey_
+authTransmission :: Maybe (THandleAuth 'TClient) -> Bool -> Maybe C.APrivateAuthKey -> Maybe C.CbNonce -> ByteString -> Either TransportError (Maybe TAuthorizations)
+authTransmission thAuth serviceAuth pKey_ nonce_ t = traverse authenticate pKey_
   where
     authenticate :: C.APrivateAuthKey -> Either TransportError TAuthorizations
     authenticate (C.APrivateAuthKey a pk) = (,serviceSig) <$> case a of
-      C.SX25519 -> case thAuth of
-        Just THAuthClient {peerServerPubKey = k} -> Right $ TAAuthenticator $ C.cbAuthenticate k pk nonce t'
-        Nothing -> Left TENoServerAuth
+      C.SX25519 -> case (thAuth, nonce_) of
+        (Just THAuthClient {peerServerPubKey = k}, Just nonce) -> Right $ TAAuthenticator $ C.cbAuthenticate k pk nonce t'
+        _ -> Left TENoServerAuth
       C.SEd25519 -> sign pk
       C.SEd448 -> sign pk
     -- When command is signed by both entity key and service key,

src/Simplex/Messaging/Crypto.hs

@@ -140,8 +140,10 @@ module Simplex.Messaging.Crypto
     gcmIV,
 
     -- * NaCl crypto_box
-    CbNonce (unCbNonce),
+    CbNonceI (..),
+    CbNonce,
     pattern CbNonce,
+    CorrCbNonce,
     cbEncrypt,
     cbEncryptNoPad,
     cbEncryptMaxLenBS,
@@ -152,8 +154,12 @@ module Simplex.Messaging.Crypto
     sbEncryptNoPad,
     sbDecryptNoPad,
     cbNonce,
+    unsafeCbNonce,
+    corrCbNonce,
     randomCbNonce,
     reverseNonce,
+    xorNonce,
+    toCbNonce,
 
     -- * NaCl crypto_secretbox
     SbKey (unSbKey),
@@ -173,12 +179,14 @@ module Simplex.Messaging.Crypto
 
     -- * pseudo-random bytes
     randomBytes,
+    randomBytes',
 
     -- * digests
     sha256Hash,
     sha512Hash,
     sha3_256,
     sha3_384,
+    keccak256,
     md5Hash,
 
     -- * Message padding / un-padding
@@ -218,7 +226,7 @@ import Crypto.Cipher.AES (AES256)
 import qualified Crypto.Cipher.Types as AES
 import qualified Crypto.Cipher.XSalsa as XSalsa
 import qualified Crypto.Error as CE
-import Crypto.Hash (Digest, MD5, SHA3_256, SHA3_384, SHA256 (..), SHA512 (..), hash, hashDigestSize)
+import Crypto.Hash (Digest, Keccak_256, MD5, SHA3_256, SHA3_384, SHA256 (..), SHA512 (..), hash, hashDigestSize)
 import qualified Crypto.KDF.HKDF as H
 import qualified Crypto.MAC.Poly1305 as Poly1305
 import qualified Crypto.PubKey.Curve25519 as X25519
@@ -232,7 +240,8 @@ import Data.ASN1.Types
 import Data.Aeson (FromJSON (..), ToJSON (..))
 import qualified Data.Attoparsec.ByteString.Char8 as A
 import Data.Bifunctor (bimap, first)
-import Data.ByteArray (ByteArrayAccess)
+import Data.Bits (xor)
+import Data.ByteArray (ByteArray, ByteArrayAccess)
 import qualified Data.ByteArray as BA
 import Data.ByteString.Base64 (decode)
 import qualified Data.ByteString.Base64.URL as U
@@ -254,7 +263,7 @@ import Simplex.Messaging.Agent.Store.DB (Binary (..), FromField (..), ToField (.
 import Simplex.Messaging.Encoding
 import Simplex.Messaging.Encoding.String
 import Simplex.Messaging.Parsers (parseAll, parseString)
-import Simplex.Messaging.Util ((<$?>))
+import Simplex.Messaging.Util (packZipWith, (<$?>))
 
 -- | Cryptographic algorithms.
 data Algorithm = Ed25519 | Ed448 | X25519 | X448
@@ -931,6 +940,8 @@ data CryptoError
     CERatchetEarlierMessage Word32
   | -- | duplicate message number
     CERatchetDuplicateMessage
+  | -- | KEM key generation failed, indicating a broken RNG
+    CryptoKEMKeyGenError
   deriving (Eq, Show, Exception)
 
 aesKeySize :: Int
@@ -1040,14 +1051,17 @@ sha3_384 :: ByteString -> ByteString
 sha3_384 = BA.convert . (hash :: ByteString -> Digest SHA3_384)
 {-# INLINE sha3_384 #-}
 
+-- | Keccak-256, the hash Ethereum uses. Not SHA3-256: they differ in the padding byte and produce entirely different digests.
+keccak256 :: ByteString -> ByteString
+keccak256 = BA.convert . (hash :: ByteString -> Digest Keccak_256)
+{-# INLINE keccak256 #-}
+
 md5Hash :: ByteString -> ByteString
 md5Hash = BA.convert . (hash :: ByteString -> Digest MD5)
 
 -- | AEAD-GCM encryption with associated data.
 --
--- Used as part of double ratchet encryption.
--- This function requires 16 bytes IV, it transforms IV in cryptonite_aes_gcm_init here:
--- https://github.com/haskell-crypto/cryptonite/blob/master/cbits/cryptonite_aes.c
+-- Used as part of double ratchet encryption, with a 16-byte IV (see @initAEAD@).
 encryptAEAD :: Key -> IV -> Int -> ByteString -> ByteString -> ExceptT CryptoError IO (AuthTag, ByteString)
 encryptAEAD aesKey ivBytes paddedLen ad msg = do
   aead <- initAEAD @AES256 aesKey ivBytes
@@ -1067,10 +1081,7 @@ encryptAEADNoPad aesKey ivBytes ad msg = do
 
 -- | AEAD-GCM decryption with associated data.
 --
--- Used as part of double ratchet encryption.
--- This function requires 16 bytes IV, it transforms IV in cryptonite_aes_gcm_init here:
--- https://github.com/haskell-crypto/cryptonite/blob/master/cbits/cryptonite_aes.c
--- To make it compatible with WebCrypto we will need to start using initAEADGCM.
+-- Used as part of double ratchet encryption, with a 16-byte IV (see @initAEAD@).
 decryptAEAD :: Key -> IV -> ByteString -> ByteString -> AuthTag -> ExceptT CryptoError IO ByteString
 decryptAEAD aesKey ivBytes ad msg (AuthTag authTag) = do
   aead <- initAEAD @AES256 aesKey ivBytes
@@ -1148,9 +1159,9 @@ maxLength :: forall i. KnownNat i => Int
 maxLength = fromIntegral (natVal $ Proxy @i)
 {-# INLINE maxLength #-}
 
--- this function requires 16 bytes IV, it transforms IV in cryptonite_aes_gcm_init here:
--- https://github.com/haskell-crypto/cryptonite/blob/master/cbits/cryptonite_aes.c
--- This is used for double ratchet encryption, so to make it compatible with WebCrypto we will need to deprecate it and start using initAEADGCM
+-- The 16-byte double ratchet IV is intentionally not the 96-bit IV recommended by NIST SP 800-38D, so GCM derives J0 = GHASH(IV || 0^64 || [128]_64),
+-- as in crypton_aes_gcm_init: https://hackage.haskell.org/package/crypton-0.34/src/cbits/crypton_aes.c
+-- WebCrypto and other SP 800-38D implementations interoperate only when given all 16 IV bytes.
 initAEAD :: forall c. AES.BlockCipher c => Key -> IV -> ExceptT CryptoError IO (AES.AEAD c)
 initAEAD (Key aesKey) (IV ivBytes) = do
   iv <- makeIV @c ivBytes
@@ -1327,8 +1338,8 @@ sbEncryptNoPad (SbKey key) (CbNonce nonce) = cryptoBox key nonce
 {-# INLINE sbEncryptNoPad #-}
 
 -- | NaCl @crypto_box@ encrypt with a shared DH secret and 192-bit nonce.
-cbEncryptMaxLenBS :: KnownNat i => DhSecret X25519 -> CbNonce -> MaxLenBS i -> ByteString
-cbEncryptMaxLenBS (DhSecretX25519 secret) (CbNonce nonce) = cryptoBox secret nonce . unMaxLenBS . padMaxLenBS
+cbEncryptMaxLenBS :: (KnownNat i, CbNonceI n) => DhSecret X25519 -> n -> MaxLenBS i -> ByteString
+cbEncryptMaxLenBS (DhSecretX25519 secret) nonce = cryptoBox secret (unCbNonce nonce) . unMaxLenBS . padMaxLenBS
 {-# INLINE cbEncryptMaxLenBS #-}
 
 cryptoBox :: ByteArrayAccess key => key -> ByteString -> ByteString -> ByteString
@@ -1338,8 +1349,8 @@ cryptoBox secret nonce s = BA.convert tag <> c
     tag = Poly1305.auth rs c
 
 -- | NaCl @crypto_box@ decrypt with a shared DH secret and 192-bit nonce.
-cbDecrypt :: DhSecret X25519 -> CbNonce -> ByteString -> Either CryptoError ByteString
-cbDecrypt (DhSecretX25519 secret) = sbDecrypt_ secret
+cbDecrypt :: CbNonceI n => DhSecret X25519 -> n -> ByteString -> Either CryptoError ByteString
+cbDecrypt (DhSecretX25519 secret) = sbDecrypt_ secret . toCbNonce
 {-# INLINE cbDecrypt #-}
 
 -- | NaCl @crypto_box@ decrypt with a shared DH secret and 192-bit nonce (without unpadding).
@@ -1386,9 +1397,16 @@ cbAuthenticate k pk nonce msg = CbAuthenticator $ cbEncryptNoPad (dh' k pk) nonc
 cbVerify :: PublicKeyX25519 -> PrivateKeyX25519 -> CbNonce -> CbAuthenticator -> ByteString -> Bool
 cbVerify k pk nonce (CbAuthenticator s) authorized = cbDecryptNoPad (dh' k pk) nonce s == Right (sha512Hash authorized)
 
-newtype CbNonce = CryptoBoxNonce {unCbNonce :: ByteString}
+class CbNonceI n where
+  unCbNonce :: n -> ByteString
+
+newtype CbNonce = CryptoBoxNonce ByteString
   deriving (Eq, Show)
-  deriving newtype (FromField)
+
+instance CbNonceI CbNonce where
+  unCbNonce (CryptoBoxNonce s) = s
+
+instance FromField CbNonce where fromField = blobFieldDecoder cbNonce
 
 instance ToField CbNonce where toField (CryptoBoxNonce s) = toField $ Binary s
 
@@ -1399,7 +1417,7 @@ pattern CbNonce s <- CryptoBoxNonce s
 
 instance StrEncoding CbNonce where
   strEncode (CbNonce s) = strEncode s
-  strP = cbNonce <$> strP
+  strP = cbNonce <$?> strP
 
 instance ToJSON CbNonce where
   toJSON = strToJSON
@@ -1408,27 +1426,52 @@ instance ToJSON CbNonce where
 instance FromJSON CbNonce where
   parseJSON = strParseJSON "CbNonce"
 
-cbNonce :: ByteString -> CbNonce
+cbNonce :: ByteString -> Either String CbNonce
 cbNonce s
-  | len == 24 = CryptoBoxNonce s
-  | len > 24 = CryptoBoxNonce . fst $ B.splitAt 24 s
-  | otherwise = CryptoBoxNonce $ s <> B.replicate (24 - len) (toEnum 0)
-  where
-    len = B.length s
+  | B.length s == 24 = Right $ CryptoBoxNonce s
+  | otherwise = Left "CbNonce: invalid length"
+
+unsafeCbNonce :: ByteString -> CbNonce
+unsafeCbNonce s = either error id $ cbNonce s
+
+corrCbNonce :: ByteString -> Either String CorrCbNonce
+corrCbNonce s = CorrCbNonce . unCbNonce <$> cbNonce s
 
 randomCbNonce :: TVar ChaChaDRG -> STM CbNonce
 randomCbNonce = fmap CryptoBoxNonce . randomBytes 24
 
 randomBytes :: Int -> TVar ChaChaDRG -> STM ByteString
-randomBytes n gVar = stateTVar gVar $ randomBytesGenerate n
+randomBytes = randomBytes'
+{-# INLINE randomBytes #-}
 
-reverseNonce :: CbNonce -> CbNonce
-reverseNonce (CryptoBoxNonce s) = CryptoBoxNonce (B.reverse s)
+randomBytes' :: ByteArray a => Int -> TVar ChaChaDRG -> STM a
+randomBytes' n gVar = stateTVar gVar $ randomBytesGenerate n
+{-# INLINE randomBytes' #-}
+
+reverseNonce :: CbNonceI n => n -> CbNonce
+reverseNonce = CryptoBoxNonce . B.reverse . unCbNonce
+
+xorNonce :: CbNonceI n => ByteString -> n -> CbNonce
+xorNonce s nonce = CryptoBoxNonce $ packZipWith xor s n <> B.drop (B.length s) n
+  where
+    n = unCbNonce nonce
+
+toCbNonce :: CbNonceI n => n -> CbNonce
+toCbNonce = CryptoBoxNonce . unCbNonce
 
 instance Encoding CbNonce where
   smpEncode = unCbNonce
   smpP = CryptoBoxNonce <$> A.take 24
 
+newtype CorrCbNonce = CorrCbNonce ByteString
+
+instance CbNonceI CorrCbNonce where
+  unCbNonce (CorrCbNonce s) = s
+
+instance Encoding CorrCbNonce where
+  smpEncode = smpEncode . unCbNonce
+  smpP = corrCbNonce <$?> smpP
+
 newtype SbKey = SecretBoxKey {unSbKey :: ByteString}
   deriving (Eq, Show)
   deriving newtype (FromField)

src/Simplex/Messaging/Crypto/BBS.hs

@@ -17,6 +17,7 @@ module Simplex.Messaging.Crypto.BBS
     BBSProof (..),
     BBSHeader (..),
     BBSPresHeader (..),
+    FixedBS (..),
     bbsKeyGen,
     bbsPublicKey,
     bbsSign,
@@ -33,7 +34,9 @@ import Data.Proxy (Proxy (..))
 import Foreign
 import Foreign.C
 import GHC.TypeLits (KnownNat, KnownSymbol, Nat, Symbol, natVal, symbolVal)
+import Simplex.Messaging.Encoding (Encoding (..), Large (..))
 import Simplex.Messaging.Encoding.String
+import Simplex.Messaging.Util ((<$?>))
 import System.IO.Unsafe (unsafePerformIO)
 
 -- Note: the data constructors below are unchecked escape hatches for trusted,
@@ -71,8 +74,8 @@ newtype BBSPresHeader = BBSPresHeader ByteString
   deriving (ToJSON, FromJSON) via (StrJSON "BBSPresHeader" BBSPresHeader)
 
 -- | A ByteString validated to be exactly @n@ bytes when parsed via StrEncoding
--- (and the JSON derived from it). Local to BBS, where every key/signature is a
--- fixed size; @name@ appears in the decode error only.
+-- (and the JSON derived from it), for keys and signatures of a fixed size;
+-- @name@ appears in the decode error only.
 newtype FixedBS (name :: Symbol) (n :: Nat) = FixedBS ByteString
 
 instance forall name n. (KnownSymbol name, KnownNat n) => StrEncoding (FixedBS name n) where
@@ -97,14 +100,20 @@ bbsProofLen :: Int -> Int
 bbsProofLen numUndisclosed = bbsProofBaseLen + numUndisclosed * bbsProofUdElemLen
 
 -- | A proof is @bbsProofBaseLen + 32 * numUndisclosed@ bytes; reject anything else.
+mkBBSProof :: ByteString -> Either String BBSProof
+mkBBSProof bs
+  | len >= bbsProofBaseLen && (len - bbsProofBaseLen) `mod` bbsProofUdElemLen == 0 = Right $ BBSProof bs
+  | otherwise = Left $ "BBS: invalid proof length " <> show len
+  where
+    len = B.length bs
+
 instance StrEncoding BBSProof where
   strEncode (BBSProof bs) = strEncode bs
-  strP = do
-    bs <- base64urlP
-    let len = B.length bs
-    if len >= bbsProofBaseLen && (len - bbsProofBaseLen) `mod` bbsProofUdElemLen == 0
-      then pure (BBSProof bs)
-      else fail $ "BBS: invalid proof length " <> show len
+  strP = mkBBSProof <$?> base64urlP
+
+instance Encoding BBSProof where
+  smpEncode (BBSProof p) = smpEncode (Large p)
+  smpP = mkBBSProof . unLarge <$?> smpP
 
 -- FFI
 

src/Simplex/Messaging/Crypto/BIP32.hs

@@ -0,0 +1,101 @@
+{-# LANGUAGE LambdaCase #-}
+{-# LANGUAGE NamedFieldPuns #-}
+{-# LANGUAGE OverloadedStrings #-}
+
+-- | BIP-32 HD derivation over secp256k1, private only: we hold the seed, so CKDpub, xpub and fingerprints are not implemented. An invalid master or child key is recomputed as SLIP-0010 specifies, up to two times.
+module Simplex.Messaging.Crypto.BIP32
+  ( ExtendedKey,
+    xkKey,
+    xkChainCode,
+    masterKey,
+    derivePath,
+    derivePath_,
+    renderPath,
+    hardened,
+    isHardened,
+  )
+where
+
+import Control.Concurrent.STM (TVar)
+import Control.Monad (foldM)
+import Control.Monad.Trans.Except (ExceptT (..), runExceptT)
+import qualified Crypto.Hash as H
+import qualified Crypto.MAC.HMAC as HMAC
+import Crypto.Random (ChaChaDRG)
+import Data.Bits ((.|.))
+import Data.ByteArray (ScrubbedBytes)
+import qualified Data.ByteArray as BA
+import Data.ByteString (ByteString)
+import qualified Data.ByteString.Char8 as BC
+import Data.List (intercalate)
+import Data.Word (Word32)
+import qualified Simplex.Messaging.Crypto.Secp256k1 as S
+import Simplex.Messaging.Encoding (smpEncode)
+
+data ExtendedKey = ExtendedKey
+  { xkKey :: S.Secp256k1PrivateKey,
+    xkChainCode :: ScrubbedBytes
+  }
+  deriving (Eq)
+
+hardenedOffset :: Word32
+hardenedOffset = 0x80000000
+
+hardened :: Word32 -> Word32
+hardened = (.|. hardenedOffset)
+
+isHardened :: Word32 -> Bool
+isHardened i = i >= hardenedOffset
+
+masterKey :: ScrubbedBytes -> Either String ExtendedKey
+masterKey seed
+  | seedLen < 16 || seedLen > 64 = Left $ "seed: expected 16 to 64 bytes, got " <> show seedLen
+  | otherwise = go attempts seed
+  where
+    seedLen = BA.length seed
+    go :: Int -> ScrubbedBytes -> Either String ExtendedKey
+    go 0 _ = Left derivationFailed
+    go n s = case S.mkPrivateKey il of
+      Right k -> Right ExtendedKey {xkKey = k, xkChainCode = ir}
+      Left _ -> go (n - 1) i
+      where
+        i = hmacSHA512 "Bitcoin seed" s
+        (il, ir) = BA.splitAt 32 i
+
+deriveChild :: S.Secp256k1Context -> ExtendedKey -> Word32 -> IO (Either String ExtendedKey)
+deriveChild ctx ExtendedKey {xkKey, xkChainCode} i = do
+  dat <-
+    if isHardened i
+      then pure $ BA.cons 0 (S.unPrivateKey xkKey)
+      else BA.convert <$> (S.serializePublicKey S.Compressed =<< S.secp256k1PublicKey_ ctx xkKey)
+  go attempts dat
+  where
+    go :: Int -> ScrubbedBytes -> IO (Either String ExtendedKey)
+    go 0 _ = pure $ Left derivationFailed
+    go n dat = do
+      let (il, ir) = BA.splitAt 32 $ hmacSHA512 xkChainCode (dat <> BA.convert (smpEncode i))
+      S.privateKeyTweakAdd xkKey il >>= \case
+        Just k -> pure $ Right ExtendedKey {xkKey = k, xkChainCode = ir}
+        Nothing -> go (n - 1) $ BA.cons 1 ir
+
+derivePath :: TVar ChaChaDRG -> ExtendedKey -> [Word32] -> IO (Either String ExtendedKey)
+derivePath g xk path = S.withContext g $ \ctx -> derivePath_ ctx xk path
+
+derivePath_ :: S.Secp256k1Context -> ExtendedKey -> [Word32] -> IO (Either String ExtendedKey)
+derivePath_ ctx xk = runExceptT . foldM (\k -> ExceptT . deriveChild ctx k) xk
+
+attempts :: Int
+attempts = 3
+
+derivationFailed :: String
+derivationFailed = "derivation failed after " <> show attempts <> " attempts"
+
+renderPath :: [Word32] -> ByteString
+renderPath is = BC.pack $ intercalate "/" ("m" : map component is)
+  where
+    component i
+      | isHardened i = show (i - hardenedOffset) <> "'"
+      | otherwise = show i
+
+hmacSHA512 :: ScrubbedBytes -> ScrubbedBytes -> ScrubbedBytes
+hmacSHA512 key msg = BA.convert (HMAC.hmac key msg :: HMAC.HMAC H.SHA512)

src/Simplex/Messaging/Crypto/BIP39.hs

@@ -0,0 +1,149 @@
+{-# LANGUAGE LambdaCase #-}
+{-# LANGUAGE OverloadedStrings #-}
+
+-- | BIP-39 mnemonics, English only: every word is ASCII, any Unicode space character separates words, and full NFKD normalization is not applied; a passphrase is bytes the caller normalizes.
+module Simplex.Messaging.Crypto.BIP39
+  ( WalletEntropy,
+    unEntropy,
+    EntropyStrength (..),
+    mkEntropy,
+    randomEntropy,
+    parsePhrase,
+    entropyPhrase,
+    entropySeed,
+  )
+where
+
+import Control.Concurrent.STM
+import Crypto.Hash (Digest, SHA256, SHA512 (..), hash)
+import qualified Crypto.KDF.PBKDF2 as PBKDF2
+import Crypto.Number.Serialize (i2ospOf_, os2ip)
+import Crypto.Random (ChaChaDRG)
+import qualified Data.Attoparsec.Text as A
+import Data.Bits (shiftL, shiftR, (.&.), (.|.))
+import Data.ByteArray (ScrubbedBytes)
+import qualified Data.ByteArray as BA
+import Data.ByteString (ByteString)
+import qualified Data.ByteString.Char8 as BC
+import Data.Char (isSpace)
+import Data.IntMap.Strict (IntMap)
+import qualified Data.IntMap.Strict as IM
+import Data.List (foldl')
+import Data.Map.Strict (Map)
+import qualified Data.Map.Strict as M
+import Data.Text (Text)
+import qualified Data.Text as T
+import Data.Text.Encoding (encodeUtf8)
+import Simplex.Messaging.Agent.Store.DB (Binary (..), FromField (..), ToField (..), blobFieldDecoder)
+import qualified Simplex.Messaging.Crypto as C
+import Simplex.Messaging.Crypto.BIP39.English (englishWordList)
+
+-- | 16, 20, 24, 28 or 32 bytes of entropy: a valid BIP-39 phrase in another encoding.
+newtype WalletEntropy = WalletEntropy ScrubbedBytes
+  deriving (Eq, Show)
+
+unEntropy :: WalletEntropy -> ScrubbedBytes
+unEntropy (WalletEntropy ent) = ent
+
+data EntropyStrength = ES128 | ES160 | ES192 | ES224 | ES256
+  deriving (Eq, Show, Bounded, Enum)
+
+newtype Mnemonic = Mnemonic [Int]
+  deriving (Eq)
+
+strengthBytes :: EntropyStrength -> Int
+strengthBytes = \case
+  ES128 -> 16
+  ES160 -> 20
+  ES192 -> 24
+  ES224 -> 28
+  ES256 -> 32
+
+wordCount :: Int -> Int
+wordCount entBytes = (entBits + entBits `div` 32) `div` 11
+  where
+    entBits = entBytes * 8
+
+seedSize :: Int
+seedSize = 64
+
+validEntropySizes :: [Int]
+validEntropySizes = map strengthBytes [minBound .. maxBound]
+
+validWordCounts :: [Int]
+validWordCounts = map wordCount validEntropySizes
+
+wordByIndex :: IntMap ByteString
+wordByIndex = IM.fromList $ zip [0 ..] englishWordList
+
+indexByWord :: Map ByteString Int
+indexByWord = M.fromList $ zip englishWordList [0 ..]
+
+mkEntropy :: ScrubbedBytes -> Either String WalletEntropy
+mkEntropy ent
+  | entLen `elem` validEntropySizes = Right $ WalletEntropy ent
+  | otherwise = Left $ "entropy: expected 16, 20, 24, 28 or 32 bytes, got " <> show entLen
+  where
+    entLen = BA.length ent
+
+instance ToField WalletEntropy where toField (WalletEntropy ent) = toField $ Binary (BA.convert ent :: ByteString)
+
+instance FromField WalletEntropy where fromField = blobFieldDecoder $ mkEntropy . BA.convert
+
+randomEntropy :: EntropyStrength -> TVar ChaChaDRG -> STM WalletEntropy
+randomEntropy s = fmap WalletEntropy . C.randomBytes' (strengthBytes s)
+
+-- | The words joined by single spaces, the PBKDF2 password BIP-39 specifies.
+entropyPhrase :: WalletEntropy -> ByteString
+entropyPhrase = BC.unwords . mnemonicWords . entropyToMnemonic
+
+-- | Indexes are in @[0, 2047]@: they are masked to 11 bits or looked up in the wordlist.
+mnemonicWords :: Mnemonic -> [ByteString]
+mnemonicWords (Mnemonic idxs) = map (wordByIndex IM.!) idxs
+
+entropyToMnemonic :: WalletEntropy -> Mnemonic
+entropyToMnemonic (WalletEntropy ent) =
+  Mnemonic [fromIntegral ((combined `shiftR` (11 * (n - 1 - i))) .&. 0x7FF) | i <- [0 .. n - 1]]
+  where
+    entBits = BA.length ent * 8
+    csBits = entBits `div` 32
+    -- csBits is at most 8 (256/32), so the first checksum byte always suffices.
+    csByte = BA.index (hash ent :: Digest SHA256) 0
+    combined = os2ip ent `shiftL` csBits .|. fromIntegral (csByte `shiftR` (8 - csBits))
+    n = (entBits + csBits) `div` 11
+
+mnemonicToEntropy :: Mnemonic -> WalletEntropy
+mnemonicToEntropy (Mnemonic idxs) = WalletEntropy $ i2ospOf_ (entBits `div` 8) (combined `shiftR` csBits)
+  where
+    totalBits = length idxs * 11
+    entBits = totalBits * 32 `div` 33
+    csBits = totalBits - entBits
+    combined = foldl' (\acc i -> acc `shiftL` 11 .|. fromIntegral i) (0 :: Integer) idxs
+
+parsePhrase :: Text -> Either String WalletEntropy
+parsePhrase = A.parseOnly (phraseP <* A.endOfInput)
+
+phraseP :: A.Parser WalletEntropy
+phraseP = do
+  ws <- A.skipSpace *> (wordP `A.sepBy'` A.takeWhile1 isSpace) <* A.skipSpace
+  let n = length ws
+  if n `notElem` validWordCounts
+    then fail $ "mnemonic: expected 12, 15, 18, 21 or 24 words, got " <> show n
+    else do
+      m <- Mnemonic <$> traverse lookupWord ws
+      let ent = mnemonicToEntropy m
+      -- recomputing the checksum bits from the decoded entropy rejects wrong checksum bits
+      if entropyToMnemonic ent == m
+        then pure ent
+        else fail "mnemonic: checksum mismatch"
+  where
+    wordP = T.toLower <$> A.takeWhile1 (not . isSpace)
+    lookupWord w = maybe (fail "mnemonic: not in wordlist") pure $ M.lookup (encodeUtf8 w) indexByWord
+
+entropySeed :: WalletEntropy -> ByteString -> ScrubbedBytes
+entropySeed ent passphrase =
+  PBKDF2.generate
+    (PBKDF2.prfHMAC SHA512)
+    PBKDF2.Parameters {PBKDF2.iterCounts = 2048, PBKDF2.outputLength = seedSize}
+    (entropyPhrase ent)
+    ("mnemonic" <> passphrase)

src/Simplex/Messaging/Crypto/BIP39/English.hs

@@ -0,0 +1,11 @@
+{-# LANGUAGE TemplateHaskell #-}
+
+-- | The BIP-39 English wordlist, embedded verbatim from <https://github.com/bitcoin/bips/blob/master/bip-0039/english.txt>, pinned by a checksum test.
+module Simplex.Messaging.Crypto.BIP39.English (englishWordList) where
+
+import Data.ByteString.Char8 (ByteString)
+import qualified Data.ByteString.Char8 as BC
+import Data.FileEmbed (embedFile)
+
+englishWordList :: [ByteString]
+englishWordList = BC.words $(embedFile "src/Simplex/Messaging/Crypto/BIP39/english.txt")

src/Simplex/Messaging/Crypto/BIP44.hs

@@ -0,0 +1,62 @@
+{-# LANGUAGE CPP #-}
+{-# LANGUAGE DerivingStrategies #-}
+{-# LANGUAGE GeneralizedNewtypeDeriving #-}
+{-# LANGUAGE LambdaCase #-}
+
+-- | BIP-44 paths, @m/44'/coin'/account'/0/0@: one account per key, at the hardened account level.
+module Simplex.Messaging.Crypto.BIP44
+  ( CoinType (..),
+    AccountIndex,
+    mkAccountIndex,
+    unAccountIndex,
+    bip44Path,
+  )
+where
+
+import Data.Aeson (FromJSON (..), ToJSON)
+import Data.Word (Word32)
+import Simplex.Messaging.Agent.Store.DB (FromField (..), ToField (..))
+import Simplex.Messaging.Crypto.BIP32 (hardened, isHardened)
+import Simplex.Messaging.Util ((<$?>))
+
+#if defined(dbPostgres)
+import Database.PostgreSQL.Simple.FromField (ResultError (..), returnError)
+#else
+import Database.SQLite.Simple (ResultError (..))
+import Database.SQLite.Simple.FromField (returnError)
+#endif
+
+-- | SLIP-44 coin types.
+data CoinType = Ethereum
+  deriving (Eq, Show)
+
+coinIndex :: CoinType -> Word32
+coinIndex = \case
+  Ethereum -> 60
+
+-- | Below 2^31, so it can be hardened without colliding with another index.
+newtype AccountIndex = AccountIndex Word32
+  deriving (Eq, Ord, Show)
+  deriving newtype (ToJSON, ToField)
+
+unAccountIndex :: AccountIndex -> Word32
+unAccountIndex (AccountIndex i) = i
+
+mkAccountIndex :: Word32 -> Either String AccountIndex
+mkAccountIndex i
+  | isHardened i = Left "account index at or above 2^31"
+  | otherwise = Right $ AccountIndex i
+
+instance FromJSON AccountIndex where
+  parseJSON v = mkAccountIndex <$?> parseJSON v
+
+#if defined(dbPostgres)
+instance FromField AccountIndex where
+  fromField f dat = fromField f dat >>= either (returnError ConversionFailed f) pure . mkAccountIndex
+#else
+instance FromField AccountIndex where
+  fromField f = fromField f >>= either (returnError ConversionFailed f) pure . mkAccountIndex
+#endif
+
+bip44Path :: CoinType -> AccountIndex -> [Word32]
+bip44Path coin (AccountIndex account) = [hardened 44, hardened (coinIndex coin), hardened account, 0, 0]

src/Simplex/Messaging/Crypto/Entitlement.hs

@@ -0,0 +1,151 @@
+{-# LANGUAGE DataKinds #-}
+{-# LANGUAGE DerivingVia #-}
+{-# LANGUAGE DuplicateRecordFields #-}
+{-# LANGUAGE GeneralizedNewtypeDeriving #-}
+{-# LANGUAGE NamedFieldPuns #-}
+{-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE TemplateHaskell #-}
+
+module Simplex.Messaging.Crypto.Entitlement
+  ( Entitlement (..),
+    EntitlementCredential (..),
+    EntitlementProof (..),
+    EntitlementVerification (..),
+    MasterKey (..),
+    randomMasterKey,
+    entitlementBBSHeader,
+    entitlementIssuerKeys,
+    signEntitlement,
+    verifyCredential,
+    generateEntitlementProof,
+    verifyEntitlement,
+  )
+where
+
+import Control.Concurrent.STM
+import Crypto.Random (ChaChaDRG)
+import Data.Aeson (FromJSON (..), ToJSON (..))
+import qualified Data.Aeson.TH as JQ
+import Data.ByteString.Char8 (ByteString)
+import qualified Data.ByteString.Char8 as B
+import Data.Either (fromRight)
+import Data.Map.Strict (Map)
+import qualified Data.Map.Strict as M
+import Data.Text (Text)
+import Data.Text.Encoding (decodeUtf8', encodeUtf8)
+import Data.Time.Clock (UTCTime)
+import Data.Word (Word16)
+import qualified Simplex.Messaging.Crypto as C
+import Simplex.Messaging.Crypto.BBS
+import Simplex.Messaging.Encoding
+import Simplex.Messaging.Encoding.String
+import Simplex.Messaging.Parsers (defaultJSON)
+import Simplex.Messaging.Util ((<$$>))
+
+newtype MasterKey = MasterKey ByteString
+  deriving newtype (Eq, Show)
+  deriving (StrEncoding) via (FixedBS "MasterKey" 32)
+  deriving (ToJSON, FromJSON) via (StrJSON "MasterKey" MasterKey)
+
+data Entitlement = Entitlement
+  { expiresAt :: UTCTime,
+    entitlementName :: Text,
+    extraInfo :: Text
+  }
+  deriving (Eq, Show)
+
+data EntitlementCredential = EntitlementCredential
+  { issuerKeyIdx :: Word16,
+    masterKey :: MasterKey,
+    entitlement :: Entitlement,
+    issuerSignature :: BBSSignature
+  }
+  deriving (Eq, Show)
+
+data EntitlementProof = EntitlementProof
+  { issuerKeyIdx :: Word16,
+    entitlement :: Entitlement,
+    entProof :: BBSProof
+  }
+  deriving (Eq, Show)
+
+data EntitlementVerification = EVValid | EVInvalid | EVUnknownIssuer
+  deriving (Eq, Show)
+
+instance Encoding Entitlement where
+  smpEncode Entitlement {expiresAt, entitlementName, extraInfo} =
+    smpEncode (strEncode expiresAt, entitlementName, Large $ encodeUtf8 extraInfo)
+  smpP = do
+    (expBs, entitlementName, Large extraBs) <- smpP
+    expiresAt <- either fail pure $ strDecode (expBs :: ByteString)
+    extraInfo <- either (fail . show) pure $ decodeUtf8' extraBs
+    pure Entitlement {expiresAt, entitlementName, extraInfo}
+
+instance Encoding EntitlementProof where
+  smpEncode EntitlementProof {issuerKeyIdx, entProof, entitlement} =
+    smpEncode (issuerKeyIdx, entProof, entitlement)
+  smpP = do
+    (issuerKeyIdx, entProof, entitlement) <- smpP
+    pure EntitlementProof {issuerKeyIdx, entProof, entitlement}
+
+entitlementBBSHeader :: BBSHeader
+entitlementBBSHeader = BBSHeader "SimpleX badges v1"
+
+entitlementMessageCount :: Int
+entitlementMessageCount = 4
+
+entitlementDisclosedIndexes :: [Int]
+entitlementDisclosedIndexes = [1, 2, 3]
+
+entitlementMessages :: MasterKey -> Entitlement -> [ByteString]
+entitlementMessages (MasterKey mk) ent = mk : disclosedMessages ent
+
+disclosedMessages :: Entitlement -> [ByteString]
+disclosedMessages Entitlement {expiresAt, entitlementName, extraInfo} =
+  [strEncode expiresAt, encodeUtf8 entitlementName, encodeUtf8 extraInfo]
+
+randomMasterKey :: TVar ChaChaDRG -> STM MasterKey
+randomMasterKey g = MasterKey <$> C.randomBytes 32 g
+
+signEntitlement :: BBSSecretKey -> Word16 -> MasterKey -> Entitlement -> IO (Either String EntitlementCredential)
+signEntitlement sk keyIdx mk ent =
+  EntitlementCredential keyIdx mk ent <$$> bbsSign sk entitlementBBSHeader (entitlementMessages mk ent)
+
+verifyCredential :: BBSPublicKey -> EntitlementCredential -> IO Bool
+verifyCredential pk EntitlementCredential {masterKey, issuerSignature, entitlement} =
+  bbsVerify pk issuerSignature entitlementBBSHeader (entitlementMessages masterKey entitlement)
+
+generateEntitlementProof :: Map Word16 BBSPublicKey -> EntitlementCredential -> BBSPresHeader -> IO (Either String EntitlementProof)
+generateEntitlementProof keys EntitlementCredential {issuerKeyIdx, masterKey, issuerSignature, entitlement} ph =
+  case M.lookup issuerKeyIdx keys of
+    Nothing -> pure $ Left $ "no issuer key " <> show issuerKeyIdx
+    Just pk -> EntitlementProof issuerKeyIdx entitlement <$$> bbsProofGen pk issuerSignature entitlementBBSHeader ph entitlementDisclosedIndexes (entitlementMessages masterKey entitlement)
+
+verifyEntitlement :: Map Word16 BBSPublicKey -> BBSPresHeader -> EntitlementProof -> IO EntitlementVerification
+verifyEntitlement keys ph EntitlementProof {issuerKeyIdx, entProof, entitlement} =
+  case M.lookup issuerKeyIdx keys of
+    Nothing -> pure EVUnknownIssuer
+    Just pk ->
+      (\valid -> if valid then EVValid else EVInvalid)
+        <$> bbsProofVerify pk entProof entitlementBBSHeader ph entitlementDisclosedIndexes entitlementMessageCount (disclosedMessages entitlement)
+
+entitlementIssuerKeys :: Map Word16 BBSPublicKey
+entitlementIssuerKeys =
+  M.fromList
+    [ (1, key "mW_5Zp1wHnXDF56wOZwFcRjGrf0GLLsfyymIQDqYoWfjfvS7oQWSfi7hH65N8JhuE9x8wbKXHidnQLO4GnOSMP_bRKUMH1qIzv5SQKFHNM8G4PaWcTcri8iZLc-3xhSI"),
+      (2, key "odGCB7uVDXTURsHgSvSciByV4Q3-3ZvEB8myDsDJqm-PwOYc5-At36uc7n_pyUDxEQEHr9i4RJgFih2FSArPW-EQBXNPNf4wTtA0znn74qLEGc4fh9pVYPEIm_ZGbnsJ"),
+      (3, key "txkT2003WMjc43KvYvPKEcR970NLmw5UZY51eUqgk91sgp53idt1HTlKYvnrEttJDFMlctYf1-bpri0e9DhBQ-xk1J4WoLN2uif_1OcA1pGCobpk9lwtsq1Idek4biy0"),
+      (4, key "q_YzegihaLYrEm9z3cAghsfDGNZfXuEpQGMJERJQS4M0Szl4gvSC_fV_muKc3NIMA_8iYuBN8qyvb5U55RctCRn3kleFQ4sqf-WBgoydX6UVo7BsYcUbXWWEFZXlOGIH"),
+      (5, key "oqymHASH_okefShrnz4HnTooUNlE1WoDRnSrgd0bTCpOacgJWBsMpwZpdmYlX-vQAKAC_zmI4VdKoOznnhW-sdUXZw6bthCi5JYjGxCR1Co27i1tix5UXCTbR5Jp901-"),
+      (6, key "kDqaB6zKSRp_97QPFj5JPDlo0vzfSTLSp9goFx1qajv4q4H6dR6BbkmWZ4xx_9Q2AxmcpqcV0ethz1OH-Jk_Sz2J1mIz1PUVM9LkdLhi_PNtqhezzO5dbVs-HJ1fNqe6"),
+      (7, key "rl36D5mg2N3NmmEybxE_RBeU9YZ_zeXNPfp7ZMLtUEuf2Mo4OQM_Up1v5rX_IqICD-AIJcuyptEBsELx_PJQzpmiNuG5I4cWO6HkRKtc6fVFvgZMrDJjaascPd1CIyxX"),
+      (8, key "joM3Bnt7JPt5JiwQwERHGjro2iVZ0mPD_clUh4hzkhxvbjuFrWuTmfSNA8PWBqGKEGNl13aRi1pMf6yY14E27c5C71JxWm7T-rZaBrGPEUWifhD-qidWuf3PU7KJCCWd")
+    ]
+  where
+    key = fromRight (error "bad base64 in BBSPublicKey") . strDecode . B.pack
+
+$(JQ.deriveJSON defaultJSON ''Entitlement)
+
+$(JQ.deriveJSON defaultJSON ''EntitlementCredential)
+
+$(JQ.deriveJSON defaultJSON ''EntitlementProof)

src/Simplex/Messaging/Crypto/Ratchet.hs

@@ -72,6 +72,7 @@ module Simplex.Messaging.Crypto.Ratchet
     rcEncryptHeader,
     rcEncryptMsg,
     rcDecrypt,
+    maxSkippedMsgKeys,
     -- used in tests
     MsgHeader (..),
     RatchetInitParams (..),
@@ -119,7 +120,7 @@ import Simplex.Messaging.Crypto.SNTRUP761.Bindings
 import Simplex.Messaging.Encoding
 import Simplex.Messaging.Encoding.String
 import Simplex.Messaging.Parsers (defaultJSON, parseE, parseE')
-import Simplex.Messaging.Util (($>>=), (<$?>))
+import Simplex.Messaging.Util ((<$?>))
 import Simplex.Messaging.Version
 import Simplex.Messaging.Version.Internal
 import UnliftIO.STM
@@ -327,9 +328,13 @@ instance StrEncoding AnyE2ERatchetParamsUri where
             Nothing -> pure $ AnyE2ERatchetParamsUri SRKSProposed a $ E2ERatchetParamsUri vr k1 k2 Nothing
       _ -> fail "bad e2e params"
     where
-      kemP query =
-        queryParam_ "kem_key" query
-          $>>= \k -> Just . kemParams k <$> queryParam_ "kem_ct" query
+      kemP query = do
+        k_ <- queryParam_ "kem_key" query
+        ct_ <- queryParam_ "kem_ct" query
+        case (k_, ct_) of
+          (Just k, _) -> pure $ Just $ kemParams k ct_
+          (Nothing, Nothing) -> pure Nothing
+          (Nothing, Just _) -> fail "bad e2e params: kem_ct without kem_key"
       kemParams k = \case
         Nothing -> ARKP SRKSProposed $ RKParamsProposed k
         Just ct -> ARKP SRKSAccepted $ RKParamsAccepted ct k
@@ -451,6 +456,7 @@ generateSndE2EParams g v = \case
 
 data RatchetInitParams = RatchetInitParams
   { assocData :: Str,
+    rcVerifyCodePQ :: Str,
     ratchetKey :: RatchetKey,
     sndHK :: HeaderKey,
     rcvNextHK :: HeaderKey,
@@ -493,14 +499,14 @@ pqX3dhRcv (rpk1, rpk2, rpKem_) (E2ERatchetParams _ sk1 sk2 sKem_) = do
 
 pqX3dh :: DhAlgorithm a => (PublicKey a, PublicKey a) -> DhSecret a -> DhSecret a -> DhSecret a -> Maybe RatchetKEMAccepted -> RatchetInitParams
 pqX3dh (sk1, rk1) dh1 dh2 dh3 kemAccepted =
-  RatchetInitParams {assocData, ratchetKey = RatchetKey sk, sndHK = Key hk, rcvNextHK = Key nhk, kemAccepted}
+  RatchetInitParams {assocData, rcVerifyCodePQ = Str vcPQ, ratchetKey = RatchetKey sk, sndHK = Key hk, rcvNextHK = Key nhk, kemAccepted}
   where
     assocData = Str $ pubKeyBytes sk1 <> pubKeyBytes rk1
     dhs = dhBytes' dh1 <> dhBytes' dh2 <> dhBytes' dh3 <> pq
     pq = maybe "" (\RatchetKEMAccepted {rcPQRss = KEMSharedKey ss} -> BA.convert ss) kemAccepted
-    (hk, nhk, sk) =
-      let salt = B.replicate 64 '\0'
-       in hkdf3 salt dhs "SimpleXX3DH"
+    salt = B.replicate 64 '\0'
+    (hk, nhk, sk) = hkdf3 salt dhs "SimpleXX3DH"
+    vcPQ = hkdf salt dhs "SimpleXVerifyCode" 32
 
 type RatchetX448 = Ratchet 'X448
 
@@ -509,6 +515,8 @@ data Ratchet a = Ratchet
     rcVersion :: RatchetVersions,
     -- associated data - must be the same in both parties ratchets
     rcAD :: Str,
+    -- verification code covering all handshake keys, absent in the ratchets created before it was added
+    rcVCPQ :: Maybe Str,
     rcDHRs :: PrivateKey a,
     rcKEM :: Maybe RatchetKEM,
     rcSupportKEM :: PQSupport, -- defines header size, can only be enabled once
@@ -637,13 +645,14 @@ instance FromField MessageKey where fromField = blobFieldDecoder smpDecode
 -- @
 initSndRatchet ::
   forall a. (AlgorithmI a, DhAlgorithm a) => RatchetVersions -> PublicKey a -> PrivateKey a -> (RatchetInitParams, Maybe KEMKeyPair) -> Ratchet a
-initSndRatchet rcVersion rcDHRr rcDHRs (RatchetInitParams {assocData, ratchetKey, sndHK, rcvNextHK, kemAccepted}, rcPQRs_) = do
+initSndRatchet rcVersion rcDHRr rcDHRs (RatchetInitParams {assocData, rcVerifyCodePQ, ratchetKey, sndHK, rcvNextHK, kemAccepted}, rcPQRs_) = do
   -- state.RK, state.CKs, state.NHKs = KDF_RK_HE(SK, DH(state.DHRs, state.DHRr) || state.PQRss)
   let (rcRK, rcCKs, rcNHKs) = rootKdf ratchetKey rcDHRr rcDHRs (rcPQRss <$> kemAccepted)
       pqOn = isJust rcPQRs_
    in Ratchet
         { rcVersion,
           rcAD = assocData,
+          rcVCPQ = Just rcVerifyCodePQ,
           rcDHRs,
           rcKEM = (`RatchetKEM` kemAccepted) <$> rcPQRs_,
           rcSupportKEM = PQSupport pqOn,
@@ -668,10 +677,11 @@ initSndRatchet rcVersion rcDHRr rcDHRs (RatchetInitParams {assocData, ratchetKey
 -- as part of the connection request and random salt was received from the sender.
 initRcvRatchet ::
   forall a. (AlgorithmI a, DhAlgorithm a) => RatchetVersions -> PrivateKey a -> (RatchetInitParams, Maybe KEMKeyPair) -> PQSupport -> Ratchet a
-initRcvRatchet rcVersion rcDHRs (RatchetInitParams {assocData, ratchetKey, sndHK, rcvNextHK, kemAccepted}, rcPQRs_) pqSupport =
+initRcvRatchet rcVersion rcDHRs (RatchetInitParams {assocData, rcVerifyCodePQ, ratchetKey, sndHK, rcvNextHK, kemAccepted}, rcPQRs_) pqSupport =
   Ratchet
     { rcVersion,
       rcAD = assocData,
+      rcVCPQ = Just rcVerifyCodePQ,
       rcDHRs,
       -- rcKEM:
       -- state.PQRs = bob_pq_kem_key_pair
@@ -953,6 +963,9 @@ type DecryptResult a = (Either CryptoError ByteString, Ratchet a, SkippedMsgDiff
 maxSkip :: Word32
 maxSkip = 512
 
+maxSkippedMsgKeys :: Int
+maxSkippedMsgKeys = 2000
+
 rcDecrypt ::
   forall a.
   (AlgorithmI a, DhAlgorithm a) =>

src/Simplex/Messaging/Crypto/SNTRUP761/Bindings.hs

@@ -16,6 +16,8 @@ module Simplex.Messaging.Crypto.SNTRUP761.Bindings
   ) where
 
 import Control.Concurrent.STM
+import Control.Exception (throwIO)
+import Control.Monad (when)
 import Crypto.Random (ChaChaDRG)
 import Data.Aeson (FromJSON (..), ToJSON (..))
 import Data.Bifunctor (bimap)
@@ -23,6 +25,7 @@ import Data.ByteArray (ScrubbedBytes)
 import qualified Data.ByteArray as BA
 import Data.ByteString (ByteString)
 import Simplex.Messaging.Agent.Store.DB (FromField (..), ToField (..))
+import qualified Simplex.Messaging.Crypto as C
 import Simplex.Messaging.Crypto.SNTRUP761.Bindings.Defines
 import Simplex.Messaging.Crypto.SNTRUP761.Bindings.FFI
 import Simplex.Messaging.Crypto.SNTRUP761.Bindings.RNG (rngFuncPtr, withDRG)
@@ -55,14 +58,13 @@ pattern KEMSharedKey s <- KEMSharedKey_ s
 type KEMKeyPair = (KEMPublicKey, KEMSecretKey)
 
 sntrup761Keypair :: TVar ChaChaDRG -> IO KEMKeyPair
-sntrup761Keypair drg =
-  bimap KEMPublicKey_ KEMSecretKey
-    <$> BA.allocRet
-      c_SNTRUP761_SECRETKEY_SIZE
-      ( \skPtr ->
-          BA.alloc c_SNTRUP761_PUBLICKEY_SIZE $ \pkPtr ->
-            withDRG drg $ \cxtPtr -> c_sntrup761_keypair pkPtr skPtr cxtPtr rngFuncPtr
-      )
+sntrup761Keypair drg = do
+  ((r, pk), sk) <-
+    BA.allocRet c_SNTRUP761_SECRETKEY_SIZE $ \skPtr ->
+      BA.allocRet c_SNTRUP761_PUBLICKEY_SIZE $ \pkPtr ->
+        withDRG drg $ \cxtPtr -> c_sntrup761_keypair pkPtr skPtr cxtPtr rngFuncPtr
+  when (r /= 0) $ throwIO C.CryptoKEMKeyGenError
+  pure (KEMPublicKey_ pk, KEMSecretKey sk)
 
 sntrup761Enc :: TVar ChaChaDRG -> KEMPublicKey -> IO (KEMCiphertext, KEMSharedKey)
 sntrup761Enc drg (KEMPublicKey pk) =

src/Simplex/Messaging/Crypto/SNTRUP761/Bindings/FFI.hs

@@ -10,9 +10,9 @@ import Foreign
 import Foreign.C
 import Simplex.Messaging.Crypto.SNTRUP761.Bindings.RNG (RNGContext, RNGFunc)
 
--- void sntrup761_keypair (uint8_t *pk, uint8_t *sk, void *random_ctx, sntrup761_random_func *random);
+-- int sntrup761_keypair (uint8_t *pk, uint8_t *sk, void *random_ctx, sntrup761_random_func *random);
 foreign import ccall "sntrup761_keypair"
-  c_sntrup761_keypair :: Ptr Word8 -> Ptr Word8 -> Ptr RNGContext -> FunPtr RNGFunc -> IO ()
+  c_sntrup761_keypair :: Ptr Word8 -> Ptr Word8 -> Ptr RNGContext -> FunPtr RNGFunc -> IO CInt
 
 -- void sntrup761_enc (uint8_t *c, uint8_t *k, const uint8_t *pk, void *random_ctx, sntrup761_random_func *random);
 foreign import ccall "sntrup761_enc"

src/Simplex/Messaging/Crypto/Secp256k1.hs

@@ -0,0 +1,195 @@
+{-# LANGUAGE ForeignFunctionInterface #-}
+{-# LANGUAGE NamedFieldPuns #-}
+{-# LANGUAGE ScopedTypeVariables #-}
+
+-- | FFI bindings to libsecp256k1.
+module Simplex.Messaging.Crypto.Secp256k1
+  ( Secp256k1PrivateKey,
+    Secp256k1PublicKey,
+    Secp256k1Context,
+    PubKeyFormat (..),
+    RecoverableSignature (..),
+    mkPrivateKey,
+    unPrivateKey,
+    withContext,
+    secp256k1PublicKey,
+    secp256k1PublicKey_,
+    serializePublicKey,
+    privateKeyTweakAdd,
+    signRecoverable,
+  )
+where
+
+import Control.Concurrent.STM
+import Control.Exception (bracket, throwIO)
+import Control.Monad (void, when)
+import Crypto.Number.Serialize (i2ospOf_)
+import qualified Crypto.PubKey.ECC.Types as ECT
+import Crypto.Random (ChaChaDRG)
+import Data.ByteArray (ScrubbedBytes)
+import qualified Data.ByteArray as BA
+import Data.ByteString (ByteString)
+import qualified Data.ByteString as B
+import Foreign hiding (void)
+import Foreign.C
+import qualified Simplex.Messaging.Crypto as C
+import System.IO.Unsafe (unsafePerformIO)
+
+-- Sizes
+
+privateKeySize :: Int
+privateKeySize = 32
+
+compressedSize :: Int
+compressedSize = 33
+
+uncompressedSize :: Int
+uncompressedSize = 65
+
+-- | Internal size of @secp256k1_pubkey@ (opaque, not a serialization).
+pubKeyInternalSize :: Int
+pubKeyInternalSize = 64
+
+-- | Internal size of @secp256k1_ecdsa_recoverable_signature@.
+recSigInternalSize :: Int
+recSigInternalSize = 65
+
+compactSigSize :: Int
+compactSigSize = 64
+
+digestSize :: Int
+digestSize = 32
+
+-- Types
+
+newtype Secp256k1PrivateKey = Secp256k1PrivateKey ScrubbedBytes
+  deriving (Eq)
+
+-- | A public key in libsecp256k1's opaque 64-byte form; 'serializePublicKey' returns the SEC1 bytes.
+newtype Secp256k1PublicKey = Secp256k1PublicKey ByteString
+
+newtype Secp256k1Context = Secp256k1Context (Ptr Ctx)
+
+data PubKeyFormat = Compressed | Uncompressed
+  deriving (Eq, Show)
+
+-- | @r@ and @s@, 32 bytes big-endian each, and the recovery id, EIP-1559's @yParity@.
+data RecoverableSignature = RecoverableSignature
+  { rsR :: ByteString,
+    rsS :: ByteString,
+    rsRecId :: Word8
+  }
+  deriving (Eq, Show)
+
+-- FFI
+
+data Ctx
+
+data PubKeyRaw
+
+data RecSigRaw
+
+foreign import ccall "secp256k1_context_create"
+  c_context_create :: CUInt -> IO (Ptr Ctx)
+
+foreign import ccall "secp256k1_context_destroy"
+  c_context_destroy :: Ptr Ctx -> IO ()
+
+foreign import ccall "secp256k1_context_randomize"
+  c_context_randomize :: Ptr Ctx -> Ptr Word8 -> IO CInt
+
+foreign import ccall "&secp256k1_context_static"
+  c_context_static :: Ptr (Ptr Ctx)
+
+foreign import ccall "secp256k1_ec_pubkey_create"
+  c_ec_pubkey_create :: Ptr Ctx -> Ptr PubKeyRaw -> Ptr Word8 -> IO CInt
+
+foreign import ccall "secp256k1_ec_pubkey_serialize"
+  c_ec_pubkey_serialize :: Ptr Ctx -> Ptr Word8 -> Ptr CSize -> Ptr PubKeyRaw -> CUInt -> IO CInt
+
+foreign import ccall "secp256k1_ec_seckey_tweak_add"
+  c_ec_seckey_tweak_add :: Ptr Ctx -> Ptr Word8 -> Ptr Word8 -> IO CInt
+
+foreign import ccall "secp256k1_ecdsa_sign_recoverable"
+  c_ecdsa_sign_recoverable :: Ptr Ctx -> Ptr RecSigRaw -> Ptr Word8 -> Ptr Word8 -> Ptr () -> Ptr () -> IO CInt
+
+foreign import ccall "secp256k1_ecdsa_recoverable_signature_serialize_compact"
+  c_ecdsa_recoverable_signature_serialize_compact :: Ptr Ctx -> Ptr Word8 -> Ptr CInt -> Ptr RecSigRaw -> IO CInt
+
+-- SECP256K1_CONTEXT_NONE = SECP256K1_FLAGS_TYPE_CONTEXT
+contextNone :: CUInt
+contextNone = 1
+
+staticContext :: Ptr Ctx
+staticContext = unsafePerformIO $ peek c_context_static
+{-# NOINLINE staticContext #-}
+
+withContext :: TVar ChaChaDRG -> (Secp256k1Context -> IO a) -> IO a
+withContext g f = bracket (c_context_create contextNone) c_context_destroy $ \ctx -> do
+  seed :: ScrubbedBytes <- atomically $ C.randomBytes' 32 g
+  rc <- BA.withByteArray seed $ c_context_randomize ctx
+  when (rc /= 1) $ throwIO (userError "secp256k1_context_randomize failed")
+  f $ Secp256k1Context ctx
+
+-- Public API
+
+-- | Reject zero and anything at or above the group order, which is what makes 'secp256k1PublicKey' total.
+mkPrivateKey :: ScrubbedBytes -> Either String Secp256k1PrivateKey
+mkPrivateKey bs
+  | BA.length bs /= privateKeySize = Left $ "private key: expected 32 bytes, got " <> show (BA.length bs)
+  | BA.all (== 0) bs || bs >= groupOrder = Left "private key: not in [1, n-1]"
+  | otherwise = Right $ Secp256k1PrivateKey bs
+
+-- | Compared as bytes: both are 32 bytes big-endian, and 'ScrubbedBytes' compares lexicographically.
+groupOrder :: ScrubbedBytes
+groupOrder = i2ospOf_ privateKeySize $ ECT.ecc_n $ ECT.common_curve $ ECT.getCurveByName ECT.SEC_p256k1
+
+unPrivateKey :: Secp256k1PrivateKey -> ScrubbedBytes
+unPrivateKey (Secp256k1PrivateKey bs) = bs
+
+secp256k1PublicKey :: TVar ChaChaDRG -> Secp256k1PrivateKey -> IO Secp256k1PublicKey
+secp256k1PublicKey g k = withContext g (`secp256k1PublicKey_` k)
+
+secp256k1PublicKey_ :: Secp256k1Context -> Secp256k1PrivateKey -> IO Secp256k1PublicKey
+secp256k1PublicKey_ (Secp256k1Context ctx) (Secp256k1PrivateKey sk) = do
+  (rc, pk) <- BA.allocRet pubKeyInternalSize $ \pkPtr -> BA.withByteArray sk $ c_ec_pubkey_create ctx pkPtr
+  when (rc /= 1) $ throwIO (userError "secp256k1_ec_pubkey_create failed on a validated key")
+  pure $ Secp256k1PublicKey pk
+
+serializePublicKey :: PubKeyFormat -> Secp256k1PublicKey -> IO ByteString
+serializePublicKey fmt (Secp256k1PublicKey pk) =
+  BA.alloc outLen $ \outPtr ->
+    with (fromIntegral outLen) $ \lenPtr ->
+      BA.withByteArray pk $ \pkPtr ->
+        void $ c_ec_pubkey_serialize staticContext outPtr lenPtr pkPtr flag
+  where
+    -- SECP256K1_EC_COMPRESSED = SECP256K1_FLAGS_TYPE_COMPRESSION | SECP256K1_FLAGS_BIT_COMPRESSION, SECP256K1_EC_UNCOMPRESSED = SECP256K1_FLAGS_TYPE_COMPRESSION
+    (flag, outLen) = case fmt of
+      Compressed -> (2 .|. 256, compressedSize)
+      Uncompressed -> (2, uncompressedSize)
+
+-- | @sk + tweak mod n@, as BIP-32 child derivation requires. 'Nothing' when the tweak is not 32 bytes or not below n, or when the result is zero.
+privateKeyTweakAdd :: Secp256k1PrivateKey -> ScrubbedBytes -> IO (Maybe Secp256k1PrivateKey)
+privateKeyTweakAdd (Secp256k1PrivateKey sk) tweak
+  | BA.length tweak /= privateKeySize = pure Nothing
+  | otherwise =
+      BA.withByteArray tweak $ \twPtr -> do
+        (rc, sk') <- BA.copyRet sk $ \skPtr -> c_ec_seckey_tweak_add staticContext skPtr twPtr
+        pure $ if rc == 1 then Just (Secp256k1PrivateKey sk') else Nothing
+
+-- | Signs a 32-byte digest, deterministically (RFC 6979) and always low-@s@, as EIP-2 requires.
+signRecoverable :: TVar ChaChaDRG -> Secp256k1PrivateKey -> ByteString -> IO (Either String RecoverableSignature)
+signRecoverable g (Secp256k1PrivateKey sk) digest
+  | B.length digest /= digestSize = pure $ Left $ "digest: expected 32 bytes, got " <> show (B.length digest)
+  | otherwise = withContext g $ \(Secp256k1Context ctx) ->
+      allocaBytes recSigInternalSize $ \sigPtr -> do
+        rc <- BA.withByteArray digest $ \msgPtr -> BA.withByteArray sk $ \skPtr -> c_ecdsa_sign_recoverable ctx sigPtr msgPtr skPtr nullPtr nullPtr
+        if rc /= 1
+          then pure $ Left "secp256k1_ecdsa_sign_recoverable failed on a validated key"
+          else do
+            (recId, compact) <- BA.allocRet compactSigSize $ \outPtr ->
+              alloca $ \recIdPtr -> do
+                void $ c_ecdsa_recoverable_signature_serialize_compact ctx outPtr recIdPtr sigPtr
+                peek recIdPtr
+            let (rsR, rsS) = B.splitAt (compactSigSize `div` 2) compact
+            pure $ Right RecoverableSignature {rsR, rsS, rsRecId = fromIntegral recId}

src/Simplex/Messaging/Crypto/ShortLink.hs

@@ -17,6 +17,7 @@ module Simplex.Messaging.Crypto.ShortLink
     encodeSignUserData,
     newOwnerAuth,
     encryptLinkData,
+    encryptFixedData,
     encryptUserData,
     decryptLinkData,
   )
@@ -84,9 +85,10 @@ newOwnerAuth g ownerId signingKey = do
   pure (ownerPrivKey, OwnerAuth {ownerId, ownerKey, authOwnerSig})
 
 encryptLinkData :: TVar ChaChaDRG -> C.SbKey -> (ByteString, ByteString) -> ExceptT AgentErrorType IO QueueLinkData
-encryptLinkData g k = bimapM (encrypt fixedDataPaddedLength) (encrypt userDataPaddedLength)
-  where
-    encrypt len = encryptData g k len
+encryptLinkData g k = bimapM (encryptFixedData g k) (encryptUserData g k)
+
+encryptFixedData :: TVar ChaChaDRG -> C.SbKey -> ByteString -> ExceptT AgentErrorType IO EncDataBytes
+encryptFixedData g k s = encryptData g k fixedDataPaddedLength s
 
 encryptUserData :: TVar ChaChaDRG -> C.SbKey -> ByteString -> ExceptT AgentErrorType IO EncDataBytes
 encryptUserData g k s = encryptData g k userDataPaddedLength s

src/Simplex/Messaging/Eth/Address.hs

@@ -0,0 +1,78 @@
+{-# LANGUAGE DataKinds #-}
+{-# LANGUAGE DerivingVia #-}
+{-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE TupleSections #-}
+
+module Simplex.Messaging.Eth.Address
+  ( Address,
+    unAddress,
+    addressFromPublicKey,
+    addressFromPrivateKey,
+    deriveAddress,
+  )
+where
+
+import Control.Applicative (optional)
+import Control.Concurrent.STM (TVar)
+import Control.Monad (unless, when, (<=<))
+import Crypto.Random (ChaChaDRG)
+import Data.Aeson (FromJSON, ToJSON)
+import qualified Data.Attoparsec.ByteString.Char8 as A
+import Data.Bits (shiftR, (.&.))
+import qualified Data.ByteArray.Encoding as BAE
+import Data.ByteString (ByteString)
+import qualified Data.ByteString as B
+import qualified Data.ByteString.Char8 as BC
+import Data.Char (isHexDigit, isLower, isUpper, toUpper)
+import Data.Word (Word32)
+import Simplex.Messaging.Crypto (keccak256)
+import Simplex.Messaging.Crypto.BIP32 (ExtendedKey, derivePath_, xkKey)
+import qualified Simplex.Messaging.Crypto.Secp256k1 as S
+import Simplex.Messaging.Encoding.String
+
+newtype Address = Address ByteString
+  deriving (Eq, Show)
+  deriving (ToJSON, FromJSON) via (StrJSON "Address" Address)
+
+instance StrEncoding Address where
+  strEncode = checksumAddress
+  strP = do
+    _ <- optional $ A.string "0x"
+    body <- A.takeWhile isHexDigit
+    unless (B.length body == addressSize * 2) $ fail $ "address: expected 40 hex digits, got " <> show (B.length body)
+    a <- Address <$> either fail pure (BAE.convertFromBase BAE.Base16 body)
+    when (mixedCase body && checksumAddress a /= "0x" <> body) $ fail "address: EIP-55 checksum mismatch"
+    pure a
+    where
+      mixedCase s = BC.any isUpper s && BC.any isLower s
+
+addressSize :: Int
+addressSize = 20
+
+unAddress :: Address -> ByteString
+unAddress (Address bs) = bs
+
+-- | The last 20 bytes of @keccak256@ of the uncompressed public key with its @0x04@ SEC1 prefix removed.
+addressFromPublicKey :: S.Secp256k1PublicKey -> IO Address
+addressFromPublicKey pk = Address . B.drop 12 . keccak256 . B.drop 1 <$> S.serializePublicKey S.Uncompressed pk
+
+addressFromPrivateKey :: TVar ChaChaDRG -> S.Secp256k1PrivateKey -> IO Address
+addressFromPrivateKey g k = S.withContext g (`addressFromPrivateKey_` k)
+
+addressFromPrivateKey_ :: S.Secp256k1Context -> S.Secp256k1PrivateKey -> IO Address
+addressFromPrivateKey_ ctx = addressFromPublicKey <=< S.secp256k1PublicKey_ ctx
+
+deriveAddress :: TVar ChaChaDRG -> ExtendedKey -> [Word32] -> IO (Either String (ExtendedKey, Address))
+deriveAddress g xk path = S.withContext g $ \ctx ->
+  derivePath_ ctx xk path >>= mapM (\k -> (k,) <$> addressFromPrivateKey_ ctx (xkKey k))
+
+-- | EIP-55: @0x@ and 40 hex digits whose case encodes a checksum over the lowercase hex form.
+checksumAddress :: Address -> ByteString
+checksumAddress (Address bs) = "0x" <> BC.pack (zipWith adjust [0 ..] (BC.unpack lowerHex))
+  where
+    lowerHex = BAE.convertToBase BAE.Base16 bs
+    hashed = keccak256 lowerHex
+    adjust i c = if isLower c && nibbleAt i >= 8 then toUpper c else c
+    nibbleAt i =
+      let byte = B.index hashed (i `div` 2)
+       in if even i then byte `shiftR` 4 else byte .&. 0x0F

src/Simplex/Messaging/Eth/RLP.hs

@@ -0,0 +1,34 @@
+{-# LANGUAGE LambdaCase #-}
+
+-- | Recursive Length Prefix, the encoding of Ethereum transactions.
+module Simplex.Messaging.Eth.RLP
+  ( RLPItem (..),
+    rlpEncode,
+    scalarItem,
+  )
+where
+
+import Data.ByteString (ByteString)
+import qualified Data.ByteString as B
+import Data.Word (Word32)
+import Simplex.Messaging.Encoding (smpEncode)
+
+data RLPItem = RLPBytes ByteString | RLPList [RLPItem]
+
+rlpEncode :: RLPItem -> ByteString
+rlpEncode = \case
+  RLPBytes b
+    | B.length b == 1 && B.head b < 0x80 -> b
+    | otherwise -> lengthPrefixed 0x80 b
+  RLPList items -> lengthPrefixed 0xc0 $ B.concat $ map rlpEncode items
+  where
+    lengthPrefixed offset payload
+      | len < 56 = B.cons (offset + fromIntegral len) payload
+      | otherwise = B.cons (offset + 55 + fromIntegral (B.length lenBytes)) lenBytes <> payload
+      where
+        len = B.length payload
+        lenBytes = B.dropWhile (== 0) $ smpEncode (fromIntegral len :: Word32)
+
+-- | Big-endian with no leading zeros, and zero as the empty string.
+scalarItem :: ByteString -> RLPItem
+scalarItem = RLPBytes . B.dropWhile (== 0)

src/Simplex/Messaging/Eth/Transaction.hs

@@ -0,0 +1,58 @@
+{-# LANGUAGE NamedFieldPuns #-}
+
+-- | EIP-1559 (type 2) transactions with an empty access list, signed for @eth_sendRawTransaction@.
+module Simplex.Messaging.Eth.Transaction
+  ( Eip1559Tx (..),
+    signEip1559Tx,
+  )
+where
+
+import Control.Concurrent.STM (TVar)
+import Crypto.Random (ChaChaDRG)
+import Data.Bits (shiftR)
+import Data.ByteString (ByteString)
+import qualified Data.ByteString as B
+import Data.WideWord.Word128 (Word128 (..))
+import Data.WideWord.Word256 (Word256 (..))
+import Data.Word (Word32, Word64)
+import Simplex.Messaging.Crypto (keccak256)
+import Simplex.Messaging.Crypto.Secp256k1 (RecoverableSignature (..), Secp256k1PrivateKey, signRecoverable)
+import Simplex.Messaging.Encoding (smpEncode)
+import Simplex.Messaging.Eth.Address (Address, unAddress)
+import Simplex.Messaging.Eth.RLP
+import Simplex.Messaging.Util ((<$$>))
+
+-- | Each width is the narrowest that both geth and reth accept.
+data Eip1559Tx = Eip1559Tx
+  { txChainId :: Word64,
+    txNonce :: Word64,
+    txMaxPriorityFeePerGas :: Word128,
+    txMaxFeePerGas :: Word128,
+    txGasLimit :: Word64,
+    txTo :: Address,
+    txValue :: Word256,
+    txData :: ByteString
+  }
+
+-- | @0x02 || rlp([chainId, nonce, maxPriorityFeePerGas, maxFeePerGas, gasLimit, to, value, data, accessList, yParity, r, s])@, signing @keccak256@ of the same without the last three.
+signEip1559Tx :: TVar ChaChaDRG -> Secp256k1PrivateKey -> Eip1559Tx -> IO (Either String ByteString)
+signEip1559Tx g sk Eip1559Tx {txChainId, txNonce, txMaxPriorityFeePerGas, txMaxFeePerGas, txGasLimit, txTo, txValue, txData} =
+  signed <$$> signRecoverable g sk (keccak256 $ typed fields)
+  where
+    signed RecoverableSignature {rsR, rsS, rsRecId} = typed $ fields <> [scalarItem (B.singleton rsRecId), scalarItem rsR, scalarItem rsS]
+    fields =
+      [ w64 txChainId,
+        w64 txNonce,
+        w128 txMaxPriorityFeePerGas,
+        w128 txMaxFeePerGas,
+        w64 txGasLimit,
+        RLPBytes $ unAddress txTo,
+        w256 txValue,
+        RLPBytes txData,
+        RLPList []
+      ]
+    w64 = scalarItem . be64
+    w128 (Word128 hi lo) = scalarItem $ be64 hi <> be64 lo
+    w256 (Word256 a3 a2 a1 a0) = scalarItem $ be64 a3 <> be64 a2 <> be64 a1 <> be64 a0
+    be64 n = smpEncode (fromIntegral (n `shiftR` 32) :: Word32) <> smpEncode (fromIntegral n :: Word32)
+    typed items = B.cons 0x02 $ rlpEncode $ RLPList items

src/Simplex/Messaging/Names/Record.hs

@@ -1,3 +1,6 @@
+{-# LANGUAGE DerivingStrategies #-}
+{-# LANGUAGE GeneralizedNewtypeDeriving #-}
+{-# LANGUAGE LambdaCase #-}
 {-# LANGUAGE NamedFieldPuns #-}
 {-# LANGUAGE OverloadedStrings #-}
 {-# LANGUAGE StrictData #-}
@@ -5,13 +8,24 @@
 
 module Simplex.Messaging.Names.Record
   ( NameRecord (..),
+    NameResponse (..),
+    NameRegistration (..),
+    NamePricing (..),
+    USDCents (..),
+    NameReservedReason (..),
   )
 where
 
+import Data.Aeson (FromJSON (..), ToJSON (..))
 import qualified Data.Aeson as J
 import qualified Data.Aeson.TH as JQ
+import Data.Int (Int64)
+import Data.Map.Strict (Map)
 import Data.Text (Text)
-import Simplex.Messaging.Parsers (defaultJSON, dropPrefix)
+import qualified Data.Text as T
+import Simplex.Messaging.Encoding.String
+import Simplex.Messaging.Parsers (defaultJSON, dropPrefix, taggedObjectJSON)
+import Simplex.Messaging.SystemTime (SystemSeconds)
 
 -- | Resolved name record returned by the names role. JSON keys match the
 --   resolver REST output; both FromJSON (resolver -> server) and ToJSON
@@ -44,3 +58,87 @@ $( JQ.deriveJSON
     defaultJSON {J.omitNothingFields = False, J.fieldLabelModifier = dropPrefix "nr"}
     ''NameRecord
  )
+
+-- | US cents.
+newtype USDCents = USDCents Int64
+  deriving (Eq, Ord, Show)
+  deriving newtype (ToJSON, FromJSON)
+
+-- | What the registry holds for a name, and the block it was read at.
+data NameResponse = NameResponse
+  { -- | absent only from a v20/v21 router, which sent the record alone
+    lastBlockTs :: Maybe SystemSeconds,
+    registration :: NameRegistration
+  }
+  deriving (Eq, Show)
+
+-- | What the registry holds for a name.
+data NameRegistration
+  = -- | Held by someone. Always carries a record, empty where none was set.
+    NRRegistered
+      { -- | absent only from a v20/v21 router, which sent the record alone
+        expires :: Maybe SystemSeconds,
+        -- | unix seconds, > expires: until here only the owner may renew
+        graceUntil :: Maybe SystemSeconds,
+        -- | held back as well, which is why it will not free up at expiry
+        reservedReason_ :: Maybe NameReservedReason,
+        nameRecord :: NameRecord
+      }
+  | -- | Held by nobody, and registrable now.
+    NRAvailable {pricing :: NamePricing}
+  | -- | Held back by the registry, and not for sale at its price.
+    NRReserved {reservedReason :: NameReservedReason}
+  deriving (Eq, Show)
+
+-- | Enough to price the name locally, which the router cannot do behind a hash.
+data NamePricing = NamePricing
+  { -- | US cents per year, for the lengths the registry prices specially
+    registrationPrices :: Map Int USDCents,
+    -- | US cents per year for every other length
+    basePrice :: USDCents,
+    -- | characters; the registry refuses shorter labels
+    minLabelLength :: Int
+  }
+  deriving (Eq, Show)
+
+-- | Why the registry holds a name back.
+data NameReservedReason
+  = -- | held for SimpleX
+    NRRInternal
+  | NRRTrademark
+  | NRRCommunity
+  | -- | added to the registry after this version, and still reserved
+    NRRUnknown Text
+  deriving (Eq, Show)
+
+instance TextEncoding NameReservedReason where
+  textEncode = \case
+    NRRInternal -> "internal"
+    NRRTrademark -> "trademark"
+    NRRCommunity -> "community"
+    NRRUnknown t -> t
+  textDecode = Just . reservedReasonOf
+
+-- | An unknown reason is kept as text, capped: it reaches a client as a word.
+-- Capping precedes the match, so what is kept encodes back to what it decoded.
+reservedReasonOf :: Text -> NameReservedReason
+reservedReasonOf t = case T.take 32 $ T.takeWhile (\c -> c > ' ' && c < '\DEL') t of
+  "internal" -> NRRInternal
+  "trademark" -> NRRTrademark
+  "community" -> NRRCommunity
+  r -> NRRUnknown r
+
+instance ToJSON NameReservedReason where
+  toJSON = textToJSON
+  toEncoding = textToEncoding
+
+instance FromJSON NameReservedReason where
+  parseJSON = textParseJSON "NameReservedReason"
+
+$(JQ.deriveJSON defaultJSON ''NamePricing)
+
+-- taggedObjectJSON, not sumTypeJSON: this JSON is the RNAME payload and the
+-- resolver contract, so it must not vary with the swift build flag.
+$(JQ.deriveJSON (taggedObjectJSON $ dropPrefix "NR") ''NameRegistration)
+
+$(JQ.deriveJSON defaultJSON ''NameResponse)

src/Simplex/Messaging/Notifications/Transport.hs

@@ -132,7 +132,7 @@ ntfClientHandshake c keyHash ntfVRange _proxyServer _serviceKeys = do
 
 ntfThHandleServer :: forall c. THandleNTF c 'TServer -> VersionNTF -> VersionRangeNTF -> C.PrivateKeyX25519 -> THandleNTF c 'TServer
 ntfThHandleServer th v vr pk =
-  let thAuth = THAuthServer {serverPrivKey = pk, peerClientService = Nothing, sessSecret' = Nothing}
+  let thAuth = THAuthServer {serverPrivKey = pk, peerClientService = Nothing, peerEntitlement = Nothing, sessSecret' = Nothing}
    in ntfThHandle_ th v vr (Just thAuth)
 
 ntfThHandleClient :: forall c. THandleNTF c 'TClient -> VersionNTF -> VersionRangeNTF -> (C.PublicKeyX25519, CertChainPubKey) -> THandleNTF c 'TClient

src/Simplex/Messaging/Protocol.hs

@@ -80,6 +80,12 @@ module Simplex.Messaging.Protocol
     ErrorType (..),
     CommandError (..),
     ProxyError (..),
+    NameQuery (..),
+    NameResponse (..),
+    NameRegistration (..),
+    NamePricing (..),
+    USDCents (..),
+    NameReservedReason (..),
     NameErrorType (..),
     BrokerErrorType (..),
     NetworkError (..),
@@ -139,7 +145,10 @@ module Simplex.Messaging.Protocol
     RcvNtfDhSecret,
     Message (..),
     RcvMessage (..),
-    MsgId,
+    MsgId (unMsgId),
+    mkMsgId,
+    unsafeMsgId,
+    randomMsgId,
     MsgBody,
     IdsHash (..),
     ServiceSub (..),
@@ -162,6 +171,7 @@ module Simplex.Messaging.Protocol
     EncFwdTransmission (..),
     EncResponse (..),
     EncTransmission (..),
+    encTransmissionNonce,
     FwdResponse (..),
     FwdTransmission (..),
     NameRecord (..),
@@ -226,8 +236,10 @@ module Simplex.Messaging.Protocol
 where
 
 import Control.Applicative (optional, (<|>))
+import Control.Concurrent.STM (STM, TVar)
 import Control.Exception (Exception, SomeException, displayException, fromException)
 import Control.Monad.Except
+import Crypto.Random (ChaChaDRG)
 import Data.Aeson (FromJSON (..), ToJSON (..))
 import qualified Data.Aeson as J
 import qualified Data.Aeson.TH as J
@@ -235,6 +247,7 @@ import Data.Attoparsec.ByteString.Char8 (Parser, (<?>))
 import qualified Data.Attoparsec.ByteString.Char8 as A
 import Data.Bifunctor (bimap, first)
 import Data.Bits (xor)
+import qualified Data.ByteArray as BA
 import qualified Data.ByteString as BS
 import qualified Data.ByteString.Base64 as B64
 import Data.ByteString.Char8 (ByteString)
@@ -261,16 +274,16 @@ import qualified GHC.TypeLits as TE
 import qualified GHC.TypeLits as Type
 import Network.Socket (ServiceName)
 import qualified Network.TLS as TLS
-import Simplex.Messaging.Agent.Store.DB (Binary (..), FromField (..), ToField (..))
+import Simplex.Messaging.Agent.Store.DB (Binary (..), FromField (..), ToField (..), blobFieldDecoder)
 import qualified Simplex.Messaging.Crypto as C
 import Simplex.Messaging.Encoding
 import Simplex.Messaging.Encoding.String
-import Simplex.Messaging.Names.Record (NameRecord (..))
+import Simplex.Messaging.Names.Record
 import Simplex.Messaging.Parsers
 import Simplex.Messaging.Protocol.Types
 import Simplex.Messaging.Server.QueueStore.QueueInfo
 import Simplex.Messaging.ServiceScheme
-import Simplex.Messaging.SimplexName (SimplexDomain)
+import Simplex.Messaging.SimplexName (LabelHash, SimplexDomain (..), SimplexTLD (..), fullDomainName, labelHash)
 import Simplex.Messaging.Transport
 import Simplex.Messaging.Transport.Client (TransportHost, TransportHosts (..))
 import Simplex.Messaging.Util (bshow, eitherToMaybe, safeDecodeUtf8, (<$?>))
@@ -603,7 +616,7 @@ data Command (p :: Party) where
   -- - corrId: unique correlation ID between proxy and relay, also used as a nonce to encrypt forwarded transmission
   RFWD :: EncFwdTransmission -> Command ProxyService -- use CorrId as CbNonce, proxy to relay
   -- Resolve SimpleX name.
-  RSLV :: SimplexDomain -> Command Resolver
+  RSLV :: NameQuery -> Command Resolver
 
 deriving instance Show (Command p)
 
@@ -695,19 +708,24 @@ instance Encoding NewNtfCreds where
 newtype EncTransmission = EncTransmission ByteString
   deriving (Show)
 
+encTransmissionNonce :: C.CbNonceI n => VersionSMP -> n -> C.CbNonce
+encTransmissionNonce v nonce
+  | v >= fwdNoncesSMPVersion = C.xorNonce (smpEncode v) nonce
+  | otherwise = C.toCbNonce nonce
+
 data FwdTransmission = FwdTransmission
-  { fwdCorrId :: CorrId,
+  { fwdCorrId :: C.CorrCbNonce,
     fwdVersion :: VersionSMP,
     fwdKey :: C.PublicKeyX25519,
     fwdTransmission :: EncTransmission
   }
 
 instance Encoding FwdTransmission where
-  smpEncode FwdTransmission {fwdCorrId = CorrId corrId, fwdVersion, fwdKey, fwdTransmission = EncTransmission t} =
-    smpEncode (corrId, fwdVersion, fwdKey, Tail t)
+  smpEncode FwdTransmission {fwdCorrId, fwdVersion, fwdKey, fwdTransmission = EncTransmission t} =
+    smpEncode (fwdCorrId, fwdVersion, fwdKey, Tail t)
   smpP = do
-    (corrId, fwdVersion, fwdKey, Tail t) <- smpP
-    pure FwdTransmission {fwdCorrId = CorrId corrId, fwdVersion, fwdKey, fwdTransmission = EncTransmission t}
+    (fwdCorrId, fwdVersion, fwdKey, Tail t) <- smpP
+    pure FwdTransmission {fwdCorrId, fwdVersion, fwdKey, fwdTransmission = EncTransmission t}
 
 newtype EncFwdTransmission = EncFwdTransmission ByteString
   deriving (Show)
@@ -730,8 +748,8 @@ data BrokerMsg where
   NMSG :: C.CbNonce -> EncNMsgMeta -> BrokerMsg
   -- Should include certificate chain
   PKEY :: SessionId -> VersionRangeSMP -> CertChainPubKey -> BrokerMsg -- TLS-signed server key for proxy shared secret and initial sender key
-  RRES :: EncFwdResponse -> BrokerMsg -- relay to proxy
-  PRES :: EncResponse -> BrokerMsg -- proxy to client
+  RRES :: Maybe C.CbNonce -> EncFwdResponse -> BrokerMsg -- relay to proxy
+  PRES :: Maybe C.CbNonce -> EncResponse -> BrokerMsg -- proxy to client
   END :: BrokerMsg
   ENDS :: Int64 -> IdsHash -> BrokerMsg
   DELD :: BrokerMsg
@@ -739,8 +757,8 @@ data BrokerMsg where
   OK :: BrokerMsg
   ERR :: ErrorType -> BrokerMsg
   PONG :: BrokerMsg
-  -- Resolved SimpleX name.
-  RNAME :: NameRecord -> BrokerMsg
+  -- What the router knows about a SimpleX name.
+  RNAME :: NameResponse -> BrokerMsg
   deriving (Eq, Show)
 
 data RcvMessage = RcvMessage
@@ -753,16 +771,16 @@ newtype EncFwdResponse = EncFwdResponse ByteString
   deriving (Eq, Show)
 
 data FwdResponse = FwdResponse
-  { fwdCorrId :: CorrId,
+  { fwdCorrId :: C.CorrCbNonce,
     fwdResponse :: EncResponse
   }
 
 instance Encoding FwdResponse where
-  smpEncode FwdResponse {fwdCorrId = CorrId corrId, fwdResponse = EncResponse t} =
-    smpEncode (corrId, Tail t)
+  smpEncode FwdResponse {fwdCorrId, fwdResponse = EncResponse t} =
+    smpEncode (fwdCorrId, Tail t)
   smpP = do
-    (corrId, Tail t) <- smpP
-    pure FwdResponse {fwdCorrId = CorrId corrId, fwdResponse = EncResponse t}
+    (fwdCorrId, Tail t) <- smpP
+    pure FwdResponse {fwdCorrId, fwdResponse = EncResponse t}
 
 newtype EncResponse = EncResponse ByteString
   deriving (Eq, Show)
@@ -785,7 +803,7 @@ toMsgInfo = \case
   Message {msgId, msgTs} -> msgInfo msgId msgTs MTMessage
   MessageQuota {msgId, msgTs} -> msgInfo msgId msgTs MTQuota
   where
-    msgInfo msgId msgTs msgType = MsgInfo {msgId = decodeLatin1 $ B64.encode msgId, msgTs = systemToUTCTime msgTs, msgType}
+    msgInfo msgId msgTs msgType = MsgInfo {msgId = decodeLatin1 $ B64.encode $ unMsgId msgId, msgTs = systemToUTCTime msgTs, msgType}
 
 messageId :: Message -> MsgId
 messageId = \case
@@ -1317,7 +1335,10 @@ instance ProtocolTypeI p => FromJSON (ProtocolServer p) where
   parseJSON = strParseJSON "ProtocolServer"
 
 newtype BasicAuth = BasicAuth {unBasicAuth :: ByteString}
-  deriving (Eq, Ord, Show)
+  deriving (Ord, Show)
+
+instance Eq BasicAuth where
+  BasicAuth s == BasicAuth s' = BA.constEq s s'
 
 instance IsString BasicAuth where fromString = BasicAuth . B.pack
 
@@ -1483,7 +1504,34 @@ type RcvNtfPublicDhKey = C.PublicKeyX25519
 type RcvNtfDhSecret = C.DhSecretX25519
 
 -- | SMP message server ID.
-type MsgId = ByteString
+newtype MsgId = MsgId {unMsgId :: ByteString}
+  deriving (Eq, Show)
+
+instance C.CbNonceI MsgId where
+  unCbNonce (MsgId s) = B.take 24 s
+
+instance Encoding MsgId where
+  smpEncode (MsgId s) = smpEncode s
+  smpP = mkMsgId <$?> smpP
+
+instance StrEncoding MsgId where
+  strEncode (MsgId s) = strEncode s
+  strP = mkMsgId <$?> strP
+
+instance FromField MsgId where fromField = blobFieldDecoder mkMsgId
+
+instance ToField MsgId where toField (MsgId s) = toField $ Binary s
+
+mkMsgId :: ByteString -> Either String MsgId
+mkMsgId s
+  | B.length s >= 24 = Right $ MsgId s
+  | otherwise = Left "MsgId: invalid length"
+
+unsafeMsgId :: ByteString -> MsgId
+unsafeMsgId s = either error id $ mkMsgId s
+
+randomMsgId :: Int -> TVar ChaChaDRG -> STM MsgId
+randomMsgId n = fmap MsgId . C.randomBytes (max 24 n)
 
 -- | SMP message body.
 type MsgBody = ByteString
@@ -1589,11 +1637,28 @@ data ErrorType
     DUPLICATE_ -- not part of SMP protocol, used internally
   deriving (Eq, Show)
 
+-- | What RSLV asks about: a name, or the hash of a second-level label.
+data NameQuery = NQDomain SimplexDomain | NQHash LabelHash SimplexTLD
+  deriving (Eq, Show)
+
+instance Encoding NameQuery where
+  smpEncode = \case
+    NQDomain d -> encodeUtf8 $ fullDomainName d
+    NQHash h tld -> strEncode h <> strEncode tld
+  smpP = NQHash <$> strP <*> strP <|> NQDomain <$> strP
+
+-- | Hashed from v22, except a name with subnames or a web TLD.
+hashedQuery :: NameQuery -> NameQuery
+hashedQuery q = case q of
+  NQDomain SimplexDomain {nameTLD, domain, subDomain}
+    | null subDomain && nameTLD /= TLDWeb -> NQHash (labelHash domain) nameTLD
+  _ -> q
+
 -- | Name resolution error
 data NameErrorType
   = -- | the names role / resolver is not configured on this server
     NO_RESOLVER
-  | -- | the name is not registered (resolver returned not-found)
+  | -- | the name does not resolve; sent only to a session below v22
     NOT_FOUND
   | -- | backing resolver/RPC failure - contains the diagnostic detail
     RESOLVER {resolverErr :: Text}
@@ -1791,8 +1856,7 @@ instance PartyI p => ProtocolEncoding SMPVersion ErrorType (Command p) where
   encodeProtocol v = \case
     NEW NewQueueReq {rcvAuthKey = rKey, rcvDhKey = dhKey, auth_, subMode, queueReqData, ntfCreds}
       | v >= newNtfCredsSMPVersion -> new <> e (subMode, queueReqData, ntfCreds)
-      | v >= shortLinksSMPVersion -> new <> e (subMode, queueReqData)
-      | otherwise -> new <> e (subMode, senderCanSecure (queueReqMode <$> queueReqData))
+      | otherwise -> new <> e (subMode, queueReqData)
       where
         new = e (NEW_, ' ', rKey, dhKey, auth_)
     SUB -> e SUB_
@@ -1822,7 +1886,7 @@ instance PartyI p => ProtocolEncoding SMPVersion ErrorType (Command p) where
     PRXY host auth_ -> e (PRXY_, ' ', host, auth_)
     PFWD fwdV pubKey (EncTransmission s) -> e (PFWD_, ' ', fwdV, pubKey, Tail s)
     RFWD (EncFwdTransmission s) -> e (RFWD_, ' ', Tail s)
-    RSLV d -> e (RSLV_, ' ', d)
+    RSLV q -> e (RSLV_, ' ', if v >= nameAvailSMPVersion then hashedQuery q else q)
     where
       e :: Encoding a => a -> ByteString
       e = smpEncode
@@ -1879,20 +1943,18 @@ instance ProtocolEncoding SMPVersion ErrorType Cmd where
     CT SCreator NEW_ -> Cmd SCreator <$> newCmd
       where
         newCmd
-          | v >= newNtfCredsSMPVersion = new smpP smpP
-          | v >= shortLinksSMPVersion = new smpP nothing
-          | otherwise = new (qReq <$> smpP) nothing
+          | v >= newNtfCredsSMPVersion = new smpP
+          | otherwise = new nothing
           where
             nothing = pure Nothing
-            new p2 p3 = NEW <$> do
+            new p3 = NEW <$> do
               rcvAuthKey <- _smpP
               rcvDhKey <- smpP
               auth_ <- smpP
               subMode <- smpP
-              queueReqData <- p2
+              queueReqData <- smpP
               ntfCreds <- p3
               pure NewQueueReq {rcvAuthKey, rcvDhKey, auth_, subMode, queueReqData, ntfCreds}
-            qReq sndSecure = Just $ if sndSecure then QRMessaging Nothing else QRContact Nothing
     CT SRecipient tag ->
       Cmd SRecipient <$> case tag of
         SUB_ -> pure SUB
@@ -1929,7 +1991,7 @@ instance ProtocolEncoding SMPVersion ErrorType Cmd where
     CT SNotifierService NSUBS_
       | v >= rcvServiceSMPVersion -> Cmd SNotifierService <$> (NSUBS <$> _smpP <*> smpP)
       | otherwise -> pure $ Cmd SNotifierService $ NSUBS (-1) mempty
-    CT SResolver RSLV_ -> Cmd SResolver . RSLV <$> _smpP <* A.takeByteString
+    CT SResolver RSLV_ -> Cmd SResolver . RSLV <$> _smpP
 
   fromProtocolError = fromProtocolError @SMPVersion @ErrorType @BrokerMsg
   {-# INLINE fromProtocolError #-}
@@ -1943,8 +2005,7 @@ instance ProtocolEncoding SMPVersion ErrorType BrokerMsg where
     IDS QIK {rcvId, sndId, rcvPublicDhKey = srvDh, queueMode, linkId, serviceId, serverNtfCreds}
       | v >= newNtfCredsSMPVersion -> ids <> e (queueMode, linkId, serviceId, serverNtfCreds)
       | v >= serviceCertsSMPVersion -> ids <> e (queueMode, linkId, serviceId)
-      | v >= shortLinksSMPVersion -> ids <> e (queueMode, linkId)
-      | otherwise -> ids <> e (senderCanSecure queueMode)
+      | otherwise -> ids <> e (queueMode, linkId)
       where
         ids = e (IDS_, ' ', rcvId, sndId, srvDh)
     LNK sId d -> e (LNK_, ' ', sId, d)
@@ -1958,8 +2019,8 @@ instance ProtocolEncoding SMPVersion ErrorType BrokerMsg where
     NID nId srvNtfDh -> e (NID_, ' ', nId, srvNtfDh)
     NMSG nmsgNonce encNMsgMeta -> e (NMSG_, ' ', nmsgNonce, encNMsgMeta)
     PKEY sid vr certKey -> e (PKEY_, ' ', sid, vr, certKey)
-    RRES (EncFwdResponse encBlock) -> e (RRES_, ' ', Tail encBlock)
-    PRES (EncResponse encBlock) -> e (PRES_, ' ', Tail encBlock)
+    RRES nonce_ (EncFwdResponse encBlock) -> fwdResp RRES_ nonce_ encBlock
+    PRES nonce_ (EncResponse encBlock) -> fwdResp PRES_ nonce_ encBlock
     END -> e END_
     ENDS n idsHash -> serviceResp ENDS_ n idsHash
     DELD -> e DELD_
@@ -1972,13 +2033,20 @@ instance ProtocolEncoding SMPVersion ErrorType BrokerMsg where
             | v < clientNoticesSMPVersion -> BLOCKED info {notice = Nothing}
           _ -> err
     PONG -> e PONG_
-    RNAME rec -> e (RNAME_, ' ', Tail $ LB.toStrict $ J.encode rec)
+    RNAME res
+      | v >= nameAvailSMPVersion -> e (RNAME_, ' ', Tail $ LB.toStrict $ J.encode res)
+      | otherwise -> case registration res of
+          NRRegistered {nameRecord} -> e (RNAME_, ' ', Tail $ LB.toStrict $ J.encode nameRecord)
+          _ -> e (ERR_, ' ', NAME NOT_FOUND)
     where
       e :: Encoding a => a -> ByteString
       e = smpEncode
       serviceResp tag n idsHash
         | v >= rcvServiceSMPVersion = e (tag, ' ', n, idsHash)
         | otherwise = e (tag, ' ', n)
+      fwdResp tag nonce_ encBlock
+        | v >= fwdNoncesSMPVersion = e (tag, ' ', nonce_, Tail encBlock)
+        | otherwise = e (tag, ' ', Tail encBlock)
 
   protocolP v = \case
     MSG_ -> do
@@ -1988,19 +2056,17 @@ instance ProtocolEncoding SMPVersion ErrorType BrokerMsg where
         bodyP = EncRcvMsgBody . unTail <$> smpP
     ALLS_ -> pure ALLS
     IDS_
-      | v >= newNtfCredsSMPVersion -> ids smpP smpP smpP smpP
-      | v >= serviceCertsSMPVersion -> ids smpP smpP smpP nothing
-      | v >= shortLinksSMPVersion -> ids smpP smpP nothing nothing
-      | otherwise -> ids (qm <$> smpP) nothing nothing nothing
+      | v >= newNtfCredsSMPVersion -> ids smpP smpP
+      | v >= serviceCertsSMPVersion -> ids smpP nothing
+      | otherwise -> ids nothing nothing
       where
-        qm sndSecure = Just $ if sndSecure then QMMessaging else QMContact
         nothing = pure Nothing
-        ids p1 p2 p3 p4 = do
+        ids p3 p4 = do
           rcvId <- _smpP
           sndId <- smpP
           rcvPublicDhKey <- smpP
-          queueMode <- p1
-          linkId <- p2
+          queueMode <- smpP
+          linkId <- smpP
           serviceId <- p3
           serverNtfCreds <- p4
           pure $ IDS QIK {rcvId, sndId, rcvPublicDhKey, queueMode, linkId, serviceId, serverNtfCreds}
@@ -2010,8 +2076,8 @@ instance ProtocolEncoding SMPVersion ErrorType BrokerMsg where
     NID_ -> NID <$> _smpP <*> smpP
     NMSG_ -> NMSG <$> _smpP <*> smpP
     PKEY_ -> PKEY <$> _smpP <*> smpP <*> smpP
-    RRES_ -> RRES <$> (EncFwdResponse . unTail <$> _smpP)
-    PRES_ -> PRES <$> (EncResponse . unTail <$> _smpP)
+    RRES_ -> fwdRespP RRES EncFwdResponse
+    PRES_ -> fwdRespP PRES EncResponse
     END_ -> pure END
     ENDS_ -> serviceRespP ENDS
     DELD_ -> pure DELD
@@ -2019,11 +2085,18 @@ instance ProtocolEncoding SMPVersion ErrorType BrokerMsg where
     OK_ -> pure OK
     ERR_ -> ERR <$> _smpP
     PONG_ -> pure PONG
-    RNAME_ -> fmap RNAME . J.eitherDecodeStrict . unTail <$?> _smpP
+    RNAME_
+      | v >= nameAvailSMPVersion -> fmap RNAME . J.eitherDecodeStrict . unTail <$?> _smpP
+      | otherwise -> fmap (RNAME . oldResponse) . J.eitherDecodeStrict . unTail <$?> _smpP
     where
+      oldResponse nameRecord =
+        NameResponse {lastBlockTs = Nothing, registration = NRRegistered {expires = Nothing, graceUntil = Nothing, reservedReason_ = Nothing, nameRecord}}
       serviceRespP resp
         | v >= rcvServiceSMPVersion = resp <$> _smpP <*> smpP
         | otherwise = resp <$> _smpP <*> pure mempty
+      fwdRespP :: (Maybe C.CbNonce -> a -> BrokerMsg) -> (ByteString -> a) -> Parser BrokerMsg
+      fwdRespP resp enc = resp <$> (A.space *> nonceP) <*> (enc <$> A.takeByteString)
+      nonceP = if v >= fwdNoncesSMPVersion then smpP else pure Nothing
 
   fromProtocolError = \case
     PECmdSyntax -> CMD SYNTAX
@@ -2040,9 +2113,9 @@ instance ProtocolEncoding SMPVersion ErrorType BrokerMsg where
     -- PONG response must not have queue ID
     PONG -> noEntityMsg
     PKEY {} -> noEntityMsg
-    RRES _ -> noEntityMsg
+    RRES {} -> noEntityMsg
     ALLS -> noEntityMsg
-    RNAME _ -> noEntityMsg
+    RNAME {} -> noEntityMsg
     -- other broker responses must have queue ID
     _
       | B.null entId -> Left $ CMD NO_ENTITY
@@ -2376,7 +2449,9 @@ tDecodeServer THandleParams {sessionId, thVersion = v, implySessId} = \case
           where
             cmdOrErr = parseProtocol @v @err @cmd v command >>= checkCredentials tAuth entityId
             t :: a -> (CorrId, EntityId, a)
-            t = (corrId,entityId,)
+            -- IDs are slices of the ~16 KB received block and are kept as subscription keys,
+            -- so without a copy one live key retains the whole block
+            t = (CorrId $ B.copy $ bs corrId,EntityId $ B.copy $ unEntityId entityId,)
         Left _ -> tError corrId PEBlock
     | otherwise -> tError corrId PESession
   Left _ -> tError "" PEBlock
@@ -2412,3 +2487,4 @@ $(J.deriveJSON defaultJSON ''BlockingInfo)
 
 -- run deriveJSON in one TH splice to allow mutual instance
 $(concat <$> mapM @[] (J.deriveJSON (sumTypeJSON id)) [''ProxyError, ''NameErrorType, ''ErrorType])
+

src/Simplex/Messaging/Server.hs

@@ -104,7 +104,6 @@ import qualified Simplex.Messaging.Crypto as C
 import Simplex.Messaging.Encoding
 import Simplex.Messaging.Encoding.String
 import Simplex.Messaging.Protocol
-import Simplex.Messaging.SimplexName (SimplexDomain)
 import Simplex.Messaging.Server.Control
 import Simplex.Messaging.Server.Env.STM as Env
 import Simplex.Messaging.Server.Expiration
@@ -1093,6 +1092,7 @@ controlPortAuth h user admin role auth = do
   readTVarIO role >>= \case
     CPRNone -> do
       atomically $ writeTVar role $! newRole
+      when (newRole == CPRNone) $ logWarn "ControlPort: failed auth"
       hPutStrLn h $ currentRole newRole
     r -> hPutStrLn h $ currentRole r <> if r == newRole then "" else ", start new session to change."
   where
@@ -1145,14 +1145,16 @@ cancelSub s = case subThread s of
 type VerifiedTransmissionOrError s = Either (Transmission BrokerMsg) (VerifiedTransmission s)
 
 receive :: forall c s. (Transport c, MsgStoreClass s) => THandleSMP c 'TServer -> s -> Client s -> M s ()
-receive h@THandle {params = THandleParams {thAuth, sessionId}} ms Client {rcvQ, sndQ, rcvActiveAt} = do
+receive h@THandle {params = THandleParams {thAuth, sessionId}} ms c@Client {rcvQ, rcvActiveAt} = do
   labelMyThread . B.unpack $ "client $" <> encode sessionId <> " receive"
   sa <- asks serverActive
   stats <- asks serverStats
+  authDelay <- asks $ authErrorDelay . config
   liftIO $ forever $ do
     ts <- tGetServer h
     unlessM (readTVarIO sa) $ throwIO $ userError "server stopped"
-    atomically . (writeTVar rcvActiveAt $!) =<< getSystemTime
+    receivedAt <- getSystemTime
+    atomically $ writeTVar rcvActiveAt $! receivedAt
     let (es, ts') = partitionEithers $ L.toList ts
         errs = map (second ERR) es
     errs' <- case ts' of
@@ -1168,7 +1170,7 @@ receive h@THandle {params = THandleParams {thAuth, sessionId}} ms Client {rcvQ,
         mapM_ (atomically . writeTBQueue rcvQ) $ L.nonEmpty cmds
         pure $ errs ++ errs'
       [] -> pure errs
-    mapM_ (atomically . writeTBQueue sndQ . (,[])) $ L.nonEmpty errs'
+    mapM_ (sendResponses c stats authDelay receivedAt . (,[])) $ L.nonEmpty errs'
   where
     sameParty :: SParty p -> SignedTransmission Cmd -> Bool
     sameParty p (_, _, (_, _, Cmd p' _)) = isJust $ testEquality p p'
@@ -1191,6 +1193,27 @@ receive h@THandle {params = THandleParams {thAuth, sessionId}} ms Client {rcvQ,
             GET -> incStat $ msgGetAuth stats
             _ -> pure ()
 
+isAuthError :: Transmission BrokerMsg -> Bool
+isAuthError (_, _, r) = case r of
+  ERR AUTH -> True
+  _ -> False
+
+sendResponses :: Client s -> ServerStats -> Int64 -> SystemTime -> (NonEmpty (Transmission BrokerMsg), [Transmission BrokerMsg]) -> IO ()
+sendResponses c@Client {sndQ} stats authDelay since rs@(ts, _)
+  | any isAuthError ts = sendAfterDelay c stats authDelay since rs
+  | otherwise = atomically $ writeTBQueue sndQ rs
+
+sendAfterDelay :: Client s -> ServerStats -> Int64 -> SystemTime -> (NonEmpty (Transmission BrokerMsg), [Transmission BrokerMsg]) -> IO ()
+sendAfterDelay c@Client {sndQ, clientTHParams = THandleParams {sessionId}} stats delay since rs
+  | delay > 0 = do
+      elapsed <- subtract (microseconds since) . microseconds <$> getSystemTime
+      when (elapsed >= delay) $ incStat $ authDelayExceeded stats
+      forkClient c (B.unpack $ "client $" <> encode sessionId <> " auth") $ threadDelay' (delay - elapsed `mod` delay) >> write
+  | otherwise = write
+  where
+    write = atomically $ writeTBQueue sndQ rs
+    microseconds (MkSystemTime s ns) = s * 1000000 + fromIntegral (ns `div` 1000)
+
 send :: Transport c => MVar (THandleSMP c 'TServer) -> Client s -> IO ()
 send th c@Client {sndQ, msgQ, clientTHParams = THandleParams {sessionId}} = do
   labelMyThread . B.unpack $ "client $" <> encode sessionId <> " send"
@@ -1319,11 +1342,6 @@ isContactQueue QueueRec {queueMode, senderKey} = case queueMode of
   Just QMContact -> True
   Nothing -> isNothing senderKey -- for backward compatibility with pre-SKEY contact addresses
 
-isSecuredMsgQueue :: QueueRec -> Bool
-isSecuredMsgQueue QueueRec {queueMode, senderKey} = case queueMode of
-  Just QMContact -> False
-  _ -> isJust senderKey
-
 -- Random correlation ID is used as a nonce in case crypto_box authenticator is used to authorize transmission
 verifyCmdAuthorization :: Maybe (THandleAuth 'TServer) -> Maybe TAuthorizations -> ByteString -> CorrId -> C.APublicAuthKey -> Bool
 verifyCmdAuthorization thAuth tAuth authorized corrId key = maybe False (verify key) tAuth
@@ -1339,7 +1357,7 @@ verifyCmdAuthorization thAuth tAuth authorized corrId key = maybe False (verify
 
 verifyCmdAuth :: Maybe (THandleAuth 'TServer) -> C.PublicKeyX25519 -> C.CbAuthenticator -> ByteString -> CorrId -> Bool
 verifyCmdAuth thAuth k authenticator authorized (CorrId corrId) = case thAuth of
-  Just THAuthServer {serverPrivKey = pk} -> C.cbVerify k pk (C.cbNonce corrId) authenticator authorized
+  Just THAuthServer {serverPrivKey = pk} -> either (const False) (\nonce -> C.cbVerify k pk nonce authenticator authorized) $ C.cbNonce corrId
   Nothing -> False
 
 dummyVerifyCmd :: Maybe (THandleAuth 'TServer) -> Maybe TAuthorizations -> ByteString -> CorrId -> Bool
@@ -1379,15 +1397,18 @@ client
   ms
   clnt@Client {clientId, rcvQ, sndQ, msgQ, clientTHParams = thParams'@THandleParams {sessionId}, procThreads} = do
     labelMyThread . B.unpack $ "client $" <> encode sessionId <> " commands"
+    stats <- asks serverStats
+    authDelay <- asks $ authErrorDelay . config
     let clntServiceId = (\THClientService {serviceId} -> serviceId) <$> (peerClientService =<< thAuth thParams')
         process batchSubs t acc@(rs, msgs) =
           (maybe acc (\(!r, !msg_) -> (r : rs, maybe msgs (: msgs) msg_)))
             <$> processCommand clntServiceId batchSubs t
     forever $ do
       batch <- atomically (readTBQueue rcvQ)
+      processedAt <- liftIO getSystemTime
       batchSubs <- prepareBatchSubs clntServiceId batch
       foldrM (process batchSubs) ([], []) batch
-        >>= \(rs_, msgs) -> mapM_ (atomically . writeTBQueue sndQ . (,msgs)) (L.nonEmpty rs_)
+        >>= \(rs_, msgs) -> mapM_ (liftIO . sendResponses clnt stats authDelay processedAt . (,msgs)) (L.nonEmpty rs_)
   where
     prepareBatchSubs ::
       Maybe ServiceId ->
@@ -1458,14 +1479,15 @@ client
         ServerStats {pMsgFwds, pMsgFwdsOwn} <- asks serverStats
         let inc = mkIncProxyStats pMsgFwds pMsgFwdsOwn
         liftIO (lookupSMPServerClient a sessId) >>= \case
-          Just (own, smp) -> do
+          Just (own, smp) | Right fwdCorrId <- C.corrCbNonce (bs corrId) -> do
             inc own pRequests
             forkProxiedCmd $ do
-              liftIO (runExceptT (forwardSMPTransmission smp corrId fwdV pubKey encBlock) `E.catches` clientHandlers)  >>= \case
-                Right r -> PRES r <$ inc own pSuccesses
+              liftIO (runExceptT (forwardSMPTransmission smp fwdCorrId fwdV pubKey encBlock) `E.catches` clientHandlers)  >>= \case
+                Right (nonce_, r) -> PRES nonce_ r <$ inc own pSuccesses
                 Left e -> ERR (smpProxyError e) <$ case e of
                   PCEProtocolError {} -> inc own pSuccesses
                   _ -> inc own pErrorsOther
+          Just _ -> pure $ Just $ ERR $ CMD SYNTAX
           Nothing -> inc False pRequests >> inc False pErrorsConnect $> Just (ERR $ PROXY NO_SESSION)
       where
         forkProxiedCmd :: M s BrokerMsg -> M s (Maybe BrokerMsg)
@@ -1494,15 +1516,19 @@ client
         Just nenv -> pure (Just nenv)
     -- Runs on a forked thread so RSLV does not block other commands;
     -- concurrency is limited by serverResolverConcurrency in forkCmd.
-    resolveNameMsg :: NamesEnv -> SimplexDomain -> M s BrokerMsg
-    resolveNameMsg nenv d = do
+    resolveNameMsg :: VersionSMP -> NamesEnv -> NameQuery -> M s BrokerMsg
+    resolveNameMsg v nenv q = do
       st <- asks (rslvStats . serverStats)
       (selector, msg) <-
-        liftIO (resolveName nenv d) <&> \case
-          Right rec -> (rslvSucc, RNAME rec)
-          Left e@NOT_FOUND -> (rslvNotFound, ERR $ NAME e)
+        liftIO (resolveName nenv q) <&> \case
+          Right res -> (if answered (registration res) then rslvSucc else rslvNotFound, RNAME res)
           Left e -> (rslvResolverErrs, ERR $ NAME e)
       incStat (selector st) $> msg
+      where
+        -- below v22 the encoder answers anything but a record as NAME NOT_FOUND
+        answered = \case
+          NRRegistered {} -> True
+          _ -> v >= nameAvailSMPVersion
     transportErr :: TransportError -> ErrorType
     transportErr = PROXY . BROKER . TRANSPORT
     mkIncProxyStats :: MonadIO m => ProxyStats -> ProxyStats -> OwnServer -> (ProxyStats -> IORef Int) -> m ()
@@ -1519,7 +1545,7 @@ client
       Cmd SProxyService (RFWD encBlock) -> (response . (corrId, NoEntity,) =<<) <$> processForwardedCommand encBlock
       Cmd SResolver (RSLV d) -> rslvNamesEnv >>= \case
         Nothing -> pure $ response (corrId, NoEntity, ERR (NAME NO_RESOLVER))
-        Just nenv -> forkCmd serverResolverConcurrency corrId NoEntity (resolveNameMsg nenv d)
+        Just nenv -> forkCmd serverResolverConcurrency corrId NoEntity (resolveNameMsg (thVersion thParams') nenv d)
       Cmd SSenderLink command -> case command of
         LKEY k -> withQueue $ \q qr -> checkMode QMMessaging qr $ secureQueue_ q k $>> getQueueLink_ q qr
         LGET -> withQueue $ \q qr -> checkContact qr $ getQueueLink_ q qr
@@ -1951,7 +1977,7 @@ client
             getDelivered :: Sub -> STM (Maybe (ServerSub, SystemSeconds))
             getDelivered Sub {delivered, subThread} = do
               readTVar delivered $>>= \(msgId', ts) ->
-                if msgId == msgId' || B.null msgId
+                if msgId == msgId'
                   then writeTVar delivered Nothing $> Just (subThread, ts)
                   else pure Nothing
             updateStats :: ServerStats -> Bool -> SystemSeconds -> Message -> IO ()
@@ -1997,7 +2023,7 @@ client
                       when (isJust (queueData qr) && isSecuredMsgQueue qr) $ void $ liftIO $
                         deleteQueueLinkData (queueStore ms) q
                       ServerConfig {messageExpiration, expireMessagesOnSend, msgIdBytes} <- asks config
-                      msgId <- randomId' msgIdBytes
+                      msgId <- atomically . randomMsgId msgIdBytes =<< asks random
                       msg_ <- liftIO $ runExceptT $ do
                         when expireMessagesOnSend $ mapM_ (expireMessages stats) messageExpiration
                         msg <- liftIO $ mkMessage msgId body
@@ -2110,7 +2136,7 @@ client
               liftIO $ storeNtf ns nId ntf
               incStat . ntfCount =<< asks serverStats
 
-            mkMessageNotification :: ByteString -> SystemTime -> RcvNtfDhSecret -> M s MsgNtf
+            mkMessageNotification :: MsgId -> SystemTime -> RcvNtfDhSecret -> M s MsgNtf
             mkMessageNotification msgId msgTs rcvNtfDhSecret = do
               ntfNonce <- atomically . C.randomCbNonce =<< asks random
               let msgMeta = NMsgMeta {msgId, msgTs}
@@ -2119,29 +2145,35 @@ client
 
         processForwardedCommand :: EncFwdTransmission -> M s (Maybe BrokerMsg)
         processForwardedCommand (EncFwdTransmission s) = fmap (either (Just . ERR) id) . runExceptT $ do
+          startedAt <- liftIO getSystemTime
           THAuthServer {serverPrivKey, sessSecret'} <- maybe (throwE $ transportErr TENoServerAuth) pure (thAuth thParams')
           sessSecret <- maybe (throwE $ transportErr TENoServerAuth) pure sessSecret'
-          let proxyNonce = C.cbNonce $ bs corrId
+          proxyNonce <- liftEitherWith (const CRYPTO) $ C.cbNonce $ bs corrId
           s' <- liftEitherWith (const CRYPTO) $ C.cbDecryptNoPad sessSecret proxyNonce s
           FwdTransmission {fwdCorrId, fwdVersion, fwdKey, fwdTransmission = EncTransmission et} <- liftEitherWith (const $ CMD SYNTAX) $ smpDecode s'
+          unless (fwdVersion `isCompatible` thServerVRange thParams') $ throwE $ transportErr TEVersion
           let clientSecret = C.dh' fwdKey serverPrivKey
-              clientNonce = C.cbNonce $ bs fwdCorrId
-          b <- liftEitherWith (const CRYPTO) $ C.cbDecrypt clientSecret clientNonce et
+              clientNonce = fwdCorrId
+          b <- liftEitherWith (const CRYPTO) $ C.cbDecrypt clientSecret (encTransmissionNonce fwdVersion clientNonce) et
           let clntTHParams = smpTHParamsSetVersion fwdVersion thParams'
           -- only allowing single forwarded transactions
           t' <- case tParse clntTHParams b of
             t :| [] -> pure $ tDecodeServer clntTHParams t
             _ -> throwE BLOCK
-          let clntThAuth = Just $ THAuthServer {serverPrivKey, peerClientService = Nothing, sessSecret' = Just clientSecret}
+          let clntThAuth = Just $ THAuthServer {serverPrivKey, peerClientService = Nothing, peerEntitlement = Nothing, sessSecret' = Just clientSecret}
               encodeResp r = do
                 r' <- case batchTransmissions clntTHParams [Right (Nothing, encodeTransmission clntTHParams r)] of
                   [] -> throwE INTERNAL -- at least 1 item is guaranteed from NonEmpty/Right
                   TBError _ _ : _ -> throwE BLOCK
                   TBTransmission b' _ : _ -> pure b'
                   TBTransmissions b' _ _ : _ -> pure b'
-                r2 <- liftEitherWith (const BLOCK) $ EncResponse <$> C.cbEncrypt clientSecret (C.reverseNonce clientNonce) r' paddedProxiedTLength
+                nonce_ <-
+                  if fwdVersion >= fwdNoncesSMPVersion
+                    then Just <$> (atomically . C.randomCbNonce =<< asks random)
+                    else pure Nothing
+                r2 <- liftEitherWith (const BLOCK) $ EncResponse <$> C.cbEncrypt clientSecret (fromMaybe (C.reverseNonce clientNonce) nonce_) r' paddedProxiedTLength
                 let fr = FwdResponse {fwdCorrId, fwdResponse = r2}
-                pure $ RRES $ EncFwdResponse $ C.cbEncryptNoPad sessSecret (C.reverseNonce proxyNonce) (smpEncode fr)
+                pure $ RRES nonce_ $ EncFwdResponse $ C.cbEncryptNoPad sessSecret (C.reverseNonce proxyNonce) (smpEncode fr)
           -- the inner response, or Nothing if forked (RSLV).
           r_ <- lift (rejectOrVerify clntThAuth t') >>= \case
             -- rejectOrVerify filters allowed commands, no need to repeat it here.
@@ -2150,14 +2182,18 @@ client
               Cmd SResolver (RSLV d) -> lift $ rslvNamesEnv >>= \case
                 Nothing -> pure $ Just (corrId', entId', ERR (NAME NO_RESOLVER))
                 Just nenv -> forkCmd serverResolverConcurrency corrId NoEntity $ do
-                  msg <- resolveNameMsg nenv d
+                  msg <- resolveNameMsg (thVersion clntTHParams) nenv d
                   either ERR id <$> runExceptT (encodeResp (corrId', entId', msg))
               -- INTERNAL because processCommand never returns Nothing for sender commands;
               -- `fst` drops the empty message only returned for SUB.
               _ -> Just . maybe (corrId', entId', ERR INTERNAL) fst <$> lift (processCommand Nothing (Right (M.empty, M.empty, M.empty)) t'')
           stats <- asks serverStats
           incStat $ pMsgFwdsRecv stats
-          traverse encodeResp r_
+          traverse encodeResp r_ >>= \case
+            Just rres | any isAuthError r_ -> do
+              authDelay <- asks $ authErrorDelay . config
+              Nothing <$ liftIO (sendAfterDelay clnt stats authDelay startedAt ([(corrId, NoEntity, rres)], []))
+            rres_ -> pure rres_
           where
             rejectOrVerify :: Maybe (THandleAuth 'TServer) -> SignedTransmissionOrError ErrorType Cmd -> M s (VerifiedTransmissionOrError s)
             rejectOrVerify clntThAuth = \case
@@ -2183,7 +2219,7 @@ client
           MessageQuota {} -> RcvMsgQuota msgTs'
           where
             encrypt :: KnownNat i => C.MaxLenBS i -> RcvMessage
-            encrypt body = RcvMessage msgId' . EncRcvMsgBody $ C.cbEncryptMaxLenBS (rcvDhSecret qr) (C.cbNonce msgId') body
+            encrypt body = RcvMessage msgId' . EncRcvMsgBody $ C.cbEncryptMaxLenBS (rcvDhSecret qr) msgId' body
             msgId' = messageId msg
             msgTs' = messageTs msg
 
@@ -2232,7 +2268,7 @@ client
                     SubPending -> QSubPending
                     SubThread _ -> QSubThread
                 ProhibitSub -> pure QProhibitSub
-              qDelivered <- decodeLatin1 . encode . fst <$$> readTVarIO delivered
+              qDelivered <- decodeLatin1 . encode . unMsgId . fst <$$> readTVarIO delivered
               pure QSub {qSubThread, qDelivered}
 
         ok :: Transmission BrokerMsg

src/Simplex/Messaging/Server/Env/STM.hs

@@ -68,6 +68,7 @@ module Simplex.Messaging.Server.Env.STM
     defaultInactiveClientExpiration,
     defaultProxyClientConcurrency,
     defaultNameResolverConcurrency,
+    defaultAuthErrorDelayMs,
     defaultMaxJournalMsgCount,
     defaultMaxJournalStateLines,
     defaultIdleQueueInterval,
@@ -187,6 +188,7 @@ data ServerConfig s = ServerConfig
     ntfDeliveryInterval :: Int,
     -- | interval between sending pending END events to unsubscribed clients, seconds
     pendingENDInterval :: Int,
+    authErrorDelay :: Int64,
     smpCredentials :: ServerCredentials,
     httpCredentials :: Maybe ServerCredentials,
     -- | SMP client-server protocol version range
@@ -255,6 +257,9 @@ defaultProxyClientConcurrency = 32
 defaultNameResolverConcurrency :: Int
 defaultNameResolverConcurrency = 1000
 
+defaultAuthErrorDelayMs :: Int64
+defaultAuthErrorDelayMs = 50
+
 journalMsgStoreDepth :: Int
 journalMsgStoreDepth = 5
 
@@ -571,7 +576,10 @@ newProhibitedSub = do
   return Sub {subThread = ProhibitSub, delivered}
 
 newEnv :: ServerConfig s -> IO (Env s)
-newEnv config@ServerConfig {smpCredentials, httpCredentials, serverStoreCfg, smpAgentCfg, information, messageExpiration, idleQueueInterval, msgQueueQuota, maxJournalMsgCount, maxJournalStateLines, namesConfig} = do
+newEnv config@ServerConfig {smpCredentials, httpCredentials, serverStoreCfg, smpAgentCfg, information, messageExpiration, idleQueueInterval, msgQueueQuota, maxJournalMsgCount, maxJournalStateLines, namesConfig, msgIdBytes} = do
+  when (msgIdBytes < 24) $ do
+    logError "Error: msgIdBytes must be at least 24"
+    exitFailure
   serverActive <- newTVarIO True
   server <- newServer
   msgStore_ <- case serverStoreCfg of

src/Simplex/Messaging/Server/Main.hs

@@ -580,6 +580,7 @@ smpServerCLI_ generateSite serveStaticFiles attachStaticFiles cfgPath logPath =
               prometheusMetricsFile = combine logPath "smp-server-metrics.txt",
               pendingENDInterval = 15000000, -- 15 seconds
               ntfDeliveryInterval = 1500000, -- 1.5 second
+              authErrorDelay = 1000 * readIniDefault defaultAuthErrorDelayMs "TRANSPORT" "auth_error_delay_ms" ini,
               smpServerVRange = supportedServerSMPRelayVRange,
               transportConfig =
                 mkTransportServerConfig

src/Simplex/Messaging/Server/Main/GitCommit.hs

@@ -6,14 +6,25 @@ module Simplex.Messaging.Server.Main.GitCommit
   ) where
 
 import Language.Haskell.TH
+import Language.Haskell.TH.Syntax (addDependentFile)
 import System.Process
 import Control.Exception
+import Control.Monad (filterM)
+import System.Directory (doesFileExist)
 import System.Exit
+import System.FilePath ((</>))
 
 gitCommit :: Q Exp
-gitCommit = stringE . commit =<< runIO (try $ readProcessWithExitCode "git" ["rev-parse", "HEAD"] "")
+gitCommit =
+  runIO gitHead >>= \case
+    Right (ExitSuccess, out, _)
+      | [commit, gitDir] <- lines out -> do
+          -- without these dependencies GHC recompilation check would not know that the commit changed.
+          -- HEAD reflog is updated on any HEAD change and, unlike branch ref files, git gc cannot remove it
+          -- mid-build; it is absent when reflogs are disabled, and addDependentFile fails on a missing file
+          mapM_ addDependentFile =<< runIO (filterM doesFileExist [gitDir </> "HEAD", gitDir </> "logs" </> "HEAD"])
+          stringE commit
+    _ -> stringE ""
   where
-    commit :: Either SomeException (ExitCode, String, String) -> String
-    commit = \case
-      Right (ExitSuccess, out, _) -> take 40 out
-      _ -> ""
+    gitHead :: IO (Either SomeException (ExitCode, String, String))
+    gitHead = try $ readProcessWithExitCode "git" ["rev-parse", "HEAD", "--git-dir"] ""

src/Simplex/Messaging/Server/Main/Init.hs

@@ -130,7 +130,9 @@ iniFileContent cfgPath logPath opts host basicAuth controlPortPwds =
     <> ("host = " <> T.pack host <> "\n")
     <> ("port = " <> defaultServerPorts <> "\n")
     <> "log_tls_errors = off\n\n\
-        \# Use `websockets = 443` to run websockets server in addition to plain TLS.\n\
+        \# Minimum time to respond with AUTH error, milliseconds.\n"
+    <> ("# auth_error_delay_ms = " <> tshow defaultAuthErrorDelayMs <> "\n\n")
+    <> "# Use `websockets = 443` to run websockets server in addition to plain TLS.\n\
         \# This option is deprecated and should be used for testing only.\n\
         \# , port 443 should be specified in port above\n\
         \websockets = off\n"

src/Simplex/Messaging/Server/MsgStore/Postgres.hs

@@ -110,12 +110,13 @@ instance MsgStoreClass PostgresMsgStore where
   expireOldMessages :: Bool -> PostgresMsgStore -> Int64 -> Int64 -> IO MessageStats
   expireOldMessages _tty ms now ttl =
     maybeFirstRow' newMessageStats toMessageStats $ withConnection st $ \db ->
-      DB.query db "CALL expire_old_messages(?,?,?,0,0,0)" (oldQueue, oldMsg, batchSize)
+      DB.query db "CALL expire_old_messages(?,?,0,0,0)" (oldMsg, batchSize)
     where
       st = dbStore $ queueStore_ ms
-      oldQueue = 0 :: Int64 -- expire all queues
       oldMsg = now - ttl
-      batchSize = 10000 :: Int
+      -- expired messages read per page in expire_old_messages, and the page is one
+      -- transaction: queues in it stay row-locked against SEND and ACK until it commits.
+      batchSize = 100 :: Int
       toMessageStats (expiredMsgsCount, storedMsgsCount, storedQueues) =
         MessageStats {expiredMsgsCount, storedMsgsCount, storedQueues}
 
@@ -201,7 +202,7 @@ instance MsgStoreClass PostgresMsgStore where
           DB.query
             db
             "SELECT quota_written, was_empty FROM write_message(?,?,?,?,?,?,?)"
-            (recipientId' q, Binary (messageId msg), systemSeconds (messageTs msg), msgQuota, ntf, Binary body, quota)
+            (recipientId' q, messageId msg, systemSeconds (messageTs msg), msgQuota, ntf, Binary body, quota)
     where
       toResult = \case
         ((msgQuota, wasEmpty) : _) -> if msgQuota then Nothing else Just (msg, wasEmpty)
@@ -270,14 +271,14 @@ instance MsgStoreClass PostgresMsgStore where
     uninterruptibleMask_ $
       withDB' "tryDelMsg" (queueStore_ ms) $ \db ->
         maybeFirstRow toMessage $
-          DB.query db "SELECT r_msg_id, r_msg_ts, r_msg_quota, r_msg_ntf_flag, r_msg_body FROM try_del_msg(?, ?)" (recipientId' q, Binary msgId)
+          DB.query db "SELECT r_msg_id, r_msg_ts, r_msg_quota, r_msg_ntf_flag, r_msg_body FROM try_del_msg(?, ?)" (recipientId' q, msgId)
 
   tryDelPeekMsg :: PostgresMsgStore -> PostgresQueue -> MsgId -> ExceptT ErrorType IO (Maybe Message, Maybe Message)
   tryDelPeekMsg ms q msgId =
     uninterruptibleMask_ $
       withDB' "tryDelPeekMsg" (queueStore_ ms) $ \db ->
         toResult . map toMessage
-          <$> DB.query db "SELECT r_msg_id, r_msg_ts, r_msg_quota, r_msg_ntf_flag, r_msg_body FROM try_del_peek_msg(?, ?)" (recipientId' q, Binary msgId)
+          <$> DB.query db "SELECT r_msg_id, r_msg_ts, r_msg_quota, r_msg_ntf_flag, r_msg_body FROM try_del_peek_msg(?, ?)" (recipientId' q, msgId)
     where
       toResult = \case
         [] -> (Nothing, Nothing)
@@ -296,13 +297,13 @@ uninterruptibleMask_ :: ExceptT ErrorType IO a -> ExceptT ErrorType IO a
 uninterruptibleMask_ = ExceptT . E.uninterruptibleMask_ . runExceptT
 {-# INLINE uninterruptibleMask_ #-}
 
-toMaybeMessage :: (Maybe (Binary MsgId), Maybe Int64, Maybe Bool, Maybe Bool, Maybe (Binary MsgBody)) -> Maybe Message
+toMaybeMessage :: (Maybe MsgId, Maybe Int64, Maybe Bool, Maybe Bool, Maybe (Binary MsgBody)) -> Maybe Message
 toMaybeMessage = \case
   (Just msgId, Just ts, Just msgQuota, Just ntf, Just body) -> Just $ toMessage (msgId, ts, msgQuota, ntf, body)
   _ -> Nothing
 
-toMessage :: (Binary MsgId, Int64, Bool, Bool, Binary MsgBody) -> Message
-toMessage (Binary msgId, ts, msgQuota, ntf, Binary body)
+toMessage :: (MsgId, Int64, Bool, Bool, Binary MsgBody) -> Message
+toMessage (msgId, ts, msgQuota, ntf, Binary body)
   | msgQuota = MessageQuota {msgId, msgTs}
   | otherwise = Message {msgId, msgTs, msgFlags = MsgFlags ntf, msgBody = C.unsafeMaxLenBS body} -- TODO [messages] unsafeMaxLenBS?
   where
@@ -364,8 +365,8 @@ deleteAllMessages ms =
       db
       [sql|
         UPDATE msg_queues
-        SET msg_queue_size = 0, msg_can_write = TRUE, msg_queue_expire = FALSE
-        WHERE msg_queue_size != 0 OR msg_can_write = FALSE OR msg_queue_expire = TRUE
+        SET msg_queue_size = 0, msg_can_write = TRUE
+        WHERE msg_queue_size != 0 OR msg_can_write = FALSE
       |]
 
 updateQueueCounts :: PostgresMsgStore -> IO ()
@@ -385,16 +386,15 @@ updateQueueCounts ms =
       db
       [sql|
         UPDATE msg_queues
-        SET msg_queue_size = 0, msg_can_write = TRUE, msg_queue_expire = FALSE
-        WHERE msg_queue_size != 0 OR msg_can_write = FALSE OR msg_queue_expire = TRUE
+        SET msg_queue_size = 0, msg_can_write = TRUE
+        WHERE msg_queue_size != 0 OR msg_can_write = FALSE
       |]
     void $ DB.execute_
       db
       [sql|
         UPDATE msg_queues q
         SET msg_queue_size = s.size,
-            msg_can_write = s.quota_count = 0,
-            msg_queue_expire = s.size > s.quota_count
+            msg_can_write = s.quota_count = 0
         FROM queue_stats s
         WHERE q.recipient_id = s.recipient_id
       |]
@@ -429,7 +429,7 @@ messageRecToText rId msg =
     tabFields = BB.char7 ',' `intersperse` fields
     fields =
       [ renderField (toField rId),
-        renderField (toField $ Binary (messageId msg)),
+        renderField (toField $ messageId msg),
         renderField (toField $ systemSeconds (messageTs msg)),
         renderField (toField msgQuota),
         renderField (toField ntf),