smp web: client with tests (#1782)

* smp web: client with tests

* fixes

* support batching and SMP proxy

---------

Co-authored-by: Evgeny @ SimpleX Chat <[email protected]>

.github/workflows/build.yml

@@ -173,7 +173,7 @@ jobs:
             -v ${{ github.workspace }}:/project \
             build/${{ matrix.os }}:latest
 
-      - name: Build smp-server (postgresql) and tests
+      - name: Build smp-server, xftp-server (postgresql) and tests
         if: matrix.should_run == true
         shell: docker exec -t builder sh -eu {0}
         run: |
@@ -182,12 +182,12 @@ jobs:
           cabal update
           cabal build --jobs=$(nproc) --enable-tests -fserver_postgres
           mkdir -p /out
-          for i in smp-server simplexmq-test; do
+          for i in smp-server xftp-server simplexmq-test; do
             bin=$(find /project/dist-newstyle -name "$i" -type f -executable)
             chmod +x "$bin"
             mv "$bin" /out/
           done
-          strip /out/smp-server
+          strip /out/smp-server /out/xftp-server
 
       - name: Copy simplexmq-test from container
         if: matrix.should_run == true
@@ -195,19 +195,29 @@ jobs:
         run: |
           docker cp builder:/out/simplexmq-test .
 
-      - name: Copy smp-server (postgresql) from container and prepare it
+      - name: Copy smp-server, xftp-server (postgresql) from container and prepare it
         if: startsWith(github.ref, 'refs/tags/v') && matrix.should_run == true
         id: prepare-postgres
         shell: bash
         run: |
-          name="smp-server-postgres-ubuntu-${{ matrix.os_underscore }}-${{ matrix.arch }}"
-          docker cp builder:/out/smp-server $name
+          printf 'bins<<EOF\n' > bins.output
+          printf 'hashes<<EOF\n' > hashes.output
 
-          path="${{ github.workspace }}/$name"
-          echo "bin=$path" >> $GITHUB_OUTPUT
+          for i in smp-server xftp-server; do
+            name="${i}-postgres-ubuntu-${{ matrix.os_underscore }}-${{ matrix.arch }}"
+            docker cp builder:/out/$i $name
 
-          hash="SHA2-256($name)= $(openssl sha256 $path | cut -d' ' -f 2)"
-          printf 'hash=%s' "$hash" >> $GITHUB_OUTPUT
+            path="${{ github.workspace }}/$name"
+            hash="SHA2-256($name)= $(openssl sha256 $path | cut -d' ' -f 2)"
+
+            printf '%s\n' "$path" >> bins.output
+            printf '%s\n\n' "$hash" >> hashes.output
+          done
+          printf 'EOF\n' >> bins.output
+          printf 'EOF\n' >> hashes.output
+
+          cat bins.output >> "$GITHUB_OUTPUT"
+          cat hashes.output >> "$GITHUB_OUTPUT"
 
       - name: Build everything else (standard)
         if: matrix.should_run == true
@@ -257,10 +267,10 @@ jobs:
           fail_on_unmatched_files: true
           body: |
             ${{ steps.prepare-regular.outputs.hashes }}
-            ${{ steps.prepare-postgres.outputs.hash }}
+            ${{ steps.prepare-postgres.outputs.hashes }}
           files: |
             ${{ steps.prepare-regular.outputs.bins }}
-            ${{ steps.prepare-postgres.outputs.bin }}
+            ${{ steps.prepare-postgres.outputs.bins }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 

CHANGELOG.md

@@ -1,3 +1,39 @@
+# 6.5.1
+
+Version 6.5.1.0
+
+XFTP client:
+- backwards compatible file header decoding.
+
+# 6.5.0
+
+Version 6.5.0.17
+
+SMP agent:
+- improve subscriptions
+  - reduce memory usage and retries during initial subscription (#1758)
+  - fix race resulting in pending subscriptions never subscribed (#1756)
+  - batch processing of subscription results and errors (#1652)
+  - reduce memory usage of active subscriptions.
+- drop message after N reception attempts (#1762)
+- fix possible deadlocks of queue overloading when processing messages (#1713)
+- improved APIs for short link management and creation.
+- support multiple link owners in link data (#1701)
+
+SMP server:
+- store messages in PostgreSQL (#1622).
+- reduce memory usage with PostgreSQL database - do not use queue cache (#1637)
+- fix in-memory server not restoring queue/service associations after 2+ restarts (#1618)
+
+XFTP server:
+- support PostgreSQL database.
+- add server page.
+- support uploads from web clients.
+
+Servers:
+- better socket leak prevention during TLS handshake, NetworkError type to bette diagnose connection errors (#1619)
+- use "=" as default INI key-value separator (#1767)
+
 # 6.4.4
 
 Servers:

README.md

@@ -33,7 +33,7 @@ To initialize the server use `smp-server init -n <fqdn>` (or `smp-server init --
 
 SMP server uses in-memory persistence with an optional append-only log of created queues that allows to re-start the server without losing the connections. This log is compacted on every server restart, permanently removing suspended and removed queues.
 
-To enable store log, initialize server using `smp-server -l` command, or modify `smp-server.ini` created during initialization (uncomment `enable: on` option in the store log section). Use `smp-server --help` for other usage tips.
+To enable store log, initialize server using `smp-server -l` command, or modify `smp-server.ini` created during initialization (uncomment `enable = on` option in the store log section). Use `smp-server --help` for other usage tips.
 
 Starting from version 2.3.0, when store log is enabled, the server would also enable saving undelivered messages on exit and restoring them on start. This can be disabled via a separate setting `restore_messages` in `smp-server.ini` file. Saving messages would only work if the server is stopped with SIGINT signal (keyboard interrupt), if it is stopped with SIGTERM signal the messages would not be saved.
 

apps/common/Web/Embedded.hs

@@ -0,0 +1,15 @@
+{-# LANGUAGE TemplateHaskell #-}
+
+module Web.Embedded where
+
+import Data.FileEmbed (embedDir, embedFile)
+import Simplex.Messaging.Server.Web (EmbeddedContent (..))
+
+embeddedContent :: EmbeddedContent
+embeddedContent =
+  EmbeddedContent
+    { indexHtml = $(embedFile "apps/common/Web/static/index.html"),
+      linkHtml = $(embedFile "apps/common/Web/static/link.html"),
+      mediaContent = $(embedDir "apps/common/Web/static/media/"),
+      wellKnown = $(embedDir "apps/common/Web/static/.well-known/")
+    }

apps/common/Web/static/index.html

@@ -105,6 +105,13 @@
                     class="text-[16px] leading-[26px] tracking-[0.01em] nav-link-text text-black dark:text-white before:bg-black dark:before:bg-white">Server
                     information</span></a>
               </li>
+              <!-- <x-xftpConfig>
+              <li class="nav-link relative"><a href="/file"
+                  class="flex items-center justify-between gap-2 lg:py-5 whitespace-nowrap"><span
+                    class="text-[16px] leading-[26px] tracking-[0.01em] nav-link-text text-black dark:text-white before:bg-black dark:before:bg-white">File
+                    transfer</span></a>
+              </li>
+              </x-xftpConfig> -->
             </ul><a target="_blank" href="https://github.com/simplex-chat/simplex-chat#help-us-with-donations"
               class="whitespace-nowrap flex items-center gap-1 self-center text-white dark:text-black text-[16px] font-medium tracking-[0.02em] rounded-[34px] bg-primary-light dark:bg-primary-dark py-3 lg:py-2 px-20 lg:px-5 mb-16 lg:mb-0">Donate</a>
           </div>
@@ -229,7 +236,7 @@
                 <td>Source code:</td>
                 <td>
                   <x-sourceCode><a href="${sourceCode}" target="_blank">${sourceCode}</a></x-sourceCode>
-                  <x-noSourceCode>add to smp-server.ini (required by <a href="https://github.com/simplex-chat/simplexmq/blob/stable/LICENSE" target="_blank">AGPLv3</a>)</x-noSourceCode>
+                  <x-noSourceCode>add to ${iniFileName} (required by <a href="https://github.com/simplex-chat/simplexmq/blob/stable/LICENSE" target="_blank">AGPLv3</a>)</x-noSourceCode>
                 </td>
               </tr>
               <x-website>
@@ -317,6 +324,7 @@
             <h2 class="text-[30px] mb-[20px] leading-[28px] text-[#606C71] dark:text-white font-bold max-w-[475px]">
               Configuration</h2>
             <table id="config">
+              <x-smpConfig>
               <tr class="text-grey-black dark:text-white text-base">
                 <td>Persistence:</td>
                 <td>${persistence}</td>
@@ -337,6 +345,25 @@
                 <td>Basic auth enabled:</td>
                 <td>${basicAuthEnabled}</td>
               </tr>
+              </x-smpConfig>
+              <x-xftpConfig>
+              <tr class="text-grey-black dark:text-white text-base">
+                <td>File expiration:</td>
+                <td>${fileExpiration}</td>
+              </tr>
+              <tr class="text-grey-black dark:text-white text-base">
+                <td>Stats enabled:</td>
+                <td>${statsEnabled}</td>
+              </tr>
+              <tr class="text-grey-black dark:text-white text-base">
+                <td>New uploads allowed:</td>
+                <td>${newUploadsAllowed}</td>
+              </tr>
+              <tr class="text-grey-black dark:text-white text-base">
+                <td>Basic auth enabled:</td>
+                <td>${basicAuthEnabled}</td>
+              </tr>
+              </x-xftpConfig>
             </table>
           </div>
         </div>

apps/smp-server/Main.hs

@@ -2,8 +2,9 @@ module Main where
 
 import Control.Logger.Simple
 import Simplex.Messaging.Server.CLI (getEnvPath)
-import Simplex.Messaging.Server.Main
-import qualified Static
+import Simplex.Messaging.Server.Main (smpServerCLI_)
+import Simplex.Messaging.Server.Web (serveStaticFiles, attachStaticAndWS)
+import SMPWeb (smpGenerateSite)
 
 defaultCfgPath :: FilePath
 defaultCfgPath = "/etc/opt/simplex"
@@ -18,4 +19,4 @@ main :: IO ()
 main = do
   cfgPath <- getEnvPath "SMP_SERVER_CFG_PATH" defaultCfgPath
   logPath <- getEnvPath "SMP_SERVER_LOG_PATH" defaultLogPath
-  withGlobalLogging logCfg $ smpServerCLI_ Static.generateSite Static.serveStaticFiles Static.attachStaticFiles cfgPath logPath
+  withGlobalLogging logCfg $ smpServerCLI_ smpGenerateSite serveStaticFiles attachStaticAndWS cfgPath logPath

apps/smp-server/SMPWeb.hs

@@ -0,0 +1,43 @@
+{-# LANGUAGE NamedFieldPuns #-}
+{-# LANGUAGE OverloadedStrings #-}
+
+module SMPWeb
+  ( smpGenerateSite,
+    serverInformation,
+  ) where
+
+import Data.ByteString (ByteString)
+import Data.String (fromString)
+import Web.Embedded (embeddedContent)
+import Simplex.Messaging.Encoding.String (strEncode)
+import Simplex.Messaging.Server.Information
+import Simplex.Messaging.Server.Main (simplexmqSource)
+import qualified Simplex.Messaging.Server.Web as Web
+import Simplex.Messaging.Server.Web (render, serverInfoSubsts, timedTTLText)
+import Simplex.Messaging.Transport.Client (TransportHost (..))
+
+smpGenerateSite :: ServerInformation -> Maybe TransportHost -> FilePath -> IO ()
+smpGenerateSite si onionHost path =
+  Web.generateSite embeddedContent (serverInformation si onionHost) smpLinkPages path
+
+smpLinkPages :: [String]
+smpLinkPages = ["contact", "invitation", "a", "c", "g", "r", "i"]
+
+serverInformation :: ServerInformation -> Maybe TransportHost -> ByteString
+serverInformation ServerInformation {config, information} onionHost = render (Web.indexHtml embeddedContent) substs
+  where
+    substs = [("smpConfig", Just "y"), ("xftpConfig", Nothing)] <> substConfig <> serverInfoSubsts simplexmqSource information <> [("onionHost", strEncode <$> onionHost), ("iniFileName", Just "smp-server.ini")]
+    substConfig =
+      [ ( "persistence",
+          Just $ case persistence config of
+            SPMMemoryOnly -> "In-memory only"
+            SPMQueues -> "Queues"
+            SPMMessages -> "Queues and messages"
+        ),
+        ("messageExpiration", Just $ maybe "Never" (fromString . timedTTLText) $ messageExpiration config),
+        ("statsEnabled", Just . yesNo $ statsEnabled config),
+        ("newQueuesAllowed", Just . yesNo $ newQueuesAllowed config),
+        ("basicAuthEnabled", Just . yesNo $ basicAuthEnabled config)
+      ]
+    yesNo True = "Yes"
+    yesNo False = "No"

apps/smp-server/static/a/index.html

@@ -1 +0,0 @@
-../link.html

apps/smp-server/static/c/index.html

@@ -1 +0,0 @@
-../link.html

apps/smp-server/static/contact/index.html

@@ -1 +0,0 @@
-../link.html

apps/smp-server/static/i/index.html

@@ -1 +0,0 @@
-../link.html

apps/smp-server/static/invitation/index.html

@@ -1 +0,0 @@
-../link.html

apps/smp-server/web/Static.hs

@@ -1,253 +0,0 @@
-{-# LANGUAGE LambdaCase #-}
-{-# LANGUAGE NamedFieldPuns #-}
-{-# LANGUAGE OverloadedStrings #-}
-
-module Static where
-
-import Control.Logger.Simple
-import Control.Monad
-import Data.ByteString (ByteString)
-import qualified Data.ByteString.Char8 as B
-import Data.Char (toUpper)
-import Data.IORef (readIORef)
-import Data.Maybe (fromMaybe)
-import Data.String (fromString)
-import qualified Data.Text as T
-import Data.Text.Encoding (encodeUtf8)
-import Network.Socket (getPeerName)
-import Network.Wai (Application, Request (..))
-import Network.Wai.Application.Static (StaticSettings (..))
-import qualified Network.Wai.Application.Static as S
-import qualified Network.Wai.Handler.Warp as W
-import qualified Network.Wai.Handler.Warp.Internal as WI
-import qualified Network.Wai.Handler.WarpTLS as WT
-import Simplex.Messaging.Encoding.String (strEncode)
-import Simplex.Messaging.Server (AttachHTTP)
-import Simplex.Messaging.Server.CLI (simplexmqCommit)
-import Simplex.Messaging.Server.Information
-import Simplex.Messaging.Server.Main (EmbeddedWebParams (..), WebHttpsParams (..), simplexmqSource)
-import Simplex.Messaging.Transport (simplexMQVersion)
-import Simplex.Messaging.Transport.Client (TransportHost (..))
-import Simplex.Messaging.Util (tshow)
-import Static.Embedded as E
-import System.Directory (createDirectoryIfMissing)
-import System.FilePath
-import UnliftIO.Concurrent (forkFinally)
-import UnliftIO.Exception (bracket, finally)
-import qualified WaiAppStatic.Types as WAT
-
-serveStaticFiles :: EmbeddedWebParams -> IO ()
-serveStaticFiles EmbeddedWebParams {webStaticPath, webHttpPort, webHttpsParams} = do
-  forM_ webHttpPort $ \port -> flip forkFinally (\e -> logError $ "HTTP server crashed: " <> tshow e) $ do
-    logInfo $ "Serving static site on port " <> tshow port
-    W.runSettings (mkSettings port) app
-  forM_ webHttpsParams $ \WebHttpsParams {port, cert, key} -> flip forkFinally (\e -> logError $ "HTTPS server crashed: " <> tshow e) $ do
-    logInfo $ "Serving static site on port " <> tshow port <> " (TLS)"
-    WT.runTLS (WT.tlsSettings cert key) (mkSettings port) app
-  where
-    app = staticFiles webStaticPath
-    mkSettings port = W.setPort port warpSettings
-
--- | Prepare context and prepare HTTP handler for TLS connections that already passed TLS.handshake and ALPN check.
-attachStaticFiles :: FilePath -> (AttachHTTP -> IO ()) -> IO ()
-attachStaticFiles path action =
-  -- Initialize global internal state for http server.
-  WI.withII warpSettings $ \ii -> do
-    action $ \socket cxt -> do
-      -- Initialize internal per-connection resources.
-      addr <- getPeerName socket
-      withConnection addr cxt $ \(conn, transport) ->
-        withTimeout ii conn $ \th ->
-          -- Run Warp connection handler to process HTTP requests for static files.
-          WI.serveConnection conn ii th addr transport warpSettings app
-  where
-    app = staticFiles path
-    -- from warp-tls
-    withConnection socket cxt = bracket (WT.attachConn socket cxt) (terminate . fst)
-    -- from warp
-    withTimeout ii conn =
-      bracket
-        (WI.registerKillThread (WI.timeoutManager ii) (WI.connClose conn))
-        WI.cancel
-    -- shared clean up
-    terminate conn = WI.connClose conn `finally` (readIORef (WI.connWriteBuffer conn) >>= WI.bufFree)
-
-warpSettings :: W.Settings
-warpSettings = W.setGracefulShutdownTimeout (Just 1) W.defaultSettings
-
-staticFiles :: FilePath -> Application
-staticFiles root = S.staticApp settings . changeWellKnownPath
-  where
-    settings = defSettings {ssListing = Nothing, ssGetMimeType = getMimeType}
-    defSettings = S.defaultFileServerSettings root
-    getMimeType f
-      | WAT.fromPiece (WAT.fileName f) == "apple-app-site-association" = pure "application/json"
-      | otherwise = (ssGetMimeType defSettings) f
-    changeWellKnownPath req = case pathInfo req of
-      ".well-known" : rest ->
-        req
-          { pathInfo = "well-known" : rest,
-            rawPathInfo = "/well-known/" <> B.drop pfxLen (rawPathInfo req)
-          }
-      _ -> req
-    pfxLen = B.length "/.well-known/"
-
-generateSite :: ServerInformation -> Maybe TransportHost -> FilePath -> IO ()
-generateSite si onionHost sitePath = do
-  createDirectoryIfMissing True sitePath
-  B.writeFile (sitePath </> "index.html") $ serverInformation si onionHost
-  copyDir "media" E.mediaContent
-  -- `.well-known` path is re-written in changeWellKnownPath,
-  -- staticApp does not allow hidden folders.
-  copyDir "well-known" E.wellKnown
-  createLinkPage "contact"
-  createLinkPage "invitation"
-  createLinkPage "a"
-  createLinkPage "c"
-  createLinkPage "g"
-  createLinkPage "r"
-  createLinkPage "i"
-  logInfo $ "Generated static site contents at " <> tshow sitePath
-  where
-    copyDir dir content = do
-      createDirectoryIfMissing True $ sitePath </> dir
-      forM_ content $ \(path, s) -> B.writeFile (sitePath </> dir </> path) s
-    createLinkPage path = do
-      createDirectoryIfMissing True $ sitePath </> path
-      B.writeFile (sitePath </> path </> "index.html") E.linkHtml
-
-serverInformation :: ServerInformation -> Maybe TransportHost -> ByteString
-serverInformation ServerInformation {config, information} onionHost = render E.indexHtml substs
-  where
-    substs = substConfig <> substInfo <> [("onionHost", strEncode <$> onionHost)]
-    substConfig =
-      [ ( "persistence",
-          Just $ case persistence config of
-            SPMMemoryOnly -> "In-memory only"
-            SPMQueues -> "Queues"
-            SPMMessages -> "Queues and messages"
-        ),
-        ("messageExpiration", Just $ maybe "Never" (fromString . timedTTLText) $ messageExpiration config),
-        ("statsEnabled", Just . yesNo $ statsEnabled config),
-        ("newQueuesAllowed", Just . yesNo $ newQueuesAllowed config),
-        ("basicAuthEnabled", Just . yesNo $ basicAuthEnabled config)
-      ]
-    yesNo True = "Yes"
-    yesNo False = "No"
-    substInfo =
-      concat
-        [ basic,
-          maybe [("usageConditions", Nothing), ("usageAmendments", Nothing)] conds (usageConditions spi),
-          maybe [("operator", Nothing)] operatorE (operator spi),
-          maybe [("admin", Nothing)] admin (adminContacts spi),
-          maybe [("complaints", Nothing)] complaints (complaintsContacts spi),
-          maybe [("hosting", Nothing)] hostingE (hosting spi),
-          server
-        ]
-      where
-        basic =
-          [ ("sourceCode", if T.null sc then Nothing else Just (encodeUtf8 sc)),
-            ("noSourceCode", if T.null sc then Just "none" else Nothing),
-            ("version", Just $ B.pack simplexMQVersion),
-            ("commitSourceCode", Just $ encodeUtf8 $ maybe (T.pack simplexmqSource) sourceCode information),
-            ("shortCommit", Just $ B.pack $ take 7 simplexmqCommit),
-            ("commit", Just $ B.pack simplexmqCommit),
-            ("website", encodeUtf8 <$> website spi)
-          ]
-        spi = fromMaybe (emptyServerInfo "") information
-        sc = sourceCode spi
-        conds ServerConditions {conditions, amendments} =
-          [ ("usageConditions", Just $ encodeUtf8 conditions),
-            ("usageAmendments", encodeUtf8 <$> amendments)
-          ]
-        operatorE Entity {name, country} =
-          [ ("operator", Just ""),
-            ("operatorEntity", Just $ encodeUtf8 name),
-            ("operatorCountry", encodeUtf8 <$> country)
-          ]
-        admin ServerContactAddress {simplex, email, pgp} =
-          [ ("admin", Just ""),
-            ("adminSimplex", strEncode <$> simplex),
-            ("adminEmail", encodeUtf8 <$> email),
-            ("adminPGP", encodeUtf8 . pkURI <$> pgp),
-            ("adminPGPFingerprint", encodeUtf8 . pkFingerprint <$> pgp)
-          ]
-        complaints ServerContactAddress {simplex, email, pgp} =
-          [ ("complaints", Just ""),
-            ("complaintsSimplex", strEncode <$> simplex),
-            ("complaintsEmail", encodeUtf8 <$> email),
-            ("complaintsPGP", encodeUtf8 . pkURI <$> pgp),
-            ("complaintsPGPFingerprint", encodeUtf8 . pkFingerprint <$> pgp)
-          ]
-        hostingE Entity {name, country} =
-          [ ("hosting", Just ""),
-            ("hostingEntity", Just $ encodeUtf8 name),
-            ("hostingCountry", encodeUtf8 <$> country)
-          ]
-        server =
-          [ ("serverCountry", encodeUtf8 <$> serverCountry spi),
-            ("hostingType",  (\s -> maybe s (\(c, rest) -> toUpper c `B.cons` rest) $ B.uncons s) . strEncode <$> hostingType spi)
-          ]
-
--- Copy-pasted from simplex-chat Simplex.Chat.Types.Preferences
-{-# INLINE timedTTLText #-}
-timedTTLText :: (Integral i, Show i) => i -> String
-timedTTLText 0 = "0 sec"
-timedTTLText ttl = do
-  let (m', s) = ttl `quotRem` 60
-      (h', m) = m' `quotRem` 60
-      (d', h) = h' `quotRem` 24
-      (mm, d) = d' `quotRem` 30
-  unwords $
-    [mms mm | mm /= 0]
-      <> [ds d | d /= 0]
-      <> [hs h | h /= 0]
-      <> [ms m | m /= 0]
-      <> [ss s | s /= 0]
-  where
-    ss s = show s <> " sec"
-    ms m = show m <> " min"
-    hs 1 = "1 hour"
-    hs h = show h <> " hours"
-    ds 1 = "1 day"
-    ds 7 = "1 week"
-    ds 14 = "2 weeks"
-    ds d = show d <> " days"
-    mms 1 = "1 month"
-    mms mm = show mm <> " months"
-
--- | Rewrite source with provided substitutions
-render :: ByteString -> [(ByteString, Maybe ByteString)] -> ByteString
-render src = \case
-  [] -> src
-  (label, content') : rest -> render (section_ label content' src) rest
-
--- | Rewrite section content inside @<x-label>...</x-label>@ markers.
--- Markers are always removed when found. Closing marker is mandatory.
--- If content is absent, whole section is removed.
--- Section content is delegated to `item_`. If no sections found, the whole source is delegated.
-section_ :: ByteString -> Maybe ByteString -> ByteString -> ByteString
-section_ label content' src =
-  case B.breakSubstring startMarker src of
-    (_, "") -> item_ label (fromMaybe "" content') src -- no section, just replace items
-    (before, afterStart') ->
-      -- found section start, search for end too
-      case B.breakSubstring endMarker $ B.drop (B.length startMarker) afterStart' of
-        (_, "") -> error $ "missing section end: " <> show endMarker
-        (inside, next') ->
-          let next = B.drop (B.length endMarker) next'
-           in case content' of
-                Just content | not (B.null content) -> before <> item_ label content inside <> section_ label content' next
-                _ -> before <> next -- collapse section
-  where
-    startMarker = "<x-" <> label <> ">"
-    endMarker = "</x-" <> label <> ">"
-
--- | Replace all occurences of @${label}@ with provided content.
-item_ :: ByteString -> ByteString -> ByteString -> ByteString
-item_ label content' src =
-  case B.breakSubstring marker src of
-    (done, "") -> done
-    (before, after') -> before <> content' <> item_ label content' (B.drop (B.length marker) after')
-  where
-    marker = "${" <> label <> "}"

apps/smp-server/web/Static/Embedded.hs

@@ -1,18 +0,0 @@
-{-# LANGUAGE TemplateHaskell #-}
-
-module Static.Embedded where
-
-import Data.FileEmbed (embedDir, embedFile)
-import Data.ByteString (ByteString)
-
-indexHtml :: ByteString
-indexHtml = $(embedFile "apps/smp-server/static/index.html")
-
-linkHtml :: ByteString
-linkHtml = $(embedFile "apps/smp-server/static/link.html")
-
-mediaContent :: [(FilePath, ByteString)]
-mediaContent = $(embedDir "apps/smp-server/static/media/")
-
-wellKnown :: [(FilePath, ByteString)]
-wellKnown = $(embedDir "apps/smp-server/static/.well-known/")

apps/xftp-server/Main.hs

@@ -1,8 +1,10 @@
 module Main where
 
 import Control.Logger.Simple
+import Simplex.FileTransfer.Server.Main (xftpServerCLI_)
 import Simplex.Messaging.Server.CLI (getEnvPath)
-import Simplex.FileTransfer.Server.Main
+import Simplex.Messaging.Server.Web (serveStaticFiles)
+import XFTPWeb (xftpGenerateSite)
 
 defaultCfgPath :: FilePath
 defaultCfgPath = "/etc/opt/simplex-xftp"
@@ -18,4 +20,4 @@ main = do
   setLogLevel LogDebug -- change to LogError in production
   cfgPath <- getEnvPath "XFTP_SERVER_CFG_PATH" defaultCfgPath
   logPath <- getEnvPath "XFTP_SERVER_LOG_PATH" defaultLogPath
-  withGlobalLogging logCfg $ xftpServerCLI cfgPath logPath
+  withGlobalLogging logCfg $ xftpServerCLI_ xftpGenerateSite serveStaticFiles cfgPath logPath

apps/xftp-server/XFTPWeb.hs

@@ -0,0 +1,67 @@
+{-# LANGUAGE NamedFieldPuns #-}
+{-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE TemplateHaskell #-}
+
+module XFTPWeb
+  ( xftpGenerateSite,
+    xftpServerInformation,
+  ) where
+
+import Control.Monad (forM_)
+import qualified Data.ByteString.Char8 as B
+import Data.ByteString (ByteString)
+import Data.FileEmbed (embedDir, embedFile)
+import Data.Maybe (isJust)
+import Data.String (fromString)
+import Web.Embedded (embeddedContent)
+import Simplex.FileTransfer.Server.Env (XFTPServerConfig (..))
+import Simplex.Messaging.Encoding.String (strEncode)
+import Simplex.Messaging.Server.Expiration (ExpirationConfig (..))
+import Simplex.Messaging.Server.Information (ServerPublicInfo)
+import Simplex.Messaging.Server.Main (simplexmqSource)
+import qualified Simplex.Messaging.Server.Web as Web
+import Simplex.Messaging.Server.Web (render, serverInfoSubsts, timedTTLText)
+import Simplex.Messaging.Transport.Client (TransportHost (..))
+import System.Directory (createDirectoryIfMissing)
+import System.FilePath ((</>))
+
+xftpWebContent :: [(FilePath, ByteString)]
+xftpWebContent = $(embedDir "apps/xftp-server/static/xftp-web-bundle/")
+
+xftpMediaContent :: [(FilePath, ByteString)]
+xftpMediaContent = $(embedDir "apps/xftp-server/static/media/")
+
+xftpFilePageHtml :: ByteString
+xftpFilePageHtml = $(embedFile "apps/xftp-server/static/file.html")
+
+xftpGenerateSite :: XFTPServerConfig s -> Maybe ServerPublicInfo -> Maybe TransportHost -> FilePath -> IO ()
+xftpGenerateSite cfg info onionHost path = do
+  let substs = xftpSubsts cfg info onionHost
+  Web.generateSite embeddedContent (render (Web.indexHtml embeddedContent) substs) [] path
+  let xftpDir = path </> "xftp-web-bundle"
+      mediaDir = path </> "media"
+      fileDir = path </> "file"
+  filePage xftpDir xftpWebContent
+  filePage mediaDir xftpMediaContent
+  createDirectoryIfMissing True fileDir
+  B.writeFile (fileDir </> "index.html") $ render xftpFilePageHtml substs
+  where
+    filePage dir content_ = do
+      createDirectoryIfMissing True dir
+      forM_ content_ $ \(fp, content) -> B.writeFile (dir </> fp) content
+
+xftpServerInformation :: XFTPServerConfig s -> Maybe ServerPublicInfo -> Maybe TransportHost -> ByteString
+xftpServerInformation cfg info onionHost = render (Web.indexHtml embeddedContent) (xftpSubsts cfg info onionHost)
+
+xftpSubsts :: XFTPServerConfig s -> Maybe ServerPublicInfo -> Maybe TransportHost -> [(ByteString, Maybe ByteString)]
+xftpSubsts XFTPServerConfig {fileExpiration, logStatsInterval, allowNewFiles, newFileBasicAuth} information onionHost =
+  [("smpConfig", Nothing), ("xftpConfig", Just "y")] <> substConfig <> serverInfoSubsts simplexmqSource information <> [("onionHost", strEncode <$> onionHost), ("iniFileName", Just "file-server.ini")]
+  where
+    substConfig =
+      [ ("fileExpiration", Just $ maybe "Never" (fromString . timedTTLText . ttl) fileExpiration),
+        ("statsEnabled", Just . yesNo $ isJust logStatsInterval),
+        ("newUploadsAllowed", Just . yesNo $ allowNewFiles),
+        ("basicAuthEnabled", Just . yesNo $ isJust newFileBasicAuth)
+      ]
+    yesNo True = "Yes"
+    yesNo False = "No"

apps/xftp-server/static/media/xftp-protocol-dark.svg

@@ -0,0 +1,115 @@
+<svg width="440" height="520" viewBox="-20 0 440 520" fill="none" xmlns="http://www.w3.org/2000/svg">
+  <!-- Sender browser -->
+  <rect x="120" y="16" width="160" height="56" rx="10" stroke="#70F0F9" stroke-width="1.5"/>
+  <text x="200" y="40" text-anchor="middle" font-family="system-ui, sans-serif" font-size="13" font-weight="600" fill="#70F0F9">Sender's browser</text>
+  <text x="200" y="56" text-anchor="middle" font-family="system-ui, sans-serif" font-size="11" fill="rgba(112,240,249,0.7)">encrypts file</text>
+
+  <!-- Arrow down from sender to chunks -->
+  <line x1="200" y1="72" x2="200" y2="120" stroke="#70F0F9" stroke-width="1.5" marker-end="url(#arrowC)"/>
+
+  <!-- Chunks row -->
+  <rect x="112" y="120" width="176" height="40" rx="8" fill="none" stroke="#70F0F9" stroke-width="1" stroke-dasharray="4 3"/>
+  <text x="200" y="145" text-anchor="middle" font-family="system-ui, sans-serif" font-size="12" fill="#70F0F9">encrypted chunks</text>
+
+  <!-- Arrows from chunks to routers -->
+  <line x1="152" y1="160" x2="80" y2="220" stroke="#70F0F9" stroke-width="1.5" marker-end="url(#arrowC)"/>
+  <line x1="200" y1="160" x2="200" y2="220" stroke="#70F0F9" stroke-width="1.5" marker-end="url(#arrowC)"/>
+  <line x1="248" y1="160" x2="320" y2="220" stroke="#70F0F9" stroke-width="1.5" marker-end="url(#arrowC)"/>
+
+  <!-- Router 1 (SimpleX) -->
+  <rect x="20" y="220" width="120" height="56" rx="6" fill="none" stroke="#70F0F9" stroke-width="1.5"/>
+  <g transform="translate(28, 227)">
+    <rect width="14" height="4" rx="1" fill="rgba(112,240,249,0.5)"/>
+    <rect y="6" width="14" height="4" rx="1" fill="rgba(112,240,249,0.5)"/>
+    <rect y="12" width="14" height="4" rx="1" fill="rgba(112,240,249,0.5)"/>
+    <circle cx="11" cy="2" r="1" fill="#70F0F9"/>
+    <circle cx="11" cy="8" r="1" fill="#70F0F9"/>
+    <circle cx="11" cy="14" r="1" fill="#70F0F9"/>
+  </g>
+  <text x="80" y="244" text-anchor="middle" font-family="system-ui, sans-serif" font-size="11" font-weight="600" fill="#70F0F9">SimpleX</text>
+  <text x="80" y="258" text-anchor="middle" font-family="system-ui, sans-serif" font-size="9" fill="rgba(112,240,249,0.7)">XFTP router</text>
+
+  <!-- Router 2 (Flux) -->
+  <rect x="155" y="220" width="90" height="56" rx="6" fill="none" stroke="#70F0F9" stroke-width="1.5"/>
+  <g transform="translate(163, 227)">
+    <rect width="14" height="4" rx="1" fill="rgba(112,240,249,0.5)"/>
+    <rect y="6" width="14" height="4" rx="1" fill="rgba(112,240,249,0.5)"/>
+    <rect y="12" width="14" height="4" rx="1" fill="rgba(112,240,249,0.5)"/>
+    <circle cx="11" cy="2" r="1" fill="#70F0F9"/>
+    <circle cx="11" cy="8" r="1" fill="#70F0F9"/>
+    <circle cx="11" cy="14" r="1" fill="#70F0F9"/>
+  </g>
+  <text x="200" y="244" text-anchor="middle" font-family="system-ui, sans-serif" font-size="11" font-weight="600" fill="#70F0F9">Flux</text>
+  <text x="200" y="258" text-anchor="middle" font-family="system-ui, sans-serif" font-size="9" fill="rgba(112,240,249,0.7)">XFTP router</text>
+
+  <!-- Router 3 (SimpleX) -->
+  <rect x="260" y="220" width="120" height="56" rx="6" fill="none" stroke="#70F0F9" stroke-width="1.5"/>
+  <g transform="translate(268, 227)">
+    <rect width="14" height="4" rx="1" fill="rgba(112,240,249,0.5)"/>
+    <rect y="6" width="14" height="4" rx="1" fill="rgba(112,240,249,0.5)"/>
+    <rect y="12" width="14" height="4" rx="1" fill="rgba(112,240,249,0.5)"/>
+    <circle cx="11" cy="2" r="1" fill="#70F0F9"/>
+    <circle cx="11" cy="8" r="1" fill="#70F0F9"/>
+    <circle cx="11" cy="14" r="1" fill="#70F0F9"/>
+  </g>
+  <text x="320" y="244" text-anchor="middle" font-family="system-ui, sans-serif" font-size="11" font-weight="600" fill="#70F0F9">SimpleX</text>
+  <text x="320" y="258" text-anchor="middle" font-family="system-ui, sans-serif" font-size="9" fill="rgba(112,240,249,0.7)">XFTP router</text>
+
+  <!-- Arrows from routers down -->
+  <line x1="80" y1="276" x2="152" y2="336" stroke="#70F0F9" stroke-width="1.5" marker-end="url(#arrowC)"/>
+  <line x1="200" y1="276" x2="200" y2="336" stroke="#70F0F9" stroke-width="1.5" marker-end="url(#arrowC)"/>
+  <line x1="320" y1="276" x2="248" y2="336" stroke="#70F0F9" stroke-width="1.5" marker-end="url(#arrowC)"/>
+
+  <!-- Re-encrypt label -->
+  <text x="330" y="310" text-anchor="start" font-family="system-ui, sans-serif" font-size="10" fill="rgba(112,240,249,0.7)">re-encrypted</text>
+  <text x="330" y="322" text-anchor="start" font-family="system-ui, sans-serif" font-size="10" fill="rgba(112,240,249,0.7)">per recipient</text>
+
+  <!-- Chunks row (download) -->
+  <rect x="112" y="336" width="176" height="40" rx="8" fill="none" stroke="#70F0F9" stroke-width="1" stroke-dasharray="4 3"/>
+  <text x="200" y="361" text-anchor="middle" font-family="system-ui, sans-serif" font-size="12" fill="#70F0F9">encrypted chunks</text>
+
+  <!-- Arrow down to recipient -->
+  <line x1="200" y1="376" x2="200" y2="424" stroke="#70F0F9" stroke-width="1.5" marker-end="url(#arrowC)"/>
+
+  <!-- Recipient browser -->
+  <rect x="120" y="424" width="160" height="56" rx="10" stroke="#70F0F9" stroke-width="1.5"/>
+  <text x="200" y="448" text-anchor="middle" font-family="system-ui, sans-serif" font-size="13" font-weight="600" fill="#70F0F9">Recipient's browser</text>
+  <text x="200" y="464" text-anchor="middle" font-family="system-ui, sans-serif" font-size="11" fill="rgba(112,240,249,0.7)">decrypts file</text>
+
+  <!-- Key path (dashed, side) -->
+  <path d="M120 44 L8 44 L8 452 L120 452" stroke="#70F0F9" stroke-width="1.5" stroke-dasharray="6 4" fill="none" marker-end="url(#arrowC)"/>
+  <text x="-6" y="240" text-anchor="middle" font-family="system-ui, sans-serif" font-size="10" fill="#70F0F9" transform="rotate(-90 -6 240)">key in URL fragment - never sent to page server or data router</text>
+
+
+  <!-- Closed padlock: encryption (between sender and chunks) -->
+  <g transform="translate(192, 88)">
+    <path d="M4,7 V4 C4,1.2 12,1.2 12,4 V7" stroke="#60a5fa" stroke-width="1.5" fill="none" stroke-linecap="round"/>
+    <rect x="2" y="7" width="12" height="9" rx="2" fill="#60a5fa"/>
+    <circle cx="8" cy="12" r="1.2" fill="#0B2A59"/>
+  </g>
+
+  <!-- Open padlock: decryption (between chunks and recipient) -->
+  <g transform="translate(192, 392)">
+    <path d="M4,7 V4 C4,1.2 12,1.2 12,4 V2" stroke="#60a5fa" stroke-width="1.5" fill="none" stroke-linecap="round"/>
+    <rect x="2" y="7" width="12" height="9" rx="2" fill="#60a5fa"/>
+    <circle cx="8" cy="12" r="1.2" fill="#0B2A59"/>
+  </g>
+
+  <!-- Key icon on dashed line -->
+  <g transform="translate(8, 410)">
+    <circle cx="0" cy="0" r="6" stroke="#FBBF24" stroke-width="2" fill="#FBBF24"/>
+    <circle cx="0" cy="0" r="2" fill="#0B2A59"/>
+    <line x1="6" y1="0" x2="16" y2="0" stroke="#FBBF24" stroke-width="2"/>
+    <line x1="14" y1="0" x2="14" y2="4" stroke="#FBBF24" stroke-width="2"/>
+    <line x1="11" y1="0" x2="11" y2="3.5" stroke="#FBBF24" stroke-width="2"/>
+  </g>
+
+  <!-- Annotation: no shared IDs -->
+  <text x="200" y="510" text-anchor="middle" font-family="system-ui, sans-serif" font-size="10" fill="rgba(112,240,249,0.7)">Each file fragment uses unique anonymous credentials - no shared identifiers</text>
+
+  <defs>
+    <marker id="arrowC" viewBox="0 0 10 10" refX="9" refY="5" markerWidth="6" markerHeight="6" orient="auto-start-reverse">
+      <path d="M 0 0 L 10 5 L 0 10 z" fill="#70F0F9"/>
+    </marker>
+  </defs>
+</svg>

apps/xftp-server/static/media/xftp-protocol.svg

@@ -0,0 +1,130 @@
+<svg width="440" height="520" viewBox="-20 0 440 520" fill="none" xmlns="http://www.w3.org/2000/svg">
+  <!-- Sender browser -->
+  <rect x="120" y="16" width="160" height="56" rx="10" fill="url(#gBox)" stroke="#606C71" stroke-width="1.5"/>
+  <text x="200" y="40" text-anchor="middle" font-family="system-ui, sans-serif" font-size="13" font-weight="600" fill="#fff">Sender's browser</text>
+  <text x="200" y="56" text-anchor="middle" font-family="system-ui, sans-serif" font-size="11" fill="rgba(255,255,255,0.8)">encrypts file</text>
+
+  <!-- Arrow down from sender to chunks -->
+  <line x1="200" y1="72" x2="200" y2="120" stroke="#606C71" stroke-width="1.5" marker-end="url(#arrowG)"/>
+
+  <!-- Chunks row -->
+  <rect x="112" y="120" width="176" height="40" rx="8" fill="#f0f7ff" stroke="#0053D0" stroke-width="1" stroke-dasharray="4 3"/>
+  <text x="200" y="145" text-anchor="middle" font-family="system-ui, sans-serif" font-size="12" fill="#0053D0">encrypted chunks</text>
+
+  <!-- Arrows from chunks to routers -->
+  <line x1="152" y1="160" x2="80" y2="220" stroke="#606C71" stroke-width="1.5" marker-end="url(#arrowG)"/>
+  <line x1="200" y1="160" x2="200" y2="220" stroke="#606C71" stroke-width="1.5" marker-end="url(#arrowG)"/>
+  <line x1="248" y1="160" x2="320" y2="220" stroke="#606C71" stroke-width="1.5" marker-end="url(#arrowG)"/>
+
+  <!-- Router 1 (SimpleX) -->
+  <rect x="20" y="220" width="120" height="56" rx="6" fill="#f0f4f8" stroke="#606C71" stroke-width="1.5"/>
+  <g transform="translate(28, 227)">
+    <rect width="14" height="4" rx="1" fill="#606C71"/>
+    <rect y="6" width="14" height="4" rx="1" fill="#606C71"/>
+    <rect y="12" width="14" height="4" rx="1" fill="#606C71"/>
+    <circle cx="11" cy="2" r="1" fill="#53C1FF"/>
+    <circle cx="11" cy="8" r="1" fill="#53C1FF"/>
+    <circle cx="11" cy="14" r="1" fill="#53C1FF"/>
+  </g>
+  <text x="80" y="244" text-anchor="middle" font-family="system-ui, sans-serif" font-size="11" font-weight="600" fill="#3F484B">SimpleX</text>
+  <text x="80" y="258" text-anchor="middle" font-family="system-ui, sans-serif" font-size="9" fill="#606C71">XFTP router</text>
+
+  <!-- Router 2 (Flux) -->
+  <rect x="155" y="220" width="90" height="56" rx="6" fill="#f0f4f8" stroke="#606C71" stroke-width="1.5"/>
+  <g transform="translate(163, 227)">
+    <rect width="14" height="4" rx="1" fill="#606C71"/>
+    <rect y="6" width="14" height="4" rx="1" fill="#606C71"/>
+    <rect y="12" width="14" height="4" rx="1" fill="#606C71"/>
+    <circle cx="11" cy="2" r="1" fill="#53C1FF"/>
+    <circle cx="11" cy="8" r="1" fill="#53C1FF"/>
+    <circle cx="11" cy="14" r="1" fill="#53C1FF"/>
+  </g>
+  <text x="200" y="244" text-anchor="middle" font-family="system-ui, sans-serif" font-size="11" font-weight="600" fill="#3F484B">Flux</text>
+  <text x="200" y="258" text-anchor="middle" font-family="system-ui, sans-serif" font-size="9" fill="#606C71">XFTP router</text>
+
+  <!-- Router 3 (SimpleX) -->
+  <rect x="260" y="220" width="120" height="56" rx="6" fill="#f0f4f8" stroke="#606C71" stroke-width="1.5"/>
+  <g transform="translate(268, 227)">
+    <rect width="14" height="4" rx="1" fill="#606C71"/>
+    <rect y="6" width="14" height="4" rx="1" fill="#606C71"/>
+    <rect y="12" width="14" height="4" rx="1" fill="#606C71"/>
+    <circle cx="11" cy="2" r="1" fill="#53C1FF"/>
+    <circle cx="11" cy="8" r="1" fill="#53C1FF"/>
+    <circle cx="11" cy="14" r="1" fill="#53C1FF"/>
+  </g>
+  <text x="320" y="244" text-anchor="middle" font-family="system-ui, sans-serif" font-size="11" font-weight="600" fill="#3F484B">SimpleX</text>
+  <text x="320" y="258" text-anchor="middle" font-family="system-ui, sans-serif" font-size="9" fill="#606C71">XFTP router</text>
+
+  <!-- Arrows from routers down -->
+  <line x1="80" y1="276" x2="152" y2="336" stroke="#606C71" stroke-width="1.5" marker-end="url(#arrowG)"/>
+  <line x1="200" y1="276" x2="200" y2="336" stroke="#606C71" stroke-width="1.5" marker-end="url(#arrowG)"/>
+  <line x1="320" y1="276" x2="248" y2="336" stroke="#606C71" stroke-width="1.5" marker-end="url(#arrowG)"/>
+
+  <!-- Re-encrypt label -->
+  <text x="330" y="310" text-anchor="start" font-family="system-ui, sans-serif" font-size="10" fill="#606C71">re-encrypted</text>
+  <text x="330" y="322" text-anchor="start" font-family="system-ui, sans-serif" font-size="10" fill="#606C71">per recipient</text>
+
+  <!-- Chunks row (download) -->
+  <rect x="112" y="336" width="176" height="40" rx="8" fill="#f0f7ff" stroke="#0053D0" stroke-width="1" stroke-dasharray="4 3"/>
+  <text x="200" y="361" text-anchor="middle" font-family="system-ui, sans-serif" font-size="12" fill="#0053D0">encrypted chunks</text>
+
+  <!-- Arrow down to recipient -->
+  <line x1="200" y1="376" x2="200" y2="424" stroke="#606C71" stroke-width="1.5" marker-end="url(#arrowG)"/>
+
+  <!-- Recipient browser -->
+  <rect x="120" y="424" width="160" height="56" rx="10" fill="url(#gBox)" stroke="#606C71" stroke-width="1.5"/>
+  <text x="200" y="448" text-anchor="middle" font-family="system-ui, sans-serif" font-size="13" font-weight="600" fill="#fff">Recipient's browser</text>
+  <text x="200" y="464" text-anchor="middle" font-family="system-ui, sans-serif" font-size="11" fill="rgba(255,255,255,0.8)">decrypts file</text>
+
+  <!-- Key path (dashed, side) -->
+  <path d="M120 44 L8 44 L8 452 L120 452" stroke="#0053D0" stroke-width="1.5" stroke-dasharray="6 4" fill="none" marker-end="url(#arrowB)"/>
+  <text x="-6" y="240" text-anchor="middle" font-family="system-ui, sans-serif" font-size="10" fill="#0053D0" transform="rotate(-90 -6 240)">key in URL fragment - never sent to page server or data router</text>
+
+
+  <!-- Closed padlock: encryption (between sender and chunks) -->
+  <g transform="translate(192, 88)">
+    <path d="M4,7 V4 C4,1.2 12,1.2 12,4 V7" stroke="#0053D0" stroke-width="1.5" fill="none" stroke-linecap="round"/>
+    <rect x="2" y="7" width="12" height="9" rx="2" fill="#0053D0"/>
+    <circle cx="8" cy="12" r="1.2" fill="#fff"/>
+  </g>
+
+  <!-- Open padlock: decryption (between chunks and recipient) -->
+  <g transform="translate(192, 392)">
+    <path d="M4,7 V4 C4,1.2 12,1.2 12,4 V2" stroke="#0053D0" stroke-width="1.5" fill="none" stroke-linecap="round"/>
+    <rect x="2" y="7" width="12" height="9" rx="2" fill="#0053D0"/>
+    <circle cx="8" cy="12" r="1.2" fill="#fff"/>
+  </g>
+
+  <!-- Key icon on dashed line -->
+  <g transform="translate(8, 410)">
+    <circle cx="0" cy="0" r="6" stroke="#D97706" stroke-width="2" fill="#D97706"/>
+    <circle cx="0" cy="0" r="2" fill="#fff"/>
+    <line x1="6" y1="0" x2="16" y2="0" stroke="#D97706" stroke-width="2"/>
+    <line x1="14" y1="0" x2="14" y2="4" stroke="#D97706" stroke-width="2"/>
+    <line x1="11" y1="0" x2="11" y2="3.5" stroke="#D97706" stroke-width="2"/>
+  </g>
+
+  <!-- Annotation: no shared IDs -->
+  <text x="200" y="510" text-anchor="middle" font-family="system-ui, sans-serif" font-size="10" fill="#606C71">Each file fragment uses unique anonymous credentials - no shared identifiers</text>
+
+  <defs>
+    <linearGradient id="gBox" x1="120" y1="16" x2="280" y2="72" gradientUnits="userSpaceOnUse">
+      <stop stop-color="#0053D0"/>
+      <stop offset="1" stop-color="#53C1FF"/>
+    </linearGradient>
+    <linearGradient id="gSrv1" x1="20" y1="220" x2="140" y2="276" gradientUnits="userSpaceOnUse">
+      <stop stop-color="#0053D0"/>
+      <stop offset="1" stop-color="#53C1FF"/>
+    </linearGradient>
+    <linearGradient id="gSrv2" x1="155" y1="220" x2="245" y2="276" gradientUnits="userSpaceOnUse">
+      <stop stop-color="#0053D0"/>
+      <stop offset="1" stop-color="#53C1FF"/>
+    </linearGradient>
+    <marker id="arrowG" viewBox="0 0 10 10" refX="9" refY="5" markerWidth="6" markerHeight="6" orient="auto-start-reverse">
+      <path d="M 0 0 L 10 5 L 0 10 z" fill="#606C71"/>
+    </marker>
+    <marker id="arrowB" viewBox="0 0 10 10" refX="9" refY="5" markerWidth="6" markerHeight="6" orient="auto-start-reverse">
+      <path d="M 0 0 L 10 5 L 0 10 z" fill="#0053D0"/>
+    </marker>
+  </defs>
+</svg>

apps/xftp-server/static/xftp-web-bundle/index.css

@@ -0,0 +1,145 @@
+#app, [data-xftp-app] {
+  font-family: system-ui, -apple-system, sans-serif;
+  color: #333;
+  width: 100%;
+  max-width: 480px;
+  padding: 16px;
+  box-sizing: border-box;
+  --xftp-ring-fg: #3b82f6;
+}
+
+:is(#app, [data-xftp-app]) .card {
+  background: #fff;
+  border-radius: 12px;
+  padding: 32px 24px;
+  box-shadow: 0 1px 3px rgba(0,0,0,.1);
+  text-align: center;
+}
+
+:is(#app, [data-xftp-app]) h1 {
+  font-size: 1.25rem;
+  font-weight: 600;
+  margin-bottom: 24px;
+}
+
+:is(#app, [data-xftp-app]) .stage { margin-top: 16px; }
+
+/* Drop zone */
+:is(#app, [data-xftp-app]) .drop-zone {
+  border: 2px dashed #ccc;
+  border-radius: 8px;
+  padding: 32px 16px;
+  transition: border-color .15s, background .15s;
+}
+:is(#app, [data-xftp-app]) .drop-zone.drag-over {
+  border-color: #3b82f6;
+  background: #eff6ff;
+}
+
+/* Buttons */
+:is(#app, [data-xftp-app]) .btn {
+  display: inline-block;
+  padding: 10px 24px;
+  border: none;
+  border-radius: 6px;
+  background: #3b82f6;
+  color: #fff;
+  font-size: .9rem;
+  font-weight: 500;
+  cursor: pointer;
+  transition: background .15s;
+}
+:is(#app, [data-xftp-app]) .btn:hover { background: #2563eb; }
+:is(#app, [data-xftp-app]) .btn-secondary { background: #6b7280; }
+:is(#app, [data-xftp-app]) .btn-secondary:hover { background: #4b5563; }
+
+/* Hints */
+:is(#app, [data-xftp-app]) .hint { color: #999; font-size: .85rem; margin-top: 8px; }
+:is(#app, [data-xftp-app]) .expiry { margin-top: 12px; }
+
+/* Progress */
+:is(#app, [data-xftp-app]) .progress-ring { display: block; margin: 0 auto 12px; }
+:is(#app, [data-xftp-app]) #upload-status,
+:is(#app, [data-xftp-app]) #dl-status { font-size: .9rem; color: #666; margin-bottom: 12px; }
+
+/* Share link row */
+:is(#app, [data-xftp-app]) .link-row {
+  display: flex;
+  gap: 8px;
+  margin-top: 12px;
+}
+:is(#app, [data-xftp-app]) .link-row input {
+  flex: 1;
+  padding: 8px 10px;
+  border: 1px solid #ccc;
+  border-radius: 6px;
+  font-size: .85rem;
+  background: #f9fafb;
+}
+
+/* Upload link */
+:is(#app, [data-xftp-app]) .upload-link {
+  margin-top: 12px;
+  color: #3b82f6;
+  font-size: .9rem;
+  text-decoration: none;
+  cursor: pointer;
+}
+:is(#app, [data-xftp-app]) .upload-link:not([hidden]) {
+  display: inline-block;
+}
+:is(#app, [data-xftp-app]) .upload-link:hover { text-decoration: underline; }
+
+/* Messages */
+:is(#app, [data-xftp-app]) .success { color: #16a34a; font-weight: 600; }
+:is(#app, [data-xftp-app]) .error { color: #dc2626; font-weight: 500; margin-bottom: 12px; }
+
+/* Security note */
+:is(#app, [data-xftp-app]) .security-note {
+  margin-top: 20px;
+  padding: 12px;
+  background: #f0fdf4;
+  border-radius: 6px;
+  font-size: .8rem;
+  color: #555;
+  text-align: left;
+}
+:is(#app, [data-xftp-app]) .security-note p + p { margin-top: 6px; }
+:is(#app, [data-xftp-app]) .security-note a { color: #3b82f6; text-decoration: none; }
+:is(#app, [data-xftp-app]) .security-note a:hover { text-decoration: underline; }
+
+/* ── Dark mode ─────────────────────────────────── */
+.dark :is(#app, [data-xftp-app]) {
+  color: #e5e7eb;
+  --xftp-ring-bg: #374151;
+  --xftp-ring-fg: #60a5fa;
+  --xftp-ring-text: #e5e7eb;
+  --xftp-ring-done: #4ade80;
+}
+.dark :is(#app, [data-xftp-app]) .card {
+  background: #1f2937;
+  box-shadow: 0 1px 3px rgba(0,0,0,.4);
+}
+.dark :is(#app, [data-xftp-app]) .drop-zone { border-color: #4b5563; }
+.dark :is(#app, [data-xftp-app]) .drop-zone.drag-over {
+  border-color: #60a5fa;
+  background: rgba(59,130,246,.15);
+}
+.dark :is(#app, [data-xftp-app]) .btn-secondary { background: #4b5563; }
+.dark :is(#app, [data-xftp-app]) .btn-secondary:hover { background: #374151; }
+.dark :is(#app, [data-xftp-app]) .hint { color: #9ca3af; }
+.dark :is(#app, [data-xftp-app]) #upload-status,
+.dark :is(#app, [data-xftp-app]) #dl-status { color: #9ca3af; }
+.dark :is(#app, [data-xftp-app]) .link-row input {
+  background: #374151;
+  border-color: #4b5563;
+  color: #e5e7eb;
+}
+.dark :is(#app, [data-xftp-app]) .success { color: #4ade80; }
+.dark :is(#app, [data-xftp-app]) .error { color: #f87171; }
+.dark :is(#app, [data-xftp-app]) .security-note {
+  background: rgba(34,197,94,.1);
+  color: #d1d5db;
+}
+.dark :is(#app, [data-xftp-app]) .upload-link { color: #60a5fa; }
+.dark :is(#app, [data-xftp-app]) .security-note a { color: #60a5fa; }

plans/2026-03-25-xftp-postgres-backend-design.md

@@ -0,0 +1,472 @@
+# XFTP Server PostgreSQL Backend
+
+## Overview
+
+Add PostgreSQL backend support to xftp-server, following the SMP server pattern. Supports bidirectional migration between STM (in-memory with StoreLog) and PostgreSQL backends.
+
+## Goals
+
+- PostgreSQL-backed file metadata storage as an alternative to STM + StoreLog
+- Polymorphic server code via `FileStoreClass` typeclass with IO-based methods (following `QueueStoreClass` pattern)
+- Bidirectional migration: StoreLog <-> PostgreSQL via CLI commands
+- Shared `server_postgres` cabal flag (same flag enables both SMP and XFTP Postgres support)
+- INI-based backend selection at runtime
+
+## Architecture
+
+### FileStoreClass Typeclass
+
+IO-based typeclass following the `QueueStoreClass` pattern — each method is a self-contained IO action, with the implementation responsible for its own atomicity (STM backend wraps in `atomically`, Postgres backend uses database transactions):
+
+```haskell
+class FileStoreClass s where
+  type FileStoreConfig s
+
+  -- Lifecycle
+  newFileStore :: FileStoreConfig s -> IO s
+  closeFileStore :: s -> IO ()
+
+  -- File operations
+  addFile :: s -> SenderId -> FileInfo -> RoundedFileTime -> ServerEntityStatus -> IO (Either XFTPErrorType ())
+  setFilePath :: s -> SenderId -> FilePath -> IO (Either XFTPErrorType ())
+  addRecipient :: s -> SenderId -> FileRecipient -> IO (Either XFTPErrorType ())
+  getFile :: s -> SFileParty p -> XFTPFileId -> IO (Either XFTPErrorType (FileRec, C.APublicAuthKey))
+  deleteFile :: s -> SenderId -> IO (Either XFTPErrorType ())
+  blockFile :: s -> SenderId -> BlockingInfo -> Bool -> IO (Either XFTPErrorType ())
+  deleteRecipient :: s -> RecipientId -> FileRec -> IO ()
+  ackFile :: s -> RecipientId -> IO (Either XFTPErrorType ())
+
+  -- Expiration (with LIMIT for Postgres; called in a loop until empty)
+  expiredFiles :: s -> Int64 -> Int -> IO [(SenderId, Maybe FilePath, Word32)]
+
+  -- Storage and stats (for init-time computation)
+  getUsedStorage :: s -> IO Int64
+  getFileCount :: s -> IO Int
+```
+
+- STM backend: each method wraps its STM transaction in `atomically` internally.
+- Postgres backend: each method runs its query via `withDB` / database connection internally.
+
+No polymorphic monad or `runStore` dispatcher needed — unlike `MsgStoreClass`, XFTP file operations are individually atomic and don't require grouping multiple operations into backend-dependent transactions.
+
+### PostgresFileStore Data Type
+
+```haskell
+data PostgresFileStore = PostgresFileStore
+  { dbStore :: DBStore,
+    dbStoreLog :: Maybe (StoreLog 'WriteMode)
+  }
+```
+
+- `dbStore` — connection pool created via `createDBStore`, runs schema migrations on init.
+- `dbStoreLog` — optional parallel log file (enabled by `db_store_log` INI setting). When present, every mutation (`addFile`, `setFilePath`, `deleteFile`, `blockFile`, `addRecipient`, `ackFile`) also writes to this log via a `withLog` wrapper. `withLog` is called AFTER the DB operation succeeds (so the log reflects committed state only). Log write failures are non-fatal (logged as warnings, do not fail the DB operation). This provides an audit trail and enables recovery via export.
+
+`closeFileStore` for Postgres calls `closeDBStore` (closes connection pool) then `mapM_ closeStoreLog dbStoreLog` (flushes and closes the parallel log). For STM, it closes the storeLog. Called from a `finally` block during server shutdown, matching SMP's `stopServer` → `closeMsgStore` → `closeQueueStore` pattern.
+
+### STMFileStore Type
+
+After extracting from current `Store.hs`, `STMFileStore` retains the file and recipient maps but no longer owns `usedStorage` (moved to `XFTPEnv`):
+
+```haskell
+data STMFileStore = STMFileStore
+  { files :: TMap SenderId FileRec,
+    recipients :: TMap RecipientId (SenderId, RcvPublicAuthKey)
+  }
+```
+
+`closeFileStore` for STM is a no-op (TMaps are garbage-collected; the env-level `storeLog` is closed separately by the server).
+
+### Error Handling
+
+Postgres operations follow SMP's `withDB` / `handleDuplicate` pattern:
+
+```haskell
+withDB :: Text -> PostgresFileStore -> (DB.Connection -> IO (Either XFTPErrorType a)) -> ExceptT XFTPErrorType IO a
+withDB op st action =
+  ExceptT $ E.try (withTransaction (dbStore st) action) >>= either logErr pure
+  where
+    logErr :: E.SomeException -> IO (Either XFTPErrorType a)
+    logErr e = logError ("STORE: " <> err) $> Left INTERNAL
+      where
+        err = op <> ", withDB, " <> tshow e
+
+handleDuplicate :: SqlError -> IO (Either XFTPErrorType a)
+handleDuplicate e = case constraintViolation e of
+  Just (UniqueViolation _) -> pure $ Left DUPLICATE_
+  _ -> E.throwIO e
+```
+
+- All DB operations wrapped in `withDB` — catches exceptions, logs, returns `INTERNAL`.
+- Unique constraint violations caught by `handleDuplicate` and mapped to `DUPLICATE_`.
+- UPDATE operations verified with `assertUpdated` — returns `AUTH` if 0 rows affected (matching SMP pattern, prevents silent failures when WHERE clause doesn't match).
+- Critical sections (DB write + TVar update) wrapped in `uninterruptibleMask_` to prevent async exceptions from leaving inconsistent state between DB and TVars.
+
+### FileRec and TVar Fields
+
+`FileRec` retains its `TVar` fields (matching SMP's `PostgresQueue` pattern):
+
+```haskell
+data FileRec = FileRec
+  { senderId :: SenderId,
+    fileInfo :: FileInfo,
+    filePath :: TVar (Maybe FilePath),
+    recipientIds :: TVar (Set RecipientId),
+    createdAt :: RoundedFileTime,
+    fileStatus :: TVar ServerEntityStatus
+  }
+```
+
+- **STM backend**: TVars are the source of truth, as currently.
+- **Postgres backend**: `getFile` reads from DB and creates a `FileRec` with fresh TVars populated from the DB row (matching SMP's `mkQ` pattern — `newTVarIO` per load). Mutation methods (`setFilePath`, `blockFile`, etc.) update both the DB (persistence) and the TVars (in-session consistency). The `recipientIds` TVar is initialized to `S.empty` — no subquery needed because no server code reads `recipientIds` directly; all recipient operations go through the typeclass methods (`addRecipient`, `deleteRecipient`, `ackFile`), which query the `recipients` table for Postgres.
+
+### usedStorage Ownership
+
+`usedStorage :: TVar Int64` moves from the store to `XFTPEnv`. The store typeclass does **not** manage `usedStorage` — it only provides `getUsedStorage` for init-time computation.
+
+- **STM init**: StoreLog replay calls `setFilePath` (which only sets the filePath TVar — the STM `setFilePath` implementation is changed to **not** update `usedStorage`). Similarly, STM `deleteFile` (Store.hs line 117) and `blockFile` (line 125) are changed to **not** update `usedStorage` — the server handles all `usedStorage` adjustments externally. After replay, `getUsedStorage` computes the sum over all file sizes (matching current `countUsedStorage` behavior).
+- **Postgres init**: `getUsedStorage` executes `SELECT COALESCE(SUM(file_size), 0) FROM files`.
+- **Runtime**: Server manages `usedStorage` TVar directly for reserve/commit/rollback during uploads, and adjusts after `deleteFile`/`blockFile` calls.
+
+**Note on `getUsedStorage` semantics**: The current STM `countUsedStorage` sums all file sizes unconditionally (including files without `filePath` set, i.e., created but not yet uploaded). The Postgres `getUsedStorage` matches this: `SELECT SUM(file_size) FROM files` (no `WHERE file_path IS NOT NULL`). In practice, orphaned files (created but never uploaded) are rare and short-lived (expired within 48h), so the difference is negligible. A future improvement could filter by `file_path IS NOT NULL` in both backends to reflect actual disk usage more accurately.
+
+### Server.hs Refactoring
+
+`Server.hs` becomes polymorphic over `FileStoreClass s`. Since all typeclass methods are IO, call sites replace `atomically` with direct IO calls to the store.
+
+**Call sites requiring changes** (exhaustive list):
+
+1. **`receiveServerFile`** (line 563): `atomically $ writeTVar filePath (Just fPath)` → `setFilePath store senderId fPath`. The `reserve` logic (line 551-555) stays as direct TVar manipulation on `usedStorage` from `XFTPEnv`.
+
+2. **`verifyXFTPTransmission`** (line 453): `atomically $ verify =<< getFile st party fId` — the `getFile` call and subsequent `readTVar fileStatus` are in a single `atomically` block. Refactored to: `getFile st party fId` (IO), then `readTVarIO (fileStatus fr)` from the returned `FileRec` (safe for both backends — STM TVar is the source of truth, Postgres TVar is a fresh snapshot from DB).
+
+3. **`retryAdd`** (line 516): Signature `XFTPFileId -> STM (Either XFTPErrorType a)` → `XFTPFileId -> IO (Either XFTPErrorType a)`. The `atomically` call (line 520) replaced with `liftIO`.
+
+4. **`deleteOrBlockServerFile_`** (line 620): Parameter `FileStore -> STM (Either XFTPErrorType ())` → `FileStoreClass s => s -> IO (Either XFTPErrorType ())`. The `atomically` call (line 626) removed — the store method is already IO. After the store action, server adjusts `usedStorage` TVar in `XFTPEnv` based on `fileInfo.size`.
+
+5. **`ackFileReception`** (line 605): `atomically $ deleteRecipient st rId fr` → `deleteRecipient st rId fr`.
+
+6. **Control port `CPDelete`/`CPBlock`** (lines 371, 377): `atomically $ getFile fs SFRecipient fileId` → `getFile fs SFRecipient fileId`.
+
+7. **`expireServerFiles`** (line 636): Replace per-file `expiredFilePath` iteration with batched `expiredFiles st old batchSize`, which returns `[(SenderId, Maybe FilePath, Word32)]` — the `Word32` file size is needed so the server can adjust the `usedStorage` TVar after each deletion. Called in a loop until the returned list is empty. The `itemDelay` between files applies to the deletion loop over each batch, not the query itself. STM backend ignores the batch size limit (returns all expired files from TMap scan); Postgres uses `LIMIT`.
+
+8. **`restoreServerStats`** (line 694): `FileStore {files, usedStorage} <- asks store` accesses store fields directly. Refactored to: `usedStorage` from `XFTPEnv` via `asks usedStorage`, file count via `getFileCount store`. STM: `M.size <$> readTVarIO files`. Postgres: `SELECT COUNT(*) FROM files`.
+
+### Store Config Selection
+
+GADT in `Env.hs`:
+
+```haskell
+data XFTPStoreConfig s where
+  XSCMemory :: Maybe FilePath -> XFTPStoreConfig STMFileStore
+#if defined(dbServerPostgres)
+  XSCDatabase :: PostgresFileStoreCfg -> XFTPStoreConfig PostgresFileStore
+#endif
+```
+
+`XFTPEnv` becomes polymorphic:
+
+```haskell
+data XFTPEnv s = XFTPEnv
+  { config :: XFTPServerConfig,
+    store :: s,
+    usedStorage :: TVar Int64,
+    storeLog :: Maybe (StoreLog 'WriteMode),
+    ...
+  }
+```
+
+The `M` monad (`ReaderT (XFTPEnv s) IO`) and all functions in `Server.hs` gain `FileStoreClass s =>` constraints.
+
+**StoreLog lifecycle per backend:**
+
+- **STM mode**: `storeLog = Just sl` (current behavior — append-only log for persistence and recovery).
+- **Postgres mode**: `storeLog = Nothing` (main storeLog disabled — Postgres is the source of truth). The optional parallel `dbStoreLog` inside `PostgresFileStore` provides audit/recovery if enabled via `db_store_log` INI setting.
+
+The existing `withFileLog` pattern in Server.hs continues to work unchanged — it maps over `Maybe (StoreLog 'WriteMode)`, which is `Nothing` in Postgres mode so the calls become no-ops.
+
+### Main.hs Store Type Dispatch
+
+The `Start` CLI command gains a `--confirm-migrations` flag (default `MCConsole` — manual prompt, matching SMP's `StartOptions`). For automated deployments, `--confirm-migrations up` auto-applies forward migrations. The import command uses `MCYesUp` (always auto-apply).
+
+Following SMP's existential dispatch pattern (`AStoreType` + `run`), `Main.hs` selects the store type from INI config and dispatches to the polymorphic server:
+
+```haskell
+runServer ini = do
+  let storeType = fromRight "memory" $ lookupValue "STORE_LOG" "store_files" ini
+  case storeType of
+    "memory" -> run $ XSCMemory (enableStoreLog $> storeLogFilePath)
+    "database" ->
+#if defined(dbServerPostgres)
+      run $ XSCDatabase PostgresFileStoreCfg {..}
+#else
+      exitError "server not compiled with Postgres support"
+#endif
+    _ -> exitError $ "Invalid store_files value: " <> storeType
+  where
+    run :: FileStoreClass s => XFTPStoreConfig s -> IO ()
+    run storeCfg = do
+      env <- newXFTPServerEnv storeCfg config
+      runReaderT (xftpServer config) env
+```
+
+**`newXFTPServerEnv` refactored signature:**
+
+```haskell
+newXFTPServerEnv :: FileStoreClass s => XFTPStoreConfig s -> XFTPServerConfig -> IO (XFTPEnv s)
+newXFTPServerEnv storeCfg config = do
+  (store, storeLog) <- case storeCfg of
+    XSCMemory storeLogPath -> do
+      st <- newFileStore ()
+      sl <- mapM (`readWriteFileStore` st) storeLogPath
+      pure (st, sl)
+    XSCDatabase dbCfg -> do
+      st <- newFileStore dbCfg
+      pure (st, Nothing)  -- main storeLog disabled for Postgres
+  usedStorage <- newTVarIO =<< getUsedStorage store
+  ...
+  pure XFTPEnv {config, store, usedStorage, storeLog, ...}
+```
+
+### Startup Config Validation
+
+Following SMP's `checkMsgStoreMode` pattern, `Main.hs` validates config before starting:
+
+- **`store_files=database` + StoreLog file exists** (without `db_store_log=on`): Error — "StoreLog file present but store_files is `database`. Use `xftp-server database import` to migrate, or set `db_store_log: on`."
+- **`store_files=database` + schema doesn't exist**: Error — "Create schema in PostgreSQL or use `xftp-server database import`."
+- **`store_files=memory` + Postgres schema exists**: Warning — "Postgres schema exists but store_files is `memory`. Data in Postgres will not be used."
+- **Binary compiled without `server_postgres` + `store_files=database`**: Error — "Server not compiled with Postgres support."
+
+## Module Structure
+
+```
+src/Simplex/FileTransfer/Server/
+  Store.hs                    -- FileStoreClass typeclass + shared types (FileRec, FileRecipient, etc.)
+  Store/
+    STM.hs                    -- STMFileStore (extracted from current Store.hs)
+    Postgres.hs               -- PostgresFileStore                [CPP-guarded]
+    Postgres/
+      Migrations.hs           -- Schema migrations                [CPP-guarded]
+      Config.hs               -- PostgresFileStoreCfg             [CPP-guarded]
+  StoreLog.hs                 -- Unchanged (interchange format for both backends + migration)
+  Env.hs                      -- XFTPStoreConfig GADT, polymorphic XFTPEnv
+  Main.hs                     -- Store selection, migration CLI commands
+  Server.hs                   -- Polymorphic over FileStoreClass
+```
+
+## PostgreSQL Schema
+
+Initial migration (`20260325_initial`):
+
+```sql
+CREATE TABLE files (
+  sender_id BYTEA NOT NULL PRIMARY KEY,
+  file_size INT4 NOT NULL,
+  file_digest BYTEA NOT NULL,
+  sender_key BYTEA NOT NULL,
+  file_path TEXT,
+  created_at INT8 NOT NULL,
+  status TEXT NOT NULL DEFAULT 'active'
+);
+
+CREATE TABLE recipients (
+  recipient_id BYTEA NOT NULL PRIMARY KEY,
+  sender_id BYTEA NOT NULL REFERENCES files ON DELETE CASCADE,
+  recipient_key BYTEA NOT NULL
+);
+
+CREATE INDEX idx_recipients_sender_id ON recipients (sender_id);
+CREATE INDEX idx_files_created_at ON files (created_at);
+```
+
+- `file_size` is `INT4` matching `Word32` in `FileInfo.size`
+- `sender_key` and `recipient_key` stored as `BYTEA` using binary encoding via `C.encodePubKey` / `C.decodePubKey` (matching SMP's `ToField`/`FromField` instances for `APublicAuthKey` — includes algorithm type tag in the binary format)
+- `file_path` nullable (set after upload completes via `setFilePath`)
+- `ON DELETE CASCADE` for recipients when file is hard-deleted
+- `created_at` stores rounded epoch seconds (1-hour precision, `RoundedFileTime`)
+- `status` as TEXT via `StrEncoding` (`ServerEntityStatus`: `EntityActive`, `EntityBlocked info`, `EntityOff`)
+- Hard deletes (no `deleted_at` column)
+- No PL/pgSQL functions needed; `setFilePath` uses `WHERE file_path IS NULL` to prevent duplicate uploads (the `UPDATE` itself acquires a row-level lock)
+- `used_storage` computed on startup: `SELECT COALESCE(SUM(file_size), 0) FROM files` (matches STM `countUsedStorage` — all files, see usedStorage Ownership section)
+
+### Migrations Module
+
+Following SMP's `QueueStore/Postgres/Migrations.hs` pattern:
+
+```haskell
+module Simplex.FileTransfer.Server.Store.Postgres.Migrations
+  ( xftpServerMigrations,
+  )
+where
+
+import Data.List (sortOn)
+import Data.Text (Text)
+import Simplex.Messaging.Agent.Store.Shared
+import Text.RawString.QQ (r)
+
+xftpSchemaMigrations :: [(String, Text, Maybe Text)]
+xftpSchemaMigrations =
+  [ ("20260325_initial", m20260325_initial, Nothing)
+  ]
+
+xftpServerMigrations :: [Migration]
+xftpServerMigrations = sortOn name $ map migration xftpSchemaMigrations
+  where
+    migration (name, up, down) = Migration {name, up, down = down}
+
+m20260325_initial :: Text
+m20260325_initial =
+  [r|
+CREATE TABLE files (
+  sender_id BYTEA NOT NULL PRIMARY KEY,
+  ...
+);
+  |]
+```
+
+The `Migration` type (from `Simplex.Messaging.Agent.Store.Shared`) has fields `{name :: String, up :: Text, down :: Maybe Text}`. Initial migration has `Nothing` for `down`. Future migrations should include `Just down_migration` for rollback support. Called via `createDBStore dbOpts xftpServerMigrations (MigrationConfig confirmMigrations Nothing)`.
+
+### Postgres Operations
+
+Key query patterns:
+
+- **`addFile`**: `INSERT INTO files (...) VALUES (...)`, return `DUPLICATE_` on unique violation.
+- **`setFilePath`**: `UPDATE files SET file_path = ? WHERE sender_id = ? AND file_path IS NULL`, verified with `assertUpdated` (returns `AUTH` if 0 rows affected — file not found or already uploaded). The `WHERE file_path IS NULL` prevents duplicate uploads; the `UPDATE` acquires a row lock implicitly. Only persists the path; `usedStorage` managed by server.
+- **`addRecipient`**: `INSERT INTO recipients (...)`, plus check for duplicates. No need for `recipientIds` TVar update — Postgres derives it from the table.
+- **`getFile`** (sender): `SELECT ... FROM files WHERE sender_id = ?`, returns auth key from `sender_key` column.
+- **`getFile`** (recipient): `SELECT f.*, r.recipient_key FROM recipients r JOIN files f ON ... WHERE r.recipient_id = ?`.
+- **`deleteFile`**: `DELETE FROM files WHERE sender_id = ?` (recipients cascade).
+- **`blockFile`**: `UPDATE files SET status = ? WHERE sender_id = ?`. When `deleted = True`, the server adjusts `usedStorage` externally (matching current STM behavior where `blockFile` only updates status and storage, not `filePath`).
+- **`expiredFiles`**: `SELECT sender_id, file_path, file_size FROM files WHERE created_at + ? < ? LIMIT ?` — batched query replaces per-file iteration, includes `file_size` for `usedStorage` adjustment. Called in a loop until no rows returned.
+
+## INI Configuration
+
+New keys in `[STORE_LOG]` section:
+
+```ini
+[STORE_LOG]
+enable: on
+store_files: memory            # memory | database
+db_connection: postgresql://xftp@/xftp_server_store
+db_schema: xftp_server
+db_pool_size: 10
+db_store_log: off
+expire_files_hours: 48
+```
+
+`store_files` selects the backend (`store_files` rather than `store_queues` because XFTP stores files, not queues):
+- `memory` -> `XSCMemory` (current behavior)
+- `database` -> `XSCDatabase` (requires `server_postgres` build flag)
+
+### INI Template Generation (`xftp-server init`)
+
+The `iniFileContent` function in `Main.hs` must be updated to generate the new keys in the `[STORE_LOG]` section. Following SMP's `iniDbOpts` pattern with `optDisabled'` (prefixes `"# "` when value equals default), Postgres keys are generated commented out by default:
+
+```ini
+[STORE_LOG]
+enable: on
+
+# File storage mode: `memory` or `database` (PostgreSQL).
+store_files: memory
+
+# Database connection settings for PostgreSQL database (`store_files: database`).
+# db_connection: postgresql://xftp@/xftp_server_store
+# db_schema: xftp_server
+# db_pool_size: 10
+
+# Write database changes to store log file
+# db_store_log: off
+
+expire_files_hours: 48
+```
+
+Reuses `iniDBOptions` from `Simplex.Messaging.Server.CLI` for runtime parsing (falls back to defaults when keys are commented out or missing). `enableDbStoreLog'` pattern (`settingIsOn "STORE_LOG" "db_store_log"`) controls `dbStoreLogPath`.
+
+### PostgresFileStoreCfg
+
+```haskell
+data PostgresFileStoreCfg = PostgresFileStoreCfg
+  { dbOpts :: DBOpts,
+    dbStoreLogPath :: Maybe FilePath,
+    confirmMigrations :: MigrationConfirmation
+  }
+```
+
+No `deletedTTL` (hard deletes).
+
+### Default DB Options
+
+```haskell
+defaultXFTPDBOpts :: DBOpts
+defaultXFTPDBOpts =
+  DBOpts
+    { connstr = "postgresql://xftp@/xftp_server_store",
+      schema = "xftp_server",
+      poolSize = 10,
+      createSchema = False
+    }
+```
+
+## Migration CLI
+
+Bidirectional migration via StoreLog as interchange format:
+
+```
+xftp-server database import [--database DB_CONN] [--schema DB_SCHEMA] [--pool-size N]
+xftp-server database export [--database DB_CONN] [--schema DB_SCHEMA] [--pool-size N]
+```
+
+No `--table` flag needed (unlike SMP which has queues/messages/all) — XFTP has a single entity type (files + recipients, always migrated together).
+
+CLI options reuse `dbOptsP` parser from `Simplex.Messaging.Server.CLI`.
+
+### Import (StoreLog -> PostgreSQL)
+
+1. Confirm: prompt user with database connection details and StoreLog path
+2. Read and replay StoreLog into temporary `STMFileStore`
+3. Connect to PostgreSQL, run schema migrations (`createSchema = True`, `confirmMigrations = MCYesUp`)
+4. Batch-insert file records into `files` table using PostgreSQL COPY protocol (matching SMP's `batchInsertQueues` pattern for performance). Progress reported every 10k files.
+5. Batch-insert recipient records into `recipients` table using COPY protocol
+6. Verify counts: `SELECT COUNT(*) FROM files` / `recipients` — warn if mismatch
+7. Rename StoreLog to `.bak` (prevents accidental re-import, preserves original for rollback)
+8. Report counts
+
+### Export (PostgreSQL -> StoreLog)
+
+1. Confirm: prompt user with database connection details and output path. Fail if output file already exists.
+2. Connect to PostgreSQL
+3. Open new StoreLog file for writing
+4. Fold over all file records, writing per file (in this order, matching existing `writeFileStore`): `AddFile` (with `ServerEntityStatus` — this preserves `EntityBlocked` state), `AddRecipients`, then `PutFile` (if `file_path` is set)
+5. Report counts
+
+Note: `AddFile` carries `ServerEntityStatus` which includes `EntityBlocked info`, so blocking state is preserved through export/import without needing separate `BlockFile` log entries.
+
+File data on disk is untouched by migration — only metadata moves between backends.
+
+## Cabal Integration
+
+Shared `server_postgres` flag. New Postgres modules added to existing conditional block:
+
+```cabal
+if flag(server_postgres)
+  cpp-options: -DdbServerPostgres
+  exposed-modules:
+    ...existing SMP modules...
+    Simplex.FileTransfer.Server.Store.Postgres
+    Simplex.FileTransfer.Server.Store.Postgres.Migrations
+    Simplex.FileTransfer.Server.Store.Postgres.Config
+```
+
+CPP guards (`#if defined(dbServerPostgres)`) in:
+- `Store.hs` — Postgres `FromField`/`ToField` instances for XFTP-specific types if needed
+- `Env.hs` — `XSCDatabase` constructor
+- `Main.hs` — database CLI commands, store selection for `database` mode, Postgres imports
+- `Server.hs` — Postgres-specific imports if needed
+
+## Testing
+
+- **Parameterized server tests**: Existing `xftpServerTests` refactored to accept a store type parameter (following SMP's `SpecWith (ASrvTransport, AStoreType)` pattern). The same server tests run against both STM and Postgres backends — STM tests run unconditionally, Postgres tests added under `#if defined(dbServerPostgres)` with `postgressBracket` for database lifecycle (drop → create → test → drop).
+- **Unit tests**: `PostgresFileStore` operations — add/get/delete/block/expire, duplicate detection, auth errors
+- **Migration round-trip**: STM store → export to StoreLog → import to Postgres → export back → verify StoreLog equality (including blocked file status)
+- **Tests location**: in `tests/` alongside existing XFTP tests, guarded by `server_postgres` CPP flag
+- **Test database**: PostgreSQL on `localhost:5432`, using a dedicated `xftp_server_test` schema (dropped and recreated per test run via `postgressBracket`, following SMP's test database lifecycle pattern)
+- **Test fixtures**: `testXFTPStoreDBOpts :: DBOpts` with `createSchema = True`, `confirmMigrations = MCYesUp`, in `tests/XFTPClient.hs`

plans/2026-03-25-xftp-postgres-implementation-plan.md

@@ -0,0 +1,648 @@
+# XFTP PostgreSQL Backend — Implementation Plan
+
+> **For agentic workers:** REQUIRED: Use superpowers-extended-cc:subagent-driven-development (if subagents available) or superpowers-extended-cc:executing-plans to implement this plan. Steps use checkbox (`- [ ]`) syntax for tracking.
+
+**Goal:** Add PostgreSQL backend support to xftp-server as an alternative to STM + StoreLog, with bidirectional migration.
+
+**Architecture:** Introduce `FileStoreClass` typeclass (IO-based, following `QueueStoreClass` pattern). Extract current STM store into `Store/STM.hs`, make `Server.hs` polymorphic, then add `Store/Postgres.hs` behind `server_postgres` CPP flag. `usedStorage` moves from store to `XFTPEnv` so the server manages quota tracking externally.
+
+**Tech Stack:** Haskell, postgresql-simple, STM, fourmolu, cabal with CPP flags
+
+**Design spec:** `plans/2026-03-25-xftp-postgres-backend-design.md`
+
+---
+
+## File Structure
+
+**Existing files modified:**
+- `src/Simplex/FileTransfer/Server/Store.hs` — rewritten: becomes typeclass + shared types
+- `src/Simplex/FileTransfer/Server/Env.hs` — polymorphic `XFTPEnv s`, `XFTPStoreConfig` GADT
+- `src/Simplex/FileTransfer/Server.hs` — polymorphic over `FileStoreClass s`
+- `src/Simplex/FileTransfer/Server/StoreLog.hs` — update for IO store functions
+- `src/Simplex/FileTransfer/Server/Main.hs` — INI config, dispatch, CLI commands
+- `simplexmq.cabal` — new modules
+- `tests/XFTPClient.hs` — Postgres test fixtures
+- `tests/Test.hs` — Postgres test group
+
+**New files created:**
+- `src/Simplex/FileTransfer/Server/Store/STM.hs` — `STMFileStore` (extracted from current `Store.hs`)
+- `src/Simplex/FileTransfer/Server/Store/Postgres.hs` — `PostgresFileStore` [CPP-guarded]
+- `src/Simplex/FileTransfer/Server/Store/Postgres/Config.hs` — `PostgresFileStoreCfg` [CPP-guarded]
+- `src/Simplex/FileTransfer/Server/Store/Postgres/Migrations.hs` — schema SQL [CPP-guarded]
+- `tests/CoreTests/XFTPStoreTests.hs` — Postgres store unit tests [CPP-guarded]
+
+---
+
+## Task 1: Move `usedStorage` from `FileStore` to `XFTPEnv`
+
+**Files:**
+- Modify: `src/Simplex/FileTransfer/Server/Store.hs`
+- Modify: `src/Simplex/FileTransfer/Server/Env.hs`
+- Modify: `src/Simplex/FileTransfer/Server.hs`
+
+- [ ] **Step 1: Remove `usedStorage` from `FileStore` in `Store.hs`**
+
+  1. Remove `usedStorage :: TVar Int64` field from `FileStore` record (line 47).
+  2. Remove `usedStorage <- newTVarIO 0` from `newFileStore` (line 75) and drop the field from the record construction (line 76).
+  3. In `setFilePath` (line 92-97): remove `modifyTVar' (usedStorage st) (+ fromIntegral (size fileInfo))` — keep only `writeTVar filePath (Just fPath)`. Change pattern from `\FileRec {fileInfo, filePath}` to `\FileRec {filePath}` (fileInfo is now unused — `-Wunused-matches` error).
+  4. In `deleteFile` (line 112-119): remove `modifyTVar' usedStorage $ subtract (fromIntegral $ size fileInfo)`. Change outer pattern match from `FileStore {files, recipients, usedStorage}` to `FileStore {files, recipients}`. Change inner pattern from `Just FileRec {fileInfo, recipientIds}` to `Just FileRec {recipientIds}` (`fileInfo` is now unused — `-Wunused-matches` error).
+  5. In `blockFile` (line 122-127): remove `when deleted $ modifyTVar' usedStorage $ subtract (fromIntegral $ size fileInfo)`. Change pattern match from `st@FileStore {usedStorage}` to `st`. The `deleted` parameter and `fileInfo` in the inner pattern become unused — prefix with `_` or remove from pattern to avoid `-Wunused-matches`.
+
+- [ ] **Step 2: Add `usedStorage` to `XFTPEnv` in `Env.hs`**
+
+  1. Add `usedStorage :: TVar Int64` field to `XFTPEnv` record (between `store` and `storeLog`, line 93).
+  2. In `newXFTPServerEnv` (line 112-126): replace lines 117-118:
+     ```
+     used <- countUsedStorage <$> readTVarIO (files store)
+     atomically $ writeTVar (usedStorage store) used
+     ```
+     with:
+     ```
+     usedStorage <- newTVarIO =<< countUsedStorage <$> readTVarIO (files store)
+     ```
+  3. Add `usedStorage` to the `pure XFTPEnv {..}` construction.
+
+- [ ] **Step 3: Update all `usedStorage` access sites in `Server.hs`**
+
+  1. Line 552: `us <- asks $ usedStorage . store` → `us <- asks usedStorage`.
+  2. Line 569: `us <- asks $ usedStorage . store` → `us <- asks usedStorage`.
+  3. Line 639: `usedStart <- readTVarIO $ usedStorage st` → `usedStart <- readTVarIO =<< asks usedStorage`.
+  4. Line 647: `usedEnd <- readTVarIO $ usedStorage st` → `usedEnd <- readTVarIO =<< asks usedStorage`.
+  5. Line 694: `FileStore {files, usedStorage} <- asks store` → split into `FileStore {files} <- asks store` and `usedStorage <- asks usedStorage`.
+  6. In `deleteOrBlockServerFile_` (line 620): after `void $ atomically $ storeAction st`, add usedStorage adjustment — `us <- asks usedStorage` then `atomically $ modifyTVar' us $ subtract (fromIntegral $ size fileInfo)` when file had a path (check `path` from `readTVarIO filePath` earlier in the function).
+
+- [ ] **Step 4: Build and verify**
+
+  Run: `cabal build`
+
+- [ ] **Step 5: Run existing tests**
+
+  Run: `cabal test --test-show-details=streaming --test-option=--match="/XFTP/"`
+
+- [ ] **Step 6: Format and commit**
+
+  ```bash
+  fourmolu -i src/Simplex/FileTransfer/Server/Store.hs src/Simplex/FileTransfer/Server/Env.hs src/Simplex/FileTransfer/Server.hs
+  git add src/Simplex/FileTransfer/Server/Store.hs src/Simplex/FileTransfer/Server/Env.hs src/Simplex/FileTransfer/Server.hs
+  git commit -m "refactor(xftp): move usedStorage from FileStore to XFTPEnv"
+  ```
+
+---
+
+## Task 2: Add `getUsedStorage`, `getFileCount`, `expiredFiles` functions
+
+**Files:**
+- Modify: `src/Simplex/FileTransfer/Server/Store.hs`
+- Modify: `src/Simplex/FileTransfer/Server/Env.hs`
+- Modify: `src/Simplex/FileTransfer/Server.hs`
+
+- [ ] **Step 1: Add three new functions to `Store.hs`**
+
+  1. Add to exports: `getUsedStorage`, `getFileCount`, `expiredFiles`.
+  2. Remove `expiredFilePath` from exports AND delete the function definition (dead code → `-Wunused-binds` error). Also remove `($>>=)` from import `Simplex.Messaging.Util (ifM, ($>>=))` → `Simplex.Messaging.Util (ifM)` — `$>>=` was only used by `expiredFilePath`.
+  3. Add import: `qualified Data.Map.Strict as M` (needed for `M.foldl'` in `getUsedStorage` and `M.toList` in `expiredFiles`).
+  4. Implement:
+     ```haskell
+     getUsedStorage :: FileStore -> IO Int64
+     getUsedStorage FileStore {files} =
+       M.foldl' (\acc FileRec {fileInfo = FileInfo {size}} -> acc + fromIntegral size) 0 <$> readTVarIO files
+
+     getFileCount :: FileStore -> IO Int
+     getFileCount FileStore {files} = M.size <$> readTVarIO files
+
+     expiredFiles :: FileStore -> Int64 -> Int -> IO [(SenderId, Maybe FilePath, Word32)]
+     expiredFiles FileStore {files} old _limit = do
+       fs <- readTVarIO files
+       fmap catMaybes . forM (M.toList fs) $ \(sId, FileRec {fileInfo = FileInfo {size}, filePath, createdAt = RoundedSystemTime createdAt}) ->
+         if createdAt + fileTimePrecision < old
+           then do
+             path <- readTVarIO filePath
+             pure $ Just (sId, path, size)
+           else pure Nothing
+     ```
+  5. Add imports: `Data.Maybe (catMaybes)`, `Data.Word (Word32)` (note: `qualified Data.Map.Strict as M` already added in item 3).
+
+- [ ] **Step 2: Replace `countUsedStorage` in `Env.hs`**
+
+  1. Replace `countUsedStorage <$> readTVarIO (files store)` with `getUsedStorage store` in `newXFTPServerEnv`.
+  2. Remove `countUsedStorage` function definition and its export.
+  3. Remove `qualified Data.Map.Strict as M` import if no longer used.
+
+- [ ] **Step 3: Update `restoreServerStats` in `Server.hs` to use `getFileCount`**
+
+  In `restoreServerStats` (line 694-696): replace `FileStore {files} <- asks store` and `_filesCount <- M.size <$> readTVarIO files` with `st <- asks store` and `_filesCount <- liftIO $ getFileCount st` (eliminates the `FileStore` pattern match — `files` binding no longer needed).
+
+- [ ] **Step 4: Replace `expireServerFiles` iteration in `Server.hs`**
+
+  1. Replace the body of `expireServerFiles` (lines 636-660). Remove `files' <- readTVarIO (files st)` and the `forM_ (M.keys files')` loop.
+  2. New body: call `expiredFiles st old 10000` in a loop. For each `(sId, filePath_, fileSize)` in returned list: apply `itemDelay`, remove disk file if present, call `atomically $ deleteFile st sId`, adjust `usedStorage` TVar by `fileSize`, increment `filesExpired` stat. Loop until `expiredFiles` returns `[]`.
+  3. Remove `Data.Map.Strict` import from Server.hs if no longer needed (was used for `M.size` and `M.keys` — now replaced by `getFileCount` and `expiredFiles`).
+
+- [ ] **Step 5: Build and verify**
+
+  Run: `cabal build`
+
+- [ ] **Step 6: Run existing tests**
+
+  Run: `cabal test --test-show-details=streaming --test-option=--match="/XFTP/"`
+
+- [ ] **Step 7: Format and commit**
+
+  ```bash
+  fourmolu -i src/Simplex/FileTransfer/Server/Store.hs src/Simplex/FileTransfer/Server/Env.hs src/Simplex/FileTransfer/Server.hs
+  git add src/Simplex/FileTransfer/Server/Store.hs src/Simplex/FileTransfer/Server/Env.hs src/Simplex/FileTransfer/Server.hs
+  git commit -m "refactor(xftp): add getUsedStorage, getFileCount, expiredFiles store functions"
+  ```
+
+---
+
+## Task 3: Change `Store.hs` functions from STM to IO
+
+**Files:**
+- Modify: `src/Simplex/FileTransfer/Server/Store.hs`
+- Modify: `src/Simplex/FileTransfer/Server.hs`
+- Modify: `src/Simplex/FileTransfer/Server/StoreLog.hs`
+
+- [ ] **Step 1: Change all Store.hs function signatures from STM to IO**
+
+  For each of: `addFile`, `setFilePath`, `addRecipient`, `getFile`, `deleteFile`, `blockFile`, `deleteRecipient`, `ackFile`:
+  1. Change return type from `STM (Either XFTPErrorType ...)` to `IO (Either XFTPErrorType ...)` (or `STM ()` to `IO ()` for `deleteRecipient`).
+  2. Wrap the function body in `atomically $ do ...`.
+  3. Keep `withFile` and `newFileRec` as internal STM helpers (called inside the `atomically` blocks).
+
+- [ ] **Step 2: Update Server.hs call sites — remove `atomically` wrappers**
+
+  1. Line 563 (`receiveServerFile`): change `atomically $ writeTVar filePath (Just fPath)` → add `st <- asks store` then `void $ liftIO $ setFilePath st senderId fPath` (design call site #1 — `store` is not in scope in `receiveServerFile`'s `receive` helper, so bind via `asks`; `void` avoids `-Wunused-do-bind` warning on the `Either` result).
+  2. Line 453 (`verifyXFTPTransmission`): split `atomically $ verify =<< getFile st party fId` into: `liftIO (getFile st party fId)` (IO→M lift), then pattern match on result, use `readTVarIO (fileStatus fr)` instead of `readTVar`.
+  3. Lines 371, 377 (control port `CPDelete`/`CPBlock`): change `ExceptT $ atomically $ getFile fs SFRecipient fileId` → `ExceptT $ liftIO $ getFile fs SFRecipient fileId` (inside `unliftIO u $ do` block which runs in M monad — `liftIO` required to lift IO into M).
+  4. Line 508 (`addFile` in `createFile`): the `ExceptT $ addFile st sId file ts EntityActive` — `addFile` is now IO, `ExceptT` wraps IO directly. Remove any `atomically`.
+  5. Line 514 (`addRecipient`): same — `ExceptT . addRecipient st sId` works directly in IO.
+  6. Line 516 (`retryAdd`): change parameter type from `(XFTPFileId -> STM (Either XFTPErrorType a))` to `(XFTPFileId -> IO (Either XFTPErrorType a))`. Line 520: change `atomically (add fId)` to `liftIO (add fId)`.
+  7. Line 605 (`ackFileReception`): change `atomically $ deleteRecipient st rId fr` to `liftIO $ deleteRecipient st rId fr`.
+  8. Line 620 (`deleteOrBlockServerFile_`): change third parameter type from `(FileStore -> STM (Either XFTPErrorType ()))` to `(FileStore -> IO (Either XFTPErrorType ()))`. Line 626: change `void $ atomically $ storeAction st` to `void $ liftIO $ storeAction st`.
+  9. `expireServerFiles` `delete` helper: change `atomically $ deleteFile st sId` to `liftIO $ deleteFile st sId` (deleteFile is now IO; `liftIO` required because the helper runs in M monad, not IO).
+
+- [ ] **Step 3: Update `StoreLog.hs` — remove `atomically` from replay**
+
+  In `readFileStore` (line 93), function `addToStore`:
+  1. Change `atomically (addToStore lr)` to `addToStore lr` — store functions are now IO.
+  2. The `addToStore` body calls `addFile`, `setFilePath`, `deleteFile`, `blockFile`, `ackFile` — all IO now, no `atomically` needed.
+  3. For `AddRecipients`: `runExceptT $ mapM_ (ExceptT . addRecipient st sId) rcps` — `addRecipient` returns `IO (Either ...)`, so `ExceptT . addRecipient st sId` works directly.
+
+- [ ] **Step 4: Build and verify**
+
+  Run: `cabal build`
+
+- [ ] **Step 5: Run existing tests**
+
+  Run: `cabal test --test-show-details=streaming --test-option=--match="/XFTP/"`
+
+- [ ] **Step 6: Format and commit**
+
+  ```bash
+  fourmolu -i src/Simplex/FileTransfer/Server/Store.hs src/Simplex/FileTransfer/Server.hs src/Simplex/FileTransfer/Server/StoreLog.hs
+  git add src/Simplex/FileTransfer/Server/Store.hs src/Simplex/FileTransfer/Server.hs src/Simplex/FileTransfer/Server/StoreLog.hs
+  git commit -m "refactor(xftp): change file store operations from STM to IO"
+  ```
+
+---
+
+## Task 4: Extract `FileStoreClass` typeclass, move STM impl to `Store/STM.hs`
+
+**Files:**
+- Rewrite: `src/Simplex/FileTransfer/Server/Store.hs`
+- Create: `src/Simplex/FileTransfer/Server/Store/STM.hs`
+- Modify: `src/Simplex/FileTransfer/Server/StoreLog.hs`
+- Modify: `src/Simplex/FileTransfer/Server/Env.hs`
+- Modify: `src/Simplex/FileTransfer/Server.hs`
+- Modify: `simplexmq.cabal`
+
+- [ ] **Step 1: Create `Store/STM.hs` — move all implementation code**
+
+  1. Create directory `src/Simplex/FileTransfer/Server/Store/`.
+  2. Create `src/Simplex/FileTransfer/Server/Store/STM.hs`.
+  3. Move from `Store.hs`: `FileStore` data type (rename to `STMFileStore`), all function implementations, internal helpers (`withFile`, `newFileRec`), all STM-specific imports.
+  4. Rename all `FileStore` references to `STMFileStore` in the new file.
+  5. Module declaration: `module Simplex.FileTransfer.Server.Store.STM` exporting only `STMFileStore (..)` — do NOT export standalone functions (`addFile`, `setFilePath`, etc.) to avoid name collisions with the typeclass methods from `Store.hs`.
+
+- [ ] **Step 2: Rewrite `Store.hs` as the typeclass module**
+
+  1. Add `{-# LANGUAGE TypeFamilies #-}` pragma to `Store.hs` (required for `type FileStoreConfig s` associated type).
+  2. Keep in `Store.hs`: `FileRec (..)`, `FileRecipient (..)`, `RoundedFileTime`, `fileTimePrecision` definitions and their `StrEncoding` instance.
+  3. Add `FileStoreClass` typeclass:
+     ```haskell
+     class FileStoreClass s where
+       type FileStoreConfig s
+
+       -- Lifecycle
+       newFileStore :: FileStoreConfig s -> IO s
+       closeFileStore :: s -> IO ()
+
+       -- File operations
+       addFile :: s -> SenderId -> FileInfo -> RoundedFileTime -> ServerEntityStatus -> IO (Either XFTPErrorType ())
+       setFilePath :: s -> SenderId -> FilePath -> IO (Either XFTPErrorType ())
+       addRecipient :: s -> SenderId -> FileRecipient -> IO (Either XFTPErrorType ())
+       getFile :: s -> SFileParty p -> XFTPFileId -> IO (Either XFTPErrorType (FileRec, C.APublicAuthKey))
+       deleteFile :: s -> SenderId -> IO (Either XFTPErrorType ())
+       blockFile :: s -> SenderId -> BlockingInfo -> Bool -> IO (Either XFTPErrorType ())
+       deleteRecipient :: s -> RecipientId -> FileRec -> IO ()
+       ackFile :: s -> RecipientId -> IO (Either XFTPErrorType ())
+
+       -- Expiration
+       expiredFiles :: s -> Int64 -> Int -> IO [(SenderId, Maybe FilePath, Word32)]
+
+       -- Stats
+       getUsedStorage :: s -> IO Int64
+       getFileCount :: s -> IO Int
+     ```
+  4. Do NOT re-export from `Store/STM.hs` — this would create a circular module dependency (Store.hs imports Store/STM.hs, Store/STM.hs imports Store.hs). Consumers must import `Store.STM` directly where they need `STMFileStore`.
+  5. Remove all STM-specific imports that are no longer needed.
+
+- [ ] **Step 3: Add `FileStoreClass` instance in `Store/STM.hs`**
+
+  1. Import `FileStoreClass` from `Simplex.FileTransfer.Server.Store`.
+  2. Inline all implementations directly in the instance body (do NOT delegate to standalone functions — the standalone names collide with typeclass method names, causing ambiguous occurrences for importers):
+     ```haskell
+     instance FileStoreClass STMFileStore where
+       type FileStoreConfig STMFileStore = ()
+       newFileStore () = do
+         files <- TM.emptyIO
+         recipients <- TM.emptyIO
+         pure STMFileStore {files, recipients}
+       closeFileStore _ = pure ()
+       addFile st sId fileInfo createdAt status = atomically $ ...
+       setFilePath st sId fPath = atomically $ ...
+       -- ... (each method's body is the existing function body, inlined)
+     ```
+  3. Remove the standalone top-level function definitions — they are now instance methods. Keep only `withFile` and `newFileRec` as internal helpers used by the instance methods.
+
+- [ ] **Step 4: Update importers**
+
+  1. `Env.hs`: add `import Simplex.FileTransfer.Server.Store.STM (STMFileStore (..))`. Change `FileStore` → `STMFileStore` in `XFTPEnv` type and `newXFTPServerEnv`. Change `store <- newFileStore` to `store <- newFileStore ()` (typeclass method now takes `FileStoreConfig STMFileStore` which is `()`). Keep `import Simplex.FileTransfer.Server.Store` for `FileRec`, `FileRecipient`, `FileStoreClass`, etc.
+  2. `Server.hs`: add `import Simplex.FileTransfer.Server.Store.STM`. Change `FileStore` → `STMFileStore` in any explicit type annotations. Import `FileStoreClass` from `Simplex.FileTransfer.Server.Store`.
+  3. `StoreLog.hs`: add `import Simplex.FileTransfer.Server.Store.STM` to access concrete `STMFileStore` type and store functions used during log replay. Change `FileStore` → `STMFileStore` in `readWriteFileStore` and `writeFileStore` parameter types.
+
+- [ ] **Step 5: Update cabal file**
+
+  Add `Simplex.FileTransfer.Server.Store.STM` to `exposed-modules` in the `!flag(client_library)` section, alongside existing XFTP server modules.
+
+- [ ] **Step 6: Build and verify**
+
+  Run: `cabal build`
+
+- [ ] **Step 7: Run existing tests**
+
+  Run: `cabal test --test-show-details=streaming --test-option=--match="/XFTP/"`
+
+- [ ] **Step 8: Format and commit**
+
+  ```bash
+  fourmolu -i src/Simplex/FileTransfer/Server/Store.hs src/Simplex/FileTransfer/Server/Store/STM.hs src/Simplex/FileTransfer/Server/Env.hs src/Simplex/FileTransfer/Server.hs src/Simplex/FileTransfer/Server/StoreLog.hs
+  git add src/Simplex/FileTransfer/Server/Store.hs src/Simplex/FileTransfer/Server/Store/STM.hs src/Simplex/FileTransfer/Server/Env.hs src/Simplex/FileTransfer/Server.hs src/Simplex/FileTransfer/Server/StoreLog.hs simplexmq.cabal
+  git commit -m "refactor(xftp): extract FileStoreClass typeclass, move STM impl to Store.STM"
+  ```
+
+---
+
+## Task 5: Make `XFTPEnv` and `Server.hs` polymorphic over `FileStoreClass`
+
+**Files:**
+- Modify: `src/Simplex/FileTransfer/Server/Env.hs`
+- Modify: `src/Simplex/FileTransfer/Server.hs`
+- Modify: `src/Simplex/FileTransfer/Server/Main.hs`
+- Modify: `tests/XFTPClient.hs` (if it calls `runXFTPServerBlocking` directly)
+
+- [ ] **Step 1: Make `XFTPEnv` polymorphic in `Env.hs`**
+
+  1. Add `XFTPStoreConfig` GADT: `data XFTPStoreConfig s where XSCMemory :: Maybe FilePath -> XFTPStoreConfig STMFileStore`.
+  2. Change `data XFTPEnv` to `data XFTPEnv s` — field `store :: FileStore` becomes `store :: s`.
+  3. Change `newXFTPServerEnv :: XFTPServerConfig -> IO XFTPEnv` to `newXFTPServerEnv :: FileStoreClass s => XFTPStoreConfig s -> XFTPServerConfig -> IO (XFTPEnv s)`.
+  4. Pattern match on `XSCMemory storeLogPath` in `newXFTPServerEnv` body. Create store via `newFileStore ()`, storeLog via `mapM (`readWriteFileStore` st) storeLogPath`.
+
+- [ ] **Step 2: Make `Server.hs` polymorphic**
+
+  1. Change `type M a = ReaderT XFTPEnv IO a` to `type M s a = ReaderT (XFTPEnv s) IO a`.
+  2. Add `FileStoreClass s =>` constraint to all functions using `M s a`. Use `forall s.` in signatures of functions that have `where`-block bindings with `M s` type annotations — `ScopedTypeVariables` requires explicit `forall` to bring `s` into scope for inner type signatures (matching SMP's `smpServer :: forall s. MsgStoreClass s => ...` pattern). Full list: `xftpServer`, `processRequest`, `verifyXFTPTransmission`, `processXFTPRequest` and all its `where`-bound functions (`createFile`, `addRecipients`, `receiveServerFile`, `sendServerFile`, `deleteServerFile`, `ackFileReception`, `retryAdd`, `addFileRetry`, `addRecipientRetry`), `deleteServerFile_`, `blockServerFile`, `deleteOrBlockServerFile_`, `expireServerFiles`, `randomId`, `getFileId`, `withFileLog`, `incFileStat`, `saveServerStats`, `restoreServerStats`, `randomDelay` (inside `#ifdef slow_servers` CPP block). Also update `encodeXftp` (line 236) and `runCPClient` (line 339) which use explicit `ReaderT XFTPEnv IO` instead of the `M` alias — change to `ReaderT (XFTPEnv s) IO`.
+  3. Change `runXFTPServerBlocking` and `runXFTPServer` to take `XFTPStoreConfig s` parameter.
+  4. Add `closeFileStore store` call to the server shutdown path (in the `finally` block or `stopServer` equivalent — after saving stats, before logging "Server stopped"). This ensures Postgres connection pool and `dbStoreLog` are properly closed. For STM this is a no-op.
+
+- [ ] **Step 3: Update `Main.hs` dispatch**
+
+  1. In `runServer`: construct `XSCMemory (enableStoreLog $> storeLogFilePath)`.
+  2. Add dispatch function that calls the updated `runXFTPServer` (which creates `started` internally):
+     ```haskell
+     run :: FileStoreClass s => XFTPStoreConfig s -> IO ()
+     run storeCfg = runXFTPServer storeCfg serverConfig
+     ```
+  3. Call `run` with the `XSCMemory` config.
+
+- [ ] **Step 4: Update test helper if needed**
+
+  If `tests/XFTPClient.hs` calls `runXFTPServerBlocking` directly, update the call to pass an `XSCMemory` config. Check the `withXFTPServer` / `serverBracket` helper.
+
+- [ ] **Step 5: Build and verify**
+
+  Run: `cabal build && cabal build test:simplexmq-test`
+
+- [ ] **Step 6: Run existing tests**
+
+  Run: `cabal test --test-show-details=streaming --test-option=--match="/XFTP/"`
+
+- [ ] **Step 7: Format and commit**
+
+  ```bash
+  fourmolu -i src/Simplex/FileTransfer/Server/Env.hs src/Simplex/FileTransfer/Server.hs src/Simplex/FileTransfer/Server/Main.hs
+  git add src/Simplex/FileTransfer/Server/Env.hs src/Simplex/FileTransfer/Server.hs src/Simplex/FileTransfer/Server/Main.hs tests/XFTPClient.hs simplexmq.cabal
+  git commit -m "refactor(xftp): make XFTPEnv and server polymorphic over FileStoreClass"
+  ```
+
+---
+
+## Task 6: Add Postgres config, migrations, and store skeleton
+
+**Files:**
+- Create: `src/Simplex/FileTransfer/Server/Store/Postgres/Config.hs`
+- Create: `src/Simplex/FileTransfer/Server/Store/Postgres/Migrations.hs`
+- Create: `src/Simplex/FileTransfer/Server/Store/Postgres.hs`
+- Modify: `src/Simplex/FileTransfer/Server/Env.hs`
+- Modify: `simplexmq.cabal`
+
+- [ ] **Step 1: Create `Store/Postgres/Config.hs`**
+
+  ```haskell
+  module Simplex.FileTransfer.Server.Store.Postgres.Config
+    ( PostgresFileStoreCfg (..),
+      defaultXFTPDBOpts,
+    )
+  where
+
+  import Simplex.Messaging.Agent.Store.Postgres.Options (DBOpts (..))
+  import Simplex.Messaging.Agent.Store.Shared (MigrationConfirmation)
+
+  data PostgresFileStoreCfg = PostgresFileStoreCfg
+    { dbOpts :: DBOpts,
+      dbStoreLogPath :: Maybe FilePath,
+      confirmMigrations :: MigrationConfirmation
+    }
+
+  defaultXFTPDBOpts :: DBOpts
+  defaultXFTPDBOpts =
+    DBOpts
+      { connstr = "postgresql://xftp@/xftp_server_store",
+        schema = "xftp_server",
+        poolSize = 10,
+        createSchema = False
+      }
+  ```
+
+- [ ] **Step 2: Create `Store/Postgres/Migrations.hs`**
+
+  Full migration module with `xftpServerMigrations :: [Migration]` and `m20260325_initial` containing CREATE TABLE SQL for `files` and `recipients` tables plus indexes. Follow SMP's `QueueStore/Postgres/Migrations.hs` pattern exactly: tuple list → `sortOn name . map migration`.
+
+- [ ] **Step 3: Create `Store/Postgres.hs` with stub instance**
+
+  1. Define `PostgresFileStore` with `dbStore :: DBStore` and `dbStoreLog :: Maybe (StoreLog 'WriteMode)`.
+  2. `instance FileStoreClass PostgresFileStore` with `error "not implemented"` for all methods except `newFileStore` (calls `createDBStore` + opens `dbStoreLog`) and `closeFileStore` (closes both). `type FileStoreConfig PostgresFileStore = PostgresFileStoreCfg`.
+  3. Add `withDB`, `handleDuplicate`, `assertUpdated`, `withLog` helpers.
+
+- [ ] **Step 4: Add `XSCDatabase` GADT constructor in `Env.hs` (CPP-guarded)**
+
+  ```haskell
+  #if defined(dbServerPostgres)
+  import Simplex.FileTransfer.Server.Store.Postgres (PostgresFileStore)
+  import Simplex.FileTransfer.Server.Store.Postgres.Config (PostgresFileStoreCfg)
+  #endif
+
+  data XFTPStoreConfig s where
+    XSCMemory :: Maybe FilePath -> XFTPStoreConfig STMFileStore
+  #if defined(dbServerPostgres)
+    XSCDatabase :: PostgresFileStoreCfg -> XFTPStoreConfig PostgresFileStore
+  #endif
+  ```
+
+- [ ] **Step 5: Update cabal**
+
+  Add to existing `if flag(server_postgres)` block:
+  ```
+  Simplex.FileTransfer.Server.Store.Postgres
+  Simplex.FileTransfer.Server.Store.Postgres.Config
+  Simplex.FileTransfer.Server.Store.Postgres.Migrations
+  ```
+
+- [ ] **Step 6: Build both ways**
+
+  Run: `cabal build && cabal build -fserver_postgres`
+
+- [ ] **Step 7: Format and commit**
+
+  ```bash
+  fourmolu -i src/Simplex/FileTransfer/Server/Store/Postgres.hs src/Simplex/FileTransfer/Server/Store/Postgres/Config.hs src/Simplex/FileTransfer/Server/Env.hs
+  git add src/Simplex/FileTransfer/Server/Store/Postgres.hs src/Simplex/FileTransfer/Server/Store/Postgres/Config.hs src/Simplex/FileTransfer/Server/Store/Postgres/Migrations.hs src/Simplex/FileTransfer/Server/Env.hs simplexmq.cabal
+  git commit -m "feat(xftp): add PostgreSQL store skeleton with schema migration"
+  ```
+
+---
+
+## Task 7: Implement `PostgresFileStore` operations
+
+**Files:**
+- Modify: `src/Simplex/FileTransfer/Server/Store/Postgres.hs`
+
+- [ ] **Step 1: Implement `addFile`**
+
+  `INSERT INTO files (sender_id, file_size, file_digest, sender_key, file_path, created_at, status) VALUES (?,?,?,?,NULL,?,?)`. Catch unique violation with `handleDuplicate` → `DUPLICATE_`. Call `withLog "addFile"` after.
+
+- [ ] **Step 2: Implement `getFile`**
+
+  For `SFSender`: `SELECT ... FROM files WHERE sender_id = ?`. Construct `FileRec` with `newTVarIO` per TVar field. `recipientIds = S.empty`.
+  For `SFRecipient`: `SELECT f.*, r.recipient_key FROM recipients r JOIN files f ON r.sender_id = f.sender_id WHERE r.recipient_id = ?`.
+
+- [ ] **Step 3: Implement `setFilePath`**
+
+  `UPDATE files SET file_path = ? WHERE sender_id = ? AND file_path IS NULL`. Use `assertUpdated`. Call `withLog "setFilePath"`.
+
+- [ ] **Step 4: Implement `addRecipient`**
+
+  `INSERT INTO recipients (recipient_id, sender_id, recipient_key) VALUES (?,?,?)`. `handleDuplicate` → `DUPLICATE_`. Call `withLog "addRecipient"`.
+
+- [ ] **Step 5: Implement `deleteFile`, `blockFile`**
+
+  `deleteFile`: `DELETE FROM files WHERE sender_id = ?` (CASCADE). `withLog "deleteFile"`.
+  `blockFile`: `UPDATE files SET status = ? WHERE sender_id = ?`. `assertUpdated`. `withLog "blockFile"`.
+
+- [ ] **Step 6: Implement `deleteRecipient`, `ackFile`**
+
+  `deleteRecipient`: `DELETE FROM recipients WHERE recipient_id = ?`. `withLog "deleteRecipient"`.
+  `ackFile`: same + return `Left AUTH` if 0 rows.
+
+- [ ] **Step 7: Implement `expiredFiles`, `getUsedStorage`, `getFileCount`**
+
+  `expiredFiles`: `SELECT sender_id, file_path, file_size FROM files WHERE created_at + ? < ? LIMIT ?`.
+  `getUsedStorage`: `SELECT COALESCE(SUM(file_size), 0) FROM files`.
+  `getFileCount`: `SELECT COUNT(*) FROM files`.
+
+- [ ] **Step 8: Add `ToField`/`FromField` instances**
+
+  For `RoundedFileTime` (Int64 wrapper), `ServerEntityStatus` (Text via StrEncoding), `C.APublicAuthKey` (Binary via `encodePubKey`/`decodePubKey`). Check SMP's `QueueStore/Postgres.hs` for existing instances to import.
+
+- [ ] **Step 9: Wrap mutation operations in `uninterruptibleMask_`**
+
+  Operations that combine a DB write with a TVar update (e.g., `getFile` constructs `FileRec` with `newTVarIO`) must be wrapped in `E.uninterruptibleMask_` to prevent async exceptions from leaving inconsistent state. Follow SMP's `addQueue_`, `deleteStoreQueue` pattern.
+
+- [ ] **Step 10: Build**
+
+  Run: `cabal build -fserver_postgres`
+
+- [ ] **Step 11: Format and commit**
+
+  ```bash
+  fourmolu -i src/Simplex/FileTransfer/Server/Store/Postgres.hs
+  git add src/Simplex/FileTransfer/Server/Store/Postgres.hs
+  git commit -m "feat(xftp): implement PostgresFileStore operations"
+  ```
+
+---
+
+## Task 8: Add INI config, Main.hs dispatch, startup validation
+
+**Files:**
+- Modify: `src/Simplex/FileTransfer/Server/Main.hs`
+- Modify: `src/Simplex/FileTransfer/Server/Env.hs`
+
+- [ ] **Step 1: Update `iniFileContent` in `Main.hs`**
+
+  Add to `[STORE_LOG]` section: `store_files: memory`, commented-out `db_connection`, `db_schema`, `db_pool_size`, `db_store_log` keys. Follow SMP's `optDisabled'` pattern for commented defaults.
+
+- [ ] **Step 2: Add `StartOptions` and `--confirm-migrations` flag**
+
+  ```haskell
+  data StartOptions = StartOptions
+    { confirmMigrations :: MigrationConfirmation
+    }
+  ```
+  Add to `Start` command parser with default `MCConsole`. Thread through to `runServer`.
+
+- [ ] **Step 3: Add store_files INI parsing and CPP-guarded Postgres dispatch**
+
+  In `runServer`: read `store_files` from INI (`fromRight "memory" $ lookupValue "STORE_LOG" "store_files" ini`). Add `"database"` branch (CPP-guarded) that constructs `PostgresFileStoreCfg` using `iniDBOptions ini defaultXFTPDBOpts` and `enableDbStoreLog'` pattern. Non-postgres build: `exitError`.
+
+- [ ] **Step 4: Add `XSCDatabase` branch in `newXFTPServerEnv` (`Env.hs`)**
+
+  CPP-guarded pattern match on `XSCDatabase dbCfg`: `newFileStore dbCfg`, `storeLog = Nothing`.
+
+- [ ] **Step 5: Add startup config validation**
+
+  Add `checkFileStoreMode` (CPP-guarded) before `run`: validate conflicting storeLog file + database mode, missing schema, etc. per design doc.
+
+- [ ] **Step 6: Build both ways**
+
+  Run: `cabal build && cabal build -fserver_postgres`
+
+- [ ] **Step 7: Format and commit**
+
+  ```bash
+  fourmolu -i src/Simplex/FileTransfer/Server/Main.hs src/Simplex/FileTransfer/Server/Env.hs
+  git add src/Simplex/FileTransfer/Server/Main.hs src/Simplex/FileTransfer/Server/Env.hs
+  git commit -m "feat(xftp): add PostgreSQL INI config, store dispatch, startup validation"
+  ```
+
+---
+
+## Task 9: Add database import/export CLI commands
+
+**Files:**
+- Modify: `src/Simplex/FileTransfer/Server/Main.hs`
+
+- [ ] **Step 1: Add `Database` CLI command (CPP-guarded)**
+
+  Add `Database StoreCmd DBOpts` constructor to `CliCommand`. Add `database` subcommand parser with `import`/`export` subcommands + `dbOptsP defaultXFTPDBOpts`.
+
+- [ ] **Step 2: Implement `importFileStoreToDatabase`**
+
+  1. `confirmOrExit` with database details.
+  2. Create temporary `STMFileStore`, replay StoreLog via `readWriteFileStore`.
+  3. Create `PostgresFileStore` with `createSchema = True`, `confirmMigrations = MCYesUp`.
+  4. Batch-insert files using PostgreSQL COPY protocol. Progress every 10k.
+  5. Batch-insert recipients using COPY protocol.
+  6. Verify counts: `SELECT COUNT(*)` — warn on mismatch.
+  7. Rename StoreLog to `.bak`.
+  8. Report counts.
+
+- [ ] **Step 3: Implement `exportDatabaseToStoreLog`**
+
+  1. `confirmOrExit`. Fail if output file exists.
+  2. Create `PostgresFileStore` from config.
+  3. Open StoreLog for writing.
+  4. Fold over file records: write `AddFile` (with status), `AddRecipients`, `PutFile` per file.
+  5. Close StoreLog, report counts.
+
+- [ ] **Step 4: Build**
+
+  Run: `cabal build -fserver_postgres`
+
+- [ ] **Step 5: Format and commit**
+
+  ```bash
+  fourmolu -i src/Simplex/FileTransfer/Server/Main.hs
+  git add src/Simplex/FileTransfer/Server/Main.hs
+  git commit -m "feat(xftp): add database import/export CLI commands"
+  ```
+
+---
+
+## Task 10: Add Postgres tests
+
+**Files:**
+- Modify: `tests/XFTPClient.hs`
+- Modify: `tests/Test.hs`
+- Create: `tests/CoreTests/XFTPStoreTests.hs`
+
+- [ ] **Step 1: Add test fixtures in `tests/XFTPClient.hs`**
+
+  ```haskell
+  testXFTPStoreDBOpts :: DBOpts
+  testXFTPStoreDBOpts =
+    DBOpts
+      { connstr = "postgresql://test_xftp_server_user@/test_xftp_server_db",
+        schema = "xftp_server_test",
+        poolSize = 10,
+        createSchema = True
+      }
+  ```
+  Add `testXFTPDBConnectInfo :: ConnectInfo` matching the connection string.
+
+- [ ] **Step 2: Add Postgres server test group in `tests/Test.hs`**
+
+  CPP-guarded block that runs existing `xftpServerTests` with Postgres store config, wrapped in `postgressBracket testXFTPDBConnectInfo`. Parameterize `withXFTPServer` to accept store config if needed.
+
+- [ ] **Step 3: Create `tests/CoreTests/XFTPStoreTests.hs` — unit tests**
+
+  Test `PostgresFileStore` operations directly:
+  - `addFile` + `getFile SFSender` round-trip.
+  - `addFile` duplicate → `DUPLICATE_`.
+  - `getFile` nonexistent → `AUTH`.
+  - `setFilePath` + verify `WHERE file_path IS NULL` guard.
+  - `addRecipient` + `getFile SFRecipient` round-trip.
+  - `deleteFile` cascades recipients.
+  - `blockFile` + verify status.
+  - `expiredFiles` batch semantics.
+  - `getUsedStorage`, `getFileCount` correctness.
+
+- [ ] **Step 4: Add migration round-trip test**
+
+  Create `STMFileStore` with test data (files + recipients + blocked status) → export to StoreLog → import to Postgres → export back → compare StoreLog files byte-for-byte.
+
+- [ ] **Step 5: Build and run tests**
+
+  ```bash
+  cabal build -fserver_postgres test:simplexmq-test
+  cabal test --test-show-details=streaming --test-option=--match="/XFTP/" -fserver_postgres
+  ```
+
+- [ ] **Step 6: Format and commit**
+
+  ```bash
+  fourmolu -i tests/CoreTests/XFTPStoreTests.hs tests/XFTPClient.hs
+  git add tests/CoreTests/XFTPStoreTests.hs tests/XFTPClient.hs tests/Test.hs
+  git commit -m "test(xftp): add PostgreSQL backend tests"
+  ```

protocol/overview-tjr.md

@@ -1,4 +1,4 @@
-Revision 2, 2024-06-22
+Revision 4, 2026-03-09
 
 Evgeny Poberezkin
 
@@ -8,16 +8,17 @@ Evgeny Poberezkin
 
 - [Introduction](#introduction)
   - [What is SimpleX](#what-is-simplex)
+  - [Network model](#network-model)
+  - [Applications](#applications)
   - [SimpleX objectives](#simplex-objectives)
   - [In Comparison](#in-comparison)
 - [Technical Details](#technical-details)
-  - [Trust in Servers](#trust-in-servers)
-  - [Client -> Server Communication](#client---server-communication)
+  - [Trust in Routers](#trust-in-routers)
+  - [Client -> Router Communication](#client---router-communication)
   - [2-hop Onion Message Routing](#2-hop-onion-message-routing)
   - [SimpleX Messaging Protocol](#simplex-messaging-protocol)
   - [SimpleX Agents](#simplex-agents)
-  - [Encryption Primitives Used](#encryption-primitives-used)
-- [Threat model](#threat-model)
+- [Security](#security)
 - [Acknowledgements](#acknowledgements)
 
 
@@ -27,27 +28,27 @@ Evgeny Poberezkin
 
 SimpleX as a whole is a platform upon which applications can be built. [SimpleX Chat](https://github.com/simplex-chat/simplex-chat) is one such application that also serves as an example and reference application.
 
- - [SimpleX Messaging Protocol](./simplex-messaging.md) (SMP) is a protocol to send messages in one direction to a recipient, relying on a server in-between. The messages are delivered via uni-directional queues created by recipients.
- 
- - SMP protocol allows to send message via a SMP server playing proxy role using 2-hop onion routing (referred to as "private routing" in messaging clients) to protect transport information of the sender (IP address and session) from the server chosen (and possibly controlled) by the recipient.
+ - [SimpleX Messaging Protocol](./simplex-messaging.md) (SMP) is a protocol to send messages in one direction to a recipient, relying on a router in-between. The messages are delivered via uni-directional queues created by recipients.
+
+ - SMP protocol allows to send message via a SMP router playing proxy role using 2-hop onion routing (referred to as "private routing" in messaging clients) to protect transport information of the sender (IP address and session) from the router chosen (and possibly controlled) by the recipient.
 
  - SMP runs over a transport protocol (shown below as TLS) that provides integrity, server authentication, confidentiality, and transport channel binding.
 
- - A SimpleX Server is one of those servers.
+ - A SimpleX router is one of those routers.
 
- - The SimpleX Network is the term used for the collective of SimpleX Servers that facilitate SMP.
+ - The SimpleX Network is the term used for the collective of SimpleX routers that facilitate SMP.
 
- - SimpleX Client libraries speak SMP to SimpleX Servers and provide a low-level API not generally intended to be used by applications.
+ - SimpleX Client libraries speak SMP to SimpleX routers and provide a low-level API not generally intended to be used by applications.
 
  - SimpleX Agents interface with SimpleX Clients to provide a more high-level API intended to be used by applications. Typically they are embedded as libraries, but can also be abstracted into local services.
 
  - SimpleX Agents communicate with other agents inside e2e encrypted envelopes provided by SMP protocol - the syntax and semantics of the messages exchanged by the agent are defined by [SMP agent protocol](./agent-protocol.md)
 
 
-*Diagram showing the SimpleX Chat app, with logical layers of the chat application interfacing with a SimpleX Agent library, which in turn interfaces with a SimpleX Client library. The Client library in turn speaks the Messaging Protocol to a SimpleX Server.*
+*Diagram showing the SimpleX Chat app, with logical layers of the chat application interfacing with a SimpleX Agent library, which in turn interfaces with a SimpleX Client library. The Client library in turn speaks the Messaging Protocol to a SimpleX router.*
 
 ```
-  User's Computer                 Internet                    Third-Party Server
+  User's Computer                 Internet                    Third-Party Router
 ------------------     |   ----------------------     |   -------------------------
                        |                              |
    SimpleX Chat        |                              |
@@ -57,11 +58,43 @@ SimpleX as a whole is a platform upon which applications can be built. [SimpleX
 +----------------+     |                              |
 |  SimpleX Agent |     |                              |
 +----------------+    -------------- TLS ----------------    +----------------+
-| SimpleX Client | ------ SimpleX Messaging Protocol ------> | SimpleX Server |
+| SimpleX Client | ------ SimpleX Messaging Protocol ------> | SimpleX router |
 +----------------+    -----------------------------------    +----------------+
                        |                              |
 ```
 
+#### Network model
+
+SimpleX is a general-purpose packet routing network built on top of the Internet. Network endpoints — end-user devices, automated services, AI-enabled applications, IoT devices — exchange data packets through SimpleX network nodes (SMP routers), which accept, buffer, and deliver packets. Each router operates independently and can be operated by any party on standard computing hardware.
+
+SimpleX routers use resource-based addressing: each address identifies a resource on a router, similar to how the World Wide Web addresses resources via URLs. Internet routers, by comparison, use endpoint-based addressing, where IP addresses identify destination devices. Because of this design, SimpleX network participants do not need globally unique addresses to communicate.
+
+SimpleX network has two resource-based addressing schemes:
+
+- *Messaging queues* ([SMP](./simplex-messaging.md)). A queue is a unidirectional, ordered sequence of fixed-size data packets (16,384 bytes each). Each queue has a resource address on a specific router, gated by cryptographic credentials that separately authorize sending and receiving.
+
+- *Data packets* ([XFTP](./xftp.md)). A data packet is an individually addressed block in one of the standard sizes. Each packet has a unique resource address on a specific router, gated by cryptographic credentials. Data packet addressing is more efficient for delivery of larger payloads than queues.
+
+Packet delivery follows a two-router path. The sending endpoint submits a packet to a first router, which forwards it to a second router, where the receiving endpoint retrieves it. The sending endpoint's IP address is known only to the first router; the receiving endpoint's IP address is known only to the second router. See [2-hop Onion Message Routing](#2-hop-onion-message-routing) for details.
+
+Routers buffer packets between submission and retrieval — from seconds to days, enabling asynchronous delivery when endpoints are online at different times. Packets are removed after delivery or after a configured expiration period.
+
+
+#### Applications
+
+Applications currently using SimpleX network:
+
+- **SimpleX Chat** — a peer-to-peer messenger using SimpleX network as a transport layer, in the same way that communication applications use WebRTC, Tor, i2p, or Nym. All communication logic — contacts, conversations, groups, message formats, end-to-end encryption — runs on endpoint devices.
+
+- **IoT devices** — using the SimpleX queue protocol directly for sensor data collection and device control.
+
+- **AI-based services** — automated services built on the SimpleX Chat application core.
+
+- **Secure monitoring and control systems** — applications for equipment monitoring and control, including robotics, using the network for command delivery and telemetry collection.
+
+[SimpleGo](https://simplego.dev), developed by an independent organization, is a microcontroller-based device running a SimpleX Chat-compatible messenger directly on a microcontroller without a general-purpose operating system. Running over 20 days on a single battery charge, it demonstrates the energy efficiency of resource-based addressing: the device receives packets without continuous polling. A microcontroller-based router implementation that functions simultaneously as a WiFi router is also in development.
+
+
 #### SimpleX objectives
 
 1. Provide messaging infrastructure for distributed applications. This infrastructure needs to have the following qualities:
@@ -70,7 +103,7 @@ SimpleX as a whole is a platform upon which applications can be built. [SimpleX
 
    - Privacy: protect against traffic correlation attacks to determine the contacts that the users communicate with.
 
-   - Reliability: the messages should be delivered even if some participating network servers or receiving clients fail, with “at least once” delivery guarantee.
+   - Reliability: the messages should be delivered even if some participating network routers or receiving clients fail, with "at least once" delivery guarantee.
 
    - Integrity: the messages sent in one direction are ordered in a way that sender and recipient agree on; the recipient can detect when a message was removed or changed.
 
@@ -78,63 +111,63 @@ SimpleX as a whole is a platform upon which applications can be built. [SimpleX
 
    - Low latency: the delay introduced by the network should not be higher than 100ms-1s in addition to the underlying TCP network latency.
 
-2. Provide better communication security and privacy than the alternative instant messaging solutions. In particular SimpleX provides better privacy of metadata (who talks to whom and when) and better security against active network attackers and malicious servers.
+2. Provide better communication security and privacy than the alternative instant messaging solutions. In particular SimpleX provides better privacy of metadata (who talks to whom and when) and better security against active network attackers and malicious routers.
 
 3. Balance user experience with privacy requirements, prioritizing experience of mobile device users.
 
 
 #### In Comparison
 
-SimpleX network has a design similar to P2P networks, but unlike most P2P networks it consists of clients and servers without depending on any centralized component.
+SimpleX network has a design similar to P2P networks, but unlike most P2P networks it consists of clients and routers without depending on any centralized component.
 In comparison to more traditional messaging applications (e.g. WhatsApp, Signal, Telegram) the key differences of SimpleX network are:
 
 - participants do not need to have globally unique addresses to communicate, instead they use redundant unidirectional (simplex) messaging queues, with a separate set of queues for each contact.
 
 - connection requests are passed out-of-band, non-optionally protecting key exchange against man-in-the-middle attack.
 
-- simple message queues provided by network servers are used by the clients to create more complex communication scenarios, such as duplex one-to-one communication, transmitting files, group communication without central servers, and content/communication channels.
+- simple message queues provided by network routers are used by the clients to create more complex communication scenarios, such as duplex one-to-one communication, transmitting files, group communication without central routers, and content/communication channels.
 
-- servers do not store any user information (no user profiles or contacts, or messages once they are delivered), and primarily use in-memory persistence.
+- routers do not store any user information (no user profiles or contacts, or messages once they are delivered), and primarily use in-memory persistence.
 
-- users can change servers with minimal disruption - even after an in-use server disappears, simply by changing the configuration on which servers the new queues are created.
+- users can change routers with minimal disruption - even after an in-use router disappears, simply by changing the configuration on which routers the new queues are created.
 
 
 ## Technical Details
 
-#### Trust in Servers
+#### Trust in Routers
 
-Clients communicate directly with servers (but not with other clients) using SimpleX Messaging Protocol (SMP) running over some transport protocol that provides integrity, server authentication, confidentiality, and transport channel binding. By default, we assume this transport protocol is TLS.
+Clients communicate directly with routers (but not with other clients) using SimpleX Messaging Protocol (SMP) running over some transport protocol that provides integrity, server authentication, confidentiality, and transport channel binding. By default, we assume this transport protocol is TLS.
 
-Users use multiple servers, and choose where to receive their messages. Accordingly, they send messages to their communication partners' chosen servers either directly, if this is a known/trusted server, or via another SMP server providing proxy functionality to protect IP address and session of the sender.
+Users use multiple routers, and choose where to receive their messages. Accordingly, they send messages to their communication partners' chosen routers either directly, if this is a known/trusted router, or via another SMP router providing proxy functionality to protect IP address and session of the sender.
 
-Although end-to-end encryption is always present, users place a degree of trust in servers they connect to. This trust decision is very similar to a user's choice of email provider; however the trust placed in a SimpleX server is significantly less. Notably, there is no re-used identifier or credential between queues on the same (or different) servers. While a user *may* re-use a transport connection to fetch messages from multiple queues, or connect to a server from the same IP address, both are choices a user may opt into to break the promise of un-correlatable queues.
+Although end-to-end encryption is always present, users place a degree of trust in routers they connect to. This trust decision is very similar to a user's choice of email provider; however the trust placed in a SimpleX router is significantly less. Notably, there is no re-used identifier or credential between queues on the same (or different) routers. While a user *may* re-use a transport connection to fetch messages from multiple queues, or connect to a router from the same IP address, both are choices a user may opt into to break the promise of un-correlatable queues.
 
-Users may trust a server because:
+Users may trust a router because:
 
-- They deploy and control the servers themselves from the available open-source code. This has the trade-offs of strong trust in the server but limited metadata obfuscation to a passive network observer. Techniques such as noise traffic, traffic mixing (incurring latency), and using an onion routing transport protocol can mitigate that.
+- They deploy and control the routers themselves from the available open-source code. This has the trade-offs of strong trust in the router but limited metadata obfuscation to a passive network observer. Techniques such as noise traffic, traffic mixing (incurring latency), and using an onion routing transport protocol can mitigate that.
 
-- They use servers from a trusted commercial provider. The more clients the provider has, the less metadata about the communication times is leaked to the network observers.
+- They use routers from a trusted commercial provider. The more clients the provider has, the less metadata about the communication times is leaked to the network observers.
 
-By default, servers do not retain access logs, and permanently delete messages and queues when requested. Messages persist only in memory until they cross a threshold of time, typically on the order of days.[0] There is still a risk that a server maliciously records all queues and messages (even though encrypted) sent via the same transport connection to gain a partial knowledge of the user’s communications graph and other meta-data.
+By default, routers do not retain access logs, and permanently delete messages and queues when requested. Messages persist in memory or in a database until they cross a threshold of time, typically on the order of days.[0] There is still a risk that a router maliciously records all queues and messages (even though encrypted) sent via the same transport connection to gain a partial knowledge of the user's communications graph and other meta-data.
 
-SimpleX supports measures (managed transparently to the user at the agent level) to mitigate the trust placed in servers.  These include rotating the queues in use between users, noise traffic, supporting overlay networks such as Tor, and isolating traffic to different queues to different transport connections (and Tor circuits, if Tor is used).
+SimpleX supports measures (managed transparently to the user at the agent level) to mitigate the trust placed in routers.  These include rotating the queues in use between users, noise traffic, supporting overlay networks such as Tor, and isolating traffic to different queues to different transport connections (and Tor circuits, if Tor is used).
 
-[0] While configurable by servers, a minimum value is enforced by the default software. SimpleX Agents can provide redundant routing over queues to mitigate against message loss.
+[0] While configurable by routers, a minimum value is enforced by the default software. SimpleX Agents can provide redundant routing over queues to mitigate against message loss.
 
 
-#### Client -> Server Communication
+#### Client -> Router Communication
 
 Utilizing TLS grants the SimpleX Messaging Protocol (SMP) server authentication and metadata protection to a passive network observer. But SMP does not rely on the transport protocol for message confidentiality or client authentication. The SMP protocol itself provides end-to-end confidentiality, authentication, and integrity of messages between communicating parties.
 
-Servers have long-lived, self-signed, offline certificates whose hash is pre-shared with clients over secure channels - either provided with the client library or provided in the secure introduction between clients, as part of the server address.  The offline certificate signs an online certificate used in the transport protocol handshake. [0]
+Routers have long-lived, self-signed, offline certificates whose hash is pre-shared with clients over secure channels - either provided with the client library or provided in the secure introduction between clients, as part of the router address.  The offline certificate signs an online certificate used in the transport protocol handshake. [0]
 
-If the transport protocol's confidentiality is broken, incoming and outgoing messages to the server cannot be correlated by message contents. Additionally, because of encryption at the SMP layer, impersonating the server is not sufficient to pass (and therefore correlate) a message from a sender to recipient - the only attack possible is to drop the messages. Only by additionally *compromising* the server can one pass and correlate messages.
+If the transport protocol's confidentiality is broken, incoming and outgoing messages to the router cannot be correlated by message contents. Additionally, because of encryption at the SMP layer, impersonating the router is not sufficient to pass (and therefore correlate) a message from a sender to recipient - the only attack possible is to drop the messages. Only by additionally *compromising* the router can one pass and correlate messages.
 
-It's important to note that the SMP protocol does not do server authentication. Instead we rely upon the fact that an attacker who tricks the transport protocol into authenticating the server incorrectly cannot do anything with the SMP messages except drop them.
+It's important to note that the SMP protocol does not do server authentication. Instead we rely upon the fact that an attacker who tricks the transport protocol into authenticating the router incorrectly cannot do anything with the SMP messages except drop them.
 
-After the connection is established, the client sends blocks of a fixed size 16KB, and the server replies with the blocks of the same size to reduce metadata observable to a network adversary. The protocol has been designed to make traffic correlation attacks difficult, adapting ideas from Tor, remailers, and more general onion and mix networks. It does not try to replace Tor though - SimpleX servers can be deployed as onion services and SimpleX clients can communicate with servers over Tor to further improve participants privacy.
+After the connection is established, the client sends blocks of a fixed size 16KB, and the router replies with the blocks of the same size to reduce metadata observable to a network adversary. The protocol has been designed to make traffic correlation attacks difficult, adapting ideas from Tor, remailers, and more general onion and mix networks. It does not try to replace Tor though - SimpleX routers can be deployed as onion services and SimpleX clients can communicate with routers over Tor to further improve participants privacy.
 
-By using fixed-size blocks, oversized for the expected content, the vast majority of traffic is uniform in nature. When enough traffic is transiting a server simultaneously, the server acts as a low-latency mix node. We can't rely on this behavior to make a security claim, but we have engineered to take advantage of it when we can. As mentioned, this holds true even if the transport connection is compromised.
+By using fixed-size blocks, oversized for the expected content, the vast majority of traffic is uniform in nature. When enough traffic is transiting a router simultaneously, the router acts as a low-latency mix node. We can't rely on this behavior to make a security claim, but we have engineered to take advantage of it when we can. As mentioned, this holds true even if the transport connection is compromised.
 
 The protocol does not protect against attacks targeted at particular users with known identities - e.g., if the attacker wants to prove that two known users are communicating, they can achieve it by observing their local traffic. At the same time, it substantially complicates large-scale traffic correlation, making determining the real user identities much less effective.
 
@@ -143,39 +176,39 @@ The protocol does not protect against attacks targeted at particular users with
 
 #### 2-hop Onion Message Routing
 
-As SimpleX Messaging Protocol servers providing messaging queues are chosen by the recipients, in case senders connect to these servers directly the server owners (who potentially can be the recipients themselves) can learn senders' IP addresses (if Tor is not used) and which other queues on the same server are accessed by the user in the same transport connection (even if Tor is used).
+As SimpleX Messaging Protocol routers providing messaging queues are chosen by the recipients, in case senders connect to these routers directly the router owners (who potentially can be the recipients themselves) can learn senders' IP addresses (if Tor is not used) and which other queues on the same router are accessed by the user in the same transport connection (even if Tor is used).
 
 While the clients support isolating the messages sent to different queues into different transport connections (and Tor circuits), this is not practical, as it consumes additional traffic and system resources.
 
-To mitigate this problem SimpleX Messaging Protocol servers support 2-hop onion message routing when the SMP server chosen by the sender forwards the messages to the servers chosen by the recipients, thus protecting both the senders IP addresses and sessions, even if connection isolation and Tor are not used.
+To mitigate this problem SimpleX Messaging Protocol routers support 2-hop onion message routing when the SMP router chosen by the sender forwards the messages to the routers chosen by the recipients, thus protecting both the senders IP addresses and sessions, even if connection isolation and Tor are not used.
 
 The design of 2-hop onion message routing prevents these potential attacks:
 
-- MITM by proxy (SMP server that forwards the messages).
+- MITM by proxy (SMP router that forwards the messages).
 
-- Identification by the proxy which and how many queues the sender sends messages to (as messages are additionally e2e encrypted between the sender and the destination SMP server).
+- Identification by the proxy which and how many queues the sender sends messages to (as messages are additionally e2e encrypted between the sender and the destination SMP router).
 
 - Correlation of messages sent to different queues via the same user session (as random correlation IDs and keys are used for each message).
 
 See more details about 2-hop onion message routing design in [SimpleX Messaging Protocol](./simplex-messaging.md#proxying-sender-commands)
 
-Also see [Threat model](#threat-model)
+Also see [Security](./security.md)
 
 
 #### SimpleX Messaging Protocol
 
-SMP is initialized with an in-person or out-of-band introduction message, where Alice provides Bob with details of a server (including IP address or host name, port, and hash of the long-lived offline certificate), a queue ID, and Alice's public keys to agree e2e encryption. These introductions are similar to the PANDA key-exchange, in that if observed, the adversary can race to establish the communication channel instead of the intended participant. [0]
+SMP is initialized with an in-person or out-of-band introduction message, where Alice provides Bob with details of a router (including IP address or host name, port, and hash of the long-lived offline certificate), a queue ID, and Alice's public keys to agree e2e encryption. These introductions are similar to the PANDA key-exchange, in that if observed, the adversary can race to establish the communication channel instead of the intended participant. [0]
 
 Because queues are uni-directional, Bob provides an identically-formatted introduction message to Alice over Alice's now-established receiving queue.
 
-When setting up a queue, the server will create separate sender and recipient queue IDs (provided to Alice during set-up and Bob during initial connection). Additionally, during set-up Alice will perform a DH exchange with the server to agree upon a shared secret. This secret will be used to re-encrypt Bob's incoming message before Alice receives it, creating the anti-correlation property earlier-described should the transport encryption be compromised.
+When setting up a queue, the router will create separate sender and recipient queue IDs (provided to Alice during set-up and Bob during initial connection). Additionally, during set-up Alice will perform a DH exchange with the router to agree upon a shared secret. This secret will be used to re-encrypt Bob's incoming message before Alice receives it, creating the anti-correlation property earlier-described should the transport encryption be compromised.
 
-[0] Users can additionally create public 'contact queues' that are only used to receive connection requests.  
+[0] Users can additionally create public 'contact queues' that are only used to receive connection requests.
 
 
 #### SimpleX Agents
 
-SimpleX agents provide higher-level operations compared to SimpleX Clients, who are primarily concerned with creating queues and communicating with servers using SMP.  Agent operations include:
+SimpleX agents provide higher-level operations compared to SimpleX Clients, who are primarily concerned with creating queues and communicating with routers using SMP.  Agent operations include:
 
 - Managing sets of bi-directional, redundant queues for communication partners
 
@@ -186,195 +219,21 @@ SimpleX agents provide higher-level operations compared to SimpleX Clients, who
 - Noise traffic
 
 
-#### Encryption Primitives Used
+## Security
 
-- Ed25519 or Curve25519 to authorize/verify commands to SMP servers (authorization algorithm is set via client/server configuration).
-- Curve25519 for DH exchange to agree:
-  - the shared secret between server and recipient (to encrypt message bodies - it avoids shared cipher-text in sender and recipient traffic)
-  - the shared secret between sender and recipient (to encrypt messages end-to-end in each queue - it avoids shared cipher-text in redundant queues).
-- [NaCl crypto_box](https://nacl.cr.yp.to/box.html) encryption scheme (curve25519xsalsa20poly1305) for message body encryption between server and recipient and for E2E per-queue encryption.
-- SHA256 to validate server offline certificates.
-- [double ratchet](https://signal.org/docs/specifications/doubleratchet/) protocol for end-to-end message encryption between the agents:
-  - Curve448 keys to agree shared secrets required for double ratchet initialization (using [X3DH](https://signal.org/docs/specifications/x3dh/) key agreement with 2 ephemeral keys for each side),
-  - AES-GCM AEAD cipher,
-  - SHA512-based HKDF for key derivation.
+For encryption primitives, threat model, and detailed security analysis, see [Security](./security.md).
 
+SimpleX provides these security properties:
 
-## Threat Model
+- **End-to-end encryption** using Double Ratchet algorithm with forward secrecy and post-quantum cryptography.
 
-#### Global Assumptions
+- **No shared identifiers** across connections — contacts cannot prove they communicate with the same user.
 
- - A user protects their local database and key material.
- - The user's application is authentic, and no local malware is running.
- - The cryptographic primitives in use are not broken.
- - A user's choice of servers is not directly tied to their identity or otherwise represents distinguishing information about the user.
- - The user's client uses 2-hop onion message routing.
+- **Sender deniability** — neither routers nor recipients can cryptographically prove message origin.
 
-#### A passive adversary able to monitor the traffic of one user
+- **Transport metadata protection** — fixed-size blocks, 2-hop onion routing, and optional connection isolation frustrate traffic correlation.
 
-*can:*
-
- - identify that and when a user is using SimpleX.
-
- - determine which servers the user receives the messages from.
-
- - observe how much traffic is being sent, and make guesses as to its purpose.
-
-*cannot:*
-
- - see who sends messages to the user and who the user sends the messages to.
-
- - determine the servers used by users' contacts.
-
-#### A passive adversary able to monitor a set of senders and recipients
-
- *can:*
-
- - identify who and when is using SimpleX.
-
- - learn which SimpleX Messaging Protocol servers are used as receive queues for which users.
-
- - learn when messages are sent and received.
-
- - perform traffic correlation attacks against senders and recipients and correlate senders and recipients within the monitored set, frustrated by the number of users on the servers.
-
- - observe how much traffic is being sent, and make guesses as to its purpose
-
-*cannot, even in case of a compromised transport protocol:*
-
- - perform traffic correlation attacks with any increase in efficiency over a non-compromised transport protocol
-
-#### SimpleX Messaging Protocol server
-
-*can:*
-
-- learn when a queue recipient is online
-
-- know how many messages are sent via the queue (although some may be noise or not content messages).
-
-- learn which messages would trigger notifications even if a user does not use [push notifications](./push-notifications.md).
-
-- perform the correlation of the queue used to receive messages (matching multiple queues to a single user) via either a re-used transport connection, user's IP Address, or connection timing regularities.
-
-- learn a recipient's IP address, track them through other IP addresses they use to access the same queue, and infer information (e.g. employer) based on the IP addresses, as long as Tor is not used.
-
-- drop all future messages inserted into a queue, detectable only over other, redundant queues.
-
-- lie about the state of a queue to the recipient and/or to the sender  (e.g. suspended or deleted when it is not).
-
-- spam a user with invalid messages.
-
-*cannot:*
-
-- undetectably add, duplicate, or corrupt individual messages.
-
-- undetectably drop individual messages, so long as a subsequent message is delivered.
-
-- learn the contents or type of messages.
-
-- distinguish noise messages from regular messages except via timing regularities.
-
-- compromise the users' end-to-end encryption with an active attack.
-
-- learn a sender's IP address, track them through other IP addresses they use to access the same queue, and infer information (e.g. employer) based on the IP addresses, even if Tor is not used (provided messages are sent via proxy SMP server).
-
-- perform senders' queue correlation (matching multiple queues to a single sender) via either a re-used transport connection, user's IP Address, or connection timing regularities, unless it has additional information from the proxy SMP server (provided messages are sent via proxy SMP server).
-
-#### SimpleX Messaging Protocol server that proxies the messages to another SMP server
-
-*can:*
-
-- learn a sender's IP address, as long as Tor is not used.
-
-- learn when a sender with a given IP address is online.
-
-- know how many messages are sent from a given IP address and to a given destination SMP server.
-
-- drop all messages from a given IP address or to a given destination server.
-
-- unless destination SMP server detects repeated public DH keys of senders, replay messages to a destination server within a single session, causing either duplicate message delivery (which will be detected and ignored by the receiving clients), or, when receiving client is not connected to SMP server, exhausting capacity of destination queues used within the session.
-
-*cannot:*
-
-- perform queue correlation (matching multiple queues to a single user), unless it has additional information from the destination SMP server.
-
-- undetectably add, duplicate, or corrupt individual messages.
-
-- undetectably drop individual messages, so long as a subsequent message is delivered.
-
-- learn the contents or type of messages.
-
-- learn which messages would trigger notifications.
-
-- learn the destination queues of messages.
-
-- distinguish noise messages from regular messages except via timing regularities.
-
-- compromise the user's end-to-end encryption with another user via an active attack.
-
-- compromise the user's end-to-end encryption with the destination SMP servers via an active attack.
-
-#### An attacker who obtained Alice's (decrypted) chat database
-
-*can:*
-
-- see the history of all messages exchanged by Alice with her communication partners.
-
-- see shared profiles of contacts and groups.
-
-- surreptitiously receive new messages sent to Alice via existing queues; until communication queues are rotated or the Double-Ratchet advances forward.
-
-- prevent Alice from receiving all new messages sent to her - either surreptitiously by emptying the queues regularly or overtly by deleting them.
-
-- send messages from the user to their contacts; recipients will detect it as soon as the user sends the next message, because the previous message hash won’t match (and potentially won’t be able to decrypt them in case they don’t keep the previous ratchet keys).
-
-*cannot:*
-
-- impersonate a sender and send messages to the user whose database was stolen. Doing so requires also compromising the server (to place the message in the queue, that is possible until the Double-Ratchet advances forward) or the user's device at a subsequent time (to place the message in the database).
-
-- undetectably communicate at the same time as Alice with her contacts. Doing so would result in the contact getting different messages with repeated IDs.
-
-- undetectably monitor message queues in realtime without alerting the user they are doing so, as a second subscription request unsubscribes the first and notifies the second.
-
-#### A user’s contact
-
-*can:*
-
-- spam the user with messages.
-
-- forever retain messages from the user.
-
-*cannot:*
-
-- cryptographically prove to a third-party that a message came from a user (assuming the user’s device is not seized).
-
-- prove that two contacts they have is the same user.
-
-- cannot collaborate with another of the user's contacts to confirm they are communicating with the same user.
-
-#### An attacker who observes Alice showing an introduction message to Bob
-
-*can:*
-
- - Impersonate Bob to Alice.
-
-*cannot:*
-
- - Impersonate Alice to Bob.
-
-#### An attacker with Internet access
-
-*can:*
-
-- Denial of Service SimpleX messaging servers.
-
-- spam a user's public “contact queue” with connection requests.
-
-*cannot:*
-
-- send messages to a user who they are not connected with.
-
-- enumerate queues on a SimpleX server.
+- **Out-of-band key exchange** — connection requests passed outside the network protect against MITM attacks.
 
 
 ## Acknowledgements

protocol/security.md

@@ -0,0 +1,215 @@
+Revision 1, 2026-03-09
+
+# SimpleX Network: Security
+
+This document describes the cryptographic primitives and threat model for the SimpleX network. For a general introduction, see [SimpleX: messaging and application platform](./overview-tjr.md).
+
+## Table of contents
+
+- [Encryption primitives](#encryption-primitives)
+- [Threat model](#threat-model)
+  - [Global Assumptions](#global-assumptions)
+  - [A passive adversary able to monitor the traffic of one user](#a-passive-adversary-able-to-monitor-the-traffic-of-one-user)
+  - [A passive adversary able to monitor a set of senders and recipients](#a-passive-adversary-able-to-monitor-a-set-of-senders-and-recipients)
+  - [SimpleX Messaging Protocol router](#simplex-messaging-protocol-router)
+  - [SimpleX Messaging Protocol router that proxies the messages to another SMP router](#simplex-messaging-protocol-router-that-proxies-the-messages-to-another-smp-router)
+  - [An attacker who obtained Alice's (decrypted) chat database](#an-attacker-who-obtained-alices-decrypted-chat-database)
+  - [A user's contact](#a-users-contact)
+  - [An attacker who observes Alice showing an introduction message to Bob](#an-attacker-who-observes-alice-showing-an-introduction-message-to-bob)
+  - [An attacker with Internet access](#an-attacker-with-internet-access)
+
+
+## Encryption primitives
+
+- **Router command authorization**: X25519 DH-based authenticated encryption (SMP v7+), providing sender deniability. Ed25519 signatures used for recipient commands and notifier commands.
+
+- **Per-queue key agreement**: Curve25519 DH exchange to agree:
+  - the shared secret between router and recipient (to encrypt message bodies — avoids shared ciphertext in sender and recipient traffic),
+  - the shared secret between sender and recipient (to encrypt messages end-to-end in each queue — avoids shared ciphertext in redundant queues).
+
+- **SMP-layer encryption**: [NaCl crypto_box](https://nacl.cr.yp.to/box.html) (curve25519xsalsa20poly1305) for message body encryption between router and recipient, and for e2e per-queue encryption.
+
+- **Certificate validation**: SHA256 to validate router offline certificates.
+
+- **End-to-end encryption**: [Double ratchet](https://signal.org/docs/specifications/doubleratchet/) protocol:
+  - Curve448 keys for shared secret agreement via [X3DH](https://signal.org/docs/specifications/x3dh/) with 2 ephemeral keys per side,
+  - optional [SNTRUP761](https://ntruprime.cr.yp.to/) post-quantum KEM running in parallel with the DH ratchet (see [PQDR](./pqdr.md)), providing post-quantum forward secrecy,
+  - AES-GCM AEAD cipher,
+  - SHA512-based HKDF for key derivation.
+
+
+## Threat Model
+
+### Global Assumptions
+
+- A user protects their local database and key material.
+- The user's application is authentic, and no local malware is running.
+- The cryptographic primitives in use are not broken.
+- A user's choice of routers is not directly tied to their identity or otherwise represents distinguishing information about the user.
+- The user's client uses 2-hop onion message routing.
+
+### A passive adversary able to monitor the traffic of one user
+
+*can:*
+
+- identify that and when a user is using SimpleX.
+
+- determine which routers the user receives messages from.
+
+- observe how much traffic is being sent, and make guesses as to its purpose.
+
+*cannot:*
+
+- see who sends messages to the user and who the user sends messages to.
+
+- determine the routers used by users' contacts.
+
+### A passive adversary able to monitor a set of senders and recipients
+
+*can:*
+
+- identify who and when is using SimpleX.
+
+- learn which SimpleX Messaging Protocol routers are used as receive queues for which users.
+
+- learn when messages are sent and received.
+
+- perform traffic correlation attacks against senders and recipients and correlate senders and recipients within the monitored set, frustrated by the number of users on the routers.
+
+- observe how much traffic is being sent, and make guesses as to its purpose.
+
+*cannot, even in case of a compromised transport protocol:*
+
+- perform traffic correlation attacks with any increase in efficiency over a non-compromised transport protocol.
+
+### SimpleX Messaging Protocol router
+
+*can:*
+
+- learn when a queue recipient is online.
+
+- know how many messages are sent via the queue (although some may be noise or not content messages).
+
+- learn which messages would trigger notifications even if a user does not use [push notifications](./push-notifications.md).
+
+- perform the correlation of the queue used to receive messages (matching multiple queues to a single user) via either a re-used transport connection, user's IP Address, or connection timing regularities.
+
+- learn a recipient's IP address, track them through other IP addresses they use to access the same queue, and infer information (e.g. employer) based on the IP addresses, as long as Tor is not used.
+
+- drop all future messages inserted into a queue, detectable only over other, redundant queues.
+
+- lie about the state of a queue to the recipient and/or to the sender  (e.g. suspended or deleted when it is not).
+
+- spam a user with invalid messages.
+
+*cannot:*
+
+- undetectably add, duplicate, or corrupt individual messages.
+
+- undetectably drop individual messages, so long as a subsequent message is delivered.
+
+- learn the contents or type of messages.
+
+- distinguish noise messages from regular messages except via timing regularities.
+
+- compromise the users' end-to-end encryption with an active attack.
+
+- learn a sender's IP address, track them through other IP addresses they use to access the same queue, and infer information (e.g. employer) based on the IP addresses, even if Tor is not used (provided messages are sent via proxy SMP router).
+
+- perform senders' queue correlation (matching multiple queues to a single sender) via either a re-used transport connection, user's IP Address, or connection timing regularities, unless it has additional information from the proxy SMP router (provided messages are sent via proxy SMP router).
+
+### SimpleX Messaging Protocol router that proxies the messages to another SMP router
+
+*can:*
+
+- learn a sender's IP address, as long as Tor is not used.
+
+- learn when a sender with a given IP address is online.
+
+- know how many messages are sent from a given IP address and to a given destination SMP router.
+
+- drop all messages from a given IP address or to a given destination router.
+
+- unless destination SMP router detects repeated public DH keys of senders, replay messages to a destination router within a single session, causing either duplicate message delivery (which will be detected and ignored by the receiving clients), or, when receiving client is not connected to SMP router, exhausting capacity of destination queues used within the session.
+
+*cannot:*
+
+- perform queue correlation (matching multiple queues to a single user), unless it has additional information from the destination SMP router.
+
+- undetectably add, duplicate, or corrupt individual messages.
+
+- undetectably drop individual messages, so long as a subsequent message is delivered.
+
+- learn the contents or type of messages.
+
+- learn which messages would trigger notifications.
+
+- learn the destination queues of messages.
+
+- distinguish noise messages from regular messages except via timing regularities.
+
+- compromise the user's end-to-end encryption with another user via an active attack.
+
+- compromise the user's end-to-end encryption with the destination SMP routers via an active attack.
+
+### An attacker who obtained Alice's (decrypted) chat database
+
+*can:*
+
+- see the history of all messages exchanged by Alice with her communication partners.
+
+- see shared profiles of contacts and groups.
+
+- surreptitiously receive new messages sent to Alice via existing queues; until communication queues are rotated or the Double-Ratchet advances forward.
+
+- prevent Alice from receiving all new messages sent to her - either surreptitiously by emptying the queues regularly or overtly by deleting them.
+
+- send messages from the user to their contacts; recipients will detect it as soon as the user sends the next message, because the previous message hash won't match (and potentially won't be able to decrypt them in case they don't keep the previous ratchet keys).
+
+*cannot:*
+
+- impersonate a sender and send messages to the user whose database was stolen. Doing so requires also compromising the router (to place the message in the queue, that is possible until the Double-Ratchet advances forward) or the user's device at a subsequent time (to place the message in the database).
+
+- undetectably communicate at the same time as Alice with her contacts. Doing so would result in the contact getting different messages with repeated IDs.
+
+- undetectably monitor message queues in realtime without alerting the user they are doing so, as a second subscription request unsubscribes the first and notifies the first.
+
+### A user's contact
+
+*can:*
+
+- spam the user with messages.
+
+- forever retain messages from the user.
+
+*cannot:*
+
+- cryptographically prove to a third-party that a message came from a user (assuming the user's device is not seized).
+
+- prove that two contacts they have is the same user.
+
+- cannot collaborate with another of the user's contacts to confirm they are communicating with the same user.
+
+### An attacker who observes Alice showing an introduction message to Bob
+
+*can:*
+
+- Impersonate Bob to Alice.
+
+*cannot:*
+
+- Impersonate Alice to Bob.
+
+### An attacker with Internet access
+
+*can:*
+
+- Denial of Service SimpleX messaging routers.
+
+- spam a user's public "contact queue" with connection requests.
+
+*cannot:*
+
+- send messages to a user who they are not connected with.
+
+- enumerate queues on a SimpleX router.

rfcs/2026-03-20-smp-agent-web.md

@@ -0,0 +1,299 @@
+# SMP Agent for Browser — Web Widget Infrastructure
+
+## 1. Problem & Goal
+
+The SimpleX web widget needs to create duplex connections, send and receive encrypted messages, and handle the full SMP agent lifecycle — all running in the browser. This requires a TypeScript implementation of the SMP protocol stack: encoding, transport, client, and agent layers, mirroring the Haskell implementation in simplexmq.
+
+This document covers the protocol infrastructure that lives in the simplexmq repository (`smp-web/`). The widget UI and chat-layer semantics (contact addresses, business addresses, group links) live in simplex-chat.
+
+## 2. Architecture
+
+Four layers, mirroring the Haskell codebase:
+
+```
+┌─────────────────────────────────────────────────────────────┐
+│  Agent Layer                                                 │
+│  Duplex connections, X3DH key agreement, double ratchet,     │
+│  message delivery, queue rotation, connection lifecycle       │
+├─────────────────────────────────────────────────────────────┤
+│  Client Layer                                                │
+│  Connection pool (per server), command/response correlation,  │
+│  reconnection, backoff                                       │
+├─────────────────────────────────────────────────────────────┤
+│  Transport Layer                                             │
+│  WebSocket, SMP handshake, block framing (16384 bytes),      │
+│  block encryption (X25519 DH + SbChainKeys)                  │
+├─────────────────────────────────────────────────────────────┤
+│  Protocol Layer                                              │
+│  SMP commands (NEW, KEY, SUB, SEND, ACK, etc.),              │
+│  binary encoding, transmission format                        │
+├─────────────────────────────────────────────────────────────┤
+│  Shared (from xftp-web)                                      │
+│  encoding.ts, secretbox.ts, padding.ts, keys.ts, digest.ts   │
+└─────────────────────────────────────────────────────────────┘
+                    │
+                    ▼ WebSocket (TLS via browser)
+            ┌───────────────┐
+            │  SMP Server    │
+            │  (SNI → Warp   │
+            │   → WS upgrade)│
+            └───────────────┘
+```
+
+### Core Principle: Mirror Haskell Structure
+
+TypeScript code mirrors the Haskell module hierarchy as closely as possible. Each Haskell module has a corresponding TypeScript file, placed in the same relative path. Functions keep the same names. This enables:
+- Easy cross-reference between codebases
+- Sync as protocol evolves
+- Code review by people who know the Haskell side
+- Byte-for-byte testing of corresponding functions
+
+### File Structure
+
+```
+smp-web/
+├── src/
+│   ├── protocol.ts              ← SMP commands, transmission format
+│   ├── protocol/
+│   │   └── types.ts             ← protocol types
+│   ├── version.ts               ← version range negotiation
+│   ├── transport.ts             ← handshake, block framing, THandle
+│   ├── transport/
+│   │   └── websockets.ts        ← WebSocket connection
+│   ├── client.ts                ← connection pool, correlation, reconnect
+│   ├── crypto/
+│   │   ├── ratchet.ts           ← double ratchet
+│   │   └── shortLink.ts         ← HKDF, link data decrypt
+│   └── agent/
+│       ├── protocol.ts          ← connection types, link data parsing
+│       └── client.ts            ← connection lifecycle, message delivery
+├── package.json
+└── tsconfig.json
+```
+
+Encoding and crypto primitives are imported directly from xftp-web (npm dependency). New files are only created where SMP-specific logic is needed.
+
+### Haskell Module → TypeScript File Mapping
+
+| Haskell Module | TypeScript File | Source |
+|---|---|---|
+| `Simplex.Messaging.Encoding` | xftp-web `protocol/encoding.ts` | import directly |
+| `Simplex.Messaging.Crypto` | xftp-web `crypto/*` | import directly |
+| `Simplex.Messaging.Protocol` | `protocol.ts` | new |
+| `Simplex.Messaging.Protocol.Types` | `protocol/types.ts` | new |
+| `Simplex.Messaging.Version` | `version.ts` | new |
+| `Simplex.Messaging.Transport` | `transport.ts` | new |
+| `Simplex.Messaging.Transport.WebSockets` | `transport/websockets.ts` | new |
+| `Simplex.Messaging.Client` | `client.ts` | new |
+| `Simplex.Messaging.Crypto.Ratchet` | `crypto/ratchet.ts` | new |
+| `Simplex.Messaging.Crypto.ShortLink` | `crypto/shortLink.ts` | new |
+| `Simplex.Messaging.Agent.Protocol` | `agent/protocol.ts` | new |
+| `Simplex.Messaging.Agent.Client` | `agent/client.ts` | new |
+
+Function names in TypeScript match Haskell names (camelCase preserved). When a Haskell function is `smpClientHandshake`, TypeScript has `smpClientHandshake`. When Haskell has `contactShortLinkKdf`, TypeScript has `contactShortLinkKdf`.
+
+## 3. Relationship to xftp-web
+
+xftp-web (`simplexmq-2/xftp-web/`) is a production TypeScript XFTP client. smp-web reuses its foundations:
+
+**Reused directly (npm dependency)**:
+- `protocol/encoding.ts` — Decoder class, Word16/Word32/Int64, ByteString, Large, Bool, Maybe, List encoding
+- `crypto/secretbox.ts` — XSalsa20-Poly1305 (cbEncrypt/cbDecrypt, streaming)
+- `crypto/padding.ts` — Block padding (2-byte length prefix + `#` fill)
+- `crypto/keys.ts` — Ed25519, X25519 key generation, signing, DH, DER encoding
+- `crypto/digest.ts` — SHA-256, SHA-512
+- `crypto/identity.ts` — X.509 certificate chain parsing, signature verification
+
+**New in smp-web**:
+- SMP protocol commands and transmission format
+- SMP handshake (different from XFTP handshake)
+- WebSocket transport (XFTP uses HTTP/2 fetch)
+- SMP client with queue-based correlation
+- Agent layer (connections, ratchet, message processing)
+- Short link operations (HKDF-SHA512, link data parsing)
+
+**Same build pattern**:
+- TypeScript strict, ES2022 modules
+- `tsc` → `dist/`
+- Haskell tests via `callNode` (same function from XFTPWebTests)
+- Each TypeScript function verified byte-for-byte against Haskell
+
+## 4. Server Changes
+
+### Done
+- `attachStaticAndWS` — unified HTTP + WebSocket handler via `wai-websockets`
+- SNI-based routing: browser (SNI) → Warp → WebSocket upgrade → SMP over WS; native (no SNI) → SMP over TLS
+- `acceptWSConnection` — constructs `WS 'TServer` from TLS connection + Warp PendingConnection, preserves peer cert chain
+- `AttachHTTP` takes `TLS 'TServer` (not raw Context), enabling proper cert chain forwarding
+- Test: `testWebSocketAndTLS` verifies native TLS and WebSocket clients on same port
+
+### Remaining
+- CORS headers for cross-origin widget embedding (pattern available in XFTP server)
+- Server CLI configuration for enabling/disabling WebSocket support per port
+
+## 5. Build Approach
+
+Bottom-up, function-by-function. Each TypeScript function tested against its Haskell counterpart before building the next.
+
+**Test infrastructure**: `SMPWebTests.hs` reuses `callNode`, `jsOut`, `jsUint8` from `XFTPWebTests.hs` (generalized, not copied).
+
+**Pattern**: for each function:
+1. Implement in TypeScript
+2. Write Haskell test that calls it via `callNode`
+3. Compare output byte-for-byte with Haskell reference
+4. Also test cross-language: Haskell encodes → TypeScript decodes, and vice versa
+
+## 6. Implementation Phases
+
+### Phase 1: Protocol Encoding + Handshake
+
+Foundation layer. SMP-specific binary encoding and handshake.
+
+**Functions**:
+- SMP transmission format: `[auth ByteString][corrId ByteString][entityId ByteString][command]`
+- `encodeTransmission` / `parseTransmission`
+- `parseSMPServerHandshake` — versionRange, sessionId, authPubKey (CertChainPubKey)
+- `encodeSMPClientHandshake` — version, keyHash, authPubKey, proxyServer, clientService
+- Server certificate chain verification (reuse xftp-web identity.ts)
+- Version negotiation
+
+**Key encoding details**:
+- `authPubKey` uses `encodeAuthEncryptCmds`: Nothing → empty (0 bytes), Just → raw smpEncode (NOT Maybe 0/1 prefix)
+- `proxyServer`: Bool 'T'/'F' (v14+)
+- `clientService`: Maybe '0'/'1' (v16+)
+
+### Phase 2: SMP Commands
+
+All commands needed for messaging.
+
+**Sender**: SKEY, SEND
+**Receiver**: NEW, KEY, SUB, ACK, OFF, DEL
+**Link**: LGET
+**Common**: PING
+
+**For each command**: encode function + decode function for its response, tested against Haskell.
+
+### Phase 3: Transport
+
+WebSocket connection with SMP block framing.
+
+**Functions**:
+- WebSocket connect (`wss://` URL)
+- Block send/receive (16384-byte binary frames)
+- SMP handshake over WebSocket
+- Block encryption: X25519 DH → HKDF-SHA512 → SbChainKeys → per-block XSalsa20-Poly1305
+
+**Block encryption flow**:
+1. Client generates ephemeral X25519 keypair, sends public key in handshake
+2. Server sends its signed DH key in handshake
+3. Both sides compute DH shared secret
+4. `sbcInit(sessionId, dhSecret)` → two 32-byte chain keys (HKDF-SHA512)
+5. Each block: `sbcHkdf(chainKey)` → ephemeral key + nonce, advance chain
+6. Encrypt/decrypt with XSalsa20-Poly1305, blockSize-16 padding target
+
+### Phase 4: Client
+
+Connection management layer.
+
+**Functions**:
+- Connection pool: one WebSocket per SMP server
+- Command/response correlation via corrId
+- Send queue + receive queue (ABQueue pattern from simplexmq-js)
+- Automatic reconnection with exponential backoff
+- Timeout handling
+
+### Phase 5: Agent — Connection Establishment
+
+Duplex SMP connections with X3DH key agreement.
+
+**Functions**:
+- Create receive queue (NEW)
+- Join connection via invitation URI
+- X3DH key agreement
+- Send confirmation (SKEY + SEND)
+- Complete handshake (HELLO exchange)
+- Connection state machine
+
+### Phase 6: Agent — Double Ratchet
+
+Message encryption/decryption.
+
+**Functions**:
+- Signal double ratchet implementation
+- Header encryption
+- Ratchet state management
+- Key derivation (HKDF)
+- Message sequence + hash chain verification
+
+### Phase 7: Agent — Message Delivery
+
+Send and receive messages through established connections.
+
+**Functions**:
+- Send path: encrypt → encode agent envelope → SEND → handle OK/delivery receipt
+- Receive path: SUB → receive MSG → decrypt → verify → ACK
+- Delivery receipts
+- Message acknowledgment
+
+### Phase 8: Short Links
+
+Entry point for the widget — parse short link, fetch profile.
+
+**Functions**:
+- Parse short link URI (contact, group, business address types)
+- HKDF key derivation (SHA-512): `contactShortLinkKdf`
+- LGET command → LNK response
+- Decrypt link data (XSalsa20-Poly1305)
+- Parse FixedLinkData, ConnLinkData, UserLinkData
+- Extract profile JSON
+
+## 7. Persistence
+
+Agent state (keys, ratchet, connections, messages) must persist across page reloads.
+
+**Open question**: storage backend.
+
+Options:
+- **IndexedDB directly** — universal browser support, async API, no additional dependencies. Downside: key-value semantics, no SQL queries, manual indexing.
+- **SQLite in browser** — sql.js (WASM-compiled SQLite) or wa-sqlite (with OPFS backend for persistence). Upside: matches Haskell agent's SQLite storage, schema can mirror `Simplex.Messaging.Agent.Store.SQLite`. Downside: additional dependency, WASM bundle size.
+- **OPFS + SQLite** — Origin Private File System for durable storage, SQLite for structured access. Best durability, but limited browser support (no Safari private browsing).
+
+**Decision criteria**: how closely we want to mirror the Haskell agent's storage schema, bundle size budget, browser compatibility requirements.
+
+## 8. Testing Strategy
+
+### Unit Tests (per function)
+
+Haskell tests in `SMPWebTests.hs` using `callNode` pattern:
+- TypeScript function called via Node.js subprocess
+- Output compared byte-for-byte with Haskell reference
+- Cross-language tests: encode in one language, decode in the other
+
+### Integration Tests
+
+Against live SMP server (spawned by test setup, same pattern as xftp-web globalSetup.ts):
+- WebSocket connect + handshake
+- Command round-trips (NEW, KEY, SUB, SEND, ACK)
+- Message delivery through server
+- Reconnection after disconnect
+
+### Browser Tests
+
+Vitest + Playwright (same as xftp-web):
+- Full connection lifecycle in browser environment
+- WebSocket transport in real browser
+- Persistence round-trips
+
+## 9. Security Model
+
+Same principles as xftp-web:
+- **TLS via browser** — browser handles certificate validation for WSS connections
+- **SNI routing** — browser connections use SNI, routed to Warp + WebSocket handler
+- **Server identity** — verified via certificate chain in SMP handshake (keyHash from short link or known servers)
+- **Block encryption** — X25519 DH + SbChainKeys provides forward secrecy per block, on top of TLS
+- **End-to-end encryption** — double ratchet between agent peers, server sees only encrypted blobs
+- **No server-side secrets** — all keys derived and stored client-side
+- **CORS** — required for cross-origin widget embedding, safe because SMP requires auth on every command
+- **CSP** — strict content security policy for widget page
+
+**Threat model**: same as xftp-web. Primary risk is page substitution (malicious JS). Mitigated by HTTPS, CSP, SRI, and optionally IPFS hosting with published fingerprints.

rfcs/2026-03-20-smp-agent-web/2026-03-20-smp-agent-web-spike.md

@@ -0,0 +1,359 @@
+# SMP Agent Web: Spike Plan
+
+Revision 4, 2026-03-20
+
+Parent RFC: [2026-03-20-smp-agent-web.md](../2026-03-20-smp-agent-web.md)
+
+## Revision History
+
+- **Rev 4**: Aligned with RFC. Restructured as bottom-up build plan with per-function Haskell tests. Router WebSocket support done. File structure mirrors Haskell modules.
+- **Rev 3**: Fixed multiple encoding errors discovered during audit (see encoding details below).
+
+## Objective
+
+Fetch and display business/contact profile from a SimpleX short link URI, via WebSocket to SMP router. This is the first milestone of the SMP agent web implementation — it proves the protocol encoding, transport, crypto, and data parsing layers work end-to-end.
+
+The spike is not throwaway code. It is the beginning of the `smp-web/` TypeScript library, built bottom-up with each function tested against its Haskell counterpart.
+
+## What This Proves
+
+- WebSocket transport to SMP router works from browser
+- SMP protocol encoding is correct (binary format, not ASCII)
+- SMP handshake works (version negotiation, server certificate parsing)
+- Crypto is compatible (HKDF-SHA512, XSalsa20-Poly1305)
+- Short link data parsing matches Haskell (FixedLinkData, ConnLinkData, profile)
+
+## Success Criteria
+
+Haskell test creates a short link, TypeScript fetches and decodes it via WebSocket, profile data matches.
+
+## Protocol Flow
+
+```
+1. Parse short link URI
+   https://simplex.chat/c#<linkKey>?h=hosts&p=port&c=keyHash
+   → server, linkKey
+
+2. Derive keys (HKDF-SHA512)
+   linkKey → (linkId, sbKey)
+
+3. WebSocket connect
+   wss://server:443 (TLS handled by browser)
+
+4. SMP handshake
+   ← SMPServerHandshake {sessionId, smpVersionRange, authPubKey}
+   → SMPClientHandshake {smpVersion, keyHash, authPubKey=Nothing, proxyServer=False, clientService=Nothing}
+
+5. Send LGET
+   → [empty auth][corrId][linkId]["LGET"]
+
+6. Receive LNK
+   ← [auth][corrId][linkId]["LNK" space senderId encFixedData encUserData]
+
+7. Decrypt
+   XSalsa20-Poly1305 with sbKey
+   → FixedLinkData, ConnLinkData (with profile JSON)
+
+8. Display profile
+```
+
+Note: spike sends `authPubKey=Nothing` so block encryption is not used (blocks are padded only). Block encryption is added in steps 12-13.
+
+
+## Build Approach
+
+Bottom-up, function-by-function. Each TypeScript function tested against its Haskell counterpart via `callNode` — the same pattern used in xftp-web (see `XFTPWebTests.hs`).
+
+**Project location**: `simplexmq-2/smp-web/`
+**Tests**: `simplexmq-2/tests/SMPWebTests.hs` — reuses `callNode`/`jsOut`/`jsUint8` from XFTPWebTests (generalized, not copied)
+**xftp-web**: npm dependency via `file:../xftp-web` (encoding, crypto, padding imported directly). Note: libsodium-wrappers-sumo is xftp-web's dependency; tests must init the same sodium instance that xftp-web's secretbox uses. If xftp-web is ever published to npm, libsodium should become a peerDependency.
+**File structure**: mirrors Haskell module hierarchy (see RFC section 2)
+
+**Pattern for each function**:
+1. Check if xftp-web already implements it (or something close). If so, import and reuse — export from xftp-web if not yet exported. Only write new code when no existing implementation covers the need.
+2. Implement in TypeScript, in the file corresponding to its Haskell module
+3. Write Haskell test that calls it via `callNode`
+4. Compare output byte-for-byte with Haskell reference
+5. Cross-language: Haskell encodes → TypeScript decodes, and vice versa
+
+### Parsing Approach
+
+All binary parsing uses xftp-web's `Decoder` class — the same class, not a copy. `Decoder` tracks position over a `Uint8Array`, throws on malformed input, returns subarray views (zero-copy).
+
+SMP command parsing follows the same pattern as xftp-web's `decodeResponse` in `commands.ts`: `readTag` reads bytes until space or end, switch dispatches on the tag string, fields are parsed sequentially with `Decoder` methods (`decodeBytes`, `decodeLarge`, `decodeBool`, etc.).
+
+**Prerequisite xftp-web change**: `readTag` and `readSpace` in xftp-web's `commands.ts` need to be exported so smp-web can import them.
+
+### WebSocket Transport Approach
+
+WebSocket transport follows the simplexmq-js `WSTransport` pattern:
+
+- `WebSocket` connects to `wss://` URL with `binaryType = 'arraybuffer'`
+- `onmessage` enqueues received frames into an `ABQueue` (async bounded queue with backpressure)
+- `onclose` closes the queue (sentinel-based)
+- `readBlock()` dequeues one frame, validates it is exactly 16384 bytes
+- `sendBlock(data)` sends one 16384-byte binary frame
+
+The `ABQueue` class from simplexmq-js provides backpressure via semaphores and clean async iteration. It can be included in smp-web or extracted as a shared utility.
+
+The SMP transport layer wraps WebSocket transport:
+- Receives raw blocks → unpad → parse transmission
+- Encodes transmission → pad → send as block
+- After handshake, if block encryption is active: decrypt before unpad, encrypt after pad
+
+
+## Encoding Reference
+
+Binary encoding rules (from `Simplex.Messaging.Encoding`):
+
+| Type | Format |
+|------|--------|
+| `Word16` | 2 bytes big-endian |
+| `Word32` | 4 bytes big-endian |
+| `ByteString` | 1-byte length + bytes (max 255) |
+| `Large` | 2-byte length (BE) + bytes (max 65535) |
+| `Bool` | 'T' (0x54) or 'F' (0x46) |
+| `Maybe a` | '0' (0x30) for Nothing, '1' (0x31) + value for Just |
+| `smpEncodeList` | 1-byte count + items |
+| `UserLinkData` | ByteString if ≤254 bytes, else 0xFF + Large |
+
+**Critical**: `encodeAuthEncryptCmds Nothing` = empty (0 bytes), NOT 'F' or '0'.
+
+**Transmission format** (binary, NOT ASCII with spaces):
+```
+[auth ByteString][corrId ByteString][entityId ByteString][command bytes]
+```
+
+For v7+ (`implySessId = True`): sessionId is NOT sent on wire, but is prepended to the `authorized` data for signature verification. For unauthenticated commands (LGET), this doesn't apply.
+
+**Block framing**: `pad(transmission, 16384)` = `[2-byte BE length][message][padding with '#' (0x23)]`
+
+
+## Server Changes — DONE
+
+WebSocket support on the same port as native TLS is implemented and tested.
+
+- `attachStaticAndWS` — unified HTTP + WebSocket handler via `wai-websockets`
+- SNI routing: browser (SNI) → Warp → WebSocket upgrade → SMP over WS
+- `acceptWSConnection` — constructs `WS 'TServer` from `TLS 'TServer` + PendingConnection
+- Test: `testWebSocketAndTLS` in `ServerTests.hs`
+
+**Remaining**: CORS headers for cross-origin widget embedding.
+
+
+## Implementation Steps
+
+Each step produces working, tested code. Steps 1-11 work without block encryption. Steps 12-13 add it.
+
+### Step 1: Project Setup + xftp-web Changes
+
+**smp-web setup**:
+- Create `smp-web/` with `package.json` (xftp-web + `@noble/hashes` as dependencies), `tsconfig.json` (ES2022, strict, same as xftp-web)
+- Build: `tsc` → `dist/`
+
+**xftp-web change**:
+- Export `readTag` and `readSpace` from `commands.ts` (currently unexported) so smp-web can import them
+
+**Test infrastructure**:
+- Create `SMPWebTests.hs`, reusing `callNode`/`jsOut`/`jsUint8` from XFTPWebTests (generalize shared utilities into a common test module, not copy)
+- First test: import `decodeBytes` from xftp-web, encode a ByteString, verify output matches Haskell `smpEncode`
+
+### Step 2: SMP Transmission Encode/Decode
+
+**File**: `protocol.ts`
+**Haskell reference**: `Simplex.Messaging.Protocol` — `encodeTransmission_`, `transmissionP`
+
+**Implementation**:
+- `encodeTransmission(corrId, entityId, command)`: `concatBytes(encodeBytes(emptyAuth), encodeBytes(corrId), encodeBytes(entityId), command)` — unsigned, empty auth byte (0x00)
+- `decodeTransmission(data)`: sequential Decoder — `decodeBytes` for auth, corrId, entityId, then `takeAll` for command bytes
+- Pad/unpad: reuse xftp-web `blockPad`/`blockUnpad` (same 2-byte length prefix + '#' padding, same 16384 block size)
+
+**Tests**: encode in TypeScript → decode in Haskell (`transmissionP`), encode in Haskell (`encodeTransmission_`) → decode in TypeScript. Byte-for-byte match.
+
+### Step 3: SMP Handshake Parse/Encode
+
+**File**: `transport.ts`
+**Haskell reference**: `Simplex.Messaging.Transport` — `SMPServerHandshake`, `SMPClientHandshake`
+
+**Implementation**:
+- `parseSMPServerHandshake(d: Decoder)`: `decodeWord16` × 2 for versionRange, `decodeBytes` for sessionId. For authPubKey: if `maxVersion >= 7` and bytes remaining, parse `CertChainPubKey` (reuse xftp-web `identity.ts` for X.509 cert chain parsing and signature extraction). If no bytes remain, authPubKey is absent (encodeAuthEncryptCmds encoded Nothing as empty).
+- `encodeSMPClientHandshake(...)`: `concatBytes(encodeWord16(version), encodeBytes(keyHash), authPubKeyBytes, encodeBool(proxyServer), encodeMaybe(encodeService, clientService))`. Where authPubKey: empty bytes for Nothing, `encodeBytes(pubkey)` for Just. proxyServer only for v14+, clientService only for v16+.
+
+**Tests**: Haskell encodes `SMPServerHandshake` → TypeScript parses, all fields match. TypeScript encodes `SMPClientHandshake` → Haskell parses via `smpP`.
+
+### Step 4: LGET Command Encode
+
+**File**: `protocol.ts`
+**Haskell reference**: `Simplex.Messaging.Protocol` — `LGET` command encoding
+
+**Implementation**:
+- `encodeLGET()`: returns `ascii("LGET")` — 4 bytes, no parameters. The LinkId is carried as entityId in the transmission (step 2), not in the command body.
+- Full LGET block: `blockPad(encodeTransmission(corrId, linkId, encodeLGET()), 16384)`
+
+**Tests**: encode full LGET block in TypeScript, Haskell unpad + `transmissionP` + `parseProtocol` decodes as `LGET` with correct corrId and linkId.
+
+### Step 5: LNK Response Parse
+
+**File**: `protocol.ts`
+**Haskell reference**: `Simplex.Messaging.Protocol` — `LNK` response encoding (line 1834)
+
+**Implementation**:
+- `decodeResponse(d: Decoder)`: `readTag(d)` → switch dispatch (same pattern as xftp-web `decodeResponse`)
+- For `"LNK"`: `readSpace(d)`, `decodeBytes(d)` for senderId, `decodeLarge(d)` for encFixedData, `decodeLarge(d)` for encUserData
+- Also handle `"ERR"` responses for error reporting
+
+**Tests**: Haskell encodes `LNK senderId (encFixed, encUser)` → TypeScript `decodeResponse` parses. All fields match byte-for-byte.
+
+### Step 6: Short Link URI Parse
+
+**File**: `agent/protocol.ts`
+**Haskell reference**: `Simplex.Messaging.Agent.Protocol` — `ConnShortLink` StrEncoding instance (lines 1599-1612)
+
+**Implementation**:
+- `parseShortLink(uri)`: regex to extract scheme (https/simplex), type char (c/g/a), linkKey (base64url, 43 chars → 32 bytes), query params (h=hosts, p=port, c=keyHash)
+- `base64UrlDecode(s)`: pad to multiple of 4, replace `-`→`+`, `_`→`/`, decode
+- Returns `{scheme, connType, server: {hosts, port, keyHash}, linkKey}`
+
+**Tests**: Haskell `strEncode` a `ConnShortLink` → TypeScript `connShortLinkStrP` parses. All fields match. Test multiple formats: with/without query params, different type chars.
+
+**Done**. Function: `connShortLinkStrP` in `agent/protocol.ts`. Uses `base64urlDecode` from xftp-web `description.ts`.
+
+**Future**:
+- Add long link parsing (`ConnectionRequestUri`) and an either-parser that handles both short and long links.
+- Add `restoreShortLink`: preset servers are shortened to host-only (`SMPServerOnlyHost` - no port, no keyHash). After parsing, `restoreShortLink` looks up the full server by hostname from a preset servers list. Without this, connections to preset servers will fail. See `Agent/Protocol.hs:1692`.
+
+### Step 7: HKDF Key Derivation
+
+**File**: `crypto/shortLink.ts`
+**Haskell reference**: `Simplex.Messaging.Crypto.ShortLink` — `contactShortLinkKdf` (line 48)
+
+**Implementation**:
+- `contactShortLinkKdf(linkKey)`: `hkdf(sha512, linkKey, new Uint8Array(0), "SimpleXContactLink", 56)` using `@noble/hashes/hkdf` + `@noble/hashes/sha512`. Split result: first 24 bytes = linkId, remaining 32 bytes = sbKey.
+
+**Note**: Haskell `C.hkdf` uses SHA-512, not SHA3-256.
+
+**Tests**: given known linkKey bytes, TypeScript and Haskell produce identical linkId and sbKey.
+
+### Step 8: Link Data Decrypt
+
+**File**: `crypto/shortLink.ts`
+**Haskell reference**: `Simplex.Messaging.Crypto.ShortLink` — `decryptLinkData` (lines 100-120)
+
+**Implementation**:
+- `decryptLinkData(sbKey, encFixedData, encUserData)`:
+  1. For each EncDataBytes: `Decoder` → `decodeBytes(d)` for nonce (24 bytes), `decodeTail(d)` for ciphertext (includes Poly1305 tag)
+  2. `cbDecrypt(sbKey, nonce, ciphertext)` via xftp-web `secretbox.ts`
+  3. From decrypted plaintext: `decodeBytes(d)` for signature (1-byte len 0x40 + 64 bytes), `decodeTail(d)` for actual data
+  4. Return both plaintext data blobs (signature verification skipped for spike)
+
+**Tests**: Haskell `encodeSignLinkData` + `sbEncrypt` with known key/nonce → TypeScript decrypts → plaintext matches.
+
+### Step 9: ConnLinkData Parse
+
+**File**: `agent/protocol.ts`
+**Haskell reference**: `Simplex.Messaging.Agent.Protocol` — `ConnLinkData`, `UserContactData`, `OwnerAuth`, `ConnShortLink`, `ProtocolServer` Encoding instances
+
+**Implementation** (proper decoding, not skipping):
+- `decodeConnLinkData(d)`: `anyByte` for connectionMode ('C'=Contact), `decodeWord16` × 2 for agentVRange, then `decodeUserContactData`
+- `decodeUserContactData(d)`: `decodeBool` for direct, `smpListP(decodeOwnerAuth, d)` for owners, `smpListP(decodeConnShortLink, d)` for relays, `decodeUserLinkData(d)` for userData
+- `decodeOwnerAuth(d)`: `decodeBytes` for outer wrapper, then parse inner: `(ownerId, ownerKey, authOwnerSig)` all as ByteStrings
+- `decodeConnShortLink(d)`: `anyByte` for mode, then Contact: `(ctTypeChar, srv, linkKey)` or Invitation: `(srv, linkId, linkKey)`
+- `decodeProtocolServer(d)`: `decodeBytes` for scheme+keyHash, `decodeBytes` for host, `decodeBytes` for port — need to verify exact encoding
+- `decodeUserLinkData(d)`: first byte 0xFF → `decodeLarge`; otherwise it's the 1-byte length of a ByteString
+- `parseProfile(userData)`: check first byte for 'X' (0x58, zstd compressed) — if so, decompress; otherwise `JSON.parse` directly
+
+**Tests**: Haskell encodes `ContactLinkData` with known values → TypeScript decodes → all fields match.
+
+**FixedLinkData**: deferred to step 15. `linkConnReq` (ConnectionRequestUri) is NOT length-prefixed in the tuple encoding — it requires full parsing. FixedLinkData is also needed to validate mutable data signature using rootKey.
+
+### Step 15: FixedLinkData Parse
+
+**File**: `agent/protocol.ts`
+**Haskell reference**: `Simplex.Messaging.Agent.Protocol` — `FixedLinkData`, `ConnectionRequestUri`, `ConnReqUriData` Encoding instances
+
+**Implementation**:
+- `decodeFixedLinkData(d)`: `decodeWord16` × 2 for agentVRange, `decodeBytes` for rootKey (32 bytes Ed25519), then parse `ConnectionRequestUri` (mode byte + `ConnReqUriData`), optional `decodeBytes` for linkEntityId
+- `decodeConnectionRequestUri(d)`: full parsing of `ConnReqUriData` including SMP queue URIs
+- Needed for: connecting to the contact, and validating mutable data signature with rootKey
+
+**Tests**: Haskell encodes full `FixedLinkData` → TypeScript decodes → rootKey and linkConnReq fields match.
+
+### Step 10: WebSocket Transport
+
+**File**: `transport/websockets.ts`
+**Pattern reference**: simplexmq-js `WSTransport` + `ABQueue`
+
+**Implementation**:
+- `ABQueue<T>` class: semaphore-based async bounded queue (from simplexmq-js `queue.ts` — reimplement or include as utility). `enqueue`/`dequeue`/`close`, sentinel-based close, async iterator.
+- `connectWS(url)`: `new WebSocket(url)`, `binaryType = 'arraybuffer'`, `onmessage` enqueues `Uint8Array` frames into ABQueue, `onclose` closes queue, `onerror` closes socket. Returns transport handle on `onopen`.
+- `readBlock(transport)`: dequeue one frame, verify `byteLength === 16384`, return `Uint8Array`
+- `sendBlock(transport, data)`: `ws.send(data)`, verify `data.length === 16384`
+- `smpHandshake(transport, keyHash)`: `readBlock` → `blockUnpad` → `parseSMPServerHandshake` → negotiate version → `encodeSMPClientHandshake` → `blockPad` → `sendBlock`. Returns `{sessionId, version}`.
+
+**Integration test**: spawn test SMP server with web credentials (reuse `cfgWebOn` from SMPClient.hs), connect via WebSocket from Node.js, complete handshake, verify sessionId received.
+
+### Step 11: End-to-End Integration
+
+Wire steps 6-10 together: `parseShortLink` → `contactShortLinkKdf` → `connectWS` → `smpHandshake` → encode LGET block → `sendBlock` → `readBlock` → `blockUnpad` → `decodeTransmission` → `decodeResponse` → `decryptLinkData` → `decodeFixedLinkData` + `decodeConnLinkData` → `parseProfile`.
+
+**Test**: Haskell creates a contact address with short link (using agent), TypeScript fetches and decodes it via WebSocket. Profile displayName matches. This is the full spike proof: browser can fetch a SimpleX contact profile via SMP protocol.
+
+### Step 12: Server Certificate Verification
+
+**File**: `transport.ts`
+**Approach**: client sends a random challenge in an HTTP header on the WebSocket upgrade request. Server includes the signed challenge in the handshake response. Client verifies the signature using the server's certificate chain.
+
+**Implementation**:
+- Generate 32-byte random challenge, send as HTTP header (e.g. `smp-web-challenge`) on WebSocket upgrade
+- Parse `CertChainPubKey` from server handshake (already parsed in step 3 as `authPubKey`)
+- Verify certificate chain fingerprint matches `keyHash` (reuse xftp-web `caFingerprint`)
+- Verify challenge signature (reuse xftp-web `identity.ts` — `extractCertPublicKeyInfo`, signature verification)
+- Requires server-side change: detect the challenge header on WebSocket connections, sign `challenge || sessionId` with server key, include proof in handshake
+
+**Tests**: connect to test server, verify challenge-response succeeds. Connect with wrong keyHash, verify rejection.
+
+### Step 13: Block Encryption (DH + SbChainKeys)
+
+**File**: `transport.ts`
+**Haskell reference**: `Simplex.Messaging.Crypto` — `sbcInit`, `sbcHkdf`; `Simplex.Messaging.Transport` — `tPutBlock`, `tGetBlock`
+
+**Implementation**:
+- `generateX25519KeyPair()`, `dh(peerPub, ownPriv)` — reuse from xftp-web `keys.ts`
+- `sbcInit(sessionId, dhSecret)`: `hkdf(sha512, dhSecret, sessionId, "SimpleXSbChainInit", 64)` → split at 32: `(sndChainKey, rcvChainKey)`. Note client swaps send/receive keys vs server (line 858 Transport.hs).
+- `sbcHkdf(chainKey)`: `hkdf(sha512, chainKey, "", "SimpleXSbChain", 88)` → split: 32 bytes new chainKey, 32 bytes sbKey, 24 bytes nonce. Returns `{sbKey, nonce, nextChainKey}`.
+- `encryptBlock(state, block)`: `sbcHkdf` → `cryptoBox(sbKey, nonce, pad(block, blockSize - 16))` → 16-byte tag + ciphertext
+- `decryptBlock(state, block)`: `sbcHkdf` → split tag (first 16 bytes) + ciphertext → `cryptoBoxOpen` → `unpad`
+
+**Tests**: Haskell and TypeScript DH with same keys → identical chain keys. Haskell encrypts block → TypeScript decrypts (and vice versa). Chain key advances identically after each block.
+
+### Step 14: Full Handshake with Auth
+
+**File**: `transport.ts`
+**Haskell reference**: `Simplex.Messaging.Transport` — `smpClientHandshake` (lines 792-842)
+
+**Implementation**:
+- Update `smpHandshake` to generate ephemeral X25519 keypair and include public key in `encodeSMPClientHandshake` as authPubKey
+- Compute DH: `dh(serverDhPub, clientPrivKey)` → shared secret
+- `sbcInit(sessionId, dhSecret)` → chain keys (with client-side swap)
+- All subsequent `readBlock`/`sendBlock` go through `decryptBlock`/`encryptBlock`
+
+**Tests**: full handshake with real server, block encryption active, exchange encrypted commands. Haskell sends encrypted response → TypeScript decrypts correctly.
+
+
+## Haskell Code References
+
+### Handshake
+- `Simplex.Messaging.Transport` — `smpClientHandshake`, `smpServerHandshake`, `SMPServerHandshake`, `SMPClientHandshake`
+- `encodeAuthEncryptCmds` — Nothing → empty, Just → raw smpEncode
+
+### Protocol
+- `Simplex.Messaging.Protocol` — `LGET`, `LNK`, `encodeTransmission_`, `transmissionP`
+- Block: `pad`/`unPad` in `Simplex.Messaging.Crypto`
+
+### Short Links
+- `Simplex.Messaging.Crypto.ShortLink` — `contactShortLinkKdf`, `decryptLinkData`
+- `Simplex.Messaging.Agent.Protocol` — `ConnShortLink`, `FixedLinkData`, `ConnLinkData`, `UserLinkData`
+
+### Block Encryption
+- `Simplex.Messaging.Crypto` — `sbcInit`, `sbcHkdf`, `sbEncrypt`, `sbDecrypt`, `dh'`
+- `Simplex.Messaging.Transport` — `blockEncryption`, `TSbChainKeys`, `tPutBlock`, `tGetBlock`

rfcs/2026-03-20-smp-agent-web/2026-05-17-smp-client.md

@@ -0,0 +1,355 @@
+# SMP Client for Browser
+
+**Parent**: [SMP Agent Web Spike](./2026-03-20-smp-agent-web-spike.md)
+**Depends on**: Spike 1 (merged) — transport, ratchet, encoding, per-queue E2E
+
+## Context
+
+The encoding spike proved all four encryption layers work cross-language. The next implementable and testable piece is the SMP client — the layer that sends commands, correlates responses by CorrId, authenticates with entity keys, and exposes typed async functions.
+
+Faithful transpilation of `Simplex.Messaging.Client` (Client.hs). Transport is WebSocket (already working), protocol logic is identical to Haskell.
+
+## Encoding path (per command)
+
+Traced from `sendSMPMessage` through every function call:
+
+```
+1. encodeTransmission_(v, (corrId, entityId, command))
+   → smpEncode(corrId, entityId) <> encodeProtocol(v, cmd)
+   Already have as encodeTransmission() in protocol.ts — update in place
+
+2. encodeTransmissionForAuth(thParams, transmission)
+   → tForAuth = smpEncode(sessionId) <> encodeTransmission_(...)
+   → tToSend = encodeTransmission_(...)  [when implySessId=true, which is always true for v>=7]
+   Note: implySessId means tToSend omits sessionId, but tForAuth includes it (for signing)
+
+3. authTransmission(thAuth, serviceAuth=false, maybePrivKey, nonce, tForAuth)
+   → thAuth contains serverPubKey (X25519) from handshake
+   → maybePrivKey is Nothing for unauthenticated commands (LGET, SEND without key)
+   → Nothing privKey: no auth, encode empty ByteString
+   → Just X25519 privKey: TAAuthenticator(cbAuthenticate(serverPubKey, privKey, nonce, tForAuth))
+   → Just Ed25519 privKey: TASignature(sign(privKey, tForAuth))
+   Note: nonce IS the CorrId (same 24 bytes used for both)
+   Note: serviceAuth is always false for browser client (no service certificates)
+
+4. tEncodeAuth(serviceAuth=false, maybeAuth)
+   → Nothing: smpEncode("")  [1-byte 0x00]
+   → Just (TAAuthenticator s, _): smpEncode(s)  [1-byte len + 80 bytes]
+   → Just (TASignature sig, _): smpEncode(signatureBytes sig)  [1-byte len + 64 bytes]
+   Note: TAuthorizations = (TransmissionAuth, Maybe serviceSig) — serviceSig always Nothing for us
+
+5. tEncode(serviceAuth, (auth, tToSend))
+   → tEncodeAuth(auth) <> tToSend
+
+6. tEncodeBatch1(serviceAuth, sentRawTransmission)
+   → lenEncode(1) + smpEncode(Large(tEncode(...)))
+   Single-command batch. Always used when batch=true (v7+).
+
+7. batchTransmissions_(blockSize, transmissions)
+   → Pack multiple Large-wrapped transmissions into ≤blockSize blocks
+   → Count byte prefix, up to 255 per block
+   → blockSize' = blockSize - 19 (2 pad + 1 count + 16 auth tag)
+```
+
+## Parsing path (per received block)
+
+```
+1. tParse(thParams, blockBytes)
+   → batch=true: parse count byte, then N Large-wrapped transmissions
+   → Each: transmissionP(thParams) parses:
+     - authenticator (ByteString, 1-byte len + data) — ignored by client
+     - rest = authorized bytes
+     - re-parse authorized: corrId (ByteString) + entityId (ByteString) + command (rest)
+     - if implySessId=true: sessionId not in wire format, prepended from thParams for verification
+   → Returns RawTransmission{authenticator, corrId, entityId, command}
+
+2. tDecodeClient(thParams, rawTransmission)
+   → Verify sessId matches (skipped when implySessId=true)
+   → parseProtocol(v, command) → Either ErrorType BrokerMsg
+   → Return (corrId, entityId, Right msg | Left err)
+
+3. clientResp classification (Client.hs:708-712):
+   → Left err (parse error) → PCEResponseError
+   → Right msg, protocolError msg = Just err → PCEProtocolError (ERR response)
+   → Right msg, protocolError msg = Nothing → Right msg (success)
+
+4. Process: lookup corrId in pendingCommands
+   → Found: resolve Promise with clientResp
+   → Not found (empty corrId = server push): deliver to event callback
+```
+
+## Functions to implement
+
+### Crypto (`src/crypto.ts` — extend)
+
+| Function | Haskell | Implementation |
+|---|---|---|
+| `sha512Hash(msg)` | `Crypto.hs:1016` | `sha512(msg)` from `@noble/hashes/sha512` |
+| `cbAuthenticator(serverPubKey, entityPrivKey, nonce, msg)` | `Crypto.hs:1367` | `cryptoBox(dh(serverPubKey, privKey), nonce, sha512Hash(msg))` → 80 bytes (16 tag + 64 hash) |
+`cryptoBox` and `dh` already available from xftp-web. `sha512` from `@noble/hashes`.
+
+Not needed in spike: `cbDecryptNoPad` (only used by `cbVerify` and proxy commands).
+
+Ed25519 signing: `crypto_sign_detached` from libsodium (already loaded and initialized via xftp-web for secretbox — no second implementation needed).
+
+Not needed: `cbVerify` (server-side only).
+
+### Transport update (`src/transport/websockets.ts` — update)
+
+**Gap: `connectSMP` must return `serverPubKey`** (raw X25519 public key bytes from the handshake). Currently it computes the DH secret and derives block keys, but discards the server's raw public key. The client needs it for `cbAuthenticate` on every command.
+
+Update `SMPConnection` to include:
+```typescript
+interface SMPConnection {
+  ws: WebSocket
+  sessionId: Uint8Array
+  smpVersion: number
+  sndKey: Uint8Array | null
+  rcvKey: Uint8Array | null
+  serverPubKey: Uint8Array | null  // raw X25519 public key — needed for command auth
+}
+```
+
+### Protocol encoding (`src/protocol.ts` — update existing)
+
+Update `encodeTransmission`, `encodeBatch`, `decodeTransmission` in place — these were spike throwaway. Replace with auth-aware versions and update existing tests accordingly.
+
+| Function | Haskell ref | Notes |
+|---|---|---|
+| `encodeTransmission_(v, corrId, entityId, command)` | `Protocol.hs:2194` | Update existing `encodeTransmission`. Also fix `encodeNEW`: QueueReqData should be `Just (QRMessaging Nothing)` not `Nothing`, and rename `sndAuthKey` param to `basicAuth` (it's server auth, not a crypto key) |
+| `encodeTransmissionForAuth(sessionId, corrId, entityId, command)` | `Protocol.hs:2186` | Returns `{tForAuth, tToSend}`. `implySessId` always true for v>=7 |
+| `authTransmission(serverPubKey, maybePrivKey, nonce, tForAuth)` | `Client.hs:1372` | `maybePrivKey` is `{type: "x25519"|"ed25519", key} | null`. Null for unauthenticated commands. X25519 → cbAuthenticator. Ed25519 → sign. |
+| `tEncodeAuth(auth)` | `Protocol.hs:507` | Handles null, authenticator (80 bytes), signature (64 bytes) |
+| `tEncode(auth, tToSend)` | `Protocol.hs:2171` | `tEncodeAuth(auth) + tToSend` |
+| `tEncodeBatch1(auth, tToSend)` | `Protocol.hs:2179` | `[count=1] + Large(tEncode(...))` |
+| `tEncodeForBatch(auth, tToSend)` | `Protocol.hs:2175` | `Large(tEncode(...))` |
+| `batchTransmissions(blockSize, transmissions)` | `Protocol.hs:2151` | Pack into ≤(blockSize-19)-byte blocks, count prefix |
+| `transmissionP(sessionId, block)` | `Protocol.hs:1629` | Skip auth bytes (1-byte len + data), parse corrId + entityId + command from rest. `implySessId`=true (sessionId not in wire, no need to verify on client side), `serviceAuth`=false (no serviceSig to skip) |
+| `tParse(sessionId, block)` | `Protocol.hs:2211` | Parse count, N×Large, each through `transmissionP` |
+| `tDecodeClient(sessionId, version, rawTransmission)` | `Protocol.hs:2256` | Parse command bytes → typed BrokerMsg |
+| `encodePING()` | | PING command for keepalive |
+
+Update `decodeResponse`:
+- Add `SOK` (subscribe response with optional serviceId, returned by SUB in v19)
+- Add `INFO` (queue info response, for `getSMPQueueInfo`)
+- Improve `ERR` parsing: currently reads just the tag string. Need to parse structured `ErrorType` (at minimum AUTH, QUOTA, NO_MSG, INTERNAL) for proper error handling in the client
+
+### Client (`src/client.ts` — new)
+
+```typescript
+interface SMPClient {
+  sessionId: Uint8Array
+  smpVersion: number
+  serverPubKey: Uint8Array  // for cbAuthenticate
+
+  // Core: send pre-encoded command, correlate response
+  // Lower-level than Haskell's sendProtocolCommand — takes pre-encoded command bytes
+  // privKey: {type: "x25519", key} | {type: "ed25519", key} | null
+  // Rejects with PCEProtocolError (ERR response), PCEResponseError (parse fail), PCEResponseTimeout
+  sendCommand(privKey: AuthKey | null, entityId: Uint8Array, command: Uint8Array): Promise<BrokerMsg>
+
+  // High-level commands (keys are DER-encoded unless noted)
+  // authKeyPair: {publicKey, privateKey, type: "x25519"} — public goes in NEW encoding, private for auth
+  createQueue(authKeyPair, dhKey, subMode): Promise<QueueIdsKeys>
+  subscribeQueue(privKey, rcvId): Promise<void>  // SUB can return MSG (queued message) → pushed to onMessage
+  sendMessage(privKey, sndId, flags, msg): Promise<void>  // privKey can be null (before queue secured)
+  ackMessage(privKey, rcvId, msgId): Promise<void>  // ACK can return MSG → pushed to onMessage
+  secureQueue(privKey, rcvId, senderKey): Promise<void>
+  secureSndQueue(privKey, sndId): Promise<void>
+  getQueueLink(linkId): Promise<{senderId, linkData}>
+  getQueueInfo(privKey, queueId): Promise<QueueInfo>
+  deleteQueue(privKey, rcvId): Promise<void>
+  suspendQueue(privKey, rcvId): Promise<void>
+
+  close(): void
+}
+
+function createSMPClient(
+  url: string,
+  keyHash: Uint8Array,
+  onMessage: (entityId: Uint8Array, msg: BrokerMsg) => void,
+  onDisconnected: () => void,
+  wsOptions?: object,
+): Promise<SMPClient>
+```
+
+Internally:
+- `connectSMP` for WebSocket + handshake (existing, updated to return serverPubKey)
+- `Map<string, {resolve, reject}>` for hex(corrId) → Promise correlation
+- WebSocket `onmessage`: `receiveEncryptedBlock` → `tParse` → for each transmission: `tDecodeClient` → classify via `protocolError` → correlate by corrId or push to `onMessage`
+- `sendCommand`: generate random 24-byte corrId/nonce → `encodeTransmissionForAuth` → `authTransmission` → `tEncodeBatch1` → `sendEncryptedBlock` → return Promise resolved by correlator
+- `setInterval` ping: send PING, count timeouts, close after N consecutive
+- Timeout per command: `setTimeout` on pending Promise, reject with PCEResponseTimeout
+
+**Message delivery model:**
+
+All MSGs reach `onMessage` regardless of how they arrive. Three sources:
+
+1. **Server push** (empty corrId): receive handler calls `onMessage` directly
+2. **SUB response**: `sendCommand` resolves with MSG → `subscribeQueue` pushes to `onMessage`, returns success to caller
+3. **ACK response**: same — `ackMessage` pushes to `onMessage`, returns success
+
+High-level functions never expose MSG to their callers. This mirrors Haskell's `processSUBResponse_` (Client.hs:858-862) and `ackSMPMessage` (Client.hs:1042-1044) which both call `writeSMPMessage` to forward MSGs to msgQ and return OK-equivalent.
+
+### Client REPL (`smp-web/tests/client-repl.ts` — new, separate from ratchet-repl.ts)
+
+Separate REPL process holding a WebSocket connection + SMP client state. Same stdin/stdout line protocol approach, different state and commands.
+
+**Message queue:** The REPL maintains an internal `Message[]` queue. The SMPClient's `onMessage` callback pushes to this queue. MSGs arrive here from three sources: server pushes (no corrId), SUB responses, and ACK responses — all handled identically by the client internals. The `RECV` command dequeues from this queue (or waits with timeout).
+
+**Concurrency:** Unlike the ratchet REPL (pure, no network), the client REPL receives messages concurrently with stdin. This works because Node's event loop handles WebSocket `onmessage` events between readline callbacks — no explicit threading needed.
+
+```
+CONNECT <url> <keyHashHex> [wsOptions]
+  → Creates SMPClient, returns "ok"
+
+NEW <rcvAuthKeyHex> <rcvDhKeyHex> [subMode]
+  → createQueue (defaults: no basic auth, SMSubscribe, QRMessaging, no ntf creds)
+  → returns "ok: <rcvIdHex> <sndIdHex> <srvDhKeyHex>"
+
+SUB <rcvIdHex> <rcvPrivKeyHex>
+  → subscribeQueue, returns "ok"
+
+SEND <sndIdHex> <sndPrivKeyHex> <bodyHex>
+  → sendMessage, returns "ok"
+
+ACK <rcvIdHex> <rcvPrivKeyHex> <msgIdHex>
+  → ackMessage, returns "ok"
+
+KEY <rcvIdHex> <rcvPrivKeyHex> <senderKeyHex>
+  → secureQueue, returns "ok"
+
+SKEY <sndIdHex> <sndPrivKeyHex>
+  → secureSndQueue, returns "ok"
+
+LGET <linkIdHex>
+  → getQueueLink, returns "ok: <senderIdHex> <linkDataHex>"
+
+RECV [timeoutMs]
+  → Dequeue next server-pushed MSG, returns "ok: <entityIdHex> <msgIdHex> <bodyHex>"
+  → Times out with "error: timeout" if no message arrives
+```
+
+### Polymorphic testing
+
+Same pattern as ratchet tests: `TestPeer` sum type with `TestPeerHS` / `TestPeerJS` dispatch. For SMP client tests:
+
+```haskell
+data TestSMPClient
+  = TestClientHS SMPClient
+  | TestClientJS Handle Handle ProcessHandle  -- stdin, stdout, process
+
+-- Dispatch functions
+tcCreateQueue :: TestSMPClient -> ... -> IO QueueIdsKeys
+tcSubscribe :: TestSMPClient -> ... -> IO ()
+tcSendMessage :: TestSMPClient -> ... -> IO ()
+tcReceiveMessage :: TestSMPClient -> IO (EntityId, MsgId, ByteString)
+tcSecureQueue :: TestSMPClient -> ... -> IO ()
+tcAckMessage :: TestSMPClient -> ... -> IO ()
+```
+
+Then the same test function runs against HS↔HS, HS↔JS, JS↔HS, JS↔JS peer combinations. The test creates two clients (one receiver, one sender) on the same SMP server, creates a queue, exchanges keys, sends messages — proving protocol compatibility.
+
+## Tests
+
+### Unit tests (callNode, no server)
+
+1. `sha512Hash` — same input → same output as Haskell
+2. `cbAuthenticator` — same serverPubKey + entityPrivKey + nonce + message → same 80 bytes as Haskell
+3. `encodeTransmissionForAuth` — same sessionId + corrId + entityId + command (encoded at v19) → same `{tForAuth, tToSend}` as Haskell
+4. `authTransmission` with X25519 key — same keys + nonce + tForAuth → same authenticated bytes as Haskell
+5. `authTransmission` with Ed25519 key — same key + tForAuth → same signature bytes as Haskell
+6. `authTransmission` with no key (Nothing) — produces empty auth, matches Haskell
+7. `tEncodeBatch1` — same auth + transmission → same block bytes as Haskell
+8. `tParse` + `tDecodeClient` — TS parses Haskell-encoded response block, extracts corrId + entityId + typed response
+9. `batchTransmissions` — given N transmissions, produces same batch boundaries and block bytes as Haskell
+
+### Integration tests (with SMP server, using REPL)
+
+10. JS client connects, sends PING, receives PONG
+11. JS client creates queue (NEW → IDS)
+12. JS receiver creates queue + subscribes, JS sender sends message, receiver gets MSG
+13. Full handshake: create queue → secure (KEY) → subscribe → send → receive MSG → ack
+
+### Polymorphic integration tests
+
+14. Same test function, peer combinations:
+    - HS sender, JS receiver
+    - JS sender, HS receiver
+    - JS sender, JS receiver
+
+## Implementation order
+
+1. Transport update — `connectSMP` returns `serverPubKey`
+2. Crypto additions — `sha512Hash`, `cbAuthenticator`, Ed25519 `sign`
+3. Protocol encoding updates — `encodeTransmissionForAuth`, `authTransmission`, `tEncode`, `tEncodeBatch1`, `batchTransmissions`, `encodePING`
+4. Protocol parsing updates — `transmissionP`, `tParse`, `tDecodeClient`, update `decodeResponse` (add `SOK`, `INFO`, structured `ERR`)
+5. Unit tests for steps 1-4
+6. Client core — `createSMPClient`, `sendCommand`, corrId correlation, receive dispatch, ping
+7. High-level command functions
+8. Client REPL
+9. Integration tests with server
+10. Polymorphic test wiring
+
+## Files
+
+| File | Action |
+|---|---|
+| `smp-web/src/transport/websockets.ts` | Update `connectSMP` to return `serverPubKey` |
+| `smp-web/src/crypto.ts` | Add `sha512Hash`, `cbAuthenticator`, Ed25519 `sign` |
+| `smp-web/src/protocol.ts` | Update transmission encoding/parsing, add auth, batching |
+| `smp-web/src/client.ts` | New — SMP client |
+| `smp-web/tests/client-repl.ts` | New — SMP client REPL for integration tests |
+| `tests/SMPWebTests.hs` | Unit + integration tests |
+
+## Scope
+
+### Client spike (this plan)
+
+Core client: connect, auth, send/receive, correlate, ping. High-level commands: NEW, SUB, KEY, SKEY, SEND, ACK, OFF, DEL, LGET, GET, QUE (getSMPQueueInfo). Single-command path. Tests against real server.
+
+### Client MVP (next, after spike)
+
+- Proxy commands (PRXY, PFWD, PRES) — essential for privacy, users must not connect directly to untrusted servers
+- Batch subscribe (subscribeSMPQueues) — needed for groups
+- `reverseNonce` — needed for proxy
+- Batch delete (deleteSMPQueues)
+
+### Post-MVP
+
+| What | Why |
+|---|---|
+| Notification commands (NKEY, NDEL, NSUB) | Value only with webpush support |
+| Service certificates (serviceAuth, serviceSig) | Browser doesn't use |
+| Stream commands (streamSubscribeSMPQueues) | Not used in Haskell client either |
+| NetworkConfig, SOCKS, host mode, transport selection | Browser connects via WebSocket directly |
+| Queue link management (LSET, LDEL, LKEY) | Only needed to create links, not join them |
+| `cbVerify` | Server-side only |
+
+## Haskell references
+
+- `Client.hs:179-200` — ProtocolClient, PClient types
+- `Client.hs:248` — `type SMPClient = ProtocolClient SMPVersion ErrorType BrokerMsg`
+- `Client.hs:506-512` — Request type
+- `Client.hs:628-642` — client connection, handshake, raceAny_ [send, process, receive, monitor]
+- `Client.hs:644-658` — send loop, receive loop
+- `Client.hs:660-678` — monitor/ping loop
+- `Client.hs:680-719` — process loop, processMsg (corrId correlation, clientResp classification)
+- `Client.hs:810-828` — createSMPQueue
+- `Client.hs:833-836` — subscribeSMPQueue
+- `Client.hs:938-939` — secureSMPQueue
+- `Client.hs:1027-1031` — sendSMPMessage
+- `Client.hs:1040-1045` — ackSMPMessage (note: ACK can return MSG)
+- `Client.hs:1239-1243` — okSMPCommand pattern
+- `Client.hs:1300-1326` — sendProtocolCommand_, sendRecv, size check, tEncodeBatch1
+- `Client.hs:1333-1344` — getResponse, timeout handling
+- `Client.hs:1349-1370` — mkTransmission_, CorrId=nonce, encodeTransmissionForAuth, authTransmission
+- `Client.hs:1372-1391` — authTransmission, authenticate (X25519 vs Ed25519), service sig
+- `Protocol.hs:488-525` — RawTransmission, TransmissionAuth, TAuthorizations, tEncodeAuth
+- `Protocol.hs:1629-1643` — transmissionP
+- `Protocol.hs:2129-2198` — batching, tEncode, tEncodeBatch1, batchTransmissions_
+- `Protocol.hs:2207-2267` — tGetClient, tParse, tDecodeClient
+- `Crypto.hs:1016` — sha512Hash
+- `Crypto.hs:1296-1298` — cbEncryptNoPad (= cryptoBox without padding)
+- `Crypto.hs:1330-1331` — cbDecryptNoPad
+- `Crypto.hs:1366-1371` — cbAuthenticate, cbVerify

rfcs/2026-03-20-smp-agent-web/2026-05-20-client-mvp.md

@@ -0,0 +1,238 @@
+# SMP Client MVP: Proxy + Batching — Transpilation Plan
+
+**Parent**: [SMP Client Spike](./2026-05-17-smp-client.md)
+
+## Rule
+
+Every TypeScript function is a faithful transpilation of a specific Haskell function at specific lines. Same name, same steps, same call chain. No inferences, no approximations. Each entry below gives the exact source to transpile from.
+
+## Crypto functions
+
+### `reverseNonce` → transpile `Crypto.hs:1409-1410`
+```haskell
+reverseNonce (CryptoBoxNonce s) = CryptoBoxNonce (B.reverse s)
+```
+TS: `function reverseNonce(nonce: Uint8Array): Uint8Array` — reverse the 24 bytes.
+
+### `cbDecryptNoPad` → transpile `Crypto.hs:1330-1331`
+```haskell
+cbDecryptNoPad (DhSecretX25519 secret) = sbDecryptNoPad_ secret
+```
+Which is `sbDecryptNoPad_` from secretbox. xftp-web's `cbDecrypt` does decrypt+unpad. Need decrypt without unpad — extract tag(16) + cipher, decrypt, verify tag, return raw (no unpad). Use xftp-web's `sbInit`/`sbDecryptChunk`/`sbAuth` directly.
+
+## Protocol encoding functions
+
+### `encodeProtocolServer` → transpile `Protocol.hs:1264-1266`
+```haskell
+smpEncode ProtocolServer {host, port, keyHash} = smpEncode (host, port, keyHash)
+```
+Where:
+- `host :: NonEmpty TransportHost` → `smpEncodeList` (1-byte count + items)
+- Each `TransportHost` → `smpEncode (strEncode host)` → `encodeBytes(ascii(hostname))` (`Transport/Client.hs:77-78`)
+- `port :: ServiceName` = ByteString → `encodeBytes(port)`
+- `keyHash :: KeyHash` = ByteString → `encodeBytes(keyHash)`
+
+File: `src/protocol.ts`
+
+### `encodePRXY` → transpile `Protocol.hs:1710`
+```haskell
+PRXY host auth_ -> e (PRXY_, ' ', host, auth_)
+```
+= `"PRXY " + smpEncode(server) + smpEncode(Maybe BasicAuth)`
+
+Where `Maybe BasicAuth` = `encodeMaybe(encodeBytes, auth)`.
+
+### `encodePFWD` → transpile `Protocol.hs:1711`
+```haskell
+PFWD fwdV pubKey (EncTransmission s) -> e (PFWD_, ' ', fwdV, pubKey, Tail s)
+```
+= `"PFWD " + encodeWord16(version) + encodeBytes(pubKeyDer) + encTransmission` (Tail = no length prefix)
+
+### `decodePKEY` → transpile `Protocol.hs:1894`
+```haskell
+PKEY_ -> PKEY <$> _smpP <*> smpP <*> smpP
+```
+= space + `decodeBytes(d)` (sessionId) + `decodeVersionRange(d)` + `decodeCertChainPubKey(d)`
+
+`VersionRange` encoding (`Version.hs`): `smpEncode (minVersion, maxVersion)` = two Word16.
+
+`CertChainPubKey` encoding (`Transport.hs:663-667`): `smpEncode (encodeCertChain chain, SignedObject signedPubKey)` — `encodeCertChain` is `Large`-encoded DER bytes, `SignedObject` is `Large`-encoded DER bytes.
+
+### `decodePRES` → transpile `Protocol.hs:1896`
+```haskell
+PRES_ -> PRES <$> (EncResponse . unTail <$> _smpP)
+```
+= space + rest of bytes (Tail) → `EncResponse`
+
+### Add to `decodeResponse`: `PKEY` and `PRES` cases.
+
+## Client functions
+
+### `sendProtocolCommands` → transpile `Client.hs:1262-1278`
+
+Call chain:
+1. `mapM (mkTransmission c) cs` — for each command: generate corrId, encode, auth, register pending request
+2. `batchTransmissions' thParams` — pack into blocks
+3. `mapM (sendBatch c nm) bs` — send each block, collect responses
+4. `validate` — verify response count matches command count
+
+In TS: `mkTransmission` = the existing `sendCommand` logic (corrId generation, `encodeTransmissionForAuth`, `authTransmission`) but separated into encode+register vs send+await. Need to refactor `sendCommand` to split these.
+
+### `batchTransmissions'` → transpile `Protocol.hs:2135-2148`
+
+Already have `batchTransmissions` in protocol.ts that does `batchTransmissions_`. Need `batchTransmissions'` which wraps with `tEncodeForBatch` before batching. Currently the TS `batchTransmissions` takes pre-encoded Large-wrapped bytes. Need to match the Haskell call chain exactly:
+
+```haskell
+batchTransmissions' params ts
+  | batch = batchTransmissions_ bSize $ L.map (first $ fmap $ tEncodeForBatch serviceAuth) ts
+```
+
+### `sendBatch` → transpile `Client.hs:1285-1298`
+
+Three cases:
+- `TBError`: return error response
+- `TBTransmissions s n rs`: send block `s`, await all `n` responses concurrently
+- `TBTransmission s r`: send block `s`, await one response
+
+In browser: "concurrently" = all promises pending simultaneously, resolved by `onBlock` handler as responses arrive.
+
+### `subscribeSMPQueues` → transpile `Client.hs:840-845`
+```haskell
+subscribeSMPQueues c qs = do
+  liftIO $ enablePings c
+  sendProtocolCommands c NRMBackground cs >>= mapM (processSUBResponse c)
+  where
+    cs = L.map (\(rId, rpKey) -> (rId, Just rpKey, Cmd SRecipient SUB)) qs
+```
+
+### `processSUBResponse` → transpile `Client.hs:854-862`
+```haskell
+processSUBResponse c (Response rId r) = pure r $>>= processSUBResponse_ c rId
+processSUBResponse_ c rId = \case
+  OK -> pure $ Right Nothing
+  SOK serviceId_ -> pure $ Right serviceId_
+  cmd@MSG {} -> writeSMPMessage c rId cmd $> Right Nothing
+  r' -> pure . Left $ unexpectedResponse r'
+```
+MSG → push to `onMessage`, return success. Same pattern as single subscribe.
+
+### `deleteSMPQueues` → transpile `Client.hs:1062-1065`
+```haskell
+deleteSMPQueues = okSMPCommands DEL
+```
+Uses `okSMPCommands` (`Client.hs:1245-1253`) which calls `sendProtocolCommands` and checks each response is OK.
+
+### `connectSMPProxiedRelay` → transpile `Client.hs:1069-1093`
+
+Call chain:
+1. Send `PRXY relayServ proxyAuth` to proxy (via `sendProtocolCommand_`, entityId = NoEntity)
+2. Receive `PKEY sessionId versionRange certChainPubKey`
+3. Check version compatibility
+4. `validateRelay chain key` — validate cert chain against relay's keyHash, extract X25519 key
+5. Return `ProxiedRelay {sessionId, version, auth, relayKey}`
+
+`validateRelay` (`Client.hs:1085-1093`):
+1. `chainIdCaCerts chain` → extract leaf, id, ca certs
+2. Check `Fingerprint kh == getFingerprint idCert SHA256`
+3. `x509validate caCert (hostName, port) chain`
+4. Extract server key from leaf cert
+5. Verify signed key against server key
+
+In browser: we already have `verifyIdentityProof` and `extractSignedKey` from xftp-web. Need to adapt for relay validation where we receive the cert chain in the PKEY response (DER-encoded, not from TLS handshake).
+
+### `proxySMPCommand` → transpile `Client.hs:1157-1206`
+
+Call chain:
+1. Construct `serverThParams` = `smpTHParamsSetVersion v proxyThParams {sessionId, thAuth = serverThAuth}`
+   - `serverThAuth = thAuth proxyThParams with peerServerPubKey = relayKey`
+2. Generate ephemeral X25519 keypair: `(cmdPubKey, cmdPrivKey)`
+3. `cmdSecret = dh(relayKey, cmdPrivKey)`
+4. Generate random nonce (also used as corrId)
+5. `encodeTransmissionForAuth serverThParams (CorrId corrId, sId, Cmd sParty command)` — encode as if sending to relay
+6. `authTransmission serverThAuth False spKey nonce tForAuth` — authenticate with entity key against relay
+7. `batchTransmissions serverThParams [Right (auth, tToSend)]` — batch into single block
+8. `cbEncrypt cmdSecret nonce batchBlock paddedProxiedTLength` → `EncTransmission`
+9. Send `PFWD version cmdPubKey encTransmission` to proxy (entityId = sessionId)
+10. Receive `PRES (EncResponse encResponse)`
+11. `cbDecrypt cmdSecret (reverseNonce nonce) encResponse` — decrypt relay's response
+12. `tParse serverThParams decrypted` — parse as relay's response
+13. `tDecodeClient serverThParams parsed` — decode command
+14. Classify: `Right (ERR e)` → throw PCEProtocolError, `Right r` → return Right r, `Left e` → throw PCEResponseError
+
+Error wrapping (`Client.hs:1200-1206`): proxy-level errors (from PFWD response itself) → `ProxyClientError` returned as `Left`. Relay-level errors (inside PRES) → `PCEProtocolError` thrown.
+
+### `paddedProxiedTLength` → `Protocol.hs:306-307` = 16226
+
+## Constants
+
+```
+paddedProxiedTLength = 16226  -- Protocol.hs:306
+serviceCertsSMPVersion = 16   -- Transport.hs:213
+```
+
+## Testing
+
+### Unit tests (callNode, no server)
+Each encoding function tested byte-for-byte against Haskell:
+1. `reverseNonce` — reverse known bytes, compare
+2. `encodeProtocolServer` — encode known server, compare with `smpEncode @SMPServer`
+3. `encodePRXY` — encode PRXY command, compare with `encodeProtocol v (Cmd SProxiedClient (PRXY srv auth))`
+4. `encodePFWD` — encode PFWD command, compare with `encodeProtocol v (Cmd SProxiedClient (PFWD v pk et))`
+5. `batchTransmissions` with multiple commands — same batch boundaries as `batchTransmissions_` in Haskell
+
+### Integration tests (with two SMP servers, from SMPProxyTests.hs pattern)
+6. `connectProxiedRelay` — JS connects to proxy, sends PRXY for relay, gets PKEY, validates cert, extracts key
+7. `proxySMPMessage` — JS sends SEND via proxy to relay, HS receiver gets MSG
+8. Full proxy roundtrip — JS creates queue on relay via proxy, sends message via proxy, HS receives
+
+### Batch tests
+9. `subscribeSMPQueues` — JS batch-subscribes to N queues, verifies all subscribed
+10. `deleteSMPQueues` — JS batch-deletes N queues
+
+## Implementation order
+
+1. `reverseNonce`, `cbDecryptNoPad`
+2. `encodeProtocolServer`, `encodePRXY`, `encodePFWD`
+3. `decodePKEY`, `decodePRES`, update `decodeResponse`
+4. Unit tests for steps 1-3
+5. Refactor `sendCommand` → split into `mkTransmission` (encode+register) and send
+6. `sendProtocolCommands`, `sendBatch`
+7. `subscribeSMPQueues`, `deleteSMPQueues`
+8. Batch integration tests
+9. `connectSMPProxiedRelay` (cert validation, PRXY/PKEY)
+10. `proxySMPCommand`, `proxySMPMessage`
+11. Proxy integration tests
+
+## Files
+
+| File | Action |
+|---|---|
+| `smp-web/src/crypto.ts` | `reverseNonce`, `cbDecryptNoPad` |
+| `smp-web/src/protocol.ts` | `encodeProtocolServer`, `encodePRXY`, `encodePFWD`, `decodePKEY`, `decodePRES` |
+| `smp-web/src/client.ts` | `sendProtocolCommands`, `sendBatch`, `subscribeSMPQueues`, `deleteSMPQueues`, `connectSMPProxiedRelay`, `proxySMPCommand` |
+| `smp-web/tests/client-repl.ts` | Proxy + batch REPL commands |
+| `tests/SMPWebTests.hs` | Tests |
+
+## Haskell source — exact lines to transpile
+
+| TS function | Haskell function | File:lines |
+|---|---|---|
+| `reverseNonce` | `reverseNonce` | `Crypto.hs:1409-1410` |
+| `cbDecryptNoPad` | `cbDecryptNoPad` / `sbDecryptNoPad_` | `Crypto.hs:1330-1331` |
+| `encodeProtocolServer` | `instance Encoding (ProtocolServer p)` | `Protocol.hs:1264-1266` |
+| `encodePRXY` | `encodeProtocol v (PRXY ...)` | `Protocol.hs:1710` |
+| `encodePFWD` | `encodeProtocol v (PFWD ...)` | `Protocol.hs:1711` |
+| `decodePKEY` | `protocolP v PKEY_` | `Protocol.hs:1894` |
+| `decodePRES` | `protocolP v PRES_` | `Protocol.hs:1896` |
+| `sendProtocolCommands` | `sendProtocolCommands` | `Client.hs:1262-1278` |
+| `sendBatch` | `sendBatch` | `Client.hs:1285-1298` |
+| `subscribeSMPQueues` | `subscribeSMPQueues` | `Client.hs:840-845` |
+| `processSUBResponse` | `processSUBResponse` + `processSUBResponse_` | `Client.hs:854-862` |
+| `deleteSMPQueues` | `deleteSMPQueues` via `okSMPCommands` | `Client.hs:1062-1065, 1245-1253` |
+| `connectSMPProxiedRelay` | `connectSMPProxiedRelay` | `Client.hs:1069-1093` |
+| `validateRelay` | `validateRelay` (inside `connectSMPProxiedRelay`) | `Client.hs:1085-1093` |
+| `proxySMPCommand` | `proxySMPCommand` | `Client.hs:1157-1206` |
+| `proxyOKSMPCommand` | `proxyOKSMPCommand` | `Client.hs:1150-1155` |
+| `smpTHParamsSetVersion` | `smpTHParamsSetVersion` | `Transport.hs:921-926` |
+| `batchTransmissions'` | `batchTransmissions'` | `Protocol.hs:2135-2148` |
+| `batchTransmissions_` | `batchTransmissions_` | `Protocol.hs:2150-2169` |

rfcs/README.md

@@ -131,7 +131,7 @@ A subset of standard specifications will be designated as Core IP under the Cons
 
 This is a legally binding commitment. Once Licensed IP is included in Core IP, the company that owns the code cannot unilaterally change it — even though they own the code, the Consortium Agreement requires a Governing Decision for any change to Core IP. This protects the fundamental properties of the network (privacy, security, decentralization) from unilateral modification by any single party.
 
-The designation of specific specifications as Core IP is itself a Governing Decision requiring unanimous approval. The transition will happen incrementally as protocols stabilize — the governance ratchet ensures that each designation is irreversible.
+The designation of specific specifications as Core IP is itself a Governing Decision that requires Consortium vote. The transition will happen incrementally as protocols stabilize — the governance ratchet ensures that each designation is irreversible.
 
 The exact mechanism for distinguishing core from standard within the RFC and protocol folder structure is TBD — it will be decided as the first protocols are designated as Core IP.
 

scripts/docker/entrypoint-smp-server

@@ -67,14 +67,14 @@ if [ ! -f "${confd}/smp-server.ini" ]; then
 
   # Fix path to certificates
   if [ -n "${WEB_MANUAL}" ]; then
-    sed -i -e 's|^[^#]*https: |#&|' \
-           -e 's|^[^#]*cert: |#&|' \
-           -e 's|^[^#]*key: |#&|' \
-           -e 's|^port:.*|port: 5223|' \
+    sed -i -e 's|^[^#]*https = |#&|' \
+           -e 's|^[^#]*cert = |#&|' \
+           -e 's|^[^#]*key = |#&|' \
+           -e 's|^port = .*|port = 5223|' \
            "${confd}/smp-server.ini"
   else
-    sed -i -e "s|cert: /etc/opt/simplex/web.crt|cert: $cert_path/$ADDR.crt|" \
-           -e "s|key: /etc/opt/simplex/web.key|key: $cert_path/$ADDR.key|" \
+    sed -i -e "s|cert = /etc/opt/simplex/web.crt|cert = $cert_path/$ADDR.crt|" \
+           -e "s|key = /etc/opt/simplex/web.key|key = $cert_path/$ADDR.key|" \
            "${confd}/smp-server.ini"
   fi
 fi

scripts/docker/entrypoint-xftp-server

@@ -76,7 +76,7 @@ if [ ! -f "${confd}/file-server.ini" ]; then
 
   # Optionally, set password
   if [ -n "${PASS}" ]; then
-    sed -i -e "/^# create_password:/a create_password: $PASS" \
+    sed -i -e "/^# create_password =/a create_password = $PASS" \
            "${confd}/file-server.ini"
   fi
 fi

simplexmq.cabal

@@ -1,7 +1,7 @@
 cabal-version: 1.12
 
 name:           simplexmq
-version:        6.5.0.9
+version:        6.5.1.0
 synopsis:       SimpleXMQ message broker
 description:    This package includes <./docs/Simplex-Messaging-Server.html server>,
                 <./docs/Simplex-Messaging-Client.html client> and
@@ -24,33 +24,38 @@ extra-source-files:
     CHANGELOG.md
     cbits/sha512.h
     cbits/sntrup761.h
-    apps/smp-server/static/index.html
-    apps/smp-server/static/link.html
-    apps/smp-server/static/media/apk_icon.png
-    apps/smp-server/static/media/apple_store.svg
-    apps/smp-server/static/media/contact.js
-    apps/smp-server/static/media/contact_page_mobile.png
-    apps/smp-server/static/media/f_droid.svg
-    apps/smp-server/static/media/favicon.ico
-    apps/smp-server/static/media/GilroyBold.woff2
-    apps/smp-server/static/media/GilroyLight.woff2
-    apps/smp-server/static/media/GilroyMedium.woff2
-    apps/smp-server/static/media/GilroyRegular.woff2
-    apps/smp-server/static/media/GilroyRegularItalic.woff2
-    apps/smp-server/static/media/google_play.svg
-    apps/smp-server/static/media/logo-dark.png
-    apps/smp-server/static/media/logo-light.png
-    apps/smp-server/static/media/logo-symbol-dark.svg
-    apps/smp-server/static/media/logo-symbol-light.svg
-    apps/smp-server/static/media/moon.svg
-    apps/smp-server/static/media/qrcode.js
-    apps/smp-server/static/media/script.js
-    apps/smp-server/static/media/style.css
-    apps/smp-server/static/media/sun.svg
-    apps/smp-server/static/media/swiper-bundle.min.css
-    apps/smp-server/static/media/swiper-bundle.min.js
-    apps/smp-server/static/media/tailwind.css
-    apps/smp-server/static/media/testflight.png
+    apps/common/Web/static/index.html
+    apps/common/Web/static/link.html
+    apps/common/Web/static/media/apk_icon.png
+    apps/common/Web/static/media/apple_store.svg
+    apps/common/Web/static/media/contact.js
+    apps/common/Web/static/media/contact_page_mobile.png
+    apps/common/Web/static/media/f_droid.svg
+    apps/common/Web/static/media/favicon.ico
+    apps/common/Web/static/media/GilroyBold.woff2
+    apps/common/Web/static/media/GilroyLight.woff2
+    apps/common/Web/static/media/GilroyMedium.woff2
+    apps/common/Web/static/media/GilroyRegular.woff2
+    apps/common/Web/static/media/GilroyRegularItalic.woff2
+    apps/common/Web/static/media/google_play.svg
+    apps/common/Web/static/media/logo-dark.png
+    apps/common/Web/static/media/logo-light.png
+    apps/common/Web/static/media/logo-symbol-dark.svg
+    apps/common/Web/static/media/logo-symbol-light.svg
+    apps/common/Web/static/media/moon.svg
+    apps/common/Web/static/media/qrcode.js
+    apps/common/Web/static/media/script.js
+    apps/common/Web/static/media/style.css
+    apps/common/Web/static/media/sun.svg
+    apps/common/Web/static/media/swiper-bundle.min.css
+    apps/common/Web/static/media/swiper-bundle.min.js
+    apps/common/Web/static/media/tailwind.css
+    apps/common/Web/static/media/testflight.png
+    apps/xftp-server/static/media/xftp-protocol.svg
+    apps/xftp-server/static/media/xftp-protocol-dark.svg
+    apps/xftp-server/static/xftp-web-bundle/crypto.worker.js
+    apps/xftp-server/static/xftp-web-bundle/index.css
+    apps/xftp-server/static/xftp-web-bundle/index.js
 
 flag swift
   description: Enable swift JSON format
@@ -168,6 +173,7 @@ library
           Simplex.Messaging.Agent.Store.Postgres.Migrations.M20251009_queue_to_subscribe
           Simplex.Messaging.Agent.Store.Postgres.Migrations.M20251010_client_notices
           Simplex.Messaging.Agent.Store.Postgres.Migrations.M20251230_strict_tables
+          Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260410_receive_attempts
   else
       exposed-modules:
           Simplex.Messaging.Agent.Store.SQLite
@@ -218,6 +224,7 @@ library
           Simplex.Messaging.Agent.Store.SQLite.Migrations.M20251009_queue_to_subscribe
           Simplex.Messaging.Agent.Store.SQLite.Migrations.M20251010_client_notices
           Simplex.Messaging.Agent.Store.SQLite.Migrations.M20251230_strict_tables
+          Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260410_receive_attempts
           Simplex.Messaging.Agent.Store.SQLite.Util
   if flag(client_postgres) || flag(server_postgres)
       exposed-modules:
@@ -245,6 +252,7 @@ library
           Simplex.Messaging.Server.Main
           Simplex.Messaging.Server.Main.GitCommit
           Simplex.Messaging.Server.Main.Init
+          Simplex.Messaging.Server.Web
           Simplex.Messaging.Server.MsgStore
           Simplex.Messaging.Server.MsgStore.Journal
           Simplex.Messaging.Server.MsgStore.Journal.SharedLock
@@ -275,6 +283,9 @@ library
               Simplex.Messaging.Notifications.Server.Store.Postgres
               Simplex.Messaging.Notifications.Server.Store.Types
               Simplex.Messaging.Notifications.Server.StoreLog
+              Simplex.FileTransfer.Server.Store.Postgres
+              Simplex.FileTransfer.Server.Store.Postgres.Config
+              Simplex.FileTransfer.Server.Store.Postgres.Migrations
               Simplex.Messaging.Server.MsgStore.Postgres
               Simplex.Messaging.Server.QueueStore.Postgres
               Simplex.Messaging.Server.QueueStore.Postgres.Migrations
@@ -346,6 +357,11 @@ library
       , optparse-applicative >=0.15 && <0.17
       , process ==1.6.*
       , temporary ==1.3.*
+      , wai >=3.2 && <3.3
+      , wai-app-static >=3.1 && <3.2
+      , wai-websockets >=3.0.1 && <3.1
+      , warp ==3.3.30
+      , warp-tls ==3.4.7
       , websockets ==0.12.*
       , zlib >=0.6 && <0.8
   if flag(client_postgres) || flag(server_postgres)
@@ -402,30 +418,22 @@ executable smp-server
     cpp-options: -DdbServerPostgres
   main-is: Main.hs
   other-modules:
-      Static
-      Static.Embedded
+      SMPWeb
+      Web.Embedded
       Paths_simplexmq
   hs-source-dirs:
       apps/smp-server
-      apps/smp-server/web
+      apps/common
   default-extensions:
       StrictData
   ghc-options: -Weverything -Wno-missing-exported-signatures -Wno-missing-import-lists -Wno-missed-specialisations -Wno-all-missed-specialisations -Wno-unsafe -Wno-safe -Wno-missing-local-signatures -Wno-missing-kind-signatures -Wno-missing-deriving-strategies -Wno-monomorphism-restriction -Wno-prepositive-qualified-module -Wno-implicit-prelude -Wno-missing-safe-haskell-mode -Wno-missing-export-lists -Wno-partial-fields -Wcompat -Werror=incomplete-record-updates -Werror=incomplete-patterns -Werror=incomplete-uni-patterns -Werror=missing-methods -Werror=tabs -Wredundant-constraints -Wincomplete-record-updates -Wunused-type-patterns -O2 -threaded -rtsopts
   build-depends:
       base
     , bytestring
-    , directory
-    , file-embed
-    , filepath
-    , network
+    , file-embed >=0.0.10 && <0.1
     , simple-logger
     , simplexmq
     , text
-    , unliftio
-    , wai
-    , wai-app-static
-    , warp ==3.3.30
-    , warp-tls ==3.4.7
   default-language: Haskell2010
 
 executable xftp
@@ -449,14 +457,21 @@ executable xftp-server
       buildable: False
   main-is: Main.hs
   other-modules:
+      XFTPWeb
+      Web.Embedded
       Paths_simplexmq
   hs-source-dirs:
       apps/xftp-server
+      apps/common
   default-extensions:
       StrictData
   ghc-options: -Weverything -Wno-missing-exported-signatures -Wno-missing-import-lists -Wno-missed-specialisations -Wno-all-missed-specialisations -Wno-unsafe -Wno-safe -Wno-missing-local-signatures -Wno-missing-kind-signatures -Wno-missing-deriving-strategies -Wno-monomorphism-restriction -Wno-prepositive-qualified-module -Wno-implicit-prelude -Wno-missing-safe-haskell-mode -Wno-missing-export-lists -Wno-partial-fields -Wcompat -Werror=incomplete-record-updates -Werror=incomplete-patterns -Werror=incomplete-uni-patterns -Werror=missing-methods -Werror=tabs -Wredundant-constraints -Wincomplete-record-updates -Wunused-type-patterns -O2 -threaded -rtsopts
   build-depends:
       base
+    , bytestring
+    , directory
+    , file-embed >=0.0.10 && <0.1
+    , filepath
     , simple-logger
     , simplexmq
   default-language: Haskell2010
@@ -498,9 +513,12 @@ test-suite simplexmq-test
       XFTPCLI
       XFTPClient
       XFTPServerTests
+      WebTests
       XFTPWebTests
-      Static
-      Static.Embedded
+      SMPWebTests
+      SMPWeb
+      XFTPWeb
+      Web.Embedded
       Paths_simplexmq
   if flag(client_postgres)
       other-modules:
@@ -512,12 +530,15 @@ test-suite simplexmq-test
   if flag(server_postgres)
       other-modules:
           AgentTests.NotificationTests
+          CoreTests.XFTPStoreTests
           NtfClient
           NtfServerTests
           PostgresSchemaDump
   hs-source-dirs:
       tests
-      apps/smp-server/web
+      apps/smp-server
+      apps/xftp-server
+      apps/common
   default-extensions:
       StrictData
   -- add -fhpc to ghc-options to run tests with coverage
@@ -535,7 +556,7 @@ test-suite simplexmq-test
     , crypton-x509-store
     , crypton-x509-validation
     , directory
-    , file-embed
+    , file-embed >=0.0.10 && <0.1
     , filepath
     , generic-random ==1.5.*
     , hashable
@@ -548,6 +569,7 @@ test-suite simplexmq-test
     , ini
     , iso8601-time
     , main-tester ==0.2.*
+    , memory
     , mtl
     , network
     , QuickCheck ==2.14.*
@@ -563,10 +585,6 @@ test-suite simplexmq-test
     , unliftio
     , unliftio-core
     , unordered-containers
-    , wai
-    , wai-app-static
-    , warp
-    , warp-tls
     , yaml
   default-language: Haskell2010
   if flag(server_postgres)
@@ -575,8 +593,7 @@ test-suite simplexmq-test
     cpp-options: -DdbPostgres
   else
     build-depends:
-        memory
-      , sqlcipher-simple
+        sqlcipher-simple
   if !flag(client_postgres) || flag(client_postgres) || flag(server_postgres)
     build-depends:
         deepseq ==1.4.*

smp-web/.gitignore

@@ -0,0 +1,4 @@
+node_modules/
+dist/
+dist-test/
+package-lock.json

smp-web/cbits/js_random.js

@@ -0,0 +1,14 @@
+addToLibrary({
+  js_random_bytes: function(buf, len) {
+    var bytes = new Uint8Array(len);
+    if (typeof crypto !== 'undefined' && crypto.getRandomValues) {
+      crypto.getRandomValues(bytes);
+    } else {
+      // Node.js fallback
+      var nodeCrypto = require('crypto');
+      var nodeBytes = nodeCrypto.randomBytes(len);
+      bytes.set(nodeBytes);
+    }
+    HEAPU8.set(bytes, buf);
+  }
+});

smp-web/cbits/sha512.c

@@ -0,0 +1,315 @@
+/*
+20080913
+D. J. Bernstein
+Public domain.
+
+SHA-512 implementation from SUPERCOP/NaCl.
+Source: https://bench.cr.yp.to/supercop.html
+  crypto_hashblocks/sha512/ref/blocks.c
+  crypto_hash/sha512/ref/hash.c
+
+Combined into a single file for WASM compilation alongside sntrup761.
+*/
+
+#include "sha512.h"
+
+typedef unsigned long long uint64;
+
+/* -- crypto_hashblocks_sha512 (blocks.c) -- */
+
+static uint64 load_bigendian(const unsigned char *x)
+{
+  return
+      (uint64) (x[7]) \
+  | (((uint64) (x[6])) << 8) \
+  | (((uint64) (x[5])) << 16) \
+  | (((uint64) (x[4])) << 24) \
+  | (((uint64) (x[3])) << 32) \
+  | (((uint64) (x[2])) << 40) \
+  | (((uint64) (x[1])) << 48) \
+  | (((uint64) (x[0])) << 56)
+  ;
+}
+
+static void store_bigendian(unsigned char *x,uint64 u)
+{
+  x[7] = u; u >>= 8;
+  x[6] = u; u >>= 8;
+  x[5] = u; u >>= 8;
+  x[4] = u; u >>= 8;
+  x[3] = u; u >>= 8;
+  x[2] = u; u >>= 8;
+  x[1] = u; u >>= 8;
+  x[0] = u;
+}
+
+#define SHR(x,c) ((x) >> (c))
+#define ROTR(x,c) (((x) >> (c)) | ((x) << (64 - (c))))
+
+#define Ch(x,y,z) ((x & y) ^ (~x & z))
+#define Maj(x,y,z) ((x & y) ^ (x & z) ^ (y & z))
+#define Sigma0(x) (ROTR(x,28) ^ ROTR(x,34) ^ ROTR(x,39))
+#define Sigma1(x) (ROTR(x,14) ^ ROTR(x,18) ^ ROTR(x,41))
+#define sigma0(x) (ROTR(x, 1) ^ ROTR(x, 8) ^ SHR(x,7))
+#define sigma1(x) (ROTR(x,19) ^ ROTR(x,61) ^ SHR(x,6))
+
+#define M(w0,w14,w9,w1) w0 = sigma1(w14) + w9 + sigma0(w1) + w0;
+
+#define EXPAND \
+  M(w0 ,w14,w9 ,w1 ) \
+  M(w1 ,w15,w10,w2 ) \
+  M(w2 ,w0 ,w11,w3 ) \
+  M(w3 ,w1 ,w12,w4 ) \
+  M(w4 ,w2 ,w13,w5 ) \
+  M(w5 ,w3 ,w14,w6 ) \
+  M(w6 ,w4 ,w15,w7 ) \
+  M(w7 ,w5 ,w0 ,w8 ) \
+  M(w8 ,w6 ,w1 ,w9 ) \
+  M(w9 ,w7 ,w2 ,w10) \
+  M(w10,w8 ,w3 ,w11) \
+  M(w11,w9 ,w4 ,w12) \
+  M(w12,w10,w5 ,w13) \
+  M(w13,w11,w6 ,w14) \
+  M(w14,w12,w7 ,w15) \
+  M(w15,w13,w8 ,w0 )
+
+#define F(w,k) \
+  T1 = h + Sigma1(e) + Ch(e,f,g) + k + w; \
+  T2 = Sigma0(a) + Maj(a,b,c); \
+  h = g; \
+  g = f; \
+  f = e; \
+  e = d + T1; \
+  d = c; \
+  c = b; \
+  b = a; \
+  a = T1 + T2;
+
+static int crypto_hashblocks_sha512(unsigned char *statebytes,const unsigned char *in,unsigned long long inlen)
+{
+  uint64 state[8];
+  uint64 a;
+  uint64 b;
+  uint64 c;
+  uint64 d;
+  uint64 e;
+  uint64 f;
+  uint64 g;
+  uint64 h;
+  uint64 T1;
+  uint64 T2;
+
+  a = load_bigendian(statebytes +  0); state[0] = a;
+  b = load_bigendian(statebytes +  8); state[1] = b;
+  c = load_bigendian(statebytes + 16); state[2] = c;
+  d = load_bigendian(statebytes + 24); state[3] = d;
+  e = load_bigendian(statebytes + 32); state[4] = e;
+  f = load_bigendian(statebytes + 40); state[5] = f;
+  g = load_bigendian(statebytes + 48); state[6] = g;
+  h = load_bigendian(statebytes + 56); state[7] = h;
+
+  while (inlen >= 128) {
+    uint64 w0  = load_bigendian(in +   0);
+    uint64 w1  = load_bigendian(in +   8);
+    uint64 w2  = load_bigendian(in +  16);
+    uint64 w3  = load_bigendian(in +  24);
+    uint64 w4  = load_bigendian(in +  32);
+    uint64 w5  = load_bigendian(in +  40);
+    uint64 w6  = load_bigendian(in +  48);
+    uint64 w7  = load_bigendian(in +  56);
+    uint64 w8  = load_bigendian(in +  64);
+    uint64 w9  = load_bigendian(in +  72);
+    uint64 w10 = load_bigendian(in +  80);
+    uint64 w11 = load_bigendian(in +  88);
+    uint64 w12 = load_bigendian(in +  96);
+    uint64 w13 = load_bigendian(in + 104);
+    uint64 w14 = load_bigendian(in + 112);
+    uint64 w15 = load_bigendian(in + 120);
+
+    F(w0 ,0x428a2f98d728ae22ULL)
+    F(w1 ,0x7137449123ef65cdULL)
+    F(w2 ,0xb5c0fbcfec4d3b2fULL)
+    F(w3 ,0xe9b5dba58189dbbcULL)
+    F(w4 ,0x3956c25bf348b538ULL)
+    F(w5 ,0x59f111f1b605d019ULL)
+    F(w6 ,0x923f82a4af194f9bULL)
+    F(w7 ,0xab1c5ed5da6d8118ULL)
+    F(w8 ,0xd807aa98a3030242ULL)
+    F(w9 ,0x12835b0145706fbeULL)
+    F(w10,0x243185be4ee4b28cULL)
+    F(w11,0x550c7dc3d5ffb4e2ULL)
+    F(w12,0x72be5d74f27b896fULL)
+    F(w13,0x80deb1fe3b1696b1ULL)
+    F(w14,0x9bdc06a725c71235ULL)
+    F(w15,0xc19bf174cf692694ULL)
+
+    EXPAND
+
+    F(w0 ,0xe49b69c19ef14ad2ULL)
+    F(w1 ,0xefbe4786384f25e3ULL)
+    F(w2 ,0x0fc19dc68b8cd5b5ULL)
+    F(w3 ,0x240ca1cc77ac9c65ULL)
+    F(w4 ,0x2de92c6f592b0275ULL)
+    F(w5 ,0x4a7484aa6ea6e483ULL)
+    F(w6 ,0x5cb0a9dcbd41fbd4ULL)
+    F(w7 ,0x76f988da831153b5ULL)
+    F(w8 ,0x983e5152ee66dfabULL)
+    F(w9 ,0xa831c66d2db43210ULL)
+    F(w10,0xb00327c898fb213fULL)
+    F(w11,0xbf597fc7beef0ee4ULL)
+    F(w12,0xc6e00bf33da88fc2ULL)
+    F(w13,0xd5a79147930aa725ULL)
+    F(w14,0x06ca6351e003826fULL)
+    F(w15,0x142929670a0e6e70ULL)
+
+    EXPAND
+
+    F(w0 ,0x27b70a8546d22ffcULL)
+    F(w1 ,0x2e1b21385c26c926ULL)
+    F(w2 ,0x4d2c6dfc5ac42aedULL)
+    F(w3 ,0x53380d139d95b3dfULL)
+    F(w4 ,0x650a73548baf63deULL)
+    F(w5 ,0x766a0abb3c77b2a8ULL)
+    F(w6 ,0x81c2c92e47edaee6ULL)
+    F(w7 ,0x92722c851482353bULL)
+    F(w8 ,0xa2bfe8a14cf10364ULL)
+    F(w9 ,0xa81a664bbc423001ULL)
+    F(w10,0xc24b8b70d0f89791ULL)
+    F(w11,0xc76c51a30654be30ULL)
+    F(w12,0xd192e819d6ef5218ULL)
+    F(w13,0xd69906245565a910ULL)
+    F(w14,0xf40e35855771202aULL)
+    F(w15,0x106aa07032bbd1b8ULL)
+
+    EXPAND
+
+    F(w0 ,0x19a4c116b8d2d0c8ULL)
+    F(w1 ,0x1e376c085141ab53ULL)
+    F(w2 ,0x2748774cdf8eeb99ULL)
+    F(w3 ,0x34b0bcb5e19b48a8ULL)
+    F(w4 ,0x391c0cb3c5c95a63ULL)
+    F(w5 ,0x4ed8aa4ae3418acbULL)
+    F(w6 ,0x5b9cca4f7763e373ULL)
+    F(w7 ,0x682e6ff3d6b2b8a3ULL)
+    F(w8 ,0x748f82ee5defb2fcULL)
+    F(w9 ,0x78a5636f43172f60ULL)
+    F(w10,0x84c87814a1f0ab72ULL)
+    F(w11,0x8cc702081a6439ecULL)
+    F(w12,0x90befffa23631e28ULL)
+    F(w13,0xa4506cebde82bde9ULL)
+    F(w14,0xbef9a3f7b2c67915ULL)
+    F(w15,0xc67178f2e372532bULL)
+
+    EXPAND
+
+    F(w0 ,0xca273eceea26619cULL)
+    F(w1 ,0xd186b8c721c0c207ULL)
+    F(w2 ,0xeada7dd6cde0eb1eULL)
+    F(w3 ,0xf57d4f7fee6ed178ULL)
+    F(w4 ,0x06f067aa72176fbaULL)
+    F(w5 ,0x0a637dc5a2c898a6ULL)
+    F(w6 ,0x113f9804bef90daeULL)
+    F(w7 ,0x1b710b35131c471bULL)
+    F(w8 ,0x28db77f523047d84ULL)
+    F(w9 ,0x32caab7b40c72493ULL)
+    F(w10,0x3c9ebe0a15c9bebcULL)
+    F(w11,0x431d67c49c100d4cULL)
+    F(w12,0x4cc5d4becb3e42b6ULL)
+    F(w13,0x597f299cfc657e2aULL)
+    F(w14,0x5fcb6fab3ad6faecULL)
+    F(w15,0x6c44198c4a475817ULL)
+
+    a += state[0];
+    b += state[1];
+    c += state[2];
+    d += state[3];
+    e += state[4];
+    f += state[5];
+    g += state[6];
+    h += state[7];
+
+    state[0] = a;
+    state[1] = b;
+    state[2] = c;
+    state[3] = d;
+    state[4] = e;
+    state[5] = f;
+    state[6] = g;
+    state[7] = h;
+
+    in += 128;
+    inlen -= 128;
+  }
+
+  store_bigendian(statebytes +  0,state[0]);
+  store_bigendian(statebytes +  8,state[1]);
+  store_bigendian(statebytes + 16,state[2]);
+  store_bigendian(statebytes + 24,state[3]);
+  store_bigendian(statebytes + 32,state[4]);
+  store_bigendian(statebytes + 40,state[5]);
+  store_bigendian(statebytes + 48,state[6]);
+  store_bigendian(statebytes + 56,state[7]);
+
+  return inlen;
+}
+
+/* -- crypto_hash_sha512 (hash.c) -- */
+
+static const unsigned char iv[64] = {
+  0x6a,0x09,0xe6,0x67,0xf3,0xbc,0xc9,0x08,
+  0xbb,0x67,0xae,0x85,0x84,0xca,0xa7,0x3b,
+  0x3c,0x6e,0xf3,0x72,0xfe,0x94,0xf8,0x2b,
+  0xa5,0x4f,0xf5,0x3a,0x5f,0x1d,0x36,0xf1,
+  0x51,0x0e,0x52,0x7f,0xad,0xe6,0x82,0xd1,
+  0x9b,0x05,0x68,0x8c,0x2b,0x3e,0x6c,0x1f,
+  0x1f,0x83,0xd9,0xab,0xfb,0x41,0xbd,0x6b,
+  0x5b,0xe0,0xcd,0x19,0x13,0x7e,0x21,0x79
+};
+
+void crypto_hash_sha512(unsigned char *out,
+                        const unsigned char *in,
+                        unsigned long long inlen)
+{
+  unsigned char h[64];
+  unsigned char padded[256];
+  int i;
+  unsigned long long bytes = inlen;
+
+  for (i = 0;i < 64;++i) h[i] = iv[i];
+
+  crypto_hashblocks_sha512(h,in,inlen);
+  in += inlen;
+  inlen &= 127;
+  in -= inlen;
+
+  for (i = 0;i < (int)inlen;++i) padded[i] = in[i];
+  padded[inlen] = 0x80;
+
+  if (inlen < 112) {
+    for (i = inlen + 1;i < 119;++i) padded[i] = 0;
+    padded[119] = bytes >> 61;
+    padded[120] = bytes >> 53;
+    padded[121] = bytes >> 45;
+    padded[122] = bytes >> 37;
+    padded[123] = bytes >> 29;
+    padded[124] = bytes >> 21;
+    padded[125] = bytes >> 13;
+    padded[126] = bytes >> 5;
+    padded[127] = bytes << 3;
+    crypto_hashblocks_sha512(h,padded,128);
+  } else {
+    for (i = inlen + 1;i < 247;++i) padded[i] = 0;
+    padded[247] = bytes >> 61;
+    padded[248] = bytes >> 53;
+    padded[249] = bytes >> 45;
+    padded[250] = bytes >> 37;
+    padded[251] = bytes >> 29;
+    padded[252] = bytes >> 21;
+    padded[253] = bytes >> 13;
+    padded[254] = bytes >> 5;
+    padded[255] = bytes << 3;
+    crypto_hashblocks_sha512(h,padded,256);
+  }
+
+  for (i = 0;i < 64;++i) out[i] = h[i];
+}

smp-web/cbits/sntrup761.d.mts

@@ -0,0 +1,11 @@
+interface Sntrup761Module {
+  _sntrup761_wasm_keypair(pk: number, sk: number): void
+  _sntrup761_wasm_enc(c: number, k: number, pk: number): void
+  _sntrup761_wasm_dec(k: number, c: number, sk: number): void
+  _malloc(size: number): number
+  _free(ptr: number): void
+  HEAPU8: Uint8Array
+}
+
+declare function createSntrup761(): Promise<Sntrup761Module>
+export default createSntrup761

smp-web/cbits/sntrup761_wasm.c

@@ -0,0 +1,34 @@
+/*
+ * WASM wrapper for sntrup761.
+ * Provides JS-callable functions with RNG from JS imports.
+ *
+ * Build: emcc sntrup761_wasm.c sntrup761.c sha512.c -O2 -o sntrup761.js \
+ *        -s EXPORTED_FUNCTIONS='["_sntrup761_wasm_keypair","_sntrup761_wasm_enc","_sntrup761_wasm_dec","_malloc","_free"]' \
+ *        -s EXPORTED_RUNTIME_METHODS='["ccall","cwrap"]'
+ */
+
+#include "sntrup761.h"
+#include <stdlib.h>
+
+/* Import RNG from JS environment */
+extern void js_random_bytes(unsigned char *buf, int len);
+
+/* RNG callback adapter for sntrup761 */
+static void wasm_random(void *ctx, size_t length, uint8_t *dst) {
+  (void)ctx;
+  js_random_bytes(dst, (int)length);
+}
+
+/* JS-callable wrappers */
+
+void sntrup761_wasm_keypair(unsigned char *pk, unsigned char *sk) {
+  sntrup761_keypair(pk, sk, NULL, wasm_random);
+}
+
+void sntrup761_wasm_enc(unsigned char *c, unsigned char *k, const unsigned char *pk) {
+  sntrup761_enc(c, k, pk, NULL, wasm_random);
+}
+
+void sntrup761_wasm_dec(unsigned char *k, const unsigned char *c, const unsigned char *sk) {
+  sntrup761_dec(k, c, sk);
+}

smp-web/package.json

@@ -0,0 +1,35 @@
+{
+  "name": "@simplex-chat/smp-web",
+  "version": "0.1.0",
+  "description": "SMP protocol client for web/browser environments",
+  "license": "AGPL-3.0-only",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/simplex-chat/simplexmq.git",
+    "directory": "smp-web"
+  },
+  "type": "module",
+  "files": [
+    "src",
+    "dist"
+  ],
+  "scripts": {
+    "build:wasm": "mkdir -p dist/wasm && npx emcc cbits/sntrup761_wasm.c ../cbits/sntrup761.c cbits/sha512.c -I../cbits -O2 -o dist/wasm/sntrup761.mjs -s EXPORTED_FUNCTIONS='[\"_sntrup761_wasm_keypair\",\"_sntrup761_wasm_enc\",\"_sntrup761_wasm_dec\",\"_malloc\",\"_free\"]' -s EXPORTED_RUNTIME_METHODS='[\"ccall\",\"cwrap\",\"HEAPU8\"]' -s MODULARIZE=1 -s EXPORT_NAME='createSntrup761' -s ALLOW_MEMORY_GROWTH=1 -s ENVIRONMENT='web,node' --js-library cbits/js_random.js && cp cbits/sntrup761.d.mts dist/wasm/",
+    "build:ts": "tsc",
+    "build:test": "tsc -p tsconfig.test.json",
+    "build": "npm run build:wasm && npm run build:ts && npm run build:test"
+  },
+  "dependencies": {
+    "@noble/ciphers": "^2.2.0",
+    "@noble/curves": "^2.2.0",
+    "@noble/hashes": "^1.5.0",
+    "@simplex-chat/xftp-web": "file:../xftp-web",
+    "emsdk": "^0.4.0"
+  },
+  "devDependencies": {
+    "@types/node": "^25.5.0",
+    "@types/ws": "^8.18.1",
+    "typescript": "^5.4.0",
+    "ws": "^8.0.0"
+  }
+}

smp-web/src/agent/message.ts

@@ -0,0 +1,207 @@
+// Agent message encoding/decoding.
+// Mirrors: Simplex.Messaging.Agent.Protocol (AgentMsgEnvelope, AgentMessage, APrivHeader, AMessage)
+
+import {
+  Decoder, concatBytes,
+  encodeBytes, decodeBytes,
+  encodeLarge, decodeLarge,
+  encodeInt64, decodeInt64,
+  encodeWord16, decodeWord16,
+  encodeMaybe, decodeMaybe,
+  encodeNonEmpty, decodeNonEmpty,
+} from "@simplex-chat/xftp-web/dist/protocol/encoding.js"
+
+// -- Constants (Agent/Protocol.hs:318-319)
+
+export const currentSMPAgentVersion = 7
+
+// -- AMessage (Agent/Protocol.hs:1001-1020)
+
+export type AMessage =
+  | {type: "HELLO"}
+  | {type: "A_MSG", body: Uint8Array}
+  | {type: "A_RCVD", receipts: AMessageReceipt[]}  // NonEmpty
+  | {type: "EREADY", lastDecryptedMsgId: bigint}
+
+// Agent/Protocol.hs:1040-1045
+export interface AMessageReceipt {
+  agentMsgId: bigint       // Int64
+  msgHash: Uint8Array      // ByteString (32-byte SHA-256)
+  rcptInfo: Uint8Array     // MsgReceiptInfo (ByteString, Large-encoded)
+}
+
+// Agent/Protocol.hs:1078-1100
+export function encodeAMessage(msg: AMessage): Uint8Array {
+  switch (msg.type) {
+    case "HELLO": return new Uint8Array([0x48]) // "H"
+    case "A_MSG": return concatBytes(new Uint8Array([0x4D]), msg.body) // "M" + Tail
+    case "A_RCVD": return concatBytes(new Uint8Array([0x56]), encodeNonEmpty(encodeAMessageReceipt, msg.receipts)) // "V" + NonEmpty
+    case "EREADY": return concatBytes(new Uint8Array([0x45]), encodeInt64(msg.lastDecryptedMsgId)) // "E" + Int64
+  }
+}
+
+export function decodeAMessage(d: Decoder): AMessage {
+  const tag = d.anyByte()
+  switch (tag) {
+    case 0x48: return {type: "HELLO"} // 'H'
+    case 0x4D: return {type: "A_MSG", body: d.takeAll()} // 'M' + Tail
+    case 0x56: return {type: "A_RCVD", receipts: decodeNonEmpty(decodeAMessageReceipt, d)} // 'V'
+    case 0x45: return {type: "EREADY", lastDecryptedMsgId: decodeInt64(d)} // 'E'
+    // Queue management tags (not needed for chat messages, but recognized for decoding)
+    case 0x51: { // 'Q'
+      const sub = d.anyByte()
+      switch (sub) {
+        case 0x43: // 'C' = A_QCONT
+        case 0x41: // 'A' = QADD
+        case 0x4B: // 'K' = QKEY
+        case 0x55: // 'U' = QUSE
+        case 0x54: // 'T' = QTEST
+          throw new Error("decodeAMessage: queue management message (Q" + String.fromCharCode(sub) + ") not implemented")
+        default:
+          throw new Error("decodeAMessage: unknown Q-subtag " + sub)
+      }
+    }
+    default:
+      throw new Error("decodeAMessage: unknown tag " + tag)
+  }
+}
+
+// Agent/Protocol.hs:1106-1111
+function encodeAMessageReceipt(r: AMessageReceipt): Uint8Array {
+  return concatBytes(encodeInt64(r.agentMsgId), encodeBytes(r.msgHash), encodeLarge(r.rcptInfo))
+}
+
+function decodeAMessageReceipt(d: Decoder): AMessageReceipt {
+  return {agentMsgId: decodeInt64(d), msgHash: decodeBytes(d), rcptInfo: decodeLarge(d)}
+}
+
+// -- APrivHeader (Agent/Protocol.hs:946-957)
+
+export interface APrivHeader {
+  sndMsgId: bigint         // AgentMsgId = Int64
+  prevMsgHash: Uint8Array  // MsgHash = ByteString
+}
+
+export function encodeAPrivHeader(h: APrivHeader): Uint8Array {
+  return concatBytes(encodeInt64(h.sndMsgId), encodeBytes(h.prevMsgHash))
+}
+
+export function decodeAPrivHeader(d: Decoder): APrivHeader {
+  return {sndMsgId: decodeInt64(d), prevMsgHash: decodeBytes(d)}
+}
+
+// -- AgentMessage (Agent/Protocol.hs:866-888)
+
+export type AgentMessage =
+  | {type: "connInfo", cInfo: Uint8Array}
+  | {type: "connInfoReply", smpQueues: Uint8Array[], cInfo: Uint8Array}  // NonEmpty raw-encoded SMPQueueInfo
+  | {type: "ratchetInfo", info: Uint8Array}
+  | {type: "message", header: APrivHeader, msg: AMessage}
+
+export function encodeAgentMessage(msg: AgentMessage): Uint8Array {
+  switch (msg.type) {
+    case "connInfo":
+      return concatBytes(new Uint8Array([0x49]), msg.cInfo) // 'I' + Tail
+    case "connInfoReply":
+      // 'D' + NonEmpty SMPQueueInfo + Tail cInfo
+      // SMPQueueInfo encoding is complex; for now encode the raw bytes
+      return concatBytes(
+        new Uint8Array([0x44]),
+        encodeNonEmpty(b => b, msg.smpQueues),
+        msg.cInfo,
+      )
+    case "ratchetInfo":
+      return concatBytes(new Uint8Array([0x52]), msg.info) // 'R' + Tail
+    case "message":
+      return concatBytes(new Uint8Array([0x4D]), encodeAPrivHeader(msg.header), encodeAMessage(msg.msg)) // 'M' + header + msg
+  }
+}
+
+export function decodeAgentMessage(d: Decoder): AgentMessage {
+  const tag = d.anyByte()
+  switch (tag) {
+    case 0x49: return {type: "connInfo", cInfo: d.takeAll()} // 'I' + Tail
+    case 0x44: { // 'D'
+      // NonEmpty SMPQueueInfo is complex to decode; skip for now, just capture raw
+      throw new Error("decodeAgentMessage: connInfoReply ('D') not implemented")
+    }
+    case 0x52: return {type: "ratchetInfo", info: d.takeAll()} // 'R' + Tail
+    case 0x4D: return {type: "message", header: decodeAPrivHeader(d), msg: decodeAMessage(d)} // 'M'
+    default:
+      throw new Error("decodeAgentMessage: unknown tag " + tag)
+  }
+}
+
+// -- AgentMsgEnvelope (Agent/Protocol.hs:812-861)
+
+export type AgentMsgEnvelope =
+  | {type: "confirmation", agentVersion: number, e2eEncryption: Uint8Array | null, encConnInfo: Uint8Array}
+  | {type: "envelope", agentVersion: number, encAgentMessage: Uint8Array}
+  | {type: "invitation", agentVersion: number, connReqBytes: Uint8Array, connInfo: Uint8Array}
+  | {type: "ratchetKey", agentVersion: number, e2eEncryption: Uint8Array, info: Uint8Array}
+
+// Agent/Protocol.hs:835-843
+export function encodeAgentMsgEnvelope(env: AgentMsgEnvelope): Uint8Array {
+  switch (env.type) {
+    case "confirmation":
+      // (agentVersion, 'C', Maybe SndE2ERatchetParams, Tail encConnInfo)
+      return concatBytes(
+        encodeWord16(env.agentVersion),
+        new Uint8Array([0x43]), // 'C'
+        encodeMaybe(b => b, env.e2eEncryption), // e2eEncryption is already smpEncoded bytes or null
+        env.encConnInfo, // Tail
+      )
+    case "envelope":
+      // (agentVersion, 'M', Tail encAgentMessage)
+      return concatBytes(
+        encodeWord16(env.agentVersion),
+        new Uint8Array([0x4D]), // 'M'
+        env.encAgentMessage, // Tail
+      )
+    case "invitation":
+      // (agentVersion, 'I', Large connReqBytes, Tail connInfo)
+      return concatBytes(
+        encodeWord16(env.agentVersion),
+        new Uint8Array([0x49]), // 'I'
+        encodeLarge(env.connReqBytes),
+        env.connInfo, // Tail
+      )
+    case "ratchetKey":
+      // (agentVersion, 'R', e2eEncryption, Tail info)
+      return concatBytes(
+        encodeWord16(env.agentVersion),
+        new Uint8Array([0x52]), // 'R'
+        env.e2eEncryption, // already smpEncoded
+        env.info, // Tail
+      )
+  }
+}
+
+// Agent/Protocol.hs:844-861
+export function decodeAgentMsgEnvelope(d: Decoder): AgentMsgEnvelope {
+  const agentVersion = decodeWord16(d)
+  const tag = d.anyByte()
+  switch (tag) {
+    case 0x43: // 'C' Confirmation
+      // e2eEncryption_ is Maybe (SndE2ERatchetParams 'X448), encConnInfo is Tail
+      // Full parsing of E2ERatchetParams needed to split the boundary — not implemented in spike
+      throw new Error("decodeAgentMsgEnvelope: confirmation ('C') not fully implemented")
+    case 0x4D: // 'M' Message envelope
+      return {type: "envelope", agentVersion, encAgentMessage: d.takeAll()} // Tail
+    case 0x49: { // 'I' Invitation
+      const connReqBytes = decodeLarge(d)
+      const connInfo = d.takeAll() // Tail
+      return {type: "invitation", agentVersion, connReqBytes, connInfo}
+    }
+    case 0x52: { // 'R' RatchetKey
+      // e2eEncryption is an E2ERatchetParams — variable-length, not Tail
+      // For now, capture remaining minus nothing (since info is Tail and comes last)
+      // This is tricky: e2eEncryption is smpEncoded E2ERatchetParams, info is Tail
+      // We can't easily split without knowing the E2ERatchetParams length
+      // For the spike, just capture all remaining as raw
+      throw new Error("decodeAgentMsgEnvelope: ratchetKey ('R') not fully implemented")
+    }
+    default:
+      throw new Error("decodeAgentMsgEnvelope: unknown tag " + tag)
+  }
+}

smp-web/src/agent/protocol.ts

@@ -0,0 +1,247 @@
+// Agent protocol types and short link parsing.
+// Mirrors: Simplex.Messaging.Agent.Protocol
+
+import {base64urlDecode} from "@simplex-chat/xftp-web/dist/protocol/description.js"
+import {
+  Decoder, decodeBytes, decodeLarge, decodeWord16, decodeBool,
+} from "@simplex-chat/xftp-web/dist/protocol/encoding.js"
+
+// -- Short link types (Agent/Protocol.hs:1462-1470)
+
+export type ShortLinkScheme = "simplex" | "https"
+
+export type ContactConnType = "contact" | "channel" | "group" | "relay"
+
+export interface ProtocolServer {
+  hosts: Uint8Array[] // NonEmpty, each is the strEncoded host bytes
+  port: Uint8Array
+  keyHash: Uint8Array
+}
+
+export type ConnShortLink =
+  | {mode: "invitation", scheme: ShortLinkScheme, server: ProtocolServer, linkId: Uint8Array, linkKey: Uint8Array}
+  | {mode: "contact", scheme: ShortLinkScheme, connType: ContactConnType, server: ProtocolServer, linkKey: Uint8Array}
+
+// -- ProtocolServer binary encoding (Protocol.hs:1264-1269)
+// smpEncode (host, port, keyHash)
+// host: NonEmpty TransportHost = smpEncodeList (1-byte count + each as ByteString)
+// port: String = ByteString (1-byte len + bytes)
+// keyHash: KeyHash = ByteString (1-byte len + bytes)
+
+export function decodeProtocolServer(d: Decoder): ProtocolServer {
+  const hostCount = d.anyByte()
+  if (hostCount === 0) throw new Error("empty server host list")
+  const hosts: Uint8Array[] = []
+  for (let i = 0; i < hostCount; i++) hosts.push(decodeBytes(d))
+  const port = decodeBytes(d)
+  const keyHash = decodeBytes(d)
+  return {hosts, port, keyHash}
+}
+
+// -- ConnShortLink binary encoding (Agent/Protocol.hs:1631-1649)
+// Contact: smpEncode (CMContact, ctTypeChar, srv, linkKey)
+// Invitation: smpEncode (CMInvitation, srv, linkId, linkKey)
+
+export interface ConnShortLinkBinary {
+  mode: "contact" | "invitation"
+  connType?: ContactConnType
+  server: ProtocolServer
+  linkId?: Uint8Array
+  linkKey: Uint8Array
+}
+
+const ctTypeFromByte: Record<number, ContactConnType> = {
+  0x41: "contact",  // 'A'
+  0x43: "channel",  // 'C'
+  0x47: "group",    // 'G'
+  0x52: "relay",    // 'R'
+}
+
+export function decodeConnShortLink(d: Decoder): ConnShortLinkBinary {
+  const mode = d.anyByte()
+  if (mode === 0x49) {
+    // Invitation: (srv, linkId, linkKey)
+    const server = decodeProtocolServer(d)
+    const linkId = decodeBytes(d)
+    const linkKey = decodeBytes(d)
+    return {mode: "invitation", server, linkId, linkKey}
+  } else if (mode === 0x43) {
+    // Contact: (ctTypeChar, srv, linkKey)
+    const ctByte = d.anyByte()
+    const connType = ctTypeFromByte[ctByte]
+    if (!connType) throw new Error("unknown contact type: 0x" + ctByte.toString(16))
+    const server = decodeProtocolServer(d)
+    const linkKey = decodeBytes(d)
+    return {mode: "contact", connType, server, linkKey}
+  }
+  throw new Error("unknown ConnShortLink mode: 0x" + mode.toString(16))
+}
+
+// -- OwnerAuth (Agent/Protocol.hs:1793-1800)
+// Outer ByteString wrapping inner: (ownerId, ownerKey, authOwnerSig)
+
+export interface OwnerAuth {
+  ownerId: Uint8Array
+  ownerKey: Uint8Array
+  authOwnerSig: Uint8Array
+}
+
+export function decodeOwnerAuth(d: Decoder): OwnerAuth {
+  const inner = decodeBytes(d)
+  const id = new Decoder(inner)
+  const ownerId = decodeBytes(id)
+  const ownerKey = decodeBytes(id)
+  const authOwnerSig = decodeBytes(id)
+  return {ownerId, ownerKey, authOwnerSig}
+}
+
+// -- UserLinkData (Agent/Protocol.hs:1891-1894)
+// If first byte is 0xFF, read Large; otherwise it's a ByteString (1-byte length)
+
+export function decodeUserLinkData(d: Decoder): Uint8Array {
+  const firstByte = d.anyByte()
+  if (firstByte === 0xFF) return decodeLarge(d)
+  return d.take(firstByte)
+}
+
+// -- UserContactData (Agent/Protocol.hs:1881-1889)
+
+export interface UserContactData {
+  direct: boolean
+  owners: OwnerAuth[]
+  relays: ConnShortLinkBinary[]
+  userData: Uint8Array
+}
+
+export function decodeUserContactData(d: Decoder): UserContactData {
+  const direct = decodeBool(d)
+  const ownerCount = d.anyByte()
+  const owners: OwnerAuth[] = []
+  for (let i = 0; i < ownerCount; i++) owners.push(decodeOwnerAuth(d))
+  const relayCount = d.anyByte()
+  const relays: ConnShortLinkBinary[] = []
+  for (let i = 0; i < relayCount; i++) relays.push(decodeConnShortLink(d))
+  const userData = decodeUserLinkData(d)
+  return {direct, owners, relays, userData}
+}
+
+// -- ConnLinkData (Agent/Protocol.hs:1838-1855)
+// Contact: 'C' + versionRange + UserContactData
+
+export interface ConnLinkDataContact {
+  mode: "contact"
+  agentVRange: {min: number; max: number}
+  userContactData: UserContactData
+}
+
+export function decodeConnLinkData(d: Decoder): ConnLinkDataContact {
+  const modeChar = d.anyByte()
+  if (modeChar !== 0x43) throw new Error("expected Contact mode 'C' (0x43), got 0x" + modeChar.toString(16))
+  const min = decodeWord16(d)
+  const max = decodeWord16(d)
+  const userContactData = decodeUserContactData(d)
+  return {mode: "contact", agentVRange: {min, max}, userContactData}
+}
+
+// -- FixedLinkData (Agent/Protocol.hs:1830-1836)
+// Encoding: smpEncode (agentVRange, rootKey, linkConnReq) <> maybe "" smpEncode linkEntityId
+// rootKey is DER-encoded Ed25519 public key (ByteString: 1-byte len + 44 bytes DER)
+// linkConnReq is ConnectionRequestUri (variable length, not length-prefixed)
+// For now, we parse agentVRange + rootKey and keep the rest as raw bytes.
+// Full ConnectionRequestUri parsing is future work.
+
+export interface FixedLinkData {
+  agentVRange: {min: number; max: number}
+  rootKey: Uint8Array // DER-encoded Ed25519 public key (44 bytes)
+  rest: Uint8Array    // raw linkConnReq + linkEntityId bytes
+}
+
+export function decodeFixedLinkData(d: Decoder): FixedLinkData {
+  const min = decodeWord16(d)
+  const max = decodeWord16(d)
+  const rootKey = decodeBytes(d)
+  const rest = d.takeAll()
+  return {agentVRange: {min, max}, rootKey, rest}
+}
+
+// -- Profile extraction
+
+export function parseProfile(userData: Uint8Array): unknown {
+  if (userData.length > 0 && userData[0] === 0x58) {
+    throw new Error("zstd-compressed profile not yet supported")
+  }
+  return JSON.parse(new TextDecoder().decode(userData))
+}
+
+// -- Short link URI parsing (below) --
+
+export interface ShortLinkServer {
+  hosts: string[]
+  port: string
+  keyHash: Uint8Array
+}
+
+export type ConnShortLinkURI =
+  | {mode: "invitation", scheme: ShortLinkScheme, server: ShortLinkServer, linkId: Uint8Array, linkKey: Uint8Array}
+  | {mode: "contact", scheme: ShortLinkScheme, connType: ContactConnType, server: ShortLinkServer, linkKey: Uint8Array}
+
+const ctTypeFromChar: Record<string, ContactConnType> = {
+  a: "contact",
+  c: "channel",
+  g: "group",
+  r: "relay",
+}
+
+// Mirrors strP for AConnShortLink (Agent/Protocol.hs:1596-1629)
+export function connShortLinkStrP(uri: string): ConnShortLinkURI {
+  let scheme: ShortLinkScheme
+  let firstHost: string | null = null
+  let rest: string
+
+  if (uri.startsWith("simplex:")) {
+    scheme = "simplex"
+    rest = uri.slice("simplex:".length)
+  } else if (uri.startsWith("https://")) {
+    scheme = "https"
+    const afterScheme = uri.slice("https://".length)
+    const slashIdx = afterScheme.indexOf("/")
+    if (slashIdx < 0) throw new Error("bad short link: no path")
+    firstHost = afterScheme.slice(0, slashIdx)
+    rest = afterScheme.slice(slashIdx)
+  } else {
+    throw new Error("bad short link scheme")
+  }
+
+  if (rest[0] !== "/") throw new Error("bad short link: expected /")
+  const typeChar = rest[1]
+  const hashIdx = rest.indexOf("#")
+  if (hashIdx < 0) throw new Error("bad short link: no #")
+  const afterHash = rest.slice(hashIdx + 1)
+
+  const qIdx = afterHash.indexOf("?")
+  const fragment = qIdx >= 0 ? afterHash.slice(0, qIdx) : afterHash
+  const queryStr = qIdx >= 0 ? afterHash.slice(qIdx + 1) : ""
+  const params = new URLSearchParams(queryStr)
+
+  const hParam = params.get("h")
+  const additionalHosts = hParam ? hParam.split(",") : []
+  const allHosts = firstHost ? [firstHost, ...additionalHosts] : additionalHosts
+  if (allHosts.length === 0) throw new Error("short link without server")
+
+  const port = params.get("p") ?? ""
+  const keyHash = params.has("c") ? base64urlDecode(params.get("c")!) : new Uint8Array(0)
+  const server: ShortLinkServer = {hosts: allHosts, port, keyHash}
+
+  if (typeChar === "i") {
+    const slashIdx = fragment.indexOf("/")
+    if (slashIdx < 0) throw new Error("invitation link must have linkId/linkKey")
+    const linkId = base64urlDecode(fragment.slice(0, slashIdx))
+    const linkKey = base64urlDecode(fragment.slice(slashIdx + 1))
+    return {mode: "invitation", scheme, server, linkId, linkKey}
+  } else {
+    const connType = ctTypeFromChar[typeChar]
+    if (!connType) throw new Error("unknown contact type: " + typeChar)
+    const linkKey = base64urlDecode(fragment)
+    return {mode: "contact", scheme, connType, server, linkKey}
+  }
+}

smp-web/src/client.ts

@@ -0,0 +1,490 @@
+// SMP client: command/response correlation, authentication, typed async API.
+// Mirrors: Simplex.Messaging.Client
+
+import {
+  encodeTransmission, encodeTransmissionForAuth, authTransmission,
+  tEncodeBatch1, tEncodeForBatch, batchTransmissions, tEncode,
+  tParse, tDecodeClient, protocolError, encodePING,
+  decodeResponse, paddedProxiedTLength, encodePRXY, encodePFWD,
+  type AuthKey, type SMPResponse, type RawTransmission,
+  encodeNEW, encodeKEY, encodeSKEY, encodeSUB, encodeACK,
+  encodeSEND, encodeOFF, encodeDEL, encodeGET, encodeQUE, encodeLGET,
+  type IDSResponse, type MSGResponse,
+} from "./protocol.js"
+import {
+  connectSMP,
+  type SMPConnection,
+} from "./transport/websockets.js"
+import {SMP_BLOCK_SIZE} from "./transport.js"
+import {sbEncryptBlock, sbDecryptBlock, cbAuthenticator, reverseNonce, cbDecryptNoPad} from "./crypto.js"
+import {blockPad, blockUnpad} from "@simplex-chat/xftp-web/dist/protocol/transmission.js"
+import {Decoder} from "@simplex-chat/xftp-web/dist/protocol/encoding.js"
+import {generateX25519KeyPair, x25519KeyPairFromPrivate, dh, encodePubKeyX25519} from "@simplex-chat/xftp-web/dist/crypto/keys.js"
+import {cbEncrypt, cbDecrypt} from "@simplex-chat/xftp-web/dist/crypto/secretbox.js"
+import {extractSignedKey} from "@simplex-chat/xftp-web/dist/protocol/handshake.js"
+
+// -- Error types (Client.hs:741-770)
+
+// ProxiedRelay (Client.hs:1095-1100)
+export interface ProxiedRelay {
+  sessionId: Uint8Array
+  version: number       // negotiated version with relay
+  basicAuth: Uint8Array | null
+  relayKey: Uint8Array  // relay's X25519 public key (raw 32 bytes)
+}
+
+// ProxyClientError (Client.hs:1102-1109)
+export type ProxyClientError =
+  | {type: "ProxyProtocolError", error: string}
+  | {type: "ProxyUnexpectedResponse", response: string}
+  | {type: "ProxyResponseError", error: string}
+
+export type SMPClientError =
+  | {type: "PROTOCOL", error: string}    // ERR response from server
+  | {type: "RESPONSE", error: string}    // failed to parse response
+  | {type: "UNEXPECTED", raw: string}    // wrong response type for command
+  | {type: "TIMEOUT"}                    // response timeout
+  | {type: "NETWORK", error: string}     // connection failure
+  | {type: "TRANSPORT", error: string}   // handshake/transport error
+
+// -- SMPClient
+
+export interface SMPClient {
+  readonly sessionId: Uint8Array
+  readonly smpVersion: number
+  readonly serverPubKey: Uint8Array
+
+  // Core: send pre-encoded command, await correlated response
+  sendCommand(privKey: AuthKey | null, entityId: Uint8Array, command: Uint8Array): Promise<SMPResponse>
+
+  // High-level commands
+  createQueue(authKeyPair: {publicKey: Uint8Array, privateKey: Uint8Array}, dhKey: Uint8Array, subscribe: boolean): Promise<IDSResponse>
+  subscribeQueue(privKey: AuthKey, rcvId: Uint8Array): Promise<void>
+  getMessage(privKey: AuthKey, rcvId: Uint8Array): Promise<MSGResponse | null>
+  sendMessage(privKey: AuthKey | null, sndId: Uint8Array, notification: boolean, msg: Uint8Array): Promise<void>
+  ackMessage(privKey: AuthKey, rcvId: Uint8Array, msgId: Uint8Array): Promise<void>
+  secureQueue(privKey: AuthKey, rcvId: Uint8Array, senderKey: Uint8Array): Promise<void>
+  secureSndQueue(privKey: AuthKey, sndId: Uint8Array): Promise<void>
+  getQueueLink(linkId: Uint8Array): Promise<SMPResponse>
+  deleteQueue(privKey: AuthKey, rcvId: Uint8Array): Promise<void>
+  suspendQueue(privKey: AuthKey, rcvId: Uint8Array): Promise<void>
+
+  // Batch commands (Client.hs:840-845, 1062-1065)
+  subscribeQueues(queues: Array<{rcvId: Uint8Array, privKey: AuthKey}>): Promise<void[]>
+  deleteQueues(queues: Array<{rcvId: Uint8Array, privKey: AuthKey}>): Promise<void[]>
+
+  // Proxy commands (Client.hs:1069-1206)
+  connectProxiedRelay(relayHosts: string[], relayPort: string, relayKeyHash: Uint8Array, basicAuth: Uint8Array | null): Promise<ProxiedRelay>
+  proxySMPCommand(relay: ProxiedRelay, privKey: AuthKey | null, entityId: Uint8Array, command: Uint8Array): Promise<SMPResponse>
+  proxySendMessage(relay: ProxiedRelay, privKey: AuthKey | null, sndId: Uint8Array, notification: boolean, msg: Uint8Array): Promise<void>
+
+  close(): void
+}
+
+interface PendingRequest {
+  resolve: (resp: SMPResponse) => void
+  reject: (err: SMPClientError) => void
+  timer: ReturnType<typeof setTimeout>
+}
+
+function toHex(bytes: Uint8Array): string {
+  return Array.from(bytes, b => b.toString(16).padStart(2, "0")).join("")
+}
+
+export async function createSMPClient(
+  url: string,
+  keyHash: Uint8Array,
+  onMessage: (entityId: Uint8Array, msg: SMPResponse) => void,
+  onDisconnected: () => void,
+  config?: {timeout?: number, pingInterval?: number, pingMaxCount?: number, wsOptions?: object},
+): Promise<SMPClient> {
+  const timeout_ = config?.timeout ?? 10_000
+  const pingInterval = config?.pingInterval ?? 600_000
+  const pingMaxCount = config?.pingMaxCount ?? 3
+
+  const conn = await connectSMP(url, keyHash, config?.wsOptions)
+  if (!conn.serverPubKey) throw new Error("createSMPClient: server has no auth key")
+
+  const serverPubKey = conn.serverPubKey
+  const pending = new Map<string, PendingRequest>()
+  let closed = false
+  let pingTimer: ReturnType<typeof setInterval> | null = null
+  let timeoutCount = 0
+
+  // -- Receive loop
+
+  function onBlock(data: ArrayBuffer | Buffer) {
+    if (closed) return
+    timeoutCount = 0
+    try {
+      const raw = data instanceof ArrayBuffer ? data : data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength)
+      const block = new Uint8Array(raw)
+      // Decrypt block
+      const decrypted = decryptBlock(block)
+      // Parse batch
+      const transmissions = tParse(decrypted)
+      for (const raw of transmissions) {
+        dispatch(raw)
+      }
+    } catch (e: any) {
+      // Parse error — log to stderr
+      process.stderr.write("SMP client receive error: " + e.message + "\n")
+    }
+  }
+
+  function decryptBlock(block: Uint8Array): Uint8Array {
+    if (conn.rcvKey) {
+      const {decrypted, nextChainKey} = sbDecryptBlock(conn.rcvKey, block)
+      conn.rcvKey = nextChainKey
+      return decrypted
+    }
+    // No block encryption — strip padding
+    return blockUnpad(block)
+  }
+
+  function dispatch(raw: RawTransmission) {
+    // Parse response
+    let response: SMPResponse
+    try {
+      response = decodeResponse(new Decoder(raw.command))
+    } catch (e: any) {
+      process.stderr.write("dispatch parse error: " + e.message + " command=" + toHex(raw.command) + "\n")
+      // If we can correlate, reject the pending request
+      const key = toHex(raw.corrId)
+      const req = pending.get(key)
+      if (req) {
+        pending.delete(key)
+        clearTimeout(req.timer)
+        req.reject({type: "RESPONSE", error: e.message})
+      }
+      return
+    }
+
+    // Classify: ERR → PCEProtocolError
+    const err = protocolError(response)
+
+    // Correlate by corrId
+    const corrIdBytes = raw.corrId
+    if (corrIdBytes.length === 0) {
+      // Server push (no corrId) — deliver to event callback
+      onMessage(raw.entityId, response)
+      return
+    }
+
+    const key = toHex(corrIdBytes)
+    const req = pending.get(key)
+    if (req) {
+      pending.delete(key)
+      clearTimeout(req.timer)
+      if (err) {
+        req.reject({type: "PROTOCOL", error: err})
+      } else {
+        req.resolve(response)
+      }
+    } else {
+      // No pending request — might be a late response or server push with corrId
+      // Deliver as event
+      if (!err) onMessage(raw.entityId, response)
+    }
+  }
+
+  // Wire up WebSocket receive
+  conn.ws.onmessage = (event) => onBlock(event.data as ArrayBuffer)
+  conn.ws.onclose = () => {
+    if (!closed) {
+      closed = true
+      cleanup()
+      onDisconnected()
+    }
+  }
+  conn.ws.onerror = () => {}
+
+  // -- Ping
+
+  function startPing() {
+    if (pingInterval <= 0) return
+    pingTimer = setInterval(async () => {
+      try {
+        await client.sendCommand(null, new Uint8Array(0), encodePING())
+      } catch {
+        timeoutCount++
+        if (pingMaxCount > 0 && timeoutCount >= pingMaxCount) {
+          client.close()
+        }
+      }
+    }, pingInterval)
+  }
+
+  function cleanup() {
+    if (pingTimer) {
+      clearInterval(pingTimer)
+      pingTimer = null
+    }
+    // Reject all pending requests
+    for (const [, req] of pending) {
+      clearTimeout(req.timer)
+      req.reject({type: "NETWORK", error: "disconnected"})
+    }
+    pending.clear()
+  }
+
+  // -- Send
+
+  // mkTransmission (Client.hs:1349-1370)
+  // Encode, authenticate, register pending request. Returns encoded transmission + promise.
+  // nonce_ parameter: if provided, used as corrId (for proxy commands where nonce = corrId)
+  function mkTransmission(privKey: AuthKey | null, entityId: Uint8Array, command: Uint8Array, nonce_?: Uint8Array): {auth: Uint8Array | null, tToSend: Uint8Array, promise: Promise<SMPResponse>} {
+    const nonce = nonce_ ?? crypto.getRandomValues(new Uint8Array(24))
+    const {tForAuth, tToSend} = encodeTransmissionForAuth(conn.sessionId, nonce, entityId, command)
+    const auth = authTransmission(serverPubKey, privKey, nonce, tForAuth)
+    const promise = new Promise<SMPResponse>((resolve, reject) => {
+      const key = toHex(nonce)
+      const timer = setTimeout(() => {
+        pending.delete(key)
+        timeoutCount++
+        reject({type: "TIMEOUT"} as SMPClientError)
+      }, timeout_)
+      pending.set(key, {resolve, reject, timer})
+    })
+    return {auth, tToSend, promise}
+  }
+
+  // Send a pre-encoded block (encrypt + write to WebSocket)
+  function sendBlock(block: Uint8Array): void {
+    if (conn.sndKey) {
+      const {encrypted, nextChainKey} = sbEncryptBlock(conn.sndKey, block, SMP_BLOCK_SIZE - 16)
+      conn.sndKey = nextChainKey
+      conn.ws.send(encrypted)
+    } else {
+      conn.ws.send(blockPad(block, SMP_BLOCK_SIZE))
+    }
+  }
+
+  // sendProtocolCommand (Client.hs:1300-1326) — single command
+  // nonce_: if provided, used as corrId (for proxy where nonce = corrId)
+  function sendCommand(privKey: AuthKey | null, entityId: Uint8Array, command: Uint8Array, nonce_?: Uint8Array): Promise<SMPResponse> {
+    if (closed) return Promise.reject({type: "NETWORK", error: "closed"} as SMPClientError)
+    const {auth, tToSend, promise} = mkTransmission(privKey, entityId, command, nonce_)
+    sendBlock(tEncodeBatch1(auth, tToSend))
+    return promise
+  }
+
+  // sendProtocolCommands (Client.hs:1262-1298) — batch multiple commands
+  function sendCommands(commands: Array<{privKey: AuthKey | null, entityId: Uint8Array, command: Uint8Array}>): Promise<SMPResponse>[] {
+    if (closed) return commands.map(() => Promise.reject({type: "NETWORK", error: "closed"} as SMPClientError))
+    // mkTransmission for each
+    const transmissions = commands.map(c => mkTransmission(c.privKey, c.entityId, c.command))
+    // Encode for batching: tEncodeForBatch each
+    const encoded = transmissions.map(t => tEncodeForBatch(t.auth, t.tToSend))
+    // Pack into blocks
+    const blocks = batchTransmissions(SMP_BLOCK_SIZE, encoded)
+    // Send each block
+    for (const block of blocks) sendBlock(block)
+    // Return all promises
+    return transmissions.map(t => t.promise)
+  }
+
+  // -- High-level commands
+
+  // okSMPCommand (Client.hs:1239-1243) — only accepts OK, not SOK
+  async function okCommand(privKey: AuthKey | null, entityId: Uint8Array, command: Uint8Array): Promise<void> {
+    const resp = await sendCommand(privKey, entityId, command)
+    if (resp.type !== "OK") {
+      throw {type: "UNEXPECTED", raw: resp.type} as SMPClientError
+    }
+  }
+
+  const client: SMPClient = {
+    sessionId: conn.sessionId,
+    smpVersion: conn.smpVersion,
+    serverPubKey,
+    sendCommand,
+
+    // createQueue (Client.hs:813-827)
+    async createQueue(authKeyPair, dhKey, subscribe) {
+      const command = encodeNEW(authKeyPair.publicKey, dhKey, null, subscribe)
+      // Auth with the X25519 private key from the keypair
+      const privKey: AuthKey = {type: "x25519", key: authKeyPair.privateKey}
+      const resp = await sendCommand(privKey, new Uint8Array(0), command)
+      if (resp.type !== "IDS") throw {type: "UNEXPECTED", raw: resp.type} as SMPClientError
+      return resp.response
+    },
+
+    // subscribeSMPQueue (Client.hs:833-836)
+    async subscribeQueue(privKey, rcvId) {
+      const resp = await sendCommand(privKey, rcvId, encodeSUB())
+      // SUB can return MSG (queued message) — push to onMessage
+      if (resp.type === "MSG") {
+        onMessage(rcvId, resp)
+        return
+      }
+      if (resp.type !== "OK" && resp.type !== "SOK") {
+        throw {type: "UNEXPECTED", raw: resp.type} as SMPClientError
+      }
+    },
+
+    // getSMPMessage (Client.hs:875-880)
+    async getMessage(privKey, rcvId) {
+      const resp = await sendCommand(privKey, rcvId, encodeGET())
+      if (resp.type === "OK") return null
+      if (resp.type === "MSG") {
+        onMessage(rcvId, resp)
+        return resp.response
+      }
+      throw {type: "UNEXPECTED", raw: resp.type} as SMPClientError
+    },
+
+    // sendSMPMessage (Client.hs:1027-1031)
+    async sendMessage(privKey, sndId, notification, msg) {
+      await okCommand(privKey, sndId, encodeSEND(notification, msg))
+    },
+
+    // ackSMPMessage (Client.hs:1040-1045)
+    async ackMessage(privKey, rcvId, msgId) {
+      const resp = await sendCommand(privKey, rcvId, encodeACK(msgId))
+      // ACK can return MSG — push to onMessage
+      if (resp.type === "MSG") {
+        onMessage(rcvId, resp)
+        return
+      }
+      if (resp.type !== "OK") {
+        throw {type: "UNEXPECTED", raw: resp.type} as SMPClientError
+      }
+    },
+
+    // secureSMPQueue (Client.hs:938-939)
+    async secureQueue(privKey, rcvId, senderKey) {
+      await okCommand(privKey, rcvId, encodeKEY(senderKey))
+    },
+
+    // secureSndSMPQueue (Client.hs:943-944)
+    // SKEY sends the public key derived from the private key
+    async secureSndQueue(privKey, sndId) {
+      // x25519KeyPairFromPrivate derives public from private
+      const pubKey = x25519KeyPairFromPrivate(privKey.key).publicKey
+      await okCommand(privKey, sndId, encodeSKEY(encodePubKeyX25519(pubKey)))
+    },
+
+    // getSMPQueueLink (Client.hs:976-980)
+    async getQueueLink(linkId) {
+      return sendCommand(null, linkId, encodeLGET())
+    },
+
+    // deleteSMPQueue (Client.hs:1058-1059)
+    async deleteQueue(privKey, rcvId) {
+      await okCommand(privKey, rcvId, encodeDEL())
+    },
+
+    // suspendSMPQueue (Client.hs:1051-1052)
+    async suspendQueue(privKey, rcvId) {
+      await okCommand(privKey, rcvId, encodeOFF())
+    },
+
+    // subscribeSMPQueues (Client.hs:840-845)
+    async subscribeQueues(queues) {
+      const commands = queues.map(q => ({privKey: q.privKey, entityId: q.rcvId, command: encodeSUB()}))
+      const promises = sendCommands(commands)
+      return Promise.all(promises.map(async (p, i) => {
+        const resp = await p
+        // processSUBResponse_ (Client.hs:857-862)
+        if (resp.type === "MSG") {
+          onMessage(queues[i].rcvId, resp)
+          return
+        }
+        if (resp.type !== "OK" && resp.type !== "SOK") {
+          throw {type: "UNEXPECTED", raw: resp.type} as SMPClientError
+        }
+      }))
+    },
+
+    // deleteSMPQueues (Client.hs:1062-1065) via okSMPCommands (Client.hs:1245-1253)
+    async deleteQueues(queues) {
+      const commands = queues.map(q => ({privKey: q.privKey, entityId: q.rcvId, command: encodeDEL()}))
+      const promises = sendCommands(commands)
+      return Promise.all(promises.map(async (p) => {
+        const resp = await p
+        if (resp.type !== "OK") {
+          throw {type: "UNEXPECTED", raw: resp.type} as SMPClientError
+        }
+      }))
+    },
+
+    // connectSMPProxiedRelay (Client.hs:1069-1093)
+    async connectProxiedRelay(relayHosts, relayPort, relayKeyHash, basicAuth) {
+      // Send PRXY to proxy server
+      const command = encodePRXY(relayHosts, relayPort, relayKeyHash, basicAuth)
+      const resp = await sendCommand(null, new Uint8Array(0), command)
+      if (resp.type !== "PKEY") throw {type: "UNEXPECTED", raw: resp.type} as SMPClientError
+      const {sessionId: relaySessId, versionRange, signedKeyDer} = resp.response
+      // Check version compatibility
+      const version = Math.min(versionRange.max, conn.smpVersion)
+      if (version < versionRange.min) throw {type: "TRANSPORT", error: "incompatible relay version"} as SMPClientError
+      // Extract relay's X25519 DH key from signed key (same as connectSMP handshake)
+      const relayKey = extractSignedKey(signedKeyDer).dhKey
+      // TODO: full certificate chain validation against relayKeyHash
+      // For now we trust the proxy's PKEY response (proxy already validated the relay)
+      return {sessionId: relaySessId, version, basicAuth, relayKey}
+    },
+
+    // proxySMPCommand (Client.hs:1157-1206)
+    async proxySMPCommand(relay, privKey, entityId, command) {
+      // Prepare relay params — encode as if sending directly to relay
+      const relaySessionId = relay.sessionId
+      // Generate ephemeral X25519 keypair for this command
+      const cmdKp = generateX25519KeyPair()
+      const cmdSecret = dh(relay.relayKey, cmdKp.privateKey)
+      const nonce = crypto.getRandomValues(new Uint8Array(24))
+      // Encode transmission for relay (using relay's sessionId)
+      const {tForAuth, tToSend} = encodeTransmissionForAuth(relaySessionId, nonce, entityId, command)
+      // Authenticate against relay's key
+      const auth = privKey
+        ? cbAuthenticator(relay.relayKey, privKey.key, nonce, tForAuth)
+        : null
+      // Batch into single block (for relay)
+      const batchBlock = tEncodeBatch1(auth, tToSend)
+      // Encrypt for relay: cbEncrypt(cmdSecret, nonce, batchBlock, paddedProxiedTLength)
+      const encTransmission = cbEncrypt(cmdSecret, nonce, batchBlock, paddedProxiedTLength)
+      // Send PFWD to proxy (entityId = relay sessionId from PKEY)
+      // IMPORTANT: nonce is also used as corrId for PFWD (Client.hs:1175,1188)
+      // The relay extracts it from FwdTransmission.fwdCorrId to decrypt
+      const cmdPubKeyDer = encodePubKeyX25519(cmdKp.publicKey)
+      const pfwdCommand = encodePFWD(relay.version, cmdPubKeyDer, encTransmission)
+      const pfwdResp = await sendCommand(null, relay.sessionId, pfwdCommand, nonce)
+      // Handle response
+      if (pfwdResp.type === "PRES") {
+        // Decrypt relay's response: cbDecrypt(cmdSecret, reverseNonce(nonce), encResponse)
+        const decrypted = cbDecrypt(cmdSecret, reverseNonce(nonce), pfwdResp.encResponse)
+        // Parse as relay's response
+        const transmissions = tParse(decrypted)
+        if (transmissions.length !== 1) throw {type: "TRANSPORT", error: "bad proxy response block"} as SMPClientError
+        const decoded = tDecodeClient(transmissions[0])
+        const relayResp = decoded.response
+        const err = protocolError(relayResp)
+        if (err) throw {type: "PROTOCOL", error: err} as SMPClientError
+        return relayResp
+      }
+      if (pfwdResp.type === "ERR") {
+        throw {type: "PROTOCOL", error: pfwdResp.error} as SMPClientError
+      }
+      throw {type: "UNEXPECTED", raw: pfwdResp.type} as SMPClientError
+    },
+
+    // proxySMPMessage — convenience for SEND via proxy
+    async proxySendMessage(relay, privKey, sndId, notification, msg) {
+      const command = encodeSEND(notification, msg)
+      const resp = await client.proxySMPCommand(relay, privKey, sndId, command)
+      if (resp.type !== "OK") throw {type: "UNEXPECTED", raw: resp.type} as SMPClientError
+    },
+
+    close() {
+      if (closed) return
+      closed = true
+      cleanup()
+      conn.ws.close()
+    },
+  }
+
+  startPing()
+  return client
+}
+

smp-web/src/crypto.ts

@@ -0,0 +1,130 @@
+// Crypto primitives.
+// Mirrors: Simplex.Messaging.Crypto
+
+import {hkdf as nobleHkdf} from "@noble/hashes/hkdf"
+import {sha512} from "@noble/hashes/sha512"
+import {gcm} from "@noble/ciphers/aes.js"
+import {cbEncrypt, cbDecrypt, cryptoBox, sbInit, sbDecryptChunk, sbAuth} from "@simplex-chat/xftp-web/dist/crypto/secretbox.js"
+import {dh} from "@simplex-chat/xftp-web/dist/crypto/keys.js"
+import {concatBytes} from "@simplex-chat/xftp-web/dist/protocol/encoding.js"
+import {pad, unPad} from "@simplex-chat/xftp-web/dist/crypto/padding.js"
+
+// C.hkdf (Crypto.hs:1461-1464)
+// HKDF-SHA512 extract + expand
+export function hkdf(salt: Uint8Array, ikm: Uint8Array, info: string, n: number): Uint8Array {
+  return nobleHkdf(sha512, ikm, salt, info, n)
+}
+
+// -- SbChainKey block encryption (Crypto.hs:1449-1464)
+
+export interface SbKeyNonce {
+  sbKey: Uint8Array   // 32 bytes
+  nonce: Uint8Array   // 24 bytes
+}
+
+// sbcInit (Crypto.hs:1452-1455)
+// hkdf(sessionId, dhSecret, "SimpleXSbChainInit", 64) -> (sndChainKey, rcvChainKey)
+export function sbcInit(sessionId: Uint8Array, dhSecret: Uint8Array): {sndKey: Uint8Array; rcvKey: Uint8Array} {
+  const derived = hkdf(sessionId, dhSecret, "SimpleXSbChainInit", 64)
+  return {sndKey: derived.slice(0, 32), rcvKey: derived.slice(32, 64)}
+}
+
+// sbcHkdf (Crypto.hs:1459-1464)
+// hkdf("", chainKey, "SimpleXSbChain", 88) -> ((sbKey, nonce), nextChainKey)
+export function sbcHkdf(chainKey: Uint8Array): {keyNonce: SbKeyNonce; nextChainKey: Uint8Array} {
+  const out = hkdf(new Uint8Array(0), chainKey, "SimpleXSbChain", 88)
+  return {
+    keyNonce: {sbKey: out.slice(32, 64), nonce: out.slice(64, 88)},
+    nextChainKey: out.slice(0, 32),
+  }
+}
+
+// sbEncrypt (Crypto.hs:1296-1301)
+// pad + cryptoBox (tag prepended to ciphertext)
+export function sbEncryptBlock(chainKey: Uint8Array, block: Uint8Array, paddedLen: number): {encrypted: Uint8Array; nextChainKey: Uint8Array} {
+  const {keyNonce: {sbKey, nonce}, nextChainKey} = sbcHkdf(chainKey)
+  return {encrypted: cbEncrypt(sbKey, nonce, block, paddedLen), nextChainKey}
+}
+
+// sbDecrypt (Crypto.hs:1330-1336)
+// cryptoBoxOpen + unpad
+export function sbDecryptBlock(chainKey: Uint8Array, block: Uint8Array): {decrypted: Uint8Array; nextChainKey: Uint8Array} {
+  const {keyNonce: {sbKey, nonce}, nextChainKey} = sbcHkdf(chainKey)
+  return {decrypted: cbDecrypt(sbKey, nonce, block), nextChainKey}
+}
+
+// -- AES-256-GCM authenticated encryption (Crypto.hs:1035-1061)
+// Uses 16-byte IVs (GCM with GHASH path per NIST SP 800-38D for IVs != 96 bits)
+
+export const AUTH_TAG_SIZE = 16
+
+// encryptAEAD (Crypto.hs:1035-1039)
+export function encryptAEAD(
+  key: Uint8Array,     // 32 bytes
+  iv: Uint8Array,      // 16 bytes
+  paddedLen: number,
+  ad: Uint8Array,
+  plaintext: Uint8Array,
+): {authTag: Uint8Array; ciphertext: Uint8Array} {
+  const padded = pad(plaintext, paddedLen)
+  const cipher = gcm(key, iv, ad)
+  const encrypted = cipher.encrypt(padded)
+  return {
+    ciphertext: encrypted.subarray(0, encrypted.length - AUTH_TAG_SIZE),
+    authTag: encrypted.subarray(encrypted.length - AUTH_TAG_SIZE),
+  }
+}
+
+// decryptAEAD (Crypto.hs:1058-1061)
+export function decryptAEAD(
+  key: Uint8Array,
+  iv: Uint8Array,
+  ad: Uint8Array,
+  ciphertext: Uint8Array,
+  authTag: Uint8Array,
+): Uint8Array {
+  const cipher = gcm(key, iv, ad)
+  const encrypted = concatBytes(ciphertext, authTag)
+  const padded = cipher.decrypt(encrypted)
+  return unPad(padded)
+}
+
+// -- SHA-512 hash (Crypto.hs:1016)
+
+export function sha512Hash(msg: Uint8Array): Uint8Array {
+  return sha512(msg)
+}
+
+// -- Command authentication (Crypto.hs:1366-1367)
+
+// cbAuthenticate (Crypto.hs:1367)
+// cryptoBox(dh(serverPubKey, entityPrivKey), nonce, sha512Hash(msg)) → 80 bytes (16 tag + 64 hash)
+export function cbAuthenticator(serverPubKey: Uint8Array, entityPrivKey: Uint8Array, nonce: Uint8Array, msg: Uint8Array): Uint8Array {
+  const dhSecret = dh(serverPubKey, entityPrivKey)
+  return cryptoBox(dhSecret, nonce, sha512Hash(msg))
+}
+
+// -- reverseNonce (Crypto.hs:1409-1410)
+
+export function reverseNonce(nonce: Uint8Array): Uint8Array {
+  const reversed = new Uint8Array(nonce.length)
+  for (let i = 0; i < nonce.length; i++) reversed[i] = nonce[nonce.length - 1 - i]
+  return reversed
+}
+
+// -- cbDecryptNoPad (Crypto.hs:1330-1331)
+// Decrypt without unpadding. Used for proxy responses.
+// Same as cbDecrypt but returns raw decrypted bytes without unPad.
+
+export function cbDecryptNoPad(dhSecret: Uint8Array, nonce: Uint8Array, packet: Uint8Array): Uint8Array {
+  const tag = packet.subarray(0, 16)
+  const cipher = packet.subarray(16)
+  const state = sbInit(dhSecret, nonce)
+  const plaintext = sbDecryptChunk(state, cipher)
+  const computedTag = sbAuth(state)
+  // constant-time compare
+  let diff = 0
+  for (let i = 0; i < 16; i++) diff |= tag[i] ^ computedTag[i]
+  if (diff !== 0) throw new Error("cbDecryptNoPad: authentication failed")
+  return plaintext
+}

smp-web/src/crypto/ratchet.ts

@@ -0,0 +1,749 @@
+// Double ratchet with X3DH key agreement and PQ KEM.
+// Faithful transpilation of Simplex.Messaging.Crypto.Ratchet
+//
+// Every type, field, and function mirrors the Haskell source.
+// Line references are to src/Simplex/Messaging/Crypto/Ratchet.hs
+
+import {x448} from "@noble/curves/ed448.js"
+import {hkdf, encryptAEAD, decryptAEAD, AUTH_TAG_SIZE} from "../crypto.js"
+import {sntrup761Keypair, sntrup761Enc, sntrup761Dec} from "./sntrup761.js"
+import type {KEMKeyPair} from "./sntrup761.js"
+import {
+  Decoder, concatBytes,
+  encodeBytes, decodeBytes, decodeWord16, decodeWord32,
+  encodeLarge, decodeLarge,
+  encodeMaybe,
+} from "@simplex-chat/xftp-web/dist/protocol/encoding.js"
+
+// -- Version constants (lines 134-155)
+
+export const pqRatchetE2EEncryptVersion = 3
+export const currentE2EEncryptVersion = 3
+
+// -- X448 key operations
+
+export interface X448KeyPair {
+  publicKey: Uint8Array  // 56 bytes
+  privateKey: Uint8Array // 56 bytes
+}
+
+export function generateX448KeyPair(): X448KeyPair {
+  const privateKey = x448.utils.randomSecretKey()
+  const publicKey = x448.getPublicKey(privateKey)
+  return {publicKey, privateKey}
+}
+
+export function x448DH(publicKey: Uint8Array, privateKey: Uint8Array): Uint8Array {
+  return x448.getSharedSecret(privateKey, publicKey)
+}
+
+// DER encoding for X448 public keys (RFC 8410, SubjectPublicKeyInfo)
+// SEQUENCE { SEQUENCE { OID 1.3.101.111 } BIT STRING { 0x00 <56 bytes> } }
+const X448_PUBKEY_DER_PREFIX = new Uint8Array([
+  0x30, 0x42, 0x30, 0x05, 0x06, 0x03, 0x2b, 0x65, 0x6f, 0x03, 0x39, 0x00,
+])
+
+export function encodePubKeyX448(rawPubKey: Uint8Array): Uint8Array {
+  return concatBytes(X448_PUBKEY_DER_PREFIX, rawPubKey)
+}
+
+export function decodePubKeyX448(der: Uint8Array): Uint8Array {
+  if (der.length !== 68) throw new Error("decodePubKeyX448: invalid length " + der.length)
+  for (let i = 0; i < X448_PUBKEY_DER_PREFIX.length; i++) {
+    if (der[i] !== X448_PUBKEY_DER_PREFIX[i]) throw new Error("decodePubKeyX448: invalid DER prefix")
+  }
+  return der.subarray(12)
+}
+
+// -- KEM types (lines 567-577)
+// KEMKeyPair imported from ./sntrup761.js
+
+export interface RatchetKEMAccepted {
+  rcPQRr: Uint8Array      // KEMPublicKey - received key (1158 bytes)
+  rcPQRss: Uint8Array     // KEMSharedKey - computed shared secret (32 bytes)
+  rcPQRct: Uint8Array     // KEMCiphertext - sent encaps (1039 bytes)
+}
+
+export interface RatchetKEM {
+  rcPQRs: KEMKeyPair
+  rcKEMs: RatchetKEMAccepted | null
+}
+
+// -- RatchetInitParams (lines 457-464)
+
+export interface RatchetInitParams {
+  assocData: Uint8Array              // Str (raw bytes)
+  ratchetKey: Uint8Array             // RatchetKey (32 bytes)
+  sndHK: Uint8Array                  // HeaderKey (32 bytes)
+  rcvNextHK: Uint8Array              // HeaderKey (32 bytes)
+  kemAccepted: RatchetKEMAccepted | null  // Maybe RatchetKEMAccepted
+}
+
+// -- hkdf3 (lines 1174-1179)
+
+function hkdf3(salt: Uint8Array, ikm: Uint8Array, info: string): [Uint8Array, Uint8Array, Uint8Array] {
+  const out = hkdf(salt, ikm, info, 96)
+  return [out.slice(0, 32), out.slice(32, 64), out.slice(64, 96)]
+}
+
+// -- pqX3dh (lines 499-508)
+
+const X3DH_SALT = new Uint8Array(64)
+
+function pqX3dh(
+  sk1: Uint8Array, rk1: Uint8Array,
+  dh1: Uint8Array, dh2: Uint8Array, dh3: Uint8Array,
+  kemAccepted: RatchetKEMAccepted | null,
+): RatchetInitParams {
+  const assocData = concatBytes(sk1, rk1)
+  const pq = kemAccepted ? kemAccepted.rcPQRss : new Uint8Array(0)
+  const dhs = concatBytes(dh1, dh2, dh3, pq)
+  const [hk, nhk, sk] = hkdf3(X3DH_SALT, dhs, "SimpleXX3DH")
+  return {assocData, ratchetKey: sk, sndHK: hk, rcvNextHK: nhk, kemAccepted}
+}
+
+// -- pqX3dhSnd (lines 467-480)
+// Used by joiner (Alice in PQDR spec, Bob in DR spec) to init SENDING ratchet.
+
+export function pqX3dhSnd(
+  spk1: Uint8Array, spk2: Uint8Array,   // our private keys
+  rk1: Uint8Array, rk2: Uint8Array,     // their public keys (raw)
+  kemAccepted: RatchetKEMAccepted | null = null,
+): RatchetInitParams {
+  const sk1Pub = x448.getPublicKey(spk1)
+  const dh1 = x448DH(rk1, spk2)
+  const dh2 = x448DH(rk2, spk1)
+  const dh3 = x448DH(rk2, spk2)
+  return pqX3dh(sk1Pub, rk1, dh1, dh2, dh3, kemAccepted)
+}
+
+// -- pqX3dhRcv (lines 483-497)
+// Used by initiator (Bob in PQDR spec, Alice in DR spec) to init RECEIVING ratchet.
+
+export function pqX3dhRcv(
+  rpk1: Uint8Array, rpk2: Uint8Array,   // our private keys
+  sk1: Uint8Array, sk2: Uint8Array,     // their public keys (raw)
+  kemAccepted: RatchetKEMAccepted | null = null,
+): RatchetInitParams {
+  const rk1Pub = x448.getPublicKey(rpk1)
+  const dh1 = x448DH(sk2, rpk1)
+  const dh2 = x448DH(sk1, rpk2)
+  const dh3 = x448DH(sk2, rpk2)
+  return pqX3dh(sk1, rk1Pub, dh1, dh2, dh3, kemAccepted)
+}
+
+// -- rootKdf (lines 1159-1166)
+
+export function rootKdf(
+  rk: Uint8Array,           // RatchetKey (32 bytes)
+  peerPubKey: Uint8Array,   // PublicKey a (raw, 56 bytes for X448)
+  ownPrivKey: Uint8Array,   // PrivateKey a (raw, 56 bytes for X448)
+  kemSecret: Uint8Array | null,  // Maybe KEMSharedKey
+): {rk: Uint8Array; ck: Uint8Array; nhk: Uint8Array} {
+  const dhOut = x448DH(peerPubKey, ownPrivKey)
+  const ss = kemSecret ? concatBytes(dhOut, kemSecret) : dhOut
+  const [rk_, ck, nhk] = hkdf3(rk, ss, "SimpleXRootRatchet")
+  return {rk: rk_, ck, nhk}
+}
+
+// -- chainKdf (lines 1168-1172)
+
+export function chainKdf(ck: Uint8Array): {ck: Uint8Array; mk: Uint8Array; iv: Uint8Array; ehIV: Uint8Array} {
+  const EMPTY = new Uint8Array(0)
+  const [ck_, mk, ivs] = hkdf3(EMPTY, ck, "SimpleXChainRatchet")
+  return {ck: ck_, mk, iv: ivs.slice(0, 16), ehIV: ivs.slice(16, 32)}
+}
+
+// -- Header padding (lines 716-719)
+
+export function paddedHeaderLen(v: number, pqSupport: boolean): number {
+  if (pqSupport && v >= pqRatchetE2EEncryptVersion) return 2310
+  return 88
+}
+
+// -- SndRatchet (lines 554-559)
+
+export interface SndRatchet {
+  rcDHRr: Uint8Array   // peer's public key (raw, 56 bytes)
+  rcCKs: Uint8Array    // sending chain key (32 bytes)
+  rcHKs: Uint8Array    // sending header key (32 bytes)
+}
+
+// -- RcvRatchet (lines 561-565)
+
+export interface RcvRatchet {
+  rcCKr: Uint8Array    // receiving chain key (32 bytes)
+  rcHKr: Uint8Array    // receiving header key (32 bytes)
+}
+
+// -- MessageKey (lines 608-609)
+
+export interface MessageKey {
+  mk: Uint8Array       // Key (32 bytes)
+  iv: Uint8Array       // IV (16 bytes)
+}
+
+// -- RatchetVersions (lines 534-538)
+
+export interface RatchetVersions {
+  current: number
+  maxSupported: number
+}
+
+// -- Ratchet (lines 512-532)
+
+export interface Ratchet {
+  rcVersion: RatchetVersions
+  rcAD: Uint8Array             // Str (associated data, raw bytes)
+  rcDHRs: Uint8Array           // PrivateKey a (raw, 56 bytes)
+  rcKEM: RatchetKEM | null
+  rcSupportKEM: boolean        // PQSupport
+  rcEnableKEM: boolean         // PQEncryption
+  rcSndKEM: boolean            // PQEncryption
+  rcRcvKEM: boolean            // PQEncryption
+  rcRK: Uint8Array             // RatchetKey (32 bytes)
+  rcSnd: SndRatchet | null
+  rcRcv: RcvRatchet | null
+  rcNs: number                 // Word32
+  rcNr: number                 // Word32
+  rcPN: number                 // Word32
+  rcNHKs: Uint8Array           // HeaderKey (32 bytes)
+  rcNHKr: Uint8Array           // HeaderKey (32 bytes)
+}
+
+// -- SkippedMsgKeys (lines 580-582)
+
+export type SkippedMsgKeys = Map<string, Map<number, MessageKey>>
+
+const MAX_SKIP = 512
+
+function hexKey(k: Uint8Array): string {
+  return Array.from(k, b => b.toString(16).padStart(2, "0")).join("")
+}
+
+function hexToBytes(hex: string): Uint8Array {
+  const bytes = new Uint8Array(hex.length / 2)
+  for (let i = 0; i < hex.length; i += 2) bytes[i / 2] = parseInt(hex.substring(i, i + 2), 16)
+  return bytes
+}
+
+// -- initSndRatchet (lines 643-666)
+
+export function initSndRatchet(
+  rcVersion: RatchetVersions,
+  rcDHRr: Uint8Array,     // peer's public key (raw)
+  rcDHRs: Uint8Array,     // our private key (raw)
+  initParams: RatchetInitParams,
+  rcPQRs_: KEMKeyPair | null,
+): Ratchet {
+  const {assocData, ratchetKey, sndHK, rcvNextHK, kemAccepted} = initParams
+  // state.RK, state.CKs, state.NHKs = KDF_RK_HE(SK, DH(state.DHRs, state.DHRr) || state.PQRss)
+  const kemSecret = kemAccepted ? kemAccepted.rcPQRss : null
+  const {rk: rcRK, ck: rcCKs, nhk: rcNHKs} = rootKdf(ratchetKey, rcDHRr, rcDHRs, kemSecret)
+  const pqOn = rcPQRs_ !== null
+  return {
+    rcVersion,
+    rcAD: assocData,
+    rcDHRs,
+    rcKEM: rcPQRs_ ? {rcPQRs: rcPQRs_, rcKEMs: kemAccepted} : null,
+    rcSupportKEM: pqOn,
+    rcEnableKEM: pqOn,
+    rcSndKEM: kemAccepted !== null,
+    rcRcvKEM: false,
+    rcRK,
+    rcSnd: {rcDHRr, rcCKs, rcHKs: sndHK},
+    rcRcv: null,
+    rcPN: 0,
+    rcNs: 0,
+    rcNr: 0,
+    rcNHKs,
+    rcNHKr: rcvNextHK,
+  }
+}
+
+// -- initRcvRatchet (lines 674-699)
+
+export function initRcvRatchet(
+  rcVersion: RatchetVersions,
+  rcDHRs: Uint8Array,     // our private key (raw)
+  initParams: RatchetInitParams,
+  rcPQRs_: KEMKeyPair | null,
+  pqSupport: boolean,
+): Ratchet {
+  const {assocData, ratchetKey, sndHK, rcvNextHK, kemAccepted} = initParams
+  return {
+    rcVersion,
+    rcAD: assocData,
+    rcDHRs,
+    rcKEM: rcPQRs_ ? {rcPQRs: rcPQRs_, rcKEMs: kemAccepted} : null,
+    rcSupportKEM: pqSupport,
+    rcEnableKEM: pqSupport,
+    rcSndKEM: false,
+    rcRcvKEM: false,
+    rcRK: ratchetKey,
+    rcSnd: null,
+    rcRcv: null,
+    rcPN: 0,
+    rcNs: 0,
+    rcNr: 0,
+    rcNHKs: rcvNextHK,
+    rcNHKr: sndHK,
+  }
+}
+
+// -- RKEMParams (lines 188-190) - parsed KEM params from message header
+
+export type RKEMParams =
+  | {type: "proposed", kemPk: Uint8Array}          // RKParamsProposed KEMPublicKey
+  | {type: "accepted", kemCt: Uint8Array, kemPk: Uint8Array}  // RKParamsAccepted KEMCiphertext KEMPublicKey
+
+// -- MsgHeader (lines 703-711)
+
+interface MsgHeader {
+  msgMaxVersion: number
+  msgDHRs: Uint8Array    // PublicKey a (raw, 56 bytes)
+  msgKEM: RKEMParams | null
+  msgPN: number          // Word32
+  msgNs: number          // Word32
+}
+
+// -- encodeMsgHeader (lines 727-730)
+
+function encodeMsgHeader(v: number, hdr: MsgHeader): Uint8Array {
+  const vBytes = new Uint8Array(2)
+  vBytes[0] = (hdr.msgMaxVersion >> 8) & 0xff
+  vBytes[1] = hdr.msgMaxVersion & 0xff
+  const dhDer = encodePubKeyX448(hdr.msgDHRs)
+  const pn = encodeWord32(hdr.msgPN)
+  const ns = encodeWord32(hdr.msgNs)
+  if (v >= pqRatchetE2EEncryptVersion) {
+    // smpEncode (msgMaxVersion, msgDHRs, msgKEM, msgPN, msgNs)
+    // msgKEM :: Maybe ARKEMParams
+    const kemBytes = hdr.msgKEM ? encodeRKEMParams(hdr.msgKEM) : new Uint8Array([0x30])  // Nothing
+    return concatBytes(vBytes, encodeBytes(dhDer), kemBytes, pn, ns)
+  }
+  // smpEncode (msgMaxVersion, msgDHRs, msgPN, msgNs)
+  return concatBytes(vBytes, encodeBytes(dhDer), pn, ns)
+}
+
+// Encode Maybe ARKEMParams: '1' + encoded params, or nothing (handled at call site with '0')
+function encodeRKEMParams(params: RKEMParams): Uint8Array {
+  if (params.type === "proposed") {
+    // Just ('P', kemPk) - smpEncode ('P', k) where k is KEMPublicKey (Large)
+    return concatBytes(new Uint8Array([0x31, 0x50]), encodeLarge(params.kemPk))
+  }
+  // Just ('A', ct, kemPk) - smpEncode ('A', ct, k)
+  return concatBytes(new Uint8Array([0x31, 0x41]), encodeLarge(params.kemCt), encodeLarge(params.kemPk))
+}
+
+function encodeWord32(n: number): Uint8Array {
+  const buf = new Uint8Array(4)
+  buf[0] = (n >> 24) & 0xff; buf[1] = (n >> 16) & 0xff
+  buf[2] = (n >> 8) & 0xff; buf[3] = n & 0xff
+  return buf
+}
+
+// -- msgHeaderP (lines 733-740)
+
+function decodeMsgHeader(v: number, data: Uint8Array): MsgHeader {
+  const d = new Decoder(data)
+  const msgMaxVersion = decodeWord16(d)
+  const dhDer = decodeBytes(d)
+  const msgDHRs = decodePubKeyX448(dhDer)
+  let msgKEM: RKEMParams | null = null
+  if (v >= pqRatchetE2EEncryptVersion) {
+    // Maybe ARKEMParams
+    const maybeByte = d.anyByte()
+    if (maybeByte === 0x31) {
+      // Just - parse ARKEMParams
+      const tag = d.anyByte()
+      if (tag === 0x50) { // 'P' - Proposed: KEMPublicKey (Large)
+        msgKEM = {type: "proposed", kemPk: decodeLarge(d)}
+      } else if (tag === 0x41) { // 'A' - Accepted: KEMCiphertext (Large) + KEMPublicKey (Large)
+        const kemCt = decodeLarge(d)
+        const kemPk = decodeLarge(d)
+        msgKEM = {type: "accepted", kemCt, kemPk}
+      } else {
+        throw new Error("decodeMsgHeader: unknown KEM tag " + tag)
+      }
+    }
+    // else '0' = Nothing, msgKEM stays null
+  }
+  const msgPN = decodeWord32(d)
+  const msgNs = decodeWord32(d)
+  return {msgMaxVersion, msgDHRs, msgKEM, msgPN, msgNs}
+}
+
+// -- EncMessageHeader (lines 742-756)
+
+interface EncMessageHeader {
+  ehVersion: number     // current ratchet version
+  ehIV: Uint8Array      // IV (raw 16 bytes)
+  ehAuthTag: Uint8Array // AuthTag (raw 16 bytes)
+  ehBody: Uint8Array    // encrypted header body
+}
+
+// smpEncode (lines 751-752)
+function encodeEncMessageHeader(emh: EncMessageHeader): Uint8Array {
+  const vBytes = new Uint8Array(2)
+  vBytes[0] = (emh.ehVersion >> 8) & 0xff
+  vBytes[1] = emh.ehVersion & 0xff
+  // smpEncode (ehVersion, ehIV, ehAuthTag) <> encodeLarge ehVersion ehBody
+  const bodyEnc = emh.ehVersion >= pqRatchetE2EEncryptVersion
+    ? encodeLarge(emh.ehBody)
+    : encodeBytes(emh.ehBody)
+  return concatBytes(vBytes, emh.ehIV, emh.ehAuthTag, bodyEnc)
+}
+
+// smpP (lines 753-756)
+function decodeEncMessageHeader(data: Uint8Array): EncMessageHeader {
+  const d = new Decoder(data)
+  const ehVersion = decodeWord16(d)
+  const ehIV = d.take(16)      // IV is raw 16 bytes
+  const ehAuthTag = d.take(16) // AuthTag is raw 16 bytes
+  // largeP: peek first byte, if < 32 then Large (2-byte len), else ByteString (1-byte len)
+  const firstByte = data[d.offset()]
+  const ehBody = firstByte < 32 ? decodeLarge(d) : decodeBytes(d)
+  return {ehVersion, ehIV, ehAuthTag, ehBody}
+}
+
+// -- EncRatchetMessage (lines 772-787)
+
+interface EncRatchetMessage {
+  emHeader: Uint8Array   // smpEncoded EncMessageHeader
+  emAuthTag: Uint8Array  // AuthTag (raw 16 bytes)
+  emBody: Uint8Array     // encrypted message body
+}
+
+// encodeEncRatchetMessage (lines 779-781)
+function encodeEncRatchetMessage(v: number, msg: EncRatchetMessage): Uint8Array {
+  // encodeLarge v emHeader <> smpEncode (emAuthTag, Tail emBody)
+  const headerEnc = v >= pqRatchetE2EEncryptVersion
+    ? encodeLarge(msg.emHeader)
+    : encodeBytes(msg.emHeader)
+  return concatBytes(headerEnc, msg.emAuthTag, msg.emBody)
+}
+
+// encRatchetMessageP (lines 783-787)
+function decodeEncRatchetMessage(data: Uint8Array): EncRatchetMessage {
+  const d = new Decoder(data)
+  // largeP
+  const firstByte = data[d.offset()]
+  const emHeader = firstByte < 32 ? decodeLarge(d) : decodeBytes(d)
+  // smpEncode (emAuthTag, Tail emBody) → raw 16 bytes + rest
+  const emAuthTag = d.take(16)
+  const emBody = d.takeAll()
+  return {emHeader, emAuthTag, emBody}
+}
+
+// -- MsgEncryptKey (lines 962-968)
+
+interface MsgEncryptKey {
+  msgRcVersion: number
+  msgKey: MessageKey
+  msgRcAD: Uint8Array
+  msgEncHeader: Uint8Array
+}
+
+// -- msgKEMParams (lines 956-958) - build KEM params from ratchet state for message header
+
+function msgKEMParams(kem: RatchetKEM): RKEMParams {
+  const {rcPQRs, rcKEMs} = kem
+  if (!rcKEMs) {
+    return {type: "proposed", kemPk: rcPQRs.publicKey}
+  }
+  return {type: "accepted", kemCt: rcKEMs.rcPQRct, kemPk: rcPQRs.publicKey}
+}
+
+// -- pqEnableSupport (line 836-837)
+
+function pqEnableSupport(v: number, sup: boolean, enc: boolean): boolean {
+  return sup || (v >= pqRatchetE2EEncryptVersion && enc)
+}
+
+// -- rcEncryptHeader + rcEncryptMsg (lines 902-975)
+
+export interface EncryptResult {
+  ciphertext: Uint8Array
+  state: Ratchet
+}
+
+export function rcEncrypt(
+  rc: Ratchet,
+  plaintext: Uint8Array,
+  paddedMsgLen: number,
+): EncryptResult {
+  if (!rc.rcSnd) throw new Error("rcEncrypt: no sending ratchet (CERatchetState)")
+  const snd = rc.rcSnd
+  const v = rc.rcVersion.current
+
+  // state.CKs, mk = KDF_CK(state.CKs)
+  const chain = chainKdf(snd.rcCKs)
+
+  // header
+  const headerPlain = encodeMsgHeader(v, {
+    msgMaxVersion: rc.rcVersion.maxSupported,
+    msgDHRs: x448.getPublicKey(rc.rcDHRs),
+    msgKEM: rc.rcKEM ? msgKEMParams(rc.rcKEM) : null,
+    msgPN: rc.rcPN,
+    msgNs: rc.rcNs,
+  })
+
+  // enc_header = HENCRYPT(state.HKs, header)
+  const phl = paddedHeaderLen(v, rc.rcSupportKEM)
+  const {authTag: ehAuthTag, ciphertext: ehBody} = encryptAEAD(snd.rcHKs, chain.ehIV, phl, rc.rcAD, headerPlain)
+
+  // smpEncode EncMessageHeader
+  const emHeader = encodeEncMessageHeader({ehVersion: v, ehBody, ehAuthTag, ehIV: chain.ehIV})
+
+  // ENCRYPT(mk, plaintext, CONCAT(AD, enc_header))
+  const bodyAD = concatBytes(rc.rcAD, emHeader)
+  const {authTag: emAuthTag, ciphertext: emBody} = encryptAEAD(chain.mk, chain.iv, paddedMsgLen, bodyAD, plaintext)
+
+  // encodeEncRatchetMessage
+  const ciphertext = encodeEncRatchetMessage(v, {emHeader, emBody, emAuthTag})
+
+  // Update state
+  const newState: Ratchet = {
+    ...rc,
+    rcSnd: {...snd, rcCKs: chain.ck},
+    rcNs: rc.rcNs + 1,
+  }
+
+  return {ciphertext, state: newState}
+}
+
+// -- rcDecrypt (lines 990-1157)
+
+export interface DecryptResult {
+  plaintext: Uint8Array
+  state: Ratchet
+  skippedKeys: SkippedMsgKeys
+}
+
+export function rcDecrypt(
+  rc: Ratchet,
+  skippedKeys: SkippedMsgKeys,
+  ciphertext: Uint8Array,
+): DecryptResult {
+  const encMsg = decodeEncRatchetMessage(ciphertext)
+  const encHdr = decodeEncMessageHeader(encMsg.emHeader)
+
+  // TrySkippedMessageKeysHE
+  const skipped = tryDecryptSkipped(rc, skippedKeys, encHdr, encMsg)
+  if (skipped) return skipped
+
+  // DecryptHeader
+  let ratchetStep: "same" | "advance" = "advance"
+  let hdr: MsgHeader | null = null
+
+  if (rc.rcRcv) {
+    hdr = tryDecryptHeader(rc.rcRcv.rcHKr, rc.rcAD, encHdr)
+    if (hdr) ratchetStep = "same"
+  }
+  if (!hdr) {
+    hdr = tryDecryptHeader(rc.rcNHKr, rc.rcAD, encHdr)
+    if (!hdr) throw new Error("rcDecrypt: header decryption failed (CERatchetHeader)")
+    ratchetStep = "advance"
+  }
+
+  // Version upgrade
+  let state = rc
+  const {current, maxSupported} = rc.rcVersion
+  if (hdr.msgMaxVersion > current) {
+    state = {...state, rcVersion: {...state.rcVersion, current: Math.max(current, Math.min(hdr.msgMaxVersion, maxSupported))}}
+  }
+
+  let newSkipped = new Map(skippedKeys)
+
+  if (ratchetStep === "advance") {
+    // SkipMessageKeysHE(state, header.pn)
+    const skip1 = skipMessageKeys(state, newSkipped, hdr.msgPN)
+    state = skip1.state; newSkipped = skip1.skippedKeys
+
+    // DHRatchetPQ2HE(state, header) - ratchet step (lines 1043-1071)
+    const {kemSS, kemSS2, rcKEM: rcKEM_} = pqRatchetStep(state, hdr.msgKEM)
+    const newDHRs = generateX448KeyPair()
+    // state.RK, state.CKr, state.NHKr = KDF_RK_HE(state.RK, DH(state.DHRs, state.DHRr) || ss)
+    const kdf1 = rootKdf(state.rcRK, hdr.msgDHRs, state.rcDHRs, kemSS)
+    // state.RK, state.CKs, state.NHKs = KDF_RK_HE(state.RK, DH(state.DHRs', state.DHRr) || state.PQRss)
+    const kdf2 = rootKdf(kdf1.rk, hdr.msgDHRs, newDHRs.privateKey, kemSS2)
+    const sndKEM = kemSS2 !== null
+    const rcvKEM = kemSS !== null
+    const rcEnableKEM_ = sndKEM || rcvKEM || rcKEM_ !== null
+
+    state = {
+      ...state,
+      rcDHRs: newDHRs.privateKey,
+      rcKEM: rcKEM_,
+      rcSupportKEM: pqEnableSupport(state.rcVersion.current, state.rcSupportKEM, rcEnableKEM_),
+      rcEnableKEM: rcEnableKEM_,
+      rcSndKEM: sndKEM,
+      rcRcvKEM: rcvKEM,
+      rcRK: kdf2.rk,
+      rcSnd: {rcDHRr: hdr.msgDHRs, rcCKs: kdf2.ck, rcHKs: state.rcNHKs},
+      rcRcv: {rcCKr: kdf1.ck, rcHKr: state.rcNHKr},
+      rcPN: rc.rcNs,
+      rcNs: 0,
+      rcNr: 0,
+      rcNHKs: kdf2.nhk,
+      rcNHKr: kdf1.nhk,
+    }
+  }
+
+  // SkipMessageKeysHE(state, header.n)
+  const skip2 = skipMessageKeys(state, newSkipped, hdr.msgNs)
+  state = skip2.state; newSkipped = skip2.skippedKeys
+
+  if (!state.rcRcv) throw new Error("rcDecrypt: no receiving ratchet after skip")
+
+  // state.CKr, mk = KDF_CK(state.CKr)
+  const chain = chainKdf(state.rcRcv.rcCKr)
+
+  // DECRYPT(mk, cipher-text, CONCAT(AD, enc_header))
+  const bodyAD = concatBytes(state.rcAD, encMsg.emHeader)
+  const plaintext = decryptAEAD(chain.mk, chain.iv, bodyAD, encMsg.emBody, encMsg.emAuthTag)
+
+  // state.Nr += 1
+  state = {
+    ...state,
+    rcRcv: {...state.rcRcv, rcCKr: chain.ck},
+    rcNr: state.rcNr + 1,
+  }
+
+  return {plaintext, state, skippedKeys: newSkipped}
+}
+
+// -- skipMessageKeys (lines 1105-1121)
+
+function skipMessageKeys(
+  rc: Ratchet,
+  skippedKeys: SkippedMsgKeys,
+  untilN: number,
+): {state: Ratchet; skippedKeys: SkippedMsgKeys} {
+  if (!rc.rcRcv) return {state: rc, skippedKeys}
+  const rcv = rc.rcRcv
+  const rcNr = rc.rcNr
+
+  if (rcNr > untilN + 1) throw new Error("rcDecrypt: earlier message (CERatchetEarlierMessage)")
+  if (rcNr === untilN + 1) throw new Error("rcDecrypt: duplicate message (CERatchetDuplicateMessage)")
+  if (rcNr + MAX_SKIP < untilN) throw new Error("rcDecrypt: too many skipped (CERatchetTooManySkipped)")
+  if (rcNr === untilN) return {state: rc, skippedKeys}
+
+  // advanceRcvRatchet
+  let ck = rcv.rcCKr
+  let nr = rcNr
+  const hkHex = hexKey(rcv.rcHKr)
+  const msgKeys = new Map(skippedKeys.get(hkHex) || new Map())
+
+  while (nr < untilN) {
+    const chain = chainKdf(ck)
+    msgKeys.set(nr, {mk: chain.mk, iv: chain.iv})
+    ck = chain.ck
+    nr++
+  }
+
+  const newSkipped = new Map(skippedKeys)
+  newSkipped.set(hkHex, msgKeys)
+
+  return {
+    state: {...rc, rcRcv: {...rcv, rcCKr: ck}, rcNr: nr},
+    skippedKeys: newSkipped,
+  }
+}
+
+// -- tryDecryptSkipped (lines 1122-1141)
+
+function tryDecryptSkipped(
+  rc: Ratchet,
+  skippedKeys: SkippedMsgKeys,
+  encHdr: EncMessageHeader,
+  encMsg: EncRatchetMessage,
+): DecryptResult | null {
+  for (const [hkHex, msgKeys] of skippedKeys) {
+    const hk = hexToBytes(hkHex)
+    const hdr = tryDecryptHeader(hk, rc.rcAD, encHdr)
+    if (hdr) {
+      const mk = msgKeys.get(hdr.msgNs)
+      if (mk) {
+        const bodyAD = concatBytes(rc.rcAD, encMsg.emHeader)
+        const plaintext = decryptAEAD(mk.mk, mk.iv, bodyAD, encMsg.emBody, encMsg.emAuthTag)
+        const newMsgKeys = new Map(msgKeys)
+        newMsgKeys.delete(hdr.msgNs)
+        const newSkipped = new Map(skippedKeys)
+        if (newMsgKeys.size === 0) newSkipped.delete(hkHex)
+        else newSkipped.set(hkHex, newMsgKeys)
+        return {plaintext, state: rc, skippedKeys: newSkipped}
+      }
+      // Header decrypted but msgNs not in skipped keys - check if same/advance ratchet
+      // For now, fall through to normal decrypt
+    }
+  }
+  return null
+}
+
+// -- pqRatchetStep (lines 1072-1104)
+// Returns (kemSS for receive rootKdf, kemSS' for send rootKdf, new RatchetKEM state)
+
+function pqRatchetStep(
+  rc: Ratchet,
+  msgKEM: RKEMParams | null,
+): {kemSS: Uint8Array | null; kemSS2: Uint8Array | null; rcKEM: RatchetKEM | null} {
+  const pqEnc = rc.rcEnableKEM
+  const v = rc.rcVersion.current
+
+  if (!msgKEM) {
+    // Received message does not have KEM in header
+    if (!rc.rcKEM && pqEnc && v >= pqRatchetE2EEncryptVersion) {
+      // User enabled KEM but no KEM state yet - generate new keypair
+      const rcPQRs = sntrup761Keypair()
+      return {kemSS: null, kemSS2: null, rcKEM: {rcPQRs, rcKEMs: null}}
+    }
+    return {kemSS: null, kemSS2: null, rcKEM: null}
+  }
+
+  // Received message has KEM in header
+  if (pqEnc && v >= pqRatchetE2EEncryptVersion) {
+    // Get shared secret from received KEM params
+    const {ss, rcPQRr} = kemSharedSecret(rc.rcKEM, msgKEM)
+    // state.PQRct = PQKEM-ENC(state.PQRr, state.PQRss)
+    const kemEncResult = sntrup761Enc(rcPQRr)
+    // state.PQRs = GENERATE_PQKEM()
+    const rcPQRs = sntrup761Keypair()
+    const kem: RatchetKEM = {
+      rcPQRs,
+      rcKEMs: {rcPQRr, rcPQRss: kemEncResult.sharedSecret, rcPQRct: kemEncResult.ciphertext},
+    }
+    return {kemSS: ss, kemSS2: kemEncResult.sharedSecret, rcKEM: kem}
+  }
+
+  // PQ not enabled but message has KEM - extract shared secret only (no new KEM state)
+  const {ss} = kemSharedSecret(rc.rcKEM, msgKEM)
+  return {kemSS: ss, kemSS2: null, rcKEM: null}
+}
+
+// Extract shared secret from received KEM params (lines 1097-1104)
+function kemSharedSecret(
+  rcKEM: RatchetKEM | null,
+  params: RKEMParams,
+): {ss: Uint8Array | null; rcPQRr: Uint8Array} {
+  if (params.type === "proposed") {
+    // RKParamsProposed k -> no shared secret yet, just received the public key
+    return {ss: null, rcPQRr: params.kemPk}
+  }
+  // RKParamsAccepted ct k -> decapsulate ct with our private KEM key
+  if (!rcKEM) throw new Error("pqRatchetStep: CERatchetKEMState - no KEM state for accepted params")
+  const ss = sntrup761Dec(params.kemCt, rcKEM.rcPQRs.secretKey)
+  return {ss, rcPQRr: params.kemPk}
+}
+
+// -- decryptHeader helper (lines 1151-1153)
+
+function tryDecryptHeader(headerKey: Uint8Array, ad: Uint8Array, encHdr: EncMessageHeader): MsgHeader | null {
+  try {
+    const plainHeader = decryptAEAD(headerKey, encHdr.ehIV, ad, encHdr.ehBody, encHdr.ehAuthTag)
+    return decodeMsgHeader(encHdr.ehVersion, plainHeader)
+  } catch {
+    return null
+  }
+}

smp-web/src/crypto/shortLink.ts

@@ -0,0 +1,50 @@
+// Short link key derivation and decryption.
+// Mirrors: Simplex.Messaging.Crypto.ShortLink
+
+import {hkdf} from "../crypto.js"
+import {cbDecrypt} from "@simplex-chat/xftp-web/dist/crypto/secretbox.js"
+import {Decoder, decodeBytes} from "@simplex-chat/xftp-web/dist/protocol/encoding.js"
+
+const emptySalt = new Uint8Array(0)
+
+// contactShortLinkKdf (Crypto/ShortLink.hs:47-50)
+// hkdf("", linkKey, "SimpleXContactLink", 56) -> (linkId[24], sbKey[32])
+export function contactShortLinkKdf(linkKey: Uint8Array): {linkId: Uint8Array; sbKey: Uint8Array} {
+  const derived = hkdf(emptySalt, linkKey, "SimpleXContactLink", 56)
+  return {
+    linkId: derived.slice(0, 24),
+    sbKey: derived.slice(24, 56),
+  }
+}
+
+// invShortLinkKdf (Crypto/ShortLink.hs:52-53)
+// hkdf("", linkKey, "SimpleXInvLink", 32) -> sbKey[32]
+export function invShortLinkKdf(linkKey: Uint8Array): Uint8Array {
+  return hkdf(emptySalt, linkKey, "SimpleXInvLink", 32)
+}
+
+// decryptLinkData (Crypto/ShortLink.hs:100-125)
+// Decrypts both EncDataBytes blobs, strips signature prefix, returns raw data.
+// Signature verification is skipped for spike.
+export function decryptLinkData(
+  sbKey: Uint8Array,
+  encFixedData: Uint8Array,
+  encUserData: Uint8Array
+): {fixedData: Uint8Array; userData: Uint8Array} {
+  return {
+    fixedData: decryptSigned(sbKey, encFixedData),
+    userData: decryptSigned(sbKey, encUserData),
+  }
+}
+
+// EncDataBytes format: [nonce 24 bytes][ciphertext with prepended Poly1305 tag]
+// After decrypt+unpad: [sig ByteString (1-byte len + 64 bytes)][data]
+function decryptSigned(sbKey: Uint8Array, encData: Uint8Array): Uint8Array {
+  const nonce = encData.subarray(0, 24)
+  const ct = encData.subarray(24)
+  const plaintext = cbDecrypt(sbKey, nonce, ct)
+  // Skip signature: decodeBytes reads 1-byte length + that many bytes
+  const d = new Decoder(plaintext)
+  decodeBytes(d) // signature, discarded
+  return d.takeAll()
+}

smp-web/src/crypto/sntrup761.ts

@@ -0,0 +1,89 @@
+// SNTRUP761 post-quantum KEM.
+// Mirrors: Simplex.Messaging.Crypto.SNTRUP761
+//
+// Uses WASM compiled from the same C source as the Haskell build
+// (cbits/sntrup761.c by djb et al., public domain).
+// SHA-512 from SUPERCOP/NaCl (djb, public domain).
+
+// Key sizes (from sntrup761.h)
+export const SNTRUP761_PUBLICKEY_SIZE = 1158
+export const SNTRUP761_SECRETKEY_SIZE = 1763
+export const SNTRUP761_CIPHERTEXT_SIZE = 1039
+export const SNTRUP761_SIZE = 32 // shared secret
+
+export interface KEMKeyPair {
+  publicKey: Uint8Array  // 1158 bytes
+  secretKey: Uint8Array  // 1763 bytes
+}
+
+export interface KEMEncResult {
+  ciphertext: Uint8Array  // 1039 bytes
+  sharedSecret: Uint8Array // 32 bytes
+}
+
+// WASM module instance
+let wasmModule: any = null
+
+export async function initSntrup761(): Promise<void> {
+  if (wasmModule) return
+  const createSntrup761 = (await import("../../dist/wasm/sntrup761.mjs")).default
+  wasmModule = await createSntrup761()
+}
+
+function getModule(): any {
+  if (!wasmModule) throw new Error("sntrup761 WASM not initialized - call initSntrup761() first")
+  return wasmModule
+}
+
+export function sntrup761Keypair(): KEMKeyPair {
+  const m = getModule()
+  const pkPtr = m._malloc(SNTRUP761_PUBLICKEY_SIZE)
+  const skPtr = m._malloc(SNTRUP761_SECRETKEY_SIZE)
+  try {
+    m._sntrup761_wasm_keypair(pkPtr, skPtr)
+    const publicKey = new Uint8Array(m.HEAPU8.buffer, pkPtr, SNTRUP761_PUBLICKEY_SIZE).slice()
+    const secretKey = new Uint8Array(m.HEAPU8.buffer, skPtr, SNTRUP761_SECRETKEY_SIZE).slice()
+    return {publicKey, secretKey}
+  } finally {
+    m._free(pkPtr)
+    m._free(skPtr)
+  }
+}
+
+export function sntrup761Enc(publicKey: Uint8Array): KEMEncResult {
+  if (publicKey.length !== SNTRUP761_PUBLICKEY_SIZE) throw new Error("bad public key length")
+  const m = getModule()
+  const pkPtr = m._malloc(SNTRUP761_PUBLICKEY_SIZE)
+  const ctPtr = m._malloc(SNTRUP761_CIPHERTEXT_SIZE)
+  const ssPtr = m._malloc(SNTRUP761_SIZE)
+  try {
+    m.HEAPU8.set(publicKey, pkPtr)
+    m._sntrup761_wasm_enc(ctPtr, ssPtr, pkPtr)
+    const ciphertext = new Uint8Array(m.HEAPU8.buffer, ctPtr, SNTRUP761_CIPHERTEXT_SIZE).slice()
+    const sharedSecret = new Uint8Array(m.HEAPU8.buffer, ssPtr, SNTRUP761_SIZE).slice()
+    return {ciphertext, sharedSecret}
+  } finally {
+    m._free(pkPtr)
+    m._free(ctPtr)
+    m._free(ssPtr)
+  }
+}
+
+export function sntrup761Dec(ciphertext: Uint8Array, secretKey: Uint8Array): Uint8Array {
+  if (ciphertext.length !== SNTRUP761_CIPHERTEXT_SIZE) throw new Error("bad ciphertext length")
+  if (secretKey.length !== SNTRUP761_SECRETKEY_SIZE) throw new Error("bad secret key length")
+  const m = getModule()
+  const ctPtr = m._malloc(SNTRUP761_CIPHERTEXT_SIZE)
+  const skPtr = m._malloc(SNTRUP761_SECRETKEY_SIZE)
+  const ssPtr = m._malloc(SNTRUP761_SIZE)
+  try {
+    m.HEAPU8.set(ciphertext, ctPtr)
+    m.HEAPU8.set(secretKey, skPtr)
+    m._sntrup761_wasm_dec(ssPtr, ctPtr, skPtr)
+    return new Uint8Array(m.HEAPU8.buffer, ssPtr, SNTRUP761_SIZE).slice()
+  } finally {
+    m._free(ctPtr)
+    m._free(skPtr)
+    m._free(ssPtr)
+  }
+}

smp-web/src/index.ts

@@ -0,0 +1,12 @@
+// SMP protocol client for web/browser environments.
+// Re-exports encoding primitives from xftp-web for convenience.
+export {
+  Decoder,
+  encodeBytes, decodeBytes,
+  encodeLarge, decodeLarge,
+  encodeWord16, decodeWord16,
+  encodeBool, decodeBool,
+  encodeMaybe, decodeMaybe,
+  encodeList, decodeList,
+  concatBytes
+} from "@simplex-chat/xftp-web/dist/protocol/encoding.js"

smp-web/src/protocol.ts

@@ -0,0 +1,571 @@
+// SMP protocol commands and transmission format.
+// Mirrors: Simplex.Messaging.Protocol + Simplex.Messaging.Client (auth)
+
+import {
+  Decoder, concatBytes,
+  encodeBytes, decodeBytes,
+  encodeLarge, decodeLarge,
+  encodeWord16, decodeWord16,
+  encodeBool, decodeBool,
+  encodeMaybe, decodeMaybe,
+} from "@simplex-chat/xftp-web/dist/protocol/encoding.js"
+import {cbEncrypt, cbDecrypt} from "@simplex-chat/xftp-web/dist/crypto/secretbox.js"
+import {sign} from "@simplex-chat/xftp-web/dist/crypto/keys.js"
+import {readTag, readSpace} from "@simplex-chat/xftp-web/dist/protocol/commands.js"
+import {cbAuthenticator} from "./crypto.js"
+
+// -- Auth key type for command authentication (Client.hs:1372-1391)
+
+export type AuthKey =
+  | {type: "x25519", key: Uint8Array}   // raw 32-byte private key → cbAuthenticator
+  | {type: "ed25519", key: Uint8Array}  // raw 64-byte private key → sign
+
+// -- Transmission encoding (Protocol.hs:2186-2198)
+
+// encodeTransmission_ (Protocol.hs:2194-2198)
+// smpEncode (corrId, entityId) <> encodeProtocol v command
+// (command is pre-encoded bytes)
+export function encodeTransmission(corrId: Uint8Array, entityId: Uint8Array, command: Uint8Array): Uint8Array {
+  return concatBytes(encodeBytes(corrId), encodeBytes(entityId), command)
+}
+
+// encodeTransmissionForAuth (Protocol.hs:2186-2192)
+// implySessId = true for v>=7 (always true for web client v19)
+// tForAuth = sessionId <> encodeTransmission_(...)
+// tToSend = encodeTransmission_(...)
+export function encodeTransmissionForAuth(
+  sessionId: Uint8Array, corrId: Uint8Array, entityId: Uint8Array, command: Uint8Array,
+): {tForAuth: Uint8Array, tToSend: Uint8Array} {
+  const tToSend = encodeTransmission(corrId, entityId, command)
+  const tForAuth = concatBytes(encodeBytes(sessionId), tToSend)
+  return {tForAuth, tToSend}
+}
+
+// -- Command authentication (Client.hs:1372-1391)
+
+// authTransmission: produce auth bytes for a transmission
+// Returns null for unauthenticated commands, Uint8Array of auth bytes otherwise
+export function authTransmission(
+  serverPubKey: Uint8Array,  // server's X25519 public key from handshake
+  privKey: AuthKey | null,   // null for unauthenticated commands (LGET, SEND without key)
+  nonce: Uint8Array,         // 24-byte CorrId/nonce (same bytes)
+  tForAuth: Uint8Array,      // transmission bytes to authenticate
+): Uint8Array | null {
+  if (privKey === null) return null
+  switch (privKey.type) {
+    case "x25519":
+      // TAAuthenticator: cbAuthenticate(serverPubKey, entityPrivKey, nonce, tForAuth)
+      return cbAuthenticator(serverPubKey, privKey.key, nonce, tForAuth)
+    case "ed25519":
+      // TASignature: sign(entityPrivKey, tForAuth)
+      return sign(privKey.key, tForAuth)
+  }
+}
+
+// tEncodeAuth (Protocol.hs:507-516)
+// For v16+ (serviceAuth=true): when auth is present, encode serviceSig as Nothing (0x30) after auth.
+// When auth is absent: just empty ByteString.
+export function tEncodeAuth(auth: Uint8Array | null): Uint8Array {
+  if (auth === null) return encodeBytes(new Uint8Array(0))  // empty ByteString: [0x00]
+  // serviceAuth=true for v16+: smpEncode (authBytes, serviceSig) where serviceSig = Nothing
+  return concatBytes(encodeBytes(auth), new Uint8Array([0x30]))  // auth + Nothing
+}
+
+// tEncode (Protocol.hs:2171-2172)
+export function tEncode(auth: Uint8Array | null, tToSend: Uint8Array): Uint8Array {
+  return concatBytes(tEncodeAuth(auth), tToSend)
+}
+
+// tEncodeBatch1 (Protocol.hs:2179-2180)
+// Single-command batch: count=1 + Large(tEncode(...))
+export function tEncodeBatch1(auth: Uint8Array | null, tToSend: Uint8Array): Uint8Array {
+  return concatBytes(new Uint8Array([1]), encodeLarge(tEncode(auth, tToSend)))
+}
+
+// tEncodeForBatch (Protocol.hs:2175-2176)
+// Large(tEncode(...)) — for multi-command batches
+export function tEncodeForBatch(auth: Uint8Array | null, tToSend: Uint8Array): Uint8Array {
+  return encodeLarge(tEncode(auth, tToSend))
+}
+
+// batchTransmissions (Protocol.hs:2151-2168)
+// Pack multiple encoded transmissions into ≤blockSize blocks.
+// Each input is an already-encoded Large-wrapped transmission.
+// Returns array of blocks, each prefixed with count byte.
+export function batchTransmissions(blockSize: number, transmissions: Uint8Array[]): Uint8Array[] {
+  const maxPayload = blockSize - 19 // 2 pad + 1 count + 16 auth tag
+  const blocks: Uint8Array[] = []
+  let currentParts: Uint8Array[] = []
+  let currentLen = 0
+  let count = 0
+  for (const t of transmissions) {
+    const tLen = t.length
+    if (tLen > maxPayload) throw new Error("batchTransmissions: transmission too large")
+    if (currentLen + tLen > maxPayload || count >= 255) {
+      if (count > 0) blocks.push(concatBytes(new Uint8Array([count]), ...currentParts))
+      currentParts = [t]
+      currentLen = tLen
+      count = 1
+    } else {
+      currentParts.push(t)
+      currentLen += tLen
+      count++
+    }
+  }
+  if (count > 0) blocks.push(concatBytes(new Uint8Array([count]), ...currentParts))
+  return blocks
+}
+
+// -- Transmission parsing (Protocol.hs:1629-1643, 2211-2267)
+
+export interface RawTransmission {
+  corrId: Uint8Array
+  entityId: Uint8Array
+  command: Uint8Array
+}
+
+// transmissionP (Protocol.hs:1629-1642)
+// Parse a single transmission from block bytes.
+// implySessId=true, serviceAuth=false for web client.
+export function transmissionP(data: Uint8Array): RawTransmission {
+  const d = new Decoder(data)
+  const auth = decodeBytes(d)  // authenticator
+  // serviceAuth=true for v16+: if auth is non-empty, skip serviceSig (Maybe Signature)
+  if (auth.length > 0) {
+    decodeMaybe(decodeBytes, d)  // skip serviceSig
+  }
+  const rest = d.takeAll()      // authorized bytes
+  // re-parse authorized: corrId + entityId + command
+  const d2 = new Decoder(rest)
+  // implySessId=true: no sessionId in wire format
+  const corrId = decodeBytes(d2)
+  const entityId = decodeBytes(d2)
+  const command = d2.takeAll()
+  return {corrId, entityId, command}
+}
+
+// tParse (Protocol.hs:2211-2217)
+// Parse a received block into individual transmissions.
+// batch=true: count byte + N Large-wrapped transmissions
+export function tParse(block: Uint8Array): RawTransmission[] {
+  const d = new Decoder(block)
+  const count = d.anyByte()
+  const transmissions: RawTransmission[] = []
+  for (let i = 0; i < count; i++) {
+    const data = decodeLarge(d)
+    transmissions.push(transmissionP(data))
+  }
+  return transmissions
+}
+
+// tDecodeClient (Protocol.hs:2256-2266)
+// Parse command bytes into typed response.
+export function tDecodeClient(raw: RawTransmission): {corrId: Uint8Array, entityId: Uint8Array, response: SMPResponse} {
+  const response = decodeResponse(new Decoder(raw.command))
+  return {corrId: raw.corrId, entityId: raw.entityId, response}
+}
+
+// -- SMP command tags
+
+const SPACE = 0x20
+
+function ascii(s: string): Uint8Array {
+  const buf = new Uint8Array(s.length)
+  for (let i = 0; i < s.length; i++) buf[i] = s.charCodeAt(i)
+  return buf
+}
+
+// -- LGET command (Protocol.hs:1709)
+// No parameters. EntityId carries LinkId in transmission.
+
+export function encodeLGET(): Uint8Array {
+  return ascii("LGET")
+}
+
+// -- LNK response (Protocol.hs:1834)
+// LNK sId d -> e (LNK_, ' ', sId, d)
+// where d = (EncFixedDataBytes, EncUserDataBytes), both Large-encoded
+
+export interface LNKResponse {
+  senderId: Uint8Array
+  encFixedData: Uint8Array
+  encUserData: Uint8Array
+}
+
+export function decodeLNK(d: Decoder): LNKResponse {
+  const senderId = decodeBytes(d)
+  const encFixedData = decodeLarge(d)
+  const encUserData = decodeLarge(d)
+  return {senderId, encFixedData, encUserData}
+}
+
+// -- Response dispatch (same pattern as xftp-web decodeResponse)
+
+export interface PKEYResponse {
+  sessionId: Uint8Array
+  versionRange: {min: number, max: number}
+  certChainDer: Uint8Array   // Large-encoded DER certificate chain
+  signedKeyDer: Uint8Array   // Large-encoded DER signed public key
+}
+
+export type SMPResponse =
+  | {type: "LNK", response: LNKResponse}
+  | {type: "IDS", response: IDSResponse}
+  | {type: "MSG", response: MSGResponse}
+  | {type: "OK"}
+  | {type: "SOK", serviceId: Uint8Array | null}
+  | {type: "PKEY", response: PKEYResponse}
+  | {type: "PRES", encResponse: Uint8Array}
+  | {type: "PONG"}
+  | {type: "END"}
+  | {type: "DELD"}
+  | {type: "ERR", error: string}
+
+// protocolError check (Client.hs:710-712)
+// Returns the error string if this is an ERR response, null otherwise
+export function protocolError(resp: SMPResponse): string | null {
+  return resp.type === "ERR" ? resp.error : null
+}
+
+export function decodeResponse(d: Decoder): SMPResponse {
+  const tag = readTag(d)
+  switch (tag) {
+    case "LNK": {
+      readSpace(d)
+      return {type: "LNK", response: decodeLNK(d)}
+    }
+    case "IDS": {
+      readSpace(d)
+      return {type: "IDS", response: decodeIDS(d)}
+    }
+    case "MSG": {
+      readSpace(d)
+      return {type: "MSG", response: decodeMSG(d)}
+    }
+    case "OK": return {type: "OK"}
+    case "SOK": {
+      // SOK serviceId_ → e(SOK_, ' ', serviceId_)
+      readSpace(d)
+      const serviceId = d.remaining() > 0 ? decodeMaybe(decodeBytes, d) : null
+      return {type: "SOK", serviceId}
+    }
+    case "PKEY": {
+      // PKEY sessionId versionRange certChainPubKey (Protocol.hs:1894)
+      // PKEY_ -> PKEY <$> _smpP <*> smpP <*> smpP
+      // sessionId: ByteString, versionRange: (Word16, Word16)
+      // certChainPubKey: (NonEmpty Large, SignedObject) (Transport.hs:663-664)
+      //   certChain = NonEmpty Large = 1-byte count + N × Large(2-byte len + DER)
+      //   signedKey = Large(2-byte len + DER)
+      readSpace(d)
+      const sessionId = decodeBytes(d)
+      const min = decodeWord16(d)
+      const max = decodeWord16(d)
+      // certChain: NonEmpty Large (1-byte count + N × Large-encoded DER certs)
+      const certCount = d.anyByte()
+      const certChainDers: Uint8Array[] = []
+      for (let i = 0; i < certCount; i++) certChainDers.push(decodeLarge(d))
+      // signedKey: Large-encoded DER
+      const signedKeyDer = decodeLarge(d)
+      return {type: "PKEY", response: {sessionId, versionRange: {min, max}, certChainDer: certChainDers[0] ?? new Uint8Array(0), signedKeyDer}}
+    }
+    case "PRES": {
+      // PRES (EncResponse encBlock) (Protocol.hs:1896)
+      // PRES_ -> PRES <$> (EncResponse . unTail <$> _smpP)
+      readSpace(d)
+      return {type: "PRES", encResponse: d.takeAll()}
+    }
+    case "PONG": return {type: "PONG"}
+    case "END": return {type: "END"}
+    case "DELD": return {type: "DELD"}
+    case "ERR": {
+      readSpace(d)
+      // Read the full error string (may be multi-word like "AUTH" or "QUOTA")
+      const errBytes = d.takeAll()
+      return {type: "ERR", error: new TextDecoder().decode(errBytes)}
+    }
+    default: throw new Error("unknown SMP response: " + tag)
+  }
+}
+
+// -- SMP command encoders (Protocol.hs:1679-1715)
+
+// MsgFlags (Protocol.hs:884-892)
+// Single byte: Bool encoding of notification flag
+export function encodeMsgFlags(notification: boolean): Uint8Array {
+  return encodeBool(notification)
+}
+
+// SubscriptionMode (Protocol.hs:651-659)
+// 'S' = SMSubscribe, 'C' = SMOnlyCreate
+export function encodeSubMode(subscribe: boolean): Uint8Array {
+  return ascii(subscribe ? "S" : "C")
+}
+
+// NEW (Protocol.hs:1682-1689)
+// For v19: e(NEW_, ' ', rKey, dhKey) <> e(auth_, subMode, queueReqData, ntfCreds)
+// QueueReqData: QRMessaging Nothing = 'M' + Nothing(0x30)
+export function encodeNEW(
+  rcvAuthKey: Uint8Array,   // DER-encoded Ed25519 or X25519 public key
+  rcvDhKey: Uint8Array,     // DER-encoded X25519 public key
+  basicAuth: Uint8Array | null,  // Maybe BasicAuth (server auth, not a crypto key)
+  subscribe: boolean,
+): Uint8Array {
+  // QRMessaging Nothing: Just('M', Nothing) = 0x31 0x4D 0x30
+  const queueReqData = new Uint8Array([0x31, 0x4D, 0x30])
+  return concatBytes(
+    ascii("NEW "),
+    encodeBytes(rcvAuthKey),
+    encodeBytes(rcvDhKey),
+    encodeMaybe(encodeBytes, basicAuth),
+    encodeSubMode(subscribe),
+    queueReqData,
+    new Uint8Array([0x30]),  // ntfCreds = Nothing
+  )
+}
+
+// KEY (Protocol.hs:1692)
+// KEY k -> e(KEY_, ' ', k)
+export function encodeKEY(senderKey: Uint8Array): Uint8Array {
+  return concatBytes(ascii("KEY "), encodeBytes(senderKey))
+}
+
+// SKEY (Protocol.hs:1703)
+// SKEY k -> e(SKEY_, ' ', k)
+export function encodeSKEY(senderKey: Uint8Array): Uint8Array {
+  return concatBytes(ascii("SKEY "), encodeBytes(senderKey))
+}
+
+// SUB (Protocol.hs:1690)
+export function encodeSUB(): Uint8Array {
+  return ascii("SUB")
+}
+
+// ACK (Protocol.hs:1699)
+// ACK msgId -> e(ACK_, ' ', msgId)
+export function encodeACK(msgId: Uint8Array): Uint8Array {
+  return concatBytes(ascii("ACK "), encodeBytes(msgId))
+}
+
+// SEND (Protocol.hs:1704)
+// SEND flags msg -> e(SEND_, ' ', flags, ' ', Tail msg)
+export function encodeSEND(notification: boolean, msgBody: Uint8Array): Uint8Array {
+  return concatBytes(
+    ascii("SEND "),
+    encodeMsgFlags(notification),
+    ascii(" "),
+    msgBody, // Tail - no length prefix
+  )
+}
+
+// OFF (Protocol.hs:1700)
+export function encodeOFF(): Uint8Array {
+  return ascii("OFF")
+}
+
+// DEL (Protocol.hs:1701)
+export function encodeDEL(): Uint8Array {
+  return ascii("DEL")
+}
+
+// GET (Protocol.hs:1698)
+export function encodeGET(): Uint8Array {
+  return ascii("GET")
+}
+
+// QUE (Protocol.hs:1702)
+export function encodeQUE(): Uint8Array {
+  return ascii("QUE")
+}
+
+// PING (Protocol.hs:1705)
+export function encodePING(): Uint8Array {
+  return ascii("PING")
+}
+
+// -- Proxy commands (Protocol.hs:1710-1711)
+
+// encodeProtocolServer (Protocol.hs:1264-1266)
+// smpEncode ProtocolServer {host, port, keyHash} = smpEncode (host, port, keyHash)
+// host :: NonEmpty TransportHost → smpEncodeList (1-byte count + encodeBytes(strEncode(host)) for each)
+// port :: ServiceName = ByteString → encodeBytes
+// keyHash :: KeyHash = ByteString → encodeBytes
+export function encodeProtocolServer(hosts: string[], port: string, keyHash: Uint8Array): Uint8Array {
+  const encodedHosts = hosts.map(h => encodeBytes(ascii(h)))
+  const hostList = concatBytes(new Uint8Array([hosts.length]), ...encodedHosts)
+  return concatBytes(hostList, encodeBytes(ascii(port)), encodeBytes(keyHash))
+}
+
+// PRXY (Protocol.hs:1710)
+// PRXY host auth_ -> e(PRXY_, ' ', host, auth_)
+export function encodePRXY(hosts: string[], port: string, keyHash: Uint8Array, basicAuth: Uint8Array | null): Uint8Array {
+  return concatBytes(
+    ascii("PRXY "),
+    encodeProtocolServer(hosts, port, keyHash),
+    encodeMaybe(encodeBytes, basicAuth),
+  )
+}
+
+// PFWD (Protocol.hs:1711)
+// PFWD fwdV pubKey (EncTransmission s) -> e(PFWD_, ' ', fwdV, pubKey, Tail s)
+export function encodePFWD(version: number, pubKeyDer: Uint8Array, encTransmission: Uint8Array): Uint8Array {
+  return concatBytes(
+    ascii("PFWD "),
+    encodeWord16(version),
+    encodeBytes(pubKeyDer),
+    encTransmission, // Tail — no length prefix
+  )
+}
+
+// paddedProxiedTLength (Protocol.hs:306-307)
+export const paddedProxiedTLength = 16226
+
+// -- SMP response decoders
+
+// IDS (Protocol.hs:1914-1921)
+// For v19: e(IDS_, ' ', rcvId, sndId, srvDh) <> e(queueMode, linkId, serviceId, ntfCreds)
+export interface IDSResponse {
+  rcvId: Uint8Array
+  sndId: Uint8Array
+  srvDhKey: Uint8Array
+  queueMode: string | null  // 'M' = Messaging, 'C' = Contact
+  linkId: Uint8Array | null
+}
+
+export function decodeIDS(d: Decoder): IDSResponse {
+  const rcvId = decodeBytes(d)
+  const sndId = decodeBytes(d)
+  const srvDhKey = decodeBytes(d)
+  // v19: queueMode (Maybe QueueMode), linkId (Maybe ByteString), serviceId, ntfCreds
+  // QueueMode is encoded as Maybe Char ('M'/'C'), not Maybe ByteString
+  let queueMode: string | null = null
+  if (d.remaining() > 0) {
+    const qmByte = d.anyByte()
+    if (qmByte === 0x31) { // '1' = Just
+      queueMode = String.fromCharCode(d.anyByte())
+    }
+    // '0' = Nothing, queueMode stays null
+  }
+  let linkId: Uint8Array | null = null
+  if (d.remaining() > 0) {
+    linkId = decodeMaybe(decodeBytes, d)
+  }
+  // serviceId and ntfCreds - skip remaining
+  return {rcvId, sndId, srvDhKey, queueMode, linkId}
+}
+
+// MSG (Protocol.hs:1927-1928)
+// MSG RcvMessage {msgId, msgBody = EncRcvMsgBody body} -> e(MSG_, ' ', msgId, Tail body)
+export interface MSGResponse {
+  msgId: Uint8Array
+  msgBody: Uint8Array
+}
+
+export function decodeMSG(d: Decoder): MSGResponse {
+  const msgId = decodeBytes(d)
+  const msgBody = d.takeAll()
+  return {msgId, msgBody}
+}
+
+// -- Per-queue E2E encryption (Protocol.hs:1071-1114)
+
+// Protocol.hs:316-320
+export const e2eEncMessageLength = 16000
+export const e2eEncConfirmationLength = 15904
+
+// Protocol.hs:1078-1086
+export interface PubHeader {
+  phVersion: number                    // VersionSMPC (Word16)
+  phE2ePubDhKey: Uint8Array | null     // Maybe PublicKeyX25519 (DER-encoded ByteString)
+}
+
+export function encodePubHeader(h: PubHeader): Uint8Array {
+  return concatBytes(encodeWord16(h.phVersion), encodeMaybe(encodeBytes, h.phE2ePubDhKey))
+}
+
+export function decodePubHeader(d: Decoder): PubHeader {
+  return {phVersion: decodeWord16(d), phE2ePubDhKey: decodeMaybe(decodeBytes, d)}
+}
+
+// Protocol.hs:1097-1110
+export type PrivHeader =
+  | {type: "PHConfirmation", key: Uint8Array}  // 'K' + DER-encoded APublicAuthKey
+  | {type: "PHEmpty"}                           // '_'
+
+export function encodePrivHeader(h: PrivHeader): Uint8Array {
+  switch (h.type) {
+    case "PHConfirmation": return concatBytes(new Uint8Array([0x4B]), encodeBytes(h.key)) // 'K' + encodeBytes
+    case "PHEmpty": return new Uint8Array([0x5F]) // '_'
+  }
+}
+
+export function decodePrivHeader(d: Decoder): PrivHeader {
+  const tag = d.anyByte()
+  switch (tag) {
+    case 0x4B: return {type: "PHConfirmation", key: decodeBytes(d)} // 'K'
+    case 0x5F: return {type: "PHEmpty"} // '_'
+    default: throw new Error("decodePrivHeader: unknown tag " + tag)
+  }
+}
+
+// Protocol.hs:1095, 1112-1114
+export interface ClientMessage {
+  privHeader: PrivHeader
+  body: Uint8Array
+}
+
+// smpEncode (ClientMessage h msg) = smpEncode h <> msg
+export function encodeClientMessage(msg: ClientMessage): Uint8Array {
+  return concatBytes(encodePrivHeader(msg.privHeader), msg.body)
+}
+
+export function decodeClientMessage(d: Decoder): ClientMessage {
+  const privHeader = decodePrivHeader(d)
+  const body = d.takeAll()
+  return {privHeader, body}
+}
+
+// Protocol.hs:1071-1093
+export interface ClientMsgEnvelope {
+  cmHeader: PubHeader
+  cmNonce: Uint8Array       // CbNonce: raw 24 bytes
+  cmEncBody: Uint8Array     // encrypted body (Tail)
+}
+
+// smpEncode (cmHeader, cmNonce, Tail cmEncBody)
+export function encodeClientMsgEnvelope(env: ClientMsgEnvelope): Uint8Array {
+  return concatBytes(encodePubHeader(env.cmHeader), env.cmNonce, env.cmEncBody)
+}
+
+export function decodeClientMsgEnvelope(d: Decoder): ClientMsgEnvelope {
+  const cmHeader = decodePubHeader(d)
+  const cmNonce = d.take(24)  // CbNonce is raw 24 bytes
+  const cmEncBody = d.takeAll()
+  return {cmHeader, cmNonce, cmEncBody}
+}
+
+// -- Per-queue E2E encrypt/decrypt (Agent/Client.hs:2074-2102)
+
+// agentCbEncrypt: encrypt a ClientMessage and wrap in ClientMsgEnvelope
+export function agentCbEncrypt(
+  e2eDhSecret: Uint8Array,     // X25519 DH shared secret (32 bytes)
+  smpClientVersion: number,    // Word16
+  e2ePubKey: Uint8Array | null, // DER-encoded X25519 public key, null for normal messages
+  msg: Uint8Array,             // smpEncode(ClientMessage)
+): Uint8Array {
+  const cmNonce = crypto.getRandomValues(new Uint8Array(24))
+  const paddedLen = e2ePubKey !== null ? e2eEncConfirmationLength : e2eEncMessageLength
+  const cmEncBody = cbEncrypt(e2eDhSecret, cmNonce, msg, paddedLen)
+  const cmHeader: PubHeader = {phVersion: smpClientVersion, phE2ePubDhKey: e2ePubKey}
+  return encodeClientMsgEnvelope({cmHeader, cmNonce, cmEncBody})
+}
+
+// agentCbDecrypt: decrypt a ClientMsgEnvelope
+export function agentCbDecrypt(
+  dhSecret: Uint8Array,        // X25519 DH shared secret (32 bytes)
+  data: Uint8Array,            // raw ClientMsgEnvelope bytes
+): {pubHeader: PubHeader, clientMessage: ClientMessage} {
+  const env = decodeClientMsgEnvelope(new Decoder(data))
+  const plaintext = cbDecrypt(dhSecret, env.cmNonce, env.cmEncBody)
+  const clientMessage = decodeClientMessage(new Decoder(plaintext))
+  return {pubHeader: env.cmHeader, clientMessage}
+}

smp-web/src/transport.ts

@@ -0,0 +1,77 @@
+// SMP transport: handshake, block framing.
+// Mirrors: Simplex.Messaging.Transport
+
+import {
+  Decoder, concatBytes,
+  encodeWord16, decodeWord16,
+  encodeBytes, decodeBytes,
+  encodeLarge, decodeLarge,
+  encodeBool,
+  encodeMaybe,
+  decodeNonEmpty
+} from "@simplex-chat/xftp-web/dist/protocol/encoding.js"
+
+// -- Version constants (Transport.hs:186-213)
+
+export const SMP_BLOCK_SIZE = 16384
+export const currentSMPVersion = 19
+export const webClientSMPVersion = 19
+
+// -- SMPServerHandshake (Transport.hs:631-640)
+
+export interface SMPServerHandshake {
+  smpVersionRange: {min: number; max: number}
+  sessionId: Uint8Array
+  authPubKey: SMPAuthPubKey | null
+  webIdentityProof: Uint8Array | null // raw signature bytes (v19+)
+}
+
+export interface SMPAuthPubKey {
+  certChainDer: Uint8Array[] // DER-encoded certificate chain
+  signedKeyDer: Uint8Array  // DER-encoded SignedExact PubKey
+}
+
+export function decodeSMPServerHandshake(d: Decoder): SMPServerHandshake {
+  const min = decodeWord16(d)
+  const max = decodeWord16(d)
+  const sessionId = decodeBytes(d)
+  // authPubKey: version-gated (v7+)
+  let authPubKey: SMPAuthPubKey | null = null
+  if (max >= 7 && d.remaining() > 0) {
+    const certChainDer = decodeNonEmpty(decodeLarge, d)
+    const signedKeyDer = decodeLarge(d)
+    authPubKey = {certChainDer, signedKeyDer}
+  }
+  // webIdentityProof: version-gated (v19+)
+  let webIdentityProof: Uint8Array | null = null
+  if (max >= webClientSMPVersion && d.remaining() > 0) {
+    webIdentityProof = decodeBytes(d)
+  }
+  return {smpVersionRange: {min, max}, sessionId, authPubKey, webIdentityProof}
+}
+
+// -- SMPClientHandshake (Transport.hs:592-604)
+
+export interface SMPClientHandshake {
+  smpVersion: number
+  keyHash: Uint8Array
+  authPubKey: Uint8Array | null // X25519 public key, or null for no block encryption
+  proxyServer: boolean
+  clientService: null // not used in web client
+}
+
+export function encodeSMPClientHandshake(h: SMPClientHandshake): Uint8Array {
+  const parts: Uint8Array[] = [
+    encodeWord16(h.smpVersion),
+    encodeBytes(h.keyHash),
+  ]
+  // authPubKey: encodeAuthEncryptCmds — empty for Nothing, encodeBytes for Just (v7+)
+  if (h.authPubKey !== null) {
+    parts.push(encodeBytes(h.authPubKey))
+  }
+  // proxyServer: Bool (v14+)
+  parts.push(encodeBool(h.proxyServer))
+  // clientService: Maybe (v16+) — Nothing = '0' (0x30)
+  parts.push(encodeMaybe(() => new Uint8Array(0), null))
+  return concatBytes(...parts)
+}

smp-web/src/transport/websockets.ts

@@ -0,0 +1,132 @@
+// WebSocket transport for SMP protocol.
+// Mirrors: Simplex.Messaging.Transport.WebSockets (client side)
+
+import WebSocket from "ws"
+import {randomBytes} from "crypto"
+import {Decoder} from "@simplex-chat/xftp-web/dist/protocol/encoding.js"
+import {base64urlEncode} from "@simplex-chat/xftp-web/dist/protocol/description.js"
+import {blockPad, blockUnpad} from "@simplex-chat/xftp-web/dist/protocol/transmission.js"
+import {verifyIdentityProof} from "@simplex-chat/xftp-web/dist/crypto/identity.js"
+import {generateX25519KeyPair, dh, encodePubKeyX25519} from "@simplex-chat/xftp-web/dist/crypto/keys.js"
+import {extractSignedKey} from "@simplex-chat/xftp-web/dist/protocol/handshake.js"
+import {decodeSMPServerHandshake, encodeSMPClientHandshake, SMP_BLOCK_SIZE, currentSMPVersion} from "../transport.js"
+import {sbcInit, sbEncryptBlock, sbDecryptBlock} from "../crypto.js"
+
+export interface SMPConnection {
+  ws: WebSocket
+  sessionId: Uint8Array
+  smpVersion: number
+  // Block encryption state (null if no auth)
+  sndKey: Uint8Array | null
+  rcvKey: Uint8Array | null
+  // Server's raw X25519 public key — needed for command auth (cbAuthenticate)
+  serverPubKey: Uint8Array | null
+}
+
+export async function connectSMP(url: string, keyHash: Uint8Array, wsOptions?: object): Promise<SMPConnection> {
+  // Generate challenge and append to URL
+  const challenge = new Uint8Array(randomBytes(32))
+  const challengeUrl = url + (url.includes("?") ? "&" : "?") + "challenge=" + base64urlEncode(challenge).replace(/=+$/, "")
+
+  const ws = new WebSocket(challengeUrl, wsOptions)
+  ws.binaryType = "arraybuffer"
+
+  await new Promise<void>((resolve, reject) => {
+    ws.onopen = () => resolve()
+    ws.onerror = (e) => reject(e)
+  })
+
+  // Receive server handshake (first block)
+  const serverBlock = await receiveBlock(ws)
+  const serverHs = decodeSMPServerHandshake(new Decoder(blockUnpad(serverBlock)))
+
+  // Negotiate version
+  const version = Math.min(serverHs.smpVersionRange.max, currentSMPVersion)
+  if (version < 6) throw new Error("Incompatible server version")
+
+  // Verify server identity and extract DH key
+  let sndKey: Uint8Array | null = null
+  let rcvKey: Uint8Array | null = null
+  let clientAuthPubKey: Uint8Array | null = null
+  let serverPubKey: Uint8Array | null = null
+
+  if (serverHs.authPubKey) {
+    // Verify server identity if server supports web challenge (v19+)
+    if (serverHs.webIdentityProof) {
+      const ok = verifyIdentityProof({
+        certChainDer: serverHs.authPubKey.certChainDer,
+        signedKeyDer: serverHs.authPubKey.signedKeyDer,
+        sigBytes: serverHs.webIdentityProof,
+        challenge,
+        sessionId: serverHs.sessionId,
+        keyHash,
+      })
+      if (!ok) throw new Error("Server identity verification failed")
+    }
+
+    // DH key exchange for block encryption (v11+)
+    serverPubKey = extractSignedKey(serverHs.authPubKey.signedKeyDer).dhKey
+    const clientKp = generateX25519KeyPair()
+    clientAuthPubKey = encodePubKeyX25519(clientKp.publicKey)
+    const dhSecret = dh(serverPubKey, clientKp.privateKey)
+    // Client swaps snd/rcv vs server (Transport.hs:880)
+    const keys = sbcInit(serverHs.sessionId, dhSecret)
+    sndKey = keys.rcvKey
+    rcvKey = keys.sndKey
+  }
+
+  // Send client handshake
+  const clientHs = encodeSMPClientHandshake({
+    smpVersion: version,
+    keyHash,
+    authPubKey: clientAuthPubKey,
+    proxyServer: false,
+    clientService: null
+  })
+  sendBlock(ws, blockPad(clientHs, SMP_BLOCK_SIZE))
+
+  return {ws, sessionId: serverHs.sessionId, smpVersion: version, sndKey, rcvKey, serverPubKey}
+}
+
+export function receiveBlock(ws: WebSocket): Promise<Uint8Array> {
+  return new Promise((resolve, reject) => {
+    ws.onmessage = (e) => {
+      const data = e.data
+      if (data instanceof ArrayBuffer) {
+        resolve(new Uint8Array(data))
+      } else if (data instanceof Buffer) {
+        resolve(new Uint8Array(data))
+      } else {
+        reject(new Error("Expected binary frame"))
+      }
+    }
+    ws.onerror = (e) => reject(e)
+  })
+}
+
+export function sendBlock(ws: WebSocket, data: Uint8Array): void {
+  if (data.length !== SMP_BLOCK_SIZE) throw new Error("Block must be " + SMP_BLOCK_SIZE + " bytes")
+  ws.send(data)
+}
+
+// Encrypted block send: pad to (blockSize - 16), encrypt (adds 16-byte tag)
+export function sendEncryptedBlock(conn: SMPConnection, plaintext: Uint8Array): void {
+  if (!conn.sndKey) throw new Error("no block encryption keys")
+  const {encrypted, nextChainKey} = sbEncryptBlock(conn.sndKey, plaintext, SMP_BLOCK_SIZE - 16)
+  conn.sndKey = nextChainKey
+  ws_send(conn.ws, encrypted)
+}
+
+// Encrypted block receive: decrypt (removes 16-byte tag + unpad)
+export async function receiveEncryptedBlock(conn: SMPConnection): Promise<Uint8Array> {
+  if (!conn.rcvKey) throw new Error("no block encryption keys")
+  const block = await receiveBlock(conn.ws)
+  const {decrypted, nextChainKey} = sbDecryptBlock(conn.rcvKey, block)
+  conn.rcvKey = nextChainKey
+  return decrypted
+}
+
+function ws_send(ws: WebSocket, data: Uint8Array): void {
+  if (data.length !== SMP_BLOCK_SIZE) throw new Error("Encrypted block must be " + SMP_BLOCK_SIZE + " bytes")
+  ws.send(data)
+}

smp-web/tests/client-repl.ts

@@ -0,0 +1,257 @@
+// SMP client REPL for cross-language testing.
+// Holds one SMPClient, reads commands from stdin, writes results to stdout.
+//
+// Commands:
+//   CONNECT <url> <keyHashHex> [wsOptionsJson]
+//   NEW <rcvAuthKeyHex> <rcvDhKeyHex> <rcvPrivKeyHex>
+//   SUB <rcvIdHex> <rcvPrivKeyHex>
+//   SEND <sndIdHex> <sndPrivKeyHex|none> <notification 0|1> <bodyHex>
+//   ACK <rcvIdHex> <rcvPrivKeyHex> <msgIdHex>
+//   KEY <rcvIdHex> <rcvPrivKeyHex> <senderKeyHex>
+//   SKEY <sndIdHex> <sndPrivKeyHex>
+//   DEL <rcvIdHex> <rcvPrivKeyHex>
+//   OFF <rcvIdHex> <rcvPrivKeyHex>
+//   PING
+//   RECV [timeoutMs]
+//   CLOSE
+
+import {createInterface} from "readline"
+import {createSMPClient, type SMPClient} from "../dist/client.js"
+import type {SMPResponse, AuthKey} from "../dist/protocol.js"
+import {generateX25519KeyPair, dh, encodePubKeyX25519, decodePubKeyX25519} from "@simplex-chat/xftp-web/dist/crypto/keys.js"
+import {cbDecrypt} from "@simplex-chat/xftp-web/dist/crypto/secretbox.js"
+import {Decoder, decodeBytes, decodeBool} from "@simplex-chat/xftp-web/dist/protocol/encoding.js"
+
+import type {ProxiedRelay} from "../dist/client.js"
+
+// -- State
+
+let client: SMPClient | null = null
+let proxiedRelay: ProxiedRelay | null = null
+// Per-queue DH shared secrets for decrypting received messages (keyed by rcvId hex)
+const queueSecrets = new Map<string, Uint8Array>()
+const messageQueue: Array<{entityId: Uint8Array, msg: SMPResponse}> = []
+let messageWaiter: {resolve: (m: {entityId: Uint8Array, msg: SMPResponse}) => void, timer: ReturnType<typeof setTimeout>} | null = null
+
+// -- Hex helpers
+
+function toHex(bytes: Uint8Array): string {
+  return Array.from(bytes, b => b.toString(16).padStart(2, "0")).join("")
+}
+
+function fromHex(hex: string): Uint8Array {
+  const bytes = new Uint8Array(hex.length / 2)
+  for (let i = 0; i < hex.length; i += 2)
+    bytes[i / 2] = parseInt(hex.substring(i, i + 2), 16)
+  return bytes
+}
+
+// -- Message delivery
+
+function onMessage(entityId: Uint8Array, msg: SMPResponse): void {
+  if (messageWaiter) {
+    const w = messageWaiter
+    messageWaiter = null
+    clearTimeout(w.timer)
+    w.resolve({entityId, msg})
+  } else {
+    messageQueue.push({entityId, msg})
+  }
+}
+
+function waitForMessage(timeoutMs: number): Promise<{entityId: Uint8Array, msg: SMPResponse}> {
+  if (messageQueue.length > 0) {
+    return Promise.resolve(messageQueue.shift()!)
+  }
+  return new Promise((resolve, reject) => {
+    const timer = setTimeout(() => {
+      messageWaiter = null
+      reject(new Error("timeout"))
+    }, timeoutMs)
+    messageWaiter = {resolve, timer}
+  })
+}
+
+function makeAuthKey(hexKey: string): AuthKey {
+  return {type: "x25519", key: fromHex(hexKey)}
+}
+
+// -- Command parser
+
+async function parseLine(line: string): Promise<string> {
+  const parts = line.split(" ")
+  const cmd = parts[0]
+
+  try {
+    switch (cmd) {
+      case "CONNECT": {
+        const url = parts[1]
+        const keyHash = fromHex(parts[2])
+        const wsOptions = parts[3] ? JSON.parse(parts[3]) : undefined
+        client = await createSMPClient(url, keyHash, onMessage, () => {
+          process.stderr.write("disconnected\n")
+        }, {wsOptions, timeout: 15000})
+        return "ok"
+      }
+
+      case "NEW": {
+        if (!client) return "error: not connected"
+        const rcvAuthKey = fromHex(parts[1])
+        const rcvPrivKey = fromHex(parts[2])
+        // Generate DH keypair for per-queue E2E
+        const dhKp = generateX25519KeyPair()
+        const dhPubDer = encodePubKeyX25519(dhKp.publicKey)
+        const resp = await client.createQueue(
+          {publicKey: rcvAuthKey, privateKey: rcvPrivKey},
+          dhPubDer,
+          true,
+        )
+        // Compute and store DH shared secret for decrypting received messages
+        const srvDhRaw = decodePubKeyX25519(resp.srvDhKey)
+        const dhShared = dh(srvDhRaw, dhKp.privateKey)
+        queueSecrets.set(toHex(resp.rcvId), dhShared)
+        return "ok: " + toHex(resp.rcvId) + " " + toHex(resp.sndId) + " " + toHex(resp.srvDhKey)
+      }
+
+      case "SUB": {
+        if (!client) return "error: not connected"
+        await client.subscribeQueue(makeAuthKey(parts[2]), fromHex(parts[1]))
+        return "ok"
+      }
+
+      case "SEND": {
+        if (!client) return "error: not connected"
+        const sndId = fromHex(parts[1])
+        const privKey: AuthKey | null = parts[2] === "none" ? null : makeAuthKey(parts[2])
+        const notification = parts[3] === "1"
+        const body = fromHex(parts[4])
+        await client.sendMessage(privKey, sndId, notification, body)
+        return "ok"
+      }
+
+      case "ACK": {
+        if (!client) return "error: not connected"
+        await client.ackMessage(makeAuthKey(parts[2]), fromHex(parts[1]), fromHex(parts[3]))
+        return "ok"
+      }
+
+      case "KEY": {
+        if (!client) return "error: not connected"
+        await client.secureQueue(makeAuthKey(parts[2]), fromHex(parts[1]), fromHex(parts[3]))
+        return "ok"
+      }
+
+      case "SKEY": {
+        if (!client) return "error: not connected"
+        await client.secureSndQueue(makeAuthKey(parts[2]), fromHex(parts[1]))
+        return "ok"
+      }
+
+      case "DEL": {
+        if (!client) return "error: not connected"
+        await client.deleteQueue(makeAuthKey(parts[2]), fromHex(parts[1]))
+        return "ok"
+      }
+
+      case "OFF": {
+        if (!client) return "error: not connected"
+        await client.suspendQueue(makeAuthKey(parts[2]), fromHex(parts[1]))
+        return "ok"
+      }
+
+      case "PING": {
+        if (!client) return "error: not connected"
+        // Optional auth key as second arg: PING <privKeyHex>
+        const pingKey: AuthKey | null = parts[1] ? makeAuthKey(parts[1]) : null
+        const resp = await client.sendCommand(pingKey, new Uint8Array(0), new TextEncoder().encode("PING"))
+        return resp.type === "PONG" ? "ok" : "error: unexpected " + resp.type
+      }
+
+      case "RECV": {
+        if (!client) return "error: not connected"
+        const timeoutMs = parts[1] ? parseInt(parts[1]) : 5000
+        const m = await waitForMessage(timeoutMs)
+        if (m.msg.type === "MSG") {
+          const {msgId, msgBody} = m.msg.response
+          // Decrypt per-queue E2E: cbDecrypt(dhShared, cbNonce(msgId), body)
+          const dhShared = queueSecrets.get(toHex(m.entityId))
+          if (dhShared) {
+            // decryptMsgV3: cbDecrypt then parse ClientRcvMsgBody (msgTs + msgFlags + space + Tail msgBody)
+            const decrypted = cbDecrypt(dhShared, msgId, msgBody)
+            const dd = new Decoder(decrypted)
+            dd.take(8) // skip msgTs (SystemTime = Int64 = 8 bytes)
+            dd.take(1) // skip msgFlags (Bool = 1 byte)
+            dd.take(1) // skip space (0x20)
+            const body = dd.takeAll()
+            return "ok: " + toHex(m.entityId) + " " + toHex(msgId) + " " + toHex(body)
+          }
+          // No DH secret (sender queue) — return raw
+          return "ok: " + toHex(m.entityId) + " " + toHex(msgId) + " " + toHex(msgBody)
+        }
+        return "ok: " + toHex(m.entityId) + " " + m.msg.type
+      }
+
+      // BSUB <rcvId1Hex>:<privKey1Hex> <rcvId2Hex>:<privKey2Hex> ...
+      case "BSUB": {
+        if (!client) return "error: not connected"
+        const queues = parts.slice(1).map(p => {
+          const [rcvIdHex, privKeyHex] = p.split(":")
+          return {rcvId: fromHex(rcvIdHex), privKey: makeAuthKey(privKeyHex)}
+        })
+        await client.subscribeQueues(queues)
+        return "ok"
+      }
+
+      // PRXY <host1,host2,...> <port> <keyHashHex> [basicAuthHex]
+      case "PRXY": {
+        if (!client) return "error: not connected"
+        const hosts = parts[1].split(",")
+        const port = parts[2]
+        const keyHash = fromHex(parts[3])
+        const auth = parts[4] ? fromHex(parts[4]) : null
+        proxiedRelay = await client.connectProxiedRelay(hosts, port, keyHash, auth)
+        return "ok: " + toHex(proxiedRelay.sessionId) + " " + proxiedRelay.version
+      }
+
+      // PSEND <sndIdHex> <sndPrivKeyHex|none> <notification 0|1> <bodyHex>
+      case "PSEND": {
+        if (!client || !proxiedRelay) return "error: not connected or no proxy session"
+        const sndId = fromHex(parts[1])
+        const privKey: AuthKey | null = parts[2] === "none" ? null : makeAuthKey(parts[2])
+        const notification = parts[3] === "1"
+        const body = fromHex(parts[4])
+        await client.proxySendMessage(proxiedRelay, privKey, sndId, notification, body)
+        return "ok"
+      }
+
+      case "CLOSE": {
+        if (client) client.close()
+        client = null
+        return "ok"
+      }
+
+      default:
+        return "error: unknown command: " + cmd
+    }
+  } catch (e: any) {
+    if (e.type) return "error: " + e.type + (e.error ? " " + e.error : "")
+    return "error: " + (e.message || String(e))
+  }
+}
+
+// -- Main
+
+async function main() {
+  const rl = createInterface({input: process.stdin, terminal: false})
+  for await (const line of rl) {
+    const trimmed = line.trim()
+    if (!trimmed) continue
+    const response = await parseLine(trimmed)
+    process.stdout.write(response + "\n")
+  }
+}
+
+main().catch(e => {
+  process.stderr.write("FATAL: " + e.message + "\n")
+  process.exit(1)
+})

smp-web/tests/ratchet-repl.ts

@@ -0,0 +1,326 @@
+// Double ratchet REPL for cross-language testing.
+// Holds one ratchet state, reads commands from stdin, writes results to stdout.
+//
+// Init protocol:
+//   INIT_RCV <version> <pqSupport 0|1>
+//     → ok: <hex E2E params>
+//   COMPLETE <hex peer E2E params>
+//     → ok
+//   INIT_SND <version> <kemMode> <hex peer E2E params>
+//     → ok: <hex E2E params>
+//     kemMode: none | propose | accept
+//
+// Encrypt/decrypt operators (same syntax as Haskell DoubleRatchetTests):
+//   \#> <plaintext>       encrypt, assert noSndKEM
+//   !#> <plaintext>       encrypt, assert hasSndKEM
+//   \#>! <plaintext>      encrypt PQEncOn, assert noSndKEM
+//   !#>! <plaintext>      encrypt PQEncOn, assert hasSndKEM
+//   !#>\ <plaintext>      encrypt PQEncOff, assert hasSndKEM
+//   \#>\ <plaintext>      encrypt PQEncOff, assert noSndKEM
+//   <#\ <hex ct> <expected>   decrypt, assert noRcvKEM
+//   <#! <hex ct> <expected>   decrypt, assert hasRcvKEM
+//
+// Plain encrypt/decrypt (no assertions):
+//   E <plaintext>         → ok: <hex ciphertext>
+//   D <hex ciphertext>    → ok: <plaintext>
+//
+// Response format: ok: <data> or error: <message>
+
+import {createInterface} from "readline"
+import {
+  generateX448KeyPair, pqX3dhSnd, pqX3dhRcv,
+  encodePubKeyX448, decodePubKeyX448,
+  initSndRatchet, initRcvRatchet,
+  rcEncrypt, rcDecrypt,
+  rootKdf,
+  type Ratchet, type SkippedMsgKeys, type RatchetVersions,
+  type RatchetInitParams, type RatchetKEMAccepted,
+} from "../dist/crypto/ratchet.js"
+import {initSntrup761, sntrup761Keypair, sntrup761Enc, sntrup761Dec} from "../dist/crypto/sntrup761.js"
+import type {KEMKeyPair} from "../dist/crypto/sntrup761.js"
+import {
+  Decoder, decodeBytes, decodeLarge, encodeBytes, encodeWord16, concatBytes,
+} from "@simplex-chat/xftp-web/dist/protocol/encoding.js"
+
+// -- State
+
+let ratchet: Ratchet | null = null
+let skippedKeys: SkippedMsgKeys = new Map()
+const PADDED_MSG_LEN = 16000
+
+// Intermediate state for RCV init (between INIT_RCV and COMPLETE)
+let rcvInitState: {
+  privKey1: Uint8Array
+  privKey2: Uint8Array
+  kemKeyPair: KEMKeyPair | null
+  pqSupport: boolean
+} | null = null
+
+// -- Hex helpers
+
+function toHex(bytes: Uint8Array): string {
+  return Array.from(bytes, b => b.toString(16).padStart(2, "0")).join("")
+}
+
+function fromHex(hex: string): Uint8Array {
+  const bytes = new Uint8Array(hex.length / 2)
+  for (let i = 0; i < hex.length; i += 2)
+    bytes[i / 2] = parseInt(hex.substring(i, i + 2), 16)
+  return bytes
+}
+
+// -- E2E params helpers
+
+// Parse E2ERatchetParams: version(Word16) + pk1(ByteString) + pk2(ByteString) + Maybe KEMParams
+interface ParsedE2EParams {
+  version: number
+  pk1Raw: Uint8Array  // raw X448 public key
+  pk2Raw: Uint8Array  // raw X448 public key
+  kemPk: Uint8Array | null  // KEM public key if proposed
+  kemCt: Uint8Array | null  // KEM ciphertext if accepted
+  kemAcceptPk: Uint8Array | null  // KEM public key in accepted
+}
+
+function parseE2EParams(data: Uint8Array): ParsedE2EParams {
+  const d = new Decoder(data)
+  const version = d.anyByte() * 256 + d.anyByte()
+  const pk1Raw = decodePubKeyX448(decodeBytes(d))
+  const pk2Raw = decodePubKeyX448(decodeBytes(d))
+  let kemPk: Uint8Array | null = null
+  let kemCt: Uint8Array | null = null
+  let kemAcceptPk: Uint8Array | null = null
+  if (version >= 3 && d.remaining() > 0) {
+    const maybeByte = d.anyByte()
+    if (maybeByte === 0x31) { // Just
+      const tag = d.anyByte()
+      if (tag === 0x50) { // 'P' Proposed
+        kemPk = decodeLarge(d)
+      } else if (tag === 0x41) { // 'A' Accepted
+        kemCt = decodeLarge(d)
+        kemAcceptPk = decodeLarge(d)
+      }
+    }
+  }
+  return {version, pk1Raw, pk2Raw, kemPk, kemCt, kemAcceptPk}
+}
+
+// Encode E2ERatchetParams for sending to peer
+function encodeE2EParams(
+  version: number,
+  pk1Raw: Uint8Array, pk2Raw: Uint8Array,
+  kemPk: Uint8Array | null,  // for proposed
+  kemCt: Uint8Array | null,  // for accepted
+  kemAcceptPk: Uint8Array | null,  // public key in accepted
+): Uint8Array {
+  const vBytes = new Uint8Array(2)
+  vBytes[0] = (version >> 8) & 0xff
+  vBytes[1] = version & 0xff
+  const parts = [vBytes, encodeBytes(encodePubKeyX448(pk1Raw)), encodeBytes(encodePubKeyX448(pk2Raw))]
+  if (version >= 3) {
+    if (kemCt && kemAcceptPk) {
+      // Just Accepted
+      parts.push(new Uint8Array([0x31, 0x41])) // Just + 'A'
+      parts.push(new Uint8Array([(kemCt.length >> 8) & 0xff, kemCt.length & 0xff]))
+      parts.push(kemCt)
+      parts.push(new Uint8Array([(kemAcceptPk.length >> 8) & 0xff, kemAcceptPk.length & 0xff]))
+      parts.push(kemAcceptPk)
+    } else if (kemPk) {
+      // Just Proposed
+      parts.push(new Uint8Array([0x31, 0x50])) // Just + 'P'
+      parts.push(new Uint8Array([(kemPk.length >> 8) & 0xff, kemPk.length & 0xff]))
+      parts.push(kemPk)
+    } else {
+      // Nothing
+      parts.push(new Uint8Array([0x30]))
+    }
+  }
+  return concatBytes(...parts)
+}
+
+// -- Init handlers
+
+function handleInitRcv(version: number, pqSupport: boolean): string {
+  const kp1 = generateX448KeyPair()
+  const kp2 = generateX448KeyPair()
+  let kemKeyPair: KEMKeyPair | null = null
+  let kemPk: Uint8Array | null = null
+  if (pqSupport) {
+    kemKeyPair = sntrup761Keypair()
+    kemPk = kemKeyPair.publicKey
+  }
+  rcvInitState = {privKey1: kp1.privateKey, privKey2: kp2.privateKey, kemKeyPair, pqSupport}
+  const params = encodeE2EParams(version, kp1.publicKey, kp2.publicKey, kemPk, null, null)
+  return "ok: " + toHex(params)
+}
+
+function handleComplete(peerParamsHex: string): string {
+  if (!rcvInitState) return "error: not in RCV init state"
+  const {privKey1, privKey2, kemKeyPair, pqSupport} = rcvInitState
+  const peerParams = parseE2EParams(fromHex(peerParamsHex))
+
+  // Build kemAccepted for X3DH if peer accepted our KEM proposal
+  let kemAccepted: RatchetKEMAccepted | null = null
+  if (peerParams.kemCt && peerParams.kemAcceptPk && kemKeyPair) {
+    const ss = sntrup761Dec(peerParams.kemCt, kemKeyPair.secretKey)
+    kemAccepted = {rcPQRr: peerParams.kemAcceptPk, rcPQRss: ss, rcPQRct: peerParams.kemCt}
+  }
+
+  // X3DH (receiver side)
+  const initParams = pqX3dhRcv(privKey1, privKey2, peerParams.pk1Raw, peerParams.pk2Raw, kemAccepted)
+
+  // Init receiving ratchet
+  const vs: RatchetVersions = {current: peerParams.version, maxSupported: peerParams.version}
+  ratchet = initRcvRatchet(vs, privKey2, initParams, kemKeyPair, pqSupport)
+  skippedKeys = new Map()
+  rcvInitState = null
+  return "ok"
+}
+
+function handleInitSnd(version: number, kemMode: string, peerParamsHex: string): string {
+  const peerParams = parseE2EParams(fromHex(peerParamsHex))
+
+  const kp1 = generateX448KeyPair()
+  const kp2 = generateX448KeyPair()
+  const kp3 = generateX448KeyPair() // fresh DH key for ratchet
+
+  // KEM handling
+  let kemAccepted: RatchetKEMAccepted | null = null
+  let ownKemKp: KEMKeyPair | null = null
+  let outKemPk: Uint8Array | null = null
+  let outKemCt: Uint8Array | null = null
+  let outKemAcceptPk: Uint8Array | null = null
+
+  if (kemMode === "accept" && peerParams.kemPk) {
+    // Accept peer's KEM proposal
+    const encResult = sntrup761Enc(peerParams.kemPk)
+    ownKemKp = sntrup761Keypair()
+    kemAccepted = {rcPQRr: peerParams.kemPk, rcPQRss: encResult.sharedSecret, rcPQRct: encResult.ciphertext}
+    outKemCt = encResult.ciphertext
+    outKemAcceptPk = ownKemKp.publicKey
+  } else if (kemMode === "propose") {
+    ownKemKp = sntrup761Keypair()
+    outKemPk = ownKemKp.publicKey
+  }
+
+  // X3DH (sender side)
+  const initParams = pqX3dhSnd(kp1.privateKey, kp2.privateKey, peerParams.pk1Raw, peerParams.pk2Raw, kemAccepted)
+
+  // Init sending ratchet
+  const vs: RatchetVersions = {current: version, maxSupported: version}
+  ratchet = initSndRatchet(vs, peerParams.pk2Raw, kp3.privateKey, initParams, ownKemKp)
+  skippedKeys = new Map()
+
+  const params = encodeE2EParams(version, kp1.publicKey, kp2.publicKey, outKemPk, outKemCt, outKemAcceptPk)
+  return "ok: " + toHex(params)
+}
+
+// -- Encrypt/decrypt handlers
+
+function handleEncrypt(kemAssert: boolean | null, _pqPref: boolean | null, plaintext: string): string {
+  if (!ratchet) return "error: not initialized"
+  try {
+    const result = rcEncrypt(ratchet, new TextEncoder().encode(plaintext), PADDED_MSG_LEN)
+    ratchet = result.state
+    if (kemAssert === true && !ratchet.rcSndKEM) return "error: expected hasSndKEM"
+    if (kemAssert === false && ratchet.rcSndKEM) return "error: expected noSndKEM"
+    return "ok: " + toHex(result.ciphertext)
+  } catch (e: any) {
+    return "error: " + e.message
+  }
+}
+
+function handleDecrypt(kemAssert: boolean | null, hexCt: string, expectedPlaintext: string | null): string {
+  if (!ratchet) return "error: not initialized"
+  try {
+    const ct = fromHex(hexCt)
+    const result = rcDecrypt(ratchet, skippedKeys, ct)
+    ratchet = result.state
+    skippedKeys = result.skippedKeys
+    const plaintext = new TextDecoder().decode(result.plaintext)
+    if (kemAssert === true && !ratchet.rcRcvKEM) return "error: expected hasRcvKEM"
+    if (kemAssert === false && ratchet.rcRcvKEM) return "error: expected noRcvKEM"
+    if (expectedPlaintext !== null && plaintext !== expectedPlaintext)
+      return "error: expected '" + expectedPlaintext + "', got '" + plaintext + "'"
+    return "ok: " + plaintext
+  } catch (e: any) {
+    return "error: " + e.message
+  }
+}
+
+// -- Command parser
+
+function parseLine(line: string): string {
+  // Init commands
+  if (line.startsWith("INIT_RCV ")) {
+    const parts = line.split(" ")
+    return handleInitRcv(parseInt(parts[1]), parts[2] === "1")
+  }
+  if (line.startsWith("COMPLETE ")) {
+    return handleComplete(line.substring(9).trim())
+  }
+  if (line.startsWith("INIT_SND ")) {
+    const parts = line.split(" ")
+    return handleInitSnd(parseInt(parts[1]), parts[2], parts[3])
+  }
+
+  // Query commands
+  if (line === "SNDKEM") {
+    if (!ratchet) return "error: not initialized"
+    return "ok: " + (ratchet.rcSndKEM ? "1" : "0")
+  }
+  if (line === "RCVKEM") {
+    if (!ratchet) return "error: not initialized"
+    return "ok: " + (ratchet.rcRcvKEM ? "1" : "0")
+  }
+
+  // Encrypt operators: \#> !#> \#>! !#>! !#>\ \#>\
+  const encMatch = line.match(/^([!\\])#(>[!\\]?)\s+(.+)$/)
+  if (encMatch) {
+    const [, kemChar, arrow, msg] = encMatch
+    const kemAssert = kemChar === "!" ? true : false
+    let pqPref: boolean | null = null
+    if (arrow === ">!") pqPref = true
+    else if (arrow === ">\\") pqPref = false
+    return handleEncrypt(kemAssert, pqPref, msg)
+  }
+
+  // Decrypt operators: <#\ <#!
+  const decMatch = line.match(/^<#([!\\])\s+(\S+)\s+(.+)$/)
+  if (decMatch) {
+    const [, kemChar, hexCt, expected] = decMatch
+    const kemAssert = kemChar === "!" ? true : false
+    return handleDecrypt(kemAssert, hexCt, expected)
+  }
+
+  // Plain encrypt (no assertion)
+  if (line.startsWith("E ")) {
+    return handleEncrypt(null, null, line.substring(2))
+  }
+
+  // Plain decrypt (no assertion, no expected)
+  if (line.startsWith("D ")) {
+    return handleDecrypt(null, line.substring(2), null)
+  }
+
+  return "error: unknown command: " + line
+}
+
+// -- Main
+
+async function main() {
+  await initSntrup761()
+
+  const rl = createInterface({input: process.stdin, terminal: false})
+
+  for await (const line of rl) {
+    const trimmed = line.trim()
+    if (!trimmed) continue
+    const response = parseLine(trimmed)
+    process.stdout.write(response + "\n")
+  }
+}
+
+main().catch(e => {
+  process.stderr.write("FATAL: " + e.message + "\n")
+  process.exit(1)
+})

smp-web/tsconfig.json

@@ -0,0 +1,19 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "ES2022",
+    "moduleResolution": "node",
+    "lib": ["ES2022"],
+    "outDir": "dist",
+    "rootDir": "src",
+    "declaration": true,
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "resolveJsonModule": true,
+    "sourceMap": true
+  },
+  "include": ["src/**/*.ts"],
+  "exclude": ["node_modules", "dist"]
+}

smp-web/tsconfig.test.json

@@ -0,0 +1,15 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "ES2022",
+    "moduleResolution": "node",
+    "lib": ["ES2022"],
+    "outDir": "dist-test",
+    "rootDir": "tests",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "sourceMap": true
+  },
+  "include": ["tests/**/*.ts"]
+}

src/Simplex/FileTransfer/Server.hs

@@ -31,7 +31,6 @@ import qualified Data.ByteString.Char8 as B
 import Data.Int (Int64)
 import Data.List.NonEmpty (NonEmpty)
 import qualified Data.List.NonEmpty as L
-import qualified Data.Map.Strict as M
 import Data.Maybe (fromMaybe, isJust)
 import qualified Data.Text as T
 import qualified Data.Text.IO as T
@@ -73,6 +72,7 @@ import Simplex.Messaging.Transport.HTTP2
 import Simplex.Messaging.Transport.HTTP2.File (fileBlockSize)
 import Simplex.Messaging.Transport.HTTP2.Server (runHTTP2Server)
 import Simplex.Messaging.Transport.Server (SNICredentialUsed, TransportServerConfig (..), runLocalTCPServer)
+import Simplex.Messaging.Server.Web (serveStaticPageH2)
 import Simplex.Messaging.Util
 import Simplex.Messaging.Version
 import System.Environment (lookupEnv)
@@ -84,10 +84,10 @@ import System.Random (getStdRandom, randomR)
 #endif
 import UnliftIO
 import UnliftIO.Concurrent (threadDelay)
-import UnliftIO.Directory (doesFileExist, removeFile, renameFile)
+import UnliftIO.Directory (canonicalizePath, doesFileExist, removeFile, renameFile)
 import qualified UnliftIO.Exception as E
 
-type M a = ReaderT XFTPEnv IO a
+type M s a = ReaderT (XFTPEnv s) IO a
 
 data XFTPTransportRequest = XFTPTransportRequest
   { thParams :: THandleParamsXFTP 'TServer,
@@ -111,19 +111,19 @@ corsPreflightHeaders =
     ("Access-Control-Max-Age", "86400")
   ]
 
-runXFTPServer :: XFTPServerConfig -> IO ()
+runXFTPServer :: FileStoreClass s => XFTPServerConfig s -> IO ()
 runXFTPServer cfg = do
   started <- newEmptyTMVarIO
   runXFTPServerBlocking started cfg
 
-runXFTPServerBlocking :: TMVar Bool -> XFTPServerConfig -> IO ()
+runXFTPServerBlocking :: FileStoreClass s => TMVar Bool -> XFTPServerConfig s -> IO ()
 runXFTPServerBlocking started cfg = newXFTPServerEnv cfg >>= runReaderT (xftpServer cfg started)
 
 data Handshake
   = HandshakeSent C.PrivateKeyX25519
   | HandshakeAccepted (THandleParams XFTPVersion 'TServer)
 
-xftpServer :: XFTPServerConfig -> TMVar Bool -> M ()
+xftpServer :: forall s. FileStoreClass s => XFTPServerConfig s -> TMVar Bool -> M s ()
 xftpServer cfg@XFTPServerConfig {xftpPort, transportConfig, inactiveClientExpiration, fileExpiration, xftpServerVRange} started = do
   mapM_ (expireServerFiles Nothing) fileExpiration
   restoreServerStats
@@ -136,7 +136,7 @@ xftpServer cfg@XFTPServerConfig {xftpPort, transportConfig, inactiveClientExpira
     )
     `finally` stopServer
   where
-    runServer :: M ()
+    runServer :: M s ()
     runServer = do
       srvCreds@(chain, pk) <- asks tlsServerCreds
       httpCreds_ <- asks httpServerCreds
@@ -147,11 +147,13 @@ xftpServer cfg@XFTPServerConfig {xftpPort, transportConfig, inactiveClientExpira
       sessions <- liftIO TM.emptyIO
       let cleanup sessionId = atomically $ TM.delete sessionId sessions
           srvParams = if isJust httpCreds_ then defaultSupportedParamsHTTPS else defaultSupportedParams
+      webCanonicalRoot_ <- liftIO $ mapM canonicalizePath (webStaticPath cfg)
       liftIO . runHTTP2Server started xftpPort defaultHTTP2BufferSize srvParams srvCreds httpCreds_ transportConfig inactiveClientExpiration cleanup $ \sniUsed sessionId sessionALPN r sendResponse -> do
         let addCORS' = sniUsed && addCORSHeaders transportConfig
-        if addCORS' && H.requestMethod r == Just "OPTIONS"
-          then sendResponse $ H.responseNoBody N.ok200 corsPreflightHeaders
-          else do
+        case H.requestMethod r of
+          Just "OPTIONS" | addCORS' -> sendResponse $ H.responseNoBody N.ok200 corsPreflightHeaders
+          Just "GET" | sniUsed -> forM_ webCanonicalRoot_ $ \root -> serveStaticPageH2 root r sendResponse
+          _ -> do
             reqBody <- getHTTP2Body r xftpBlockSize
             let v = VersionXFTP 1
                 thServerVRange = versionToRange v
@@ -165,7 +167,7 @@ xftpServer cfg@XFTPServerConfig {xftpPort, transportConfig, inactiveClientExpira
                       Nothing -> pure ()
                       Just thParams -> processRequest req0 {thParams}
                 | otherwise -> liftIO . sendResponse $ H.responseNoBody N.ok200 (corsHeaders addCORS')
-    xftpServerHandshakeV1 :: X.CertificateChain -> C.APrivateSignKey -> TMap SessionId Handshake -> XFTPTransportRequest -> M (Maybe (THandleParams XFTPVersion 'TServer))
+    xftpServerHandshakeV1 :: X.CertificateChain -> C.APrivateSignKey -> TMap SessionId Handshake -> XFTPTransportRequest -> M s (Maybe (THandleParams XFTPVersion 'TServer))
     xftpServerHandshakeV1 chain serverSignKey sessions XFTPTransportRequest {thParams = thParams0@THandleParams {sessionId}, request, reqBody = HTTP2Body {bodyHead}, sendResponse, sniUsed, addCORS} = do
       s <- atomically $ TM.lookup sessionId sessions
       r <- runExceptT $ case s of
@@ -224,39 +226,40 @@ xftpServer cfg@XFTPServerConfig {xftpPort, transportConfig, inactiveClientExpira
               liftIO . sendResponse $ H.responseNoBody N.ok200 (corsHeaders addCORS)
               pure Nothing
             Nothing -> throwE HANDSHAKE
-        sendError :: XFTPErrorType -> M (Maybe (THandleParams XFTPVersion 'TServer))
+        sendError :: XFTPErrorType -> M s (Maybe (THandleParams XFTPVersion 'TServer))
         sendError err = do
           runExceptT (encodeXftp err) >>= \case
             Right bs -> liftIO . sendResponse $ H.responseBuilder N.ok200 (corsHeaders addCORS) bs
             Left _ -> logError $ "Error encoding handshake error: " <> tshow err
           pure Nothing
-        encodeXftp :: Encoding a => a -> ExceptT XFTPErrorType (ReaderT XFTPEnv IO) Builder
+        encodeXftp :: Encoding a => a -> ExceptT XFTPErrorType (ReaderT (XFTPEnv s) IO) Builder
         encodeXftp a = byteString <$> liftHS (C.pad (smpEncode a) xftpBlockSize)
         liftHS = liftEitherWith (const HANDSHAKE)
 
-    stopServer :: M ()
+    stopServer :: M s ()
     stopServer = do
-      withFileLog closeStoreLog
+      st <- asks fileStore
+      liftIO $ closeFileStore st
       saveServerStats
       logNote "Server stopped"
 
-    expireFilesThread_ :: XFTPServerConfig -> [M ()]
+    expireFilesThread_ :: XFTPServerConfig s -> [M s ()]
     expireFilesThread_ XFTPServerConfig {fileExpiration = Just fileExp} = [expireFiles fileExp]
     expireFilesThread_ _ = []
 
-    expireFiles :: ExpirationConfig -> M ()
+    expireFiles :: ExpirationConfig -> M s ()
     expireFiles expCfg = do
       let interval = checkInterval expCfg * 1000000
       forever $ do
         liftIO $ threadDelay' interval
         expireServerFiles (Just 100000) expCfg
 
-    serverStatsThread_ :: XFTPServerConfig -> [M ()]
+    serverStatsThread_ :: XFTPServerConfig s -> [M s ()]
     serverStatsThread_ XFTPServerConfig {logStatsInterval = Just interval, logStatsStartTime, serverStatsLogFile} =
       [logServerStats logStatsStartTime interval serverStatsLogFile]
     serverStatsThread_ _ = []
 
-    logServerStats :: Int64 -> Int64 -> FilePath -> M ()
+    logServerStats :: Int64 -> Int64 -> FilePath -> M s ()
     logServerStats startAt logInterval statsFilePath = do
       initialDelay <- (startAt -) . fromIntegral . (`div` 1000000_000000) . diffTimeToPicoseconds . utctDayTime <$> liftIO getCurrentTime
       liftIO $ putStrLn $ "server stats log enabled: " <> statsFilePath
@@ -297,12 +300,12 @@ xftpServer cfg@XFTPServerConfig {xftpPort, transportConfig, inactiveClientExpira
               ]
         liftIO $ threadDelay' interval
 
-    prometheusMetricsThread_ :: XFTPServerConfig -> [M ()]
+    prometheusMetricsThread_ :: XFTPServerConfig s -> [M s ()]
     prometheusMetricsThread_ XFTPServerConfig {prometheusInterval = Just interval, prometheusMetricsFile} =
       [savePrometheusMetrics interval prometheusMetricsFile]
     prometheusMetricsThread_ _ = []
 
-    savePrometheusMetrics :: Int -> FilePath -> M ()
+    savePrometheusMetrics :: Int -> FilePath -> M s ()
     savePrometheusMetrics saveInterval metricsFile = do
       labelMyThread "savePrometheusMetrics"
       liftIO $ putStrLn $ "Prometheus metrics saved every " <> show saveInterval <> " seconds to " <> metricsFile
@@ -321,11 +324,11 @@ xftpServer cfg@XFTPServerConfig {xftpPort, transportConfig, inactiveClientExpira
       let fd = periodStatDataCounts $ _filesDownloaded d
       pure FileServerMetrics {statsData = d, filesDownloadedPeriods = fd, rtsOptions}
 
-    controlPortThread_ :: XFTPServerConfig -> [M ()]
+    controlPortThread_ :: XFTPServerConfig s -> [M s ()]
     controlPortThread_ XFTPServerConfig {controlPort = Just port} = [runCPServer port]
     controlPortThread_ _ = []
 
-    runCPServer :: ServiceName -> M ()
+    runCPServer :: ServiceName -> M s ()
     runCPServer port = do
       cpStarted <- newEmptyTMVarIO
       u <- askUnliftIO
@@ -333,7 +336,7 @@ xftpServer cfg@XFTPServerConfig {xftpPort, transportConfig, inactiveClientExpira
         labelMyThread "control port server"
         runLocalTCPServer cpStarted port $ runCPClient u
       where
-        runCPClient :: UnliftIO (ReaderT XFTPEnv IO) -> Socket -> IO ()
+        runCPClient :: UnliftIO (ReaderT (XFTPEnv s) IO) -> Socket -> IO ()
         runCPClient u sock = do
           labelMyThread "control port client"
           h <- socketToHandle sock ReadWriteMode
@@ -363,21 +366,20 @@ xftpServer cfg@XFTPServerConfig {xftpPort, transportConfig, inactiveClientExpira
                   XFTPServerConfig {controlPortUserAuth = user, controlPortAdminAuth = admin} = cfg
               CPStatsRTS -> E.tryAny getRTSStats >>= either (hPrint h) (hPrint h)
               CPDelete fileId -> withUserRole $ unliftIO u $ do
-                fs <- asks store
+                fs <- asks fileStore
                 r <- runExceptT $ do
-                  (fr, _) <- ExceptT $ atomically $ getFile fs SFRecipient fileId
+                  (fr, _) <- ExceptT $ liftIO $ getFile fs SFRecipient fileId
                   ExceptT $ deleteServerFile_ fr
                 liftIO . hPutStrLn h $ either (\e -> "error: " <> show e) (\() -> "ok") r
               CPBlock fileId info -> withUserRole $ unliftIO u $ do
-                fs <- asks store
+                fs <- asks fileStore
                 r <- runExceptT $ do
-                  (fr, _) <- ExceptT $ atomically $ getFile fs SFRecipient fileId
+                  (fr, _) <- ExceptT $ liftIO $ getFile fs SFRecipient fileId
                   ExceptT $ blockServerFile fr info
                 liftIO . hPutStrLn h $ either (\e -> "error: " <> show e) (\() -> "ok") r
               CPHelp -> hPutStrLn h "commands: stats-rts, delete, help, quit"
               CPQuit -> pure ()
               CPSkip -> pure ()
-              _ -> hPutStrLn h "unsupported command"
               where
                 withUserRole action =
                   readTVarIO role >>= \case
@@ -393,7 +395,7 @@ data ServerFile = ServerFile
     sbState :: LC.SbState
   }
 
-processRequest :: XFTPTransportRequest -> M ()
+processRequest :: FileStoreClass s => XFTPTransportRequest -> M s ()
 processRequest XFTPTransportRequest {thParams, reqBody = body@HTTP2Body {bodyHead}, sendResponse, addCORS}
   | B.length bodyHead /= xftpBlockSize = sendXFTPResponse ("", NoEntity, FRErr BLOCK) Nothing
   | otherwise =
@@ -428,7 +430,7 @@ processRequest XFTPTransportRequest {thParams, reqBody = body@HTTP2Body {bodyHea
           done
 
 #ifdef slow_servers
-randomDelay :: M ()
+randomDelay :: M s ()
 randomDelay = do
   d <- asks $ responseDelay . config
   when (d > 0) $ do
@@ -438,20 +440,20 @@ randomDelay = do
 
 data VerificationResult = VRVerified XFTPRequest | VRFailed XFTPErrorType
 
-verifyXFTPTransmission :: Maybe (THandleAuth 'TServer) -> SignedTransmission FileCmd -> M VerificationResult
+verifyXFTPTransmission :: forall s. FileStoreClass s => Maybe (THandleAuth 'TServer) -> SignedTransmission FileCmd -> M s VerificationResult
 verifyXFTPTransmission thAuth (tAuth, authorized, (corrId, fId, cmd)) =
   case cmd of
     FileCmd SFSender (FNEW file rcps auth') -> pure $ XFTPReqNew file rcps auth' `verifyWith` sndKey file
     FileCmd SFRecipient PING -> pure $ VRVerified XFTPReqPing
     FileCmd party _ -> verifyCmd party
   where
-    verifyCmd :: SFileParty p -> M VerificationResult
+    verifyCmd :: SFileParty p -> M s VerificationResult
     verifyCmd party = do
-      st <- asks store
-      atomically $ verify =<< getFile st party fId
+      st <- asks fileStore
+      liftIO $ verify =<< getFile st party fId
       where
         verify = \case
-          Right (fr, k) -> result <$> readTVar (fileStatus fr)
+          Right (fr, k) -> result <$> readTVarIO (fileStatus fr)
             where
               result = \case
                 EntityActive -> XFTPReqCmd fId fr cmd `verifyWith` k
@@ -462,7 +464,7 @@ verifyXFTPTransmission thAuth (tAuth, authorized, (corrId, fId, cmd)) =
     -- TODO verify with DH authorization
     req `verifyWith` k = if verifyCmdAuthorization thAuth tAuth authorized corrId k then VRVerified req else VRFailed AUTH
 
-processXFTPRequest :: HTTP2Body -> XFTPRequest -> M (FileResponse, Maybe ServerFile)
+processXFTPRequest :: forall s. FileStoreClass s => HTTP2Body -> XFTPRequest -> M s (FileResponse, Maybe ServerFile)
 processXFTPRequest HTTP2Body {bodyPart} = \case
   XFTPReqNew file rks auth -> noFile =<< ifM allowNew (createFile file rks) (pure $ FRErr AUTH)
     where
@@ -481,9 +483,9 @@ processXFTPRequest HTTP2Body {bodyPart} = \case
   XFTPReqPing -> noFile FRPong
   where
     noFile resp = pure (resp, Nothing)
-    createFile :: FileInfo -> NonEmpty RcvPublicAuthKey -> M FileResponse
+    createFile :: FileInfo -> NonEmpty RcvPublicAuthKey -> M s FileResponse
     createFile file rks = do
-      st <- asks store
+      st <- asks fileStore
       r <- runExceptT $ do
         sizes <- asks $ allowedChunkSizes . config
         unless (size file `elem` sizes) $ throwE SIZE
@@ -500,27 +502,27 @@ processXFTPRequest HTTP2Body {bodyPart} = \case
         let rIds = L.map (\(FileRecipient rId _) -> rId) rcps
         pure $ FRSndIds sId rIds
       pure $ either FRErr id r
-    addFileRetry :: FileStore -> FileInfo -> Int -> RoundedFileTime -> M (Either XFTPErrorType XFTPFileId)
+    addFileRetry :: s -> FileInfo -> Int -> RoundedFileTime -> M s (Either XFTPErrorType XFTPFileId)
     addFileRetry st file n ts =
       retryAdd n $ \sId -> runExceptT $ do
         ExceptT $ addFile st sId file ts EntityActive
         pure sId
-    addRecipientRetry :: FileStore -> Int -> XFTPFileId -> RcvPublicAuthKey -> M (Either XFTPErrorType FileRecipient)
+    addRecipientRetry :: s -> Int -> XFTPFileId -> RcvPublicAuthKey -> M s (Either XFTPErrorType FileRecipient)
     addRecipientRetry st n sId rpk =
       retryAdd n $ \rId -> runExceptT $ do
         let rcp = FileRecipient rId rpk
         ExceptT $ addRecipient st sId rcp
         pure rcp
-    retryAdd :: Int -> (XFTPFileId -> STM (Either XFTPErrorType a)) -> M (Either XFTPErrorType a)
+    retryAdd :: Int -> (XFTPFileId -> IO (Either XFTPErrorType a)) -> M s (Either XFTPErrorType a)
     retryAdd 0 _ = pure $ Left INTERNAL
     retryAdd n add = do
       fId <- getFileId
-      atomically (add fId) >>= \case
+      liftIO (add fId) >>= \case
         Left DUPLICATE_ -> retryAdd (n - 1) add
         r -> pure r
-    addRecipients :: XFTPFileId -> NonEmpty RcvPublicAuthKey -> M FileResponse
+    addRecipients :: XFTPFileId -> NonEmpty RcvPublicAuthKey -> M s FileResponse
     addRecipients sId rks = do
-      st <- asks store
+      st <- asks fileStore
       r <- runExceptT $ do
         rcps <- mapM (ExceptT . addRecipientRetry st 3 sId) rks
         lift $ withFileLog $ \sl -> logAddRecipients sl sId rcps
@@ -529,7 +531,7 @@ processXFTPRequest HTTP2Body {bodyPart} = \case
         let rIds = L.map (\(FileRecipient rId _) -> rId) rcps
         pure $ FRRcvIds rIds
       pure $ either FRErr id r
-    receiveServerFile :: FileRec -> M FileResponse
+    receiveServerFile :: FileRec -> M s FileResponse
     receiveServerFile FileRec {senderId, fileInfo = FileInfo {size, digest}, filePath} = case bodyPart of
       Nothing -> pure $ FRErr SIZE
       -- TODO validate body size from request before downloading, once it's populated
@@ -547,7 +549,7 @@ processXFTPRequest HTTP2Body {bodyPart} = \case
                   | bs == 0 || bs > s -> pure $ FRErr SIZE
                   | otherwise -> drain (s - bs)
           reserve = do
-            us <- asks $ usedStorage . store
+            us <- asks usedStorage
             quota <- asks $ fromMaybe maxBound . fileSizeQuota . config
             atomically . stateTVar us $
               \used -> let used' = used + fromIntegral size in if used' <= quota then (True, used') else (False, used)
@@ -557,21 +559,28 @@ processXFTPRequest HTTP2Body {bodyPart} = \case
             receiveChunk (XFTPRcvChunkSpec fPath size digest) >>= \case
               Right () -> do
                 stats <- asks serverStats
-                withFileLog $ \sl -> logPutFile sl senderId fPath
-                atomically $ writeTVar filePath (Just fPath)
-                incFileStat filesUploaded
-                incFileStat filesCount
-                liftIO $ atomicModifyIORef'_ (filesSize stats) (+ fromIntegral size)
-                pure FROk
+                st <- asks fileStore
+                liftIO (setFilePath st senderId fPath) >>= \case
+                  Right () -> do
+                    withFileLog $ \sl -> logPutFile sl senderId fPath
+                    incFileStat filesUploaded
+                    incFileStat filesCount
+                    liftIO $ atomicModifyIORef'_ (filesSize stats) (+ fromIntegral size)
+                    pure FROk
+                  Left _e -> do
+                    us <- asks usedStorage
+                    atomically $ modifyTVar' us $ subtract (fromIntegral size)
+                    liftIO $ whenM (doesFileExist fPath) (removeFile fPath) `catch` logFileError
+                    pure $ FRErr AUTH
               Left e -> do
-                us <- asks $ usedStorage . store
+                us <- asks usedStorage
                 atomically $ modifyTVar' us $ subtract (fromIntegral size)
                 liftIO $ whenM (doesFileExist fPath) (removeFile fPath) `catch` logFileError
                 pure $ FRErr e
           receiveChunk spec = do
             t <- asks $ fileTimeout . config
             liftIO $ fromMaybe (Left TIMEOUT) <$> timeout t (runExceptT $ receiveFile getBody spec)
-    sendServerFile :: FileRec -> RcvPublicDhKey -> M (FileResponse, Maybe ServerFile)
+    sendServerFile :: FileRec -> RcvPublicDhKey -> M s (FileResponse, Maybe ServerFile)
     sendServerFile FileRec {senderId, filePath, fileInfo = FileInfo {size}} rDhKey = do
       readTVarIO filePath >>= \case
         Just path -> ifM (doesFileExist path) sendFile (pure (FRErr AUTH, Nothing))
@@ -590,38 +599,41 @@ processXFTPRequest HTTP2Body {bodyPart} = \case
                 _ -> pure (FRErr INTERNAL, Nothing)
         _ -> pure (FRErr NO_FILE, Nothing)
 
-    deleteServerFile :: FileRec -> M FileResponse
+    deleteServerFile :: FileRec -> M s FileResponse
     deleteServerFile fr = either FRErr (\() -> FROk) <$> deleteServerFile_ fr
 
     logFileError :: SomeException -> IO ()
     logFileError e = logError $ "Error deleting file: " <> tshow e
 
-    ackFileReception :: RecipientId -> FileRec -> M FileResponse
+    ackFileReception :: RecipientId -> FileRec -> M s FileResponse
     ackFileReception rId fr = do
       withFileLog (`logAckFile` rId)
-      st <- asks store
-      atomically $ deleteRecipient st rId fr
+      st <- asks fileStore
+      liftIO $ deleteRecipient st rId fr
       incFileStat fileDownloadAcks
       pure FROk
 
-deleteServerFile_ :: FileRec -> M (Either XFTPErrorType ())
+deleteServerFile_ :: FileStoreClass s => FileRec -> M s (Either XFTPErrorType ())
 deleteServerFile_ fr@FileRec {senderId} = do
   withFileLog (`logDeleteFile` senderId)
   deleteOrBlockServerFile_ fr filesDeleted (`deleteFile` senderId)
 
 -- this also deletes the file from storage, but doesn't include it in delete statistics
-blockServerFile :: FileRec -> BlockingInfo -> M (Either XFTPErrorType ())
+blockServerFile :: FileStoreClass s => FileRec -> BlockingInfo -> M s (Either XFTPErrorType ())
 blockServerFile fr@FileRec {senderId} info = do
   withFileLog $ \sl -> logBlockFile sl senderId info
   deleteOrBlockServerFile_ fr filesBlocked $ \st -> blockFile st senderId info True
 
-deleteOrBlockServerFile_ :: FileRec -> (FileServerStats -> IORef Int) -> (FileStore -> STM (Either XFTPErrorType ())) -> M (Either XFTPErrorType ())
+deleteOrBlockServerFile_ :: FileStoreClass s => FileRec -> (FileServerStats -> IORef Int) -> (s -> IO (Either XFTPErrorType ())) -> M s (Either XFTPErrorType ())
 deleteOrBlockServerFile_ FileRec {filePath, fileInfo} stat storeAction = runExceptT $ do
   path <- readTVarIO filePath
   stats <- asks serverStats
   ExceptT $ first (\(_ :: SomeException) -> FILE_IO) <$> try (forM_ path $ \p -> whenM (doesFileExist p) (removeFile p >> deletedStats stats))
-  st <- asks store
-  void $ atomically $ storeAction st
+  st <- asks fileStore
+  ExceptT $ liftIO $ storeAction st
+  forM_ path $ \_ -> do
+    us <- asks usedStorage
+    atomically $ modifyTVar' us $ subtract (fromIntegral $ size fileInfo)
   lift $ incFileStat stat
   where
     deletedStats stats = do
@@ -631,47 +643,50 @@ deleteOrBlockServerFile_ FileRec {filePath, fileInfo} stat storeAction = runExce
 getFileTime :: IO RoundedFileTime
 getFileTime = getRoundedSystemTime
 
-expireServerFiles :: Maybe Int -> ExpirationConfig -> M ()
+expireServerFiles :: FileStoreClass s => Maybe Int -> ExpirationConfig -> M s ()
 expireServerFiles itemDelay expCfg = do
-  st <- asks store
-  usedStart <- readTVarIO $ usedStorage st
+  st <- asks fileStore
+  us <- asks usedStorage
+  usedStart <- readTVarIO us
   old <- liftIO $ expireBeforeEpoch expCfg
-  files' <- readTVarIO (files st)
-  logNote $ "Expiration check: " <> tshow (M.size files') <> " files"
-  forM_ (M.keys files') $ \sId -> do
-    mapM_ threadDelay itemDelay
-    atomically (expiredFilePath st sId old)
-      >>= mapM_ (maybeRemove $ delete st sId)
-  usedEnd <- readTVarIO $ usedStorage st
+  filesCount <- liftIO $ getFileCount st
+  logNote $ "Expiration check: " <> tshow filesCount <> " files"
+  expireLoop st us old
+  usedEnd <- readTVarIO us
   logNote $ "Used " <> mbs usedStart <> " -> " <> mbs usedEnd <> ", " <> mbs (usedStart - usedEnd) <> " reclaimed."
   where
     mbs bs = tshow (bs `div` 1048576) <> "mb"
-    maybeRemove del = maybe del (remove del)
-    remove del filePath =
-      ifM
-        (doesFileExist filePath)
-        ((removeFile filePath >> del) `catch` \(e :: SomeException) -> logError $ "failed to remove expired file " <> tshow filePath <> ": " <> tshow e)
-        del
-    delete st sId = do
-      withFileLog (`logDeleteFile` sId)
-      void . atomically $ deleteFile st sId -- will not update usedStorage if sId isn't in store
-      incFileStat filesExpired
+    expireLoop st us old = do
+      expired <- liftIO $ expiredFiles st old 10000
+      forM_ expired $ \(sId, filePath_, fileSize) -> do
+        mapM_ threadDelay itemDelay
+        forM_ filePath_ $ \fp ->
+          whenM (doesFileExist fp) $
+            removeFile fp `catch` \(e :: SomeException) -> logError $ "failed to remove expired file " <> tshow fp <> ": " <> tshow e
+        forM_ filePath_ $ \_ ->
+          atomically $ modifyTVar' us $ subtract (fromIntegral fileSize)
+        incFileStat filesExpired
+      let sIds = map (\(sId, _, _) -> sId) expired
+      unless (null sIds) $ do
+        withFileLog $ \sl -> mapM_ (logDeleteFile sl) sIds
+        liftIO $ deleteFiles st sIds
+        expireLoop st us old
 
-randomId :: Int -> M ByteString
+randomId :: Int -> M s ByteString
 randomId n = atomically . C.randomBytes n =<< asks random
 
-getFileId :: M XFTPFileId
+getFileId :: M s XFTPFileId
 getFileId = fmap EntityId . randomId =<< asks (fileIdSize . config)
 
-withFileLog :: (StoreLog 'WriteMode -> IO a) -> M ()
+withFileLog :: (StoreLog 'WriteMode -> IO a) -> M s ()
 withFileLog action = liftIO . mapM_ action =<< asks storeLog
 
-incFileStat :: (FileServerStats -> IORef Int) -> M ()
+incFileStat :: (FileServerStats -> IORef Int) -> M s ()
 incFileStat statSel = do
   stats <- asks serverStats
   liftIO $ atomicModifyIORef'_ (statSel stats) (+ 1)
 
-saveServerStats :: M ()
+saveServerStats :: M s ()
 saveServerStats =
   asks (serverStatsBackupFile . config)
     >>= mapM_ (\f -> asks serverStats >>= liftIO . getFileServerStatsData >>= liftIO . saveStats f)
@@ -681,7 +696,7 @@ saveServerStats =
       B.writeFile f $ strEncode stats
       logNote "server stats saved"
 
-restoreServerStats :: M ()
+restoreServerStats :: FileStoreClass s => M s ()
 restoreServerStats = asks (serverStatsBackupFile . config) >>= mapM_ restoreStats
   where
     restoreStats f = whenM (doesFileExist f) $ do
@@ -689,9 +704,9 @@ restoreServerStats = asks (serverStatsBackupFile . config) >>= mapM_ restoreStat
       liftIO (strDecode <$> B.readFile f) >>= \case
         Right d@FileServerStatsData {_filesCount = statsFilesCount, _filesSize = statsFilesSize} -> do
           s <- asks serverStats
-          FileStore {files, usedStorage} <- asks store
-          _filesCount <- M.size <$> readTVarIO files
-          _filesSize <- readTVarIO usedStorage
+          st <- asks fileStore
+          _filesCount <- liftIO $ getFileCount st
+          _filesSize <- readTVarIO =<< asks usedStorage
           liftIO $ setFileServerStats s d {_filesCount, _filesSize}
           renameFile f $ f <> ".bak"
           logNote "server stats restored"

src/Simplex/FileTransfer/Server/Env.hs

@@ -1,21 +1,35 @@
+{-# LANGUAGE CPP #-}
 {-# LANGUAGE DataKinds #-}
 {-# LANGUAGE DuplicateRecordFields #-}
+{-# LANGUAGE FlexibleContexts #-}
 {-# LANGUAGE GADTs #-}
 {-# LANGUAGE KindSignatures #-}
+{-# LANGUAGE RankNTypes #-}
 {-# LANGUAGE LambdaCase #-}
 {-# LANGUAGE NamedFieldPuns #-}
 {-# LANGUAGE OverloadedStrings #-}
 {-# LANGUAGE StrictData #-}
+{-# LANGUAGE TypeFamilies #-}
 
 module Simplex.FileTransfer.Server.Env
   ( XFTPServerConfig (..),
+    XFTPStoreConfig (..),
     XFTPEnv (..),
     XFTPRequest (..),
+    XFTPStoreType,
+    FileStore (..),
+    AFStoreType (..),
+    fileStore,
+    fromFileStore,
     defaultInactiveClientExpiration,
     defFileExpirationHours,
     defaultFileExpiration,
     newXFTPServerEnv,
-    countUsedStorage,
+    readFileStoreType,
+    runWithStoreConfig,
+    checkFileStoreMode,
+    importToDatabase,
+    exportFromDatabase,
   ) where
 
 import Control.Logger.Simple
@@ -23,7 +37,6 @@ import Control.Monad
 import Crypto.Random
 import Data.Int (Int64)
 import Data.List.NonEmpty (NonEmpty)
-import qualified Data.Map.Strict as M
 import Data.Time.Clock (getCurrentTime)
 import Data.Word (Word32)
 import Data.X509.Validation (Fingerprint (..))
@@ -31,7 +44,21 @@ import Network.Socket
 import qualified Network.TLS as T
 import Simplex.FileTransfer.Protocol (FileCmd, FileInfo (..), XFTPFileId)
 import Simplex.FileTransfer.Server.Stats
+import Data.Either (fromRight)
+import Data.Ini (Ini, lookupValue)
+import qualified Data.Text as T
 import Simplex.FileTransfer.Server.Store
+import Simplex.Messaging.Agent.Store.Shared (MigrationConfirmation)
+import Data.Functor (($>))
+import Simplex.Messaging.Server.CLI (settingIsOn)
+import System.Exit (exitFailure)
+#if defined(dbServerPostgres)
+import Data.Maybe (isNothing)
+import Simplex.FileTransfer.Server.Store.Postgres (PostgresFileStore, importFileStore, exportFileStore)
+import Simplex.FileTransfer.Server.Store.Postgres.Config (PostgresFileStoreCfg (..), defaultXFTPDBOpts)
+import Simplex.Messaging.Server.CLI (iniDBOptions)
+import System.Directory (doesFileExist)
+#endif
 import Simplex.FileTransfer.Server.StoreLog
 import Simplex.FileTransfer.Transport (VersionRangeXFTP)
 import qualified Simplex.Messaging.Crypto as C
@@ -42,10 +69,11 @@ import Simplex.Messaging.Util (tshow)
 import System.IO (IOMode (..))
 import UnliftIO.STM
 
-data XFTPServerConfig = XFTPServerConfig
+data XFTPServerConfig s = XFTPServerConfig
   { xftpPort :: ServiceName,
     controlPort :: Maybe ServiceName,
     fileIdSize :: Int,
+    serverStoreCfg :: XFTPStoreConfig s,
     storeLogFile :: Maybe FilePath,
     filesPath :: FilePath,
     -- | server storage quota
@@ -77,7 +105,8 @@ data XFTPServerConfig = XFTPServerConfig
     prometheusInterval :: Maybe Int,
     prometheusMetricsFile :: FilePath,
     transportConfig :: TransportServerConfig,
-    responseDelay :: Int
+    responseDelay :: Int,
+    webStaticPath :: Maybe FilePath
   }
 
 defaultInactiveClientExpiration :: ExpirationConfig
@@ -87,9 +116,10 @@ defaultInactiveClientExpiration =
       checkInterval = 3600 -- seconds, 1 hours
     }
 
-data XFTPEnv = XFTPEnv
-  { config :: XFTPServerConfig,
-    store :: FileStore,
+data XFTPEnv s = XFTPEnv
+  { config :: XFTPServerConfig s,
+    store :: FileStore s,
+    usedStorage :: TVar Int64,
     storeLog :: Maybe (StoreLog 'WriteMode),
     random :: TVar ChaChaDRG,
     serverIdentity :: C.KeyHash,
@@ -98,6 +128,38 @@ data XFTPEnv = XFTPEnv
     serverStats :: FileServerStats
   }
 
+fileStore :: XFTPEnv s -> s
+fileStore = fromFileStore . store
+{-# INLINE fileStore #-}
+
+data XFTPStoreConfig s where
+  XSCMemory :: Maybe FilePath -> XFTPStoreConfig STMFileStore
+#if defined(dbServerPostgres)
+  XSCDatabase :: PostgresFileStoreCfg -> XFTPStoreConfig PostgresFileStore
+#endif
+
+type family XFTPStoreType (fs :: FSType) where
+  XFTPStoreType 'FSMemory = STMFileStore
+#if defined(dbServerPostgres)
+  XFTPStoreType 'FSPostgres = PostgresFileStore
+#endif
+
+data FileStore s where
+  StoreMemory :: STMFileStore -> FileStore STMFileStore
+#if defined(dbServerPostgres)
+  StoreDatabase :: PostgresFileStore -> FileStore PostgresFileStore
+#endif
+
+data AFStoreType = forall fs. AFSType (SFSType fs)
+
+fromFileStore :: FileStore s -> s
+fromFileStore = \case
+  StoreMemory s -> s
+#if defined(dbServerPostgres)
+  StoreDatabase s -> s
+#endif
+{-# INLINE fromFileStore #-}
+
 defFileExpirationHours :: Int64
 defFileExpirationHours = 48
 
@@ -108,13 +170,22 @@ defaultFileExpiration =
       checkInterval = 2 * 3600 -- seconds, 2 hours
     }
 
-newXFTPServerEnv :: XFTPServerConfig -> IO XFTPEnv
-newXFTPServerEnv config@XFTPServerConfig {storeLogFile, fileSizeQuota, xftpCredentials, httpCredentials} = do
+newXFTPServerEnv :: FileStoreClass s => XFTPServerConfig s -> IO (XFTPEnv s)
+newXFTPServerEnv config@XFTPServerConfig {serverStoreCfg, fileSizeQuota, xftpCredentials, httpCredentials} = do
   random <- C.newRandom
-  store <- newFileStore
-  storeLog <- mapM (`readWriteFileStore` store) storeLogFile
-  used <- countUsedStorage <$> readTVarIO (files store)
-  atomically $ writeTVar (usedStorage store) used
+  (store, storeLog) <- case serverStoreCfg of
+    XSCMemory storeLogPath -> do
+      st <- newFileStore ()
+      sl <- mapM (`readWriteFileStore` st) storeLogPath
+      atomically $ writeTVar (stmStoreLog st) sl
+      pure (StoreMemory st, sl)
+#if defined(dbServerPostgres)
+    XSCDatabase dbCfg -> do
+      st <- newFileStore dbCfg
+      pure (StoreDatabase st, Nothing)
+#endif
+  used <- getUsedStorage (fromFileStore store)
+  usedStorage <- newTVarIO used
   forM_ fileSizeQuota $ \quota -> do
     logNote $ "Total / available storage: " <> tshow quota <> " / " <> tshow (quota - used)
     when (quota < used) $ logWarn "WARNING: storage quota is less than used storage, no files can be uploaded!"
@@ -122,12 +193,76 @@ newXFTPServerEnv config@XFTPServerConfig {storeLogFile, fileSizeQuota, xftpCrede
   httpServerCreds <- mapM loadServerCredential httpCredentials
   Fingerprint fp <- loadFingerprint xftpCredentials
   serverStats <- newFileServerStats =<< getCurrentTime
-  pure XFTPEnv {config, store, storeLog, random, tlsServerCreds, httpServerCreds, serverIdentity = C.KeyHash fp, serverStats}
-
-countUsedStorage :: M.Map k FileRec -> Int64
-countUsedStorage = M.foldl' (\acc FileRec {fileInfo = FileInfo {size}} -> acc + fromIntegral size) 0
+  pure XFTPEnv {config, store, usedStorage, storeLog, random, tlsServerCreds, httpServerCreds, serverIdentity = C.KeyHash fp, serverStats}
 
 data XFTPRequest
   = XFTPReqNew FileInfo (NonEmpty RcvPublicAuthKey) (Maybe BasicAuth)
   | XFTPReqCmd XFTPFileId FileRec FileCmd
   | XFTPReqPing
+
+readFileStoreType :: Ini -> Either String AFStoreType
+readFileStoreType ini = case fromRight "memory" $ T.unpack <$> lookupValue "STORE_LOG" "store_files" ini of
+  "memory" -> Right $ AFSType SFSMemory
+  "database" -> Right $ AFSType SFSPostgres
+  other -> Left $ "Invalid store_files value: " <> other
+
+-- | Dispatch store config from AFStoreType singleton and run the callback.
+-- CPP guards for Postgres are handled here so Main.hs stays CPP-free.
+runWithStoreConfig ::
+  AFStoreType ->
+  Ini ->
+  FilePath ->
+  MigrationConfirmation ->
+  (forall s. FileStoreClass s => XFTPStoreConfig s -> IO ()) ->
+  IO ()
+runWithStoreConfig (AFSType SFSMemory) ini storeLogFilePath _confirmMigrations run =
+  run $ XSCMemory (enableStoreLog' $> storeLogFilePath)
+  where
+    enableStoreLog' = settingIsOn "STORE_LOG" "enable" ini
+runWithStoreConfig (AFSType SFSPostgres) ini storeLogFilePath confirmMigrations run =
+#if defined(dbServerPostgres)
+  run $ XSCDatabase dbCfg
+  where
+    enableDbStoreLog' = settingIsOn "STORE_LOG" "db_store_log" ini
+    dbStoreLogPath = enableDbStoreLog' $> storeLogFilePath
+    dbCfg = PostgresFileStoreCfg {dbOpts = iniDBOptions ini defaultXFTPDBOpts, dbStoreLogPath, confirmMigrations}
+#else
+  error "server binary is compiled without support for PostgreSQL database"
+#endif
+
+-- | Validate startup config when store_files=database.
+checkFileStoreMode :: Ini -> AFStoreType -> FilePath -> IO ()
+checkFileStoreMode ini (AFSType SFSPostgres) storeLogFilePath = do
+#if defined(dbServerPostgres)
+  storeLogExists <- doesFileExist storeLogFilePath
+  let dbStoreLogOn = settingIsOn "STORE_LOG" "db_store_log" ini
+  when (storeLogExists && isNothing dbStoreLogOn) $ do
+    putStrLn $ "Error: store log file " <> storeLogFilePath <> " exists but store_files is `database`."
+    putStrLn "Use `file-server database import` to migrate, or set `db_store_log: on`."
+    exitFailure
+#else
+  putStrLn "Error: server binary is compiled without support for PostgreSQL database."
+  putStrLn "Please re-compile with `cabal build -fserver_postgres`."
+  exitFailure
+#endif
+checkFileStoreMode _ (AFSType SFSMemory) _ = pure ()
+
+-- | Import StoreLog to PostgreSQL database.
+importToDatabase :: FilePath -> Ini -> MigrationConfirmation -> IO ()
+#if defined(dbServerPostgres)
+importToDatabase storeLogFilePath ini _confirmMigrations = do
+  let dbCfg = PostgresFileStoreCfg {dbOpts = iniDBOptions ini defaultXFTPDBOpts, dbStoreLogPath = Nothing, confirmMigrations = _confirmMigrations}
+  importFileStore storeLogFilePath dbCfg
+#else
+importToDatabase _ _ _ = error "Error: server binary is compiled without support for PostgreSQL database.\nPlease re-compile with `cabal build -fserver_postgres`."
+#endif
+
+-- | Export PostgreSQL database to StoreLog.
+exportFromDatabase :: FilePath -> Ini -> MigrationConfirmation -> IO ()
+#if defined(dbServerPostgres)
+exportFromDatabase storeLogFilePath ini _confirmMigrations = do
+  let dbCfg = PostgresFileStoreCfg {dbOpts = iniDBOptions ini defaultXFTPDBOpts, dbStoreLogPath = Nothing, confirmMigrations = _confirmMigrations}
+  exportFileStore storeLogFilePath dbCfg
+#else
+exportFromDatabase _ _ _ = error "Error: server binary is compiled without support for PostgreSQL database.\nPlease re-compile with `cabal build -fserver_postgres`."
+#endif

src/Simplex/FileTransfer/Server/Main.hs

@@ -5,16 +5,23 @@
 {-# LANGUAGE OverloadedLists #-}
 {-# LANGUAGE OverloadedStrings #-}
 {-# LANGUAGE PatternSynonyms #-}
+{-# LANGUAGE RankNTypes #-}
+{-# LANGUAGE TypeApplications #-}
 
 module Simplex.FileTransfer.Server.Main
   ( xftpServerCLI,
+    xftpServerCLI_,
   ) where
 
+import Control.Monad (unless, when)
 import Data.Either (fromRight)
 import Data.Functor (($>))
 import Data.Ini (lookupValue, readIniFile)
 import Data.Int (Int64)
+import Data.List (find)
+import qualified Data.List.NonEmpty as L
 import Data.Maybe (fromMaybe, isJust)
+import Data.Text.Encoding (encodeUtf8)
 import qualified Data.Text as T
 import qualified Data.Text.IO as T
 import Network.Socket (HostName)
@@ -22,13 +29,17 @@ import Options.Applicative
 import Simplex.FileTransfer.Chunks
 import Simplex.FileTransfer.Description (FileSize (..))
 import Simplex.FileTransfer.Server (runXFTPServer)
-import Simplex.FileTransfer.Server.Env (XFTPServerConfig (..), defFileExpirationHours, defaultFileExpiration, defaultInactiveClientExpiration)
+import Simplex.FileTransfer.Server.Env (XFTPServerConfig (..), XFTPStoreConfig, AFStoreType (..), defFileExpirationHours, defaultFileExpiration, defaultInactiveClientExpiration, readFileStoreType, runWithStoreConfig, checkFileStoreMode, importToDatabase, exportFromDatabase)
 import Simplex.FileTransfer.Transport (alpnSupportedXFTPhandshakes, supportedFileServerVRange)
 import qualified Simplex.Messaging.Crypto as C
 import Simplex.Messaging.Encoding.String
 import Simplex.Messaging.Protocol (ProtoServerWithAuth (..), pattern XFTPServer)
+import Simplex.Messaging.Agent.Store.Shared (MigrationConfirmation (..))
 import Simplex.Messaging.Server.CLI
 import Simplex.Messaging.Server.Expiration
+import Simplex.Messaging.Server.Information (ServerPublicInfo (..))
+import Simplex.Messaging.Server.Main (serverPublicInfo, printSourceCode)
+import Simplex.Messaging.Server.Web (EmbeddedWebParams (..), WebHttpsParams (..))
 import Simplex.Messaging.Transport.Client (TransportHost (..))
 import Simplex.Messaging.Transport.HTTP2 (httpALPN)
 import Simplex.Messaging.Transport.Server (ServerCredentials (..), TransportServerConfig (..), mkTransportServerConfig)
@@ -39,7 +50,15 @@ import System.IO (BufferMode (..), hSetBuffering, stderr, stdout)
 import Text.Read (readMaybe)
 
 xftpServerCLI :: FilePath -> FilePath -> IO ()
-xftpServerCLI cfgPath logPath = do
+xftpServerCLI = xftpServerCLI_ (\_ _ _ _ -> pure ()) (\_ -> pure ())
+
+xftpServerCLI_ ::
+  (forall s. XFTPServerConfig s -> Maybe ServerPublicInfo -> Maybe TransportHost -> FilePath -> IO ()) ->
+  (EmbeddedWebParams -> IO ()) ->
+  FilePath ->
+  FilePath ->
+  IO ()
+xftpServerCLI_ generateSite serveStaticFiles cfgPath logPath = do
   getCliCommand' (cliCommandP cfgPath logPath iniFile) serverVersion >>= \case
     Init opts ->
       doesFileExist iniFile >>= \case
@@ -49,9 +68,13 @@ xftpServerCLI cfgPath logPath = do
       doesFileExist iniFile >>= \case
         True -> genOnline cfgPath certOpts
         _ -> exitError $ "Error: server is not initialized (" <> iniFile <> " does not exist).\nRun `" <> executableName <> " init`."
-    Start ->
+    Start opts ->
       doesFileExist iniFile >>= \case
-        True -> readIniFile iniFile >>= either exitError runServer
+        True -> readIniFile iniFile >>= either exitError (runServer opts)
+        _ -> exitError $ "Error: server is not initialized (" <> iniFile <> " does not exist).\nRun `" <> executableName <> " init`."
+    Database cmd ->
+      doesFileExist iniFile >>= \case
+        True -> readIniFile iniFile >>= either exitError (runDatabaseCmd cmd)
         _ -> exitError $ "Error: server is not initialized (" <> iniFile <> " does not exist).\nRun `" <> executableName <> " init`."
     Delete -> do
       confirmOrExit
@@ -66,7 +89,23 @@ xftpServerCLI cfgPath logPath = do
     defaultServerPort = "443"
     executableName = "file-server"
     storeLogFilePath = combine logPath "file-server-store.log"
-    initializeServer InitOptions {enableStoreLog, signAlgorithm, ip, fqdn, filesPath, fileSizeQuota} = do
+    defaultStaticPath = combine logPath "www"
+    runDatabaseCmd cmd ini = case cmd of
+      SCImport -> do
+        storeLogExists <- doesFileExist storeLogFilePath
+        unless storeLogExists $ exitError $ "Error: store log file " <> storeLogFilePath <> " does not exist."
+        confirmOrExit
+          ("Import store log " <> storeLogFilePath <> " to PostgreSQL database?")
+          "Import cancelled."
+        importToDatabase storeLogFilePath ini MCYesUp
+      SCExport -> do
+        storeLogExists <- doesFileExist storeLogFilePath
+        when storeLogExists $ exitError $ "Error: store log file " <> storeLogFilePath <> " already exists."
+        confirmOrExit
+          ("Export PostgreSQL database to store log " <> storeLogFilePath <> "?")
+          "Export cancelled."
+        exportFromDatabase storeLogFilePath ini MCConsole
+    initializeServer InitOptions {enableStoreLog, signAlgorithm, ip, fqdn, filesPath, fileSizeQuota, webStaticPath = webStaticPath_} = do
       clearDirIfExists cfgPath
       clearDirIfExists logPath
       createDirectoryIfMissing True cfgPath
@@ -81,56 +120,89 @@ xftpServerCLI cfgPath logPath = do
       printServiceInfo serverVersion srv
       where
         iniFileContent host =
-          "[STORE_LOG]\n\
+          "[INFORMATION]\n\
+          \# AGPLv3 license requires that you make any source code modifications\n\
+          \# available to the end users of the server.\n\
+          \# LICENSE: https://github.com/simplex-chat/simplexmq/blob/stable/LICENSE\n\
+          \# Include correct source code URI in case the server source code is modified in any way.\n\
+          \# source_code = https://github.com/simplex-chat/simplexmq\n\
+          \\n\
+          \# Declaring all below information is optional, any of these fields can be omitted.\n\
+          \# server_country = ISO-3166 2-letter code\n\
+          \# operator = entity (organization or person name)\n\
+          \# operator_country = ISO-3166 2-letter code\n\
+          \# website =\n\
+          \# admin_simplex = SimpleX address\n\
+          \# admin_email =\n\
+          \# complaints_simplex = SimpleX address\n\
+          \# complaints_email =\n\
+          \# hosting = entity (organization or person name)\n\
+          \# hosting_country = ISO-3166 2-letter code\n\
+          \# hosting_type = virtual\n\
+          \\n\
+          \[STORE_LOG]\n\
           \# The server uses STM memory for persistence,\n\
           \# that will be lost on restart (e.g., as with redis).\n\
           \# This option enables saving memory to append only log,\n\
           \# and restoring it when the server is started.\n\
           \# Log is compacted on start (deleted objects are removed).\n"
-            <> ("enable: " <> onOff enableStoreLog <> "\n\n")
+            <> ("enable = " <> onOff enableStoreLog <> "\n\n")
+            <> "# File storage mode: `memory` or `database` (PostgreSQL).\n\
+               \store_files = memory\n\n\
+               \# Database connection settings for PostgreSQL database (`store_files = database`).\n\
+               \# db_connection = postgresql://xftp@/xftp_server_store\n\
+               \# db_schema = xftp_server\n\
+               \# db_pool_size = 10\n\n\
+               \# Write database changes to store log file\n\
+               \# db_store_log = off\n\n"
             <> "# Expire files after the specified number of hours.\n"
-            <> ("expire_files_hours: " <> tshow defFileExpirationHours <> "\n\n")
-            <> "log_stats: off\n\
+            <> ("expire_files_hours = " <> tshow defFileExpirationHours <> "\n\n")
+            <> "log_stats = off\n\
                \\n\
                \# Log interval for real-time Prometheus metrics\n\
-               \# prometheus_interval: 60\n\
+               \# prometheus_interval = 60\n\
                \\n\
                \[AUTH]\n\
                \# Set new_files option to off to completely prohibit uploading new files.\n\
                \# This can be useful when you want to decommission the server, but still allow downloading the existing files.\n\
-               \new_files: on\n\
+               \new_files = on\n\
                \\n\
                \# Use create_password option to enable basic auth to upload new files.\n\
                \# The password should be used as part of server address in client configuration:\n\
                \# xftp://fingerprint:password@host1,host2\n\
                \# The password will not be shared with file recipients, you must share it only\n\
                \# with the users who you want to allow uploading files to your server.\n\
-               \# create_password: password to upload files (any printable ASCII characters without whitespace, '@', ':' and '/')\n\
+               \# create_password = password to upload files (any printable ASCII characters without whitespace, '@', ':' and '/')\n\
                \\n\
-               \# control_port_admin_password:\n\
-               \# control_port_user_password:\n\
+               \# control_port_admin_password =\n\
+               \# control_port_user_password =\n\
                \\n\
                \[TRANSPORT]\n\
                \# host is only used to print server address on start\n"
-            <> ("host: " <> T.pack host <> "\n")
-            <> ("port: " <> T.pack defaultServerPort <> "\n")
-            <> "log_tls_errors: off\n\
-               \# control_port: 5226\n\
+            <> ("host = " <> T.pack host <> "\n")
+            <> ("port = " <> T.pack defaultServerPort <> "\n")
+            <> "log_tls_errors = off\n\
+               \# control_port = 5226\n\
                \\n\
                \[FILES]\n"
-            <> ("path: " <> T.pack filesPath <> "\n")
-            <> ("storage_quota: " <> safeDecodeUtf8 (strEncode fileSizeQuota) <> "\n")
+            <> ("path = " <> T.pack filesPath <> "\n")
+            <> ("storage_quota = " <> safeDecodeUtf8 (strEncode fileSizeQuota) <> "\n")
             <> "\n\
                \[INACTIVE_CLIENTS]\n\
                \# TTL and interval to check inactive clients\n\
-               \disconnect: off\n"
-            <> ("# ttl: " <> tshow (ttl defaultInactiveClientExpiration) <> "\n")
-            <> ("# check_interval: " <> tshow (checkInterval defaultInactiveClientExpiration) <> "\n")
+               \disconnect = off\n"
+            <> ("# ttl = " <> tshow (ttl defaultInactiveClientExpiration) <> "\n")
+            <> ("# check_interval = " <> tshow (checkInterval defaultInactiveClientExpiration) <> "\n")
             <> "\n\
                \[WEB]\n\
-               \# cert: /etc/opt/simplex-xftp/web.crt\n\
-               \# key: /etc/opt/simplex-xftp/web.key\n"
-    runServer ini = do
+               \# Set path to generate static mini-site for server information\n"
+            <> ("static_path = " <> T.pack (fromMaybe defaultStaticPath webStaticPath_) <> "\n\n")
+            <> "# Run an embedded HTTP server on this port.\n\
+               \# http = 8000\n\n\
+               \# TLS credentials for HTTPS web server on the same port as XFTP.\n\
+               \# cert = " <> T.pack (cfgPath `combine` "web.crt") <> "\n\
+               \# key = " <> T.pack (cfgPath `combine` "web.key") <> "\n"
+    runServer StartOptions {confirmMigrations} ini = do
       hSetBuffering stdout LineBuffering
       hSetBuffering stderr LineBuffering
       fp <- checkSavedFingerprint cfgPath defaultX509Config
@@ -138,9 +210,28 @@ xftpServerCLI cfgPath logPath = do
           port = T.unpack $ strictIni "TRANSPORT" "port" ini
           srv = ProtoServerWithAuth (XFTPServer [THDomainName host] (if port == "443" then "" else port) (C.KeyHash fp)) Nothing
       printServiceInfo serverVersion srv
-      printXFTPConfig serverConfig
-      runXFTPServer serverConfig
+      let information = serverPublicInfo ini
+      printSourceCode (sourceCode <$> information)
+      case readFileStoreType ini of
+        Left err -> error err
+        Right fsType -> do
+          checkFileStoreMode ini fsType storeLogFilePath
+          runWithStoreConfig fsType ini storeLogFilePath confirmMigrations $ \storeCfg -> do
+            let cfg = serverConfig storeCfg
+            printXFTPConfig cfg
+            case webStaticPath' of
+              Just path -> do
+                let onionHost =
+                      either (const Nothing) (find isOnion) $
+                        strDecode @(L.NonEmpty TransportHost) . encodeUtf8 =<< lookupValue "TRANSPORT" "host" ini
+                    webHttpPort = eitherToMaybe (lookupValue "WEB" "http" ini) >>= readMaybe . T.unpack
+                generateSite cfg information onionHost path
+                when (isJust webHttpPort || isJust webHttpsParams') $
+                  serveStaticFiles EmbeddedWebParams {webStaticPath = path, webHttpPort, webHttpsParams = webHttpsParams'}
+              Nothing -> pure ()
+            runXFTPServer cfg
       where
+        isOnion = \case THOnionHost _ -> True; _ -> False
         enableStoreLog = settingIsOn "STORE_LOG" "enable" ini
         logStats = settingIsOn "STORE_LOG" "log_stats" ini
         c = combine cfgPath . ($ defaultX509Config)
@@ -172,11 +263,21 @@ xftpServerCLI cfgPath logPath = do
                   privateKeyFile = key
                 }
 
-        serverConfig =
+        webHttpsParams' = do
+          httpsPort <- eitherToMaybe (lookupValue "WEB" "https" ini) >>= readMaybe . T.unpack
+          cert <- eitherToMaybe $ T.unpack <$> lookupValue "WEB" "cert" ini
+          key <- eitherToMaybe $ T.unpack <$> lookupValue "WEB" "key" ini
+          pure WebHttpsParams {port = httpsPort, cert, key}
+
+        webStaticPath' = eitherToMaybe $ T.unpack <$> lookupValue "WEB" "static_path" ini
+
+        serverConfig :: XFTPStoreConfig s -> XFTPServerConfig s
+        serverConfig serverStoreCfg =
           XFTPServerConfig
             { xftpPort = T.unpack $ strictIni "TRANSPORT" "port" ini,
               controlPort = either (const Nothing) (Just . T.unpack) $ lookupValue "TRANSPORT" "control_port" ini,
               fileIdSize = 16,
+              serverStoreCfg,
               storeLogFile = enableStoreLog $> storeLogFilePath,
               filesPath = T.unpack $ strictIni "FILES" "path" ini,
               fileSizeQuota = either error unFileSize <$> strDecodeIni "FILES" "storage_quota" ini,
@@ -209,7 +310,7 @@ xftpServerCLI cfgPath logPath = do
               logStatsStartTime = 0, -- seconds from 00:00 UTC
               serverStatsLogFile = combine logPath "file-server-stats.daily.log",
               serverStatsBackupFile = logStats $> combine logPath "file-server-stats.log",
-              prometheusInterval = eitherToMaybe $ read . T.unpack <$> lookupValue "STORE_LOG" "prometheus_interval" ini,
+              prometheusInterval = eitherToMaybe (lookupValue "STORE_LOG" "prometheus_interval" ini) >>= readMaybe . T.unpack,
               prometheusMetricsFile = combine logPath "xftp-server-metrics.txt",
               transportConfig =
                 let cfg =
@@ -218,22 +319,31 @@ xftpServerCLI cfgPath logPath = do
                         (Just $ alpnSupportedXFTPhandshakes <> httpALPN)
                         False
                  in cfg {addCORSHeaders = isJust httpCredentials_},
-              responseDelay = 0
+              responseDelay = 0,
+              webStaticPath = webStaticPath'
             }
 
 data CliCommand
   = Init InitOptions
   | OnlineCert CertOptions
-  | Start
+  | Start StartOptions
+  | Database StoreCmd
   | Delete
 
+data StoreCmd = SCImport | SCExport
+
+newtype StartOptions = StartOptions
+  { confirmMigrations :: MigrationConfirmation
+  }
+
 data InitOptions = InitOptions
   { enableStoreLog :: Bool,
     signAlgorithm :: SignAlgorithm,
     ip :: HostName,
     fqdn :: Maybe HostName,
     filesPath :: FilePath,
-    fileSizeQuota :: FileSize Int64
+    fileSizeQuota :: FileSize Int64,
+    webStaticPath :: Maybe FilePath
   }
   deriving (Show)
 
@@ -242,7 +352,8 @@ cliCommandP cfgPath logPath iniFile =
   hsubparser
     ( command "init" (info (Init <$> initP) (progDesc $ "Initialize server - creates " <> cfgPath <> " and " <> logPath <> " directories and configuration files"))
         <> command "cert" (info (OnlineCert <$> certOptionsP) (progDesc $ "Generate new online TLS server credentials (configuration: " <> iniFile <> ")"))
-        <> command "start" (info (pure Start) (progDesc $ "Start server (configuration: " <> iniFile <> ")"))
+        <> command "start" (info (Start <$> startOptsP) (progDesc $ "Start server (configuration: " <> iniFile <> ")"))
+        <> command "database" (info (Database <$> storeCmdP) (progDesc "Import/export file store to/from PostgreSQL database"))
         <> command "delete" (info (pure Delete) (progDesc "Delete configuration and log files"))
     )
   where
@@ -302,4 +413,27 @@ cliCommandP cfgPath logPath iniFile =
               <> help "File storage quota (e.g. 100gb)"
               <> metavar "QUOTA"
           )
-      pure InitOptions {enableStoreLog, signAlgorithm, ip, fqdn, filesPath, fileSizeQuota}
+      webStaticPath <-
+        (optional . strOption)
+          ( long "web-path"
+              <> help "Directory to store generated static site with server information"
+              <> metavar "PATH"
+          )
+      pure InitOptions {enableStoreLog, signAlgorithm, ip, fqdn, filesPath, fileSizeQuota, webStaticPath}
+    startOptsP :: Parser StartOptions
+    startOptsP = do
+      confirmMigrations <-
+        option
+          parseConfirmMigrations
+          ( long "confirm-migrations"
+              <> metavar "CONFIRM_MIGRATIONS"
+              <> help "Confirm PostgreSQL database migration: up, down (default is manual confirmation)"
+              <> value MCConsole
+          )
+      pure StartOptions {confirmMigrations}
+    storeCmdP :: Parser StoreCmd
+    storeCmdP =
+      hsubparser
+        ( command "import" (info (pure SCImport) (progDesc "Import store log file into PostgreSQL database"))
+            <> command "export" (info (pure SCExport) (progDesc "Export PostgreSQL database to store log file"))
+        )

src/Simplex/FileTransfer/Server/Store.hs

@@ -1,51 +1,53 @@
 {-# LANGUAGE DataKinds #-}
 {-# LANGUAGE GADTs #-}
+{-# LANGUAGE KindSignatures #-}
 {-# LANGUAGE LambdaCase #-}
 {-# LANGUAGE NamedFieldPuns #-}
 {-# LANGUAGE OverloadedStrings #-}
 {-# LANGUAGE TupleSections #-}
+{-# LANGUAGE TypeFamilies #-}
 
 module Simplex.FileTransfer.Server.Store
-  ( FileStore (..),
+  ( FSType (..),
+    SFSType (..),
+    FileStoreClass (..),
     FileRec (..),
     FileRecipient (..),
+    STMFileStore (..),
     RoundedFileTime,
-    newFileStore,
-    addFile,
-    setFilePath,
-    addRecipient,
-    deleteFile,
-    blockFile,
-    deleteRecipient,
-    expiredFilePath,
-    getFile,
-    ackFile,
     fileTimePrecision,
   )
 where
 
+import Data.Kind (Type)
+
 import Control.Concurrent.STM
-import Control.Monad
+import Control.Monad (forM, void)
 import qualified Data.Attoparsec.ByteString.Char8 as A
 import Data.Int (Int64)
+import qualified Data.Map.Strict as M
+import Data.Maybe (catMaybes)
 import Data.Set (Set)
 import qualified Data.Set as S
+import Data.Word (Word32)
 import Simplex.FileTransfer.Protocol (FileInfo (..), SFileParty (..), XFTPFileId)
 import Simplex.FileTransfer.Transport (XFTPErrorType (..))
 import qualified Simplex.Messaging.Crypto as C
 import Simplex.Messaging.Encoding.String
 import Simplex.Messaging.Protocol (BlockingInfo, RcvPublicAuthKey, RecipientId, SenderId)
 import Simplex.Messaging.Server.QueueStore (ServerEntityStatus (..))
+import Simplex.Messaging.Server.StoreLog (StoreLog, closeStoreLog)
+import System.IO (IOMode (..))
 import Simplex.Messaging.SystemTime
 import Simplex.Messaging.TMap (TMap)
 import qualified Simplex.Messaging.TMap as TM
-import Simplex.Messaging.Util (ifM, ($>>=))
+import Simplex.Messaging.Util (ifM)
 
-data FileStore = FileStore
-  { files :: TMap SenderId FileRec,
-    recipients :: TMap RecipientId (SenderId, RcvPublicAuthKey),
-    usedStorage :: TVar Int64
-  }
+data FSType = FSMemory | FSPostgres
+
+data SFSType :: FSType -> Type where
+  SFSMemory :: SFSType 'FSMemory
+  SFSPostgres :: SFSType 'FSPostgres
 
 data FileRec = FileRec
   { senderId :: SenderId,
@@ -59,28 +61,126 @@ data FileRec = FileRec
 type RoundedFileTime = RoundedSystemTime 3600
 
 fileTimePrecision :: Int64
-fileTimePrecision = 3600 -- truncate creation time to 1 hour
+fileTimePrecision = 3600
 
-data FileRecipient = FileRecipient RecipientId RcvPublicAuthKey
+data FileRecipient = FileRecipient RecipientId C.APublicAuthKey
   deriving (Show)
 
 instance StrEncoding FileRecipient where
   strEncode (FileRecipient rId rKey) = strEncode rId <> ":" <> strEncode rKey
   strP = FileRecipient <$> strP <* A.char ':' <*> strP
 
-newFileStore :: IO FileStore
-newFileStore = do
-  files <- TM.emptyIO
-  recipients <- TM.emptyIO
-  usedStorage <- newTVarIO 0
-  pure FileStore {files, recipients, usedStorage}
+class FileStoreClass s where
+  type FileStoreConfig s
+  newFileStore :: FileStoreConfig s -> IO s
+  closeFileStore :: s -> IO ()
+  addFile :: s -> SenderId -> FileInfo -> RoundedFileTime -> ServerEntityStatus -> IO (Either XFTPErrorType ())
+  setFilePath :: s -> SenderId -> FilePath -> IO (Either XFTPErrorType ())
+  addRecipient :: s -> SenderId -> FileRecipient -> IO (Either XFTPErrorType ())
+  deleteFile :: s -> SenderId -> IO (Either XFTPErrorType ())
+  deleteFiles :: s -> [SenderId] -> IO ()
+  deleteFiles s = mapM_ (void . deleteFile s)
+  blockFile :: s -> SenderId -> BlockingInfo -> Bool -> IO (Either XFTPErrorType ())
+  deleteRecipient :: s -> RecipientId -> FileRec -> IO ()
+  getFile :: s -> SFileParty p -> XFTPFileId -> IO (Either XFTPErrorType (FileRec, C.APublicAuthKey))
+  ackFile :: s -> RecipientId -> IO (Either XFTPErrorType ())
+  expiredFiles :: s -> Int64 -> Int -> IO [(SenderId, Maybe FilePath, Word32)]
+  getUsedStorage :: s -> IO Int64
+  getFileCount :: s -> IO Int
 
-addFile :: FileStore -> SenderId -> FileInfo -> RoundedFileTime -> ServerEntityStatus -> STM (Either XFTPErrorType ())
-addFile FileStore {files} sId fileInfo createdAt status =
-  ifM (TM.member sId files) (pure $ Left DUPLICATE_) $ do
-    f <- newFileRec sId fileInfo createdAt status
-    TM.insert sId f files
-    pure $ Right ()
+-- STM in-memory store
+
+data STMFileStore = STMFileStore
+  { files :: TMap SenderId FileRec,
+    recipients :: TMap RecipientId (SenderId, RcvPublicAuthKey),
+    stmStoreLog :: TVar (Maybe (StoreLog 'WriteMode))
+  }
+
+instance FileStoreClass STMFileStore where
+  type FileStoreConfig STMFileStore = ()
+
+  newFileStore () = do
+    files <- TM.emptyIO
+    recipients <- TM.emptyIO
+    stmStoreLog <- newTVarIO Nothing
+    pure STMFileStore {files, recipients, stmStoreLog}
+
+  closeFileStore STMFileStore {stmStoreLog} = readTVarIO stmStoreLog >>= mapM_ closeStoreLog
+
+  addFile STMFileStore {files} sId fileInfo createdAt status = atomically $
+    ifM (TM.member sId files) (pure $ Left DUPLICATE_) $ do
+      f <- newFileRec sId fileInfo createdAt status
+      TM.insert sId f files
+      pure $ Right ()
+
+  setFilePath st sId fPath = atomically $
+    withFile st sId $ \FileRec {filePath, fileStatus} -> do
+      readTVar filePath >>= \case
+        Just _ -> pure $ Left AUTH
+        Nothing ->
+          readTVar fileStatus >>= \case
+            EntityActive -> do
+              writeTVar filePath (Just fPath)
+              pure $ Right ()
+            _ -> pure $ Left AUTH
+
+  addRecipient st@STMFileStore {recipients} senderId (FileRecipient rId rKey) = atomically $
+    withFile st senderId $ \FileRec {recipientIds} -> do
+      rIds <- readTVar recipientIds
+      mem <- TM.member rId recipients
+      if rId `S.member` rIds || mem
+        then pure $ Left DUPLICATE_
+        else do
+          writeTVar recipientIds $! S.insert rId rIds
+          TM.insert rId (senderId, rKey) recipients
+          pure $ Right ()
+
+  deleteFile STMFileStore {files, recipients} senderId = atomically $ do
+    TM.lookupDelete senderId files >>= \case
+      Just FileRec {recipientIds} -> do
+        readTVar recipientIds >>= mapM_ (`TM.delete` recipients)
+        pure $ Right ()
+      _ -> pure $ Left AUTH
+
+  blockFile st senderId info _deleted = atomically $
+    withFile st senderId $ \FileRec {fileStatus} -> do
+      writeTVar fileStatus $! EntityBlocked info
+      pure $ Right ()
+
+  deleteRecipient STMFileStore {recipients} rId FileRec {recipientIds} = atomically $ do
+    TM.delete rId recipients
+    modifyTVar' recipientIds $ S.delete rId
+
+  getFile st party fId = atomically $ case party of
+    SFSender -> withFile st fId $ pure . Right . (\f -> (f, sndKey $ fileInfo f))
+    SFRecipient ->
+      TM.lookup fId (recipients st) >>= \case
+        Just (sId, rKey) -> withFile st sId $ pure . Right . (,rKey)
+        _ -> pure $ Left AUTH
+
+  ackFile st@STMFileStore {recipients} recipientId = atomically $ do
+    TM.lookupDelete recipientId recipients >>= \case
+      Just (sId, _) ->
+        withFile st sId $ \FileRec {recipientIds} -> do
+          modifyTVar' recipientIds $ S.delete recipientId
+          pure $ Right ()
+      _ -> pure $ Left AUTH
+
+  expiredFiles STMFileStore {files} old _limit = do
+    fs <- readTVarIO files
+    fmap catMaybes . forM (M.toList fs) $ \(sId, FileRec {fileInfo = FileInfo {size}, filePath, createdAt = RoundedSystemTime createdAt}) ->
+      if createdAt + fileTimePrecision < old
+        then do
+          path <- readTVarIO filePath
+          pure $ Just (sId, path, size)
+        else pure Nothing
+
+  getUsedStorage STMFileStore {files} =
+    M.foldl' (\acc FileRec {fileInfo = FileInfo {size}} -> acc + fromIntegral size) 0 <$> readTVarIO files
+
+  getFileCount STMFileStore {files} = M.size <$> readTVarIO files
+
+-- Internal STM helpers
 
 newFileRec :: SenderId -> FileInfo -> RoundedFileTime -> ServerEntityStatus -> STM FileRec
 newFileRec senderId fileInfo createdAt status = do
@@ -89,75 +189,8 @@ newFileRec senderId fileInfo createdAt status = do
   fileStatus <- newTVar status
   pure FileRec {senderId, fileInfo, filePath, recipientIds, createdAt, fileStatus}
 
-setFilePath :: FileStore -> SenderId -> FilePath -> STM (Either XFTPErrorType ())
-setFilePath st sId fPath =
-  withFile st sId $ \FileRec {fileInfo, filePath} -> do
-    writeTVar filePath (Just fPath)
-    modifyTVar' (usedStorage st) (+ fromIntegral (size fileInfo))
-    pure $ Right ()
-
-addRecipient :: FileStore -> SenderId -> FileRecipient -> STM (Either XFTPErrorType ())
-addRecipient st@FileStore {recipients} senderId (FileRecipient rId rKey) =
-  withFile st senderId $ \FileRec {recipientIds} -> do
-    rIds <- readTVar recipientIds
-    mem <- TM.member rId recipients
-    if rId `S.member` rIds || mem
-      then pure $ Left DUPLICATE_
-      else do
-        writeTVar recipientIds $! S.insert rId rIds
-        TM.insert rId (senderId, rKey) recipients
-        pure $ Right ()
-
--- this function must be called after the file is deleted from the file system
-deleteFile :: FileStore -> SenderId -> STM (Either XFTPErrorType ())
-deleteFile FileStore {files, recipients, usedStorage} senderId = do
-  TM.lookupDelete senderId files >>= \case
-    Just FileRec {fileInfo, recipientIds} -> do
-      readTVar recipientIds >>= mapM_ (`TM.delete` recipients)
-      modifyTVar' usedStorage $ subtract (fromIntegral $ size fileInfo)
-      pure $ Right ()
-    _ -> pure $ Left AUTH
-
--- this function must be called after the file is deleted from the file system
-blockFile :: FileStore -> SenderId -> BlockingInfo -> Bool -> STM (Either XFTPErrorType ())
-blockFile st@FileStore {usedStorage} senderId info deleted =
-  withFile st senderId $ \FileRec {fileInfo, fileStatus} -> do
-    when deleted $ modifyTVar' usedStorage $ subtract (fromIntegral $ size fileInfo)
-    writeTVar fileStatus $! EntityBlocked info
-    pure $ Right ()
-
-deleteRecipient :: FileStore -> RecipientId -> FileRec -> STM ()
-deleteRecipient FileStore {recipients} rId FileRec {recipientIds} = do
-  TM.delete rId recipients
-  modifyTVar' recipientIds $ S.delete rId
-
-getFile :: FileStore -> SFileParty p -> XFTPFileId -> STM (Either XFTPErrorType (FileRec, C.APublicAuthKey))
-getFile st party fId = case party of
-  SFSender -> withFile st fId $ pure . Right . (\f -> (f, sndKey $ fileInfo f))
-  SFRecipient ->
-    TM.lookup fId (recipients st) >>= \case
-      Just (sId, rKey) -> withFile st sId $ pure . Right . (,rKey)
-      _ -> pure $ Left AUTH
-
-expiredFilePath :: FileStore -> XFTPFileId -> Int64 -> STM (Maybe (Maybe FilePath))
-expiredFilePath FileStore {files} sId old =
-  TM.lookup sId files
-    $>>= \FileRec {filePath, createdAt = RoundedSystemTime createdAt} ->
-      if createdAt + fileTimePrecision < old
-        then Just <$> readTVar filePath
-        else pure Nothing
-
-ackFile :: FileStore -> RecipientId -> STM (Either XFTPErrorType ())
-ackFile st@FileStore {recipients} recipientId = do
-  TM.lookupDelete recipientId recipients >>= \case
-    Just (sId, _) ->
-      withFile st sId $ \FileRec {recipientIds} -> do
-        modifyTVar' recipientIds $ S.delete recipientId
-        pure $ Right ()
-    _ -> pure $ Left AUTH
-
-withFile :: FileStore -> SenderId -> (FileRec -> STM (Either XFTPErrorType a)) -> STM (Either XFTPErrorType a)
-withFile FileStore {files} sId a =
+withFile :: STMFileStore -> SenderId -> (FileRec -> STM (Either XFTPErrorType a)) -> STM (Either XFTPErrorType a)
+withFile STMFileStore {files} sId a =
   TM.lookup sId files >>= \case
     Just f -> a f
     _ -> pure $ Left AUTH

src/Simplex/FileTransfer/Server/Store/Postgres.hs

@@ -0,0 +1,370 @@
+{-# LANGUAGE BangPatterns #-}
+{-# LANGUAGE DataKinds #-}
+{-# LANGUAGE GADTs #-}
+{-# LANGUAGE LambdaCase #-}
+{-# LANGUAGE NamedFieldPuns #-}
+{-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE ScopedTypeVariables #-}
+{-# LANGUAGE TypeFamilies #-}
+
+module Simplex.FileTransfer.Server.Store.Postgres
+  ( PostgresFileStore (..),
+    importFileStore,
+    exportFileStore,
+  )
+where
+
+import qualified Control.Exception as E
+import Control.Logger.Simple
+import Control.Monad
+import Control.Monad.Except
+import Control.Monad.IO.Class
+import Control.Monad.Trans.Except (throwE)
+import Data.ByteString (ByteString)
+import qualified Data.ByteString.Char8 as B
+import Data.ByteString.Builder (Builder)
+import qualified Data.ByteString.Builder as BB
+import qualified Data.ByteString.Lazy as LB
+import Data.Functor (($>))
+import Data.Int (Int32, Int64)
+import Data.List (intersperse)
+import qualified Data.List.NonEmpty as L
+import qualified Data.Map.Strict as M
+import qualified Data.Set as S
+import Data.Text (Text)
+import Data.Word (Word32)
+import Database.PostgreSQL.Simple (Binary (..), In (..), Only (..), SqlError, (:.) (..))
+import qualified Database.PostgreSQL.Simple as DB
+import qualified Database.PostgreSQL.Simple.Copy as DB
+import Database.PostgreSQL.Simple.Errors (ConstraintViolation (..), constraintViolation)
+import Database.PostgreSQL.Simple.ToField (Action (..), ToField (..))
+import GHC.IO (catchAny)
+import Simplex.FileTransfer.Protocol (FileInfo (..), SFileParty (..))
+import Simplex.FileTransfer.Server.Store
+import Simplex.FileTransfer.Server.Store.Postgres.Config
+import Simplex.FileTransfer.Server.Store.Postgres.Migrations (xftpServerMigrations)
+import Simplex.FileTransfer.Server.StoreLog
+import Simplex.FileTransfer.Transport (XFTPErrorType (..))
+import Simplex.Messaging.Agent.Store.Postgres (closeDBStore, createDBStore)
+import Simplex.Messaging.Agent.Store.Postgres.Common (DBStore, withTransaction)
+import Simplex.Messaging.Agent.Store.Postgres.Options (DBOpts (..))
+import Simplex.Messaging.Agent.Store.Shared (MigrationConfig (..), MigrationConfirmation (..))
+import qualified Simplex.Messaging.Crypto as C
+import Simplex.Messaging.Protocol (RcvPublicAuthKey, RecipientId, SenderId)
+import Simplex.Messaging.Transport (EntityId (..))
+import Simplex.Messaging.Server.QueueStore (ServerEntityStatus (..))
+import Simplex.Messaging.Server.QueueStore.Postgres ()
+import Simplex.Messaging.Server.StoreLog (openWriteStoreLog)
+import Simplex.Messaging.Util (firstRow, tshow)
+import System.Directory (renameFile)
+import System.Exit (exitFailure)
+import System.IO (IOMode (..), hFlush, stdout)
+import UnliftIO.STM
+
+data PostgresFileStore = PostgresFileStore
+  { dbStore :: DBStore,
+    dbStoreLog :: Maybe (StoreLog 'WriteMode)
+  }
+
+instance FileStoreClass PostgresFileStore where
+  type FileStoreConfig PostgresFileStore = PostgresFileStoreCfg
+
+  newFileStore PostgresFileStoreCfg {dbOpts, dbStoreLogPath, confirmMigrations} = do
+    dbStore <- either err pure =<< createDBStore dbOpts xftpServerMigrations (MigrationConfig confirmMigrations Nothing)
+    dbStoreLog <- mapM (openWriteStoreLog True) dbStoreLogPath
+    pure PostgresFileStore {dbStore, dbStoreLog}
+    where
+      err e = do
+        logError $ "STORE: newFileStore, error opening PostgreSQL database, " <> tshow e
+        exitFailure
+
+  closeFileStore PostgresFileStore {dbStore, dbStoreLog} = do
+    closeDBStore dbStore
+    mapM_ closeStoreLog dbStoreLog
+
+  addFile st sId fileInfo@FileInfo {sndKey, size, digest} createdAt status =
+    E.uninterruptibleMask_ $ runExceptT $ do
+      void $ withDB "addFile" st $ \db ->
+        E.try
+          ( DB.execute
+              db
+              "INSERT INTO files (sender_id, file_size, file_digest, sender_key, created_at, status) VALUES (?,?,?,?,?,?)"
+              (sId, (fromIntegral size :: Int32), Binary digest, Binary (C.encodePubKey sndKey), createdAt, status)
+          )
+          >>= either handleDuplicate (pure . Right)
+      withLog "addFile" st $ \s -> logAddFile s sId fileInfo createdAt status
+
+  setFilePath st sId fPath = E.uninterruptibleMask_ $ runExceptT $ do
+    assertUpdated $ withDB' "setFilePath" st $ \db ->
+      DB.execute db "UPDATE files SET file_path = ? WHERE sender_id = ? AND file_path IS NULL AND status = 'active'" (fPath, sId)
+    withLog "setFilePath" st $ \s -> logPutFile s sId fPath
+
+  addRecipient st senderId (FileRecipient rId rKey) = E.uninterruptibleMask_ $ runExceptT $ do
+    void $ withDB "addRecipient" st $ \db ->
+      E.try
+        ( DB.execute
+            db
+            "INSERT INTO recipients (recipient_id, sender_id, recipient_key) VALUES (?,?,?)"
+            (rId, senderId, Binary (C.encodePubKey rKey))
+        )
+        >>= either handleDuplicate (pure . Right)
+    withLog "addRecipient" st $ \s -> logAddRecipients s senderId (pure $ FileRecipient rId rKey)
+
+  deleteFile st sId = E.uninterruptibleMask_ $ runExceptT $ do
+    assertUpdated $ withDB' "deleteFile" st $ \db ->
+      DB.execute db "DELETE FROM files WHERE sender_id = ?" (Only sId)
+    withLog "deleteFile" st $ \s -> logDeleteFile s sId
+
+  deleteFiles st sIds = E.uninterruptibleMask_ $ do
+    withTransaction (dbStore st) $ \db ->
+      DB.execute db "DELETE FROM files WHERE sender_id IN ?" (Only (In sIds))
+    withLog "deleteFiles" st $ \s -> mapM_ (logDeleteFile s) sIds
+
+  blockFile st sId info _deleted = E.uninterruptibleMask_ $ runExceptT $ do
+    assertUpdated $ withDB' "blockFile" st $ \db ->
+      DB.execute db "UPDATE files SET status = ? WHERE sender_id = ?" (EntityBlocked info, sId)
+    withLog "blockFile" st $ \s -> logBlockFile s sId info
+
+  deleteRecipient st rId _fr =
+    void $ runExceptT $ withDB' "deleteRecipient" st $ \db ->
+      DB.execute db "DELETE FROM recipients WHERE recipient_id = ?" (Only rId)
+
+  getFile st party fId = runExceptT $ case party of
+    SFSender -> do
+      row <- loadFileRow "SELECT sender_id, file_size, file_digest, sender_key, file_path, created_at, status FROM files WHERE sender_id = ?"
+      fr <- ExceptT $ rowToFileRec row
+      pure (fr, sndKey (fileInfo fr))
+    SFRecipient -> do
+      row :. Only rcpKeyBs <-
+        loadFileRow
+          "SELECT f.sender_id, f.file_size, f.file_digest, f.sender_key, f.file_path, f.created_at, f.status, r.recipient_key FROM files f JOIN recipients r ON r.sender_id = f.sender_id WHERE r.recipient_id = ?"
+      fr <- ExceptT $ rowToFileRec row
+      rcpKey <- either (const $ throwE INTERNAL) pure $ C.decodePubKey rcpKeyBs
+      pure (fr, rcpKey)
+    where
+      loadFileRow :: DB.FromRow r => DB.Query -> ExceptT XFTPErrorType IO r
+      loadFileRow q =
+        withDB "getFile" st $ \db ->
+          firstRow id AUTH $ DB.query db q (Only fId)
+
+  ackFile st rId = E.uninterruptibleMask_ $ runExceptT $ do
+    assertUpdated $ withDB' "ackFile" st $ \db ->
+      DB.execute db "DELETE FROM recipients WHERE recipient_id = ?" (Only rId)
+    withLog "ackFile" st $ \s -> logAckFile s rId
+
+  expiredFiles st old limit =
+    fmap toResult $ withTransaction (dbStore st) $ \db ->
+      DB.query
+        db
+        "SELECT sender_id, file_path, file_size FROM files WHERE created_at + ? < ? ORDER BY created_at LIMIT ?"
+        (fileTimePrecision, old, limit)
+    where
+      toResult :: [(SenderId, Maybe FilePath, Int32)] -> [(SenderId, Maybe FilePath, Word32)]
+      toResult = map (\(sId, path, size) -> (sId, path, fromIntegral size))
+
+  getUsedStorage st =
+    withTransaction (dbStore st) $ \db -> do
+      [Only total] <- DB.query_ db "SELECT COALESCE(SUM(file_size::BIGINT), 0)::BIGINT FROM files"
+      pure total
+
+  getFileCount st =
+    withTransaction (dbStore st) $ \db -> do
+      [Only count] <- DB.query_ db "SELECT COUNT(*) FROM files"
+      pure (fromIntegral (count :: Int64))
+
+-- Internal helpers
+
+mkFileRec :: SenderId -> FileInfo -> Maybe FilePath -> RoundedFileTime -> ServerEntityStatus -> IO FileRec
+mkFileRec senderId fileInfo path createdAt status = do
+  filePath <- newTVarIO path
+  recipientIds <- newTVarIO S.empty
+  fileStatus <- newTVarIO status
+  pure FileRec {senderId, fileInfo, filePath, recipientIds, createdAt, fileStatus}
+
+type FileRecRow = (SenderId, Int32, ByteString, ByteString, Maybe FilePath, RoundedFileTime, ServerEntityStatus)
+
+rowToFileRec :: FileRecRow -> IO (Either XFTPErrorType FileRec)
+rowToFileRec (sId, size, digest, sndKeyBs, path, createdAt, status) =
+  case C.decodePubKey sndKeyBs of
+    Right sndKey -> do
+      let fileInfo = FileInfo {sndKey, size = fromIntegral size, digest}
+      Right <$> mkFileRec sId fileInfo path createdAt status
+    Left _ -> pure $ Left INTERNAL
+
+-- DB helpers
+
+withDB :: forall a. Text -> PostgresFileStore -> (DB.Connection -> IO (Either XFTPErrorType a)) -> ExceptT XFTPErrorType IO a
+withDB op st action =
+  ExceptT $ E.try (withTransaction (dbStore st) action) >>= either logErr pure
+  where
+    logErr :: E.SomeException -> IO (Either XFTPErrorType a)
+    logErr e = logError ("STORE: " <> err) $> Left INTERNAL
+      where
+        err = op <> ", withDB, " <> tshow e
+
+withDB' :: Text -> PostgresFileStore -> (DB.Connection -> IO a) -> ExceptT XFTPErrorType IO a
+withDB' op st action = withDB op st $ fmap Right . action
+
+assertUpdated :: ExceptT XFTPErrorType IO Int64 -> ExceptT XFTPErrorType IO ()
+assertUpdated = (>>= \n -> when (n == 0) (throwE AUTH))
+
+handleDuplicate :: SqlError -> IO (Either XFTPErrorType a)
+handleDuplicate e = case constraintViolation e of
+  Just (UniqueViolation _) -> pure $ Left DUPLICATE_
+  Just (ForeignKeyViolation _ _) -> pure $ Left AUTH
+  _ -> E.throwIO e
+
+withLog :: MonadIO m => Text -> PostgresFileStore -> (StoreLog 'WriteMode -> IO ()) -> m ()
+withLog op PostgresFileStore {dbStoreLog} action =
+  forM_ dbStoreLog $ \sl -> liftIO $ action sl `catchAny` \e ->
+    logWarn $ "STORE: " <> op <> ", withLog, " <> tshow e
+
+-- Import: StoreLog -> PostgreSQL
+
+importFileStore :: FilePath -> PostgresFileStoreCfg -> IO ()
+importFileStore storeLogFilePath dbCfg = do
+  putStrLn $ "Reading store log: " <> storeLogFilePath
+  stmStore <- newFileStore () :: IO STMFileStore
+  sl <- readWriteFileStore storeLogFilePath stmStore
+  closeStoreLog sl
+  allFiles <- readTVarIO (files stmStore)
+  allRcps <- readTVarIO (recipients stmStore)
+  let fileCount = M.size allFiles
+      rcpCount = M.size allRcps
+  putStrLn $ "Loaded " <> show fileCount <> " files, " <> show rcpCount <> " recipients."
+  let dbCfg' = dbCfg {dbOpts = (dbOpts dbCfg) {createSchema = True}, confirmMigrations = MCYesUp}
+  pgStore <- newFileStore dbCfg' :: IO PostgresFileStore
+  existingCount <- getFileCount pgStore
+  when (existingCount > 0) $ do
+    putStrLn $ "WARNING: database already contains " <> show existingCount <> " files. Import will fail on duplicate keys."
+    putStrLn "Drop the existing schema first or use a fresh database."
+    exitFailure
+  putStrLn "Importing files..."
+  fCnt <- withTransaction (dbStore pgStore) $ \db -> do
+    DB.copy_
+      db
+      "COPY files (sender_id, file_size, file_digest, sender_key, file_path, created_at, status) FROM STDIN WITH (FORMAT csv)"
+    iforM_ (M.toList allFiles) $ \i (sId, fr) -> do
+      DB.putCopyData db =<< fileRecToCSV sId fr
+      when (i > 0 && i `mod` 10000 == 0) $ putStr ("  " <> show i <> " files\r") >> hFlush stdout
+    DB.putCopyEnd db
+    [Only cnt] <- DB.query_ db "SELECT COUNT(*) FROM files"
+    pure (cnt :: Int64)
+  putStrLn $ "Imported " <> show fCnt <> " files."
+  putStrLn "Importing recipients..."
+  rCnt <- withTransaction (dbStore pgStore) $ \db -> do
+    DB.copy_
+      db
+      "COPY recipients (recipient_id, sender_id, recipient_key) FROM STDIN WITH (FORMAT csv)"
+    iforM_ (M.toList allRcps) $ \i (rId, (sId, rKey)) -> do
+      DB.putCopyData db $ recipientToCSV rId sId rKey
+      when (i > 0 && i `mod` 10000 == 0) $ putStr ("  " <> show i <> " recipients\r") >> hFlush stdout
+    DB.putCopyEnd db
+    [Only cnt] <- DB.query_ db "SELECT COUNT(*) FROM recipients"
+    pure (cnt :: Int64)
+  putStrLn $ "Imported " <> show rCnt <> " recipients."
+  when (fromIntegral fileCount /= fCnt) $
+    putStrLn $ "WARNING: expected " <> show fileCount <> " files, got " <> show fCnt
+  when (fromIntegral rcpCount /= rCnt) $
+    putStrLn $ "WARNING: expected " <> show rcpCount <> " recipients, got " <> show rCnt
+  closeFileStore pgStore
+  renameFile storeLogFilePath (storeLogFilePath <> ".bak")
+  putStrLn $ "Store log renamed to " <> storeLogFilePath <> ".bak"
+
+-- Export: PostgreSQL -> StoreLog
+
+exportFileStore :: FilePath -> PostgresFileStoreCfg -> IO ()
+exportFileStore storeLogFilePath dbCfg = do
+  pgStore <- newFileStore dbCfg :: IO PostgresFileStore
+  sl <- openWriteStoreLog False storeLogFilePath
+  -- Fold 1: stream files, write FNEW + FPUT per file
+  putStrLn "Exporting files..."
+  !fCnt <- withTransaction (dbStore pgStore) $ \db ->
+    DB.fold_
+      db
+      "SELECT sender_id, file_size, file_digest, sender_key, file_path, created_at, status FROM files ORDER BY created_at"
+      (0 :: Int)
+      ( \(!fc) (sId, size :: Int32, digest :: ByteString, sndKeyBs :: ByteString, path :: Maybe String, createdAt, status) ->
+          case C.decodePubKey sndKeyBs of
+            Right sndKey -> do
+              let fileInfo = FileInfo {sndKey, size = fromIntegral size, digest}
+              logAddFile sl sId fileInfo createdAt status
+              forM_ path $ logPutFile sl sId
+              pure (fc + 1)
+            Left _ -> do
+              putStrLn $ "WARNING: invalid sender key for " <> show sId
+              pure fc
+      )
+  -- Fold 2: stream recipients ordered by sender_id, flush FADD on sender change
+  putStrLn "Exporting recipients..."
+  !rCnt <- withTransaction (dbStore pgStore) $ \db ->
+    DB.fold_
+      db
+      "SELECT sender_id, recipient_id, recipient_key FROM recipients ORDER BY sender_id"
+      (Nothing :: Maybe SenderId, [] :: [FileRecipient], 0 :: Int)
+      ( \(!prevSId, !buf, !rc) (sId, rId, rKeyBs :: ByteString) ->
+          case C.decodePubKey rKeyBs of
+            Right rKey -> do
+              let rcp = FileRecipient rId rKey
+              case prevSId of
+                Just prev | prev /= sId -> do
+                  forM_ (L.nonEmpty buf) $ logAddRecipients sl prev
+                  pure (Just sId, [rcp], rc + length buf)
+                _ -> pure (Just sId, rcp : buf, rc)
+            Left _ -> putStrLn ("WARNING: invalid recipient key for " <> show rId) $> (prevSId, buf, rc)
+      )
+      >>= \(lastSId, buf, rc) -> do
+        forM_ lastSId $ \sId -> forM_ (L.nonEmpty buf) $ logAddRecipients sl sId
+        pure (rc + length buf)
+  closeStoreLog sl
+  closeFileStore pgStore
+  putStrLn $ "Exported " <> show fCnt <> " files, " <> show rCnt <> " recipients to " <> storeLogFilePath
+
+-- CSV helpers for COPY protocol
+
+iforM_ :: Monad m => [a] -> (Int -> a -> m ()) -> m ()
+iforM_ xs f = zipWithM_ f [0 ..] xs
+
+fileRecToCSV :: SenderId -> FileRec -> IO ByteString
+fileRecToCSV sId FileRec {fileInfo = FileInfo {sndKey, size, digest}, filePath, createdAt, fileStatus} = do
+  path <- readTVarIO filePath
+  status <- readTVarIO fileStatus
+  pure $ LB.toStrict $ BB.toLazyByteString $ mconcat (BB.char7 ',' `intersperse` fields path status) <> BB.char7 '\n'
+  where
+    fields path status =
+      [ renderField (toField (Binary (unEntityId sId))),
+        renderField (toField (fromIntegral size :: Int32)),
+        renderField (toField (Binary digest)),
+        renderField (toField (Binary (C.encodePubKey sndKey))),
+        nullable (toField <$> path),
+        renderField (toField createdAt),
+        quotedField (toField status)
+      ]
+
+recipientToCSV :: RecipientId -> SenderId -> RcvPublicAuthKey -> ByteString
+recipientToCSV rId sId rKey =
+  LB.toStrict $ BB.toLazyByteString $ mconcat (BB.char7 ',' `intersperse` fields) <> BB.char7 '\n'
+  where
+    fields =
+      [ renderField (toField (Binary (unEntityId rId))),
+        renderField (toField (Binary (unEntityId sId))),
+        renderField (toField (Binary (C.encodePubKey rKey)))
+      ]
+
+renderField :: Action -> Builder
+renderField = \case
+  Plain bld -> bld
+  Escape s -> BB.byteString s
+  EscapeByteA s -> BB.string7 "\\x" <> BB.byteStringHex s
+  EscapeIdentifier s -> BB.byteString s
+  Many as -> mconcat (map renderField as)
+
+nullable :: Maybe Action -> Builder
+nullable = maybe mempty renderField
+
+quotedField :: Action -> Builder
+quotedField a = BB.char7 '"' <> escapeQuotes (renderField a) <> BB.char7 '"'
+  where
+    escapeQuotes bld =
+      let bs = LB.toStrict $ BB.toLazyByteString bld
+       in BB.byteString $ B.concatMap (\c -> if c == '"' then "\"\"" else B.singleton c) bs

src/Simplex/FileTransfer/Server/Store/Postgres/Config.hs

@@ -0,0 +1,25 @@
+{-# LANGUAGE OverloadedStrings #-}
+
+module Simplex.FileTransfer.Server.Store.Postgres.Config
+  ( PostgresFileStoreCfg (..),
+    defaultXFTPDBOpts,
+  )
+where
+
+import Simplex.Messaging.Agent.Store.Postgres.Options (DBOpts (..))
+import Simplex.Messaging.Agent.Store.Shared (MigrationConfirmation)
+
+data PostgresFileStoreCfg = PostgresFileStoreCfg
+  { dbOpts :: DBOpts,
+    dbStoreLogPath :: Maybe FilePath,
+    confirmMigrations :: MigrationConfirmation
+  }
+
+defaultXFTPDBOpts :: DBOpts
+defaultXFTPDBOpts =
+  DBOpts
+    { connstr = "postgresql://xftp@/xftp_server_store",
+      schema = "xftp_server",
+      poolSize = 10,
+      createSchema = False
+    }

src/Simplex/FileTransfer/Server/Store/Postgres/Migrations.hs

@@ -0,0 +1,47 @@
+{-# LANGUAGE NamedFieldPuns #-}
+{-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE QuasiQuotes #-}
+
+module Simplex.FileTransfer.Server.Store.Postgres.Migrations
+  ( xftpServerMigrations,
+  )
+where
+
+import Data.List (sortOn)
+import Data.Text (Text)
+import Simplex.Messaging.Agent.Store.Shared
+import Text.RawString.QQ (r)
+
+xftpSchemaMigrations :: [(String, Text, Maybe Text)]
+xftpSchemaMigrations =
+  [ ("20260325_initial", m20260325_initial, Nothing)
+  ]
+
+-- | The list of migrations in ascending order by date
+xftpServerMigrations :: [Migration]
+xftpServerMigrations = sortOn name $ map migration xftpSchemaMigrations
+  where
+    migration (name, up, down) = Migration {name, up, down = down}
+
+m20260325_initial :: Text
+m20260325_initial =
+  [r|
+CREATE TABLE files (
+  sender_id BYTEA NOT NULL PRIMARY KEY,
+  file_size INTEGER NOT NULL,
+  file_digest BYTEA NOT NULL,
+  sender_key BYTEA NOT NULL,
+  file_path TEXT,
+  created_at BIGINT NOT NULL,
+  status TEXT NOT NULL DEFAULT 'active'
+);
+
+CREATE TABLE recipients (
+  recipient_id BYTEA NOT NULL PRIMARY KEY,
+  sender_id BYTEA NOT NULL REFERENCES files ON DELETE CASCADE,
+  recipient_key BYTEA NOT NULL
+);
+
+CREATE INDEX idx_recipients_sender_id ON recipients (sender_id);
+CREATE INDEX idx_files_created_at ON files (created_at);
+|]

src/Simplex/FileTransfer/Server/StoreLog.hs

@@ -10,6 +10,7 @@ module Simplex.FileTransfer.Server.StoreLog
     FileStoreLogRecord (..),
     closeStoreLog,
     readWriteFileStore,
+    writeFileStore,
     logAddFile,
     logPutFile,
     logAddRecipients,
@@ -32,6 +33,7 @@ import Data.Map.Strict (Map)
 import qualified Data.Map.Strict as M
 import Simplex.FileTransfer.Protocol (FileInfo (..))
 import Simplex.FileTransfer.Server.Store
+import Simplex.FileTransfer.Transport (XFTPErrorType (..))
 import Simplex.Messaging.Encoding.String
 import Simplex.Messaging.Protocol (BlockingInfo, RcvPublicAuthKey, RecipientId, SenderId)
 import Simplex.Messaging.Server.QueueStore (ServerEntityStatus (..))
@@ -87,20 +89,22 @@ logBlockFile s fId = logFileStoreRecord s . BlockFile fId
 logAckFile :: StoreLog 'WriteMode -> RecipientId -> IO ()
 logAckFile s = logFileStoreRecord s . AckFile
 
-readWriteFileStore :: FilePath -> FileStore -> IO (StoreLog 'WriteMode)
+readWriteFileStore :: FilePath -> STMFileStore -> IO (StoreLog 'WriteMode)
 readWriteFileStore = readWriteStoreLog readFileStore writeFileStore
 
-readFileStore :: FilePath -> FileStore -> IO ()
+readFileStore :: FilePath -> STMFileStore -> IO ()
 readFileStore f st = mapM_ (addFileLogRecord . LB.toStrict) . LB.lines =<< LB.readFile f
   where
     addFileLogRecord s = case strDecode s of
       Left e -> B.putStrLn $ "Log parsing error (" <> B.pack e <> "): " <> B.take 100 s
       Right lr ->
-        atomically (addToStore lr) >>= \case
+        addToStore lr >>= \case
           Left e -> B.putStrLn $ "Log processing error (" <> bshow e <> "): " <> B.take 100 s
           _ -> pure ()
     addToStore = \case
-      AddFile sId file createdAt status -> addFile st sId file createdAt status
+      AddFile sId file createdAt status
+        | size file > 0 -> addFile st sId file createdAt status
+        | otherwise -> pure $ Left SIZE
       PutFile qId path -> setFilePath st qId path
       AddRecipients sId rcps -> runExceptT $ addRecipients sId rcps
       DeleteFile sId -> deleteFile st sId
@@ -108,8 +112,8 @@ readFileStore f st = mapM_ (addFileLogRecord . LB.toStrict) . LB.lines =<< LB.re
       AckFile rId -> ackFile st rId
     addRecipients sId rcps = mapM_ (ExceptT . addRecipient st sId) rcps
 
-writeFileStore :: StoreLog 'WriteMode -> FileStore -> IO ()
-writeFileStore s FileStore {files, recipients} = do
+writeFileStore :: StoreLog 'WriteMode -> STMFileStore -> IO ()
+writeFileStore s STMFileStore {files, recipients} = do
   allRcps <- readTVarIO recipients
   readTVarIO files >>= mapM_ (logFile allRcps)
   where

src/Simplex/FileTransfer/Types.hs

@@ -46,6 +46,7 @@ import Simplex.Messaging.Encoding
 import Simplex.Messaging.Encoding.String
 import Simplex.Messaging.Parsers
 import Simplex.Messaging.Protocol (XFTPServer)
+import Simplex.Messaging.Util (safeDecodeUtf8, (<$$>))
 import System.FilePath ((</>))
 
 type RcvFileId = ByteString -- Agent entity ID
@@ -65,7 +66,8 @@ data FileHeader = FileHeader
 instance Encoding FileHeader where
   smpEncode FileHeader {fileName, fileExtra} = smpEncode (fileName, fileExtra)
   smpP = do
-    (fileName, fileExtra) <- smpP
+    fileName <- safeDecodeUtf8 <$> smpP
+    fileExtra <- safeDecodeUtf8 <$$> smpP
     pure FileHeader {fileName, fileExtra}
 
 type DBRcvFileId = Int64

src/Simplex/Messaging/Agent.hs

@@ -64,6 +64,7 @@ module Simplex.Messaging.Agent
     setConnShortLink,
     deleteConnShortLink,
     getConnShortLink,
+    getConnLinkPrivKey,
     deleteLocalInvShortLink,
     changeConnectionUser,
     prepareConnectionToJoin,
@@ -354,9 +355,9 @@ setConnShortLinkAsync :: AgentClient -> ACorrId -> ConnId -> UserConnLinkData 'C
 setConnShortLinkAsync c = withAgentEnv c .:: setConnShortLinkAsync' c
 {-# INLINE setConnShortLinkAsync #-}
 
--- | Get and verify data from short link (LGET/LKEY command) asynchronously, synchronous response is new connection id
-getConnShortLinkAsync :: AgentClient -> UserId -> ACorrId -> ConnShortLink 'CMContact -> AE ConnId
-getConnShortLinkAsync c = withAgentEnv c .:. getConnShortLinkAsync' c
+-- | Get and verify data from short link (LGET/LKEY command) asynchronously, synchronous response is new/passed connection id
+getConnShortLinkAsync :: AgentClient -> UserId -> ACorrId -> Maybe ConnId -> ConnShortLink 'CMContact -> AE ConnId
+getConnShortLinkAsync c = withAgentEnv c .:: getConnShortLinkAsync' c
 {-# INLINE getConnShortLinkAsync #-}
 
 -- | Join SMP agent connection (JOIN command) asynchronously, synchronous response is new connection id.
@@ -401,10 +402,11 @@ createConnection c nm userId enableNtfs checkNotices = withAgentEnv c .::. newCo
 {-# INLINE createConnection #-}
 
 -- | Prepare connection link for contact mode (no network call).
--- Returns root key pair (for signing OwnerAuth), the created link, and internal params.
+-- Caller provides root signing key pair and link entity ID.
+-- Returns the created link and internal params.
 -- The link address is fully determined at this point.
-prepareConnectionLink :: AgentClient -> UserId -> Maybe ByteString -> Bool -> Maybe CRClientData -> AE (C.KeyPairEd25519, CreatedConnLink 'CMContact, PreparedLinkParams)
-prepareConnectionLink c userId linkEntityId checkNotices = withAgentEnv c . prepareConnectionLink' c userId linkEntityId checkNotices
+prepareConnectionLink :: AgentClient -> UserId -> C.KeyPairEd25519 -> ByteString -> Bool -> Maybe CRClientData -> AE (CreatedConnLink 'CMContact, PreparedLinkParams)
+prepareConnectionLink c userId rootKey linkEntityId checkNotices = withAgentEnv c . prepareConnectionLink' c userId rootKey linkEntityId checkNotices
 {-# INLINE prepareConnectionLink #-}
 
 -- | Create connection for prepared link (single network call).
@@ -427,6 +429,10 @@ getConnShortLink :: AgentClient -> NetworkRequestMode -> UserId -> ConnShortLink
 getConnShortLink c = withAgentEnv c .:. getConnShortLink' c
 {-# INLINE getConnShortLink #-}
 
+getConnLinkPrivKey :: AgentClient -> ConnId -> AE (Maybe C.PrivateKeyEd25519)
+getConnLinkPrivKey c = withAgentEnv c . getConnLinkPrivKey' c
+{-# INLINE getConnLinkPrivKey #-}
+
 -- | This irreversibly deletes short link data, and it won't be retrievable again
 deleteLocalInvShortLink :: AgentClient -> ConnShortLink 'CMInvitation -> AE ()
 deleteLocalInvShortLink c = withAgentEnv c . deleteLocalInvShortLink' c
@@ -918,23 +924,22 @@ newConn c nm userId enableNtfs checkNotices cMode linkData_ clientData pqInitKey
       `catchE` \e -> withStore' c (`deleteConnRecord` connId) >> throwE e
 
 -- | Prepare connection link for contact mode (no network, no database).
--- Generates all cryptographic material and returns the link that will be created.
-prepareConnectionLink' :: AgentClient -> UserId -> Maybe ByteString -> Bool -> Maybe CRClientData -> AM (C.KeyPairEd25519, CreatedConnLink 'CMContact, PreparedLinkParams)
-prepareConnectionLink' c userId linkEntityId checkNotices clientData = do
+-- Caller provides root signing key pair and link entity ID.
+prepareConnectionLink' :: AgentClient -> UserId -> C.KeyPairEd25519 -> ByteString -> Bool -> Maybe CRClientData -> AM (CreatedConnLink 'CMContact, PreparedLinkParams)
+prepareConnectionLink' c userId rootKey@(_, plpRootPrivKey) linkEntityId checkNotices clientData = do
   g <- asks random
   plpSrvWithAuth@(ProtoServerWithAuth srv _) <- getSMPServer c userId
   when checkNotices $ checkClientNotices c plpSrvWithAuth
   AgentConfig {smpClientVRange, smpAgentVRange} <- asks config
   plpNonce@(C.CbNonce corrId) <- atomically $ C.randomCbNonce g
-  sigKeys@(_, plpRootPrivKey) <- atomically $ C.generateKeyPair g
   plpQueueE2EKeys@(e2ePubKey, _) <- atomically $ C.generateKeyPair g
   let sndId = SMP.EntityId $ B.take 24 $ C.sha3_384 corrId
       qUri = SMPQueueUri smpClientVRange $ SMPQueueAddress srv sndId e2ePubKey (Just QMContact)
       connReq = CRContactUri $ ConnReqUriData SSSimplex smpAgentVRange [qUri] clientData
-      (plpLinkKey, plpSignedFixedData) = SL.encodeSignFixedData sigKeys smpAgentVRange connReq linkEntityId
+      (plpLinkKey, plpSignedFixedData) = SL.encodeSignFixedData rootKey smpAgentVRange connReq (Just linkEntityId)
       ccLink = CCLink connReq $ Just $ CSLContact SLSServer CCTContact srv plpLinkKey
       params = PreparedLinkParams {plpNonce, plpQueueE2EKeys, plpLinkKey, plpRootPrivKey, plpSignedFixedData, plpSrvWithAuth}
-  pure (sigKeys, ccLink, params)
+  pure (ccLink, params)
 
 -- | Create connection for prepared link (single network call).
 createConnectionForLink' :: AgentClient -> NetworkRequestMode -> UserId -> Bool -> CreatedConnLink 'CMContact -> PreparedLinkParams -> UserConnLinkData 'CMContact -> CR.InitialKeys -> SubscriptionMode -> AM ConnId
@@ -1001,14 +1006,22 @@ setConnShortLinkAsync' c corrId connId userLinkData clientData =
       _ -> throwE $ CMD PROHIBITED "setConnShortLinkAsync: invalid connection or mode"
     enqueueCommand c corrId connId (Just srv) $ AClientCommand $ LSET userLinkData clientData
 
-getConnShortLinkAsync' :: AgentClient -> UserId -> ACorrId -> ConnShortLink 'CMContact -> AM ConnId
-getConnShortLinkAsync' c userId corrId shortLink@(CSLContact _ _ srv _) = do
-  g <- asks random
-  connId <- withStore c $ \db -> do
-    -- server is created so the command is processed in server queue,
-    -- not blocking other "no server" commands
-    void $ createServer db srv
-    prepareNewConn db g
+getConnShortLinkAsync' :: AgentClient -> UserId -> ACorrId -> Maybe ConnId -> ConnShortLink 'CMContact -> AM ConnId
+getConnShortLinkAsync' c userId corrId connId_ shortLink@(CSLContact _ _ srv _) = do
+  connId <- case connId_ of
+    Just existingConnId -> do
+      -- connId and srv can be unrelated: connId is used as "mailbox" for LDATA delivery,
+      -- while srv is the short link's server for the LGET request.
+      -- E.g., owner's relay connection (connId, on server A) fetches relay's group link data (srv = server B).
+      -- This works because enqueueCommand stores (connId, srv) independently in the commands table,
+      -- the network request targets srv, and event delivery uses connId via corrId correlation.
+      withStore' c $ \db -> void $ createServer db srv
+      pure existingConnId
+    Nothing -> do
+      g <- asks random
+      withStore c $ \db -> do
+        void $ createServer db srv
+        prepareNewConn db g
   enqueueCommand c corrId connId (Just srv) $ AClientCommand $ LGET shortLink
   pure connId
   where
@@ -1079,6 +1092,14 @@ deleteConnShortLink' c nm connId cMode =
       (RcvConnection _ rq, SCMInvitation) -> deleteQueueLink c nm rq
       _ -> throwE $ CMD PROHIBITED "deleteConnShortLink: not contact address"
 
+getConnLinkPrivKey' :: AgentClient -> ConnId -> AM (Maybe C.PrivateKeyEd25519)
+getConnLinkPrivKey' c connId = do
+  SomeConn _ conn <- withStore c (`getConn` connId)
+  pure $ case conn of
+    ContactConnection _ rq -> linkPrivSigKey <$> shortLink rq
+    RcvConnection _ rq -> linkPrivSigKey <$> shortLink rq
+    _ -> Nothing
+
 -- TODO [short links] remove 1-time invitation data and link ID from the server after the message is sent.
 getConnShortLink' :: forall c. AgentClient -> NetworkRequestMode -> UserId -> ConnShortLink c -> AM (FixedLinkData c, ConnLinkData c)
 getConnShortLink' c nm userId = \case
@@ -1556,12 +1577,11 @@ subscribeAllConnections' :: AgentClient -> Bool -> Maybe UserId -> AM ()
 subscribeAllConnections' c onlyNeeded activeUserId_ = handleErr $ do
   userSrvs <- withStore' c (`getSubscriptionServers` onlyNeeded)
   unless (null userSrvs) $ do
-    maxPending <- asks $ maxPendingSubscriptions . config
-    currPending <- newTVarIO 0
+    batchSize <- asks $ subsBatchSize . config
     let userSrvs' = case activeUserId_ of
           Just activeUserId -> sortOn (\(uId, _) -> if uId == activeUserId then 0 else 1 :: Int) userSrvs
           Nothing -> userSrvs
-    rs <- lift $ mapConcurrently (subscribeUserServer maxPending currPending) userSrvs'
+    rs <- lift $ mapConcurrently (subscribeUserServer batchSize) userSrvs'
     let (errs, oks) = partitionEithers rs
     logInfo $ "subscribed " <> tshow (sum oks) <> " queues"
     forM_ (L.nonEmpty errs) $ notifySub c . ERRS . L.map ("",)
@@ -1570,18 +1590,16 @@ subscribeAllConnections' c onlyNeeded activeUserId_ = handleErr $ do
   resumeAllCommands c
   where
     handleErr = (`catchAllErrors` \e -> notifySub' c "" (ERR e) >> throwE e)
-    subscribeUserServer :: Int -> TVar Int -> (UserId, SMPServer) -> AM' (Either AgentErrorType Int)
-    subscribeUserServer maxPending currPending (userId, srv) = do
-      atomically $ whenM ((maxPending <=) <$> readTVar currPending) retry
-      tryAllErrors' $ do
-        qs <- withStore' c $ \db -> do
-          qs <- getUserServerRcvQueueSubs db userId srv onlyNeeded
-          unless (null qs) $ atomically $ modifyTVar' currPending (+ length qs) -- update before leaving transaction
-          pure qs
-        let n = length qs
-        unless (null qs) $ lift $ subscribe qs `E.finally` atomically (modifyTVar' currPending $ subtract n)
-        pure n
+    subscribeUserServer :: Int -> (UserId, SMPServer) -> AM' (Either AgentErrorType Int)
+    subscribeUserServer batchSize (userId, srv) = tryAllErrors' $ loop 0 Nothing
       where
+        loop !n cursor_ = do
+          qs <- withStore' c $ \db -> getUserServerRcvQueueSubs db userId srv onlyNeeded batchSize cursor_
+          if null qs then pure n else do
+            lift $ subscribe qs
+            let n' = n + length qs
+                lastRcvId = Just $ queueId $ last qs
+            if length qs < batchSize then pure n' else loop n' lastRcvId
         subscribe qs = do
           rs <- subscribeUserServerQueues c userId srv qs
           -- TODO [certs rcv] storeClientServiceAssocs store associations of queues with client service ID
@@ -3140,18 +3158,28 @@ processSMPTransmissions c@AgentClient {subQ} (tSess@(userId, srv, _), _v, sessId
                                     pure conn''
                                 | otherwise = pure conn'
                           Right Nothing -> prohibited "msg: bad agent msg" >> ack
-                          Left e@(AGENT A_DUPLICATE) -> do
+                          Left e@(AGENT A_DUPLICATE {}) -> do
                             atomically $ incSMPServerStat c userId srv recvDuplicates
                             withStore' c (\db -> getLastMsg db connId srvMsgId) >>= \case
                               Just RcvMsg {internalId, msgMeta, msgBody = agentMsgBody, userAck}
                                 | userAck -> ackDel internalId
-                                | otherwise ->
-                                    liftEither (parse smpP (AGENT A_MESSAGE) agentMsgBody) >>= \case
-                                      AgentMessage _ (A_MSG body) -> do
-                                        logServer "<--" c srv rId $ "MSG <MSG>:" <> logSecret' srvMsgId
-                                        notify $ MSG msgMeta msgFlags body
-                                        pure ACKPending
-                                      _ -> ack
+                                | otherwise -> do
+                                    attempts <- withStore' c $ \db -> incMsgRcvAttempts db connId internalId
+                                    AgentConfig {rcvExpireCount, rcvExpireInterval} <- asks config
+                                    let firstTs = snd $ recipient msgMeta
+                                        brokerTs = snd $ broker msgMeta
+                                    now <- liftIO getCurrentTime
+                                    if attempts >= rcvExpireCount && diffUTCTime now firstTs >= rcvExpireInterval
+                                      then do
+                                        notify $ ERR (AGENT $ A_DUPLICATE $ Just DroppedMsg {brokerTs, attempts})
+                                        ackDel internalId
+                                      else
+                                        liftEither (parse smpP (AGENT A_MESSAGE) agentMsgBody) >>= \case
+                                          AgentMessage _ (A_MSG body) -> do
+                                            logServer "<--" c srv rId $ "MSG <MSG>:" <> logSecret' srvMsgId
+                                            notify $ MSG msgMeta msgFlags body
+                                            pure ACKPending
+                                          _ -> ack
                               _ -> checkDuplicateHash e encryptedMsgHash >> ack
                           Left (AGENT (A_CRYPTO e)) -> do
                             atomically $ incSMPServerStat c userId srv recvCryptoErrs

src/Simplex/Messaging/Agent/Client.hs

@@ -764,12 +764,15 @@ resubscribeSMPSession c@AgentClient {smpSubWorkers, workerSeq} tSess = do
         (pure Nothing) -- prevent race with cleanup and adding pending queues in another call
         (Just <$> getSessVar workerSeq tSess smpSubWorkers ts)
     newSubWorker v = do
-      a <- async $ void (E.tryAny runSubWorker) >> atomically (cleanup v)
+      a <- async $ void $ E.tryAny $ runSubWorker v
       atomically $ putTMVar (sessionVar v) a
-    runSubWorker = do
+    runSubWorker v = do
       ri <- asks $ reconnectInterval . config
       withRetryForeground ri isForeground (isNetworkOnline c) $ \_ loop -> do
-        pending <- atomically $ SS.getPendingSubs tSess $ currentSubs c
+        pending <- atomically $ do
+          qs <- SS.getPendingSubs tSess $ currentSubs c
+          when (M.null qs) $ cleanup v
+          pure qs
         unless (M.null pending) $ do
           liftIO $ waitUntilForeground c
           liftIO $ waitForUserNetwork c
@@ -1595,9 +1598,15 @@ checkQueues c = fmap partitionEithers . mapM checkQueue
 -- and that they are already added to pending subscriptions.
 resubscribeSessQueues :: AgentClient -> SMPTransportSession -> [RcvQueueSub] -> AM' ()
 resubscribeSessQueues c tSess qs = do
+  batchSize <- asks $ subsBatchSize . config
   (errs, qs_) <- checkQueues c qs
-  forM_ (L.nonEmpty qs_) $ \qs' -> void $ subscribeSessQueues_ c True (tSess, qs')
+  subscribeChunks $ toChunks batchSize qs_
   forM_ (L.nonEmpty errs) $ notifySub c . ERRS . L.map (first qConnId)
+  where
+    subscribeChunks [] = pure ()
+    subscribeChunks (qs' : rest) = do
+      (_, active) <- subscribeSessQueues_ c True (tSess, qs')
+      when active $ subscribeChunks rest
 
 subscribeSessQueues_ :: AgentClient -> Bool -> (SMPTransportSession, NonEmpty RcvQueueSub) -> AM' (BatchResponses RcvQueueSub AgentErrorType (Maybe ServiceId), Bool)
 subscribeSessQueues_ c withEvents qs = sendClientBatch_ "SUB" False subscribe_ c NRMBackground qs
@@ -2096,7 +2105,7 @@ cryptoError :: C.CryptoError -> AgentErrorType
 cryptoError = \case
   C.CryptoLargeMsgError -> CMD LARGE "CryptoLargeMsgError"
   C.CryptoHeaderError _ -> AGENT A_MESSAGE -- parsing error
-  C.CERatchetDuplicateMessage -> AGENT A_DUPLICATE
+  C.CERatchetDuplicateMessage -> AGENT $ A_DUPLICATE Nothing
   C.AESDecryptError -> c DECRYPT_AES
   C.CBDecryptError -> c DECRYPT_CB
   C.CERatchetHeader -> c RATCHET_HEADER

src/Simplex/Messaging/Agent/Env/SQLite.hs

@@ -168,10 +168,12 @@ data AgentConfig = AgentConfig
     ntfBatchSize :: Int,
     ntfSubFirstCheckInterval :: NominalDiffTime,
     ntfSubCheckInterval :: NominalDiffTime,
-    maxPendingSubscriptions :: Int,
+    subsBatchSize :: Int,
     caCertificateFile :: FilePath,
     privateKeyFile :: FilePath,
     certificateFile :: FilePath,
+    rcvExpireCount :: Int,
+    rcvExpireInterval :: NominalDiffTime,
     e2eEncryptVRange :: VersionRangeE2E,
     smpAgentVRange :: VersionRangeSMPA,
     smpClientVRange :: VersionRangeSMPC
@@ -241,12 +243,14 @@ defaultAgentConfig =
       ntfBatchSize = 150,
       ntfSubFirstCheckInterval = nominalDay,
       ntfSubCheckInterval = 3 * nominalDay,
-      maxPendingSubscriptions = 35000,
+      subsBatchSize = 1350,
       -- CA certificate private key is not needed for initialization
       -- ! we do not generate these
       caCertificateFile = "/etc/opt/simplex-agent/ca.crt",
       privateKeyFile = "/etc/opt/simplex-agent/agent.key",
       certificateFile = "/etc/opt/simplex-agent/agent.crt",
+      rcvExpireCount = 8,
+      rcvExpireInterval = nominalDay,
       e2eEncryptVRange = supportedE2EEncryptVRange,
       smpAgentVRange = supportedSMPAgentVRange,
       smpClientVRange = supportedSMPClientVRange

src/Simplex/Messaging/Agent/Protocol.hs

@@ -146,6 +146,7 @@ module Simplex.Messaging.Agent.Protocol
     ConnectionErrorType (..),
     BrokerErrorType (..),
     SMPAgentError (..),
+    DroppedMsg (..),
     AgentCryptoError (..),
     cryptoErrToSyncState,
     ATransmission,
@@ -788,6 +789,12 @@ data MsgMeta = MsgMeta
   }
   deriving (Eq, Show)
 
+data DroppedMsg = DroppedMsg
+  { brokerTs :: UTCTime,
+    attempts :: Int
+  }
+  deriving (Eq, Show)
+
 data SMPConfirmation = SMPConfirmation
   { -- | sender's public key to use for authentication of sender's commands at the recepient's server
     senderKey :: Maybe SndPublicAuthKey,
@@ -2050,12 +2057,13 @@ data SMPAgentError
     A_LINK {linkErr :: String}
   | -- | cannot decrypt message
     A_CRYPTO {cryptoErr :: AgentCryptoError}
-  | -- | duplicate message - this error is detected by ratchet decryption - this message will be ignored and not shown
-    -- it may also indicate a loss of ratchet synchronization (when only one message is sent via copied ratchet)
-    A_DUPLICATE
+  | -- | duplicate message - this error is detected by ratchet decryption - this message will be ignored and not shown.
+    -- it may also indicate a loss of ratchet synchronization (when only one message is sent via copied ratchet).
+    -- when message is dropped after too many reception attempts, DroppedMsg is included.
+    A_DUPLICATE {droppedMsg_ :: Maybe DroppedMsg}
   | -- | error in the message to add/delete/etc queue in connection
     A_QUEUE {queueErr :: String}
-  deriving (Eq, Read, Show, Exception)
+  deriving (Eq, Show, Exception)
 
 data AgentCryptoError
   = -- | AES decryption error
@@ -2165,6 +2173,8 @@ $(J.deriveJSON (sumTypeJSON id) ''ConnectionErrorType)
 
 $(J.deriveJSON (sumTypeJSON id) ''AgentCryptoError)
 
+$(J.deriveJSON defaultJSON ''DroppedMsg)
+
 $(J.deriveJSON (sumTypeJSON id) ''SMPAgentError)
 
 $(J.deriveJSON (sumTypeJSON id) ''AgentErrorType)

src/Simplex/Messaging/Agent/Store/AgentStore.hs

@@ -127,6 +127,7 @@ module Simplex.Messaging.Agent.Store.AgentStore
     setMsgUserAck,
     getRcvMsg,
     getLastMsg,
+    incMsgRcvAttempts,
     checkRcvMsgHashExists,
     getRcvMsgBrokerTs,
     deleteMsg,
@@ -457,7 +458,7 @@ checkConfirmedSndQueueExists_ db SndQueue {server, sndId} =
     DB.query
       db
       ( "SELECT 1 FROM snd_queues WHERE host = ? AND port = ? AND snd_id = ? AND status != ? LIMIT 1"
-#if defined(dpPostgres)
+#if defined(dbPostgres)
           <> " FOR UPDATE"
 #endif
       )
@@ -1110,6 +1111,19 @@ toRcvMsg ((agentMsgId, internalTs, brokerId, brokerTs) :. (sndMsgId, integrity,
       msgReceipt = MsgReceipt <$> rcptInternalId_ <*> rcptStatus_
    in RcvMsg {internalId = InternalId agentMsgId, msgMeta, msgType, msgBody, internalHash, msgReceipt, userAck}
 
+incMsgRcvAttempts :: DB.Connection -> ConnId -> InternalId -> IO Int
+incMsgRcvAttempts db connId (InternalId msgId) =
+  fromOnly . head
+    <$> DB.query
+      db
+      [sql|
+        UPDATE rcv_messages
+        SET receive_attempts = receive_attempts + 1
+        WHERE conn_id = ? AND internal_id = ?
+        RETURNING receive_attempts
+      |]
+      (connId, msgId)
+
 checkRcvMsgHashExists :: DB.Connection -> ConnId -> ByteString -> IO Bool
 checkRcvMsgHashExists db connId hash =
   maybeFirstRow' False fromOnlyBI $
@@ -2211,14 +2225,14 @@ getSubscriptionServers db onlyNeeded =
     toUserServer :: (UserId, NonEmpty TransportHost, ServiceName, C.KeyHash) -> (UserId, SMPServer)
     toUserServer (userId, host, port, keyHash) = (userId, SMPServer host port keyHash)
 
-getUserServerRcvQueueSubs :: DB.Connection -> UserId -> SMPServer -> Bool -> IO [RcvQueueSub]
-getUserServerRcvQueueSubs db userId (SMPServer h p kh) onlyNeeded =
-  map toRcvQueueSub
-    <$> DB.query
-      db
-      (rcvQueueSubQuery <> toSubscribe <> " c.deleted = 0 AND q.deleted = 0 AND c.user_id = ? AND q.host = ? AND q.port = ? AND COALESCE(q.server_key_hash, s.key_hash) = ?")
-      (userId, h, p, kh)
+getUserServerRcvQueueSubs :: DB.Connection -> UserId -> SMPServer -> Bool -> Int -> Maybe SMP.RecipientId -> IO [RcvQueueSub]
+getUserServerRcvQueueSubs db userId (SMPServer h p kh) onlyNeeded limit cursor_ =
+  map toRcvQueueSub <$> case cursor_ of
+    Nothing -> DB.query db (q <> orderLimit) (userId, h, p, kh, limit)
+    Just cursor -> DB.query db (q <> " AND q.rcv_id > ? " <> orderLimit) (userId, h, p, kh, cursor, limit)
   where
+    q = rcvQueueSubQuery <> toSubscribe <> " c.deleted = 0 AND q.deleted = 0 AND c.user_id = ? AND q.host = ? AND q.port = ? AND COALESCE(q.server_key_hash, s.key_hash) = ?"
+    orderLimit = " ORDER BY q.rcv_id LIMIT ?"
     toSubscribe
       | onlyNeeded = " WHERE q.to_subscribe = 1 AND "
       | otherwise = " WHERE "

src/Simplex/Messaging/Agent/Store/Entity.hs

@@ -10,6 +10,8 @@
 
 module Simplex.Messaging.Agent.Store.Entity
   ( DBStored (..),
+    SDBStored (..),
+    DBStoredI (..),
     DBEntityId,
     DBEntityId' (..),
   )

src/Simplex/Messaging/Agent/Store/Postgres/Migrations/App.hs

@@ -11,6 +11,7 @@ import Simplex.Messaging.Agent.Store.Postgres.Migrations.M20250702_conn_invitati
 import Simplex.Messaging.Agent.Store.Postgres.Migrations.M20251009_queue_to_subscribe
 import Simplex.Messaging.Agent.Store.Postgres.Migrations.M20251010_client_notices
 import Simplex.Messaging.Agent.Store.Postgres.Migrations.M20251230_strict_tables
+import Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260410_receive_attempts
 import Simplex.Messaging.Agent.Store.Shared (Migration (..))
 
 schemaMigrations :: [(String, Text, Maybe Text)]
@@ -21,7 +22,8 @@ schemaMigrations =
     ("20250702_conn_invitations_remove_cascade_delete", m20250702_conn_invitations_remove_cascade_delete, Just down_m20250702_conn_invitations_remove_cascade_delete),
     ("20251009_queue_to_subscribe", m20251009_queue_to_subscribe, Just down_m20251009_queue_to_subscribe),
     ("20251010_client_notices", m20251010_client_notices, Just down_m20251010_client_notices),
-    ("20251230_strict_tables", m20251230_strict_tables, Just down_m20251230_strict_tables)
+    ("20251230_strict_tables", m20251230_strict_tables, Just down_m20251230_strict_tables),
+    ("20260410_receive_attempts", m20260410_receive_attempts, Just down_m20260410_receive_attempts)
   ]
 
 -- | The list of migrations in ascending order by date

src/Simplex/Messaging/Agent/Store/Postgres/Migrations/M20260410_receive_attempts.hs

@@ -0,0 +1,19 @@
+{-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE QuasiQuotes #-}
+
+module Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260410_receive_attempts where
+
+import Data.Text (Text)
+import Text.RawString.QQ (r)
+
+m20260410_receive_attempts :: Text
+m20260410_receive_attempts =
+  [r|
+ALTER TABLE rcv_messages ADD COLUMN receive_attempts SMALLINT NOT NULL DEFAULT 0;
+|]
+
+down_m20260410_receive_attempts :: Text
+down_m20260410_receive_attempts =
+  [r|
+ALTER TABLE rcv_messages DROP COLUMN receive_attempts;
+|]