mirror of
https://github.com/simplex-chat/simplexmq.git
synced 2026-10-12 11:39:56 +00:00
Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 shum
|
df7bc4b54d | ||
|
shum
|
58e4f0eaa7 | ||
|
shum
|
23d912eb92 |
@@ -9,10 +9,6 @@ on:
|
||||
- "v*"
|
||||
pull_request:
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: ${{ !startsWith(github.ref, 'refs/tags/v') }}
|
||||
|
||||
jobs:
|
||||
|
||||
# =============================
|
||||
@@ -305,29 +301,3 @@ jobs:
|
||||
echo "All "$attempts" attempts failed."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# =============================
|
||||
# Resolver test job
|
||||
# =============================
|
||||
|
||||
# The SNRC resolver is Python, so this job needs none of the Haskell toolchain
|
||||
# above and runs independently of it.
|
||||
|
||||
resolver-test:
|
||||
name: "resolver (python)"
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Clone project
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Set up uv
|
||||
# uv takes Python from .python-version, the version of the Dockerfile's runtime stage.
|
||||
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
|
||||
|
||||
- name: Lint
|
||||
working-directory: scripts/resolver/service
|
||||
run: uv run --frozen ruff check .
|
||||
|
||||
- name: Test
|
||||
working-directory: scripts/resolver/service
|
||||
run: uv run --frozen pytest -v
|
||||
|
||||
@@ -1,8 +1,4 @@
|
||||
*.lock
|
||||
# the resolver's Docker image and CI install exactly these
|
||||
!/scripts/resolver/service/uv.lock
|
||||
# local overrides of scripts/resolver/.env.example
|
||||
/scripts/resolver/.env
|
||||
*.db
|
||||
*.db.bak
|
||||
*.session.sql
|
||||
@@ -16,4 +12,3 @@ cabal.project.local~
|
||||
*.tix
|
||||
.coverage
|
||||
|
||||
__pycache__/
|
||||
|
||||
+6
-20
@@ -719,32 +719,23 @@ Small_random (small * out, void *random_ctx, sntrup761_random_func * random)
|
||||
|
||||
/* ----- Streamlined NTRU Prime Core */
|
||||
|
||||
/* x^p-x-1 has a degree-19 factor mod 3, so a random g is not invertible in R3
|
||||
with probability about 3^-19; KeyGen_attempts failures in a row mean a broken RNG */
|
||||
#define KeyGen_attempts 10
|
||||
|
||||
/* h,(f,ginv) = KeyGen() */
|
||||
/* returns 0 if KeyGen succeeded; else -1 */
|
||||
static int
|
||||
static void
|
||||
KeyGen (Fq * h, small * f, small * ginv, void *random_ctx,
|
||||
sntrup761_random_func * random)
|
||||
{
|
||||
small g[p];
|
||||
Fq finv[p];
|
||||
int i;
|
||||
|
||||
for (i = 0; i < KeyGen_attempts; ++i)
|
||||
for (;;)
|
||||
{
|
||||
Small_random (g, random_ctx, random);
|
||||
if (R3_recip (ginv, g) == 0)
|
||||
break;
|
||||
}
|
||||
if (i == KeyGen_attempts)
|
||||
return -1;
|
||||
Short_random (f, random_ctx, random);
|
||||
Rq_recip3 (finv, f); /* always works */
|
||||
Rq_mult_small (h, finv, g);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* c = Encrypt(r,h) */
|
||||
@@ -893,21 +884,18 @@ typedef small Inputs[p]; /* passed by reference */
|
||||
#define PublicKeys_bytes Rq_bytes
|
||||
|
||||
/* pk,sk = ZKeyGen() */
|
||||
/* returns 0 if KeyGen succeeded; else -1 */
|
||||
static int
|
||||
static void
|
||||
ZKeyGen (unsigned char *pk, unsigned char *sk, void *random_ctx,
|
||||
sntrup761_random_func * random)
|
||||
{
|
||||
Fq h[p];
|
||||
small f[p], v[p];
|
||||
|
||||
if (KeyGen (h, f, v, random_ctx, random) != 0)
|
||||
return -1;
|
||||
KeyGen (h, f, v, random_ctx, random);
|
||||
Rq_encode (pk, h);
|
||||
Small_encode (sk, f);
|
||||
sk += Small_bytes;
|
||||
Small_encode (sk, v);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* C = ZEncrypt(r,pk) */
|
||||
@@ -972,21 +960,19 @@ HashSession (unsigned char *k, int b, const unsigned char *y,
|
||||
/* ----- Streamlined NTRU Prime */
|
||||
|
||||
/* pk,sk = KEM_KeyGen() */
|
||||
int
|
||||
void
|
||||
sntrup761_keypair (unsigned char *pk, unsigned char *sk, void *random_ctx,
|
||||
sntrup761_random_func * random)
|
||||
{
|
||||
int i;
|
||||
|
||||
if (ZKeyGen (pk, sk, random_ctx, random) != 0)
|
||||
return -1;
|
||||
ZKeyGen (pk, sk, random_ctx, random);
|
||||
sk += SecretKeys_bytes;
|
||||
for (i = 0; i < PublicKeys_bytes; ++i)
|
||||
*sk++ = pk[i];
|
||||
random (random_ctx, Inputs_bytes, sk);
|
||||
sk += Inputs_bytes;
|
||||
Hash_prefix (sk, 4, pk, PublicKeys_bytes);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* c,r_enc = Hide(r,pk,cache); cache is Hash4(pk) */
|
||||
|
||||
+1
-2
@@ -19,8 +19,7 @@
|
||||
|
||||
typedef void sntrup761_random_func (void *ctx, size_t length, uint8_t *dst);
|
||||
|
||||
/* returns 0 on success, -1 if the RNG never produced an invertible polynomial */
|
||||
int
|
||||
void
|
||||
sntrup761_keypair (uint8_t *pk, uint8_t *sk,
|
||||
void *random_ctx, sntrup761_random_func *random);
|
||||
|
||||
|
||||
@@ -7,7 +7,6 @@ This file provides guidance on coding style and approaches and on building the c
|
||||
When designing code and planning implementations:
|
||||
- Apply adversarial thinking, and consider what may happen if one of the communicating parties is malicious.
|
||||
- Formulate an explicit threat model for each change - who can do which undesirable things and under which circumstances.
|
||||
- Use the default cryptographic primitive for each purpose from the [primitive registry](../protocol/crypto-registry.md); any exception requires review, and the registry must be updated with every new primitive use or domain-separation string.
|
||||
|
||||
## Code Quality Standards
|
||||
|
||||
|
||||
@@ -1,198 +0,0 @@
|
||||
# SMP server database load: root causes and fixes
|
||||
|
||||
## Summary
|
||||
|
||||
The recurring `pMsgFwdsOwn_pErrorsOther` spikes are a symptom of chronic disk saturation of the
|
||||
`smp_server` PostgreSQL database, not a proxy or forwarding bug. The baseline read load is the
|
||||
Prometheus scrape running whole-table `COUNT` scans of `msg_queues` (48.5M rows) every 30 seconds,
|
||||
compounded by the message-expiration sweep. The write load comes from indexing `updated_at`:
|
||||
`updateQueueTime` stamps it on the first send to each active queue per UTC day, and these non-HOT
|
||||
updates bloat the indexes to 104 GB and amplify WAL to about 425 GB per day (37% full-page images,
|
||||
~31% from `updateQueueTime`). Because the stamp is day-granular, this write burst clusters at 00:00 UTC,
|
||||
tipping the saturated disk into the observed spike and loading the pgBackRest backup repository.
|
||||
|
||||
Code fixes replace the largest metric scan with an estimate and index the sweep and notifier counts.
|
||||
Reclaiming the index bloat, which also clears the remaining metric scan, and keeping it down are
|
||||
operator actions, because PostgreSQL never shrinks indexes automatically.
|
||||
|
||||
## Environment
|
||||
|
||||
The affected deployment runs the pure PostgreSQL backend (`store_queues: database`,
|
||||
`store_messages: database`), schema `smp_server`, `db_pool_size 20`, `prometheus_interval 30`,
|
||||
`[NAMES]` enabled, own-server proxying enabled, and forwards routed over a SOCKS proxy.
|
||||
|
||||
`msg_queues`: 48.5M live rows, 12.3% dead tuples, 14 GB heap, 104 GB indexes, 118 GB total.
|
||||
|
||||
## Evidence
|
||||
|
||||
`pg_stat_statements` over a 5 minute window, ordered by disk reads (`track_io_timing` was off, so
|
||||
`shared_blks_read` is the disk proxy; the two `EXPLAIN ANALYZE` rows are manual and excluded):
|
||||
|
||||
| query | calls | mean ms | reads | share of all reads |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| `getEntityCounts` (the six-`COUNT` metric query) | 30 | 111,456 | 990 GB | 70.7% |
|
||||
| queue-record load (`SELECT recipient_id, …`) | 47,027 | 121 | 89 GB | 6.3% |
|
||||
| expiration sweep batch (`array_agg` in `expire_old_messages`) | 13 | 233,143 | 62 GB | 4.5% |
|
||||
| message peek (`DISTINCT ON (recipient_id)`) | 46,932 | 76 | 43 GB | 3.0% |
|
||||
| `updateQueueTime` (`UPDATE msg_queues SET updated_at`) | 172,885 | 4 | 7.7 GB | 0.5% |
|
||||
| `write_message` | 110,911 | 5 | 7.0 GB | 0.5% |
|
||||
|
||||
`EXPLAIN (ANALYZE, BUFFERS)` of each `msg_queues` count inside `getEntityCounts`:
|
||||
|
||||
| count | plan | time |
|
||||
| --- | --- | --- |
|
||||
| `queue_count` (`deleted_at IS NULL`) | Parallel Seq Scan | 30 s |
|
||||
| `notifier_count` (`deleted_at IS NULL AND notifier_id IS NOT NULL`) | Parallel Seq Scan | 32.6 s |
|
||||
| `ntf_service_queues_count` (`ntf_service_id IS NOT NULL AND deleted_at IS NULL`) | Parallel Seq Scan | 30.2 s |
|
||||
| `rcv_service_queues_count` (`rcv_service_id IS NOT NULL AND deleted_at IS NULL`) | Index Only Scan | 1.4 ms |
|
||||
|
||||
The expiration sweep batch: 229 s for one 10,000-row batch, walking `msg_queues_pkey` and discarding
|
||||
809,729 rows by filter to find 10,000 expirable ones.
|
||||
|
||||
`pg_stat_wal` over the measurement window: 27 TB of WAL accumulated, 37.2% of records full-page
|
||||
images, averaging about 425 GB per day; recent days reach ~470 GB (from the `archive-push` log) as
|
||||
the queue count grows.
|
||||
|
||||
Per-statement WAL from `pg_stat_statements` (`wal_bytes`), leaf statements only (this instance runs
|
||||
`pg_stat_statements.track = all`, so the `write_message` wrapper double-counts the `INSERT` and
|
||||
`UPDATE` it runs, and is excluded):
|
||||
|
||||
| category | share of WAL | main statements |
|
||||
| --- | --- | --- |
|
||||
| `msg_queues` updates | ~42% | `updateQueueTime` ~31%, `msg_can_write` flags ~9%, insert/delete ~2% |
|
||||
| `messages` insert and delete | ~34% | message `INSERT`, `DELETE` by `message_id` and `recipient_id` |
|
||||
| read path (hint bits, FPIs on SELECT) | ~24% | message peek ~15%, `msg_queue_size`, `getEntityCounts` |
|
||||
|
||||
## Root causes
|
||||
|
||||
1. Prometheus scrape. `getEntityCounts` (`QueueStore/Postgres.hs:154`) runs six `COUNT(1)` subqueries
|
||||
every `prometheus_interval` (30 s), called from the metrics path (`Server.hs:698`). Four count
|
||||
`msg_queues`; three of those seq-scan the 48.5M-row heap. At 70.7% of all reads it is the dominant
|
||||
disk consumer, and it runs continuously (mean 111 s per scrape exceeds the 30 s interval).
|
||||
|
||||
2. Message-expiration sweep. `expireMessagesThread` (`Server.hs:477`) calls the `expire_old_messages`
|
||||
procedure (`server_schema.sql`), whose inner batch query selects expirable queues ordered by
|
||||
`recipient_id`. With `oldQueue = 0` the `updated_at` filter matches everything, so the planner
|
||||
walks the primary key and filters `msg_queue_expire`, discarding ~99% of rows per batch.
|
||||
|
||||
3. Index bloat. 104 GB of indexes on a 14 GB table. `updated_at` is part of
|
||||
`idx_msg_queues_updated_at_recipient_id` (`server_schema.sql:526`) and `updateQueueTime`
|
||||
(`QueueStore/Postgres.hs:443`) updates it on the day's first send per queue (172,885 times in the
|
||||
window above). Updating an indexed column prevents HOT updates, so every update writes a new row
|
||||
version plus new entries in every index on the table, and the old index entries accumulate as bloat.
|
||||
|
||||
4. WAL amplification. Only 15.6% of `msg_queues` updates are HOT, because its composite index covers
|
||||
`updated_at` and `msg_queue_expire`, both changed on the hot path (cause 3); the rest rewrite every
|
||||
index on the table. With the 104 GB of bloated indexes and frequent checkpoints (`max_wal_size 4GB`,
|
||||
`checkpoint_timeout 5min`, `wal_compression off`), 37% of WAL records are full-page images, and the
|
||||
server generates about 425 GB of WAL per day (27 TB accumulated; recent days near 470 GB).
|
||||
`updateQueueTime`, stamping `updated_at` on the ~5.3M daily-active queues, is the single largest
|
||||
statement at ~31% of WAL (see Evidence). pgBackRest `archive-push` ships all of it, loading the
|
||||
primary disk and the backup repository. The load peaks
|
||||
at 00:00 UTC because `getSystemDate` rounds `updated_at` to the UTC day, so the first send to each
|
||||
active queue after midnight runs `updateQueueTime`, clustering these writes and the `archive-push`
|
||||
volume in the 00h hour.
|
||||
|
||||
## Index bloat and autovacuum
|
||||
|
||||
Autovacuum is running (60 runs on this table) and is not misconfigured, but two limits apply.
|
||||
|
||||
Autovacuum reclaims heap dead tuples and refreshes planner statistics; it does not shrink indexes.
|
||||
Only `REINDEX` or `pg_repack` rebuilds a bloated B-tree. Index bloat therefore has no automatic
|
||||
remedy.
|
||||
|
||||
Default autovacuum pacing assumes moderate churn. `autovacuum_vacuum_scale_factor = 0.2` waits until
|
||||
20% of the table is dead (about 9.7M rows here) before vacuuming, and cost-based throttling caps its
|
||||
I/O, so a hot table stays around 12% dead. Per-table tuning makes it run sooner and faster.
|
||||
|
||||
The amplifier is indexing `updated_at`, a column updated on every send, which the fixes and operator
|
||||
actions below address.
|
||||
|
||||
## Implemented fixes (batch 1)
|
||||
|
||||
Branch `sh/leaks-batch-1`, two commits.
|
||||
|
||||
| commit | change | effect |
|
||||
| --- | --- | --- |
|
||||
| `88673eee` | migration `20260916_prometheus_indexes` adds partial indexes `idx_msg_queues_expire (recipient_id) WHERE deleted_at IS NULL AND msg_queue_expire` and `idx_msg_queues_notifier_active (notifier_id) WHERE deleted_at IS NULL AND notifier_id IS NOT NULL` | the sweep batch and `notifier_count` become index scans |
|
||||
| `2912be1e` | `getEntityCounts.queue_count` uses `pg_class.reltuples` (`QueueStore/Postgres.hs:154`) | removes the 30 s, 990 GB whole-table `COUNT` on every scrape |
|
||||
|
||||
Verified: both indexes are created and used as index-only scans, the schema-dump test passes, and
|
||||
`smp-server` builds. `queueCount` is consumed only by metrics, logs, and display
|
||||
(`Server.hs:528,698,815,2369`), so an estimate is safe.
|
||||
|
||||
Not fixed by new indexes:
|
||||
|
||||
- `ntf_service_queues_count` seq-scans only because of index bloat; `idx_msg_queues_ntf_service_id`
|
||||
already covers it. `REINDEX` restores the index-only scan, as proven by `rcv_service_queues_count`.
|
||||
- `rcv_service_queues_count` and the two `services` counts are already cheap.
|
||||
|
||||
## Implemented fixes (batch 2)
|
||||
|
||||
Migration `20260917_msg_queues_hot` and `MsgStore/Postgres.hs`.
|
||||
|
||||
| change | effect |
|
||||
| --- | --- |
|
||||
| drop `idx_msg_queues_updated_at_recipient_id`, set `msg_queues` `fillfactor = 80` | `updateQueueTime` updates become HOT, so they rewrite no index entries; removes its index bloat and cuts its WAL |
|
||||
| remove the dead `updated_at > p_old_queue` filter and its parameter from `expire_old_messages`, matching the `CALL` (`MsgStore/Postgres.hs:113`) | the sweep query runs as an index-only scan on `idx_msg_queues_expire`; `updated_at` is no longer read on the hot path |
|
||||
| autovacuum reloptions on `msg_queues`, `messages` (and its TOAST), and `services` | keeps dead tuples and bloat low per table (see the table below) |
|
||||
|
||||
Verified on PostgreSQL 16 with the daily-active pattern (~11% of queues updated per day): `updateQueueTime`
|
||||
goes from 0% HOT (indexed `updated_at`) to 100% HOT at `fillfactor = 80` (96.7% at 90), and per-update
|
||||
WAL drops about 4x. The HOT ratio depends on how many rows per page change before autovacuum reclaims
|
||||
space, so it is workload-dependent; `fillfactor = 80` reached 100% for this fraction, and updating a much
|
||||
larger share of queues at once would need a lower value. `fillfactor` applies to pages rewritten after
|
||||
the change, so the ratio ramps up as the heap turns over, or immediately after `pg_repack`.
|
||||
|
||||
The Postgres message store does not read `updated_at`, so no query depends on the dropped index.
|
||||
|
||||
Autovacuum reloptions applied by the migration:
|
||||
|
||||
| table | reloptions | reason |
|
||||
| --- | --- | --- |
|
||||
| `msg_queues` | `fillfactor 80`, `autovacuum_vacuum_scale_factor 0.02`, `autovacuum_analyze_scale_factor 0.01`, `autovacuum_vacuum_cost_limit 1000` | HOT headroom; vacuum/analyze at ~2%/1% churn instead of 20%/10%; faster vacuum |
|
||||
| `messages` (and TOAST) | `autovacuum_vacuum_scale_factor 0.02`, `autovacuum_analyze_scale_factor 0.01`, `toast.autovacuum_vacuum_scale_factor 0.02` | high insert and delete churn; 152 GB, mostly TOASTed message bodies |
|
||||
| `services` | `fillfactor 70`, `autovacuum_vacuum_threshold 1000`, `autovacuum_vacuum_scale_factor 0` | tiny, very hot table was autovacuumed tens of thousands of times; make updates HOT and vacuum it far less |
|
||||
|
||||
## Operator actions
|
||||
|
||||
1. Reclaim index bloat (online, no lock; needs free disk near the index size; run largest first):
|
||||
```
|
||||
REINDEX INDEX CONCURRENTLY smp_server.idx_msg_queues_updated_at_recipient_id;
|
||||
REINDEX INDEX CONCURRENTLY smp_server.idx_msg_queues_notifier_id;
|
||||
REINDEX INDEX CONCURRENTLY smp_server.idx_msg_queues_ntf_service_id;
|
||||
REINDEX INDEX CONCURRENTLY smp_server.idx_msg_queues_rcv_service_id;
|
||||
REINDEX INDEX CONCURRENTLY smp_server.idx_msg_queues_sender_id;
|
||||
REINDEX INDEX CONCURRENTLY smp_server.idx_msg_queues_link_id;
|
||||
ANALYZE smp_server.msg_queues;
|
||||
```
|
||||
Expect `pg_indexes_size('smp_server.msg_queues')` to drop from 104 GB to roughly 12 GB (measured
|
||||
after reindexing). `ANALYZE` restores the index-only scan for `ntf_service_queues_count`. With the
|
||||
batch-2 `fillfactor` the index bloat no longer rebuilds, so this is a one-time reclaim.
|
||||
|
||||
2. Autovacuum reloptions are applied by the batch-2 migration (`msg_queues`, `messages` and its TOAST,
|
||||
`services`), so no manual `ALTER TABLE` is needed. Reclaim the current `messages` bloat once with
|
||||
`pg_repack smp_server.messages` (152 GB heap and TOAST, roughly 30-50 GB reclaimable); running
|
||||
`pg_repack` on `msg_queues` also makes `fillfactor = 80` effective immediately rather than as pages
|
||||
turn over. With `fillfactor = 80` the `msg_queues` index bloat no longer rebuilds, so scheduled
|
||||
reindexing is rarely needed. To automate it anyway, run `reindexdb --concurrently` or `pg_repack`
|
||||
from a scheduler; `pg_cron` runs each job inside a transaction and cannot execute
|
||||
`REINDEX ... CONCURRENTLY`, so use `pg_cron` only for `VACUUM`/`ANALYZE` or a non-concurrent
|
||||
maintenance-window `REINDEX`.
|
||||
|
||||
3. Cut WAL and checkpoint pressure (all reloadable, no restart):
|
||||
```
|
||||
ALTER SYSTEM SET wal_compression = 'lz4';
|
||||
ALTER SYSTEM SET max_wal_size = '32GB';
|
||||
ALTER SYSTEM SET checkpoint_timeout = '30min';
|
||||
SELECT pg_reload_conf();
|
||||
```
|
||||
Fewer checkpoints produce fewer full-page images; `wal_compression` shrinks those that remain.
|
||||
Together these reduce the WAL written on the primary and shipped by `archive-push`. The message
|
||||
insert and delete volume itself is fixed by traffic, but most of its WAL cost is amplification
|
||||
(full-page images and hint-bit writes on bloated pages), which these settings and `REINDEX` reduce.
|
||||
This is separate from pgBackRest repo-side compression, which does not reduce WAL on the primary.
|
||||
Verify with `pg_stat_reset_shared('wal')`, wait, then re-check `wal_fpi` and `wal_bytes` in
|
||||
`pg_stat_wal`.
|
||||
|
||||
4. Enable `track_io_timing = on` so disk wait time is measurable in `pg_stat_statements`.
|
||||
@@ -0,0 +1,186 @@
|
||||
# SMP server: memory leaks and CPU/RAM exhaustion paths
|
||||
|
||||
Date: 2026-09-08
|
||||
Scope: server-side code at HEAD 200129af. Defensive review for the operator. All file:line references verified against the current tree.
|
||||
|
||||
## Summary
|
||||
|
||||
Scope is the PostgreSQL backend only (`store_queues: database`, `store_messages: database`), which selects `SSCDatabase` -> `PostgresMsgStore`. In that backend the queue store is created with `useCache = False` (`MsgStore/Postgres.hs:100`) and `getMsgQueue` is a no-op (`:172`), so there is no in-memory queue cache and no journal message files.
|
||||
|
||||
Consequence: the queue-cache and journal issues below do not occur on this backend. Specifically, memory-leaks 1 and 2 (queue cache) require `useCache = True`, which only the journal-with-Postgres-queues mode (`store_messages: journal`) uses; and the SUBS journal-FD spike requires journal files. On PostgreSQL, SUBS is a single streaming DB query (`MsgStore/Postgres.hs:122`), with no FD/RAM spike.
|
||||
|
||||
Real CPU/RAM issues on the PostgreSQL backend (all store-independent): unauthenticated RSLV outbound fan-out, the proxy session never torn down on forward timeout, the absence of any global connection/thread cap, and the service `subscriptions` map not pruned on ACK. Reproduction tests: `testRslvFanOut` and `testProxyForwardTimeoutStuckSession`. No fixes are applied. Verified against a local PostgreSQL (`server_postgres` build).
|
||||
|
||||
## Memory leaks
|
||||
|
||||
### 1. Queue cache retains deleted queues (not the PostgreSQL backend)
|
||||
|
||||
Only with `useCache = True` (journal-with-Postgres-queues mode). The pure PostgreSQL backend uses `useCache = False`, so the `queues` cache is never populated and nothing leaks. Documented for completeness.
|
||||
|
||||
`deleteStoreQueue` nulls the queue's `queueRec` TVar and cleans the `senders`, service, and notifier maps, but never removes the entry from the `queues` map.
|
||||
|
||||
- STM: `QueueStore/STM.hs` (no `TM.delete rId queues`).
|
||||
- Postgres: `QueueStore/Postgres.hs:451-464` (sets `deleted_at`, nulls `queueRec`, never removed from the cache; `compactQueues` at `:148-152` deletes DB rows only).
|
||||
- Add sites: STM `QueueStore/STM.hs:126`; Postgres `:182` and `:245` (any load caches).
|
||||
|
||||
Effect: each create+delete cycle leaves a permanent `STMQueue`/`JournalQueue` object with its TVars. A client looping NEW then DEL grows RAM without bound; freed only by restart. `getEntityCounts` counts `M.size queues` (`QueueStore/STM.hs:101`), so the reported `queueCount` also drifts up. Reproduced on STM, Journal, and Postgres+journal: 200 create+delete cycles retain 200 cache entries.
|
||||
|
||||
Fix direction: drop the entry from the cache on delete in both stores. Safe because `loadQueue` filters `AND deleted_at IS NULL` (`QueueStore/Postgres.hs:231,286,309`), so a deleted queue is never reloaded.
|
||||
|
||||
```haskell
|
||||
delete q = do
|
||||
writeTVar qr Nothing
|
||||
TM.delete rId $ queues st -- add: drop deleted queue from the cache
|
||||
TM.delete senderId $ senders st
|
||||
...
|
||||
```
|
||||
|
||||
### 2. Postgres queue cache has no eviction (not the pure PostgreSQL backend)
|
||||
|
||||
Only with `useCache = True` (journal-with-Postgres-queues mode). The pure PostgreSQL backend (`store_messages: database`) uses `useCache = False`, so there is no cache to grow. Documented for completeness.
|
||||
|
||||
Every distinct queue touched by SUB/SEND/NSUB/GET is cached in `queues` via `cacheQueue` (`QueueStore/Postgres.hs:245`) and removed only on delete. There is no idle or size-based eviction, so the working set trends toward every queue accessed since boot, defeating the purpose of the database backing store. `idleQueueInterval` (`Env/STM.hs:233`, 4h) closes journal message-queue handles, not this cache. Requires the Postgres backend.
|
||||
|
||||
### 3. Service subscriptions map not pruned on ACK
|
||||
|
||||
A receiving-service client's `subscriptions` map gains a `Sub` per delivered queue at `Server.hs:1896` (initial scan) and `Server.hs:2073` (`newServiceDeliverySub` on each SEND to an associated queue). `acknowledgeMsg` (`Server.hs:1929-1951`) clears delivery state but never deletes the map entry. Entries are removed only on queue delete, unassociation, or disconnect. A long-lived service session grows toward the number of associated queues regardless of ACKs.
|
||||
|
||||
### 4. Notification store grows without a subscriber (not fixed)
|
||||
|
||||
`storeNtf` prepends to a per-notifier list with no bound (`NtfStore.hs:37-42`). The list is drained only by an active notifier subscription or the notification TTL (default 24h, `Env/STM.hs:238-243`). With no notifier subscribed, one entry accumulates per notification-flagged SEND. `deleteExpiredNtfs` also leaves the emptied notifier key in the outer map (`NtfStore.hs:57-61`).
|
||||
|
||||
Not fixed here: the per-notifier list is TTL-bounded, and the empty-key retention is bounded by the number of active notifiers (freed on queue/notifier delete), so it is not truly unbounded. Removing the emptied key in `deleteExpiredNtfs` races with `storeNtf`, which reads the outer map outside a transaction; a safe fix requires making `storeNtf` fully transactional, which changes a hot path. Deferred as not worth that trade.
|
||||
|
||||
### 5. Proxy client maps grow with distinct PRXY targets
|
||||
|
||||
`smpClients` (keyed by `SMPServer`) and `smpSessions` grow per distinct proxied relay. On connect failure with `persistErrorInterval /= 0` (default 30s), the error var is kept in `smpClients` (`Client/Agent.hs:245-247`) and removed only lazily on re-query. Client-controlled `PRXY srv` targets make the target set attacker-chosen.
|
||||
|
||||
### 6. Blocked writer threads on a stuck relay
|
||||
|
||||
When a relay stops draining, the proxy client's `sndQ` fills and each further command forks a thread parked in `writeTBQueue` (`Client.hs:1367-1370`), each retaining a `Request`. This is the memory amplifier of the proxy stuck-session issue.
|
||||
|
||||
### 7. sentCommands entries leak for unanswered commands
|
||||
|
||||
`getResponse` sets `pending = False` on timeout but does not delete the `sentCommands` entry (`Client.hs:1372-1383`); removal happens only on a matching response (`Client.hs:704`). Commands a relay never answers stay for the connection lifetime.
|
||||
|
||||
## CPU/RAM exhaustion paths
|
||||
|
||||
Ranked by ease of trigger times impact.
|
||||
|
||||
1. No global connection cap plus a 120s handshake hold. Sockets are accepted and handlers forked with no admission control (`Transport/Server.hs:176-184`); the SMP handshake may hang for `smpHandshakeTimeout` (`Main.hs:535`, 120s). Unauthenticated thread, FD, and buffer exhaustion.
|
||||
2. `PRXY` establishes an outbound TLS connection to an arbitrary client-named host and port (`Server.hs:1417-1429`), which is server-side request forgery plus unbounded cached outbound sessions. Gated by `allowSMPProxy` and optional `newQueueBasicAuth`; open on public relays.
|
||||
3. `NEW` creates a persistent queue with keys, a store-log record, and disk state (`Server.hs:1569-1628`), with no maximum queue count. On public or no-password servers this is unauthenticated and compounds leak 1.
|
||||
4. `RSLV` forks up to `serverResolverConcurrency` (1000) outbound HTTP requests per connection through a per-client counter with no global cap (`Server.hs:1279,1522`, `Env/STM.hs:256`). Only when `[NAMES]` is enabled. No result cache (`Names.hs:61-74`).
|
||||
5. `SUBS` on a recipient service folds over every queue of the service and reruns on every reconnect since `hasSub` is false on a fresh connection. On the PostgreSQL backend this is one streaming DB query joining each service queue with its head message (`MsgStore/Postgres.hs:122`): O(N) DB work, but no journal opens and no FD/RAM spike (the journal-open spike, `Journal.hs:449-467`, is journal-mode only). The store-independent cost that remains is the service `subscriptions` map growing (leak 3).
|
||||
6. `RFWD` runs one X25519 DH per request inline on the connection thread before validating the inner ciphertext (`Server.hs:2129`), unauthenticated (`Server.hs:1278`).
|
||||
7. Batch multiplier: one 16 KB block carries up to 255 transmissions (`Protocol.hs:2292`), multiplying every per-command cost. Unknown-queue transmissions still run a full signature verify via the dummy-verify path (`Server.hs:1345-1350`).
|
||||
|
||||
Interaction: `RSLV` (limit 1000) and `PRXY`/`PFWD` (limit 32) share one `procThreads` counter (`Server.hs:1476-1489`), so an RSLV flood also starves proxying on the same connection.
|
||||
|
||||
## Config-gated exposure to confirm
|
||||
|
||||
- `newQueueBasicAuth`: gates NEW and PRXY. Absent on public servers by design.
|
||||
- `allowSMPProxy`: enables PRXY (hardcoded true in the default config path, `Main.hs:610`).
|
||||
- `[NAMES] enable`: gates RSLV.
|
||||
|
||||
## Reproduction tests
|
||||
|
||||
Two store-independent issues have sound reproductions on the PostgreSQL backend. Both are pending (`xit`) specs asserting the correct behavior, each verified to fail (reproduce) when activated (`it`).
|
||||
|
||||
- `tests/RSLVTests.hs`
|
||||
- `testRslvFanOut`: one connection must not fan out to many concurrent resolver requests. Verified: 64 concurrent from one connection.
|
||||
- `tests/SMPProxyTests.hs`
|
||||
- `testProxyForwardTimeoutStuckSession`: after repeated forward timeouts the session must be dropped, so a later forward returns `NO_SESSION`. Verified: all 10 forwards return `BROKER TIMEOUT` (session never torn down; the operator-reported `pMsgFwdsOwn_pErrorsOther`).
|
||||
|
||||
Running the Postgres tests requires a local PostgreSQL with roles `test_server_user` and `postgres` and database `test_server_db`, reachable via the default Unix socket (set `PGHOST`) and `localhost:5432`, plus a `-fserver_postgres` build.
|
||||
|
||||
## Sanity of the reproductions
|
||||
|
||||
Verified by tracing the exercised code path.
|
||||
|
||||
- `testRslvFanOut`: sound, store-independent, timing-based. Full E2E: real RSLV commands to a real names-enabled server; a 3s resolver delay vs a 0.8s sample separates concurrent from sequential; 64 outbound requests are in flight from one connection. `managerConnCount = 10` does not serialize them (64 confirmed). Couples to a resolver-limit the fix must add and the test set.
|
||||
- `testProxyForwardTimeoutStuckSession`: sound, store-independent. Real PFWD path; 10 forwards on one session all return `BROKER TIMEOUT`, none `NO_SESSION`, so the session is never torn down. Asserts the desired behavior, so it fails now and passes once a teardown/circuit-breaker fix lands.
|
||||
|
||||
Issues without a sound reproduction on the PostgreSQL backend:
|
||||
|
||||
- Queue cache leak / no eviction (memory-leaks 1, 2): do not occur, `useCache = False`. The earlier `testDeleteQueueNoLeak`/`testPostgresCacheBounded` only reproduced in journal/cache modes and were removed.
|
||||
- SUBS journal-FD spike: journal-mode only; `getMsgQueue` is a no-op on PostgreSQL. The earlier `testSubsServiceScanBounded` was removed.
|
||||
- Global admission control: not testable black-box. `connect()` succeeds regardless of an app-level cap (kernel backlog), and an admission fix accepts-then-closes rather than refusing the connect, so a black-box client cannot distinguish fixed from unfixed. Needs a configurable connection cap the test can set or a server-exposed active-connection metric.
|
||||
- Service `subscriptions` map not pruned on ACK (memory-leaks 3): real and store-independent, but not reproduced here; needs a messaging-service (service-cert) client that SUBS then receives and ACKs, asserting the map does not grow.
|
||||
|
||||
Note on issue 6: task 6 is "bound proxy writer threads to a stuck relay" (`nonBlockingWriteTBQueue` forking when the relay stops reading). The shipped test does not reproduce that; it reproduces the related, higher-impact and operator-observed "session not torn down on forward timeout". The writer-thread fork bomb needs a relay that accepts TCP but stops reading, and `nonBlockingWriteTBQueue` is not exported, so it is not reproduced here.
|
||||
|
||||
## Fix directions (general)
|
||||
|
||||
Sketches only, one per high-impact issue. Not applied.
|
||||
|
||||
Issue 1, drop deleted queues from the cache (both stores):
|
||||
|
||||
```haskell
|
||||
delete q = do
|
||||
writeTVar qr Nothing
|
||||
TM.delete rId $ queues st -- add
|
||||
...
|
||||
```
|
||||
|
||||
Issue 2, evict the Postgres queue cache working set (idle or bounded LRU):
|
||||
|
||||
```haskell
|
||||
-- periodic sweep, or on insert past a size bound:
|
||||
evictIdleQueues st now = atomically $
|
||||
modifyTVar' (queues st) (M.filter (not . idleSince now)) -- keep only recently used
|
||||
```
|
||||
|
||||
Issue 3, bound the SUBS scan: close each queue after peek and paginate; do not rerun the full scan on every reconnect:
|
||||
|
||||
```haskell
|
||||
f' a (q, qr) =
|
||||
(runExceptT (tryPeekMsg ms q) >>= f a (recipientId q) . ...) `finally` closeMsgQueue ms q
|
||||
```
|
||||
|
||||
Issue 4, global RSLV concurrency cap (semaphore in NamesEnv, acquired around the outbound call):
|
||||
|
||||
```haskell
|
||||
data NamesEnv = NamesEnv { ..., resolverSem :: TVar Int, resolverLimit :: Int }
|
||||
|
||||
withResolverSlot NamesEnv {resolverSem, resolverLimit} =
|
||||
bracket_ (atomically $ readTVar resolverSem >>= \n -> if n >= resolverLimit then retry else writeTVar resolverSem (n + 1))
|
||||
(atomically $ modifyTVar' resolverSem (subtract 1))
|
||||
```
|
||||
|
||||
Issue 5, global connection admission at accept (refuse beyond a cap):
|
||||
|
||||
```haskell
|
||||
-- in the accept loop: gate on a shared counter before forking the handler
|
||||
n <- atomically $ stateTVar activeConns (\n -> (n, n + 1))
|
||||
if n >= maxConns then close sock else forkHandler sock `finally` atomically (modifyTVar' activeConns (subtract 1))
|
||||
```
|
||||
|
||||
Issue 6, bound proxy writers and add a forward-timeout circuit breaker:
|
||||
|
||||
```haskell
|
||||
-- fail fast instead of forking an unbounded blocked writer:
|
||||
nonBlockingWriteTBQueue q x = atomically (tryWriteTBQueue q x)
|
||||
>>= (`unless` throwIO PCEResponseTimeout) -- or count and drop
|
||||
-- and drop the relay session after N consecutive forward timeouts (independent of received bytes)
|
||||
```
|
||||
|
||||
Also: prune the service `subscriptions` entry on ACK (leak 3, `acknowledgeMsg`); the ntf empty-key (leak 4) needs a transactional `storeNtf` first.
|
||||
|
||||
## Additional findings (verified 2026-09-09)
|
||||
|
||||
Found by a subagent sweep of the PostgreSQL backend and store-independent hot paths, then each confirmed by reading the cited code. Not yet fixed or tested (except where noted).
|
||||
|
||||
Attacker-triggerable (DoS):
|
||||
|
||||
1. Batched crypto + per-transmission DB SELECT amplification (HIGH). `dummyVerifyCmd` runs a real Ed25519/Ed448 verify or attacker-chosen X25519 DH per transmission on an unknown queue (`Server.hs:1345-1350`). SEND is a non-batch party (`batchParty` = Recipient/Notifier only, `Protocol.hs:420-423`), so it uses the per-transmission path `mapM (verifyTransmission ...)` (`Server.hs:1167`): one crypto op plus one `getQueueRec` DB SELECT (no cache) per transmission. A 16 KB block holds ~130-250 transmissions (`Protocol.hs:2292`), so one write by any handshake-completing peer causes ~130-250 asymmetric verifications and ~130-250 individual SELECTs. Testable.
|
||||
2. `clientService` handshake grows the `services` table without bound (MED-HIGH). `getClientService` accepts a self-signed chain (`CCSelf`, `Transport.hs:755-768`) and verifies an attacker cert per connection; `getCreateService` inserts a `services` row on any new cert fingerprint (`QueueStore/Postgres.hs:469-482`). A fresh self-signed cert per handshake yields an unbounded persistent row per connection, pre-auth. Testable (count rows across N handshakes).
|
||||
3. Unbounded delivery threads to a slow consumer (MED). When a subscribed recipient's `sndQ` is full, `tryDeliverMessage` forks a bare `forkIO` parked on `writeTBQueue` (`Server.hs:2068-2072,2085-2105`), not gated by `procThreads`. One blocked thread per queue for a slow consumer subscribed to many queues.
|
||||
|
||||
Operational / scale:
|
||||
|
||||
4. DB pool is the throughput ceiling (HIGH for scale). Default `poolSize=10` (`Main/Init.hs:44`), one MVar-gated pool shared by all clients. Each SEND is 3 transactions: `getQueueRec` + `deleteExpiredMsgs` + `writeMsg`, and `expireMessagesOnSend=True` by default (`Main.hs:563`, `Server.hs:2004`) forces the middle one on every SEND to a non-empty queue. A second pool (`dbPriorityPool`, another `poolSize` backends) is opened but never used by the SMP server (no `withTransactionPriority _ True` in server code) -> 2xpoolSize backends, half idle.
|
||||
5. Prometheus scrape cost (MED-HIGH, if enabled). Every `prometheus_interval` (default 60s), `getDeliveredMetrics` folds over all clients x all subscriptions in memory (`Server.hs:723-734`) and `getEntityCounts` runs 6 `COUNT(1)` scans over `msg_queues`/`services` (`QueueStore/Postgres.hs:154-170`, `Server.hs:699`).
|
||||
6. Subscription serialization bottleneck (MED). All subscription churn funnels through one `subQ` + one `serverThread` mutating the single `queueSubscribers` Map-in-a-TVar (`Server.hs:263-360`, `Env/STM.hs:375-391`); a batched SUB of N queues is N separate STM writes to one contended TVar.
|
||||
|
||||
Lower / noted (chain verified, not deep-dived): `expire_old_messages` does a full `COUNT(1) FROM messages` every 2h; legacy non-service `deliverNtfsThread` is O(ntf-clients x pending) every 1.5s; `PeriodStats` week/month `IntSet`s grow with distinct active queues; no server-side prepared statements; full-row queue decode per command.
|
||||
@@ -85,7 +85,7 @@ SMP agent protocol has 2 main parts:
|
||||
|
||||
|
||||
|
||||
The procedure of establishing a duplex connection is explained on the example of Alice (the initiating party) and Bob (the joining party) creating a bi-directional connection consisting of two unidirectional (simplex) queues, using SMP agents (A and B) to facilitate it, and two different SMP routers (which could be the same router). It is shown on the diagram above and has these steps:
|
||||
The procedure of establishing a duplex connection is explained on the example of Alice and Bob creating a bi-directional connection consisting of two unidirectional (simplex) queues, using SMP agents (A and B) to facilitate it, and two different SMP routers (which could be the same router). It is shown on the diagram above and has these steps:
|
||||
|
||||
1. Alice requests the new connection from the SMP agent A using agent `createConnection` api function.
|
||||
2. Agent A creates an SMP queue on the router (using [SMP protocol](./simplex-messaging.md) `NEW` command) and responds to Alice with the invitation that contains queue information and the encryption keys Bob's agent B should use. The invitation format is described in [Connection link](connection-link-1-time-invitation-and-contact-address).
|
||||
@@ -208,7 +208,7 @@ This syntax of decrypted SMP client message body is defined by `decryptedAgentMe
|
||||
Decrypted SMP message client body can be one of 4 types:
|
||||
- `agentConnInfo` - used by the initiating party when confirming reply queue - sent in `agentConfirmation` envelope.
|
||||
- `agentConnInfoReply` - used by accepting party, includes reply queue(s) in the initial confirmation - sent in `agentConfirmation` envelope.
|
||||
- `agentRatchetInfo` - used to pass additional information when renegotiating double ratchet encryption - sent in `agentRatchetKey` envelope. A key sent in reply to another key includes the hash of that key; agents do not reply to such keys.
|
||||
- `agentRatchetInfo` - used to pass additional information when renegotiating double ratchet encryption - sent in `agentRatchetKey` envelope.
|
||||
- `agentMessage` - all other agent messages.
|
||||
|
||||
`agentMessage` contains these parts:
|
||||
@@ -233,8 +233,7 @@ connInfo = *OCTET
|
||||
agentConnInfoReply = %s"D" smpQueues connInfo
|
||||
smpQueues = length 1*newQueueInfo ; NonEmpty list of reply queues
|
||||
agentRatchetInfo = %s"R" ratchetInfo
|
||||
ratchetInfo = [answeredKeyHash *OCTET] ; bytes after answeredKeyHash are ignored
|
||||
answeredKeyHash = %s"0" / (%s"1" shortString) ; "0" in a key that starts renegotiation, otherwise SHA-256 of the two raw public keys of the answered key
|
||||
ratchetInfo = *OCTET
|
||||
|
||||
agentMessage = %s"M" agentMsgHeader aMessage
|
||||
agentMsgHeader = agentMsgId prevMsgHash
|
||||
|
||||
@@ -1,189 +0,0 @@
|
||||
Revision 1, 2026-10-01
|
||||
|
||||
# SimpleX Network: cryptographic primitive registry
|
||||
|
||||
The cryptographic primitives, constructions and domain-separation strings used by this repository, and which of them new code may use. For the threat model see [Security](./security.md).
|
||||
|
||||
Module aliases follow the codebase: `C` = [Simplex.Messaging.Crypto](../src/Simplex/Messaging/Crypto.hs), `LC` = [Crypto.Lazy](../src/Simplex/Messaging/Crypto/Lazy.hs), `CR` = [Crypto.Ratchet](../src/Simplex/Messaging/Crypto/Ratchet.hs), `SL` = [Crypto.ShortLink](../src/Simplex/Messaging/Crypto/ShortLink.hs), `KEM` = [Crypto.SNTRUP761](../src/Simplex/Messaging/Crypto/SNTRUP761.hs) and [its bindings](../src/Simplex/Messaging/Crypto/SNTRUP761/Bindings.hs), `BBS` = [Crypto.BBS](../src/Simplex/Messaging/Crypto/BBS.hs). Lengths are in bytes.
|
||||
|
||||
## Table of contents
|
||||
|
||||
- [Policy](#policy)
|
||||
- [Defaults for new code](#defaults-for-new-code)
|
||||
- [Primitives](#primitives)
|
||||
- [SimpleX box constructions](#simplex-box-constructions)
|
||||
- [Domain separation and KDF inputs](#domain-separation-and-kdf-inputs)
|
||||
- [Nonces and IVs](#nonces-and-ivs)
|
||||
- [Implementation sources](#implementation-sources)
|
||||
- [xftp-web (TypeScript)](#xftp-web-typescript)
|
||||
- [Tests](#tests)
|
||||
|
||||
## Policy
|
||||
|
||||
- New code uses the default primitive for its purpose from [Defaults for new code](#defaults-for-new-code).
|
||||
- Using any other primitive, or a primitive marked *restricted* outside its listed purpose, requires a written justification in the pull request and review by a maintainer responsible for cryptography.
|
||||
- *Legacy-only* primitives stay only for the existing wire formats and stored data listed below. Do not add call sites.
|
||||
- Every new HKDF info string or hash prefix must be unique, start with `SimpleX`, and be added to [Domain separation and KDF inputs](#domain-separation-and-kdf-inputs).
|
||||
- Changing a primitive, key length, KDF input, info string or nonce construction changes the wire format. It requires a protocol version and an update of this registry and of the affected `protocol/` specification.
|
||||
|
||||
Status values used below:
|
||||
|
||||
| Status | Meaning |
|
||||
|---|---|
|
||||
| Default | Approved and preferred for new code for this purpose |
|
||||
| Approved | Approved for new code for this purpose when the default does not fit |
|
||||
| Restricted | Approved only for the listed purpose (external interoperability or a single protocol) |
|
||||
| Legacy-only | Kept for existing wire formats or stored data; no new call sites |
|
||||
|
||||
## Defaults for new code
|
||||
|
||||
| Purpose | Default | Functions |
|
||||
|---|---|---|
|
||||
| Signature | Ed25519 | `C.sign'`, `C.verify'` with `C.SEd25519` |
|
||||
| Key agreement | X25519 | `C.generateKeyPair`, `C.dh'` |
|
||||
| Post-quantum KEM | sntrup761 | `KEM.sntrup761Keypair`, `sntrup761Enc`, `sntrup761Dec` |
|
||||
| Hybrid DH + KEM secret | HKDF-SHA512 over `dh \|\| kemSecret` with a new info string, as in `CR.rootKdf` | `C.hkdf` |
|
||||
| Authenticated encryption | XSalsa20-Poly1305 [SimpleX secretbox](#simplex-box-constructions) | `C.sbEncrypt`/`C.sbDecrypt`, `LC.sbEncryptTailTag` for streams |
|
||||
| Authenticated encryption with a DH secret | XSalsa20-Poly1305 [SimpleX crypto_box](#simplex-box-constructions) | `C.cbEncrypt`/`C.cbDecrypt` |
|
||||
| Forward-secret symmetric session | HKDF-SHA512 key chain | `C.sbcInit`, `C.sbcHkdf` |
|
||||
| KDF | HKDF-SHA512 | `C.hkdf` |
|
||||
| Hash commitment, derived identifier | SHA3-256 | `C.sha3_256` |
|
||||
| Certificate fingerprint | SHA-256 X.509 fingerprint | `C.signedFingerprint`, `C.certificateFingerprint` |
|
||||
| Randomness | ChaChaDRG seeded from system entropy | `C.newRandom`, `C.randomBytes` and the `random*` helpers |
|
||||
|
||||
## Primitives
|
||||
|
||||
| Primitive | Status | Purpose and protocol | Implementation | Lengths |
|
||||
|---|---|---|---|---|
|
||||
| Ed25519 | Default | SMP/NTF/XFTP queue and file keys (agent default `rcvAuthAlg`, `sndAuthAlg` in [Agent/Env/SQLite.hs](../src/Simplex/Messaging/Agent/Env/SQLite.hs)), short-link signatures and owner auth (`SL.encodeSign`, `SL.newOwnerAuth`), XRCP identity and session keys ([RemoteControl/Invitation.hs](../src/Simplex/RemoteControl/Invitation.hs)), agent service request signatures, client/service TLS certificates ([Transport/Credentials.hs](../src/Simplex/Messaging/Transport/Credentials.hs)) | crypton `Crypto.PubKey.Ed25519` via `C.sign'`/`C.verify'` | pub 32, priv 32, sig 64; X.509 SPKI 44 |
|
||||
| Ed448 | Approved | Server CA and online certificates (default `signAlgorithm = ED448` in [Server/CLI.hs](../src/Simplex/Messaging/Server/CLI.hs)); accepted for SMP command signatures and TLS | crypton `Crypto.PubKey.Ed448`; server key generation by the `openssl genpkey` CLI | pub 57, priv 57, sig 114; SPKI 69 |
|
||||
| X25519 | Default | Per-queue e2e and server-to-recipient `crypto_box` keys, SMP session DH, command authenticator (SMP v7+), proxy (PRXY/PFWD), notification tokens, XFTP chunk transport, XRCP hello and announcements | crypton `Crypto.PubKey.Curve25519` via `C.dh'` | pub 32, priv 32, secret 32; SPKI 44 |
|
||||
| X448 | Restricted | E2E double ratchet only: X3DH and DH ratchet (`CR.RatchetX448`, E2E v3) | crypton `Crypto.PubKey.Curve448` | pub 56, priv 56, secret 56 |
|
||||
| sntrup761 | Default | PQ KEM in double ratchet (E2E v3, agent v5+) and XRCP v1 hello | Vendored C [cbits/sntrup761.c](../cbits/sntrup761.c) via FFI; randomness from ChaChaDRG through `haskell_rng_func` ([Bindings/RNG.hs](../src/Simplex/Messaging/Crypto/SNTRUP761/Bindings/RNG.hs)) | pk 1158, sk 1763, ct 1039, shared 32 |
|
||||
| XSalsa20-Poly1305, SimpleX crypto_box | Default | SMP message bodies (server to recipient), per-queue e2e envelope, confirmations, proxy forwarding, notifications, XRCP hello, XFTP chunk transport | crypton `Crypto.Cipher.XSalsa`, `Crypto.MAC.Poly1305`; `C.cryptoBox`, `LC.cbInit` | key 32 (DH secret), nonce 24, tag 16 |
|
||||
| XSalsa20-Poly1305, SimpleX secretbox | Default | SMP transport block encryption (SMP v11+), short-link data (SMP v15+), XFTP file encryption, local file encryption ([Crypto/File.hs](../src/Simplex/Messaging/Crypto/File.hs)), XRCP session | same; `C.sbEncrypt`, `LC.sbEncryptTailTag`, `LC.sbInit` | key 32, nonce 24, tag 16 |
|
||||
| crypto_box authenticator | Approved | Deniable sender command authorization with X25519 queue keys (SMP v7+); agent currently creates Ed25519 keys | `C.cbAuthenticate`, `C.cbVerify`: `crypto_box(sha512(msg))` | 80 = 64 + 16 |
|
||||
| AES-256-GCM, 16-byte IV | Legacy-only | Double ratchet header and body (E2E v3, [pqdr.md](./pqdr.md)) | crypton `Crypto.Cipher.AES`, `Crypto.Cipher.Types`; `C.encryptAEAD`, `C.decryptAEAD`, `C.initAEAD`; J0 = GHASH(IV) as NIST SP 800-38D defines for non-96-bit IVs | key 32, IV 16, tag 16; padded header 88 (PQ off) or 2310 (PQ on), `CR.paddedHeaderLen` |
|
||||
| AES-256-GCM, 12-byte IV | Restricted | WebRTC frame encryption in simplex-chat (`Simplex.Chat.Mobile.WebRTC`), for WebCrypto interoperability | `C.encryptAESNoPad`, `C.decryptAESNoPad`, `C.initAEADGCM`, `C.GCMIV` | key 32, IV 12, tag 16 |
|
||||
| HKDF-SHA512 | Default | All KDFs listed in [Domain separation](#domain-separation-and-kdf-inputs) | crypton `Crypto.KDF.HKDF` with `SHA512`; `C.hkdf` (extract + expand) | output per use, at most 255 * 64 |
|
||||
| SHA-256 | Default for X.509 fingerprints, otherwise Restricted | X.509 fingerprints (`C.KeyHash`, server identity, XRCP CA, service certificate hash, SMP v16+), XFTP chunk digests, agent message hashes, `requestCode`, ratchet key dedup hash, security code `codeAD = sha256(rcAD)` | crypton; `C.sha256Hash`, `LC.sha256Hash`, `getFingerprint ... HashSHA256` | 32 |
|
||||
| SHA-512 | Restricted | XFTP file digests and file description hash, input of the crypto_box authenticator | crypton; `C.sha512Hash`, `LC.sha512Hash` | 64 |
|
||||
| SHA-512 (inside sntrup761) | Restricted | sntrup761 internal hash | [cbits/sha512.c](../cbits/sha512.c) calls OpenSSL `SHA512()` from the system `libcrypto` (`extra-libraries: crypto`) | 64 |
|
||||
| SHA3-256 | Default | Short-link key `sha3_256(fixedData)` (SMP v15+), XRCP hybrid secret, service request binding (agent v8) | crypton; `C.sha3_256`, `KEM.kemHybridSecret` | 32 |
|
||||
| SHA3-384 | Restricted | Client-supplied sender ID: first 24 bytes of `sha3_384(corrId)`, computed by the agent (`prepareConnectionLink'`, `newRcvConnSrv`) and the server (`createQueue`) | crypton; `C.sha3_384` | 48, truncated to 24 |
|
||||
| Keccak-256 | Restricted | Label hash for SMP name queries (`NameQuery NQHash`), must match the on-chain registry | crypton `Keccak_256`; `labelHash` in [SimplexName.hs](../src/Simplex/Messaging/SimplexName.hs) | 32 |
|
||||
| MD5 | Legacy-only, non-security | XOR-aggregated queue ID hash `IdsHash` for service subscription reconciliation (SUBS, NSUBS, SOKS, ENDS) | crypton (`C.md5Hash`, `Protocol.queueIdHash`); SQLite UDF `simplex_xor_md5_combine` ([Agent/Store/SQLite.hs](../src/Simplex/Messaging/Agent/Store/SQLite.hs)); PostgreSQL pgcrypto `digest(..., 'md5')` in server and NTF schemas | 16 |
|
||||
| ChaChaDRG | Default | Keys, nonces, correlation IDs, random IDs, sntrup761 randomness | crypton `Crypto.Random`; `C.newRandom` seeds with `drgNew` from system entropy | n/a |
|
||||
| BBS+ over BLS12-381, SHA-256 suite | Restricted | Badge entitlement credentials and proofs ([Crypto/Entitlement.hs](../src/Simplex/Messaging/Crypto/Entitlement.hs)), XFTP storage-time extension | Vendored submodules libbbs and blst (C and assembly) via FFI; randomness from libbbs `getentropy`, `SecRandomCopyBytes` with the `commoncrypto` flag, `BCryptGenRandom` on Windows ([cbits/getentropy_win.c](../cbits/getentropy_win.c)) | sk 32, pk 96, sig 80, proof 272 + 32 per undisclosed message |
|
||||
| ECDSA with SHA-256 (ES256) | Restricted | APNS provider JWT ([Push/APNS.hs](../src/Simplex/Messaging/Notifications/Server/Push/APNS.hs)) | crypton `Crypto.PubKey.ECC.ECDSA.sign`; key read by cryptostore `Crypto.Store.PKCS8`; curve taken from the key file | signature DER `SEQUENCE {r, s}`, base64url |
|
||||
| TLS 1.3 / 1.2 | Restricted | SMP, XFTP, NTF, XRCP transport: `TLS_CHACHA20_POLY1305_SHA256` (1.3), `ECDHE_ECDSA_CHACHA20POLY1305_SHA256` (1.2), groups X448 and X25519, Ed448 and Ed25519 signatures (`defaultSupportedParams`) | `tls` 1.9 | n/a |
|
||||
| TLS for browsers | Restricted | XFTP server with HTTPS credentials (`defaultSupportedParamsHTTPS`: `ciphersuite_strong`, FFDHE, P-521, ECDSA and RSA signatures); SMP server HTTPS requires RSA-4096 (`checkHTTPSCredentials`) | `tls`, `warp-tls` | n/a |
|
||||
| X.509 | Restricted | Certificate chains, fingerprints, signed session keys (`C.signX509`, `C.verifyX509`) | `crypton-x509`, `crypton-x509-validation`, `cryptostore` | n/a |
|
||||
| SQLCipher | Restricted | Agent SQLite database at rest (`PRAGMA key`); cipher parameters are SQLCipher defaults, not set in this repository | `direct-sqlcipher` (git dependency, [cabal.project](../cabal.project)) | n/a |
|
||||
|
||||
## SimpleX box constructions
|
||||
|
||||
`C.cryptoBox` and `LC.sbInit_` compute, for a 32-byte key `k` and 24-byte nonce `n`:
|
||||
|
||||
```
|
||||
k1 = HSalsa20(k, 0^16)
|
||||
subkey = HSalsa20(k1, n[0..16])
|
||||
stream = Salsa20(subkey, n[16..24])
|
||||
polyKey = stream[0..32]
|
||||
ct = msg XOR stream[32..]
|
||||
tag = Poly1305(polyKey, ct)
|
||||
```
|
||||
|
||||
| Construction | Key `k` | Equivalent libsodium call | Layout |
|
||||
|---|---|---|---|
|
||||
| SimpleX crypto_box (`C.cbEncrypt`, `LC.cbInit`) | X25519 shared secret | `crypto_box_easy` (`crypto_box_beforenm` is `HSalsa20(dh, 0^16)`) | strict: `tag \|\| ct`; lazy tail-tag: `ct \|\| tag` |
|
||||
| SimpleX secretbox (`C.sbEncrypt`, `LC.sbInit`, `KEM.kcb*`) | 32-byte symmetric key | `crypto_secretbox_easy` with key `crypto_core_hsalsa20(0^16, k)`, not with `k` | same |
|
||||
|
||||
Padding before encryption: `C.pad` prefixes a 2-byte big-endian length and fills with `#` (message at most 65533 bytes); `LC.pad` prefixes an 8-byte length. `*NoPad` variants skip padding.
|
||||
|
||||
## Domain separation and KDF inputs
|
||||
|
||||
HKDF is `C.hkdf salt ikm info len` (HKDF-SHA512).
|
||||
|
||||
| Info string | Salt | IKM | Output (split) | Function | Use |
|
||||
|---|---|---|---|---|---|
|
||||
| `"SimpleXX3DH"` | 64 zero bytes | `dh1 \|\| dh2 \|\| dh3 [\|\| kemShared]` | 96: `hk`, `nhk`, root key (32 each) | `CR.pqX3dh` | Ratchet initialization, added in E2E v2; KEM secret from E2E v3 (current minimum) |
|
||||
| `"SimpleXVerifyCode"` | 64 zero bytes | same as `SimpleXX3DH` | 32: `rcVCPQ` | `CR.pqX3dh` | Verification code covering all handshake keys, new ratchets |
|
||||
| `"SimpleXRootRatchet"` | root key | `dh [\|\| kemShared]` | 96: root key, chain key, next header key | `CR.rootKdf` | DH/PQ ratchet step |
|
||||
| `"SimpleXChainRatchet"` | empty | chain key | 96: chain key, message key (32 each), message IV (16), header IV (16) | `CR.chainKdf` | Per-message keys |
|
||||
| `"SimpleXSbChainInit"` | SMP session ID | X25519 session secret | 64: two 32-byte chain keys | `C.sbcInit` from `Transport.blockEncryption` | SMP transport block encryption, SMP v11+ |
|
||||
| `"SimpleXSbChainInit"` | empty | XRCP hybrid secret (below) | 64: two 32-byte chain keys | `C.sbcInit` in [RemoteControl/Client.hs](../src/Simplex/RemoteControl/Client.hs) | XRCP v1 session |
|
||||
| `"SimpleXSbChain"` | empty | chain key | 88: chain key (32), secretbox key (32), nonce (24) | `C.sbcHkdf` | Each SMP block and XRCP message |
|
||||
| `"SimpleXContactLink"` | empty | link key (32) | 56: link ID (24), secretbox key (32) | `SL.contactShortLinkKdf` | Contact short links, SMP v15+ |
|
||||
| `"SimpleXInvLink"` | empty | link key (32) | 32: secretbox key | `SL.invShortLinkKdf` | Invitation short links, SMP v15+ |
|
||||
|
||||
Other derivations:
|
||||
|
||||
| Value | Construction | Function | Use |
|
||||
|---|---|---|---|
|
||||
| Service request binding | `sha3_256("SimpleXService" \|\| rcAD)` | `serviceReqBinding` in [Agent.hs](../src/Simplex/Messaging/Agent.hs) | Agent RPC, agent v8 |
|
||||
| BBS header | `"SimpleX badges v1"` | `entitlementBBSHeader` | Badge credentials |
|
||||
| XRCP hybrid secret | `sha3_256(x25519Dh \|\| kemShared)` | `KEM.kemHybridSecret` | XRCP v1 |
|
||||
| Short-link key | `sha3_256(smpEncode FixedLinkData)` | `SL.encodeSignFixedData`, checked in `SL.decryptLinkData` | SMP v15+ |
|
||||
| Sender ID | `take 24 (sha3_384 corrId)` | `prepareConnectionLink'`, `newRcvConnSrv` in Agent.hs; `createQueue` in [Server.hs](../src/Simplex/Messaging/Server.hs) | Client-supplied sender ID with link data, SMP v15+; the server rejects a mismatch to prevent an ID oracle |
|
||||
| Ratchet associated data | `pubKeyBytes sk1 \|\| pubKeyBytes rk1` (raw X448, 112) | `CR.pqX3dh` | AD of every ratchet AEAD |
|
||||
| Security code | `sha256(rcAD)` | `ratchetVerifyCodes` in [AgentStore.hs](../src/Simplex/Messaging/Agent/Store/AgentStore.hs) | Connection verification |
|
||||
| Contact request code | `sha256(smpEncode (k1, k2, kem, sndId))` | `requestCode` in Agent.hs | Contact request binding |
|
||||
| Command authenticator | `crypto_box(sha512(authorized))` | `C.cbAuthenticate` | SMP v7+ |
|
||||
| Queue IDs hash | `xor` of `md5(queueId)` | `Protocol.queueIdsHash` | Service subscriptions |
|
||||
|
||||
`"SimpleXSbChainInit"` and `"SimpleXSbChain"` are shared by SMP block encryption and XRCP. Their outputs are separated only by the IKM (and the salt for `sbcInit`).
|
||||
|
||||
## Nonces and IVs
|
||||
|
||||
| Use | Construction |
|
||||
|---|---|
|
||||
| SMP command authenticator, proxied command | `C.cbNonce corrId`, where `corrId` is a random 24-byte nonce (`C.randomCbNonce`) |
|
||||
| SMP proxied responses | `C.reverseNonce` of the request nonce |
|
||||
| Server-to-recipient message body | `C.cbNonce msgId`; `msgIdBytes = 24` in [Server/Main.hs](../src/Simplex/Messaging/Server/Main.hs) |
|
||||
| Per-queue e2e envelope, short-link data, notifications, XRCP hello, XFTP chunk download | Random 24-byte nonce, sent with the ciphertext |
|
||||
| XFTP file, local encrypted file | Random key and nonce per file (`C.randomSbKey`, `C.randomCbNonce`) |
|
||||
| SMP block, XRCP session messages | Key and nonce from `C.sbcHkdf`, one per block |
|
||||
| Ratchet header, body | 16-byte IVs from `CR.chainKdf`; header IV is sent, body IV is not |
|
||||
| WebRTC frames | 12-byte `C.GCMIV` supplied by the caller in simplex-chat |
|
||||
|
||||
## Implementation sources
|
||||
|
||||
| Source | Version or pin | Provides |
|
||||
|---|---|---|
|
||||
| crypton | 0.34 | Ed25519, Ed448, X25519, X448, XSalsa20, Poly1305, AES-GCM, HKDF, SHA-2, SHA-3, Keccak, MD5, ChaChaDRG, ECDSA |
|
||||
| tls, crypton-x509, crypton-x509-validation, cryptostore | 1.9.0, 1.7.6, 1.6.12, 0.3.0.1 | TLS, X.509, PKCS#8 |
|
||||
| [cbits/sntrup761.c](../cbits/sntrup761.c) | Copy of draft-josefsson-ntruprime-streamlined-00 ([cbits/README.md](../cbits/README.md)) | sntrup761 |
|
||||
| [cbits/sha512.c](../cbits/sha512.c) and system OpenSSL `libcrypto` | system | SHA-512 for sntrup761 |
|
||||
| `cbits/libbbs` submodule ([simplex-chat/libbbs](https://github.com/simplex-chat/libbbs)) | `59a0f4bf` | BBS+ (`bbs_sha256_ciphersuite`), SHA-256, SHAKE256 |
|
||||
| `cbits/blst` submodule ([supranational/blst](https://github.com/supranational/blst)) | `db3defd0` | BLS12-381 (C and `build/assembly.S`, `-D__BLST_PORTABLE__`) |
|
||||
| direct-sqlcipher | git `f814ee68` | SQLCipher |
|
||||
| `openssl` CLI | system | Server CA and certificate key generation (`createServerX509_`) |
|
||||
|
||||
## xftp-web (TypeScript)
|
||||
|
||||
[xftp-web](../xftp-web/) reimplements a subset for the browser XFTP client. It has no AES-GCM, HKDF, SHA-3, MD5, X448 or sntrup761 code.
|
||||
|
||||
| Primitive | Implementation | Haskell counterpart |
|
||||
|---|---|---|
|
||||
| Ed25519 sign, verify, keys | libsodium-wrappers-sumo (`crypto_sign_*`), [src/crypto/keys.ts](../xftp-web/src/crypto/keys.ts) | `C.sign'`, `C.verify'` |
|
||||
| Ed448 verify | `@noble/curves/ed448`, keys.ts `verifyEd448` | `C.verify'` |
|
||||
| X25519 | libsodium `crypto_scalarmult`, `crypto_box_keypair` | `C.dh'` |
|
||||
| SimpleX crypto_box, secretbox, tail-tag streaming | Salsa20 block function written in TypeScript, libsodium `crypto_core_hsalsa20` and `crypto_onetimeauth_*`, [src/crypto/secretbox.ts](../xftp-web/src/crypto/secretbox.ts) | `C.cryptoBox`, `LC.sbInit_` |
|
||||
| crypto_box authenticator | [src/protocol/client.ts](../xftp-web/src/protocol/client.ts) `cbAuthenticate`, `cbVerify` | `C.cbAuthenticate`, `C.cbVerify` |
|
||||
| SHA-256, SHA-512 | libsodium `crypto_hash_sha256`, `crypto_hash_sha512*`, [src/crypto/digest.ts](../xftp-web/src/crypto/digest.ts) | `C.sha256Hash`, `C.sha512Hash` |
|
||||
| Random | WebCrypto `crypto.getRandomValues`; libsodium for key generation | `C.randomBytes` |
|
||||
|
||||
## Tests
|
||||
|
||||
None of the tests below compares against published reference vectors (NIST, RFC 8032, RFC 7748, RFC 5869, NaCl, the sntrup761 draft or the BBS draft). Interoperability is tested between this repository's own implementations.
|
||||
|
||||
| Area | Test module (hspec group) | Kind |
|
||||
|---|---|---|
|
||||
| Ed25519, Ed448, X25519 crypto_box, secretbox, lazy and tail-tag secretbox, AES-GCM 12-byte IV, X.509 key encoding, X.509 chains, sntrup761, BBS+, entitlements, padding | [tests/CoreTests/CryptoTests.hs](../tests/CoreTests/CryptoTests.hs) | Round-trip; fixed lengths for BBS+; fixed bytes for padding |
|
||||
| Double ratchet (X25519 and X448), PQ KEM agreement, AES-GCM 16-byte IV | [tests/AgentTests/DoubleRatchetTests.hs](../tests/AgentTests/DoubleRatchetTests.hs) | Round-trip; decoding of a stored v2 ratchet JSON |
|
||||
| Short links (HKDF info strings, SHA3-256 link key) | [tests/AgentTests/ShortLinkTests.hs](../tests/AgentTests/ShortLinkTests.hs) | Round-trip, tamper rejection |
|
||||
| File encryption | [tests/CoreTests/CryptoFileTests.hs](../tests/CoreTests/CryptoFileTests.hs) | Round-trip |
|
||||
| XRCP session (SHA3-256 hybrid, sb chain) | [tests/RemoteControl.hs](../tests/RemoteControl.hs) | End-to-end |
|
||||
| SMP authenticators, block encryption, `IdsHash` (MD5) | [tests/ServerTests.hs](../tests/ServerTests.hs) | End-to-end; with the PostgreSQL queue store this checks Haskell MD5 against pgcrypto |
|
||||
| Haskell and xftp-web: SHA-256/512, Ed25519, Ed448 identity proof, X25519, DER keys, crypto_box, secretbox, tail-tag, authenticator, file encryption | [tests/XFTPWebTests.hs](../tests/XFTPWebTests.hs) (`XFTP Web Client`) | Byte-identical cross-language output |
|
||||
@@ -17,11 +17,11 @@ note over CC, HC: TLS handshake
|
||||
par
|
||||
note over CC: validate client X509 credentials
|
||||
CC->>CI: session code from tlsUnique
|
||||
CI-->>CC: user confirmation (optional if known)
|
||||
CI-->>CC: user confirmation
|
||||
and
|
||||
note over HC: validate server X509 credentials
|
||||
HC->>HI: session code from tlsUnique
|
||||
HI-->>HC: user confirmation (optional if known)
|
||||
HI-->>HC: user confirmation
|
||||
end
|
||||
|
||||
note over CI, HI: 3. Session verification and protocol negotiation
|
||||
|
||||
File diff suppressed because one or more lines are too long
|
Before Width: | Height: | Size: 31 KiB After Width: | Height: | Size: 31 KiB |
+4
-8
@@ -62,16 +62,14 @@ It is possible to reduce size overhead by using only one KEM agreement and makin
|
||||
|
||||
## Double ratchet with encrypted headers augmented with double PQ KEM
|
||||
|
||||
Algorithm below assumes that in addition to shared secret from the initial key agreement, there will be an encapsulation key available from the initiating party, that sent its keys in the connection invitation to the joining party (see [agent protocol](./agent-protocol.md)). Following the double ratchet specification, the pseudo-code below names the joining party Alice and the initiating party Bob.
|
||||
|
||||
Unlike X3DH prekey bundles, these keys are not reusable keys published for any party to use. The initiating party generates two X448 key pairs and, optionally, a sntrup761 KEM key pair for each connection, sends the public keys in the invitation, and deletes the stored private keys once the ratchet is initialized; only the second X448 private key and the agreed KEM key pair remain in the ratchet state as its initial ratchet keys. The joining party generates its keys when joining and keeps only its KEM key pair, in the ratchet state. The exception is a contact address that publishes ratchet keys in its link data: all requesters use these keys until the address owner rotates them, and the owner keeps the private keys of a few recent generations.
|
||||
Algorithm below assumes that in addition to shared secret from the initial key agreement, there will be an encapsulation key available from the party that published its keys (Bob).
|
||||
|
||||
### Initialization
|
||||
|
||||
The double ratchet initialization is defined in pseudo-code. This pseudo-code is identical to Signal algorithm specification except for that parts that add post-quantum key agreement.
|
||||
|
||||
```
|
||||
// Alice (joining party) received Bob's keys in the invitation and initializes ratchet first
|
||||
// Alice obtained Bob's keys and initializes ratchet first
|
||||
def RatchetInitAlicePQ2HE(state, SK, bob_dh_public_key, shared_hka, shared_nhkb, bob_pq_kem_encapsulation_key):
|
||||
state.DHRs = GENERATE_DH()
|
||||
state.DHRr = bob_dh_public_key
|
||||
@@ -91,7 +89,7 @@ def RatchetInitAlicePQ2HE(state, SK, bob_dh_public_key, shared_hka, shared_nhkb,
|
||||
state.HKr = None
|
||||
state.NHKr = shared_nhkb
|
||||
|
||||
// Bob (initiating party) initializes ratchet second, having received Alice's first message
|
||||
// Bob initializes ratchet second, having received Alice's connection request
|
||||
def RatchetInitBobPQ2HE(state, SK, bob_dh_key_pair, shared_hka, shared_nhkb, bob_pq_kem_key_pair):
|
||||
state.DHRs = bob_dh_key_pair
|
||||
state.DHRr = None
|
||||
@@ -212,8 +210,6 @@ The outer envelope contains the encrypted header (used as associated data for bo
|
||||
|
||||
The message body is encrypted with AES-256-GCM using the message key derived from the sending chain key (`KDF_CK`). The associated data for body encryption is the concatenation of the ratchet associated data and the encoded encrypted header.
|
||||
|
||||
`KDF_CK(CK)` is HKDF-SHA512 with empty salt, `CK` as input key material and info `"SimpleXChainRatchet"`, producing 96 bytes split into the next chain key (32 bytes), the message key (32 bytes), the message body IV (16 bytes, not transmitted) and `headerIV` (16 bytes). Both IVs are used as 16-byte AES-256-GCM IVs, not the 12-byte IVs recommended by NIST SP 800-38D, so the initial counter block is J0 = GHASH(IV || 0^64 || [128]_64) as defined there for non-96-bit IVs. WebCrypto and other conforming implementations compute it when given the full 16-byte IV; truncating the IV to 12 bytes produces different ciphertext.
|
||||
|
||||
```abnf
|
||||
encRatchetMessage = versionedLength encMessageHeader msgAuthTag encMsgBody
|
||||
; encMessageHeader is used as associated data for body decryption: AD = rcAD || encMessageHeader
|
||||
@@ -278,7 +274,7 @@ As SimpleX Messaging Protocol pads messages to a fixed size, using 16kb transpor
|
||||
|
||||
Sharing the initial keys in case of SimpleX Chat it is equivalent to sharing the invitation link. As encapsulation key is large, it may be inconvenient to share it in the link in some contexts, e.g. when QR codes are used.
|
||||
|
||||
It is possible to postpone sharing the encapsulation key until the first message from the joining party (confirmation message in SMP protocol). The upside here is that the invitation link size would not increase. The downside is that the user profile shared in this confirmation will not be encrypted with PQ-resistant algorithm.
|
||||
It is possible to postpone sharing the encapsulation key until the first message from Alice (confirmation message in SMP protocol), the party sending connection request. The upside here is that the invitation link size would not increase. The downside is that the user profile shared in this confirmation will not be encrypted with PQ-resistant algorithm.
|
||||
|
||||
Another consideration is pairwise ratchets in groups. Key generation in sntrup761 is quite slow - on slow devices it can be as slow as 10-20 keys per second, so using this primitive in groups larger than 10-20 members would result in slow performance.
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@ This document describes the cryptographic primitives and threat model for the Si
|
||||
- [SimpleX Messaging Protocol router that proxies the messages to another SMP router](#simplex-messaging-protocol-router-that-proxies-the-messages-to-another-smp-router)
|
||||
- [An attacker who obtained Alice's (decrypted) chat database](#an-attacker-who-obtained-alices-decrypted-chat-database)
|
||||
- [A user's contact](#a-users-contact)
|
||||
- [An attacker who observes the initiating party showing an introduction message to the joining party](#an-attacker-who-observes-the-initiating-party-showing-an-introduction-message-to-the-joining-party)
|
||||
- [An attacker who observes Alice showing an introduction message to Bob](#an-attacker-who-observes-alice-showing-an-introduction-message-to-bob)
|
||||
- [An attacker with Internet access](#an-attacker-with-internet-access)
|
||||
|
||||
|
||||
@@ -37,8 +37,6 @@ This document describes the cryptographic primitives and threat model for the Si
|
||||
- AES-GCM AEAD cipher,
|
||||
- SHA512-based HKDF for key derivation.
|
||||
|
||||
All primitives in use, their lengths, domain-separation strings and the policy for new code are listed in the [cryptographic primitive registry](./crypto-registry.md).
|
||||
|
||||
|
||||
## Threat Model
|
||||
|
||||
@@ -170,7 +168,7 @@ All primitives in use, their lengths, domain-separation strings and the policy f
|
||||
|
||||
*cannot:*
|
||||
|
||||
- impersonate a sender and send messages to the user whose database was stolen, unless the database contains a new rotation queue that is not secured yet. Doing so requires also compromising the router (to place the message in the queue, that is possible until the Double-Ratchet advances forward) or the user's device at a subsequent time (to place the message in the database).
|
||||
- impersonate a sender and send messages to the user whose database was stolen. Doing so requires also compromising the router (to place the message in the queue, that is possible until the Double-Ratchet advances forward) or the user's device at a subsequent time (to place the message in the database).
|
||||
|
||||
- undetectably communicate at the same time as Alice with her contacts. Doing so would result in the contact getting different messages with repeated IDs.
|
||||
|
||||
@@ -192,15 +190,15 @@ All primitives in use, their lengths, domain-separation strings and the policy f
|
||||
|
||||
- cannot collaborate with another of the user's contacts to confirm they are communicating with the same user.
|
||||
|
||||
### An attacker who observes the initiating party showing an introduction message to the joining party
|
||||
### An attacker who observes Alice showing an introduction message to Bob
|
||||
|
||||
*can:*
|
||||
|
||||
- Impersonate the joining party to the initiating party.
|
||||
- Impersonate Bob to Alice.
|
||||
|
||||
*cannot:*
|
||||
|
||||
- Impersonate the initiating party to the joining party.
|
||||
- Impersonate Alice to Bob.
|
||||
|
||||
### An attacker with Internet access
|
||||
|
||||
|
||||
+47
-131
@@ -86,7 +86,7 @@ It's designed with the focus on communication security and integrity, under the
|
||||
|
||||
It is designed as a low level protocol for other application protocols to solve the problem of secure and private message transmission, making [MITM attack][1] very difficult at any part of the message transmission system.
|
||||
|
||||
This document describes SMP protocol version 23. Versions 1-5 are discontinued. The version history:
|
||||
This document describes SMP protocol version 20. Versions 1-5 are discontinued. The version history:
|
||||
|
||||
- v1: binary protocol encoding
|
||||
- v2: message flags (used to control notifications)
|
||||
@@ -108,8 +108,6 @@ This document describes SMP protocol version 23. Versions 1-5 are discontinued.
|
||||
- v19: service subscriptions to messages (SUBS, NSUBS, SOKS, ENDS, ALLS commands)
|
||||
- v20: public namespaces resolver (RSLV command, RNAME response) — direct or forwarded via PFWD
|
||||
- v21: server public information in handshake
|
||||
- v22: `RNAME` says whether a name can be registered, not only what it resolves to
|
||||
- v23: version in nonces of forwarded commands, random nonces for forwarded responses
|
||||
|
||||
## Introduction
|
||||
|
||||
@@ -1121,15 +1119,13 @@ When the client receives PKEY response it MUST validate that:
|
||||
- the fingerprint of the received certificate matches fingerprint in the router address - it mitigates MITM attack by proxy.
|
||||
- the router session key is correctly signed with the received certificate.
|
||||
|
||||
`smpVersionRange` in `PKEY` is not signed.
|
||||
|
||||
The proxy router may respond with error response in case the destination router is not available or in case it has an earlier version that does not support proxied commands.
|
||||
|
||||
#### Send command via proxy
|
||||
|
||||
Sender can send `SKEY` and `SEND` commands via proxy after obtaining the session ID with `PRXY` command (see [Request proxied session](#request-proxied-session)).
|
||||
|
||||
Transmission sent to proxy router should use session ID as entity ID and use a random correlation ID of 24 bytes as a nonce for crypto_box encryption of transmission to the destination router. When `smpVersion` in `PFWD` is 23 or higher, the first 2 bytes of this nonce are XOR-ed with `smpVersion`. The random ephemeral X25519 key to encrypt transmission should be unique per command, and it should be combined with the key sent by the router in the handshake header to proxy and to the client in `PKEY` command.
|
||||
Transmission sent to proxy router should use session ID as entity ID and use a random correlation ID of 24 bytes as a nonce for crypto_box encryption of transmission to the destination router. The random ephemeral X25519 key to encrypt transmission should be unique per command, and it should be combined with the key sent by the router in the handshake header to proxy and to the client in `PKEY` command.
|
||||
|
||||
Encrypted transmission should use the received session ID from the connection between proxy router and destination router in the authorized body.
|
||||
|
||||
@@ -1141,11 +1137,10 @@ commandKey = length x509encoded
|
||||
|
||||
The proxy router will forward the encrypted transmission in `RFWD` command (see below).
|
||||
|
||||
Having received the `RRES` response from the destination router, proxy router will forward `PRES` response to the client. `PRES` response should use the same correlation ID as `PFWD` command. The destination router will use this correlation ID increased by 1 as a nonce for encryption of the response. When `smpVersion` in `PFWD` is 23 or higher, the destination router uses a random nonce instead, and sends it before the encrypted response.
|
||||
Having received the `RRES` response from the destination router, proxy router will forward `PRES` response to the client. `PRES` response should use the same correlation ID as `PFWD` command. The destination router will use this correlation ID increased by 1 as a nonce for encryption of the response.
|
||||
|
||||
```abnf
|
||||
proxyResponse = %s"PRES" SP [responseNonce] <encrypted padded(forwardedResponse, 16226)>
|
||||
responseNonce = %s"0" / (%s"1" 24*24 OCTET) ; from v23
|
||||
proxyResponse = %s"PRES" SP <encrypted padded(forwardedResponse, 16226)>
|
||||
forwardedResponse = *OCTET ; client-encrypted SMP response, decrypted by client using per-command DH secret
|
||||
```
|
||||
|
||||
@@ -1172,7 +1167,7 @@ The shared secret for encrypting transmission bodies between proxy router and de
|
||||
|
||||
|
||||
```abnf
|
||||
relayResponse = %s"RRES" SP [responseNonce] <encrypted(responseTransmission)>
|
||||
relayResponse = %s"RRES" SP <encrypted(responseTransmission)>
|
||||
responseTransmission = fwdCorrId forwardedResponse
|
||||
; fwdCorrId and forwardedResponse defined above in RFWD section
|
||||
```
|
||||
@@ -1456,150 +1451,53 @@ reads `NameRecord` from. The reference implementation forwards each RSLV to a
|
||||
companion REST resolver process (`scripts/resolver/snrc-resolve.py`) that
|
||||
queries the SNRC contract on Ethereum; alternative backings (different chains,
|
||||
DHT, etc.) are valid as long as they expose the documented HTTP shape (`GET
|
||||
/v2/resolve/<query>` returning a `NameRegistration` on 200 for every
|
||||
registration shape, 400 for unknown TLDs, 502 for upstream failures) or
|
||||
substitute a different transport returning the same JSON. The resolver API is
|
||||
versioned separately from this protocol: `/v1/resolve/<name>` returns a bare
|
||||
`NameRecord` and is what relays before v22 call as `/resolve/<name>`.
|
||||
/resolve/<name>` returning a `NameRecord` on 200, 404 / 400 for unknown names
|
||||
or TLDs, 502 for upstream RPC failures) or substitute a different transport
|
||||
while still returning a `NameRecord` matching the encoding below.
|
||||
|
||||
#### Resolve name command
|
||||
|
||||
The `RSLV` command carries the query as text, not JSON. A client sends the
|
||||
hashed form only from v22, and the name itself below it:
|
||||
The `RSLV` command carries the canonical fully-qualified name directly as the
|
||||
payload (not JSON):
|
||||
|
||||
```abnf
|
||||
rslv = %s"RSLV" SP query
|
||||
query = domain / hashed ; hashed only from v22
|
||||
domain = 1*253 OCTET ; the name as text
|
||||
hashed = "[" 64HEXDIG "]" tld ; keccak-256 of the second-level label
|
||||
tld = %s".simplex" / %s".testing"
|
||||
rslv = %s"RSLV" SP domain ; domain = canonical name as non-space bytes, consuming the remainder of the transmission
|
||||
```
|
||||
|
||||
`domain` is the UTF-8 canonical fully-qualified name with the TLD always
|
||||
explicit (e.g. `privacy.simplex`, `test.testing`, `example.com`), bounded to
|
||||
253 bytes.
|
||||
|
||||
**Hashed labels.** The query is a name, or the keccak-256 of a second-level
|
||||
label in ENS's bracketed form. A label can only be letters, digits and hyphens,
|
||||
so `[` tells the two apart and no tag is needed.
|
||||
|
||||
Only a second-level name may be hashed. A name with subnames is sent as text: it
|
||||
is resolved rather than priced, and its record names it anyway. A web TLD has no
|
||||
registry to key a hash on.
|
||||
|
||||
From v22 a client MUST send the hash. Older routers can only read the name, so a
|
||||
client on an older session sends the name. A router answering a hashed query
|
||||
does not know the label's length, so it cannot check a minimum-length policy
|
||||
either: the client does that, from the pricing it is sent.
|
||||
|
||||
The same form reaches the backing resolver, which is what its HTTP API takes, so
|
||||
the query is one string end to end.
|
||||
|
||||
A hashed query still answers with the name. The registrar records the plaintext
|
||||
label when a name is registered, keyed by the hash of that label, so a router can
|
||||
look up what the hash stands for without ever being told. The router is not
|
||||
trusted for it: a client MUST check that the record names the name it asked
|
||||
about, and reject the answer otherwise. A registry that does not record the
|
||||
label cannot answer a hashed query at all, and the router answers `ERR NAME
|
||||
RESOLVER` rather than a record it knows the client will reject. An unregistered
|
||||
name has no recorded label, so it cannot be looked up either.
|
||||
|
||||
**Server-side validation.** The names router parses `domain` as a
|
||||
fully-qualified name (TLD required — bare labels are rejected) and forwards it
|
||||
to the configured backing resolver, which is the source of truth for which
|
||||
on-chain registry maps to each TLD.
|
||||
|
||||
The names router responds with either an `RNAME` response saying what it knows
|
||||
about the name, or an `ERR NAME` error whose subcode a client iterating across
|
||||
several configured servers can act on distinctly:
|
||||
The names router responds with either an `RNAME` response carrying the resolved
|
||||
record, or an `ERR NAME` error whose subcode a client iterating across several
|
||||
configured servers can act on distinctly:
|
||||
|
||||
| Response | Condition | Client action |
|
||||
|---|---|---|
|
||||
| `RNAME` | the router read the registry | use it |
|
||||
| `ERR NAME NOT_FOUND` | below v22 only: every name that does not resolve. From v22 a router never sends it | stop, and do not read it as registrable |
|
||||
| `RNAME` | record resolved | use it |
|
||||
| `ERR NAME NOT_FOUND` | name not registered, unknown TLD, or malformed name | authoritative "no such name" — stop |
|
||||
| `ERR NAME NO_RESOLVER` | this router has no resolver (names role not enabled) | skip this server, try the next |
|
||||
| `ERR NAME RESOLVER <detail>` | the router cannot state an answer completely: no registrar or price oracle for the TLD, an unreachable chain, a transport failure, a timeout, a registration it could not date or resolve | surface `<detail>`; retry only if it reads as transient |
|
||||
| `ERR NAME RESOLVER <detail>` | transient failure: backing resolver error (upstream 5xx, transport, timeout, decode) | transient — retry or surface, do not treat as "not found" |
|
||||
|
||||
A client SHOULD NOT broadcast a `name` to further servers after a name-capable
|
||||
router has answered (`NOT_FOUND` or `RESOLVER`), since that router has already
|
||||
seen the lookup key; `NO_RESOLVER` discloses nothing about the name beyond the
|
||||
fact that this router cannot resolve, so iterating past it is safe.
|
||||
|
||||
#### Name response
|
||||
#### Name record response
|
||||
|
||||
`RNAME` answers both what a name resolves to and whether it can be registered.
|
||||
The `RNAME` response carries a JSON-encoded record as the payload:
|
||||
|
||||
```abnf
|
||||
rname = %s"RNAME" SP nameResponse
|
||||
rname = %s"RNAME" SP json-bytes ; json-bytes consumes the remainder of the transmission
|
||||
```
|
||||
|
||||
`nameResponse` is a UTF-8 JSON object consuming the remainder of the
|
||||
transmission: the `registration` below, and `lastBlockTs`, the block timestamp the
|
||||
registry was read at. Money is US cents, times are seconds since the Unix epoch, and lengths are
|
||||
characters.
|
||||
|
||||
| Field | JSON type | Constraints |
|
||||
|---|---|---|
|
||||
| `lastBlockTs` | number | block timestamp the answer was read at. Absent only from a v20/v21 router, which sent the record alone |
|
||||
| `registration` | object | below |
|
||||
|
||||
A router reads the registry through a node of its own, which can lag. A name
|
||||
registered or changed after `lastBlockTs` still reads as it did before, so a client
|
||||
MUST NOT treat an answer as current without checking it.
|
||||
|
||||
`registration`'s `type` selects which of the three answers it is.
|
||||
|
||||
| `type` | Meaning |
|
||||
|---|---|
|
||||
| `registered` | held by someone until `expires`, renewable by its owner alone until `graceUntil`. It always carries `nameRecord`: where the owner set none, every field is unset and the resolver address is zero |
|
||||
| `available` | held by nobody and registrable now, at `pricing` |
|
||||
| `reserved` | held back by the registry and not registered. It carries no price, and a router MUST NOT quote one |
|
||||
|
||||
| Field | On | JSON type | Constraints |
|
||||
|---|---|---|---|
|
||||
| `expires` | `registered` | number | absent only from a v20/v21 router, which sent the record alone |
|
||||
| `graceUntil` | `registered` | number | after `expires`; until here only the owner may renew. Absent on the same condition as `expires` |
|
||||
| `reservedReason_` | `registered` | string | a reason word, present only when the name is held back as well |
|
||||
| `nameRecord` | `registered` | object | the record, schema below |
|
||||
| `pricing` | `available` | object | `registrationPrices`, `basePrice` and `minLabelLength`, below |
|
||||
| `reservedReason` | `reserved` | string | a reason word |
|
||||
|
||||
| `pricing` field | JSON type | Constraints |
|
||||
|---|---|---|
|
||||
| `registrationPrices` | object | label length, as a decimal string, to US cents per year, for the lengths the registry prices specially |
|
||||
| `basePrice` | number | US cents per year for every other length |
|
||||
| `minLabelLength` | number | characters; the registry refuses shorter labels |
|
||||
|
||||
A reason word is `internal`, `trademark`, `community`, or a word a later version
|
||||
reserves under, at most 32 printable ASCII characters. A router truncates an
|
||||
unknown word to that and otherwise passes it through unchanged. A client MUST
|
||||
read a word it does not know as unknown and still treat the name as reserved.
|
||||
|
||||
**Computing the price.** In US cents, for a duration in seconds:
|
||||
|
||||
```
|
||||
price len duration = tier len * duration / 31536000
|
||||
tier len = the entry for len in registrationPrices, or basePrice when there is none
|
||||
```
|
||||
|
||||
The registry's minimum registration is 730 days, a contract constant, so it is
|
||||
specified here rather than sent. `registrationPrices` omits any length below
|
||||
`minLabelLength`. A client MUST NOT show a quote for a label the registry
|
||||
refuses: a hashed query carries no length, so only the client can check it.
|
||||
|
||||
Below v22, `RNAME` carries the bare record and nothing else, and every answer
|
||||
without one is `ERR NAME NOT_FOUND`. A v22 client reads such an answer as
|
||||
`registered` with no expiry, grace or reservation.
|
||||
|
||||
From v22 a client MUST NOT read `ERR NAME NOT_FOUND` as "registrable": only
|
||||
`available` says that.
|
||||
|
||||
A router that cannot state an answer completely MUST send `ERR NAME RESOLVER
|
||||
<detail>` rather than answer partially or guess. That covers a TLD with no
|
||||
registrar or price oracle, an unreachable chain, a timeout, a registration it
|
||||
could not date or resolve, and any status word it does not recognise.
|
||||
|
||||
`nameRecord` MUST be a UTF-8 JSON object with the following schema:
|
||||
`json-bytes` MUST be a UTF-8 JSON object with the following schema:
|
||||
|
||||
| Field | JSON type | Constraints |
|
||||
|---|---|---|
|
||||
@@ -1616,16 +1514,34 @@ could not date or resolve, and any status word it does not recognise.
|
||||
| `owner` | string | `"0x"` followed by 40 lowercase hex characters (20 raw bytes) |
|
||||
| `resolver` | string | `"0x"` followed by 40 lowercase hex characters; the resolver contract address that produced the record |
|
||||
|
||||
Testnet-vs-mainnet status is derived from the queried TLD, not from the record.
|
||||
Text fields (`nickname`, `website`, `location`) use the empty string `""` as
|
||||
the "unset" sentinel: a backing resolver with no value for the field MUST emit
|
||||
an empty string, not JSON `null` and not an absent key. Link fields
|
||||
(`simplexContact`, `simplexChannel`) are arrays, primary link first, and use the
|
||||
empty array `[]` when unset. Coin fields (`eth`, `btc`, `xmr`, `dot`) use JSON
|
||||
`null` as the "unset" sentinel and MAY also be absent from the object entirely.
|
||||
|
||||
Receivers MUST tolerate extra unknown fields; adding a required field is a
|
||||
breaking change requiring an SMP version bump. Receivers parse by key name, so
|
||||
peers MUST NOT rely on a byte-canonical form.
|
||||
The backing resolver filters records that are expired or otherwise unavailable
|
||||
(the names router then returns `ERR NAME NOT_FOUND` to the client), so the wire
|
||||
format carries no expiry field. Testnet-vs-mainnet status is derived from the
|
||||
queried TLD rather than an in-record flag.
|
||||
|
||||
The names router caps the resolver response it will accept
|
||||
(`resolver_max_response_bytes`, at most 16000 bytes) so the re-encoded `RNAME`
|
||||
stays within the SMP proxied transmission budget of 16224 bytes; a response over
|
||||
the cap is `ERR NAME RESOLVER`.
|
||||
Receivers MUST tolerate extra unknown fields (forward-compatibility for future
|
||||
field additions). Adding a required field is a breaking change requiring an
|
||||
SMP version bump.
|
||||
|
||||
**Field order is not significant.** Receivers parse JSON by key name, so object
|
||||
key order, insignificant whitespace, and number formatting carry no meaning;
|
||||
records are interpreted by decoded value, never compared byte-for-byte. Peers
|
||||
MUST NOT rely on a byte-canonical form — a different resolver or server may emit
|
||||
the same record with different key order or spacing. This order-independence is
|
||||
what makes the format forward-compatible (see the unknown-field rule above).
|
||||
|
||||
**Wire-size budget.** The names router caps the resolver response it will
|
||||
accept (`resolver_max_response_bytes`, ≤ 16000 bytes, the default) so the
|
||||
re-encoded `RNAME` stays within the SMP proxied transmission budget of 16224
|
||||
bytes; a response over the cap is rejected as `ERR NAME RESOLVER`. The link
|
||||
arrays are bounded by this overall budget rather than a fixed per-field count.
|
||||
|
||||
## Transport connection with the SMP router
|
||||
|
||||
|
||||
+4
-19
@@ -71,7 +71,7 @@ The session invitation contains this data:
|
||||
|
||||
Host application decrypts (except the first session) and validates the invitation:
|
||||
- Session signature is valid.
|
||||
- Timestamp of a multicast announcement is not earlier than 3660 seconds before and not later than 3600 seconds after the current time of the host. The host ignores announcements outside of this interval and continues listening.
|
||||
- Timestamp is within some window from the current time.
|
||||
- Long-term key signature is valid.
|
||||
- Long-term CA and signature key are the same as in the first session.
|
||||
- Some version in the offered range is supported.
|
||||
@@ -121,15 +121,9 @@ During TLS handshake, parties validate certificate chains against previously kno
|
||||
|
||||
### Session verification and protocol negotiation
|
||||
|
||||
Once TLS session is established, both the host and controller devices present a "session security code" to the user. The session security code must be a digest of tlsunique channel binding. As it is computed as a digest of the TLS handshake for both the controller and the host, it will validate that the same TLS certificates are used on both sides, and that the same TLS session is established, mitigating the possibility of MITM attack in the connection.
|
||||
Once TLS session is established, both the host and controller devices present a "session security code" to the user who must match them (e.g., visually or via QR code scan) and confirm on the host device. The session security code must be a digest of tlsunique channel binding. As it is computed as a digest of the TLS handshake for both the controller and the host, it will validate that the same TLS certificates are used on both sides, and that the same TLS session is established, mitigating the possibility of MITM attack in the connection.
|
||||
|
||||
To verify the session, the user matches the codes (e.g., visually or via QR code scan) and confirms the session on the host device.
|
||||
|
||||
The user MUST verify the first session with a new controller. The host stores the long-term identity of the controller (CA fingerprint and Ed25519 public key) only after this session is verified. A controller with the stored long-term identity is a known controller.
|
||||
|
||||
The user MAY verify the subsequent sessions with a known controller. In a session that the host confirms without user action, the controller is authenticated only by its long-term identity stored in the first session (see [Threat model](#threat-model)). SimpleX Chat mobile apps require user verification of sessions with known controllers only when the user enables the "Verify connections" option.
|
||||
|
||||
Once the session is confirmed, the host sends HELLO block to the controller.
|
||||
Once the session is confirmed by the user, the host sends HELLO block to the controller.
|
||||
|
||||
XRCP blocks inside TLS are padded to 16384 bytes.
|
||||
|
||||
@@ -311,7 +305,7 @@ to receive: (rcvKey', sk, nonce) = HKDF(rcvKey, "SimpleXSbChain", 88)
|
||||
- prevent host and controller devices from establishing the session
|
||||
|
||||
*cannot:*
|
||||
- same as passive adversary, provided that user visually verified session code out-of-band in the first session with the controller.
|
||||
- same as passive adversary, provided that user visually verified session code out-of-band.
|
||||
|
||||
#### An active adversary with the access to the network:
|
||||
|
||||
@@ -330,15 +324,6 @@ to receive: (rcvKey', sk, nonce) = HKDF(rcvKey, "SimpleXSbChain", 88)
|
||||
*cannot:*
|
||||
- connect to the host or make host connect to itself.
|
||||
|
||||
#### An active adversary with the access to the network who also obtained the long-term private keys of a known controller (e.g., from the controller device):
|
||||
|
||||
*can:*
|
||||
- make host connect to itself instead of the controller.
|
||||
- access any data of the controlled host application, within the capabilities of the provided API, if the host confirms the session without user action.
|
||||
|
||||
*cannot:*
|
||||
- access host application data, provided that user visually verified session code out-of-band.
|
||||
|
||||
#### Compromised controller device:
|
||||
|
||||
*can:*
|
||||
|
||||
@@ -1,8 +1,5 @@
|
||||
# Copy to .env (next to docker-compose.yml) only to change a setting; without it
|
||||
# docker-compose.yml uses the values shown here.
|
||||
|
||||
# ============================================================================
|
||||
# Chain settings.
|
||||
# Required settings — the stack will not start without these.
|
||||
# ============================================================================
|
||||
|
||||
# Ethereum network: mainnet (the SNRC `.testing` contracts live on mainnet)
|
||||
@@ -17,16 +14,9 @@ NETWORK=mainnet
|
||||
TRUSTED_NODE_URL=https://mainnet-checkpoint-sync.attestant.io
|
||||
|
||||
# ============================================================================
|
||||
# Optional overrides — leave commented unless you need to change them.
|
||||
# Optional overrides — sensible defaults are baked into docker-compose.yml,
|
||||
# so leave these commented unless you need to change them.
|
||||
# ============================================================================
|
||||
|
||||
# Nimbus NAT (default: any). For a stable public node set an explicit IP:
|
||||
# NAT=extip:1.2.3.4 # your public IPv4: curl -s ifconfig.me
|
||||
|
||||
# Resolver (defaults in the README's "Load and scaling" and "Logs" sections)
|
||||
# SNRC_WORKERS=4 # worker processes; default CPU count, at most 4
|
||||
# SNRC_RPC_TIMEOUT=5 # seconds per request to reth
|
||||
# SNRC_LOG_FORMAT=json # default text (key=value)
|
||||
# SNRC_LOG_COLOR=always # coloured `docker compose logs`
|
||||
# SNRC_LOG_LEVEL=debug # also log health checks and idle connections
|
||||
# SNRC_TRUSTED_PROXIES=172.16.0.0/12 # behind a reverse proxy on the host; see README
|
||||
+16
-339
@@ -19,15 +19,15 @@ against **Ethereum mainnet** (where the `.testing` contracts live):
|
||||
|
||||
## 1. Configure
|
||||
|
||||
The defaults in `docker-compose.yml` work as-is:
|
||||
Edit `.env` — the defaults work as-is; override only if needed:
|
||||
|
||||
```sh
|
||||
NETWORK=mainnet # default
|
||||
TRUSTED_NODE_URL=https://mainnet-checkpoint-sync.attestant.io # default
|
||||
```
|
||||
|
||||
To override these or NAT, copy `.env.example` to `.env` and edit it; `.env` is
|
||||
not committed.
|
||||
Everything else (NAT) has a working default baked into `docker-compose.yml`;
|
||||
uncomment the hints in `.env` only to override.
|
||||
|
||||
## 2. Run
|
||||
|
||||
@@ -62,26 +62,15 @@ curl -s -X POST http://127.0.0.1:8545 \
|
||||
**2. resolver is healthy:**
|
||||
```sh
|
||||
curl -s http://127.0.0.1:8000/health | jq
|
||||
# → {"ok": true, "rpc": "http://reth:8545", "registries": {"testing": "0x…", "simplex": ""},
|
||||
# "blockNumber": 23400000, "chainLagSeconds": 12}
|
||||
# → {"ok": true, "rpc": "http://reth:8545", "registries": {"testing": "0x…", "simplex": ""}}
|
||||
```
|
||||
|
||||
`chainLagSeconds` is how far the node is behind the wall clock. A resolver that
|
||||
is reachable and answering can still be hours behind, and every name it reports
|
||||
is that stale. `null` means the node could not be reached.
|
||||
|
||||
**3. resolver resolves a live name** (`foobar.testing` is a populated test name):
|
||||
```sh
|
||||
curl -s http://127.0.0.1:8000/resolve/foobar.testing | jq
|
||||
# → {"name":"foobar.testing","nickname":"Foo","simplexContact":["https://smp16.simplex.im/a#…"], … }
|
||||
```
|
||||
|
||||
**4. the route your router will call** (check 3 passes on an older resolver too):
|
||||
```sh
|
||||
curl -s http://127.0.0.1:8000/v2/resolve/foobar.testing | jq
|
||||
# → {"type":"registered","expires":1780000000,"graceUntil":…,"nameRecord":{…}}
|
||||
```
|
||||
|
||||
**Wire your smp-server:** in its `[NAMES]` section set
|
||||
`resolver_endpoint: http://127.0.0.1:8000` (no auth needed for loopback).
|
||||
|
||||
@@ -93,7 +82,7 @@ curl -s http://127.0.0.1:8000/v2/resolve/foobar.testing | jq
|
||||
| reth p2p | `:30303` tcp/udp | Ethereum sync (open on firewall) |
|
||||
| nimbus p2p | `:9000` tcp/udp | beacon sync (open on firewall) |
|
||||
| nimbus REST | `127.0.0.1:5052` | beacon API |
|
||||
| **resolver** | `127.0.0.1:8000` | SNRC REST (`/v2/resolve`, `/resolve`, `/health`) |
|
||||
| **resolver** | `127.0.0.1:8000` | SNRC REST (`/resolve`, `/health`) |
|
||||
|
||||
## Caveats
|
||||
|
||||
@@ -121,66 +110,7 @@ standalone for local dev (no Docker), via [`uv`](https://docs.astral.sh/uv/):
|
||||
uv run scripts/resolver/service/snrc-resolve.py # defaults to local reth + mainnet .testing
|
||||
```
|
||||
|
||||
Tests and lint run from `scripts/resolver/service`:
|
||||
|
||||
```sh
|
||||
uv run pytest
|
||||
uv run ruff check .
|
||||
```
|
||||
|
||||
Dependencies are locked in `uv.lock`; the Docker image installs exactly those.
|
||||
|
||||
Three routes, versioned separately from the protocol so each only changes when
|
||||
its own shape does:
|
||||
|
||||
| Route | Called by | Answers |
|
||||
|---|---|---|
|
||||
| `/v2/resolve/<query>` | routers from SMP v22 | a `NameRegistration` |
|
||||
| `/resolve/<name>` | routers before SMP v22 | a name record, flat |
|
||||
| `/health` | anyone | readiness |
|
||||
|
||||
`/v1/resolve/<name>` is an alias for `/resolve/<name>`.
|
||||
|
||||
### v2: `/v2/resolve/<query>`
|
||||
|
||||
The body is the SMP protocol's `NameResponse`, which the router decodes as is
|
||||
and forwards; translating the registry's model to it is this resolver's job. It
|
||||
is the registration below, plus `lastBlockTs`, the timestamp of the block it was read
|
||||
at: a node that lags answers with names it has not seen registered yet.
|
||||
Its `type` is `registered`, `available` or `reserved`, and the fields each one
|
||||
carries are specified once, in the **Name response** section of
|
||||
[`protocol/simplex-messaging.md`](../../protocol/simplex-messaging.md). It is
|
||||
the wire format, so it is documented with the wire.
|
||||
|
||||
Two things follow from that and are worth stating here. Expiry and grace belong
|
||||
to the registration, not to the record: `expires` and `graceUntil` sit beside
|
||||
`nameRecord`, not inside it. And a name nobody holds is not an error: it
|
||||
answers 200 with `type: available` and its price, so no status code from this
|
||||
route means "not registered".
|
||||
|
||||
| Status | Meaning |
|
||||
|---|---|
|
||||
| 200 | a registration: `registered`, `available` or `reserved` |
|
||||
| 400 | `tldNotConfigured`, `notFullyQualified` |
|
||||
| 502 | `noPriceOracle`, `labelNotRecorded`, an unreadable status, or `upstreamError` |
|
||||
|
||||
Error bodies carry `name` and a fixed `error` code to branch on. Only
|
||||
`upstreamError` adds a `message`; the v1 route always adds one.
|
||||
|
||||
`labelNotRecorded` means the registrar holds the name but never recorded its
|
||||
label, so a hashed query cannot be answered with a name. See
|
||||
[Querying by labelhash](#querying-by-labelhash).
|
||||
|
||||
A subname reports the expiry and grace of the 2LD above it, since that is what
|
||||
bounds its lifetime. A subname nobody created reports as not registered.
|
||||
|
||||
### v1: `/resolve/<name>`
|
||||
|
||||
What routers before SMP v22 call. Its shape is unrelated to v2's: the record is
|
||||
flat, and `status`, `expires`, `graceEnds`, `reasonCode` and `reason` sit
|
||||
alongside its fields.
|
||||
|
||||
#### v1 response shape
|
||||
### Response shape
|
||||
|
||||
```jsonc
|
||||
{
|
||||
@@ -189,281 +119,28 @@ alongside its fields.
|
||||
"simplexContact": ["https://smp16.simplex.im/a#…", "https://smp11…"], // primary first, fallbacks after
|
||||
"simplexChannel": [],
|
||||
"eth": null, "btc": "bc1q…", "xmr": "4ANz…", "dot": "139G…",
|
||||
"owner": "0xd83b…", "resolver": "0x80fa…",
|
||||
"status": "registered", // registered | grace | expired | unregistered | unknown
|
||||
"expires": 1780000000, // Unix seconds; when the registration ends
|
||||
"graceEnds": 1787776000, // expires + GRACE_PERIOD; last moment the owner can renew
|
||||
"reasonCode": null, // set when the name is held back as well
|
||||
"reason": null // set when the name is held back as well
|
||||
"owner": "0xd83b…", "resolver": "0x80fa…"
|
||||
}
|
||||
```
|
||||
|
||||
`simplexContact`/`simplexChannel` are arrays (a name can advertise multiple SMP
|
||||
servers; clients try them in order). On-chain they're a single `;`-separated
|
||||
servers; clients try them in order). On-chain they're a single comma-separated
|
||||
text record; the resolver splits/trims/drops-empties. Address encodings are
|
||||
canonical per chain (EIP-55 / bech32 / SS58 / Monero-base58). Subnames work
|
||||
identically (`bar.foobar.testing`).
|
||||
|
||||
#### v1 registration status and expiry
|
||||
|
||||
A response carries `status`, `expires` and `graceEnds` whenever the resolver
|
||||
read them, a successful resolve included, so a client that has just resolved a
|
||||
name already knows when it expires. Both timestamps are Unix seconds, and
|
||||
`null` when they could not be read.
|
||||
|
||||
| `status` | Meaning |
|
||||
|---|---|
|
||||
| `registered` | live; `expires` is when that ends |
|
||||
| `grace` | lapsed, but only the previous owner may renew it, until `graceEnds` |
|
||||
| `expired` | lapsed and past grace; anyone may register it |
|
||||
| `unregistered` | never registered, and free to take |
|
||||
| `unknown` | no `SNRC_REGISTRAR_<TLD>` configured, so status could not be read |
|
||||
|
||||
A reservation is orthogonal to the status: a name held back by the registry
|
||||
carries `reasonCode` and `reason` whether or not it is registered.
|
||||
|
||||
`grace` and `expired` are told apart by the registrar's own `available(id)`
|
||||
rule, `expires + GRACE_PERIOD < now`. `GRACE_PERIOD` is read from the contract
|
||||
rather than assumed, and `now` is the latest block's timestamp rather than the
|
||||
host clock, which the registrar compares against too, so a machine with a wrong
|
||||
clock cannot misreport a registration. That rule alone is not enough: it also
|
||||
holds for a name nobody ever registered (`0 + GRACE_PERIOD < now`), so a zero
|
||||
expiry is what separates *never registered* from *registered and since
|
||||
released*.
|
||||
|
||||
A subname reports the status of the 2LD above it, which is only as good as the
|
||||
name it sits under. A subname nobody created answers 404 `unregistered`.
|
||||
|
||||
#### v1 errors
|
||||
|
||||
Every non-2xx body carries two fields: `error` is a fixed code to branch on,
|
||||
and `message` is a sentence for a human. Match on `error`, never on `message`,
|
||||
which is free to change.
|
||||
|
||||
```jsonc
|
||||
{"name": "nope.testing", "error": "unregistered",
|
||||
"message": "this name has never been registered",
|
||||
"status": "unregistered", "expires": null, "graceEnds": null}
|
||||
```
|
||||
|
||||
The codes are `tldNotConfigured`, `notFullyQualified`, `unregistered`,
|
||||
`expired`, `noSuchRoute` and `upstreamError`. When the registration is what went
|
||||
wrong, `error` and `status` hold the same value, so one field is enough to read.
|
||||
|
||||
`upstreamError` says only which exception type the RPC call raised. The text
|
||||
goes to the resolver's log instead, because `SNRC_RPC` can carry a provider key
|
||||
and the exception can carry the URL it failed on. It is also the answer
|
||||
when a registrar, controller or oracle address has no contract behind it: the
|
||||
empty reply is refused rather than read as zero, which would make every name
|
||||
look free.
|
||||
|
||||
#### v1 status codes
|
||||
### Status codes
|
||||
|
||||
| Status | Meaning |
|
||||
|---|---|
|
||||
| 200 | resolved (`status` is `registered` or `grace`, or `unknown` when no registrar is configured) |
|
||||
| 200 | resolved |
|
||||
| 400 | TLD not configured, or not a fully-qualified name |
|
||||
| 404 | `unregistered` |
|
||||
| 410 | `expired`: lapsed and past grace, so anyone may take it |
|
||||
| 404 | name has no resolver set on the registry |
|
||||
| 502 | upstream RPC error / reth not synced |
|
||||
|
||||
### Querying by labelhash
|
||||
### Configuring registries
|
||||
|
||||
A client asking whether a name is free is usually about to register it, and
|
||||
whoever runs the resolver could register it first. To avoid that, send the
|
||||
keccak hash of the label in ENS's `[<64 hex>]` form instead of the label:
|
||||
|
||||
```sh
|
||||
# instead of /resolve/acme.testing
|
||||
curl -s "http://127.0.0.1:8000/resolve/[$(printf acme | keccak-256sum | cut -d' ' -f1)].testing"
|
||||
```
|
||||
|
||||
namehash is `keccak(parent || keccak(label))`, so this reaches the same node and
|
||||
returns the same record. The registrar keys `nameExpires` and `reservedNames` on
|
||||
the labelhash too, so the status fields do not need the label either. The
|
||||
resolver learns the name only by guessing the label and hashing it.
|
||||
|
||||
Only the second-level label is a registry key, and `status` decodes a bracket
|
||||
there at any depth. The record does not: a bracket is decoded only in a
|
||||
two-label name, so `sub.[<hash>].testing` is not a supported query. Subname
|
||||
labels stay text; a bracket label left of the 2LD is an ordinary label. Routers
|
||||
from v22 send every 2LD this way, so a registrable name normally never reaches
|
||||
this service.
|
||||
|
||||
On v2, read `type`: only `available` means the name is free. On v1, read
|
||||
`status`: a name is free on `unregistered` (404) and on `expired` (410), every
|
||||
other status means somebody holds it, and a `reasonCode` means the registry will
|
||||
refuse it whatever the status says.
|
||||
|
||||
The hash must be keccak-256. `openssl dgst -sha3-256` and `sha3sum` compute
|
||||
SHA3-256, a different function that returns 64 valid-looking hex characters
|
||||
pointing at the wrong node.
|
||||
|
||||
The resolver lowercases the query before matching, so uppercase hex works too.
|
||||
Clients that refuse raw brackets in a path can percent-encode them as `%5B` and
|
||||
`%5D`.
|
||||
|
||||
Brackets cannot collide with a real name: they are invalid in a normalised ENS
|
||||
name, and a `[<64 hex>]` label is 66 bytes against the registrar's
|
||||
`maxLabelLength` of 63. A plain `0x…` label is not treated as a hash, since that
|
||||
is an ordinary, registrable name.
|
||||
|
||||
Only 2LDs can be queried this way, as only a 2LD can be raced for: subnames are
|
||||
created by the 2LD's owner. A bracket label in a subname is hashed as written,
|
||||
so it points at a node nobody can own. ENS tooling accepts the bracketed form at
|
||||
any depth; this resolver does not, on purpose.
|
||||
|
||||
This hides interest in a name and nothing else: the registration itself is
|
||||
public, and commit-reveal covers that step. A short or well-known label is easy
|
||||
to guess by hashing candidates, and the reveal publishes the labelhash, so an
|
||||
operator who logged the query can match it to the name afterwards.
|
||||
|
||||
### What a name costs
|
||||
|
||||
The controller's `prices()` names the price oracle, so no extra configuration is
|
||||
needed beyond `SNRC_CONTROLLER_<TLD>`. A `SimplexPriceOracle` exposes its curve
|
||||
through `prices()`, in US cents per year, which is the unit this API carries.
|
||||
|
||||
An ENS-shaped oracle exposes only `price1Letter()`..`price6Letter()`, in attoUSD
|
||||
per second, and charges a premium on a lapsed name that it does not expose. A
|
||||
quote from one is therefore only safe for a name that was never registered: an
|
||||
`expired` name gets no price rather than one below what the registrar charges.
|
||||
|
||||
**Set `SNRC_CONTROLLER_<TLD>` wherever `SNRC_REGISTRAR_<TLD>` is.** Without a
|
||||
controller there is no oracle, so no name can be priced.
|
||||
|
||||
**Upgrade this service before the routers that query it.** Routers from SMP v22
|
||||
call `/v2/resolve`, which an older resolver does not serve. Every name then
|
||||
answers `ERR NAME RESOLVER "HTTP 404"` until this service is upgraded, while
|
||||
`/health` still reports it as ready.
|
||||
|
||||
### Why a name is reserved
|
||||
|
||||
v2 carries the controller's reason as `reservedReason`. v1 carries the same word
|
||||
as `reasonCode`, plus `reason`, an English sentence for a human reading this API.
|
||||
Clients should branch on the word and phrase it themselves, in the user's
|
||||
language.
|
||||
|
||||
| `reasonCode` | Meaning |
|
||||
|---|---|
|
||||
| `internal` | reserved for SimpleX |
|
||||
| `trademark` | reserved to protect a trademark |
|
||||
| `community` | reserved for the community |
|
||||
| `unknown` | a reason added to the contract after this resolver; still reserved |
|
||||
|
||||
These are `SimplexController.Reason`, where 0 means not reserved. A controller
|
||||
from before the enum stores a boolean, whose `true` decodes as 1, which is why
|
||||
1 reads as `internal`, so nothing needs migrating.
|
||||
|
||||
### Configuring addresses
|
||||
|
||||
The resolver reads three contracts, each configured per TLD.
|
||||
|
||||
The **registry** answers who owns a node, and `/resolve` reads the records from
|
||||
it. The **registrar** (ERC-721) holds `nameExpires` and `GRACE_PERIOD`, which
|
||||
is where every expiry field comes from, and `labelOf`, which is how a hashed
|
||||
query is answered with a name. A name registered without recording its label
|
||||
cannot answer one, and `/v2/resolve` refuses it rather than answer with a name
|
||||
the client will reject. With no registrar for a TLD, `/resolve` still works and
|
||||
reports `"status": "unknown"`. The **controller** holds
|
||||
`reservedNames`, which is where `reasonCode` comes from. With no controller a
|
||||
held-back name reads as not reserved, and no name can be priced.
|
||||
|
||||
All three default to the mainnet `.testing` deployment. `.simplex` is unset
|
||||
until it is deployed.
|
||||
|
||||
The controller default is the **proxy**, not `SimplexControllerImpl`. Storage
|
||||
lives in the proxy, so the implementation address answers nothing. The two
|
||||
deployment files use different names for that proxy:
|
||||
`deployments.mainnet.testing.json` records it under the ENS role name
|
||||
`ETHRegistrarController`, and `verification.mainnet.testing.json` calls it
|
||||
`SimplexControllerProxy`. Both are the same address, and it is the one used
|
||||
here.
|
||||
|
||||
To override any of them, set `SNRC_REGISTRY_<TLD>`, `SNRC_REGISTRAR_<TLD>` or
|
||||
`SNRC_CONTROLLER_<TLD>` on the `resolver` service in `docker-compose.yml`, or
|
||||
as env vars when you run the script directly.
|
||||
|
||||
### Load and scaling
|
||||
|
||||
A lookup reads the chain in at most three rounds: the name's status and
|
||||
registry entries, then its record from its resolver, then prices for a name
|
||||
that is free. The node runs the calls of a JSON-RPC batch one after another,
|
||||
so the contract reads of a round go to the chain as one `eth_call` to
|
||||
[Multicall3](https://github.com/mds1/multicall). A node that is slow for a
|
||||
moment therefore delays a lookup a few times, not once per read. Connections to
|
||||
the node are kept open and reused.
|
||||
|
||||
The resolver answers HTTP/1.1, so each smp-server keeps its connections to it
|
||||
open instead of connecting for every lookup. An idle connection is closed after
|
||||
60 s; the smp-server drops its own idle ones after 30 s, so it closes them first.
|
||||
|
||||
Each access log line ends with how long the request took, so slow requests
|
||||
show up in `docker compose logs resolver`.
|
||||
|
||||
| Variable | Default | Meaning |
|
||||
|---|---|---|
|
||||
| `SNRC_WORKERS` | CPU count, at most 4 | processes sharing the port. If one exits, the others stop, so the container restarts |
|
||||
| `SNRC_RPC_TIMEOUT` | `5` | seconds to wait for each request to the node; the smp-server gives up after 3 |
|
||||
| `SNRC_MULTICALL` | `0xcA11bde05977b3631167028862bE2a173976CA11` | Multicall3 address. If it does not answer, each round is sent as a plain batch and the log says so once |
|
||||
|
||||
### Logs
|
||||
|
||||
Each event is one line: UTC time, level, event name, then `key=value` fields.
|
||||
|
||||
```
|
||||
2026-09-26T08:06:19.992Z INFO request client=203.0.113.7 worker=12 method=GET path=/v2/resolve/[4fdd…].testing status=200 bytes=701 ms=35
|
||||
2026-09-26T08:06:20.051Z WARN upstream_error name=foobar.testing error=ConnectionRefusedError message="[Errno 111] Connection refused"
|
||||
```
|
||||
|
||||
| Event | Level | Meaning |
|
||||
|---|---|---|
|
||||
| `listening` | info | started: address, workers, RPC endpoint, registries, trusted proxies |
|
||||
| `request` | info | one answered request; `/health` only at debug |
|
||||
| `upstream_error` | warn | a read from the node failed; the caller got 502 |
|
||||
| `multicall_unavailable` | warn | Multicall3 did not answer; rounds are sent as plain batches |
|
||||
| `client_gone` | warn | the caller hung up before the answer, usually an smp-server past its timeout |
|
||||
| `http_error` | warn | a malformed request; idle keep-alive timeouts only at debug |
|
||||
| `request_failed` | error | a request failed unexpectedly, with its traceback |
|
||||
| `worker_exited`, `worker_failed` | error | a worker stopped; the others stop too, so the container restarts |
|
||||
| `stopping` | info | stopped by a signal |
|
||||
|
||||
| Variable | Default | Meaning |
|
||||
|---|---|---|
|
||||
| `SNRC_LOG_FORMAT` | `text` | `text` (`key=value`) or `json`, one object per line |
|
||||
| `SNRC_LOG_COLOR` | `auto` | `auto` colours only a terminal, `always`, `never`. Docker output is no terminal, so set `always` for coloured `docker compose logs` |
|
||||
| `SNRC_LOG_LEVEL` | `info` | `debug` also logs health checks and idle connections closing |
|
||||
| `SNRC_TRUSTED_PROXIES` | none | addresses or CIDRs of reverse proxies whose `X-Forwarded-For` names the client |
|
||||
|
||||
With docker compose, set these, and those in "Load and scaling", in `.env`
|
||||
(`.env.example` lists them).
|
||||
|
||||
Behind a reverse proxy such as Caddy on the host, the resolver sees the Docker
|
||||
network's gateway rather than the client. Trust that gateway, and the logged
|
||||
client is the last `X-Forwarded-For` address no trusted proxy added, so a caller
|
||||
through the proxy cannot claim another address. Everything else on the host that
|
||||
connects to `127.0.0.1:8000`, the smp-servers included, arrives through the same
|
||||
gateway and could set the logged address too; run the proxy in the compose network
|
||||
and trust only its address to avoid that.
|
||||
|
||||
The gateway is
|
||||
`docker network inspect resolver_default --format '{{(index .IPAM.Config 0).Gateway}}'`,
|
||||
with the network named after the compose project. It changes when the network is
|
||||
recreated, as `docker compose down` does, so trust one of, in `.env`:
|
||||
|
||||
- `SNRC_TRUSTED_PROXIES=172.16.0.0/12`: Docker's default pools for compose
|
||||
networks, whatever subnet it picks. This also trusts every other container on the
|
||||
host. Add the gateway too if Docker ever gives the network a `192.168.x.x` one.
|
||||
- a pinned subnet, so the gateway stays fixed, and only that gateway:
|
||||
|
||||
```yaml
|
||||
networks:
|
||||
default:
|
||||
ipam:
|
||||
config:
|
||||
- subnet: 172.30.0.0/24
|
||||
gateway: 172.30.0.1
|
||||
```
|
||||
|
||||
in `docker-compose.yml`, with `SNRC_TRUSTED_PROXIES=172.30.0.1`, on a subnet no
|
||||
other network on the host uses.
|
||||
Defaults to mainnet `.testing` (`0x03f438…`); `.simplex` is unset until
|
||||
deployed. Override per TLD via env on the `resolver` service in
|
||||
`docker-compose.yml` (`SNRC_REGISTRY_TESTING` / `SNRC_REGISTRY_SIMPLEX`), or as
|
||||
env vars for the standalone script.
|
||||
|
||||
@@ -38,14 +38,14 @@ services:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
if [ -d /home/user/nimbus-eth2/build/data/${NETWORK:-mainnet}/db ]; then
|
||||
if [ -d /home/user/nimbus-eth2/build/data/${NETWORK}/db ]; then
|
||||
echo "Nimbus data dir already initialised — skipping checkpoint sync";
|
||||
exit 0;
|
||||
fi;
|
||||
/home/user/nimbus-eth2/build/nimbus_beacon_node trustedNodeSync \
|
||||
--network=${NETWORK:-mainnet} \
|
||||
--data-dir=/home/user/nimbus-eth2/build/data/${NETWORK:-mainnet} \
|
||||
--trusted-node-url=${TRUSTED_NODE_URL:-https://mainnet-checkpoint-sync.attestant.io} \
|
||||
--network=${NETWORK} \
|
||||
--data-dir=/home/user/nimbus-eth2/build/data/${NETWORK} \
|
||||
--trusted-node-url=${TRUSTED_NODE_URL} \
|
||||
--backfill=false
|
||||
restart: "no"
|
||||
|
||||
@@ -69,8 +69,8 @@ services:
|
||||
echo "Reth data already initialised — skipping snapshot download";
|
||||
exit 0;
|
||||
fi;
|
||||
echo "Downloading Reth ${NETWORK:-mainnet} --minimal snapshot...";
|
||||
reth download --datadir /data --chain ${NETWORK:-mainnet} --minimal && \
|
||||
echo "Downloading Reth ${NETWORK} --minimal snapshot...";
|
||||
reth download --datadir /data --chain ${NETWORK} --minimal && \
|
||||
touch /data/.snapshot-done && \
|
||||
echo "Snapshot download complete"
|
||||
restart: "no"
|
||||
@@ -92,7 +92,7 @@ services:
|
||||
command: >
|
||||
node
|
||||
--datadir /data
|
||||
--chain ${NETWORK:-mainnet}
|
||||
--chain ${NETWORK}
|
||||
--minimal
|
||||
--authrpc.jwtsecret /jwt/jwt.hex
|
||||
--authrpc.addr 0.0.0.0 --authrpc.port 8551
|
||||
@@ -117,8 +117,8 @@ services:
|
||||
- "9000:9000/udp"
|
||||
- "127.0.0.1:5052:5052"
|
||||
command: >
|
||||
--network=${NETWORK:-mainnet}
|
||||
--data-dir=/home/user/nimbus-eth2/build/data/${NETWORK:-mainnet}
|
||||
--network=${NETWORK}
|
||||
--data-dir=/home/user/nimbus-eth2/build/data/${NETWORK}
|
||||
--el=http://reth:8551
|
||||
--jwt-secret=/jwt/jwt.hex
|
||||
--non-interactive
|
||||
@@ -131,7 +131,6 @@ services:
|
||||
# smp-server points its [NAMES] resolver_endpoint at this URL.
|
||||
# To change the host port, edit the LEFT side of the port mapping below.
|
||||
resolver:
|
||||
image: snrc-resolve:local
|
||||
build:
|
||||
context: ./service
|
||||
dockerfile: Dockerfile
|
||||
@@ -151,24 +150,8 @@ services:
|
||||
# only if you're deploying against a different network or contract.
|
||||
# SNRC_REGISTRY_TESTING: 0x...
|
||||
# SNRC_REGISTRY_SIMPLEX: 0x...
|
||||
# Registrar and controller, same cascade. Without the registrar `status`
|
||||
# is "unknown"; without the controller a reserved name is "unregistered".
|
||||
# SNRC_REGISTRAR_TESTING: 0x...
|
||||
# SNRC_REGISTRAR_SIMPLEX: 0x...
|
||||
# SNRC_CONTROLLER_TESTING: 0x...
|
||||
# SNRC_CONTROLLER_SIMPLEX: 0x...
|
||||
# Set in .env (see .env.example); unset or empty means the resolver's default.
|
||||
SNRC_WORKERS: ${SNRC_WORKERS:-}
|
||||
SNRC_RPC_TIMEOUT: ${SNRC_RPC_TIMEOUT:-}
|
||||
SNRC_LOG_FORMAT: ${SNRC_LOG_FORMAT:-}
|
||||
SNRC_LOG_COLOR: ${SNRC_LOG_COLOR:-}
|
||||
SNRC_LOG_LEVEL: ${SNRC_LOG_LEVEL:-}
|
||||
SNRC_TRUSTED_PROXIES: ${SNRC_TRUSTED_PROXIES:-}
|
||||
ports:
|
||||
- "127.0.0.1:8000:8000"
|
||||
# a line per lookup; the local driver keeps 5 rotated files of 20 MB
|
||||
logging:
|
||||
driver: local
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
.venv
|
||||
**/__pycache__
|
||||
.pytest_cache
|
||||
.ruff_cache
|
||||
test_snrc_resolve.py
|
||||
@@ -1 +0,0 @@
|
||||
3.13
|
||||
@@ -1,10 +1,9 @@
|
||||
# syntax=docker/dockerfile:1.7
|
||||
# ---------- builder ----------
|
||||
# The official uv image (Astral) on top of a slim Python base installs the
|
||||
# locked dependencies into a portable .venv for the runtime stage.
|
||||
# Pinned to the uv that wrote uv.lock (its bookworm images stopped at 0.9.30), on the
|
||||
# runtime's Debian release, so the .venv's Python path exists there.
|
||||
FROM ghcr.io/astral-sh/uv:0.12.16-python3.13-trixie-slim AS builder
|
||||
# Use the official uv image (Astral) on top of a slim Python base.
|
||||
# uv resolves and installs the lockfile-free pyproject.toml in seconds and
|
||||
# produces a portable .venv we can copy into the runtime stage.
|
||||
FROM ghcr.io/astral-sh/uv:python3.13-bookworm-slim AS builder
|
||||
|
||||
ENV UV_LINK_MODE=copy \
|
||||
UV_COMPILE_BYTECODE=1 \
|
||||
@@ -13,17 +12,17 @@ ENV UV_LINK_MODE=copy \
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Dependencies first (separate layer), exactly as locked: script edits don't bust this cache.
|
||||
COPY pyproject.toml uv.lock .python-version ./
|
||||
# Install deps first (separate layer) — script edits won't bust this cache.
|
||||
COPY pyproject.toml ./
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
uv sync --frozen --no-dev --no-install-project
|
||||
uv sync --no-dev --no-install-project
|
||||
|
||||
# Script is added after the dep layer for cache friendliness.
|
||||
COPY snrc-resolve.py ./
|
||||
|
||||
# ---------- runtime ----------
|
||||
# Slim runtime — only the venv + script. No uv, no apt.
|
||||
FROM python:3.13-slim-trixie AS runtime
|
||||
FROM python:3.13-slim AS runtime
|
||||
|
||||
ENV PYTHONUNBUFFERED=1 \
|
||||
PYTHONDONTWRITEBYTECODE=1 \
|
||||
@@ -34,15 +33,15 @@ RUN groupadd --system --gid 10001 snrc && \
|
||||
useradd --system --uid 10001 --gid snrc --no-create-home --shell /usr/sbin/nologin snrc
|
||||
|
||||
WORKDIR /app
|
||||
# owned by root, so the service cannot rewrite its own code
|
||||
COPY --from=builder /app /app
|
||||
COPY --from=builder --chown=snrc:snrc /app /app
|
||||
|
||||
USER snrc:snrc
|
||||
|
||||
EXPOSE 8000
|
||||
|
||||
# Liveness check hits the script's own /health route. Three failures mark the
|
||||
# container unhealthy; compose does not restart it for that, but `docker ps` shows it.
|
||||
# Liveness check hits the script's own /health route. ThreadingHTTPServer is
|
||||
# fast enough that 3s is generous for a localhost probe; restart if it stops
|
||||
# responding entirely.
|
||||
HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
|
||||
CMD ["python", "-c", "import urllib.request, sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=3).status == 200 else 1)"]
|
||||
|
||||
|
||||
@@ -9,14 +9,5 @@ dependencies = [
|
||||
"eth-hash[pycryptodome]>=0.7",
|
||||
]
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"pytest>=8",
|
||||
"ruff>=0.6",
|
||||
]
|
||||
|
||||
[tool.uv]
|
||||
package = false
|
||||
|
||||
[tool.ruff.lint]
|
||||
select = ["E9", "F", "B"]
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
Generated
-156
@@ -1,156 +0,0 @@
|
||||
version = 1
|
||||
revision = 3
|
||||
requires-python = ">=3.11"
|
||||
|
||||
[[package]]
|
||||
name = "colorama"
|
||||
version = "0.4.6"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "eth-hash"
|
||||
version = "0.8.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/3c/f5/c67fc24f2f676aa9b7ab29679d44f113f314c817207cd4319353356f62da/eth_hash-0.8.0.tar.gz", hash = "sha256:b009752b620da2e9c7668014849d1f5fadbe4f138603f1871cc5d4ca706896b1", size = 12225, upload-time = "2026-03-25T16:36:55.099Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/87/87/b36792150ca0b28e4df683a34be15a61461ca0e349e5b5cf3ec8f694edb9/eth_hash-0.8.0-py3-none-any.whl", hash = "sha256:523718a51b369ab89866b929a5c93c52978cd866ea309192ad980dd8271f9fac", size = 7965, upload-time = "2026-03-25T16:36:54.205Z" },
|
||||
]
|
||||
|
||||
[package.optional-dependencies]
|
||||
pycryptodome = [
|
||||
{ name = "pycryptodome" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "iniconfig"
|
||||
version = "2.3.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "packaging"
|
||||
version = "26.3"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pluggy"
|
||||
version = "1.6.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pycryptodome"
|
||||
version = "3.23.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/8e/a6/8452177684d5e906854776276ddd34eca30d1b1e15aa1ee9cefc289a33f5/pycryptodome-3.23.0.tar.gz", hash = "sha256:447700a657182d60338bab09fdb27518f8856aecd80ae4c6bdddb67ff5da44ef", size = 4921276, upload-time = "2025-05-17T17:21:45.242Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/04/5d/bdb09489b63cd34a976cc9e2a8d938114f7a53a74d3dd4f125ffa49dce82/pycryptodome-3.23.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:0011f7f00cdb74879142011f95133274741778abba114ceca229adbf8e62c3e4", size = 2495152, upload-time = "2025-05-17T17:20:20.833Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a7/ce/7840250ed4cc0039c433cd41715536f926d6e86ce84e904068eb3244b6a6/pycryptodome-3.23.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:90460fc9e088ce095f9ee8356722d4f10f86e5be06e2354230a9880b9c549aae", size = 1639348, upload-time = "2025-05-17T17:20:23.171Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ee/f0/991da24c55c1f688d6a3b5a11940567353f74590734ee4a64294834ae472/pycryptodome-3.23.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4764e64b269fc83b00f682c47443c2e6e85b18273712b98aa43bcb77f8570477", size = 2184033, upload-time = "2025-05-17T17:20:25.424Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/54/16/0e11882deddf00f68b68dd4e8e442ddc30641f31afeb2bc25588124ac8de/pycryptodome-3.23.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eb8f24adb74984aa0e5d07a2368ad95276cf38051fe2dc6605cbcf482e04f2a7", size = 2270142, upload-time = "2025-05-17T17:20:27.808Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d5/fc/4347fea23a3f95ffb931f383ff28b3f7b1fe868739182cb76718c0da86a1/pycryptodome-3.23.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d97618c9c6684a97ef7637ba43bdf6663a2e2e77efe0f863cce97a76af396446", size = 2309384, upload-time = "2025-05-17T17:20:30.765Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6e/d9/c5261780b69ce66d8cfab25d2797bd6e82ba0241804694cd48be41add5eb/pycryptodome-3.23.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9a53a4fe5cb075075d515797d6ce2f56772ea7e6a1e5e4b96cf78a14bac3d265", size = 2183237, upload-time = "2025-05-17T17:20:33.736Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5a/6f/3af2ffedd5cfa08c631f89452c6648c4d779e7772dfc388c77c920ca6bbf/pycryptodome-3.23.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:763d1d74f56f031788e5d307029caef067febf890cd1f8bf61183ae142f1a77b", size = 2343898, upload-time = "2025-05-17T17:20:36.086Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9a/dc/9060d807039ee5de6e2f260f72f3d70ac213993a804f5e67e0a73a56dd2f/pycryptodome-3.23.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:954af0e2bd7cea83ce72243b14e4fb518b18f0c1649b576d114973e2073b273d", size = 2269197, upload-time = "2025-05-17T17:20:38.414Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f9/34/e6c8ca177cb29dcc4967fef73f5de445912f93bd0343c9c33c8e5bf8cde8/pycryptodome-3.23.0-cp313-cp313t-win32.whl", hash = "sha256:257bb3572c63ad8ba40b89f6fc9d63a2a628e9f9708d31ee26560925ebe0210a", size = 1768600, upload-time = "2025-05-17T17:20:40.688Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e4/1d/89756b8d7ff623ad0160f4539da571d1f594d21ee6d68be130a6eccb39a4/pycryptodome-3.23.0-cp313-cp313t-win_amd64.whl", hash = "sha256:6501790c5b62a29fcb227bd6b62012181d886a767ce9ed03b303d1f22eb5c625", size = 1799740, upload-time = "2025-05-17T17:20:42.413Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5d/61/35a64f0feaea9fd07f0d91209e7be91726eb48c0f1bfc6720647194071e4/pycryptodome-3.23.0-cp313-cp313t-win_arm64.whl", hash = "sha256:9a77627a330ab23ca43b48b130e202582e91cc69619947840ea4d2d1be21eb39", size = 1703685, upload-time = "2025-05-17T17:20:44.388Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/db/6c/a1f71542c969912bb0e106f64f60a56cc1f0fabecf9396f45accbe63fa68/pycryptodome-3.23.0-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:187058ab80b3281b1de11c2e6842a357a1f71b42cb1e15bce373f3d238135c27", size = 2495627, upload-time = "2025-05-17T17:20:47.139Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6e/4e/a066527e079fc5002390c8acdd3aca431e6ea0a50ffd7201551175b47323/pycryptodome-3.23.0-cp37-abi3-macosx_10_9_x86_64.whl", hash = "sha256:cfb5cd445280c5b0a4e6187a7ce8de5a07b5f3f897f235caa11f1f435f182843", size = 1640362, upload-time = "2025-05-17T17:20:50.392Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/50/52/adaf4c8c100a8c49d2bd058e5b551f73dfd8cb89eb4911e25a0c469b6b4e/pycryptodome-3.23.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:67bd81fcbe34f43ad9422ee8fd4843c8e7198dd88dd3d40e6de42ee65fbe1490", size = 2182625, upload-time = "2025-05-17T17:20:52.866Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5f/e9/a09476d436d0ff1402ac3867d933c61805ec2326c6ea557aeeac3825604e/pycryptodome-3.23.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c8987bd3307a39bc03df5c8e0e3d8be0c4c3518b7f044b0f4c15d1aa78f52575", size = 2268954, upload-time = "2025-05-17T17:20:55.027Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f9/c5/ffe6474e0c551d54cab931918127c46d70cab8f114e0c2b5a3c071c2f484/pycryptodome-3.23.0-cp37-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aa0698f65e5b570426fc31b8162ed4603b0c2841cbb9088e2b01641e3065915b", size = 2308534, upload-time = "2025-05-17T17:20:57.279Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/18/28/e199677fc15ecf43010f2463fde4c1a53015d1fe95fb03bca2890836603a/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:53ecbafc2b55353edcebd64bf5da94a2a2cdf5090a6915bcca6eca6cc452585a", size = 2181853, upload-time = "2025-05-17T17:20:59.322Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ce/ea/4fdb09f2165ce1365c9eaefef36625583371ee514db58dc9b65d3a255c4c/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_i686.whl", hash = "sha256:156df9667ad9f2ad26255926524e1c136d6664b741547deb0a86a9acf5ea631f", size = 2342465, upload-time = "2025-05-17T17:21:03.83Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/22/82/6edc3fc42fe9284aead511394bac167693fb2b0e0395b28b8bedaa07ef04/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:dea827b4d55ee390dc89b2afe5927d4308a8b538ae91d9c6f7a5090f397af1aa", size = 2267414, upload-time = "2025-05-17T17:21:06.72Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/59/fe/aae679b64363eb78326c7fdc9d06ec3de18bac68be4b612fc1fe8902693c/pycryptodome-3.23.0-cp37-abi3-win32.whl", hash = "sha256:507dbead45474b62b2bbe318eb1c4c8ee641077532067fec9c1aa82c31f84886", size = 1768484, upload-time = "2025-05-17T17:21:08.535Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/54/2f/e97a1b8294db0daaa87012c24a7bb714147c7ade7656973fd6c736b484ff/pycryptodome-3.23.0-cp37-abi3-win_amd64.whl", hash = "sha256:c75b52aacc6c0c260f204cbdd834f76edc9fb0d8e0da9fbf8352ef58202564e2", size = 1799636, upload-time = "2025-05-17T17:21:10.393Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/18/3d/f9441a0d798bf2b1e645adc3265e55706aead1255ccdad3856dbdcffec14/pycryptodome-3.23.0-cp37-abi3-win_arm64.whl", hash = "sha256:11eeeb6917903876f134b56ba11abe95c0b0fd5e3330def218083c7d98bbcb3c", size = 1703675, upload-time = "2025-05-17T17:21:13.146Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pygments"
|
||||
version = "2.21.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz", hash = "sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", size = 5005329, upload-time = "2026-08-17T08:02:48.824Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pytest"
|
||||
version = "9.1.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "colorama", marker = "sys_platform == 'win32'" },
|
||||
{ name = "iniconfig" },
|
||||
{ name = "packaging" },
|
||||
{ name = "pluggy" },
|
||||
{ name = "pygments" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ruff"
|
||||
version = "0.16.9"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/96/bf/c935ca98e73fe8ce65b87ef08a280c0c1e85295d569228c15e87d8fdfaf1/ruff-0.16.9.tar.gz", hash = "sha256:12b625c6cfba78d285d9f48eda5f053374f1e53cb10ef17342a383750db99161", size = 4948764, upload-time = "2026-09-24T20:37:49.416Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/0d/26/df51322b52ee1ada7eff2d071ea09d11d5c2d1dcc9f02594f5785c4d1635/ruff-0.16.9-py3-none-linux_armv6l.whl", hash = "sha256:95e6f022090368ab3b824c36276839c53b2adf1a3f4c09fefc33dfc400f6da96", size = 10082922, upload-time = "2026-09-24T20:37:13.045Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a5/27/7bf51f5a7aa375e9f339280a303aab44ca75dc1525f1cdc5991761685b0f/ruff-0.16.9-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:a5f27be168556594a86d2f415db0cf43f5291917849318f873c7e2791f7a8c67", size = 10236360, upload-time = "2026-09-24T20:37:16.049Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b6/63/09659283f92f02dff45809da194a70da2f688d87c55d8875c4fae3536072/ruff-0.16.9-py3-none-macosx_11_0_arm64.whl", hash = "sha256:1632eb1d6197f33bd00b1acbc5b71009e89a8895c158e2d2b03a834fac964ab6", size = 9892940, upload-time = "2026-09-24T20:37:17.957Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/24/58/98de1b72ec172f5f8f1731236fe21585b3998bf7dfe9fcc44ae9ba626012/ruff-0.16.9-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b3f951b14d865d5952c89d40a5ca07e87abe24fa5453299878411e127748fb1c", size = 10032114, upload-time = "2026-09-24T20:37:19.942Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c8/7d/f1e17c54ab59d4bad1dce8ee3e22a7a1d0ef4745240decacdcf3832b5bb2/ruff-0.16.9-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:447fc07e1573afff7cb02803462b12b6c8ece7cf10e2cd78565fa6d7a1c0bf8d", size = 9910227, upload-time = "2026-09-24T20:37:21.872Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/34/19/436f647a65075bbd3bab2668b3bdaa5120559b294694018cdcefabbbf30b/ruff-0.16.9-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8a3e039a6a40ed976c491722b60e0ae4a4aa1a86057f540ee7a37a5d19ae9120", size = 10547484, upload-time = "2026-09-24T20:37:24.229Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/03/59/38430a6bc2f6d8095447ac39625cf8b6e9344a47e6c26225c8ba1bff3ffb/ruff-0.16.9-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4684dded7db60aa57cb118fa158630f5feade4af5782903b6053484bdf9bd129", size = 11412367, upload-time = "2026-09-24T20:37:26.307Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c8/bd/bbb6d7fc7f208c8b8c50dd5dc8206e4cfdb1e7a8fb852606a3adf370c880/ruff-0.16.9-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d29c934357e45642fda2f34c0b1f4025b4a6c01e15e4bf0016879d60078a142c", size = 10869787, upload-time = "2026-09-24T20:37:28.35Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/bc/b8/9c543074918061abbefc3bd139bee22de35abedb00dde0f2d27288838962/ruff-0.16.9-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a21713e629d3e5bdb2f5c2def1cc7f04f47fa8e1a7eb0571b4a28e1da64bc728", size = 10406494, upload-time = "2026-09-24T20:37:30.624Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/35/7a/5a8851bd146e7ccf8fd4b003f6c75c11f8fbdb0e60673b45097986b6bf41/ruff-0.16.9-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:7baa24ef5fc8e77aa93879e1d3f43754a01ae488e869f1ae30cf431afd4d2452", size = 10590083, upload-time = "2026-09-24T20:37:32.439Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/87/f0/4c3467188f23f806960b46fa76575a7cd0514c9ba90562650b71efc96980/ruff-0.16.9-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:a41aac6230aadfaa133bdfa1614488531ffa3e0837567ae04c0da2058a9c0f9e", size = 10119151, upload-time = "2026-09-24T20:37:34.581Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/15/34/5a4def5adea572ce6aea0bb64f21f928ee317b80e0db747d7979b01d7261/ruff-0.16.9-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:c2529fb5896d49115b0e9aa8f887490b34bbe76baf879ec2264ac59406869ce7", size = 9911544, upload-time = "2026-09-24T20:37:36.796Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/62/5d/d15ebea7499eef9373318c0ee6ca127832927c6529731f6d48e18dce7ca9/ruff-0.16.9-py3-none-musllinux_1_2_i686.whl", hash = "sha256:41e3870277694177429b56406d65dfbdb2c2802c52b715edaf6a0b829c69d4ee", size = 10269884, upload-time = "2026-09-24T20:37:38.857Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d1/56/c5d3cd119ded7a3c7aba0e961b69cb3df701c91662c98ad694467d060ce1/ruff-0.16.9-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:8adbe4e58af167f767d7b2ba5e83c42e878350796cf78c2f5e14ab9903a92588", size = 10749366, upload-time = "2026-09-24T20:37:41.042Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ac/fe/734ec7527029ac757ecf821f143c9f3fcf69149c21f53a044a899b430f5a/ruff-0.16.9-py3-none-win32.whl", hash = "sha256:0e1dbc2073624dee6618d41d0098690a7244654af746704b64759e12b6b6b385", size = 10152355, upload-time = "2026-09-24T20:37:43.025Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/14/21/26e4643629b3ebb44f0a06f9c9a53058d63d989415f63a9a3c28e2ee7f22/ruff-0.16.9-py3-none-win_amd64.whl", hash = "sha256:6bd40fec8cd4c8a3d4dd589bd8ad4e6320c13c29234159bfd959a40d529d597b", size = 10592965, upload-time = "2026-09-24T20:37:44.944Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/51/60/5fb1a39dbb5ae314d5f59bc7348a63c1d5c20f3cd83914c4b5cb0be31d2d/ruff-0.16.9-py3-none-win_arm64.whl", hash = "sha256:ed1a252039200f57a59eebc063b54beabea67bfbaaca0eeaa7f54b5fbcda2284", size = 10458649, upload-time = "2026-09-24T20:37:46.882Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "snrc-resolve"
|
||||
version = "0.1.0"
|
||||
source = { virtual = "." }
|
||||
dependencies = [
|
||||
{ name = "eth-hash", extra = ["pycryptodome"] },
|
||||
]
|
||||
|
||||
[package.dev-dependencies]
|
||||
dev = [
|
||||
{ name = "pytest" },
|
||||
{ name = "ruff" },
|
||||
]
|
||||
|
||||
[package.metadata]
|
||||
requires-dist = [{ name = "eth-hash", extras = ["pycryptodome"], specifier = ">=0.7" }]
|
||||
|
||||
[package.metadata.requires-dev]
|
||||
dev = [
|
||||
{ name = "pytest", specifier = ">=8" },
|
||||
{ name = "ruff", specifier = ">=0.6" },
|
||||
]
|
||||
+3
-5
@@ -1,7 +1,7 @@
|
||||
cabal-version: 3.0
|
||||
|
||||
name: simplexmq
|
||||
version: 7.1.0.10
|
||||
version: 7.1.0.5
|
||||
synopsis: SimpleXMQ message broker
|
||||
description: This package includes <./docs/Simplex-Messaging-Server.html server>,
|
||||
<./docs/Simplex-Messaging-Client.html client> and
|
||||
@@ -194,8 +194,6 @@ library
|
||||
Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260411_service_certs
|
||||
Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260712_address_dr_rpc
|
||||
Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260823_snd_files_entitlement
|
||||
Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260919_ratchet_verify_codes
|
||||
Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260929_ratchet_indexes
|
||||
else
|
||||
exposed-modules:
|
||||
Simplex.Messaging.Agent.Store.SQLite
|
||||
@@ -250,8 +248,6 @@ library
|
||||
Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260411_service_certs
|
||||
Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260712_address_dr_rpc
|
||||
Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260823_snd_files_entitlement
|
||||
Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260919_ratchet_verify_codes
|
||||
Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260929_ratchet_indexes
|
||||
Simplex.Messaging.Agent.Store.SQLite.Util
|
||||
if flag(client_postgres) || flag(server_postgres)
|
||||
exposed-modules:
|
||||
@@ -281,6 +277,8 @@ library
|
||||
Simplex.Messaging.Server.Main.Init
|
||||
Simplex.Messaging.Server.Web
|
||||
Simplex.Messaging.Server.MsgStore
|
||||
Simplex.Messaging.Server.MsgStore.Journal
|
||||
Simplex.Messaging.Server.MsgStore.Journal.SharedLock
|
||||
Simplex.Messaging.Server.MsgStore.STM
|
||||
Simplex.Messaging.Server.MsgStore.Types
|
||||
Simplex.Messaging.Server.Names
|
||||
|
||||
@@ -17,16 +17,11 @@ module Simplex.FileTransfer.Agent
|
||||
toFSFilePath,
|
||||
-- Receiving files
|
||||
xftpReceiveFile',
|
||||
xftpPrepareReceiveFile',
|
||||
xftpStartReceiveFile',
|
||||
xftpDeleteRcvFile',
|
||||
xftpDeleteRcvFiles',
|
||||
-- Sending files
|
||||
xftpSendFile',
|
||||
xftpSendDescription',
|
||||
xftpPrepareSendFile',
|
||||
xftpPrepareSendDescription',
|
||||
xftpStartSendFile',
|
||||
deleteSndFileInternal,
|
||||
deleteSndFilesInternal,
|
||||
deleteSndFileRemote,
|
||||
@@ -132,12 +127,7 @@ closeXFTPAgent a = do
|
||||
stopWorkers workers = atomically (swapTVar workers M.empty) >>= mapM_ (liftIO . cancelWorker)
|
||||
|
||||
xftpReceiveFile' :: AgentClient -> UserId -> ValidFileDescription 'FRecipient -> Maybe CryptoFileArgs -> Bool -> AM RcvFileId
|
||||
xftpReceiveFile' c userId vfd cfArgs approvedRelays = do
|
||||
fId <- xftpPrepareReceiveFile' c userId vfd cfArgs approvedRelays
|
||||
fId <$ xftpStartReceiveFile' c fId
|
||||
|
||||
xftpPrepareReceiveFile' :: AgentClient -> UserId -> ValidFileDescription 'FRecipient -> Maybe CryptoFileArgs -> Bool -> AM RcvFileId
|
||||
xftpPrepareReceiveFile' c userId (ValidFileDescription fd@FileDescription {redirect}) cfArgs approvedRelays = do
|
||||
xftpReceiveFile' c userId (ValidFileDescription fd@FileDescription {chunks, redirect}) cfArgs approvedRelays = do
|
||||
g <- asks random
|
||||
prefixPath <- lift $ getPrefixPath "rcv.xftp"
|
||||
createDirectory prefixPath
|
||||
@@ -147,7 +137,7 @@ xftpPrepareReceiveFile' c userId (ValidFileDescription fd@FileDescription {redir
|
||||
lift $ createDirectory =<< toFSFilePath relTmpPath
|
||||
lift $ createEmptyFile =<< toFSFilePath relSavePath
|
||||
let saveFile = CryptoFile relSavePath cfArgs
|
||||
case redirect of
|
||||
fId <- case redirect of
|
||||
Nothing -> withStore c $ \db -> createRcvFile db g userId fd relPrefixPath relTmpPath saveFile approvedRelays
|
||||
Just _ -> do
|
||||
-- prepare description paths
|
||||
@@ -159,11 +149,8 @@ xftpPrepareReceiveFile' c userId (ValidFileDescription fd@FileDescription {redir
|
||||
let saveFileRedirect = CryptoFile relSavePathRedirect $ Just cfArgsRedirect
|
||||
-- create download tasks
|
||||
withStore c $ \db -> createRcvFileRedirect db g userId fd relPrefixPath relTmpPathRedirect saveFileRedirect relTmpPath saveFile approvedRelays
|
||||
|
||||
xftpStartReceiveFile' :: AgentClient -> RcvFileId -> AM ()
|
||||
xftpStartReceiveFile' c rcvFileEntityId = do
|
||||
srvs <- withStore c (`startPreparedRcvFile` rcvFileEntityId)
|
||||
lift $ forM_ srvs $ void . getXFTPRcvWorker True c . Just
|
||||
forM_ chunks (downloadChunk c)
|
||||
pure fId
|
||||
|
||||
downloadChunk :: AgentClient -> FileChunk -> AM ()
|
||||
downloadChunk c FileChunk {replicas = (FileChunkReplica {server} : _)} = do
|
||||
@@ -366,11 +353,6 @@ notify c entId cmd = atomically $ writeTBQueue (subQ c) ("", entId, AEvt (sAEnti
|
||||
|
||||
xftpSendFile' :: AgentClient -> UserId -> CryptoFile -> Int -> Maybe Word32 -> AM SndFileId
|
||||
xftpSendFile' c userId file numRecipients storageHours = do
|
||||
fId <- xftpPrepareSendFile' c userId file numRecipients storageHours
|
||||
fId <$ xftpStartSendFile' c fId
|
||||
|
||||
xftpPrepareSendFile' :: AgentClient -> UserId -> CryptoFile -> Int -> Maybe Word32 -> AM SndFileId
|
||||
xftpPrepareSendFile' c userId file numRecipients storageHours = do
|
||||
g <- asks random
|
||||
prefixPath <- lift $ getPrefixPath "snd.xftp"
|
||||
createDirectory prefixPath
|
||||
@@ -378,15 +360,12 @@ xftpPrepareSendFile' c userId file numRecipients storageHours = do
|
||||
key <- atomically $ C.randomSbKey g
|
||||
nonce <- atomically $ C.randomCbNonce g
|
||||
-- saving absolute filePath will not allow to restore file encryption after app update, but it's a short window
|
||||
withStore c $ \db -> createSndFile db g userId file numRecipients relPrefixPath key nonce Nothing storageHours
|
||||
fId <- withStore c $ \db -> createSndFile db g userId file numRecipients relPrefixPath key nonce Nothing storageHours
|
||||
lift . void $ getXFTPSndWorker True c Nothing
|
||||
pure fId
|
||||
|
||||
xftpSendDescription' :: AgentClient -> UserId -> ValidFileDescription 'FRecipient -> Int -> AM SndFileId
|
||||
xftpSendDescription' c userId vfd numRecipients = do
|
||||
fId <- xftpPrepareSendDescription' c userId vfd numRecipients
|
||||
fId <$ xftpStartSendFile' c fId
|
||||
|
||||
xftpPrepareSendDescription' :: AgentClient -> UserId -> ValidFileDescription 'FRecipient -> Int -> AM SndFileId
|
||||
xftpPrepareSendDescription' c userId (ValidFileDescription fdDirect@FileDescription {size, digest}) numRecipients = do
|
||||
xftpSendDescription' c userId (ValidFileDescription fdDirect@FileDescription {size, digest}) numRecipients = do
|
||||
g <- asks random
|
||||
prefixPath <- lift $ getPrefixPath "snd.xftp"
|
||||
createDirectory prefixPath
|
||||
@@ -397,12 +376,9 @@ xftpPrepareSendDescription' c userId (ValidFileDescription fdDirect@FileDescript
|
||||
liftError (FILE . FILE_IO . show) $ CF.writeFile file (LB.fromStrict $ strEncode fdDirect)
|
||||
key <- atomically $ C.randomSbKey g
|
||||
nonce <- atomically $ C.randomCbNonce g
|
||||
withStore c $ \db -> createSndFile db g userId file numRecipients relPrefixPath key nonce (Just RedirectFileInfo {size, digest}) Nothing
|
||||
|
||||
xftpStartSendFile' :: AgentClient -> SndFileId -> AM ()
|
||||
xftpStartSendFile' c sndFileEntityId = do
|
||||
withStore c (`startPreparedSndFile` sndFileEntityId)
|
||||
fId <- withStore c $ \db -> createSndFile db g userId file numRecipients relPrefixPath key nonce (Just RedirectFileInfo {size, digest}) Nothing
|
||||
lift . void $ getXFTPSndWorker True c Nothing
|
||||
pure fId
|
||||
|
||||
resumeXFTPSndWork :: AgentClient -> Maybe XFTPServer -> AM' ()
|
||||
resumeXFTPSndWork = void .: getXFTPSndWorker False
|
||||
|
||||
@@ -441,6 +441,7 @@ cliReceiveFile ReceiveOptions {fileDescription, filePath, retryCount, tempPath,
|
||||
| depth > 0 -> do
|
||||
CryptoFile tmpFile _ <- withExceptT cliCryptoError $ decryptChunks encSize chunkPaths key nonce $ \_ ->
|
||||
fmap CF.plain $ uniqueCombine encPath "redirect.yaml"
|
||||
forM_ chunks $ acknowledgeFileChunk a
|
||||
yaml <- liftIO $ B.readFile tmpFile
|
||||
whenM (doesPathExist encPath) $ removeDirectoryRecursive encPath
|
||||
innerVfd <- either (throwE . CLIError . ("Redirect: invalid file description: " <>)) pure $ strDecode yaml
|
||||
@@ -449,6 +450,7 @@ cliReceiveFile ReceiveOptions {fileDescription, filePath, retryCount, tempPath,
|
||||
Nothing -> do
|
||||
liftIO $ printNoNewLine "Decrypting file..."
|
||||
CryptoFile path _ <- withExceptT cliCryptoError $ decryptChunks encSize chunkPaths key nonce $ fmap CF.plain . getFilePath
|
||||
forM_ chunks $ acknowledgeFileChunk a
|
||||
whenM (doesPathExist encPath) $ removeDirectoryRecursive encPath
|
||||
liftIO $ do
|
||||
printNoNewLine $ "File downloaded: " <> path
|
||||
@@ -477,6 +479,12 @@ cliReceiveFile ReceiveOptions {fileDescription, filePath, retryCount, tempPath,
|
||||
_ -> (`uniqueCombine` name') . (</> "Downloads") =<< getHomeDirectory
|
||||
where
|
||||
name' = T.unpack name
|
||||
acknowledgeFileChunk :: XFTPClientAgent -> FileChunk -> ExceptT CLIError IO ()
|
||||
acknowledgeFileChunk a FileChunk {replicas = replica : _} = do
|
||||
let FileChunkReplica {server, replicaId, replicaKey} = replica
|
||||
c <- withRetry retryCount $ getXFTPServerClient a server
|
||||
withRetry retryCount $ ackXFTPChunk c replicaKey (unChunkReplicaId replicaId)
|
||||
acknowledgeFileChunk _ _ = throwE $ CLIError "chunk has no replicas"
|
||||
|
||||
printProgress :: String -> Int64 -> Int64 -> IO ()
|
||||
printProgress s part total = printNoNewLine $ s <> " " <> show ((part * 100) `div` total) <> "%"
|
||||
|
||||
@@ -29,7 +29,6 @@ module Simplex.FileTransfer.Description
|
||||
YAMLFileDescription (..), -- for tests
|
||||
YAMLServerReplicas (..), -- for tests
|
||||
validateFileDescription,
|
||||
sharedDescriptionHash,
|
||||
groupReplicasByServer,
|
||||
fdSeparator,
|
||||
kb,
|
||||
@@ -73,7 +72,6 @@ import Simplex.FileTransfer.Protocol
|
||||
import Simplex.Messaging.Agent.QueryString
|
||||
import Simplex.Messaging.Agent.Store.DB (Binary (..), FromField (..), ToField (..))
|
||||
import qualified Simplex.Messaging.Crypto as C
|
||||
import Simplex.Messaging.Encoding (Encoding (..))
|
||||
import Simplex.Messaging.Encoding.String
|
||||
import Simplex.Messaging.Parsers (defaultJSON, parseAll)
|
||||
import Simplex.Messaging.Protocol (XFTPServer)
|
||||
@@ -229,13 +227,6 @@ validateFileDescription fd@FileDescription {size, chunks}
|
||||
chunkNos = map (\FileChunk {chunkNo} -> chunkNo) chunks
|
||||
chunksSize = foldl' (\(s :: Int64) FileChunk {chunkSize} -> s + fromIntegral (unFileSize chunkSize)) 0
|
||||
|
||||
sharedDescriptionHash :: FileDescription p -> ByteString
|
||||
sharedDescriptionHash FileDescription {size, digest, key, nonce, chunkSize, chunks} =
|
||||
C.sha512Hash $ smpEncode (unFileSize size, unFileDigest digest, C.unSbKey key, nonce, unFileSize chunkSize) <> foldMap chunkFields chunks
|
||||
where
|
||||
chunkFields FileChunk {chunkNo, chunkSize = cSize, digest = cDigest} =
|
||||
smpEncode (fromIntegral chunkNo :: Int64, unFileSize cSize, unFileDigest cDigest)
|
||||
|
||||
encodeFileDescription :: FileDescription p -> YAMLFileDescription
|
||||
encodeFileDescription FileDescription {party, size, digest, key, nonce, chunkSize, chunks, redirect} =
|
||||
YAMLFileDescription
|
||||
|
||||
@@ -25,6 +25,7 @@ import Control.Monad.Reader
|
||||
import Control.Monad.Trans.Except
|
||||
import qualified Data.Aeson as J
|
||||
import Data.Bifunctor (first)
|
||||
import qualified Data.ByteString.Base64.URL as B64
|
||||
import Data.ByteString.Builder (Builder, byteString)
|
||||
import Data.ByteString.Char8 (ByteString)
|
||||
import qualified Data.ByteString.Char8 as B
|
||||
@@ -62,7 +63,7 @@ import Simplex.Messaging.Crypto.Entitlement (Entitlement (..), EntitlementProof
|
||||
import qualified Simplex.Messaging.Crypto.Lazy as LC
|
||||
import Simplex.Messaging.Encoding
|
||||
import Simplex.Messaging.Encoding.String
|
||||
import Simplex.Messaging.Protocol (BlockingInfo, CommandError (..), EntityId (..), RcvPublicAuthKey, RcvPublicDhKey, RecipientId, SignedTransmission, pattern NoEntity)
|
||||
import Simplex.Messaging.Protocol (BlockingInfo, EntityId (..), RcvPublicAuthKey, RcvPublicDhKey, RecipientId, SignedTransmission, pattern NoEntity)
|
||||
import Simplex.Messaging.Server (controlPortAuth, dummyVerifyCmd, verifyCmdAuthorization)
|
||||
import Simplex.Messaging.Server.Control (CPClientRole (..))
|
||||
import Simplex.Messaging.Server.Expiration
|
||||
@@ -593,7 +594,7 @@ processXFTPRequest ent HTTP2Body {bodyPart} = \case
|
||||
\used -> let used' = used + fromIntegral size in if used' <= quota then (True, used') else (False, used)
|
||||
receive = do
|
||||
path <- asks $ filesPath . config
|
||||
let fPath = path </> serverFileName senderId
|
||||
let fPath = path </> B.unpack (B64.encode $ unEntityId senderId)
|
||||
receiveChunk (XFTPRcvChunkSpec fPath size digest) >>= \case
|
||||
Right () -> do
|
||||
stats <- asks serverStats
|
||||
@@ -645,12 +646,11 @@ processXFTPRequest ent HTTP2Body {bodyPart} = \case
|
||||
|
||||
ackFileReception :: RecipientId -> FileRec -> M s FileResponse
|
||||
ackFileReception rId fr = do
|
||||
pure $ FRErr $ CMD PROHIBITED
|
||||
-- withFileLog (`logAckFile` rId)
|
||||
-- st <- asks fileStore
|
||||
-- liftIO $ deleteRecipient st rId fr
|
||||
-- incFileStat fileDownloadAcks
|
||||
-- pure FROk
|
||||
withFileLog (`logAckFile` rId)
|
||||
st <- asks fileStore
|
||||
liftIO $ deleteRecipient st rId fr
|
||||
incFileStat fileDownloadAcks
|
||||
pure FROk
|
||||
|
||||
deleteServerFile_ :: FileStoreClass s => FileRec -> M s (Either XFTPErrorType ())
|
||||
deleteServerFile_ fr@FileRec {senderId} = do
|
||||
|
||||
@@ -17,7 +17,6 @@ module Simplex.FileTransfer.Server.StoreLog
|
||||
logDeleteFile,
|
||||
logBlockFile,
|
||||
logAckFile,
|
||||
serverFileName,
|
||||
)
|
||||
where
|
||||
|
||||
@@ -25,7 +24,6 @@ import Control.Applicative (optional, (<|>))
|
||||
import Control.Concurrent.STM
|
||||
import Control.Monad.Except
|
||||
import qualified Data.Attoparsec.ByteString.Char8 as A
|
||||
import qualified Data.ByteString.Base64.URL as B64
|
||||
import qualified Data.ByteString.Char8 as B
|
||||
import qualified Data.ByteString.Lazy.Char8 as LB
|
||||
import Data.Composition ((.:), (.::.))
|
||||
@@ -37,11 +35,10 @@ import Simplex.FileTransfer.Protocol (FileInfo (..))
|
||||
import Simplex.FileTransfer.Server.Store
|
||||
import Simplex.FileTransfer.Transport (XFTPErrorType (..))
|
||||
import Simplex.Messaging.Encoding.String
|
||||
import Simplex.Messaging.Protocol (BlockingInfo, EntityId (..), RcvPublicAuthKey, RecipientId, SenderId)
|
||||
import Simplex.Messaging.Protocol (BlockingInfo, RcvPublicAuthKey, RecipientId, SenderId)
|
||||
import Simplex.Messaging.Server.QueueStore (ServerEntityStatus (..))
|
||||
import Simplex.Messaging.Server.StoreLog
|
||||
import Simplex.Messaging.Util (bshow)
|
||||
import System.FilePath (takeFileName)
|
||||
import System.IO
|
||||
|
||||
data FileStoreLogRecord
|
||||
@@ -102,9 +99,6 @@ logBlockFile s fId = logFileStoreRecord s . BlockFile fId
|
||||
logAckFile :: StoreLog 'WriteMode -> RecipientId -> IO ()
|
||||
logAckFile s = logFileStoreRecord s . AckFile
|
||||
|
||||
serverFileName :: SenderId -> FilePath
|
||||
serverFileName = B.unpack . B64.encode . unEntityId
|
||||
|
||||
readWriteFileStore :: FilePath -> STMFileStore -> IO (StoreLog 'WriteMode)
|
||||
readWriteFileStore = readWriteStoreLog readFileStore writeFileStore
|
||||
|
||||
@@ -121,9 +115,7 @@ readFileStore f st = mapM_ (addFileLogRecord . LB.toStrict) . LB.lines =<< LB.re
|
||||
AddFile sId file createdAt expiresAt status
|
||||
| size file > 0 -> addFile st sId file createdAt expiresAt status
|
||||
| otherwise -> pure $ Left SIZE
|
||||
PutFile qId path
|
||||
| takeFileName path == serverFileName qId -> setFilePath st qId path
|
||||
| otherwise -> pure $ Left AUTH
|
||||
PutFile qId path -> setFilePath st qId path
|
||||
AddRecipients sId rcps -> runExceptT $ addRecipients sId rcps
|
||||
DeleteFile sId -> deleteFile st sId
|
||||
BlockFile sId info -> blockFile st sId info True
|
||||
|
||||
@@ -93,8 +93,7 @@ data RcvFile = RcvFile
|
||||
deriving (Show)
|
||||
|
||||
data RcvFileStatus
|
||||
= RFSPrepared
|
||||
| RFSReceiving
|
||||
= RFSReceiving
|
||||
| RFSReceived
|
||||
| RFSDecrypting
|
||||
| RFSComplete
|
||||
@@ -107,7 +106,6 @@ instance ToField RcvFileStatus where toField = toField . textEncode
|
||||
|
||||
instance TextEncoding RcvFileStatus where
|
||||
textDecode = \case
|
||||
"prepared" -> Just RFSPrepared
|
||||
"receiving" -> Just RFSReceiving
|
||||
"received" -> Just RFSReceived
|
||||
"decrypting" -> Just RFSDecrypting
|
||||
@@ -115,7 +113,6 @@ instance TextEncoding RcvFileStatus where
|
||||
"error" -> Just RFSError
|
||||
_ -> Nothing
|
||||
textEncode = \case
|
||||
RFSPrepared -> "prepared"
|
||||
RFSReceiving -> "receiving"
|
||||
RFSReceived -> "received"
|
||||
RFSDecrypting -> "decrypting"
|
||||
@@ -180,8 +177,7 @@ sndFileEncPath :: FilePath -> FilePath
|
||||
sndFileEncPath prefixPath = prefixPath </> "xftp.encrypted"
|
||||
|
||||
data SndFileStatus
|
||||
= SFSPrepared
|
||||
| SFSNew -- db record created
|
||||
= SFSNew -- db record created
|
||||
| SFSEncrypting -- encryption started
|
||||
| SFSEncrypted -- encryption complete
|
||||
| SFSUploading -- all chunk replicas are created on servers
|
||||
@@ -195,7 +191,6 @@ instance ToField SndFileStatus where toField = toField . textEncode
|
||||
|
||||
instance TextEncoding SndFileStatus where
|
||||
textDecode = \case
|
||||
"prepared" -> Just SFSPrepared
|
||||
"new" -> Just SFSNew
|
||||
"encrypting" -> Just SFSEncrypting
|
||||
"encrypted" -> Just SFSEncrypted
|
||||
@@ -204,7 +199,6 @@ instance TextEncoding SndFileStatus where
|
||||
"error" -> Just SFSError
|
||||
_ -> Nothing
|
||||
textEncode = \case
|
||||
SFSPrepared -> "prepared"
|
||||
SFSNew -> "new"
|
||||
SFSEncrypting -> "encrypting"
|
||||
SFSEncrypted -> "encrypted"
|
||||
|
||||
+171
-250
@@ -106,8 +106,7 @@ module Simplex.Messaging.Agent
|
||||
deleteConnection,
|
||||
deleteConnections,
|
||||
getConnectionServers,
|
||||
getConnectionVerifyCodes,
|
||||
getConnectionsVerifyCodes,
|
||||
getConnectionRatchetAdHash,
|
||||
setProtocolServers,
|
||||
setUserEntitlement,
|
||||
checkUserServers,
|
||||
@@ -127,15 +126,10 @@ module Simplex.Messaging.Agent
|
||||
xftpStartWorkers,
|
||||
xftpStartSndWorkers,
|
||||
xftpReceiveFile,
|
||||
xftpPrepareReceiveFile,
|
||||
xftpStartReceiveFile,
|
||||
xftpDeleteRcvFile,
|
||||
xftpDeleteRcvFiles,
|
||||
xftpSendFile,
|
||||
xftpSendDescription,
|
||||
xftpPrepareSendFile,
|
||||
xftpPrepareSendDescription,
|
||||
xftpStartSendFile,
|
||||
xftpDeleteSndFileInternal,
|
||||
xftpDeleteSndFilesInternal,
|
||||
xftpDeleteSndFileRemote,
|
||||
@@ -195,7 +189,7 @@ import Data.Time.Clock
|
||||
import Data.Time.Clock.System (systemToUTCTime)
|
||||
import Data.Traversable (mapAccumL)
|
||||
import Data.Word (Word16, Word32)
|
||||
import Simplex.FileTransfer.Agent (closeXFTPAgent, deleteSndFileInternal, deleteSndFileRemote, deleteSndFilesInternal, deleteSndFilesRemote, startXFTPSndWorkers, startXFTPWorkers, toFSFilePath, xftpDeleteRcvFile', xftpDeleteRcvFiles', xftpPrepareReceiveFile', xftpPrepareSendDescription', xftpPrepareSendFile', xftpReceiveFile', xftpSendDescription', xftpSendFile', xftpStartReceiveFile', xftpStartSendFile')
|
||||
import Simplex.FileTransfer.Agent (closeXFTPAgent, deleteSndFileInternal, deleteSndFileRemote, deleteSndFilesInternal, deleteSndFilesRemote, startXFTPSndWorkers, startXFTPWorkers, toFSFilePath, xftpDeleteRcvFile', xftpDeleteRcvFiles', xftpReceiveFile', xftpSendDescription', xftpSendFile')
|
||||
import Simplex.FileTransfer.Description (ValidFileDescription)
|
||||
import Simplex.FileTransfer.Protocol (FileParty (..))
|
||||
import Simplex.FileTransfer.Types (RcvFileId, SndFileId)
|
||||
@@ -234,7 +228,7 @@ import Simplex.Messaging.Protocol
|
||||
ErrorType (AUTH),
|
||||
MsgBody,
|
||||
MsgFlags (..),
|
||||
NameResponse,
|
||||
NameRecord,
|
||||
NtfServer,
|
||||
ProtoServerWithAuth (..),
|
||||
ProtocolServer (..),
|
||||
@@ -467,7 +461,7 @@ getConnShortLink c = withAgentEnv c .:. getConnShortLink' c
|
||||
-- | Resolve a SimpleX name (PFWD RSLV). The agent owns server selection: it
|
||||
-- picks a names-capable server (ServerRoles.names) from the user's nameSrvs, so
|
||||
-- chat clients just pass the parsed domain.
|
||||
resolveSimplexName :: AgentClient -> NetworkRequestMode -> UserId -> SimplexDomain -> AE NameResponse
|
||||
resolveSimplexName :: AgentClient -> NetworkRequestMode -> UserId -> SimplexDomain -> AE NameRecord
|
||||
resolveSimplexName c nm userId domain = withAgentEnv c $ resolveSimplexName' c nm userId domain
|
||||
{-# INLINE resolveSimplexName #-}
|
||||
|
||||
@@ -490,12 +484,12 @@ changeConnectionUser c oldUserId connId newUserId = withAgentEnv c $ changeConne
|
||||
-- the caller of joinConnection saves connection ID to the database.
|
||||
-- Instead of it we could send confirmation asynchronously, but then it would be harder to report
|
||||
-- "link deleted" (SMP AUTH) interactively, so this approach is simpler overall.
|
||||
prepareConnectionToJoin :: AgentClient -> UserId -> Bool -> ConnectionRequestUri c -> PQSupport -> AE (ConnId, ContactRequestBinding)
|
||||
prepareConnectionToJoin :: AgentClient -> UserId -> Bool -> ConnectionRequestUri c -> PQSupport -> AE ConnId
|
||||
prepareConnectionToJoin c userId enableNtfs = withAgentEnv c .: newConnToJoin c userId "" enableNtfs Nothing
|
||||
{-# INLINE prepareConnectionToJoin #-}
|
||||
|
||||
-- | Create SMP agent connection without queue (to be joined with acceptContact passing invitation ID).
|
||||
prepareConnectionToAccept :: AgentClient -> UserId -> Bool -> InvitationId -> PQSupport -> AE (ConnId, ContactRequestBinding)
|
||||
prepareConnectionToAccept :: AgentClient -> UserId -> Bool -> InvitationId -> PQSupport -> AE ConnId
|
||||
prepareConnectionToAccept c userId enableNtfs = withAgentEnv c .: newConnToAccept c userId "" enableNtfs
|
||||
{-# INLINE prepareConnectionToAccept #-}
|
||||
|
||||
@@ -679,13 +673,9 @@ getConnectionServers c = withAgentEnv c . getConnectionServers' c
|
||||
{-# INLINE getConnectionServers #-}
|
||||
|
||||
-- | get connection ratchet associated data hash for verification (should match peer AD hash)
|
||||
getConnectionVerifyCodes :: AgentClient -> ConnId -> AE ConnVerifyCodes
|
||||
getConnectionVerifyCodes c = withAgentEnv c . getConnectionVerifyCodes' c
|
||||
{-# INLINE getConnectionVerifyCodes #-}
|
||||
|
||||
getConnectionsVerifyCodes :: AgentClient -> [ConnId] -> AE (Map ConnId ConnVerifyCodes)
|
||||
getConnectionsVerifyCodes c = withAgentEnv c . getConnectionsVerifyCodes' c
|
||||
{-# INLINE getConnectionsVerifyCodes #-}
|
||||
getConnectionRatchetAdHash :: AgentClient -> ConnId -> AE ByteString
|
||||
getConnectionRatchetAdHash c = withAgentEnv c . getConnectionRatchetAdHash' c
|
||||
{-# INLINE getConnectionRatchetAdHash #-}
|
||||
|
||||
-- | Test protocol server
|
||||
testProtocolServer :: forall p. ProtocolTypeI p => AgentClient -> NetworkRequestMode -> UserId -> ProtoServerWithAuth p -> IO (Either ProtocolTestFailure (Maybe (Either String ServerPublicInfo)))
|
||||
@@ -774,14 +764,6 @@ xftpReceiveFile :: AgentClient -> UserId -> ValidFileDescription 'FRecipient ->
|
||||
xftpReceiveFile c = withAgentEnv c .:: xftpReceiveFile' c
|
||||
{-# INLINE xftpReceiveFile #-}
|
||||
|
||||
xftpPrepareReceiveFile :: AgentClient -> UserId -> ValidFileDescription 'FRecipient -> Maybe CryptoFileArgs -> Bool -> AE RcvFileId
|
||||
xftpPrepareReceiveFile c = withAgentEnv c .:: xftpPrepareReceiveFile' c
|
||||
{-# INLINE xftpPrepareReceiveFile #-}
|
||||
|
||||
xftpStartReceiveFile :: AgentClient -> RcvFileId -> AE ()
|
||||
xftpStartReceiveFile c = withAgentEnv c . xftpStartReceiveFile' c
|
||||
{-# INLINE xftpStartReceiveFile #-}
|
||||
|
||||
-- | Delete XFTP rcv file (deletes work files from file system and db records)
|
||||
xftpDeleteRcvFile :: AgentClient -> RcvFileId -> IO ()
|
||||
xftpDeleteRcvFile c = withAgentEnv' c . xftpDeleteRcvFile' c
|
||||
@@ -802,18 +784,6 @@ xftpSendDescription :: AgentClient -> UserId -> ValidFileDescription 'FRecipient
|
||||
xftpSendDescription c = withAgentEnv c .:. xftpSendDescription' c
|
||||
{-# INLINE xftpSendDescription #-}
|
||||
|
||||
xftpPrepareSendFile :: AgentClient -> UserId -> CryptoFile -> Int -> Maybe Word32 -> AE SndFileId
|
||||
xftpPrepareSendFile c = withAgentEnv c .:: xftpPrepareSendFile' c
|
||||
{-# INLINE xftpPrepareSendFile #-}
|
||||
|
||||
xftpPrepareSendDescription :: AgentClient -> UserId -> ValidFileDescription 'FRecipient -> Int -> AE SndFileId
|
||||
xftpPrepareSendDescription c = withAgentEnv c .:. xftpPrepareSendDescription' c
|
||||
{-# INLINE xftpPrepareSendDescription #-}
|
||||
|
||||
xftpStartSendFile :: AgentClient -> SndFileId -> AE ()
|
||||
xftpStartSendFile c = withAgentEnv c . xftpStartSendFile' c
|
||||
{-# INLINE xftpStartSendFile #-}
|
||||
|
||||
-- | Delete XFTP snd file internally (deletes work files from file system and db records)
|
||||
xftpDeleteSndFileInternal :: AgentClient -> SndFileId -> IO ()
|
||||
xftpDeleteSndFileInternal c = withAgentEnv' c . deleteSndFileInternal c
|
||||
@@ -1300,7 +1270,7 @@ getConnShortLink' c nm userId = \case
|
||||
deleteLocalInvShortLink' :: AgentClient -> ConnShortLink 'CMInvitation -> AM ()
|
||||
deleteLocalInvShortLink' c (CSLInvitation _ srv linkId _) = withStore' c $ \db -> deleteInvShortLink db srv linkId
|
||||
|
||||
resolveSimplexName' :: AgentClient -> NetworkRequestMode -> UserId -> SimplexDomain -> AM NameResponse
|
||||
resolveSimplexName' :: AgentClient -> NetworkRequestMode -> UserId -> SimplexDomain -> AM NameRecord
|
||||
resolveSimplexName' c nm userId domain = do
|
||||
resolverSrv <- getNextNameServer c userId
|
||||
resolveName c nm userId resolverSrv domain
|
||||
@@ -1325,7 +1295,7 @@ newRcvConnSrv c nm userId connId enableNtfs cMode userLinkData_ clientData pqIni
|
||||
Just d -> do
|
||||
(nonce, qUri, cReq, qd) <- prepareLinkData addrKeys_ (setLinkDataRatchetKeys addrKeys_ d) $ fst e2eKeys
|
||||
(rq, qUri') <- createRcvQueue c nm userId connId srvWithAuth enableNtfs subMode (Just nonce) qd e2eKeys
|
||||
connReqWithShortLink pqInitKeys qUri cReq qUri' rq
|
||||
connReqWithShortLink qUri cReq qUri' (shortLink rq)
|
||||
Nothing -> do
|
||||
let qd = case cMode of SCMContact -> CQRContact Nothing; SCMInvitation -> CQRMessaging Nothing
|
||||
(_rq, qUri) <- createRcvQueue c nm userId connId srvWithAuth enableNtfs subMode Nothing qd e2eKeys
|
||||
@@ -1341,13 +1311,8 @@ newRcvConnSrv c nm userId connId enableNtfs cMode userLinkData_ clientData pqIni
|
||||
SCMInvitation -> do
|
||||
g <- asks random
|
||||
let pqEnc = CR.initialPQEncryption (isJust userLinkData_) pqInitKeys
|
||||
e2eRcvParams <- withStore' c $ \db -> do
|
||||
lockConnForUpdate db connId
|
||||
getRatchetX3dhKeys db connId >>= \case
|
||||
Right keys -> pure $ CR.mkRcvE2ERatchetParams (maxVersion e2eEncryptVRange) keys
|
||||
Left _ -> do
|
||||
(pks, e2eRcvParams) <- CR.generateRcvE2EParams g (maxVersion e2eEncryptVRange) pqEnc
|
||||
e2eRcvParams <$ createRatchetX3dhKeys db connId pks
|
||||
(pks, e2eRcvParams) <- liftIO $ CR.generateRcvE2EParams g (maxVersion e2eEncryptVRange) pqEnc
|
||||
withStore' c $ \db -> createRatchetX3dhKeys db connId pks
|
||||
pure $ CRInvitationUri crData $ toVersionRangeT e2eRcvParams e2eEncryptVRange
|
||||
setLinkDataRatchetKeys :: Maybe AddressRatchetKeys -> UserConnLinkData c -> UserConnLinkData c
|
||||
setLinkDataRatchetKeys ks = \case
|
||||
@@ -1372,19 +1337,23 @@ newRcvConnSrv c nm userId connId enableNtfs cMode userLinkData_ clientData pqIni
|
||||
srvData <- liftError id $ SL.encryptLinkData g k linkData
|
||||
pure $ CQRMessaging $ Just CQRData {linkKey, privSigKey, srvReq = (sndId, srvData)}
|
||||
pure (nonce, qUri, connReq, qd)
|
||||
|
||||
connReqWithShortLink :: CR.InitialKeys -> SMPQueueUri -> ConnectionRequestUri c -> SMPQueueUri -> RcvQueue -> AM (CreatedConnLink c)
|
||||
connReqWithShortLink pqInitKeys qUri cReq qUri' RcvQueue {server = srv, shortLink} = case shortLink of
|
||||
Just ShortLinkCreds {shortLinkId, shortLinkKey}
|
||||
| qUri == qUri' -> pure $ case cReq of
|
||||
CRContactUri _ _ -> CCLink cReq $ Just $ CSLContact SLSServer CCTContact srv shortLinkKey
|
||||
CRInvitationUri crData (CR.E2ERatchetParamsUri vr k1 k2 _) ->
|
||||
let cReq' = case pqInitKeys of
|
||||
CR.IKPQOn -> CRInvitationUri crData $ CR.E2ERatchetParamsUri vr k1 k2 Nothing -- remove PQ keys
|
||||
_ -> cReq -- either PQ is disabled, or disabled for initial request because there is no short link
|
||||
in CCLink cReq' $ Just $ CSLInvitation SLSServer srv shortLinkId shortLinkKey
|
||||
| otherwise -> throwE $ INTERNAL "different rcv queue address"
|
||||
Nothing -> throwE $ INTERNAL "no short link credentials"
|
||||
connReqWithShortLink :: SMPQueueUri -> ConnectionRequestUri c -> SMPQueueUri -> Maybe ShortLinkCreds -> AM (CreatedConnLink c)
|
||||
connReqWithShortLink qUri cReq qUri' shortLink = case shortLink of
|
||||
Just ShortLinkCreds {shortLinkId, shortLinkKey}
|
||||
| qUri == qUri' -> pure $ case cReq of
|
||||
CRContactUri _ _ -> CCLink cReq $ Just $ CSLContact SLSServer CCTContact srv shortLinkKey
|
||||
CRInvitationUri crData (CR.E2ERatchetParamsUri vr k1 k2 _) ->
|
||||
let cReq' = case pqInitKeys of
|
||||
CR.IKPQOn -> CRInvitationUri crData $ CR.E2ERatchetParamsUri vr k1 k2 Nothing -- remove PQ keys
|
||||
_ -> cReq -- either PQ is disabled, or disabled for initial request because there is no short link
|
||||
in CCLink cReq' $ Just $ CSLInvitation SLSServer srv shortLinkId shortLinkKey
|
||||
| otherwise -> throwE $ INTERNAL "different rcv queue address"
|
||||
Nothing ->
|
||||
let updated (ConnReqUriData _ vr _ _) = (ConnReqUriData SSSimplex vr [qUri'] clientData)
|
||||
cReq' = case cReq of
|
||||
CRContactUri crData rk -> CRContactUri (updated crData) rk
|
||||
CRInvitationUri crData e2eParams -> CRInvitationUri (updated crData) e2eParams
|
||||
in pure $ CCLink cReq' Nothing
|
||||
|
||||
newQueueNtfServer :: AM (Maybe NtfServer)
|
||||
newQueueNtfServer = fmap ntfServer_ . readTVarIO . ntfTkn =<< asks ntfSupervisor
|
||||
@@ -1401,47 +1370,30 @@ newQueueNtfSubscription c RcvQueue {userId, connId, server, clientNtfCreds} ntfS
|
||||
ns <- asks ntfSupervisor
|
||||
liftIO $ sendNtfSubCommand ns (NSCCreate, [connId])
|
||||
|
||||
newConnToJoin :: forall c. AgentClient -> UserId -> ConnId -> Bool -> Maybe UTCTime -> ConnectionRequestUri c -> PQSupport -> AM (ConnId, ContactRequestBinding)
|
||||
newConnToJoin :: forall c. AgentClient -> UserId -> ConnId -> Bool -> Maybe UTCTime -> ConnectionRequestUri c -> PQSupport -> AM ConnId
|
||||
newConnToJoin c userId connId enableNtfs serviceRequestExpiresAt cReq pqSupport = case cReq of
|
||||
CRInvitationUri {} ->
|
||||
lift (compatibleInvitationUri cReq) >>= \case
|
||||
Just (_, Compatible e2eRcvParams, aVersion) -> create aVersion $ Right e2eRcvParams
|
||||
Just (_, _, aVersion) -> create aVersion
|
||||
Nothing -> throwE $ AGENT A_VERSION
|
||||
CRContactUri {} ->
|
||||
lift (compatibleContactUri cReq) >>= \case
|
||||
Just (Compatible SMPQueueInfo {queueAddress = SMPQueueAddress {senderId}}, ratchet_, aVersion) ->
|
||||
create aVersion $ case ratchet_ of
|
||||
Just (_, Compatible e2eRcvParams) -> Right e2eRcvParams
|
||||
Nothing -> Left senderId
|
||||
Just (_, _, aVersion) -> create aVersion
|
||||
Nothing -> throwE $ AGENT A_VERSION
|
||||
where
|
||||
create :: Compatible VersionSMPA -> Either SMP.SenderId (CR.RcvE2ERatchetParams 'C.X448) -> AM (ConnId, ContactRequestBinding)
|
||||
create (Compatible connAgentVersion) addrOrParams = do
|
||||
create :: Compatible VersionSMPA -> AM ConnId
|
||||
create (Compatible connAgentVersion) = do
|
||||
g <- asks random
|
||||
maxSupported <- asks $ maxVersion . e2eEncryptVRange . config
|
||||
let cData = ConnData {userId, connId, connAgentVersion, enableNtfs, lastExternalSndId = 0, deleted = False, ratchetSyncState = RSOk, pqSupport, serviceRequestExpiresAt}
|
||||
withStore c $ \db -> runExceptT $ do
|
||||
connId' <- ExceptT $ createNewConn db g cData SCMInvitation
|
||||
binding <- case addrOrParams of
|
||||
Right e2eRcvParams -> CRBRatchet . ratchetVerifyCodes . fst <$> createRatchet_ db g connId' maxSupported pqSupport e2eRcvParams
|
||||
Left senderId -> do
|
||||
let pqEnc = CR.initialPQEncryption False $ CR.joinContactInitialKeys pqSupport
|
||||
(pks, CR.E2ERatchetParams _ k1 k2 kem_) <- liftIO $ CR.generateRcvE2EParams g maxSupported pqEnc
|
||||
liftIO $ CRBRequest (requestCode k1 k2 kem_ senderId) <$ createRatchetX3dhKeys db connId' pks
|
||||
pure (connId', binding)
|
||||
withStore c $ \db -> createNewConn db g cData SCMInvitation
|
||||
|
||||
requestCode :: C.PublicKeyX448 -> C.PublicKeyX448 -> Maybe (CR.RKEMParams 'CR.RKSProposed) -> SMP.SenderId -> ByteString
|
||||
requestCode k1 k2 kem_ sndId = C.sha256Hash $ smpEncode (k1, k2, kem_, sndId)
|
||||
|
||||
newConnToAccept :: AgentClient -> UserId -> ConnId -> Bool -> InvitationId -> PQSupport -> AM (ConnId, ContactRequestBinding)
|
||||
newConnToAccept :: AgentClient -> UserId -> ConnId -> Bool -> InvitationId -> PQSupport -> AM ConnId
|
||||
newConnToAccept c userId connId enableNtfs invId pqSup = do
|
||||
Invitation {connReq} <- withStore c $ \db -> getInvitation db "newConnToAccept" invId
|
||||
case connReq of
|
||||
CRInvitation cReq -> newConnToJoin c userId connId enableNtfs Nothing cReq pqSup
|
||||
CRInvitationDR dr@DRInvitation {ratchetState} -> do
|
||||
ConnData {connId = connId'} <- newConnToAcceptDR c userId connId dr enableNtfs
|
||||
pure (connId', CRBRatchet $ ratchetVerifyCodes ratchetState)
|
||||
|
||||
CRInvitationDR dr -> (\ConnData {connId = connId'} -> connId') <$> newConnToAcceptDR c userId connId dr enableNtfs
|
||||
|
||||
newConnToAcceptDR :: AgentClient -> UserId -> ConnId -> DRInvitation -> Bool -> AM ConnData
|
||||
newConnToAcceptDR c userId connId DRInvitation {agentVersion, pqSupport} enableNtfs = do
|
||||
g <- asks random
|
||||
@@ -1484,7 +1436,7 @@ startJoinInvitation c userId connId sq_ enableNtfs cReqUri pqSupport =
|
||||
(q, _) <- lift $ newSndQueue userId "" qInfo sndKey_
|
||||
withStore c $ \db -> runExceptT $ do
|
||||
liftIO $ lockConnForUpdate db connId
|
||||
(_, e2eSndParams) <- liftIO (getSndRatchet db connId v) >>= either (const $ createRatchet_ db g connId maxSupported pqSupport e2eRcvParams) pure
|
||||
e2eSndParams <- snd <$> createRatchet_ db g connId maxSupported pqSupport e2eRcvParams
|
||||
sq' <- maybe (ExceptT $ updateNewConnSnd db connId q) pure sq_
|
||||
pure ((cData, sq'), (Just e2eSndParams, lnkId_))
|
||||
Nothing -> throwE $ AGENT A_VERSION
|
||||
@@ -1777,7 +1729,7 @@ serviceRequest_ c userId cReqUri@(CRContactUri _ addrKeys_) timeout_ doSend = do
|
||||
when (isNothing addrKeys_) $ throwE $ AGENT $ A_SERVICE ASENotDRAddress
|
||||
reqTimeout <- maybe (asks $ serviceRequestTimeout . config) pure timeout_
|
||||
expiresAt <- addUTCTime reqTimeout <$> liftIO getCurrentTime
|
||||
(connId, _) <- newConnToJoin c userId "" False (Just expiresAt) cReqUri PQSupportOn
|
||||
connId <- newConnToJoin c userId "" False (Just expiresAt) cReqUri PQSupportOn
|
||||
var <- atomically newEmptyTMVar
|
||||
atomically $ TM.insert connId var (serviceRequests c)
|
||||
r <- tryAllErrors $ do
|
||||
@@ -2164,21 +2116,20 @@ data CommandCompletion = CCMoved | CCCompleted
|
||||
|
||||
runCommandProcessing :: AgentClient -> ConnId -> Maybe SMPServer -> Worker -> AM ()
|
||||
runCommandProcessing c@AgentClient {subQ} connId server_ Worker {doWork} = do
|
||||
AgentConfig {messageRetryInterval = RetryInterval2 {riFast}, commandQuotaRetryInterval} <- asks config
|
||||
let ri = RetryInterval2 {riSlow = commandQuotaRetryInterval, riFast}
|
||||
ri <- asks $ messageRetryInterval . config -- different retry interval?
|
||||
forever $ do
|
||||
atomically $ endAgentOperation c AOSndNetwork
|
||||
lift $ waitForWork doWork
|
||||
liftIO $ throwWhenInactive c
|
||||
atomically $ beginAgentOperation c AOSndNetwork
|
||||
withWork c doWork (\db -> getPendingServerCommand db connId server_) $ runProcessCmd ri
|
||||
withWork c doWork (\db -> getPendingServerCommand db connId server_) $ runProcessCmd (riFast ri)
|
||||
where
|
||||
runProcessCmd ri cmd = do
|
||||
pending <- newTVarIO []
|
||||
processCmd ri cmd pending
|
||||
mapM_ (atomically . writeTBQueue subQ) . reverse =<< readTVarIO pending
|
||||
processCmd :: RetryInterval2 -> PendingCommand -> TVar [ATransmission] -> AM ()
|
||||
processCmd ri PendingCommand {cmdId, corrId, userId, command, createdAt} pendingCmds = case command of
|
||||
processCmd :: RetryInterval -> PendingCommand -> TVar [ATransmission] -> AM ()
|
||||
processCmd ri PendingCommand {cmdId, corrId, userId, command} pendingCmds = case command of
|
||||
AClientCommand cmd -> case cmd of
|
||||
NEW enableNtfs (ACM cMode) pqEnc subMode useDR -> noServer $ do
|
||||
triedHosts <- newTVarIO S.empty
|
||||
@@ -2373,18 +2324,12 @@ runCommandProcessing c@AgentClient {subQ} connId server_ Worker {doWork} = do
|
||||
SomeConn _ conn@DuplexConnection {} -> a conn
|
||||
_ -> internalErr "command requires duplex connection"
|
||||
tryCommand action = tryMoveableCommand (action $> CCCompleted)
|
||||
tryMoveableCommand action = withRetryInterval2 ri $ \_ loop -> do
|
||||
tryMoveableCommand action = withRetryInterval ri $ \_ loop -> do
|
||||
liftIO $ waitWhileSuspended c
|
||||
liftIO $ waitForUserNetwork c
|
||||
tryAllErrors action >>= \case
|
||||
Left e@(SMP _ SMP.QUOTA) -> do
|
||||
AgentConfig {quotaExceededTimeout} <- asks config
|
||||
expireTs <- addUTCTime (-quotaExceededTimeout) <$> liftIO getCurrentTime
|
||||
if createdAt < expireTs
|
||||
then cmdError e
|
||||
else retrySndOp c $ loop RISlow
|
||||
Left e
|
||||
| temporaryOrHostError e -> retrySndOp c $ loop RIFast
|
||||
| temporaryOrHostError e -> retrySndOp c loop
|
||||
| otherwise -> cmdError e
|
||||
Right CCCompleted -> withStore' c (`deleteCommand` cmdId)
|
||||
Right CCMoved -> pure () -- command processing moved to another command queue
|
||||
@@ -2431,66 +2376,62 @@ enqueueMessageB c reqs = do
|
||||
cfg <- asks config
|
||||
(_, reqMids) <- unsafeWithStore c $ \db -> do
|
||||
mapAccumLM (\ids r -> storeSentMsg db cfg ids r `E.catchAny` \e -> (ids,) <$> handleInternal e) IM.empty reqs
|
||||
forME reqMids $ submitSentMsg c
|
||||
forME reqMids $ \((csqs_, _, _, _), InternalId msgId, pqSecr) -> forM csqs_ $ \(cData, sq :| sqs) -> do
|
||||
submitPendingMsg c sq
|
||||
let sqs' = filter (isActiveSndQ cData) sqs
|
||||
pure ((msgId, pqSecr), if null sqs' then Nothing else Just (sqs', msgId))
|
||||
where
|
||||
storeSentMsg ::
|
||||
DB.Connection ->
|
||||
AgentConfig ->
|
||||
IntMap (Maybe Int64, AMessage) ->
|
||||
Either AgentErrorType (Either AgentErrorType (ConnData, NonEmpty SndQueue), Maybe PQEncryption, MsgFlags, ValueOrRef AMessage) ->
|
||||
IO (IntMap (Maybe Int64, AMessage), Either AgentErrorType ((Either AgentErrorType (ConnData, NonEmpty SndQueue), Maybe PQEncryption, MsgFlags, ValueOrRef AMessage), InternalId, PQEncryption))
|
||||
storeSentMsg db cfg aMessageIds = \case
|
||||
Left e -> pure (aMessageIds, Left e)
|
||||
Right req@(csqs_, pqEnc_, msgFlags, mbr) -> case mbr of
|
||||
VRValue i_ aMessage -> case i_ >>= (`IM.lookup` aMessageIds) of
|
||||
Just _ -> pure (aMessageIds, Left $ INTERNAL "enqueueMessageB: storeSentMsg duplicate saved message body")
|
||||
Nothing -> do
|
||||
(mbId_, r) <- case csqs_ of
|
||||
Left e -> pure (Nothing, Left e)
|
||||
Right (cData, sq :| _) -> do
|
||||
mbId <- createSndMsgBody db aMessage
|
||||
(Just mbId,) <$> storeSentMsg_ cData sq mbId aMessage
|
||||
let aMessageIds' = maybe id (`IM.insert` (mbId_, aMessage)) i_ aMessageIds
|
||||
pure (aMessageIds', r)
|
||||
VRRef i -> case csqs_ of
|
||||
Left e -> pure $ (aMessageIds, Left e)
|
||||
Right (cData, sq :| _) -> case IM.lookup i aMessageIds of
|
||||
Just (Just mbId, aMessage) -> (aMessageIds,) <$> storeSentMsg_ cData sq mbId aMessage
|
||||
Just (Nothing, aMessage) -> do
|
||||
mbId <- createSndMsgBody db aMessage
|
||||
let aMessageIds' = IM.insert i (Just mbId, aMessage) aMessageIds
|
||||
(aMessageIds',) <$> storeSentMsg_ cData sq mbId aMessage
|
||||
Nothing -> pure (aMessageIds, Left $ INTERNAL "enqueueMessageB: storeSentMsg missing saved message body id")
|
||||
where
|
||||
storeSentMsg_ cData@ConnData {connId} sq sndMsgBodyId aMessage = fmap (first storeError) $ runExceptT $ do
|
||||
let AgentConfig {e2eEncryptVRange} = cfg
|
||||
internalTs <- liftIO getCurrentTime
|
||||
(internalId, internalSndId, prevMsgHash) <- ExceptT $ updateSndIds db connId
|
||||
-- We need to do pre-flight encoding that is not stored in database
|
||||
-- to calculate its hash and remember it on connection (createSndMsg -> updateSndMsgHash)
|
||||
-- to enable next enqueue.
|
||||
-- (As encoding is different per connection, we can't store shared body, so it's repeated on delivery)
|
||||
let agentMsgStr = encodeAgentMsgStr aMessage internalSndId prevMsgHash
|
||||
internalHash = C.sha256Hash agentMsgStr
|
||||
currentE2EVersion = maxVersion e2eEncryptVRange
|
||||
(mek, paddedLen, pqEnc) <- agentRatchetEncryptHeader db cData e2eEncAgentMsgLength pqEnc_ currentE2EVersion
|
||||
withExceptT (SEAgentError . cryptoError) $ CR.rcCheckCanPad paddedLen agentMsgStr
|
||||
let msgType = aMessageType aMessage
|
||||
-- msgBody is empty, because snd_messages record is linked to snd_message_bodies
|
||||
msgData = SndMsgData {internalId, internalSndId, internalTs, msgType, msgFlags, msgBody = "", pqEncryption = pqEnc, internalHash, prevMsgHash, sndMsgPrepData_ = Just SndMsgPrepData {encryptKey = mek, paddedLen, sndMsgBodyId}}
|
||||
liftIO $ createSndMsg db connId msgData
|
||||
liftIO $ createSndMsgDelivery db sq internalId
|
||||
pure (req, internalId, pqEnc)
|
||||
handleInternal :: E.SomeException -> IO (Either AgentErrorType b)
|
||||
handleInternal = pure . Left . INTERNAL . show
|
||||
|
||||
submitSentMsg :: AgentClient -> ((Either AgentErrorType (ConnData, NonEmpty SndQueue), Maybe PQEncryption, MsgFlags, ValueOrRef AMessage), InternalId, PQEncryption) -> AM' (Either AgentErrorType ((AgentMsgId, PQEncryption), Maybe ([SndQueue], AgentMsgId)))
|
||||
submitSentMsg c ((csqs_, _, _, _), InternalId msgId, pqSecr) = forM csqs_ $ \(cData, sq :| sqs) -> do
|
||||
submitPendingMsg c sq
|
||||
let sqs' = filter (isActiveSndQ cData) sqs
|
||||
pure ((msgId, pqSecr), if null sqs' then Nothing else Just (sqs', msgId))
|
||||
|
||||
storeSentMsg ::
|
||||
DB.Connection ->
|
||||
AgentConfig ->
|
||||
IntMap (Maybe Int64, AMessage) ->
|
||||
Either AgentErrorType (Either AgentErrorType (ConnData, NonEmpty SndQueue), Maybe PQEncryption, MsgFlags, ValueOrRef AMessage) ->
|
||||
IO (IntMap (Maybe Int64, AMessage), Either AgentErrorType ((Either AgentErrorType (ConnData, NonEmpty SndQueue), Maybe PQEncryption, MsgFlags, ValueOrRef AMessage), InternalId, PQEncryption))
|
||||
storeSentMsg db cfg aMessageIds = \case
|
||||
Left e -> pure (aMessageIds, Left e)
|
||||
Right req@(csqs_, pqEnc_, msgFlags, mbr) -> case mbr of
|
||||
VRValue i_ aMessage -> case i_ >>= (`IM.lookup` aMessageIds) of
|
||||
Just _ -> pure (aMessageIds, Left $ INTERNAL "enqueueMessageB: storeSentMsg duplicate saved message body")
|
||||
Nothing -> do
|
||||
(mbId_, r) <- case csqs_ of
|
||||
Left e -> pure (Nothing, Left e)
|
||||
Right (cData, sq :| _) -> do
|
||||
mbId <- createSndMsgBody db aMessage
|
||||
(Just mbId,) <$> storeSentMsg_ cData sq mbId aMessage
|
||||
let aMessageIds' = maybe id (`IM.insert` (mbId_, aMessage)) i_ aMessageIds
|
||||
pure (aMessageIds', r)
|
||||
VRRef i -> case csqs_ of
|
||||
Left e -> pure $ (aMessageIds, Left e)
|
||||
Right (cData, sq :| _) -> case IM.lookup i aMessageIds of
|
||||
Just (Just mbId, aMessage) -> (aMessageIds,) <$> storeSentMsg_ cData sq mbId aMessage
|
||||
Just (Nothing, aMessage) -> do
|
||||
mbId <- createSndMsgBody db aMessage
|
||||
let aMessageIds' = IM.insert i (Just mbId, aMessage) aMessageIds
|
||||
(aMessageIds',) <$> storeSentMsg_ cData sq mbId aMessage
|
||||
Nothing -> pure (aMessageIds, Left $ INTERNAL "enqueueMessageB: storeSentMsg missing saved message body id")
|
||||
where
|
||||
storeSentMsg_ cData@ConnData {connId} sq sndMsgBodyId aMessage = fmap (first storeError) $ runExceptT $ do
|
||||
let AgentConfig {e2eEncryptVRange} = cfg
|
||||
internalTs <- liftIO getCurrentTime
|
||||
(internalId, internalSndId, prevMsgHash) <- ExceptT $ updateSndIds db connId
|
||||
-- We need to do pre-flight encoding that is not stored in database
|
||||
-- to calculate its hash and remember it on connection (createSndMsg -> updateSndMsgHash)
|
||||
-- to enable next enqueue.
|
||||
-- (As encoding is different per connection, we can't store shared body, so it's repeated on delivery)
|
||||
let agentMsgStr = encodeAgentMsgStr aMessage internalSndId prevMsgHash
|
||||
internalHash = C.sha256Hash agentMsgStr
|
||||
currentE2EVersion = maxVersion e2eEncryptVRange
|
||||
(mek, paddedLen, pqEnc) <- agentRatchetEncryptHeader db cData e2eEncAgentMsgLength pqEnc_ currentE2EVersion
|
||||
withExceptT (SEAgentError . cryptoError) $ CR.rcCheckCanPad paddedLen agentMsgStr
|
||||
let msgType = aMessageType aMessage
|
||||
-- msgBody is empty, because snd_messages record is linked to snd_message_bodies
|
||||
msgData = SndMsgData {internalId, internalSndId, internalTs, msgType, msgFlags, msgBody = "", pqEncryption = pqEnc, internalHash, prevMsgHash, sndMsgPrepData_ = Just SndMsgPrepData {encryptKey = mek, paddedLen, sndMsgBodyId}}
|
||||
liftIO $ createSndMsg db connId msgData
|
||||
liftIO $ createSndMsgDelivery db sq internalId
|
||||
pure (req, internalId, pqEnc)
|
||||
|
||||
encodeAgentMsgStr :: AMessage -> InternalSndId -> PrevSndMsgHash -> ByteString
|
||||
encodeAgentMsgStr aMessage internalSndId prevMsgHash = do
|
||||
let privHeader = APrivHeader (unSndId internalSndId) prevMsgHash
|
||||
@@ -2865,18 +2806,15 @@ synchronizeRatchet' c connId pqSupport' force = withConnLock c connId "synchroni
|
||||
SomeConn _ (DuplexConnection cData@ConnData {pqSupport} rqs sqs)
|
||||
| ratchetSyncAllowed cData || force -> do
|
||||
-- check queues are not switching?
|
||||
when (pqSupport' /= pqSupport) $ withStore' c $ \db -> setConnPQSupport db connId pqSupport'
|
||||
let cData' = cData {pqSupport = pqSupport'} :: ConnData
|
||||
AgentConfig {e2eEncryptVRange, smpAgentVRange} <- asks config
|
||||
AgentConfig {e2eEncryptVRange} <- asks config
|
||||
g <- asks random
|
||||
(pks, e2eParams) <- liftIO $ CR.generateRcvE2EParams g (maxVersion e2eEncryptVRange) pqSupport'
|
||||
msgId <- withStore c $ \db -> runExceptT $ do
|
||||
msgId <- storeRatchetKey db (L.head sqs) (maxVersion smpAgentVRange) e2eParams Nothing
|
||||
liftIO $ do
|
||||
when (pqSupport' /= pqSupport) $ setConnPQSupport db connId pqSupport'
|
||||
setConnRatchetSync db connId RSStarted
|
||||
setRatchetX3dhKeys db connId pks
|
||||
pure msgId
|
||||
lift $ enqueueRatchetKeyMsgs c cData' sqs msgId
|
||||
enqueueRatchetKeyMsgs c cData' sqs e2eParams
|
||||
withStore' c $ \db -> do
|
||||
setConnRatchetSync db connId RSStarted
|
||||
setRatchetX3dhKeys db connId pks
|
||||
let cData'' = cData' {ratchetSyncState = RSStarted} :: ConnData
|
||||
conn' = DuplexConnection cData'' rqs sqs
|
||||
connectionStats c conn'
|
||||
@@ -3051,12 +2989,10 @@ getConnectionServers' c connId = do
|
||||
SomeConn _ conn <- withStore c (`getConn` connId)
|
||||
connectionStats c conn
|
||||
|
||||
getConnectionVerifyCodes' :: AgentClient -> ConnId -> AM ConnVerifyCodes
|
||||
getConnectionVerifyCodes' c connId =
|
||||
getConnectionsVerifyCodes' c [connId] >>= maybe (throwE $ CONN NOT_FOUND "") pure . M.lookup connId
|
||||
|
||||
getConnectionsVerifyCodes' :: AgentClient -> [ConnId] -> AM (Map ConnId ConnVerifyCodes)
|
||||
getConnectionsVerifyCodes' c connIds = withStore' c (`getRatchetVerifyCodes` connIds)
|
||||
getConnectionRatchetAdHash' :: AgentClient -> ConnId -> AM ByteString
|
||||
getConnectionRatchetAdHash' c connId = do
|
||||
CR.Ratchet {rcAD = Str rcAD} <- withStore c (`getRatchet` connId)
|
||||
pure $ C.sha256Hash rcAD
|
||||
|
||||
connectionStats :: AgentClient -> Connection c -> AM ConnectionStats
|
||||
connectionStats c = \case
|
||||
@@ -3407,9 +3343,6 @@ subscriber c@AgentClient {msgQ, subQ} = run $ forever $ do
|
||||
run a = a `catchOwn` \e -> notify $ CRITICAL True $ "Agent subscriber stopped: " <> show e
|
||||
notify err = atomically $ writeTBQueue subQ ("", "", AEvt SAEConn $ ERR err)
|
||||
|
||||
maxRatchetKeyHashes :: Int
|
||||
maxRatchetKeyHashes = 100
|
||||
|
||||
cleanupManager :: AgentClient -> AM' ()
|
||||
cleanupManager c@AgentClient {subQ} = do
|
||||
AgentConfig {initialCleanupDelay, cleanupInterval = int, storedMsgDataTTL = ttl, cleanupBatchSize = limit} <-
|
||||
@@ -3419,7 +3352,7 @@ cleanupManager c@AgentClient {subQ} = do
|
||||
run ERR deleteConns
|
||||
run ERR $ withStore' c $ \db -> deleteRcvMsgHashesExpired db ttl limit
|
||||
run ERR $ withStore' c $ \db -> deleteSndMsgsExpired db ttl limit
|
||||
run ERR $ withStore' c $ \db -> deleteRatchetKeyHashesExpired db ttl maxRatchetKeyHashes
|
||||
run ERR $ withStore' c $ \db -> deleteRatchetKeyHashesExpired db ttl limit
|
||||
run ERR $ withStore' c (`deleteExpiredNtfTokensToDelete` ttl)
|
||||
run RFERR deleteRcvFilesExpired
|
||||
run RFERR deleteRcvFilesDeleted
|
||||
@@ -3616,9 +3549,9 @@ processSMPTransmissions c@AgentClient {subQ} (tSess@(userId, srv, _), THandlePar
|
||||
_ -> prohibited "handshake: incorrect state" >> ack
|
||||
(Just e2eDh, Nothing) -> do
|
||||
decryptClientMessage e2eDh clientMsg >>= \case
|
||||
(SMP.PHEmpty, AgentRatchetKey {agentVersion, e2eEncryption, info}) -> do
|
||||
(SMP.PHEmpty, AgentRatchetKey {agentVersion, e2eEncryption}) -> do
|
||||
conn' <- updateConnVersion conn cData agentVersion
|
||||
qDuplex conn' "AgentRatchetKey" $ \a -> newRatchetKey e2eEncryption info a >> ack
|
||||
qDuplex conn' "AgentRatchetKey" $ \a -> newRatchetKey e2eEncryption a >> ack
|
||||
(SMP.PHEmpty, AgentMsgEnvelope {agentVersion, encAgentMessage}) -> do
|
||||
conn' <- updateConnVersion conn cData agentVersion
|
||||
-- primary queue is set as Active in helloMsg, below is to set additional queues Active
|
||||
@@ -3661,7 +3594,7 @@ processSMPTransmissions c@AgentClient {subQ} (tSess@(userId, srv, _), THandlePar
|
||||
qDuplexAckDel conn'' name a = qDuplex conn'' name a >> ackDel msgId
|
||||
resetRatchetSync :: AM (Connection c)
|
||||
resetRatchetSync
|
||||
| rss `notElem` ([RSOk, RSStarted] :: [RatchetSyncState]) = ifM ((RSStarted ==) <$> getRatchetSyncState) (pure conn') $ do
|
||||
| rss `notElem` ([RSOk, RSStarted] :: [RatchetSyncState]) = do
|
||||
let cData'' = (toConnData conn') {ratchetSyncState = RSOk} :: ConnData
|
||||
conn'' = updateConnection cData'' conn'
|
||||
cStats <- connectionStats c conn''
|
||||
@@ -3702,7 +3635,7 @@ processSMPTransmissions c@AgentClient {subQ} (tSess@(userId, srv, _), THandlePar
|
||||
notifySync :: AM ()
|
||||
notifySync = qDuplex conn' "AGENT A_CRYPTO error" $ \connDuplex -> do
|
||||
let rss' = cryptoErrToSyncState e
|
||||
whenM ((`elem` ([RSOk, RSAllowed, RSRequired] :: [RatchetSyncState])) <$> getRatchetSyncState) $ do
|
||||
when (rss `elem` ([RSOk, RSAllowed, RSRequired] :: [RatchetSyncState])) $ do
|
||||
let cData'' = (toConnData conn') {ratchetSyncState = rss'} :: ConnData
|
||||
conn'' = updateConnection cData'' connDuplex
|
||||
cStats <- connectionStats c conn''
|
||||
@@ -3798,9 +3731,6 @@ processSMPTransmissions c@AgentClient {subQ} (tSess@(userId, srv, _), THandlePar
|
||||
enqueueCmd :: InternalCommand -> AM ()
|
||||
enqueueCmd = enqueueCommand c "" connId (Just srv) . AInternalCommand
|
||||
|
||||
getRatchetSyncState :: AM RatchetSyncState
|
||||
getRatchetSyncState = withStore c (`getConnRatchetSync` connId)
|
||||
|
||||
unexpected :: BrokerMsg -> AM ()
|
||||
unexpected r = do
|
||||
logServer "<--" c srv rId $ "unexpected: " <> bshow r
|
||||
@@ -4110,14 +4040,14 @@ processSMPTransmissions c@AgentClient {subQ} (tSess@(userId, srv, _), THandlePar
|
||||
enqueueMessages' c cData' sqs SMP.MsgFlags {notification = True} (EREADY lastExternalSndId)
|
||||
|
||||
smpInvitation :: SMP.MsgId -> Connection c -> ConnectionRequestUri 'CMInvitation -> ConnInfo -> AM ()
|
||||
smpInvitation srvMsgId conn' connReq@(CRInvitationUri crData (CR.E2ERatchetParamsUri _ k1 k2 kem_)) cInfo = do
|
||||
smpInvitation srvMsgId conn' connReq@(CRInvitationUri crData _) cInfo = do
|
||||
logServer "<--" c srv rId $ "MSG <KEY>:" <> logSecret' srvMsgId
|
||||
case conn' of
|
||||
ContactConnection _ RcvQueue {sndId} -> do
|
||||
ContactConnection {} -> do
|
||||
-- show connection request even if invitaion via contact address is not compatible.
|
||||
invId <- storeInvitation (CRInvitation connReq) cInfo False
|
||||
let srvs = L.map qServer $ crSmpQueues crData
|
||||
notify $ REQ invId PQSupportOn srvs cInfo (CRBRequest $ requestCode k1 k2 kem_ sndId) False
|
||||
notify $ REQ invId PQSupportOn srvs cInfo False
|
||||
_ -> prohibited "inv: sent to message conn"
|
||||
|
||||
storeInvitation :: ContactRequest -> ConnInfo -> Bool -> AM InvitationId
|
||||
@@ -4144,7 +4074,7 @@ processSMPTransmissions c@AgentClient {subQ} (tSess@(userId, srv, _), THandlePar
|
||||
parseMessage "3" agentMsgBody >>= \case
|
||||
AgentConnInfoReply (replyQueue :| _) cInfo -> do
|
||||
invId <- storeInvitation (CRInvitationDR $ mkDR replyQueue) cInfo False
|
||||
notify $ REQ invId PQSupportOn (qServer replyQueue :| []) cInfo (CRBRatchet $ ratchetVerifyCodes ratchetState) True
|
||||
notify $ REQ invId PQSupportOn (qServer replyQueue :| []) cInfo True
|
||||
AgentServiceRequest (replyQueue :| _) sig_ payload ->
|
||||
case verifyServiceReq rc payload sig_ of
|
||||
Left err -> logError ("service request: " <> T.pack err) >> notify (ERR $ AGENT $ A_SERVICE ASEBadSignature)
|
||||
@@ -4169,47 +4099,42 @@ processSMPTransmissions c@AgentClient {subQ} (tSess@(userId, srv, _), THandlePar
|
||||
DuplexConnection {} -> action conn'
|
||||
_ -> qError $ name <> ": message must be sent to duplex connection"
|
||||
|
||||
newRatchetKey :: CR.RcvE2ERatchetParams 'C.X448 -> ByteString -> Connection 'CDuplex -> AM ()
|
||||
newRatchetKey e2eOtherPartyParams@(CR.E2ERatchetParams e2eVersion k1Rcv k2Rcv _) info conn'@(DuplexConnection cData'@ConnData {lastExternalSndId, pqSupport} _ sqs) =
|
||||
newRatchetKey :: CR.RcvE2ERatchetParams 'C.X448 -> Connection 'CDuplex -> AM ()
|
||||
newRatchetKey e2eOtherPartyParams@(CR.E2ERatchetParams e2eVersion k1Rcv k2Rcv _) conn'@(DuplexConnection cData'@ConnData {lastExternalSndId, pqSupport} _ sqs) =
|
||||
unlessM ratchetExists $ do
|
||||
AgentConfig {e2eEncryptVRange} <- asks config
|
||||
unless (e2eVersion `isCompatible` e2eEncryptVRange) (throwE $ AGENT A_VERSION)
|
||||
keys_ <- getSendRatchetKeys
|
||||
keys <- getSendRatchetKeys
|
||||
let rcVs = CR.RatchetVersions {current = e2eVersion, maxSupported = maxVersion e2eEncryptVRange}
|
||||
case keys_ of
|
||||
Just (keys, replyKey_) -> initRatchet rcVs keys replyKey_ >> notifyAgreed
|
||||
Nothing -> withStore' c $ \db -> addProcessedRatchetKeyHash db connId rkHashRcv
|
||||
initRatchet rcVs keys
|
||||
notifyAgreed
|
||||
where
|
||||
rkHashRcv = rkHash k1Rcv k2Rcv
|
||||
rkHash k1 k2 = C.sha256Hash $ C.pubKeyBytes k1 <> C.pubKeyBytes k2
|
||||
answeredKeyHash_ = case smpDecode info of
|
||||
Right (AgentRatchetInfo RatchetInfo {answeredKeyHash}) -> answeredKeyHash
|
||||
_ -> Nothing
|
||||
ratchetExists :: AM Bool
|
||||
ratchetExists = withStore' c $ \db -> do
|
||||
exists <- checkRatchetKeyHashExists db connId rkHashRcv
|
||||
unless exists $ addProcessedRatchetKeyHash db connId rkHashRcv
|
||||
pure exists
|
||||
getSendRatchetKeys :: AM (Maybe (CR.RcvE2EPrivRatchetParams 'C.X448, Maybe (CR.RcvE2ERatchetParams 'C.X448)))
|
||||
getSendRatchetKeys = getRatchetSyncState >>= \rss' -> case (rss', answeredKeyHash_) of
|
||||
(RSStarted, Nothing) -> Just . (,Nothing) <$> withStore c (`getRatchetX3dhKeys` connId) -- initiating client
|
||||
(RSStarted, Just h) -> fmap (,Nothing) . answeredKeys h <$> withStore c (`getRatchetX3dhKeys` connId)
|
||||
(_, Nothing) -> Just <$> sendReplyKey -- receiving client
|
||||
(RSAgreed, Just _) -> do
|
||||
withStore' c $ \db -> addProcessedRatchetKeyHash db connId rkHashRcv >> setConnRatchetSync db connId RSRequired
|
||||
getSendRatchetKeys :: AM (CR.RcvE2EPrivRatchetParams 'C.X448)
|
||||
getSendRatchetKeys = case rss of
|
||||
RSOk -> sendReplyKey -- receiving client
|
||||
RSAllowed -> sendReplyKey
|
||||
RSRequired -> sendReplyKey
|
||||
RSStarted -> withStore c (`getRatchetX3dhKeys` connId) -- initiating client
|
||||
RSAgreed -> do
|
||||
withStore' c $ \db -> setConnRatchetSync db connId RSRequired
|
||||
notifyRatchetSyncError
|
||||
-- can communicate for other client to reset to RSRequired
|
||||
-- - need to add new AgentMsgEnvelope, AgentMessage, AgentMessageType
|
||||
-- - need to deduplicate on receiving side
|
||||
throwE $ AGENT (A_CRYPTO RATCHET_SYNC)
|
||||
(_, Just _) -> pure Nothing
|
||||
where
|
||||
answeredKeys h keys@(pk1, pk2, _)
|
||||
| rkHash (C.publicKey pk1) (C.publicKey pk2) == h = Just keys
|
||||
| otherwise = Nothing
|
||||
sendReplyKey = do
|
||||
g <- asks random
|
||||
(pks, e2eParams) <- liftIO $ CR.generateRcvE2EParams g e2eVersion pqSupport
|
||||
pure (pks, Just e2eParams)
|
||||
enqueueRatchetKeyMsgs c cData' sqs e2eParams
|
||||
pure pks
|
||||
notifyRatchetSyncError = do
|
||||
let cData'' = cData' {ratchetSyncState = RSRequired} :: ConnData
|
||||
conn'' = updateConnection cData'' conn'
|
||||
@@ -4221,33 +4146,23 @@ processSMPTransmissions c@AgentClient {subQ} (tSess@(userId, srv, _), THandlePar
|
||||
conn'' = updateConnection cData'' conn'
|
||||
cStats <- connectionStats c conn''
|
||||
notify $ RSYNC RSAgreed Nothing cStats
|
||||
recreateRatchet :: Maybe (CR.RcvE2ERatchetParams 'C.X448) -> Maybe AMessage -> CR.Ratchet 'C.X448 -> AM ()
|
||||
recreateRatchet replyKey_ eready_ rc = do
|
||||
cfg@AgentConfig {smpAgentVRange} <- asks config
|
||||
(msgId_, r_) <- withStore c $ \db -> runExceptT $ do
|
||||
msgId_ <- forM replyKey_ $ \e2eParams -> storeRatchetKey db (L.head sqs) (maxVersion smpAgentVRange) e2eParams (Just rkHashRcv)
|
||||
liftIO $ do
|
||||
addProcessedRatchetKeyHash db connId rkHashRcv
|
||||
setConnRatchetSync db connId RSAgreed
|
||||
deleteRatchet db connId
|
||||
createRatchet db connId rc
|
||||
r_ <- forM eready_ $ \msg -> liftIO $ storeSentMsg db cfg IM.empty (Right (Right (cData', sqs), Nothing, SMP.MsgFlags {notification = True}, vrValue msg)) >>= either E.throwIO pure . snd
|
||||
pure (msgId_, r_)
|
||||
forM_ msgId_ $ lift . enqueueRatchetKeyMsgs c cData' sqs
|
||||
forM_ r_ $ \r -> lift $ do
|
||||
r' <- submitSentMsg c r
|
||||
enqueueSavedMessageB c $ mapMaybe snd $ rights [r']
|
||||
recreateRatchet :: CR.Ratchet 'C.X448 -> AM ()
|
||||
recreateRatchet rc = withStore' c $ \db -> do
|
||||
setConnRatchetSync db connId RSAgreed
|
||||
deleteRatchet db connId
|
||||
createRatchet db connId rc
|
||||
-- compare public keys `k1` in AgentRatchetKey messages sent by self and other party
|
||||
-- to determine ratchet initilization ordering
|
||||
initRatchet :: CR.RatchetVersions -> CR.RcvE2EPrivRatchetParams 'C.X448 -> Maybe (CR.RcvE2ERatchetParams 'C.X448) -> AM ()
|
||||
initRatchet rcVs (pk1, pk2, pKem) replyKey_
|
||||
initRatchet :: CR.RatchetVersions -> CR.RcvE2EPrivRatchetParams 'C.X448 -> AM ()
|
||||
initRatchet rcVs (pk1, pk2, pKem)
|
||||
| rkHash (C.publicKey pk1) (C.publicKey pk2) <= rkHashRcv = do
|
||||
rcParams <- liftError cryptoError $ CR.pqX3dhRcv (pk1, pk2, pKem) e2eOtherPartyParams
|
||||
recreateRatchet replyKey_ Nothing $ CR.initRcvRatchet rcVs pk2 rcParams pqSupport
|
||||
recreateRatchet $ CR.initRcvRatchet rcVs pk2 rcParams pqSupport
|
||||
| otherwise = do
|
||||
(_, rcDHRs) <- atomically . C.generateKeyPair =<< asks random
|
||||
rcParams <- liftEitherWith cryptoError $ CR.pqX3dhSnd (pk1, pk2, CR.APRKP CR.SRKSProposed <$> pKem) e2eOtherPartyParams
|
||||
recreateRatchet replyKey_ (Just $ EREADY lastExternalSndId) $ CR.initSndRatchet rcVs k2Rcv rcDHRs rcParams
|
||||
recreateRatchet $ CR.initSndRatchet rcVs k2Rcv rcDHRs rcParams
|
||||
void . enqueueMessages' c cData' sqs SMP.MsgFlags {notification = True} $ EREADY lastExternalSndId
|
||||
|
||||
checkMsgIntegrity :: PrevExternalSndId -> ExternalSndId -> PrevRcvMsgHash -> ByteString -> MsgIntegrity
|
||||
checkMsgIntegrity prevExtSndId extSndId internalPrevMsgHash receivedPrevMsgHash
|
||||
@@ -4360,25 +4275,32 @@ storeConfirmation c cData@ConnData {connId, pqSupport, connAgentVersion = v} sq
|
||||
liftIO $ createSndMsg db connId msgData
|
||||
liftIO $ createSndMsgDelivery db sq internalId
|
||||
|
||||
enqueueRatchetKeyMsgs :: AgentClient -> ConnData -> NonEmpty SndQueue -> InternalId -> AM' ()
|
||||
enqueueRatchetKeyMsgs c cData (sq :| sqs) (InternalId msgId) = do
|
||||
submitPendingMsg c sq
|
||||
mapM_ (enqueueSavedMessage c msgId) $ filter (isActiveSndQ cData) sqs
|
||||
enqueueRatchetKeyMsgs :: AgentClient -> ConnData -> NonEmpty SndQueue -> CR.RcvE2ERatchetParams 'C.X448 -> AM ()
|
||||
enqueueRatchetKeyMsgs c cData (sq :| sqs) e2eEncryption = do
|
||||
msgId <- enqueueRatchetKey c sq e2eEncryption
|
||||
mapM_ (lift . enqueueSavedMessage c msgId) $ filter (isActiveSndQ cData) sqs
|
||||
|
||||
storeRatchetKey :: DB.Connection -> SndQueue -> VersionSMPA -> CR.RcvE2ERatchetParams 'C.X448 -> Maybe ByteString -> ExceptT StoreError IO InternalId
|
||||
storeRatchetKey db sq@SndQueue {connId} agentVersion e2eEncryption answeredKeyHash_ = do
|
||||
internalTs <- liftIO getCurrentTime
|
||||
(internalId, internalSndId, prevMsgHash) <- ExceptT $ updateSndIds db connId
|
||||
let agentMsg = AgentRatchetInfo RatchetInfo {answeredKeyHash = answeredKeyHash_}
|
||||
agentMsgStr = smpEncode agentMsg
|
||||
internalHash = C.sha256Hash agentMsgStr
|
||||
let msgBody = smpEncode $ AgentRatchetKey {agentVersion, e2eEncryption, info = agentMsgStr}
|
||||
msgType = agentMessageType agentMsg
|
||||
-- this message is e2e encrypted with queue key, not with double ratchet
|
||||
msgData = SndMsgData {internalId, internalSndId, internalTs, msgType, msgBody, pqEncryption = PQEncOff, msgFlags = SMP.MsgFlags {notification = True}, internalHash, prevMsgHash, sndMsgPrepData_ = Nothing}
|
||||
liftIO $ createSndMsg db connId msgData
|
||||
liftIO $ createSndMsgDelivery db sq internalId
|
||||
pure internalId
|
||||
enqueueRatchetKey :: AgentClient -> SndQueue -> CR.RcvE2ERatchetParams 'C.X448 -> AM AgentMsgId
|
||||
enqueueRatchetKey c sq@SndQueue {connId} e2eEncryption = do
|
||||
aVRange <- asks $ smpAgentVRange . config
|
||||
msgId <- storeRatchetKey $ maxVersion aVRange
|
||||
lift $ submitPendingMsg c sq
|
||||
pure $ unId msgId
|
||||
where
|
||||
storeRatchetKey :: VersionSMPA -> AM InternalId
|
||||
storeRatchetKey agentVersion = withStore c $ \db -> runExceptT $ do
|
||||
internalTs <- liftIO getCurrentTime
|
||||
(internalId, internalSndId, prevMsgHash) <- ExceptT $ updateSndIds db connId
|
||||
let agentMsg = AgentRatchetInfo ""
|
||||
agentMsgStr = smpEncode agentMsg
|
||||
internalHash = C.sha256Hash agentMsgStr
|
||||
let msgBody = smpEncode $ AgentRatchetKey {agentVersion, e2eEncryption, info = agentMsgStr}
|
||||
msgType = agentMessageType agentMsg
|
||||
-- this message is e2e encrypted with queue key, not with double ratchet
|
||||
msgData = SndMsgData {internalId, internalSndId, internalTs, msgType, msgBody, pqEncryption = PQEncOff, msgFlags = SMP.MsgFlags {notification = True}, internalHash, prevMsgHash, sndMsgPrepData_ = Nothing}
|
||||
liftIO $ createSndMsg db connId msgData
|
||||
liftIO $ createSndMsgDelivery db sq internalId
|
||||
pure internalId
|
||||
|
||||
-- encoded AgentMessage -> encoded EncAgentMessage
|
||||
agentRatchetEncrypt :: DB.Connection -> ConnData -> ByteString -> (PQSupport -> Int) -> Maybe PQEncryption -> CR.VersionE2E -> ExceptT StoreError IO (ByteString, PQEncryption)
|
||||
@@ -4403,11 +4325,10 @@ agentRatchetDecrypt g db connId encAgentMsg = do
|
||||
|
||||
agentRatchetDecrypt' :: TVar ChaChaDRG -> DB.Connection -> ConnId -> CR.RatchetX448 -> ByteString -> ExceptT StoreError IO (ByteString, PQEncryption)
|
||||
agentRatchetDecrypt' g db connId rc encAgentMsg = do
|
||||
skipped <- liftIO $ getSkippedMsgKeys db connId CR.maxSkippedMsgKeys
|
||||
skipped <- liftIO $ getSkippedMsgKeys db connId
|
||||
(agentMsgBody_, rc', skippedDiff) <- withExceptT (SEAgentError . cryptoError) $ CR.rcDecrypt g rc skipped encAgentMsg
|
||||
agentMsgBody <- liftEither $ first (SEAgentError . cryptoError) agentMsgBody_
|
||||
liftIO $ updateRatchet db connId rc' skippedDiff
|
||||
pure (agentMsgBody, CR.rcRcvKEM rc')
|
||||
liftEither $ bimap (SEAgentError . cryptoError) (,CR.rcRcvKEM rc') agentMsgBody_
|
||||
|
||||
newSndQueue :: UserId -> ConnId -> Compatible SMPQueueInfo -> Maybe (C.APrivateAuthKey) -> AM' (NewSndQueue, C.PublicKeyX25519)
|
||||
newSndQueue userId connId (Compatible (SMPQueueInfo smpClientVersion SMPQueueAddress {smpServer, senderId, queueMode, dhPublicKey = rcvE2ePubDhKey})) sndKey_ = do
|
||||
|
||||
@@ -34,6 +34,7 @@ module Simplex.Messaging.Agent.Client
|
||||
withInvLock,
|
||||
withLockMap,
|
||||
withLocksMap,
|
||||
getMapLock,
|
||||
ipAddressProtected,
|
||||
closeAgentClient,
|
||||
closeProtocolServerClients,
|
||||
@@ -271,7 +272,7 @@ import Simplex.Messaging.Protocol
|
||||
NetworkError (..),
|
||||
MsgFlags (..),
|
||||
MsgId,
|
||||
NameResponse,
|
||||
NameRecord,
|
||||
NtfServer,
|
||||
NtfServerWithAuth,
|
||||
ProtoServer,
|
||||
@@ -311,7 +312,7 @@ import Simplex.Messaging.Session
|
||||
import Simplex.Messaging.SystemTime
|
||||
import Simplex.Messaging.TMap (TMap)
|
||||
import qualified Simplex.Messaging.TMap as TM
|
||||
import Simplex.Messaging.Transport (HandshakeError (..), SMPServiceRole (..), SMPVersion, ServiceCredentials (..), SessionId, THClientService' (..), THandleAuth (..), THandleParams (sessionId, thAuth, thVersion, serverInfo), TransportError (..), newNtfCredsSMPVersion)
|
||||
import Simplex.Messaging.Transport (HandshakeError (..), SMPServiceRole (..), SMPVersion, ServiceCredentials (..), SessionId, THClientService' (..), THandleAuth (..), THandleParams (sessionId, thAuth, thVersion, serverInfo), TransportError (..), TransportPeer (..), shortLinksSMPVersion, newNtfCredsSMPVersion)
|
||||
import Simplex.Messaging.Transport.Client (TransportHost (..))
|
||||
import Simplex.Messaging.Transport.Credentials
|
||||
import Simplex.Messaging.Util
|
||||
@@ -1260,12 +1261,9 @@ sendOrProxySMPCommand c nm userId destSrv@ProtocolServer {host = destHosts} conn
|
||||
Left e -> throwE e
|
||||
|
||||
ipAddressProtected :: NetworkConfig -> ProtocolServer p -> Bool
|
||||
ipAddressProtected NetworkConfig {socksProxy, socksMode, hostMode} (ProtocolServer _ hosts _ _)
|
||||
| isJust socksProxy = socksMode == SMAlways || if hostMode == HMPublic then allOnion else anyOnion
|
||||
| otherwise = hostMode == HMOnion && anyOnion
|
||||
ipAddressProtected NetworkConfig {socksProxy, hostMode} (ProtocolServer _ hosts _ _) = do
|
||||
isJust socksProxy || (hostMode == HMOnion && any isOnionHost hosts)
|
||||
where
|
||||
anyOnion = any isOnionHost hosts
|
||||
allOnion = all isOnionHost hosts
|
||||
isOnionHost = \case THOnionHost _ -> True; _ -> False
|
||||
|
||||
withNtfClient :: AgentClient -> NetworkRequestMode -> NtfServer -> EntityId -> ByteString -> (NtfClient -> ExceptT NtfClientError IO a) -> AM a
|
||||
@@ -1498,7 +1496,7 @@ newRcvQueue_ c nm userId connId (ProtoServerWithAuth srv auth) vRange cqrd enabl
|
||||
let sessServiceId = (\THClientService {serviceId = sId} -> sId) <$> (clientService =<< thAuth thParams')
|
||||
when (isJust serviceId && serviceId /= sessServiceId) $ logError "incorrect service ID in NEW response"
|
||||
liftIO . logServer "<--" c srv NoEntity $ B.unwords ["IDS", logSecret rcvId, logSecret sndId]
|
||||
shortLink <- mkShortLinkCreds qik
|
||||
shortLink <- mkShortLinkCreds thParams' qik
|
||||
let rq =
|
||||
RcvQueue
|
||||
{ userId,
|
||||
@@ -1539,8 +1537,8 @@ newRcvQueue_ c nm userId connId (ProtoServerWithAuth srv auth) vRange cqrd enabl
|
||||
(Just ((ntfPublicKey, ntfPrivateKey), dhpk), Just (ServerNtfCreds notifierId dhk')) ->
|
||||
Just ClientNtfCreds {ntfPublicKey, ntfPrivateKey, notifierId, rcvNtfDhSecret = C.dh' dhk' dhpk}
|
||||
_ -> Nothing
|
||||
mkShortLinkCreds :: QueueIdsKeys -> AM (Maybe ShortLinkCreds)
|
||||
mkShortLinkCreds QIK {sndId, queueMode, linkId} = case (cqrd, queueMode) of
|
||||
mkShortLinkCreds :: THandleParams SMPVersion 'TClient -> QueueIdsKeys -> AM (Maybe ShortLinkCreds)
|
||||
mkShortLinkCreds thParams' QIK {sndId, queueMode, linkId} = case (cqrd, queueMode) of
|
||||
(CQRMessaging ld, Just QMMessaging) ->
|
||||
withLinkData ld $ \lnkId CQRData {linkKey, privSigKey, srvReq = (sndId', d)} ->
|
||||
if sndId == sndId'
|
||||
@@ -1554,9 +1552,11 @@ newRcvQueue_ c nm userId connId (ProtoServerWithAuth srv auth) vRange cqrd enabl
|
||||
(_, Nothing) -> newErr "unexpected link ID"
|
||||
_ -> newErr "unexpected queue mode"
|
||||
where
|
||||
v = thVersion thParams'
|
||||
withLinkData :: Maybe d -> (SMP.LinkId -> d -> AM (Maybe ShortLinkCreds)) -> AM (Maybe ShortLinkCreds)
|
||||
withLinkData ld_ mkLink = case (ld_, linkId) of
|
||||
(Just ld, Just lnkId) -> mkLink lnkId ld
|
||||
(Just _, Nothing) | v < shortLinksSMPVersion -> pure Nothing
|
||||
(Nothing, Nothing) -> pure Nothing
|
||||
_ -> newErr "unexpected or absent link ID"
|
||||
newErr :: String -> AM (Maybe ShortLinkCreds)
|
||||
@@ -2022,7 +2022,7 @@ getQueueLink c nm userId server lnkId =
|
||||
-- resolver) and falls back to a direct send when the proxy is unavailable
|
||||
-- (faster but exposes the client IP). Mode selection is delegated to
|
||||
-- `sendOrProxySMPCommand`, which honours the network config (SPMNever etc.).
|
||||
resolveName :: AgentClient -> NetworkRequestMode -> UserId -> SMPServer -> SimplexDomain -> AM NameResponse
|
||||
resolveName :: AgentClient -> NetworkRequestMode -> UserId -> SMPServer -> SimplexDomain -> AM NameRecord
|
||||
resolveName c nm userId server domain =
|
||||
snd <$> sendOrProxySMPCommand c nm userId server "" "RSLV" NoEntity resolveViaProxy resolveDirectly
|
||||
where
|
||||
|
||||
@@ -162,7 +162,6 @@ data AgentConfig = AgentConfig
|
||||
connDeleteDeliveryTimeout :: NominalDiffTime,
|
||||
helloTimeout :: NominalDiffTime,
|
||||
quotaExceededTimeout :: NominalDiffTime,
|
||||
commandQuotaRetryInterval :: RetryInterval,
|
||||
persistErrorInterval :: NominalDiffTime,
|
||||
initialCleanupDelay :: Int64,
|
||||
cleanupInterval :: Int64,
|
||||
@@ -242,7 +241,6 @@ defaultAgentConfig =
|
||||
connDeleteDeliveryTimeout = 2 * nominalDay,
|
||||
helloTimeout = 2 * nominalDay,
|
||||
quotaExceededTimeout = 7 * nominalDay,
|
||||
commandQuotaRetryInterval = RetryInterval {initialInterval = 30_000000, increaseAfter = 0, maxInterval = 3600_000000},
|
||||
persistErrorInterval = 3, -- seconds
|
||||
initialCleanupDelay = 30 * 1000000, -- 30 seconds
|
||||
cleanupInterval = 5 * 60 * 1000000, -- 5 minutes
|
||||
|
||||
@@ -6,6 +6,7 @@ module Simplex.Messaging.Agent.Lock
|
||||
withLock',
|
||||
withGetLock,
|
||||
withGetLocks,
|
||||
getPutLock,
|
||||
)
|
||||
where
|
||||
|
||||
|
||||
@@ -79,7 +79,6 @@ module Simplex.Messaging.Agent.Protocol
|
||||
SMPConfirmation (..),
|
||||
AgentMsgEnvelope (..),
|
||||
AgentMessage (..),
|
||||
RatchetInfo (..),
|
||||
RequestSignature (..),
|
||||
AgentMessageType (..),
|
||||
APrivHeader (..),
|
||||
@@ -167,8 +166,6 @@ module Simplex.Messaging.Agent.Protocol
|
||||
ConnId,
|
||||
ConfirmationId,
|
||||
InvitationId,
|
||||
ConnVerifyCodes (..),
|
||||
ContactRequestBinding (..),
|
||||
MsgIntegrity (..),
|
||||
MsgErrorType (..),
|
||||
QueueStatus (..),
|
||||
@@ -407,7 +404,7 @@ data AEvent (e :: AEntity) where
|
||||
LINK :: ConnShortLink 'CMContact -> UserConnLinkData 'CMContact -> AEvent AEConn
|
||||
LDATA :: FixedLinkData 'CMContact -> ConnLinkData 'CMContact -> ConnectionRequestUri 'CMContact -> AEvent AEConn
|
||||
CONF :: ConfirmationId -> PQSupport -> [SMPServer] -> ConnInfo -> AEvent AEConn -- ConnInfo is from sender, [SMPServer] will be empty only in v1 handshake
|
||||
REQ :: InvitationId -> PQSupport -> NonEmpty SMPServer -> ConnInfo -> ContactRequestBinding -> Bool -> AEvent AEConn -- ConnInfo is from sender; Bool - rejection reason can be sent
|
||||
REQ :: InvitationId -> PQSupport -> NonEmpty SMPServer -> ConnInfo -> Bool -> AEvent AEConn -- ConnInfo is from sender; Bool - rejection reason can be sent
|
||||
SREQ :: InvitationId -> Maybe C.PublicKeyEd25519 -> MsgBody -> AEvent AEConn
|
||||
SSENT :: AgentMsgId -> Maybe SMPServer -> AEvent AEConn
|
||||
RJCT :: ConnInfo -> AEvent AEConn
|
||||
@@ -929,7 +926,7 @@ data AgentMessage
|
||||
| -- AgentConnInfoReply is used by accepting party in duplexHandshake mode (v2), allowing to include reply queue(s) in the initial confirmation.
|
||||
-- It made removed REPLY message unnecessary.
|
||||
AgentConnInfoReply (NonEmpty SMPQueueInfo) ConnInfo
|
||||
| AgentRatchetInfo RatchetInfo
|
||||
| AgentRatchetInfo ByteString
|
||||
| AgentMessage APrivHeader AMessage
|
||||
| AgentServiceRequest (NonEmpty SMPQueueInfo) (Maybe RequestSignature) MsgBody
|
||||
| AgentServiceResponse MsgBody
|
||||
@@ -940,7 +937,7 @@ instance Encoding AgentMessage where
|
||||
smpEncode = \case
|
||||
AgentConnInfo cInfo -> smpEncode ('I', Tail cInfo)
|
||||
AgentConnInfoReply smpQueues cInfo -> smpEncode ('D', smpQueues, Tail cInfo) -- 'D' stands for "duplex"
|
||||
AgentRatchetInfo info -> smpEncode ('R', info)
|
||||
AgentRatchetInfo info -> smpEncode ('R', Tail info)
|
||||
AgentMessage hdr aMsg -> smpEncode ('M', hdr, aMsg)
|
||||
AgentServiceRequest qs sig_ body -> smpEncode ('A', qs, sig_, Tail body)
|
||||
AgentServiceResponse body -> smpEncode ('P', Tail body)
|
||||
@@ -949,23 +946,13 @@ instance Encoding AgentMessage where
|
||||
smpP >>= \case
|
||||
'I' -> AgentConnInfo . unTail <$> smpP
|
||||
'D' -> AgentConnInfoReply <$> smpP <*> (unTail <$> smpP)
|
||||
'R' -> AgentRatchetInfo <$> smpP
|
||||
'R' -> AgentRatchetInfo . unTail <$> smpP
|
||||
'M' -> AgentMessage <$> smpP <*> smpP
|
||||
'A' -> AgentServiceRequest <$> smpP <*> smpP <*> (unTail <$> smpP)
|
||||
'P' -> AgentServiceResponse . unTail <$> smpP
|
||||
'J' -> AgentRejection . unTail <$> smpP
|
||||
_ -> fail "bad AgentMessage"
|
||||
|
||||
data RatchetInfo = RatchetInfo {answeredKeyHash :: Maybe ByteString}
|
||||
deriving (Eq, Show)
|
||||
|
||||
instance Encoding RatchetInfo where
|
||||
smpEncode RatchetInfo {answeredKeyHash} = smpEncode answeredKeyHash
|
||||
smpP = do
|
||||
answeredKeyHash <- smpP <|> pure Nothing
|
||||
_ <- A.takeByteString
|
||||
pure RatchetInfo {answeredKeyHash}
|
||||
|
||||
-- internal type for storing message type in the database
|
||||
data AgentMessageType
|
||||
= AM_CONN_INFO
|
||||
@@ -1385,15 +1372,6 @@ type ConfirmationId = ByteString
|
||||
|
||||
type InvitationId = ByteString
|
||||
|
||||
data ConnVerifyCodes = ConnVerifyCodes
|
||||
{ codeAD :: ByteString,
|
||||
codePQ :: Maybe ByteString
|
||||
}
|
||||
deriving (Eq, Show)
|
||||
|
||||
data ContactRequestBinding = CRBRatchet ConnVerifyCodes | CRBRequest ByteString
|
||||
deriving (Eq, Show)
|
||||
|
||||
extraSMPServerHosts :: Map TransportHost TransportHost
|
||||
extraSMPServerHosts =
|
||||
M.fromList
|
||||
|
||||
@@ -10,7 +10,6 @@ module Simplex.Messaging.Agent.RetryInterval
|
||||
withRetryInterval,
|
||||
withRetryIntervalCount,
|
||||
withRetryForeground,
|
||||
withRetryInterval2,
|
||||
withRetryLock2,
|
||||
updateRetryInterval2,
|
||||
nextRetryDelay,
|
||||
@@ -87,24 +86,9 @@ withRetryForeground ri isForeground isOnline action = callAction 0 $ initialInte
|
||||
| otherwise = (elapsed + delay, nextRetryDelay elapsed' delay ri)
|
||||
callAction elapsed' delay'
|
||||
|
||||
withRetryInterval2 :: forall m. MonadIO m => RetryInterval2 -> (RI2State -> (RetryIntervalMode -> m ()) -> m ()) -> m ()
|
||||
withRetryInterval2 = withRetryWait2 $ liftIO . threadDelay'
|
||||
|
||||
-- This function allows action to toggle between slow and fast retry intervals.
|
||||
withRetryLock2 :: forall m. MonadIO m => RetryInterval2 -> TMVar () -> (RI2State -> (RetryIntervalMode -> m ()) -> m ()) -> m ()
|
||||
withRetryLock2 ri lock = withRetryWait2 wait ri
|
||||
where
|
||||
wait delay = do
|
||||
waiting <- newTVarIO True
|
||||
_ <- liftIO . forkIO $ do
|
||||
threadDelay' delay
|
||||
atomically $ whenM (readTVar waiting) $ void $ tryPutTMVar lock ()
|
||||
atomically $ do
|
||||
takeTMVar lock
|
||||
writeTVar waiting False
|
||||
|
||||
withRetryWait2 :: forall m. Monad m => (Int64 -> m ()) -> RetryInterval2 -> (RI2State -> (RetryIntervalMode -> m ()) -> m ()) -> m ()
|
||||
withRetryWait2 wait RetryInterval2 {riSlow, riFast} action =
|
||||
withRetryLock2 RetryInterval2 {riSlow, riFast} lock action =
|
||||
callAction (0, initialInterval riSlow) (0, initialInterval riFast)
|
||||
where
|
||||
callAction :: (Int64, Int64) -> (Int64, Int64) -> m ()
|
||||
@@ -118,6 +102,14 @@ withRetryWait2 wait RetryInterval2 {riSlow, riFast} action =
|
||||
let elapsed' = elapsed + delay
|
||||
delay' = nextRetryDelay elapsed' delay ri
|
||||
call (elapsed', delay')
|
||||
wait delay = do
|
||||
waiting <- newTVarIO True
|
||||
_ <- liftIO . forkIO $ do
|
||||
threadDelay' delay
|
||||
atomically $ whenM (readTVar waiting) $ void $ tryPutTMVar lock ()
|
||||
atomically $ do
|
||||
takeTMVar lock
|
||||
writeTVar waiting False
|
||||
|
||||
nextRetryDelay :: Int64 -> Int64 -> RetryInterval -> Int64
|
||||
nextRetryDelay elapsed delay RetryInterval {increaseAfter, maxInterval} =
|
||||
|
||||
@@ -489,8 +489,7 @@ data PendingCommand = PendingCommand
|
||||
corrId :: ACorrId,
|
||||
userId :: UserId,
|
||||
connId :: ConnId,
|
||||
command :: AgentCommand,
|
||||
createdAt :: UTCTime
|
||||
command :: AgentCommand
|
||||
}
|
||||
|
||||
data AgentCmdType = ACClient | ACInternal
|
||||
|
||||
@@ -76,7 +76,6 @@ module Simplex.Messaging.Agent.Store.AgentStore
|
||||
getExpiredServiceConns,
|
||||
deleteExpiredServiceRequests,
|
||||
getDeletedWaitingDeliveryConnIds,
|
||||
getConnRatchetSync,
|
||||
setConnRatchetSync,
|
||||
addProcessedRatchetKeyHash,
|
||||
checkRatchetKeyHashExists,
|
||||
@@ -166,8 +165,6 @@ module Simplex.Messaging.Agent.Store.AgentStore
|
||||
createRatchet,
|
||||
deleteRatchet,
|
||||
getRatchet,
|
||||
getRatchetVerifyCodes,
|
||||
ratchetVerifyCodes,
|
||||
getRatchetForUpdate,
|
||||
getSkippedMsgKeys,
|
||||
updateRatchet,
|
||||
@@ -216,7 +213,6 @@ module Simplex.Messaging.Agent.Store.AgentStore
|
||||
-- Rcv files
|
||||
createRcvFile,
|
||||
createRcvFileRedirect,
|
||||
startPreparedRcvFile,
|
||||
lockRcvFileForUpdate,
|
||||
getRcvFile,
|
||||
getRcvFileByEntityId,
|
||||
@@ -238,7 +234,6 @@ module Simplex.Messaging.Agent.Store.AgentStore
|
||||
getRcvFilesExpired,
|
||||
-- Snd files
|
||||
createSndFile,
|
||||
startPreparedSndFile,
|
||||
lockSndFileForUpdate,
|
||||
getSndFile,
|
||||
getSndFileByEntityId,
|
||||
@@ -296,7 +291,6 @@ import Data.ByteString (ByteString)
|
||||
import qualified Data.ByteString.Base64.URL as U
|
||||
import qualified Data.ByteString.Char8 as B
|
||||
import qualified Data.ByteString.Lazy as LB
|
||||
import Data.Either (partitionEithers)
|
||||
import Data.Functor (($>))
|
||||
import Data.Int (Int64)
|
||||
import Data.List (foldl', sortBy)
|
||||
@@ -1480,11 +1474,9 @@ createSndRatchet db connId ratchetState (CR.AE2ERatchetParams s (CR.E2ERatchetPa
|
||||
db
|
||||
[sql|
|
||||
INSERT INTO ratchets
|
||||
(conn_id, ratchet_state, rc_verify_code_ad, rc_verify_code_pq, x3dh_pub_key_1, x3dh_pub_key_2, pq_pub_kem) VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
(conn_id, ratchet_state, x3dh_pub_key_1, x3dh_pub_key_2, pq_pub_kem) VALUES (?, ?, ?, ?, ?)
|
||||
ON CONFLICT (conn_id) DO UPDATE SET
|
||||
ratchet_state = EXCLUDED.ratchet_state,
|
||||
rc_verify_code_ad = EXCLUDED.rc_verify_code_ad,
|
||||
rc_verify_code_pq = EXCLUDED.rc_verify_code_pq,
|
||||
x3dh_priv_key_1 = NULL,
|
||||
x3dh_priv_key_2 = NULL,
|
||||
x3dh_pub_key_1 = EXCLUDED.x3dh_pub_key_1,
|
||||
@@ -1492,13 +1484,7 @@ createSndRatchet db connId ratchetState (CR.AE2ERatchetParams s (CR.E2ERatchetPa
|
||||
pq_priv_kem = NULL,
|
||||
pq_pub_kem = EXCLUDED.pq_pub_kem
|
||||
|]
|
||||
((connId, ratchetState) :. verifyCodesRow (ratchetVerifyCodes ratchetState) :. (x3dhPubKey1, x3dhPubKey2, CR.ARKP s <$> pqPubKem))
|
||||
|
||||
ratchetVerifyCodes :: RatchetX448 -> ConnVerifyCodes
|
||||
ratchetVerifyCodes CR.Ratchet {rcAD, rcVCPQ} = ConnVerifyCodes {codeAD = C.sha256Hash $ unStr rcAD, codePQ = unStr <$> rcVCPQ}
|
||||
|
||||
verifyCodesRow :: ConnVerifyCodes -> (Binary ByteString, Maybe (Binary ByteString))
|
||||
verifyCodesRow ConnVerifyCodes {codeAD, codePQ} = (Binary codeAD, Binary <$> codePQ)
|
||||
(connId, ratchetState, x3dhPubKey1, x3dhPubKey2, CR.ARKP s <$> pqPubKem)
|
||||
|
||||
getSndRatchet :: DB.Connection -> ConnId -> CR.VersionE2E -> IO (Either StoreError (RatchetX448, CR.AE2ERatchetParams 'C.X448))
|
||||
getSndRatchet db connId v =
|
||||
@@ -1519,12 +1505,10 @@ createRatchet db connId rc =
|
||||
DB.execute
|
||||
db
|
||||
[sql|
|
||||
INSERT INTO ratchets (conn_id, ratchet_state, rc_verify_code_ad, rc_verify_code_pq)
|
||||
VALUES (?, ?, ?, ?)
|
||||
INSERT INTO ratchets (conn_id, ratchet_state)
|
||||
VALUES (?, ?)
|
||||
ON CONFLICT (conn_id) DO UPDATE SET
|
||||
ratchet_state = EXCLUDED.ratchet_state,
|
||||
rc_verify_code_ad = EXCLUDED.rc_verify_code_ad,
|
||||
rc_verify_code_pq = EXCLUDED.rc_verify_code_pq,
|
||||
ratchet_state = ?,
|
||||
x3dh_priv_key_1 = NULL,
|
||||
x3dh_priv_key_2 = NULL,
|
||||
x3dh_pub_key_1 = NULL,
|
||||
@@ -1532,42 +1516,7 @@ createRatchet db connId rc =
|
||||
pq_priv_kem = NULL,
|
||||
pq_pub_kem = NULL
|
||||
|]
|
||||
((connId, rc) :. verifyCodesRow (ratchetVerifyCodes rc))
|
||||
|
||||
getRatchetVerifyCodes :: DB.Connection -> [ConnId] -> IO (Map ConnId ConnVerifyCodes)
|
||||
getRatchetVerifyCodes db connIds = do
|
||||
(saved, computed) <- partitionEithers . mapMaybe rowCodes <$> selectCodes
|
||||
stored <- saveCodes computed
|
||||
pure $ M.fromList $ saved <> stored
|
||||
where
|
||||
rowCodes (connId, codeAD_, codePQ_, rc_) = case codeAD_ of
|
||||
Just codeAD -> Just $ Left $ toCodes (connId, codeAD, codePQ_)
|
||||
Nothing -> Right . (connId,) . ratchetVerifyCodes <$> rc_
|
||||
toCodes (connId, Binary codeAD, codePQ_) = (connId, ConnVerifyCodes {codeAD, codePQ = fromBinary <$> codePQ_})
|
||||
codesRow (connId, cs) = verifyCodesRow cs :. Only connId
|
||||
codesQuery :: Query
|
||||
codesQuery = "SELECT conn_id, rc_verify_code_ad, rc_verify_code_pq, CASE WHEN rc_verify_code_ad IS NULL THEN ratchet_state END FROM ratchets"
|
||||
selectCodes :: IO [(ConnId, Maybe (Binary ByteString), Maybe (Binary ByteString), Maybe RatchetX448)]
|
||||
saveCodes :: [(ConnId, ConnVerifyCodes)] -> IO [(ConnId, ConnVerifyCodes)]
|
||||
#if defined(dbPostgres)
|
||||
selectCodes = DB.query db (codesQuery <> " WHERE conn_id IN ?") (Only (In connIds))
|
||||
saveCodes cs =
|
||||
map toCodes
|
||||
<$> DB.returning
|
||||
db
|
||||
[sql|
|
||||
UPDATE ratchets r
|
||||
SET rc_verify_code_ad = COALESCE(r.rc_verify_code_ad, (upd.rc_verify_code_ad :: BYTEA)),
|
||||
rc_verify_code_pq = CASE WHEN r.rc_verify_code_ad IS NULL THEN (upd.rc_verify_code_pq :: BYTEA) ELSE r.rc_verify_code_pq END
|
||||
FROM (VALUES(?, ?, ?)) AS upd(rc_verify_code_ad, rc_verify_code_pq, conn_id)
|
||||
WHERE r.conn_id = (upd.conn_id :: BYTEA)
|
||||
RETURNING r.conn_id, r.rc_verify_code_ad, r.rc_verify_code_pq
|
||||
|]
|
||||
(map codesRow cs)
|
||||
#else
|
||||
selectCodes = concat <$> mapM (DB.query db (codesQuery <> " WHERE conn_id = ?") . Only) connIds
|
||||
saveCodes cs = cs <$ unless (null cs) (DB.executeMany db "UPDATE ratchets SET rc_verify_code_ad = ?, rc_verify_code_pq = ? WHERE conn_id = ?" $ map codesRow cs)
|
||||
#endif
|
||||
(connId, rc, rc)
|
||||
|
||||
deleteRatchet :: DB.Connection -> ConnId -> IO ()
|
||||
deleteRatchet db connId =
|
||||
@@ -1595,16 +1544,12 @@ getRatchet_ q db connId =
|
||||
where
|
||||
ratchet = maybe (Left SERatchetNotFound) Right . fromOnly
|
||||
|
||||
getSkippedMsgKeys :: DB.Connection -> ConnId -> Int -> IO SkippedMsgKeys
|
||||
getSkippedMsgKeys db connId maxKeys = do
|
||||
(keys, oldKeys) <- splitAt maxKeys <$> DB.query db "SELECT skipped_message_id, header_key, msg_n, msg_key FROM skipped_messages WHERE conn_id = ? ORDER BY skipped_message_id DESC LIMIT ?" (connId, maxKeys + 1)
|
||||
case oldKeys of
|
||||
(skippedMsgId :: Int64, _, _, _) : _ -> DB.execute db "DELETE FROM skipped_messages WHERE conn_id = ? AND skipped_message_id <= ?" (connId, skippedMsgId)
|
||||
[] -> pure ()
|
||||
pure $ skipped keys
|
||||
getSkippedMsgKeys :: DB.Connection -> ConnId -> IO SkippedMsgKeys
|
||||
getSkippedMsgKeys db connId =
|
||||
skipped <$> DB.query db "SELECT header_key, msg_n, msg_key FROM skipped_messages WHERE conn_id = ?" (Only connId)
|
||||
where
|
||||
skipped = foldl' addSkippedKey M.empty
|
||||
addSkippedKey smks (_, hk, msgN, mk) = M.alter (Just . addMsgKey) hk smks
|
||||
addSkippedKey smks (hk, msgN, mk) = M.alter (Just . addMsgKey) hk smks
|
||||
where
|
||||
addMsgKey = maybe (M.singleton msgN mk) (M.insert msgN mk)
|
||||
|
||||
@@ -1723,7 +1668,7 @@ getPendingServerCommand db connId srv_ = getWorkItem "command" getCmdId getComma
|
||||
DB.query
|
||||
db
|
||||
[sql|
|
||||
SELECT c.corr_id, cs.user_id, c.command, c.created_at
|
||||
SELECT c.corr_id, cs.user_id, c.command
|
||||
FROM commands c
|
||||
JOIN connections cs USING (conn_id)
|
||||
WHERE c.command_id = ?
|
||||
@@ -1731,7 +1676,7 @@ getPendingServerCommand db connId srv_ = getWorkItem "command" getCmdId getComma
|
||||
(Only cmdId)
|
||||
where
|
||||
err = SEInternal $ "command " <> bshow cmdId <> " returned []"
|
||||
pendingCommand (corrId, userId, command, createdAt) = PendingCommand {cmdId, corrId, userId, connId, command, createdAt}
|
||||
pendingCommand (corrId, userId, command) = PendingCommand {cmdId, corrId, userId, connId, command}
|
||||
markCommandFailed cmdId = DB.execute db "UPDATE commands SET failed = 1 WHERE command_id = ?" (Only cmdId)
|
||||
|
||||
updateCommandServer :: DB.Connection -> AsyncCmdId -> SMPServer -> IO (Either StoreError ())
|
||||
@@ -2505,12 +2450,12 @@ getSubscriptionServers db onlyNeeded =
|
||||
|
||||
-- TODO [certs rcv] check index for getting queues with service present
|
||||
getUserServerRcvQueueSubs :: DB.Connection -> UserId -> SMPServer -> Bool -> ServiceAssoc -> Int -> Maybe SMP.RecipientId -> IO [RcvQueueSub]
|
||||
getUserServerRcvQueueSubs db userId srv@(SMPServer h p kh) onlyNeeded hasService limit cursor_ =
|
||||
map (toServerRcvQueueSub srv) <$> case cursor_ of
|
||||
getUserServerRcvQueueSubs db userId (SMPServer h p kh) onlyNeeded hasService limit cursor_ =
|
||||
map toRcvQueueSub <$> case cursor_ of
|
||||
Nothing -> DB.query db (q <> orderLimit) (userId, h, p, kh, limit)
|
||||
Just cursor -> DB.query db (q <> " AND q.rcv_id > ? " <> orderLimit) (userId, h, p, kh, cursor, limit)
|
||||
where
|
||||
q = serverRcvQueueSubQuery <> toSubscribe <> " c.deleted = 0 AND q.deleted = 0 AND c.user_id = ? AND q.host = ? AND q.port = ? AND COALESCE(q.server_key_hash, s.key_hash) = ?" <> serviceCond
|
||||
q = rcvQueueSubQuery <> toSubscribe <> " c.deleted = 0 AND q.deleted = 0 AND c.user_id = ? AND q.host = ? AND q.port = ? AND COALESCE(q.server_key_hash, s.key_hash) = ?" <> serviceCond
|
||||
orderLimit = " ORDER BY q.rcv_id LIMIT ?"
|
||||
toSubscribe
|
||||
| onlyNeeded = " WHERE q.to_subscribe = 1 AND "
|
||||
@@ -2523,7 +2468,7 @@ unassocUserServerRcvQueueSubs :: DB.Connection -> UserId -> SMPServer -> IO [Rcv
|
||||
unassocUserServerRcvQueueSubs db userId srv@(SMPServer h p kh) = do
|
||||
deleteClientService db userId srv
|
||||
#if defined(dbPostgres)
|
||||
map (toServerRcvQueueSub srv)
|
||||
map toRcvQueueSub
|
||||
<$> DB.query
|
||||
db
|
||||
(removeRcvAssocsQuery <> " " <> returningColumns)
|
||||
@@ -2531,15 +2476,15 @@ unassocUserServerRcvQueueSubs db userId srv@(SMPServer h p kh) = do
|
||||
where
|
||||
returningColumns =
|
||||
[sql|
|
||||
RETURNING c.user_id, rcv_queues.conn_id,
|
||||
RETURNING c.user_id, rcv_queues.conn_id, rcv_queues.host, rcv_queues.port, COALESCE(rcv_queues.server_key_hash, s.key_hash),
|
||||
rcv_queues.rcv_id, rcv_queues.rcv_private_key, rcv_queues.status, c.enable_ntfs, rcv_queues.client_notice_id,
|
||||
rcv_queues.rcv_queue_id, rcv_queues.rcv_primary, rcv_queues.replace_rcv_queue_id
|
||||
|]
|
||||
#else
|
||||
qs <- map (toServerRcvQueueSub srv)
|
||||
qs <- map toRcvQueueSub
|
||||
<$> DB.query
|
||||
db
|
||||
(serverRcvQueueSubQuery <> " WHERE c.user_id = ? AND q.host = ? AND q.port = ? AND COALESCE(q.server_key_hash, s.key_hash) = ? AND q.rcv_service_assoc = 1")
|
||||
(rcvQueueSubQuery <> " WHERE c.user_id = ? AND q.host = ? AND q.port = ? AND COALESCE(q.server_key_hash, s.key_hash) = ? AND q.rcv_service_assoc = 1")
|
||||
(userId, h, p, kh)
|
||||
DB.execute db removeRcvAssocsQuery (h, p, userId, kh)
|
||||
pure qs
|
||||
@@ -2798,11 +2743,6 @@ getDeletedWaitingDeliveryConnIds :: DB.Connection -> IO [ConnId]
|
||||
getDeletedWaitingDeliveryConnIds db =
|
||||
map fromOnly <$> DB.query_ db "SELECT conn_id FROM connections WHERE deleted_at_wait_delivery IS NOT NULL"
|
||||
|
||||
getConnRatchetSync :: DB.Connection -> ConnId -> IO (Either StoreError RatchetSyncState)
|
||||
getConnRatchetSync db connId =
|
||||
firstRow fromOnly SEConnNotFound $
|
||||
DB.query db "SELECT ratchet_sync_state FROM connections WHERE conn_id = ?" (Only connId)
|
||||
|
||||
setConnRatchetSync :: DB.Connection -> ConnId -> RatchetSyncState -> IO ()
|
||||
setConnRatchetSync db connId ratchetSyncState =
|
||||
DB.execute db "UPDATE connections SET ratchet_sync_state = ? WHERE conn_id = ?" (ratchetSyncState, connId)
|
||||
@@ -2824,35 +2764,21 @@ checkRatchetKeyHashExists db connId hash =
|
||||
(connId, Binary hash)
|
||||
|
||||
deleteRatchetKeyHashesExpired :: DB.Connection -> NominalDiffTime -> Int -> IO ()
|
||||
deleteRatchetKeyHashesExpired db ttl maxConnHashes = do
|
||||
deleteRatchetKeyHashesExpired db ttl limit = do
|
||||
cutoffTs <- addUTCTime (-ttl) <$> getCurrentTime
|
||||
#if defined(dbPostgres)
|
||||
DB.execute
|
||||
db
|
||||
("DELETE FROM processed_ratchet_key_hashes h USING (" <> maxExcessIdsQuery <> ") e WHERE h.conn_id = e.conn_id AND h.processed_ratchet_key_hash_id <= e.max_excess_id AND h.created_at < ?")
|
||||
(maxConnHashes, maxConnHashes, cutoffTs)
|
||||
#else
|
||||
maxExcessIds <- DB.query db maxExcessIdsQuery (maxConnHashes, maxConnHashes)
|
||||
DB.executeMany
|
||||
db
|
||||
"DELETE FROM processed_ratchet_key_hashes WHERE conn_id = ? AND processed_ratchet_key_hash_id <= ? AND created_at < ?"
|
||||
(map (\(connId :: ConnId, maxExcessId :: Int64) -> (connId, maxExcessId, cutoffTs)) maxExcessIds)
|
||||
#endif
|
||||
where
|
||||
maxExcessIdsQuery :: Query
|
||||
maxExcessIdsQuery =
|
||||
[sql|
|
||||
SELECT conn_id, (
|
||||
SELECT processed_ratchet_key_hash_id
|
||||
FROM processed_ratchet_key_hashes
|
||||
WHERE conn_id = c.conn_id
|
||||
ORDER BY processed_ratchet_key_hash_id DESC
|
||||
LIMIT 1 OFFSET ?
|
||||
) AS max_excess_id
|
||||
FROM processed_ratchet_key_hashes c
|
||||
GROUP BY conn_id
|
||||
HAVING COUNT(*) > ?
|
||||
|]
|
||||
[sql|
|
||||
DELETE FROM processed_ratchet_key_hashes
|
||||
WHERE processed_ratchet_key_hash_id IN (
|
||||
SELECT processed_ratchet_key_hash_id
|
||||
FROM processed_ratchet_key_hashes
|
||||
WHERE created_at < ?
|
||||
ORDER BY created_at ASC
|
||||
LIMIT ?
|
||||
)
|
||||
|]
|
||||
(cutoffTs, limit)
|
||||
|
||||
-- | returns all connection queues, the first queue is the primary one
|
||||
getRcvQueuesByConnId_ :: DB.Connection -> ConnId -> IO (Maybe (NonEmpty RcvQueue))
|
||||
@@ -2909,32 +2835,17 @@ getRcvQueueSubsByConnId_ db connId =
|
||||
rcvQueueSubQuery :: Query
|
||||
rcvQueueSubQuery =
|
||||
[sql|
|
||||
SELECT q.host, q.port, COALESCE(q.server_key_hash, s.key_hash), c.user_id, q.conn_id, q.rcv_id, q.rcv_private_key, q.status, c.enable_ntfs, q.client_notice_id,
|
||||
SELECT c.user_id, q.conn_id, q.host, q.port, COALESCE(q.server_key_hash, s.key_hash), q.rcv_id, q.rcv_private_key, q.status, c.enable_ntfs, q.client_notice_id,
|
||||
q.rcv_queue_id, q.rcv_primary, q.replace_rcv_queue_id
|
||||
FROM rcv_queues q
|
||||
JOIN servers s ON q.host = s.host AND q.port = s.port
|
||||
JOIN connections c ON q.conn_id = c.conn_id
|
||||
|]
|
||||
|
||||
serverRcvQueueSubQuery :: Query
|
||||
serverRcvQueueSubQuery =
|
||||
[sql|
|
||||
SELECT c.user_id, q.conn_id, q.rcv_id, q.rcv_private_key, q.status, c.enable_ntfs, q.client_notice_id,
|
||||
q.rcv_queue_id, q.rcv_primary, q.replace_rcv_queue_id
|
||||
FROM rcv_queues q
|
||||
JOIN servers s ON q.host = s.host AND q.port = s.port
|
||||
JOIN connections c ON q.conn_id = c.conn_id
|
||||
|]
|
||||
|
||||
type RcvQueueSubRow = (UserId, ConnId, SMP.RecipientId, SMP.RcvPrivateAuthKey) :. (QueueStatus, Maybe BoolInt, Maybe NoticeId, Int64, BoolInt, Maybe Int64)
|
||||
|
||||
toRcvQueueSub :: (NonEmpty TransportHost, ServiceName, C.KeyHash) :. RcvQueueSubRow -> RcvQueueSub
|
||||
toRcvQueueSub ((host, port, keyHash) :. r) = toServerRcvQueueSub (SMPServer host port keyHash) r
|
||||
|
||||
toServerRcvQueueSub :: SMPServer -> RcvQueueSubRow -> RcvQueueSub
|
||||
toServerRcvQueueSub server ((userId, connId, rcvId, rcvPrivateKey) :. (status, enableNtfs_, clientNoticeId, dbQueueId, BI primary, dbReplaceQueueId)) =
|
||||
toRcvQueueSub :: (UserId, ConnId, NonEmpty TransportHost, ServiceName, C.KeyHash, SMP.RecipientId, SMP.RcvPrivateAuthKey) :. (QueueStatus, Maybe BoolInt, Maybe NoticeId, Int64, BoolInt, Maybe Int64) -> RcvQueueSub
|
||||
toRcvQueueSub ((userId, connId, host, port, keyHash, rcvId, rcvPrivateKey) :. (status, enableNtfs_, clientNoticeId, dbQueueId, BI primary, dbReplaceQueueId)) =
|
||||
let enableNtfs = maybe True unBI enableNtfs_
|
||||
in RcvQueueSub {userId, connId, server, rcvId, rcvPrivateKey, status, enableNtfs, clientNoticeId, dbQueueId, primary, dbReplaceQueueId}
|
||||
in RcvQueueSub {userId, connId, server = SMPServer host port keyHash, rcvId, rcvPrivateKey, status, enableNtfs, clientNoticeId, dbQueueId, primary, dbReplaceQueueId}
|
||||
|
||||
getRcvQueueById :: DB.Connection -> ConnId -> Int64 -> IO (Either StoreError RcvQueue)
|
||||
getRcvQueueById db connId dbRcvId =
|
||||
@@ -3204,29 +3115,6 @@ createRcvFileRedirect db gVar userId redirectFd@FileDescription {chunks = redire
|
||||
chunks = []
|
||||
}
|
||||
|
||||
startPreparedRcvFile :: DB.Connection -> RcvFileId -> IO (Either StoreError [XFTPServer])
|
||||
startPreparedRcvFile db rcvFileEntityId = runExceptT $ do
|
||||
rcvFileId <- ExceptT $ getRcvFileIdByEntityId_ db rcvFileEntityId
|
||||
liftIO $ do
|
||||
updatedAt <- getCurrentTime
|
||||
DB.execute
|
||||
db
|
||||
"UPDATE rcv_files SET status = ?, updated_at = ? WHERE (rcv_file_id = ? OR redirect_id = ?) AND status = ?"
|
||||
(RFSReceiving, updatedAt, rcvFileId, rcvFileId, RFSPrepared)
|
||||
map toXFTPServer
|
||||
<$> DB.query
|
||||
db
|
||||
[sql|
|
||||
SELECT DISTINCT
|
||||
s.xftp_host, s.xftp_port, s.xftp_key_hash
|
||||
FROM rcv_file_chunk_replicas r
|
||||
JOIN xftp_servers s ON s.xftp_server_id = r.xftp_server_id
|
||||
JOIN rcv_file_chunks c ON c.rcv_file_chunk_id = r.rcv_file_chunk_id
|
||||
JOIN rcv_files f ON f.rcv_file_id = c.rcv_file_id
|
||||
WHERE (f.rcv_file_id = ? OR f.redirect_id = ?) AND r.replica_number = 1
|
||||
|]
|
||||
(rcvFileId, rcvFileId)
|
||||
|
||||
insertRcvFile :: DB.Connection -> TVar ChaChaDRG -> UserId -> FileDescription 'FRecipient -> FilePath -> FilePath -> CryptoFile -> Maybe DBRcvFileId -> Maybe RcvFileId -> Bool -> IO (Either StoreError (RcvFileId, DBRcvFileId))
|
||||
insertRcvFile db gVar userId FileDescription {size, digest, key, nonce, chunkSize, redirect} prefixPath tmpPath (CryptoFile savePath cfArgs) redirectId_ redirectEntityId_ approvedRelays = runExceptT $ do
|
||||
let (redirectDigest_, redirectSize_) = case redirect of
|
||||
@@ -3237,7 +3125,7 @@ insertRcvFile db gVar userId FileDescription {size, digest, key, nonce, chunkSiz
|
||||
DB.execute
|
||||
db
|
||||
"INSERT INTO rcv_files (rcv_file_entity_id, user_id, size, digest, key, nonce, chunk_size, prefix_path, tmp_path, save_path, save_file_key, save_file_nonce, status, redirect_id, redirect_entity_id, redirect_digest, redirect_size, approved_relays) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
||||
((Binary rcvFileEntityId, userId, size, digest, key, nonce, chunkSize, prefixPath, tmpPath) :. (savePath, fileKey <$> cfArgs, fileNonce <$> cfArgs, RFSPrepared, redirectId_, Binary <$> redirectEntityId_, redirectDigest_, redirectSize_, BI approvedRelays))
|
||||
((Binary rcvFileEntityId, userId, size, digest, key, nonce, chunkSize, prefixPath, tmpPath) :. (savePath, fileKey <$> cfArgs, fileNonce <$> cfArgs, RFSReceiving, redirectId_, Binary <$> redirectEntityId_, redirectDigest_, redirectSize_, BI approvedRelays))
|
||||
rcvFileId <- liftIO $ insertedRowId db
|
||||
pure (rcvFileEntityId, rcvFileId)
|
||||
|
||||
@@ -3542,23 +3430,13 @@ createSndFile db gVar userId (CryptoFile path cfArgs) numRecipients prefixPath k
|
||||
DB.execute
|
||||
db
|
||||
"INSERT INTO snd_files (snd_file_entity_id, user_id, path, src_file_key, src_file_nonce, num_recipients, prefix_path, key, nonce, status, redirect_size, redirect_digest, storage_time) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
||||
((Binary sndFileEntityId, userId, path, fileKey <$> cfArgs, fileNonce <$> cfArgs, numRecipients) :. (prefixPath, key, nonce, SFSPrepared, redirectSize_, redirectDigest_, storageHours))
|
||||
((Binary sndFileEntityId, userId, path, fileKey <$> cfArgs, fileNonce <$> cfArgs, numRecipients) :. (prefixPath, key, nonce, SFSNew, redirectSize_, redirectDigest_, storageHours))
|
||||
where
|
||||
(redirectSize_, redirectDigest_) =
|
||||
case redirect_ of
|
||||
Nothing -> (Nothing, Nothing)
|
||||
Just RedirectFileInfo {size, digest} -> (Just size, Just digest)
|
||||
|
||||
startPreparedSndFile :: DB.Connection -> SndFileId -> IO (Either StoreError ())
|
||||
startPreparedSndFile db sndFileEntityId = runExceptT $ do
|
||||
sndFileId <- ExceptT $ getSndFileIdByEntityId_ db sndFileEntityId
|
||||
liftIO $ do
|
||||
updatedAt <- getCurrentTime
|
||||
DB.execute
|
||||
db
|
||||
"UPDATE snd_files SET status = ?, updated_at = ? WHERE snd_file_id = ? AND status = ?"
|
||||
(SFSNew, updatedAt, sndFileId, SFSPrepared)
|
||||
|
||||
getSndFileByEntityId :: DB.Connection -> SndFileId -> IO (Either StoreError SndFile)
|
||||
getSndFileByEntityId db sndFileEntityId = runExceptT $ do
|
||||
sndFileId <- ExceptT $ getSndFileIdByEntityId_ db sndFileEntityId
|
||||
|
||||
@@ -13,7 +13,6 @@ module Simplex.Messaging.Agent.Store.Postgres.DB
|
||||
execute,
|
||||
execute_,
|
||||
executeMany,
|
||||
returning,
|
||||
query,
|
||||
query_,
|
||||
blobFieldDecoder,
|
||||
@@ -61,10 +60,6 @@ executeMany :: ToRow q => PSQL.Connection -> Query -> [q] -> IO ()
|
||||
executeMany db q qs = void $ PSQL.executeMany db q qs `E.catch` addSql q
|
||||
{-# INLINE executeMany #-}
|
||||
|
||||
returning :: (ToRow q, FromRow r) => PSQL.Connection -> Query -> [q] -> IO [r]
|
||||
returning db q qs = PSQL.returning db q qs `E.catch` addSql q
|
||||
{-# INLINE returning #-}
|
||||
|
||||
query :: (ToRow q, FromRow r) => PSQL.Connection -> Query -> q -> IO [r]
|
||||
query db q qs = PSQL.query db q qs `E.catch` addSql q
|
||||
{-# INLINE query #-}
|
||||
|
||||
@@ -15,8 +15,6 @@ import Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260410_receive_attem
|
||||
import Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260411_service_certs
|
||||
import Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260712_address_dr_rpc
|
||||
import Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260823_snd_files_entitlement
|
||||
import Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260919_ratchet_verify_codes
|
||||
import Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260929_ratchet_indexes
|
||||
import Simplex.Messaging.Agent.Store.Shared (Migration (..))
|
||||
|
||||
schemaMigrations :: [(String, Text, Maybe Text)]
|
||||
@@ -31,9 +29,7 @@ schemaMigrations =
|
||||
("20260410_receive_attempts", m20260410_receive_attempts, Just down_m20260410_receive_attempts),
|
||||
("20260411_service_certs", m20260411_service_certs, Just down_m20260411_service_certs),
|
||||
("20260712_address_dr_rpc", m20260712_address_dr_rpc, Just down_m20260712_address_dr_rpc),
|
||||
("20260823_snd_files_entitlement", m20260823_snd_files_entitlement, Just down_m20260823_snd_files_entitlement),
|
||||
("20260919_ratchet_verify_codes", m20260919_ratchet_verify_codes, Just down_m20260919_ratchet_verify_codes),
|
||||
("20260929_ratchet_indexes", m20260929_ratchet_indexes, Just down_m20260929_ratchet_indexes)
|
||||
("20260823_snd_files_entitlement", m20260823_snd_files_entitlement, Just down_m20260823_snd_files_entitlement)
|
||||
]
|
||||
|
||||
-- | The list of migrations in ascending order by date
|
||||
|
||||
-21
@@ -1,21 +0,0 @@
|
||||
{-# LANGUAGE OverloadedStrings #-}
|
||||
{-# LANGUAGE QuasiQuotes #-}
|
||||
|
||||
module Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260919_ratchet_verify_codes where
|
||||
|
||||
import Data.Text (Text)
|
||||
import Text.RawString.QQ (r)
|
||||
|
||||
m20260919_ratchet_verify_codes :: Text
|
||||
m20260919_ratchet_verify_codes =
|
||||
[r|
|
||||
ALTER TABLE ratchets ADD COLUMN rc_verify_code_ad BYTEA;
|
||||
ALTER TABLE ratchets ADD COLUMN rc_verify_code_pq BYTEA;
|
||||
|]
|
||||
|
||||
down_m20260919_ratchet_verify_codes :: Text
|
||||
down_m20260919_ratchet_verify_codes =
|
||||
[r|
|
||||
ALTER TABLE ratchets DROP COLUMN rc_verify_code_pq;
|
||||
ALTER TABLE ratchets DROP COLUMN rc_verify_code_ad;
|
||||
|]
|
||||
@@ -1,34 +0,0 @@
|
||||
{-# LANGUAGE OverloadedStrings #-}
|
||||
{-# LANGUAGE QuasiQuotes #-}
|
||||
|
||||
module Simplex.Messaging.Agent.Store.Postgres.Migrations.M20260929_ratchet_indexes where
|
||||
|
||||
import Data.Text (Text)
|
||||
import Text.RawString.QQ (r)
|
||||
|
||||
m20260929_ratchet_indexes :: Text
|
||||
m20260929_ratchet_indexes =
|
||||
[r|
|
||||
DROP INDEX idx_skipped_messages_conn_id;
|
||||
CREATE INDEX idx_skipped_messages_conn_id ON skipped_messages(conn_id, skipped_message_id);
|
||||
|
||||
DELETE FROM processed_ratchet_key_hashes AS h
|
||||
WHERE EXISTS (
|
||||
SELECT 1 FROM processed_ratchet_key_hashes d
|
||||
WHERE d.conn_id = h.conn_id AND d.hash = h.hash AND d.processed_ratchet_key_hash_id < h.processed_ratchet_key_hash_id
|
||||
);
|
||||
|
||||
DROP INDEX idx_processed_ratchet_key_hashes_hash;
|
||||
CREATE UNIQUE INDEX idx_processed_ratchet_key_hashes_hash ON processed_ratchet_key_hashes(conn_id, hash);
|
||||
CREATE INDEX idx_processed_ratchet_key_hashes_conn_id ON processed_ratchet_key_hashes(conn_id, processed_ratchet_key_hash_id);
|
||||
|]
|
||||
|
||||
down_m20260929_ratchet_indexes :: Text
|
||||
down_m20260929_ratchet_indexes =
|
||||
[r|
|
||||
DROP INDEX idx_processed_ratchet_key_hashes_conn_id;
|
||||
DROP INDEX idx_processed_ratchet_key_hashes_hash;
|
||||
CREATE INDEX idx_processed_ratchet_key_hashes_hash ON processed_ratchet_key_hashes(conn_id, hash);
|
||||
DROP INDEX idx_skipped_messages_conn_id;
|
||||
CREATE INDEX idx_skipped_messages_conn_id ON skipped_messages(conn_id);
|
||||
|]
|
||||
@@ -448,9 +448,7 @@ CREATE TABLE smp_agent_test_protocol_schema.ratchets (
|
||||
x3dh_pub_key_1 bytea,
|
||||
x3dh_pub_key_2 bytea,
|
||||
pq_priv_kem bytea,
|
||||
pq_pub_kem bytea,
|
||||
rc_verify_code_ad bytea,
|
||||
rc_verify_code_pq bytea
|
||||
pq_pub_kem bytea
|
||||
);
|
||||
|
||||
|
||||
@@ -1169,15 +1167,11 @@ CREATE INDEX idx_ntf_tokens_ntf_host_ntf_port ON smp_agent_test_protocol_schema.
|
||||
|
||||
|
||||
|
||||
CREATE INDEX idx_processed_ratchet_key_hashes_conn_id ON smp_agent_test_protocol_schema.processed_ratchet_key_hashes USING btree (conn_id, processed_ratchet_key_hash_id);
|
||||
|
||||
|
||||
|
||||
CREATE INDEX idx_processed_ratchet_key_hashes_created_at ON smp_agent_test_protocol_schema.processed_ratchet_key_hashes USING btree (created_at);
|
||||
|
||||
|
||||
|
||||
CREATE UNIQUE INDEX idx_processed_ratchet_key_hashes_hash ON smp_agent_test_protocol_schema.processed_ratchet_key_hashes USING btree (conn_id, hash);
|
||||
CREATE INDEX idx_processed_ratchet_key_hashes_hash ON smp_agent_test_protocol_schema.processed_ratchet_key_hashes USING btree (conn_id, hash);
|
||||
|
||||
|
||||
|
||||
@@ -1245,7 +1239,7 @@ CREATE UNIQUE INDEX idx_server_certs_user_id_host_port ON smp_agent_test_protoco
|
||||
|
||||
|
||||
|
||||
CREATE INDEX idx_skipped_messages_conn_id ON smp_agent_test_protocol_schema.skipped_messages USING btree (conn_id, skipped_message_id);
|
||||
CREATE INDEX idx_skipped_messages_conn_id ON smp_agent_test_protocol_schema.skipped_messages USING btree (conn_id);
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -51,8 +51,6 @@ import Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260410_receive_attempt
|
||||
import Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260411_service_certs
|
||||
import Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260712_address_dr_rpc
|
||||
import Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260823_snd_files_entitlement
|
||||
import Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260919_ratchet_verify_codes
|
||||
import Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260929_ratchet_indexes
|
||||
import Simplex.Messaging.Agent.Store.Shared (Migration (..))
|
||||
|
||||
schemaMigrations :: [(String, Query, Maybe Query)]
|
||||
@@ -103,9 +101,7 @@ schemaMigrations =
|
||||
("m20260410_receive_attempts", m20260410_receive_attempts, Just down_m20260410_receive_attempts),
|
||||
("m20260411_service_certs", m20260411_service_certs, Just down_m20260411_service_certs),
|
||||
("m20260712_address_dr_rpc", m20260712_address_dr_rpc, Just down_m20260712_address_dr_rpc),
|
||||
("m20260823_snd_files_entitlement", m20260823_snd_files_entitlement, Just down_m20260823_snd_files_entitlement),
|
||||
("m20260919_ratchet_verify_codes", m20260919_ratchet_verify_codes, Just down_m20260919_ratchet_verify_codes),
|
||||
("m20260929_ratchet_indexes", m20260929_ratchet_indexes, Just down_m20260929_ratchet_indexes)
|
||||
("m20260823_snd_files_entitlement", m20260823_snd_files_entitlement, Just down_m20260823_snd_files_entitlement)
|
||||
]
|
||||
|
||||
-- | The list of migrations in ascending order by date
|
||||
|
||||
@@ -1,20 +0,0 @@
|
||||
{-# LANGUAGE QuasiQuotes #-}
|
||||
|
||||
module Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260919_ratchet_verify_codes where
|
||||
|
||||
import Database.SQLite.Simple (Query)
|
||||
import Database.SQLite.Simple.QQ (sql)
|
||||
|
||||
m20260919_ratchet_verify_codes :: Query
|
||||
m20260919_ratchet_verify_codes =
|
||||
[sql|
|
||||
ALTER TABLE ratchets ADD COLUMN rc_verify_code_ad BLOB;
|
||||
ALTER TABLE ratchets ADD COLUMN rc_verify_code_pq BLOB;
|
||||
|]
|
||||
|
||||
down_m20260919_ratchet_verify_codes :: Query
|
||||
down_m20260919_ratchet_verify_codes =
|
||||
[sql|
|
||||
ALTER TABLE ratchets DROP COLUMN rc_verify_code_pq;
|
||||
ALTER TABLE ratchets DROP COLUMN rc_verify_code_ad;
|
||||
|]
|
||||
@@ -1,29 +0,0 @@
|
||||
{-# LANGUAGE QuasiQuotes #-}
|
||||
|
||||
module Simplex.Messaging.Agent.Store.SQLite.Migrations.M20260929_ratchet_indexes where
|
||||
|
||||
import Database.SQLite.Simple (Query)
|
||||
import Database.SQLite.Simple.QQ (sql)
|
||||
|
||||
-- idx_skipped_messages_conn_id is not changed: SQLite index entries include rowid (skipped_message_id)
|
||||
m20260929_ratchet_indexes :: Query
|
||||
m20260929_ratchet_indexes =
|
||||
[sql|
|
||||
DELETE FROM processed_ratchet_key_hashes AS h
|
||||
WHERE EXISTS (
|
||||
SELECT 1 FROM processed_ratchet_key_hashes d
|
||||
WHERE d.conn_id = h.conn_id AND d.hash = h.hash AND d.processed_ratchet_key_hash_id < h.processed_ratchet_key_hash_id
|
||||
);
|
||||
|
||||
DROP INDEX idx_processed_ratchet_key_hashes_hash;
|
||||
CREATE UNIQUE INDEX idx_processed_ratchet_key_hashes_hash ON processed_ratchet_key_hashes(conn_id, hash);
|
||||
CREATE INDEX idx_processed_ratchet_key_hashes_conn_id ON processed_ratchet_key_hashes(conn_id, processed_ratchet_key_hash_id);
|
||||
|]
|
||||
|
||||
down_m20260929_ratchet_indexes :: Query
|
||||
down_m20260929_ratchet_indexes =
|
||||
[sql|
|
||||
DROP INDEX idx_processed_ratchet_key_hashes_conn_id;
|
||||
DROP INDEX idx_processed_ratchet_key_hashes_hash;
|
||||
CREATE INDEX idx_processed_ratchet_key_hashes_hash ON processed_ratchet_key_hashes(conn_id, hash);
|
||||
|]
|
||||
@@ -182,9 +182,7 @@ CREATE TABLE ratchets(
|
||||
x3dh_pub_key_1 BLOB,
|
||||
x3dh_pub_key_2 BLOB,
|
||||
pq_priv_kem BLOB,
|
||||
pq_pub_kem BLOB,
|
||||
rc_verify_code_ad BLOB,
|
||||
rc_verify_code_pq BLOB
|
||||
pq_pub_kem BLOB
|
||||
) WITHOUT ROWID, STRICT;
|
||||
CREATE TABLE skipped_messages(
|
||||
skipped_message_id INTEGER PRIMARY KEY,
|
||||
@@ -573,6 +571,10 @@ CREATE INDEX idx_encrypted_rcv_message_hashes_hash ON encrypted_rcv_message_hash
|
||||
conn_id,
|
||||
hash
|
||||
);
|
||||
CREATE INDEX idx_processed_ratchet_key_hashes_hash ON processed_ratchet_key_hashes(
|
||||
conn_id,
|
||||
hash
|
||||
);
|
||||
CREATE INDEX idx_snd_messages_rcpt_internal_id ON snd_messages(
|
||||
conn_id,
|
||||
rcpt_internal_id
|
||||
@@ -637,14 +639,6 @@ CREATE INDEX idx_connections_deleted ON connections(deleted);
|
||||
CREATE INDEX idx_connections_service_request_expires_at ON connections(
|
||||
service_request_expires_at
|
||||
);
|
||||
CREATE UNIQUE INDEX idx_processed_ratchet_key_hashes_hash ON processed_ratchet_key_hashes(
|
||||
conn_id,
|
||||
hash
|
||||
);
|
||||
CREATE INDEX idx_processed_ratchet_key_hashes_conn_id ON processed_ratchet_key_hashes(
|
||||
conn_id,
|
||||
processed_ratchet_key_hash_id
|
||||
);
|
||||
CREATE TRIGGER tr_rcv_queue_insert
|
||||
AFTER INSERT ON rcv_queues
|
||||
FOR EACH ROW
|
||||
|
||||
@@ -166,7 +166,7 @@ import Simplex.Messaging.Parsers (defaultJSON, dropPrefix, enumJSON, sumTypeJSON
|
||||
import Simplex.Messaging.Protocol
|
||||
import Simplex.Messaging.Protocol.Types
|
||||
import Simplex.Messaging.Server.QueueStore.QueueInfo
|
||||
import Simplex.Messaging.SimplexName (SimplexDomain, fullDomainName)
|
||||
import Simplex.Messaging.SimplexName (SimplexDomain)
|
||||
import Simplex.Messaging.TMap (TMap)
|
||||
import qualified Simplex.Messaging.TMap as TM
|
||||
import Simplex.Messaging.Transport
|
||||
@@ -195,7 +195,6 @@ data PClient v err msg = PClient
|
||||
transportHost :: TransportHost,
|
||||
tcpConnectTimeout :: NetworkTimeout,
|
||||
tcpTimeout :: NetworkTimeout,
|
||||
proxiedRelayVRange :: VersionRange v,
|
||||
sendPings :: TVar Bool,
|
||||
lastReceived :: TVar UTCTime,
|
||||
timeoutErrorCount :: TVar Int,
|
||||
@@ -241,7 +240,6 @@ smpClientStub g sessionId thVersion thAuth = do
|
||||
transportHost = "localhost",
|
||||
tcpConnectTimeout,
|
||||
tcpTimeout,
|
||||
proxiedRelayVRange = supportedClientSMPRelayVRange,
|
||||
sendPings,
|
||||
lastReceived,
|
||||
timeoutErrorCount,
|
||||
@@ -479,7 +477,6 @@ data ProtocolClientConfig v = ProtocolClientConfig
|
||||
serviceCredentials :: Maybe ServiceCredentials,
|
||||
-- | client-server protocol version range
|
||||
serverVRange :: VersionRange v,
|
||||
proxiedRelayVRange :: VersionRange v,
|
||||
-- | agree shared session secret (used in SMP proxy for additional encryption layer)
|
||||
agreeSecret :: Bool,
|
||||
-- | Whether connecting client is a proxy server. See comment in ClientHandshake
|
||||
@@ -498,7 +495,6 @@ defaultClientConfig clientALPN useSNI serverVRange =
|
||||
clientALPN,
|
||||
serviceCredentials = Nothing,
|
||||
serverVRange,
|
||||
proxiedRelayVRange = serverVRange,
|
||||
agreeSecret = False,
|
||||
proxyServer = False,
|
||||
useSNI
|
||||
@@ -509,7 +505,6 @@ defaultSMPClientConfig :: ProtocolClientConfig SMPVersion
|
||||
defaultSMPClientConfig =
|
||||
(defaultClientConfig (Just alpnSupportedSMPHandshakes) False supportedClientSMPRelayVRange)
|
||||
{ defaultTransport = (show defaultSMPPort, transport @TLS),
|
||||
proxiedRelayVRange = supportedClientSMPRelayVRange,
|
||||
agreeSecret = True
|
||||
}
|
||||
{-# INLINE defaultSMPClientConfig #-}
|
||||
@@ -573,7 +568,7 @@ type SMPTransportSession = TransportSession BrokerMsg
|
||||
-- A single queue can be used for multiple 'SMPClient' instances,
|
||||
-- as 'SMPServerTransmission' includes server information.
|
||||
getProtocolClient :: forall v err msg. Protocol v err msg => TVar ChaChaDRG -> NetworkRequestMode -> TransportSession msg -> ProtocolClientConfig v -> [HostName] -> Maybe (TBQueue (ServerTransmissionBatch v err msg)) -> UTCTime -> (ProtocolClient v err msg -> IO ()) -> IO (Either (ProtocolClientError err) (ProtocolClient v err msg))
|
||||
getProtocolClient g nm transportSession@(_, srv, _) cfg@ProtocolClientConfig {qSize, networkConfig, clientALPN, serviceCredentials, serverVRange, proxiedRelayVRange, agreeSecret, proxyServer, useSNI} presetDomains msgQ proxySessTs disconnected = do
|
||||
getProtocolClient g nm transportSession@(_, srv, _) cfg@ProtocolClientConfig {qSize, networkConfig, clientALPN, serviceCredentials, serverVRange, agreeSecret, proxyServer, useSNI} presetDomains msgQ proxySessTs disconnected = do
|
||||
case chooseTransportHost networkConfig (host srv) of
|
||||
Right useHost ->
|
||||
(getCurrentTime >>= mkProtocolClient useHost >>= runClient useTransport useHost)
|
||||
@@ -598,7 +593,6 @@ getProtocolClient g nm transportSession@(_, srv, _) cfg@ProtocolClientConfig {qS
|
||||
transportHost,
|
||||
tcpConnectTimeout,
|
||||
tcpTimeout,
|
||||
proxiedRelayVRange,
|
||||
sendPings,
|
||||
lastReceived,
|
||||
timeoutErrorCount,
|
||||
@@ -1060,11 +1054,11 @@ proxySMPMessage c nm proxiedRelay spKey sId flags msg = proxyOKSMPCommand c nm p
|
||||
-- through `proxySMPCommand` and pattern-matches the expected RNAME response.
|
||||
-- Version-gated on the destination relay (mirrors `connectSMPProxiedRelay`):
|
||||
-- the client never sends RSLV to a relay that predates names support.
|
||||
proxyResolveName :: SMPClient -> NetworkRequestMode -> ProxiedRelay -> SimplexDomain -> ExceptT SMPClientError IO (Either ProxyClientError NameResponse)
|
||||
proxyResolveName :: SMPClient -> NetworkRequestMode -> ProxiedRelay -> SimplexDomain -> ExceptT SMPClientError IO (Either ProxyClientError NameRecord)
|
||||
proxyResolveName c nm proxiedRelay name
|
||||
| prVersion proxiedRelay >= namesSMPVersion =
|
||||
proxySMPCommand c nm proxiedRelay Nothing NoEntity (RSLV (NQDomain name)) >>= \case
|
||||
Right (RNAME reg) | resolvedNameOrNotFound name reg -> pure $ Right reg
|
||||
proxySMPCommand c nm proxiedRelay Nothing NoEntity (RSLV name) >>= \case
|
||||
Right (RNAME nr) -> pure $ Right nr
|
||||
Right r -> throwE $ unexpectedResponse r
|
||||
Left e -> pure $ Left e
|
||||
| otherwise = throwE $ PCETransportError TEVersion
|
||||
@@ -1072,21 +1066,16 @@ proxyResolveName c nm proxiedRelay name
|
||||
-- | Direct (non-PFWD) name resolution. Exposes the client IP to the resolver;
|
||||
-- callers that want anonymity should use `proxyResolveName` via the standard
|
||||
-- proxy fallback in the agent. RSLV requires no entity ID or authorization
|
||||
-- (see `noAuthCmd` in Protocol.hs). Gated on the session version, below which
|
||||
-- the server has no RSLV at all; the encoder gates the query format separately.
|
||||
directResolveName :: SMPClient -> NetworkRequestMode -> SimplexDomain -> ExceptT SMPClientError IO NameResponse
|
||||
-- (see `noAuthCmd` in Protocol.hs). Version-gated on the session here, not the
|
||||
-- encoder, so an old server never receives RSLV.
|
||||
directResolveName :: SMPClient -> NetworkRequestMode -> SimplexDomain -> ExceptT SMPClientError IO NameRecord
|
||||
directResolveName c nm name
|
||||
| thVersion (thParams c) >= namesSMPVersion =
|
||||
sendProtocolCommand c nm Nothing NoEntity (Cmd SResolver (RSLV (NQDomain name))) >>= \case
|
||||
RNAME reg | resolvedNameOrNotFound name reg -> pure reg
|
||||
sendProtocolCommand c nm Nothing NoEntity (Cmd SResolver (RSLV name)) >>= \case
|
||||
RNAME nr -> pure nr
|
||||
r -> throwE $ unexpectedResponse r
|
||||
| otherwise = throwE $ PCETransportError TEVersion
|
||||
|
||||
resolvedNameOrNotFound :: SimplexDomain -> NameResponse -> Bool
|
||||
resolvedNameOrNotFound d NameResponse {registration} = case registration of
|
||||
NRRegistered {nameRecord} -> T.toLower (nrName nameRecord) == fullDomainName d
|
||||
_ -> True
|
||||
|
||||
-- | Acknowledge message delivery (server deletes the message).
|
||||
--
|
||||
-- https://github.com/simplex-chat/simplexmq/blob/master/protocol/simplex-messaging.md#acknowledge-message-delivery
|
||||
@@ -1121,10 +1110,10 @@ deleteSMPQueues = okSMPCommands DEL
|
||||
-- send PRXY :: SMPServer -> Maybe BasicAuth -> Command Sender
|
||||
-- receives PKEY :: SessionId -> X.CertificateChain -> X.SignedExact X.PubKey -> BrokerMsg
|
||||
connectSMPProxiedRelay :: SMPClient -> NetworkRequestMode -> SMPServer -> Maybe BasicAuth -> ExceptT SMPClientError IO ProxiedRelay
|
||||
connectSMPProxiedRelay c@ProtocolClient {client_ = PClient {tcpConnectTimeout, tcpTimeout, proxiedRelayVRange}} nm relayServ@ProtocolServer {port = relayPort, keyHash = C.KeyHash kh} proxyAuth =
|
||||
connectSMPProxiedRelay c@ProtocolClient {client_ = PClient {tcpConnectTimeout, tcpTimeout}} nm relayServ@ProtocolServer {port = relayPort, keyHash = C.KeyHash kh} proxyAuth =
|
||||
sendProtocolCommand_ c nm Nothing tOut Nothing NoEntity (Cmd SProxiedClient (PRXY relayServ proxyAuth)) >>= \case
|
||||
PKEY sId vr (CertChainPubKey chain key) ->
|
||||
case proxiedRelayVRange `compatibleVersion` vr of
|
||||
case supportedClientSMPRelayVRange `compatibleVersion` vr of
|
||||
Nothing -> throwE $ transportErr TEVersion
|
||||
Just (Compatible v) -> do
|
||||
relayKey <- liftEitherWith (const $ transportErr $ TEHandshake IDENTITY) =<< liftIO (runExceptT $ validateRelay chain key)
|
||||
@@ -1175,7 +1164,7 @@ instance StrEncoding ProxyClientError where
|
||||
-- consider how to process slow responses - is it handled somehow locally or delegated to the caller
|
||||
-- this method is used in the client
|
||||
-- sends PFWD :: C.PublicKeyX25519 -> EncTransmission -> Command Sender
|
||||
-- receives PRES :: Maybe C.CbNonce -> EncResponse -> BrokerMsg -- proxy to client
|
||||
-- receives PRES :: EncResponse -> BrokerMsg -- proxy to client
|
||||
|
||||
-- When client sends message via proxy, there may be one successful scenario and 9 error scenarios
|
||||
-- as shown below (WTF stands for unexpected response, ??? for response that failed to parse).
|
||||
@@ -1234,14 +1223,14 @@ proxySMPCommand c@ProtocolClient {thParams = proxyThParams, client_ = PClient {c
|
||||
TBError e _ : _ -> throwE $ PCETransportError e
|
||||
TBTransmission s _ : _ -> pure s
|
||||
TBTransmissions s _ _ : _ -> pure s
|
||||
et <- liftEitherWith PCECryptoError $ EncTransmission <$> C.cbEncrypt cmdSecret (encTransmissionNonce v nonce) b paddedProxiedTLength
|
||||
et <- liftEitherWith PCECryptoError $ EncTransmission <$> C.cbEncrypt cmdSecret nonce b paddedProxiedTLength
|
||||
-- proxy interaction errors are wrapped
|
||||
let tOut = Just $ 2 * netTimeoutInt tcpTimeout nm
|
||||
tryE (sendProtocolCommand_ c nm (Just nonce) tOut Nothing (EntityId sessionId) (Cmd SProxiedClient (PFWD v cmdPubKey et))) >>= \case
|
||||
Right r -> case r of
|
||||
PRES nonce_ (EncResponse er) -> do
|
||||
PRES (EncResponse er) -> do
|
||||
-- server interaction errors are thrown directly
|
||||
t' <- liftEitherWith PCECryptoError $ C.cbDecrypt cmdSecret (fromMaybe (C.reverseNonce nonce) nonce_) er
|
||||
t' <- liftEitherWith PCECryptoError $ C.cbDecrypt cmdSecret (C.reverseNonce nonce) er
|
||||
case tParse serverThParams t' of
|
||||
t'' :| [] -> case tDecodeClient serverThParams t'' of
|
||||
(_, _, cmd) -> case cmd of
|
||||
@@ -1259,10 +1248,10 @@ proxySMPCommand c@ProtocolClient {thParams = proxyThParams, client_ = PClient {c
|
||||
|
||||
-- this method is used in the proxy
|
||||
-- sends RFWD :: EncFwdTransmission -> Command Sender
|
||||
-- receives RRES :: Maybe C.CbNonce -> EncFwdResponse -> BrokerMsg
|
||||
-- receives RRES :: EncFwdResponse -> BrokerMsg
|
||||
-- proxy should send PRES to the client with EncResponse
|
||||
-- Always uses background timeout mode
|
||||
forwardSMPTransmission :: SMPClient -> CorrId -> VersionSMP -> C.PublicKeyX25519 -> EncTransmission -> ExceptT SMPClientError IO (Maybe C.CbNonce, EncResponse)
|
||||
forwardSMPTransmission :: SMPClient -> CorrId -> VersionSMP -> C.PublicKeyX25519 -> EncTransmission -> ExceptT SMPClientError IO EncResponse
|
||||
forwardSMPTransmission c@ProtocolClient {thParams, client_ = PClient {clientCorrId = g}} fwdCorrId fwdVersion fwdKey fwdTransmission = do
|
||||
-- prepare params
|
||||
sessSecret <- case thAuth thParams of
|
||||
@@ -1274,11 +1263,11 @@ forwardSMPTransmission c@ProtocolClient {thParams, client_ = PClient {clientCorr
|
||||
eft = EncFwdTransmission $ C.cbEncryptNoPad sessSecret nonce (smpEncode fwdT)
|
||||
-- send
|
||||
sendProtocolCommand_ c NRMBackground (Just nonce) Nothing Nothing NoEntity (Cmd SProxyService (RFWD eft)) >>= \case
|
||||
RRES nonce_ (EncFwdResponse efr) -> do
|
||||
RRES (EncFwdResponse efr) -> do
|
||||
-- unwrap
|
||||
r' <- liftEitherWith PCECryptoError $ C.cbDecryptNoPad sessSecret (C.reverseNonce nonce) efr
|
||||
FwdResponse {fwdCorrId = _, fwdResponse} <- liftEitherWith (const $ PCEResponseError BLOCK) $ smpDecode r'
|
||||
pure (nonce_, fwdResponse)
|
||||
pure fwdResponse
|
||||
r -> throwE $ unexpectedResponse r
|
||||
|
||||
-- get queue information - always sent interactively
|
||||
|
||||
@@ -931,8 +931,6 @@ data CryptoError
|
||||
CERatchetEarlierMessage Word32
|
||||
| -- | duplicate message number
|
||||
CERatchetDuplicateMessage
|
||||
| -- | KEM key generation failed, indicating a broken RNG
|
||||
CryptoKEMKeyGenError
|
||||
deriving (Eq, Show, Exception)
|
||||
|
||||
aesKeySize :: Int
|
||||
@@ -1047,7 +1045,9 @@ md5Hash = BA.convert . (hash :: ByteString -> Digest MD5)
|
||||
|
||||
-- | AEAD-GCM encryption with associated data.
|
||||
--
|
||||
-- Used as part of double ratchet encryption, with a 16-byte IV (see @initAEAD@).
|
||||
-- Used as part of double ratchet encryption.
|
||||
-- This function requires 16 bytes IV, it transforms IV in cryptonite_aes_gcm_init here:
|
||||
-- https://github.com/haskell-crypto/cryptonite/blob/master/cbits/cryptonite_aes.c
|
||||
encryptAEAD :: Key -> IV -> Int -> ByteString -> ByteString -> ExceptT CryptoError IO (AuthTag, ByteString)
|
||||
encryptAEAD aesKey ivBytes paddedLen ad msg = do
|
||||
aead <- initAEAD @AES256 aesKey ivBytes
|
||||
@@ -1067,7 +1067,10 @@ encryptAEADNoPad aesKey ivBytes ad msg = do
|
||||
|
||||
-- | AEAD-GCM decryption with associated data.
|
||||
--
|
||||
-- Used as part of double ratchet encryption, with a 16-byte IV (see @initAEAD@).
|
||||
-- Used as part of double ratchet encryption.
|
||||
-- This function requires 16 bytes IV, it transforms IV in cryptonite_aes_gcm_init here:
|
||||
-- https://github.com/haskell-crypto/cryptonite/blob/master/cbits/cryptonite_aes.c
|
||||
-- To make it compatible with WebCrypto we will need to start using initAEADGCM.
|
||||
decryptAEAD :: Key -> IV -> ByteString -> ByteString -> AuthTag -> ExceptT CryptoError IO ByteString
|
||||
decryptAEAD aesKey ivBytes ad msg (AuthTag authTag) = do
|
||||
aead <- initAEAD @AES256 aesKey ivBytes
|
||||
@@ -1145,9 +1148,9 @@ maxLength :: forall i. KnownNat i => Int
|
||||
maxLength = fromIntegral (natVal $ Proxy @i)
|
||||
{-# INLINE maxLength #-}
|
||||
|
||||
-- The 16-byte double ratchet IV is intentionally not the 96-bit IV recommended by NIST SP 800-38D, so GCM derives J0 = GHASH(IV || 0^64 || [128]_64),
|
||||
-- as in crypton_aes_gcm_init: https://hackage.haskell.org/package/crypton-0.34/src/cbits/crypton_aes.c
|
||||
-- WebCrypto and other SP 800-38D implementations interoperate only when given all 16 IV bytes.
|
||||
-- this function requires 16 bytes IV, it transforms IV in cryptonite_aes_gcm_init here:
|
||||
-- https://github.com/haskell-crypto/cryptonite/blob/master/cbits/cryptonite_aes.c
|
||||
-- This is used for double ratchet encryption, so to make it compatible with WebCrypto we will need to deprecate it and start using initAEADGCM
|
||||
initAEAD :: forall c. AES.BlockCipher c => Key -> IV -> ExceptT CryptoError IO (AES.AEAD c)
|
||||
initAEAD (Key aesKey) (IV ivBytes) = do
|
||||
iv <- makeIV @c ivBytes
|
||||
|
||||
@@ -72,7 +72,6 @@ module Simplex.Messaging.Crypto.Ratchet
|
||||
rcEncryptHeader,
|
||||
rcEncryptMsg,
|
||||
rcDecrypt,
|
||||
maxSkippedMsgKeys,
|
||||
-- used in tests
|
||||
MsgHeader (..),
|
||||
RatchetInitParams (..),
|
||||
@@ -120,7 +119,7 @@ import Simplex.Messaging.Crypto.SNTRUP761.Bindings
|
||||
import Simplex.Messaging.Encoding
|
||||
import Simplex.Messaging.Encoding.String
|
||||
import Simplex.Messaging.Parsers (defaultJSON, parseE, parseE')
|
||||
import Simplex.Messaging.Util ((<$?>))
|
||||
import Simplex.Messaging.Util (($>>=), (<$?>))
|
||||
import Simplex.Messaging.Version
|
||||
import Simplex.Messaging.Version.Internal
|
||||
import UnliftIO.STM
|
||||
@@ -328,13 +327,9 @@ instance StrEncoding AnyE2ERatchetParamsUri where
|
||||
Nothing -> pure $ AnyE2ERatchetParamsUri SRKSProposed a $ E2ERatchetParamsUri vr k1 k2 Nothing
|
||||
_ -> fail "bad e2e params"
|
||||
where
|
||||
kemP query = do
|
||||
k_ <- queryParam_ "kem_key" query
|
||||
ct_ <- queryParam_ "kem_ct" query
|
||||
case (k_, ct_) of
|
||||
(Just k, _) -> pure $ Just $ kemParams k ct_
|
||||
(Nothing, Nothing) -> pure Nothing
|
||||
(Nothing, Just _) -> fail "bad e2e params: kem_ct without kem_key"
|
||||
kemP query =
|
||||
queryParam_ "kem_key" query
|
||||
$>>= \k -> Just . kemParams k <$> queryParam_ "kem_ct" query
|
||||
kemParams k = \case
|
||||
Nothing -> ARKP SRKSProposed $ RKParamsProposed k
|
||||
Just ct -> ARKP SRKSAccepted $ RKParamsAccepted ct k
|
||||
@@ -456,7 +451,6 @@ generateSndE2EParams g v = \case
|
||||
|
||||
data RatchetInitParams = RatchetInitParams
|
||||
{ assocData :: Str,
|
||||
rcVerifyCodePQ :: Str,
|
||||
ratchetKey :: RatchetKey,
|
||||
sndHK :: HeaderKey,
|
||||
rcvNextHK :: HeaderKey,
|
||||
@@ -499,14 +493,14 @@ pqX3dhRcv (rpk1, rpk2, rpKem_) (E2ERatchetParams _ sk1 sk2 sKem_) = do
|
||||
|
||||
pqX3dh :: DhAlgorithm a => (PublicKey a, PublicKey a) -> DhSecret a -> DhSecret a -> DhSecret a -> Maybe RatchetKEMAccepted -> RatchetInitParams
|
||||
pqX3dh (sk1, rk1) dh1 dh2 dh3 kemAccepted =
|
||||
RatchetInitParams {assocData, rcVerifyCodePQ = Str vcPQ, ratchetKey = RatchetKey sk, sndHK = Key hk, rcvNextHK = Key nhk, kemAccepted}
|
||||
RatchetInitParams {assocData, ratchetKey = RatchetKey sk, sndHK = Key hk, rcvNextHK = Key nhk, kemAccepted}
|
||||
where
|
||||
assocData = Str $ pubKeyBytes sk1 <> pubKeyBytes rk1
|
||||
dhs = dhBytes' dh1 <> dhBytes' dh2 <> dhBytes' dh3 <> pq
|
||||
pq = maybe "" (\RatchetKEMAccepted {rcPQRss = KEMSharedKey ss} -> BA.convert ss) kemAccepted
|
||||
salt = B.replicate 64 '\0'
|
||||
(hk, nhk, sk) = hkdf3 salt dhs "SimpleXX3DH"
|
||||
vcPQ = hkdf salt dhs "SimpleXVerifyCode" 32
|
||||
(hk, nhk, sk) =
|
||||
let salt = B.replicate 64 '\0'
|
||||
in hkdf3 salt dhs "SimpleXX3DH"
|
||||
|
||||
type RatchetX448 = Ratchet 'X448
|
||||
|
||||
@@ -515,8 +509,6 @@ data Ratchet a = Ratchet
|
||||
rcVersion :: RatchetVersions,
|
||||
-- associated data - must be the same in both parties ratchets
|
||||
rcAD :: Str,
|
||||
-- verification code covering all handshake keys, absent in the ratchets created before it was added
|
||||
rcVCPQ :: Maybe Str,
|
||||
rcDHRs :: PrivateKey a,
|
||||
rcKEM :: Maybe RatchetKEM,
|
||||
rcSupportKEM :: PQSupport, -- defines header size, can only be enabled once
|
||||
@@ -645,14 +637,13 @@ instance FromField MessageKey where fromField = blobFieldDecoder smpDecode
|
||||
-- @
|
||||
initSndRatchet ::
|
||||
forall a. (AlgorithmI a, DhAlgorithm a) => RatchetVersions -> PublicKey a -> PrivateKey a -> (RatchetInitParams, Maybe KEMKeyPair) -> Ratchet a
|
||||
initSndRatchet rcVersion rcDHRr rcDHRs (RatchetInitParams {assocData, rcVerifyCodePQ, ratchetKey, sndHK, rcvNextHK, kemAccepted}, rcPQRs_) = do
|
||||
initSndRatchet rcVersion rcDHRr rcDHRs (RatchetInitParams {assocData, ratchetKey, sndHK, rcvNextHK, kemAccepted}, rcPQRs_) = do
|
||||
-- state.RK, state.CKs, state.NHKs = KDF_RK_HE(SK, DH(state.DHRs, state.DHRr) || state.PQRss)
|
||||
let (rcRK, rcCKs, rcNHKs) = rootKdf ratchetKey rcDHRr rcDHRs (rcPQRss <$> kemAccepted)
|
||||
pqOn = isJust rcPQRs_
|
||||
in Ratchet
|
||||
{ rcVersion,
|
||||
rcAD = assocData,
|
||||
rcVCPQ = Just rcVerifyCodePQ,
|
||||
rcDHRs,
|
||||
rcKEM = (`RatchetKEM` kemAccepted) <$> rcPQRs_,
|
||||
rcSupportKEM = PQSupport pqOn,
|
||||
@@ -677,11 +668,10 @@ initSndRatchet rcVersion rcDHRr rcDHRs (RatchetInitParams {assocData, rcVerifyCo
|
||||
-- as part of the connection request and random salt was received from the sender.
|
||||
initRcvRatchet ::
|
||||
forall a. (AlgorithmI a, DhAlgorithm a) => RatchetVersions -> PrivateKey a -> (RatchetInitParams, Maybe KEMKeyPair) -> PQSupport -> Ratchet a
|
||||
initRcvRatchet rcVersion rcDHRs (RatchetInitParams {assocData, rcVerifyCodePQ, ratchetKey, sndHK, rcvNextHK, kemAccepted}, rcPQRs_) pqSupport =
|
||||
initRcvRatchet rcVersion rcDHRs (RatchetInitParams {assocData, ratchetKey, sndHK, rcvNextHK, kemAccepted}, rcPQRs_) pqSupport =
|
||||
Ratchet
|
||||
{ rcVersion,
|
||||
rcAD = assocData,
|
||||
rcVCPQ = Just rcVerifyCodePQ,
|
||||
rcDHRs,
|
||||
-- rcKEM:
|
||||
-- state.PQRs = bob_pq_kem_key_pair
|
||||
@@ -963,9 +953,6 @@ type DecryptResult a = (Either CryptoError ByteString, Ratchet a, SkippedMsgDiff
|
||||
maxSkip :: Word32
|
||||
maxSkip = 512
|
||||
|
||||
maxSkippedMsgKeys :: Int
|
||||
maxSkippedMsgKeys = 2000
|
||||
|
||||
rcDecrypt ::
|
||||
forall a.
|
||||
(AlgorithmI a, DhAlgorithm a) =>
|
||||
|
||||
@@ -16,8 +16,6 @@ module Simplex.Messaging.Crypto.SNTRUP761.Bindings
|
||||
) where
|
||||
|
||||
import Control.Concurrent.STM
|
||||
import Control.Exception (throwIO)
|
||||
import Control.Monad (when)
|
||||
import Crypto.Random (ChaChaDRG)
|
||||
import Data.Aeson (FromJSON (..), ToJSON (..))
|
||||
import Data.Bifunctor (bimap)
|
||||
@@ -25,7 +23,6 @@ import Data.ByteArray (ScrubbedBytes)
|
||||
import qualified Data.ByteArray as BA
|
||||
import Data.ByteString (ByteString)
|
||||
import Simplex.Messaging.Agent.Store.DB (FromField (..), ToField (..))
|
||||
import qualified Simplex.Messaging.Crypto as C
|
||||
import Simplex.Messaging.Crypto.SNTRUP761.Bindings.Defines
|
||||
import Simplex.Messaging.Crypto.SNTRUP761.Bindings.FFI
|
||||
import Simplex.Messaging.Crypto.SNTRUP761.Bindings.RNG (rngFuncPtr, withDRG)
|
||||
@@ -58,13 +55,14 @@ pattern KEMSharedKey s <- KEMSharedKey_ s
|
||||
type KEMKeyPair = (KEMPublicKey, KEMSecretKey)
|
||||
|
||||
sntrup761Keypair :: TVar ChaChaDRG -> IO KEMKeyPair
|
||||
sntrup761Keypair drg = do
|
||||
((r, pk), sk) <-
|
||||
BA.allocRet c_SNTRUP761_SECRETKEY_SIZE $ \skPtr ->
|
||||
BA.allocRet c_SNTRUP761_PUBLICKEY_SIZE $ \pkPtr ->
|
||||
withDRG drg $ \cxtPtr -> c_sntrup761_keypair pkPtr skPtr cxtPtr rngFuncPtr
|
||||
when (r /= 0) $ throwIO C.CryptoKEMKeyGenError
|
||||
pure (KEMPublicKey_ pk, KEMSecretKey sk)
|
||||
sntrup761Keypair drg =
|
||||
bimap KEMPublicKey_ KEMSecretKey
|
||||
<$> BA.allocRet
|
||||
c_SNTRUP761_SECRETKEY_SIZE
|
||||
( \skPtr ->
|
||||
BA.alloc c_SNTRUP761_PUBLICKEY_SIZE $ \pkPtr ->
|
||||
withDRG drg $ \cxtPtr -> c_sntrup761_keypair pkPtr skPtr cxtPtr rngFuncPtr
|
||||
)
|
||||
|
||||
sntrup761Enc :: TVar ChaChaDRG -> KEMPublicKey -> IO (KEMCiphertext, KEMSharedKey)
|
||||
sntrup761Enc drg (KEMPublicKey pk) =
|
||||
|
||||
@@ -10,9 +10,9 @@ import Foreign
|
||||
import Foreign.C
|
||||
import Simplex.Messaging.Crypto.SNTRUP761.Bindings.RNG (RNGContext, RNGFunc)
|
||||
|
||||
-- int sntrup761_keypair (uint8_t *pk, uint8_t *sk, void *random_ctx, sntrup761_random_func *random);
|
||||
-- void sntrup761_keypair (uint8_t *pk, uint8_t *sk, void *random_ctx, sntrup761_random_func *random);
|
||||
foreign import ccall "sntrup761_keypair"
|
||||
c_sntrup761_keypair :: Ptr Word8 -> Ptr Word8 -> Ptr RNGContext -> FunPtr RNGFunc -> IO CInt
|
||||
c_sntrup761_keypair :: Ptr Word8 -> Ptr Word8 -> Ptr RNGContext -> FunPtr RNGFunc -> IO ()
|
||||
|
||||
-- void sntrup761_enc (uint8_t *c, uint8_t *k, const uint8_t *pk, void *random_ctx, sntrup761_random_func *random);
|
||||
foreign import ccall "sntrup761_enc"
|
||||
|
||||
@@ -1,6 +1,3 @@
|
||||
{-# LANGUAGE DerivingStrategies #-}
|
||||
{-# LANGUAGE GeneralizedNewtypeDeriving #-}
|
||||
{-# LANGUAGE LambdaCase #-}
|
||||
{-# LANGUAGE NamedFieldPuns #-}
|
||||
{-# LANGUAGE OverloadedStrings #-}
|
||||
{-# LANGUAGE StrictData #-}
|
||||
@@ -8,24 +5,13 @@
|
||||
|
||||
module Simplex.Messaging.Names.Record
|
||||
( NameRecord (..),
|
||||
NameResponse (..),
|
||||
NameRegistration (..),
|
||||
NamePricing (..),
|
||||
USDCents (..),
|
||||
NameReservedReason (..),
|
||||
)
|
||||
where
|
||||
|
||||
import Data.Aeson (FromJSON (..), ToJSON (..))
|
||||
import qualified Data.Aeson as J
|
||||
import qualified Data.Aeson.TH as JQ
|
||||
import Data.Int (Int64)
|
||||
import Data.Map.Strict (Map)
|
||||
import Data.Text (Text)
|
||||
import qualified Data.Text as T
|
||||
import Simplex.Messaging.Encoding.String
|
||||
import Simplex.Messaging.Parsers (defaultJSON, dropPrefix, taggedObjectJSON)
|
||||
import Simplex.Messaging.SystemTime (SystemSeconds)
|
||||
import Simplex.Messaging.Parsers (defaultJSON, dropPrefix)
|
||||
|
||||
-- | Resolved name record returned by the names role. JSON keys match the
|
||||
-- resolver REST output; both FromJSON (resolver -> server) and ToJSON
|
||||
@@ -58,87 +44,3 @@ $( JQ.deriveJSON
|
||||
defaultJSON {J.omitNothingFields = False, J.fieldLabelModifier = dropPrefix "nr"}
|
||||
''NameRecord
|
||||
)
|
||||
|
||||
-- | US cents.
|
||||
newtype USDCents = USDCents Int64
|
||||
deriving (Eq, Ord, Show)
|
||||
deriving newtype (ToJSON, FromJSON)
|
||||
|
||||
-- | What the registry holds for a name, and the block it was read at.
|
||||
data NameResponse = NameResponse
|
||||
{ -- | absent only from a v20/v21 router, which sent the record alone
|
||||
lastBlockTs :: Maybe SystemSeconds,
|
||||
registration :: NameRegistration
|
||||
}
|
||||
deriving (Eq, Show)
|
||||
|
||||
-- | What the registry holds for a name.
|
||||
data NameRegistration
|
||||
= -- | Held by someone. Always carries a record, empty where none was set.
|
||||
NRRegistered
|
||||
{ -- | absent only from a v20/v21 router, which sent the record alone
|
||||
expires :: Maybe SystemSeconds,
|
||||
-- | unix seconds, > expires: until here only the owner may renew
|
||||
graceUntil :: Maybe SystemSeconds,
|
||||
-- | held back as well, which is why it will not free up at expiry
|
||||
reservedReason_ :: Maybe NameReservedReason,
|
||||
nameRecord :: NameRecord
|
||||
}
|
||||
| -- | Held by nobody, and registrable now.
|
||||
NRAvailable {pricing :: NamePricing}
|
||||
| -- | Held back by the registry, and not for sale at its price.
|
||||
NRReserved {reservedReason :: NameReservedReason}
|
||||
deriving (Eq, Show)
|
||||
|
||||
-- | Enough to price the name locally, which the router cannot do behind a hash.
|
||||
data NamePricing = NamePricing
|
||||
{ -- | US cents per year, for the lengths the registry prices specially
|
||||
registrationPrices :: Map Int USDCents,
|
||||
-- | US cents per year for every other length
|
||||
basePrice :: USDCents,
|
||||
-- | characters; the registry refuses shorter labels
|
||||
minLabelLength :: Int
|
||||
}
|
||||
deriving (Eq, Show)
|
||||
|
||||
-- | Why the registry holds a name back.
|
||||
data NameReservedReason
|
||||
= -- | held for SimpleX
|
||||
NRRInternal
|
||||
| NRRTrademark
|
||||
| NRRCommunity
|
||||
| -- | added to the registry after this version, and still reserved
|
||||
NRRUnknown Text
|
||||
deriving (Eq, Show)
|
||||
|
||||
instance TextEncoding NameReservedReason where
|
||||
textEncode = \case
|
||||
NRRInternal -> "internal"
|
||||
NRRTrademark -> "trademark"
|
||||
NRRCommunity -> "community"
|
||||
NRRUnknown t -> t
|
||||
textDecode = Just . reservedReasonOf
|
||||
|
||||
-- | An unknown reason is kept as text, capped: it reaches a client as a word.
|
||||
-- Capping precedes the match, so what is kept encodes back to what it decoded.
|
||||
reservedReasonOf :: Text -> NameReservedReason
|
||||
reservedReasonOf t = case T.take 32 $ T.takeWhile (\c -> c > ' ' && c < '\DEL') t of
|
||||
"internal" -> NRRInternal
|
||||
"trademark" -> NRRTrademark
|
||||
"community" -> NRRCommunity
|
||||
r -> NRRUnknown r
|
||||
|
||||
instance ToJSON NameReservedReason where
|
||||
toJSON = textToJSON
|
||||
toEncoding = textToEncoding
|
||||
|
||||
instance FromJSON NameReservedReason where
|
||||
parseJSON = textParseJSON "NameReservedReason"
|
||||
|
||||
$(JQ.deriveJSON defaultJSON ''NamePricing)
|
||||
|
||||
-- taggedObjectJSON, not sumTypeJSON: this JSON is the RNAME payload and the
|
||||
-- resolver contract, so it must not vary with the swift build flag.
|
||||
$(JQ.deriveJSON (taggedObjectJSON $ dropPrefix "NR") ''NameRegistration)
|
||||
|
||||
$(JQ.deriveJSON defaultJSON ''NameResponse)
|
||||
|
||||
@@ -80,12 +80,6 @@ module Simplex.Messaging.Protocol
|
||||
ErrorType (..),
|
||||
CommandError (..),
|
||||
ProxyError (..),
|
||||
NameQuery (..),
|
||||
NameResponse (..),
|
||||
NameRegistration (..),
|
||||
NamePricing (..),
|
||||
USDCents (..),
|
||||
NameReservedReason (..),
|
||||
NameErrorType (..),
|
||||
BrokerErrorType (..),
|
||||
NetworkError (..),
|
||||
@@ -168,7 +162,6 @@ module Simplex.Messaging.Protocol
|
||||
EncFwdTransmission (..),
|
||||
EncResponse (..),
|
||||
EncTransmission (..),
|
||||
encTransmissionNonce,
|
||||
FwdResponse (..),
|
||||
FwdTransmission (..),
|
||||
NameRecord (..),
|
||||
@@ -242,7 +235,6 @@ import Data.Attoparsec.ByteString.Char8 (Parser, (<?>))
|
||||
import qualified Data.Attoparsec.ByteString.Char8 as A
|
||||
import Data.Bifunctor (bimap, first)
|
||||
import Data.Bits (xor)
|
||||
import qualified Data.ByteArray as BA
|
||||
import qualified Data.ByteString as BS
|
||||
import qualified Data.ByteString.Base64 as B64
|
||||
import Data.ByteString.Char8 (ByteString)
|
||||
@@ -273,15 +265,15 @@ import Simplex.Messaging.Agent.Store.DB (Binary (..), FromField (..), ToField (.
|
||||
import qualified Simplex.Messaging.Crypto as C
|
||||
import Simplex.Messaging.Encoding
|
||||
import Simplex.Messaging.Encoding.String
|
||||
import Simplex.Messaging.Names.Record
|
||||
import Simplex.Messaging.Names.Record (NameRecord (..))
|
||||
import Simplex.Messaging.Parsers
|
||||
import Simplex.Messaging.Protocol.Types
|
||||
import Simplex.Messaging.Server.QueueStore.QueueInfo
|
||||
import Simplex.Messaging.ServiceScheme
|
||||
import Simplex.Messaging.SimplexName (LabelHash, SimplexDomain (..), SimplexTLD (..), fullDomainName, labelHash)
|
||||
import Simplex.Messaging.SimplexName (SimplexDomain)
|
||||
import Simplex.Messaging.Transport
|
||||
import Simplex.Messaging.Transport.Client (TransportHost, TransportHosts (..))
|
||||
import Simplex.Messaging.Util (bshow, eitherToMaybe, packZipWith, safeDecodeUtf8, (<$?>))
|
||||
import Simplex.Messaging.Util (bshow, eitherToMaybe, safeDecodeUtf8, (<$?>))
|
||||
import Simplex.Messaging.Version
|
||||
import Simplex.Messaging.Version.Internal
|
||||
|
||||
@@ -611,7 +603,7 @@ data Command (p :: Party) where
|
||||
-- - corrId: unique correlation ID between proxy and relay, also used as a nonce to encrypt forwarded transmission
|
||||
RFWD :: EncFwdTransmission -> Command ProxyService -- use CorrId as CbNonce, proxy to relay
|
||||
-- Resolve SimpleX name.
|
||||
RSLV :: NameQuery -> Command Resolver
|
||||
RSLV :: SimplexDomain -> Command Resolver
|
||||
|
||||
deriving instance Show (Command p)
|
||||
|
||||
@@ -703,11 +695,6 @@ instance Encoding NewNtfCreds where
|
||||
newtype EncTransmission = EncTransmission ByteString
|
||||
deriving (Show)
|
||||
|
||||
encTransmissionNonce :: VersionSMP -> C.CbNonce -> C.CbNonce
|
||||
encTransmissionNonce v nonce@(C.CbNonce s)
|
||||
| v >= fwdNoncesSMPVersion = C.cbNonce $ packZipWith xor (smpEncode v) s <> BS.drop 2 s
|
||||
| otherwise = nonce
|
||||
|
||||
data FwdTransmission = FwdTransmission
|
||||
{ fwdCorrId :: CorrId,
|
||||
fwdVersion :: VersionSMP,
|
||||
@@ -743,8 +730,8 @@ data BrokerMsg where
|
||||
NMSG :: C.CbNonce -> EncNMsgMeta -> BrokerMsg
|
||||
-- Should include certificate chain
|
||||
PKEY :: SessionId -> VersionRangeSMP -> CertChainPubKey -> BrokerMsg -- TLS-signed server key for proxy shared secret and initial sender key
|
||||
RRES :: Maybe C.CbNonce -> EncFwdResponse -> BrokerMsg -- relay to proxy
|
||||
PRES :: Maybe C.CbNonce -> EncResponse -> BrokerMsg -- proxy to client
|
||||
RRES :: EncFwdResponse -> BrokerMsg -- relay to proxy
|
||||
PRES :: EncResponse -> BrokerMsg -- proxy to client
|
||||
END :: BrokerMsg
|
||||
ENDS :: Int64 -> IdsHash -> BrokerMsg
|
||||
DELD :: BrokerMsg
|
||||
@@ -752,8 +739,8 @@ data BrokerMsg where
|
||||
OK :: BrokerMsg
|
||||
ERR :: ErrorType -> BrokerMsg
|
||||
PONG :: BrokerMsg
|
||||
-- What the router knows about a SimpleX name.
|
||||
RNAME :: NameResponse -> BrokerMsg
|
||||
-- Resolved SimpleX name.
|
||||
RNAME :: NameRecord -> BrokerMsg
|
||||
deriving (Eq, Show)
|
||||
|
||||
data RcvMessage = RcvMessage
|
||||
@@ -1330,10 +1317,7 @@ instance ProtocolTypeI p => FromJSON (ProtocolServer p) where
|
||||
parseJSON = strParseJSON "ProtocolServer"
|
||||
|
||||
newtype BasicAuth = BasicAuth {unBasicAuth :: ByteString}
|
||||
deriving (Ord, Show)
|
||||
|
||||
instance Eq BasicAuth where
|
||||
BasicAuth s == BasicAuth s' = BA.constEq s s'
|
||||
deriving (Eq, Ord, Show)
|
||||
|
||||
instance IsString BasicAuth where fromString = BasicAuth . B.pack
|
||||
|
||||
@@ -1605,28 +1589,11 @@ data ErrorType
|
||||
DUPLICATE_ -- not part of SMP protocol, used internally
|
||||
deriving (Eq, Show)
|
||||
|
||||
-- | What RSLV asks about: a name, or the hash of a second-level label.
|
||||
data NameQuery = NQDomain SimplexDomain | NQHash LabelHash SimplexTLD
|
||||
deriving (Eq, Show)
|
||||
|
||||
instance Encoding NameQuery where
|
||||
smpEncode = \case
|
||||
NQDomain d -> encodeUtf8 $ fullDomainName d
|
||||
NQHash h tld -> strEncode h <> strEncode tld
|
||||
smpP = NQHash <$> strP <*> strP <|> NQDomain <$> strP
|
||||
|
||||
-- | Hashed from v22, except a name with subnames or a web TLD.
|
||||
hashedQuery :: NameQuery -> NameQuery
|
||||
hashedQuery q = case q of
|
||||
NQDomain SimplexDomain {nameTLD, domain, subDomain}
|
||||
| null subDomain && nameTLD /= TLDWeb -> NQHash (labelHash domain) nameTLD
|
||||
_ -> q
|
||||
|
||||
-- | Name resolution error
|
||||
data NameErrorType
|
||||
= -- | the names role / resolver is not configured on this server
|
||||
NO_RESOLVER
|
||||
| -- | the name does not resolve; sent only to a session below v22
|
||||
| -- | the name is not registered (resolver returned not-found)
|
||||
NOT_FOUND
|
||||
| -- | backing resolver/RPC failure - contains the diagnostic detail
|
||||
RESOLVER {resolverErr :: Text}
|
||||
@@ -1824,7 +1791,8 @@ instance PartyI p => ProtocolEncoding SMPVersion ErrorType (Command p) where
|
||||
encodeProtocol v = \case
|
||||
NEW NewQueueReq {rcvAuthKey = rKey, rcvDhKey = dhKey, auth_, subMode, queueReqData, ntfCreds}
|
||||
| v >= newNtfCredsSMPVersion -> new <> e (subMode, queueReqData, ntfCreds)
|
||||
| otherwise -> new <> e (subMode, queueReqData)
|
||||
| v >= shortLinksSMPVersion -> new <> e (subMode, queueReqData)
|
||||
| otherwise -> new <> e (subMode, senderCanSecure (queueReqMode <$> queueReqData))
|
||||
where
|
||||
new = e (NEW_, ' ', rKey, dhKey, auth_)
|
||||
SUB -> e SUB_
|
||||
@@ -1854,7 +1822,7 @@ instance PartyI p => ProtocolEncoding SMPVersion ErrorType (Command p) where
|
||||
PRXY host auth_ -> e (PRXY_, ' ', host, auth_)
|
||||
PFWD fwdV pubKey (EncTransmission s) -> e (PFWD_, ' ', fwdV, pubKey, Tail s)
|
||||
RFWD (EncFwdTransmission s) -> e (RFWD_, ' ', Tail s)
|
||||
RSLV q -> e (RSLV_, ' ', if v >= nameAvailSMPVersion then hashedQuery q else q)
|
||||
RSLV d -> e (RSLV_, ' ', d)
|
||||
where
|
||||
e :: Encoding a => a -> ByteString
|
||||
e = smpEncode
|
||||
@@ -1911,18 +1879,20 @@ instance ProtocolEncoding SMPVersion ErrorType Cmd where
|
||||
CT SCreator NEW_ -> Cmd SCreator <$> newCmd
|
||||
where
|
||||
newCmd
|
||||
| v >= newNtfCredsSMPVersion = new smpP
|
||||
| otherwise = new nothing
|
||||
| v >= newNtfCredsSMPVersion = new smpP smpP
|
||||
| v >= shortLinksSMPVersion = new smpP nothing
|
||||
| otherwise = new (qReq <$> smpP) nothing
|
||||
where
|
||||
nothing = pure Nothing
|
||||
new p3 = NEW <$> do
|
||||
new p2 p3 = NEW <$> do
|
||||
rcvAuthKey <- _smpP
|
||||
rcvDhKey <- smpP
|
||||
auth_ <- smpP
|
||||
subMode <- smpP
|
||||
queueReqData <- smpP
|
||||
queueReqData <- p2
|
||||
ntfCreds <- p3
|
||||
pure NewQueueReq {rcvAuthKey, rcvDhKey, auth_, subMode, queueReqData, ntfCreds}
|
||||
qReq sndSecure = Just $ if sndSecure then QRMessaging Nothing else QRContact Nothing
|
||||
CT SRecipient tag ->
|
||||
Cmd SRecipient <$> case tag of
|
||||
SUB_ -> pure SUB
|
||||
@@ -1959,7 +1929,7 @@ instance ProtocolEncoding SMPVersion ErrorType Cmd where
|
||||
CT SNotifierService NSUBS_
|
||||
| v >= rcvServiceSMPVersion -> Cmd SNotifierService <$> (NSUBS <$> _smpP <*> smpP)
|
||||
| otherwise -> pure $ Cmd SNotifierService $ NSUBS (-1) mempty
|
||||
CT SResolver RSLV_ -> Cmd SResolver . RSLV <$> _smpP
|
||||
CT SResolver RSLV_ -> Cmd SResolver . RSLV <$> _smpP <* A.takeByteString
|
||||
|
||||
fromProtocolError = fromProtocolError @SMPVersion @ErrorType @BrokerMsg
|
||||
{-# INLINE fromProtocolError #-}
|
||||
@@ -1973,7 +1943,8 @@ instance ProtocolEncoding SMPVersion ErrorType BrokerMsg where
|
||||
IDS QIK {rcvId, sndId, rcvPublicDhKey = srvDh, queueMode, linkId, serviceId, serverNtfCreds}
|
||||
| v >= newNtfCredsSMPVersion -> ids <> e (queueMode, linkId, serviceId, serverNtfCreds)
|
||||
| v >= serviceCertsSMPVersion -> ids <> e (queueMode, linkId, serviceId)
|
||||
| otherwise -> ids <> e (queueMode, linkId)
|
||||
| v >= shortLinksSMPVersion -> ids <> e (queueMode, linkId)
|
||||
| otherwise -> ids <> e (senderCanSecure queueMode)
|
||||
where
|
||||
ids = e (IDS_, ' ', rcvId, sndId, srvDh)
|
||||
LNK sId d -> e (LNK_, ' ', sId, d)
|
||||
@@ -1987,8 +1958,8 @@ instance ProtocolEncoding SMPVersion ErrorType BrokerMsg where
|
||||
NID nId srvNtfDh -> e (NID_, ' ', nId, srvNtfDh)
|
||||
NMSG nmsgNonce encNMsgMeta -> e (NMSG_, ' ', nmsgNonce, encNMsgMeta)
|
||||
PKEY sid vr certKey -> e (PKEY_, ' ', sid, vr, certKey)
|
||||
RRES nonce_ (EncFwdResponse encBlock) -> fwdResp RRES_ nonce_ encBlock
|
||||
PRES nonce_ (EncResponse encBlock) -> fwdResp PRES_ nonce_ encBlock
|
||||
RRES (EncFwdResponse encBlock) -> e (RRES_, ' ', Tail encBlock)
|
||||
PRES (EncResponse encBlock) -> e (PRES_, ' ', Tail encBlock)
|
||||
END -> e END_
|
||||
ENDS n idsHash -> serviceResp ENDS_ n idsHash
|
||||
DELD -> e DELD_
|
||||
@@ -2001,20 +1972,13 @@ instance ProtocolEncoding SMPVersion ErrorType BrokerMsg where
|
||||
| v < clientNoticesSMPVersion -> BLOCKED info {notice = Nothing}
|
||||
_ -> err
|
||||
PONG -> e PONG_
|
||||
RNAME res
|
||||
| v >= nameAvailSMPVersion -> e (RNAME_, ' ', Tail $ LB.toStrict $ J.encode res)
|
||||
| otherwise -> case registration res of
|
||||
NRRegistered {nameRecord} -> e (RNAME_, ' ', Tail $ LB.toStrict $ J.encode nameRecord)
|
||||
_ -> e (ERR_, ' ', NAME NOT_FOUND)
|
||||
RNAME rec -> e (RNAME_, ' ', Tail $ LB.toStrict $ J.encode rec)
|
||||
where
|
||||
e :: Encoding a => a -> ByteString
|
||||
e = smpEncode
|
||||
serviceResp tag n idsHash
|
||||
| v >= rcvServiceSMPVersion = e (tag, ' ', n, idsHash)
|
||||
| otherwise = e (tag, ' ', n)
|
||||
fwdResp tag nonce_ encBlock
|
||||
| v >= fwdNoncesSMPVersion = e (tag, ' ', nonce_, Tail encBlock)
|
||||
| otherwise = e (tag, ' ', Tail encBlock)
|
||||
|
||||
protocolP v = \case
|
||||
MSG_ -> do
|
||||
@@ -2024,17 +1988,19 @@ instance ProtocolEncoding SMPVersion ErrorType BrokerMsg where
|
||||
bodyP = EncRcvMsgBody . unTail <$> smpP
|
||||
ALLS_ -> pure ALLS
|
||||
IDS_
|
||||
| v >= newNtfCredsSMPVersion -> ids smpP smpP
|
||||
| v >= serviceCertsSMPVersion -> ids smpP nothing
|
||||
| otherwise -> ids nothing nothing
|
||||
| v >= newNtfCredsSMPVersion -> ids smpP smpP smpP smpP
|
||||
| v >= serviceCertsSMPVersion -> ids smpP smpP smpP nothing
|
||||
| v >= shortLinksSMPVersion -> ids smpP smpP nothing nothing
|
||||
| otherwise -> ids (qm <$> smpP) nothing nothing nothing
|
||||
where
|
||||
qm sndSecure = Just $ if sndSecure then QMMessaging else QMContact
|
||||
nothing = pure Nothing
|
||||
ids p3 p4 = do
|
||||
ids p1 p2 p3 p4 = do
|
||||
rcvId <- _smpP
|
||||
sndId <- smpP
|
||||
rcvPublicDhKey <- smpP
|
||||
queueMode <- smpP
|
||||
linkId <- smpP
|
||||
queueMode <- p1
|
||||
linkId <- p2
|
||||
serviceId <- p3
|
||||
serverNtfCreds <- p4
|
||||
pure $ IDS QIK {rcvId, sndId, rcvPublicDhKey, queueMode, linkId, serviceId, serverNtfCreds}
|
||||
@@ -2044,8 +2010,8 @@ instance ProtocolEncoding SMPVersion ErrorType BrokerMsg where
|
||||
NID_ -> NID <$> _smpP <*> smpP
|
||||
NMSG_ -> NMSG <$> _smpP <*> smpP
|
||||
PKEY_ -> PKEY <$> _smpP <*> smpP <*> smpP
|
||||
RRES_ -> fwdRespP RRES EncFwdResponse
|
||||
PRES_ -> fwdRespP PRES EncResponse
|
||||
RRES_ -> RRES <$> (EncFwdResponse . unTail <$> _smpP)
|
||||
PRES_ -> PRES <$> (EncResponse . unTail <$> _smpP)
|
||||
END_ -> pure END
|
||||
ENDS_ -> serviceRespP ENDS
|
||||
DELD_ -> pure DELD
|
||||
@@ -2053,18 +2019,11 @@ instance ProtocolEncoding SMPVersion ErrorType BrokerMsg where
|
||||
OK_ -> pure OK
|
||||
ERR_ -> ERR <$> _smpP
|
||||
PONG_ -> pure PONG
|
||||
RNAME_
|
||||
| v >= nameAvailSMPVersion -> fmap RNAME . J.eitherDecodeStrict . unTail <$?> _smpP
|
||||
| otherwise -> fmap (RNAME . oldResponse) . J.eitherDecodeStrict . unTail <$?> _smpP
|
||||
RNAME_ -> fmap RNAME . J.eitherDecodeStrict . unTail <$?> _smpP
|
||||
where
|
||||
oldResponse nameRecord =
|
||||
NameResponse {lastBlockTs = Nothing, registration = NRRegistered {expires = Nothing, graceUntil = Nothing, reservedReason_ = Nothing, nameRecord}}
|
||||
serviceRespP resp
|
||||
| v >= rcvServiceSMPVersion = resp <$> _smpP <*> smpP
|
||||
| otherwise = resp <$> _smpP <*> pure mempty
|
||||
fwdRespP :: (Maybe C.CbNonce -> a -> BrokerMsg) -> (ByteString -> a) -> Parser BrokerMsg
|
||||
fwdRespP resp enc = resp <$> (A.space *> nonceP) <*> (enc <$> A.takeByteString)
|
||||
nonceP = if v >= fwdNoncesSMPVersion then smpP else pure Nothing
|
||||
|
||||
fromProtocolError = \case
|
||||
PECmdSyntax -> CMD SYNTAX
|
||||
@@ -2081,9 +2040,9 @@ instance ProtocolEncoding SMPVersion ErrorType BrokerMsg where
|
||||
-- PONG response must not have queue ID
|
||||
PONG -> noEntityMsg
|
||||
PKEY {} -> noEntityMsg
|
||||
RRES {} -> noEntityMsg
|
||||
RRES _ -> noEntityMsg
|
||||
ALLS -> noEntityMsg
|
||||
RNAME {} -> noEntityMsg
|
||||
RNAME _ -> noEntityMsg
|
||||
-- other broker responses must have queue ID
|
||||
_
|
||||
| B.null entId -> Left $ CMD NO_ENTITY
|
||||
@@ -2453,4 +2412,3 @@ $(J.deriveJSON defaultJSON ''BlockingInfo)
|
||||
|
||||
-- run deriveJSON in one TH splice to allow mutual instance
|
||||
$(concat <$> mapM @[] (J.deriveJSON (sumTypeJSON id)) [''ProxyError, ''NameErrorType, ''ErrorType])
|
||||
|
||||
|
||||
@@ -33,6 +33,9 @@ module Simplex.Messaging.Server
|
||||
( runSMPServer,
|
||||
runSMPServerBlocking,
|
||||
controlPortAuth,
|
||||
importMessages,
|
||||
exportMessages,
|
||||
printMessageStats,
|
||||
disconnectTransport,
|
||||
verifyCmdAuthorization,
|
||||
dummyVerifyCmd,
|
||||
@@ -95,16 +98,18 @@ import Network.Socket (ServiceName, Socket, socketToHandle)
|
||||
import qualified Network.TLS as TLS
|
||||
import Numeric.Natural (Natural)
|
||||
import Simplex.Messaging.Agent.Lock
|
||||
import Simplex.Messaging.Client (ProtocolClient (thParams), ProtocolClientError (..), SMPClient, SMPClientError, clientHandlers, forwardSMPTransmission, smpProxyError, temporaryClientError)
|
||||
import Simplex.Messaging.Client (ProtocolClient (thParams), ProtocolClientError (..), SMPClient, SMPClientError, clientHandlers, forwardSMPTransmission, smpProxyError, temporaryClientError, transportHost')
|
||||
import Simplex.Messaging.Client.Agent (OwnServer, SMPClientAgent (..), SMPClientAgentEvent (..), closeSMPClientAgent, getSMPServerClient'', isOwnServer, lookupSMPServerClient, getConnectedSMPServerClient)
|
||||
import qualified Simplex.Messaging.Crypto as C
|
||||
import Simplex.Messaging.Encoding
|
||||
import Simplex.Messaging.Encoding.String
|
||||
import Simplex.Messaging.Protocol
|
||||
import Simplex.Messaging.SimplexName (SimplexDomain)
|
||||
import Simplex.Messaging.Server.Control
|
||||
import Simplex.Messaging.Server.Env.STM as Env
|
||||
import Simplex.Messaging.Server.Expiration
|
||||
import Simplex.Messaging.Server.MsgStore
|
||||
import Simplex.Messaging.Server.MsgStore.Journal (JournalMsgStore, JournalQueue (..), getJournalQueueMessages)
|
||||
import Simplex.Messaging.Server.Names (NamesEnv, closeNamesEnv, resolveName)
|
||||
import Simplex.Messaging.Server.MsgStore.STM
|
||||
import Simplex.Messaging.Server.MsgStore.Types
|
||||
@@ -112,7 +117,6 @@ import Simplex.Messaging.Server.NtfStore
|
||||
import Simplex.Messaging.Server.Prometheus
|
||||
import Simplex.Messaging.Server.QueueStore
|
||||
import Simplex.Messaging.Server.QueueStore.QueueInfo
|
||||
import Simplex.Messaging.Server.QueueStore.STM (withLoadedQueues)
|
||||
import Simplex.Messaging.Server.QueueStore.Types
|
||||
import Simplex.Messaging.Server.Stats
|
||||
import Simplex.Messaging.Server.StoreLog (foldLogLines)
|
||||
@@ -142,7 +146,7 @@ import GHC.Conc.Sync (threadLabel)
|
||||
#endif
|
||||
|
||||
#if defined(dbServerPostgres)
|
||||
import Simplex.Messaging.Server.MsgStore.Postgres (getDbMessageStats)
|
||||
import Simplex.Messaging.Server.MsgStore.Postgres (exportDbMessages, getDbMessageStats)
|
||||
#endif
|
||||
|
||||
-- | Runs an SMP server using passed configuration.
|
||||
@@ -707,11 +711,7 @@ smpServer started cfg@ServerConfig {transports, transportConfig = tCfg, startOpt
|
||||
(deliveredSubs, deliveredTimes) <- getDeliveredMetrics =<< getSystemSeconds
|
||||
smpSubs <- getSubscribersMetrics subscribers
|
||||
ntfSubs <- getSubscribersMetrics ntfSubscribers
|
||||
loadedCounts <- case ms of
|
||||
StoreMemory st -> Just <$> loadedQueueCounts st
|
||||
#if defined(dbServerPostgres)
|
||||
StoreDatabase _ -> pure Nothing
|
||||
#endif
|
||||
loadedCounts <- loadedQueueCounts $ fromMsgStore ms
|
||||
pure RealTimeMetrics {socketStats, threadsCount, clientsCount, deliveredSubs, deliveredTimes, smpSubs, ntfSubs, loadedCounts}
|
||||
where
|
||||
getSubscribersMetrics ServerSubscribers {queueSubscribers, serviceSubscribers, totalServiceSubs, subClients} = do
|
||||
@@ -1093,7 +1093,6 @@ controlPortAuth h user admin role auth = do
|
||||
readTVarIO role >>= \case
|
||||
CPRNone -> do
|
||||
atomically $ writeTVar role $! newRole
|
||||
when (newRole == CPRNone) $ logWarn "ControlPort: failed auth"
|
||||
hPutStrLn h $ currentRole newRole
|
||||
r -> hPutStrLn h $ currentRole r <> if r == newRole then "" else ", start new session to change."
|
||||
where
|
||||
@@ -1320,6 +1319,11 @@ isContactQueue QueueRec {queueMode, senderKey} = case queueMode of
|
||||
Just QMContact -> True
|
||||
Nothing -> isNothing senderKey -- for backward compatibility with pre-SKEY contact addresses
|
||||
|
||||
isSecuredMsgQueue :: QueueRec -> Bool
|
||||
isSecuredMsgQueue QueueRec {queueMode, senderKey} = case queueMode of
|
||||
Just QMContact -> False
|
||||
_ -> isJust senderKey
|
||||
|
||||
-- Random correlation ID is used as a nonce in case crypto_box authenticator is used to authorize transmission
|
||||
verifyCmdAuthorization :: Maybe (THandleAuth 'TServer) -> Maybe TAuthorizations -> ByteString -> CorrId -> C.APublicAuthKey -> Bool
|
||||
verifyCmdAuthorization thAuth tAuth authorized corrId key = maybe False (verify key) tAuth
|
||||
@@ -1458,10 +1462,12 @@ client
|
||||
inc own pRequests
|
||||
forkProxiedCmd $ do
|
||||
liftIO (runExceptT (forwardSMPTransmission smp corrId fwdV pubKey encBlock) `E.catches` clientHandlers) >>= \case
|
||||
Right (nonce_, r) -> PRES nonce_ r <$ inc own pSuccesses
|
||||
Right r -> PRES r <$ inc own pSuccesses
|
||||
Left e -> ERR (smpProxyError e) <$ case e of
|
||||
PCEProtocolError {} -> inc own pSuccesses
|
||||
_ -> inc own pErrorsOther
|
||||
_ -> do
|
||||
logWarn $ "Error forwarding to relay: " <> decodeLatin1 (strEncode $ transportHost' smp) <> " own=" <> tshow own <> " " <> tshow e
|
||||
inc own pErrorsOther
|
||||
Nothing -> inc False pRequests >> inc False pErrorsConnect $> Just (ERR $ PROXY NO_SESSION)
|
||||
where
|
||||
forkProxiedCmd :: M s BrokerMsg -> M s (Maybe BrokerMsg)
|
||||
@@ -1490,19 +1496,15 @@ client
|
||||
Just nenv -> pure (Just nenv)
|
||||
-- Runs on a forked thread so RSLV does not block other commands;
|
||||
-- concurrency is limited by serverResolverConcurrency in forkCmd.
|
||||
resolveNameMsg :: VersionSMP -> NamesEnv -> NameQuery -> M s BrokerMsg
|
||||
resolveNameMsg v nenv q = do
|
||||
resolveNameMsg :: NamesEnv -> SimplexDomain -> M s BrokerMsg
|
||||
resolveNameMsg nenv d = do
|
||||
st <- asks (rslvStats . serverStats)
|
||||
(selector, msg) <-
|
||||
liftIO (resolveName nenv q) <&> \case
|
||||
Right res -> (if answered (registration res) then rslvSucc else rslvNotFound, RNAME res)
|
||||
liftIO (resolveName nenv d) <&> \case
|
||||
Right rec -> (rslvSucc, RNAME rec)
|
||||
Left e@NOT_FOUND -> (rslvNotFound, ERR $ NAME e)
|
||||
Left e -> (rslvResolverErrs, ERR $ NAME e)
|
||||
incStat (selector st) $> msg
|
||||
where
|
||||
-- below v22 the encoder answers anything but a record as NAME NOT_FOUND
|
||||
answered = \case
|
||||
NRRegistered {} -> True
|
||||
_ -> v >= nameAvailSMPVersion
|
||||
transportErr :: TransportError -> ErrorType
|
||||
transportErr = PROXY . BROKER . TRANSPORT
|
||||
mkIncProxyStats :: MonadIO m => ProxyStats -> ProxyStats -> OwnServer -> (ProxyStats -> IORef Int) -> m ()
|
||||
@@ -1519,7 +1521,7 @@ client
|
||||
Cmd SProxyService (RFWD encBlock) -> (response . (corrId, NoEntity,) =<<) <$> processForwardedCommand encBlock
|
||||
Cmd SResolver (RSLV d) -> rslvNamesEnv >>= \case
|
||||
Nothing -> pure $ response (corrId, NoEntity, ERR (NAME NO_RESOLVER))
|
||||
Just nenv -> forkCmd serverResolverConcurrency corrId NoEntity (resolveNameMsg (thVersion thParams') nenv d)
|
||||
Just nenv -> forkCmd serverResolverConcurrency corrId NoEntity (resolveNameMsg nenv d)
|
||||
Cmd SSenderLink command -> case command of
|
||||
LKEY k -> withQueue $ \q qr -> checkMode QMMessaging qr $ secureQueue_ q k $>> getQueueLink_ q qr
|
||||
LGET -> withQueue $ \q qr -> checkContact qr $ getQueueLink_ q qr
|
||||
@@ -2124,10 +2126,9 @@ client
|
||||
let proxyNonce = C.cbNonce $ bs corrId
|
||||
s' <- liftEitherWith (const CRYPTO) $ C.cbDecryptNoPad sessSecret proxyNonce s
|
||||
FwdTransmission {fwdCorrId, fwdVersion, fwdKey, fwdTransmission = EncTransmission et} <- liftEitherWith (const $ CMD SYNTAX) $ smpDecode s'
|
||||
unless (fwdVersion `isCompatible` thServerVRange thParams') $ throwE $ transportErr TEVersion
|
||||
let clientSecret = C.dh' fwdKey serverPrivKey
|
||||
clientNonce = C.cbNonce $ bs fwdCorrId
|
||||
b <- liftEitherWith (const CRYPTO) $ C.cbDecrypt clientSecret (encTransmissionNonce fwdVersion clientNonce) et
|
||||
b <- liftEitherWith (const CRYPTO) $ C.cbDecrypt clientSecret clientNonce et
|
||||
let clntTHParams = smpTHParamsSetVersion fwdVersion thParams'
|
||||
-- only allowing single forwarded transactions
|
||||
t' <- case tParse clntTHParams b of
|
||||
@@ -2140,13 +2141,9 @@ client
|
||||
TBError _ _ : _ -> throwE BLOCK
|
||||
TBTransmission b' _ : _ -> pure b'
|
||||
TBTransmissions b' _ _ : _ -> pure b'
|
||||
nonce_ <-
|
||||
if fwdVersion >= fwdNoncesSMPVersion
|
||||
then Just <$> (atomically . C.randomCbNonce =<< asks random)
|
||||
else pure Nothing
|
||||
r2 <- liftEitherWith (const BLOCK) $ EncResponse <$> C.cbEncrypt clientSecret (fromMaybe (C.reverseNonce clientNonce) nonce_) r' paddedProxiedTLength
|
||||
r2 <- liftEitherWith (const BLOCK) $ EncResponse <$> C.cbEncrypt clientSecret (C.reverseNonce clientNonce) r' paddedProxiedTLength
|
||||
let fr = FwdResponse {fwdCorrId, fwdResponse = r2}
|
||||
pure $ RRES nonce_ $ EncFwdResponse $ C.cbEncryptNoPad sessSecret (C.reverseNonce proxyNonce) (smpEncode fr)
|
||||
pure $ RRES $ EncFwdResponse $ C.cbEncryptNoPad sessSecret (C.reverseNonce proxyNonce) (smpEncode fr)
|
||||
-- the inner response, or Nothing if forked (RSLV).
|
||||
r_ <- lift (rejectOrVerify clntThAuth t') >>= \case
|
||||
-- rejectOrVerify filters allowed commands, no need to repeat it here.
|
||||
@@ -2155,7 +2152,7 @@ client
|
||||
Cmd SResolver (RSLV d) -> lift $ rslvNamesEnv >>= \case
|
||||
Nothing -> pure $ Just (corrId', entId', ERR (NAME NO_RESOLVER))
|
||||
Just nenv -> forkCmd serverResolverConcurrency corrId NoEntity $ do
|
||||
msg <- resolveNameMsg (thVersion clntTHParams) nenv d
|
||||
msg <- resolveNameMsg nenv d
|
||||
either ERR id <$> runExceptT (encodeResp (corrId', entId', msg))
|
||||
-- INTERNAL because processCommand never returns Nothing for sender commands;
|
||||
-- `fst` drops the empty message only returned for SUB.
|
||||
@@ -2273,27 +2270,40 @@ randomId = fmap EntityId . randomId'
|
||||
|
||||
saveServerMessages :: Bool -> MsgStore s -> IO ()
|
||||
saveServerMessages drainMsgs ms = case ms of
|
||||
StoreMemory ms'@STMMsgStore {storeConfig = STMStoreConfig {storePath}} -> case storePath of
|
||||
Just f -> exportMessages ms' f drainMsgs
|
||||
StoreMemory STMMsgStore {storeConfig = STMStoreConfig {storePath}} -> case storePath of
|
||||
Just f -> exportMessages False ms f drainMsgs
|
||||
Nothing -> logNote "undelivered messages are not saved"
|
||||
StoreJournal _ -> logNote "closed journal message storage"
|
||||
#if defined(dbServerPostgres)
|
||||
StoreDatabase _ -> logNote "closed postgres message storage"
|
||||
#endif
|
||||
|
||||
exportMessages :: STMMsgStore -> FilePath -> Bool -> IO ()
|
||||
exportMessages ms f drainMsgs = do
|
||||
exportMessages :: forall s. MsgStoreClass s => Bool -> MsgStore s -> FilePath -> Bool -> IO ()
|
||||
exportMessages tty st f drainMsgs = do
|
||||
logNote $ "saving messages to file " <> T.pack f
|
||||
run $ fmap (\(Sum n) -> n) . withLoadedQueues (queueStore ms) . saveQueueMsgs
|
||||
run $ case st of
|
||||
StoreMemory ms -> exportMessages_ ms $ getMsgs ms
|
||||
StoreJournal ms -> exportMessages_ ms $ getJournalMsgs ms
|
||||
#if defined(dbServerPostgres)
|
||||
StoreDatabase ms -> exportDbMessages tty ms
|
||||
#endif
|
||||
where
|
||||
exportMessages_ ms get = fmap (\(Sum n) -> n) . unsafeWithAllMsgQueues tty ms . saveQueueMsgs get
|
||||
run :: (Handle -> IO Int) -> IO ()
|
||||
run a = liftIO $ withFile f WriteMode $ tryAny . a >=> \case
|
||||
Right n -> logNote $ "messages saved: " <> tshow n
|
||||
Left e -> do
|
||||
logError $ "error exporting messages: " <> tshow e
|
||||
exitFailure
|
||||
saveQueueMsgs :: Handle -> STMQueue -> IO (Sum Int)
|
||||
saveQueueMsgs h q = do
|
||||
msgs <- getQueueMessages drainMsgs q
|
||||
getJournalMsgs ms q =
|
||||
readTVarIO (msgQueue' q) >>= \case
|
||||
Just _ -> getMsgs ms q
|
||||
Nothing -> getJournalQueueMessages ms q
|
||||
getMsgs :: MsgStoreClass s' => s' -> StoreQueue s' -> IO [Message]
|
||||
getMsgs ms q = unsafeRunStore q "saveQueueMsgs" $ getQueueMessages_ drainMsgs q =<< getMsgQueue ms q False
|
||||
saveQueueMsgs :: (StoreQueue s -> IO [Message]) -> Handle -> StoreQueue s -> IO (Sum Int)
|
||||
saveQueueMsgs get h q = do
|
||||
msgs <- get q
|
||||
unless (null msgs) $ BLD.hPutBuilder h $ encodeMessages (recipientId q) msgs
|
||||
pure $ Sum $ length msgs
|
||||
encodeMessages rId = mconcat . map (\msg -> BLD.byteString (strEncode $ MLRv3 rId msg) <> BLD.char8 '\n')
|
||||
@@ -2305,16 +2315,37 @@ processServerMessages StartOptions {skipWarnings} = do
|
||||
asks msgStore_ >>= liftIO . processMessages old_ expire
|
||||
where
|
||||
processMessages :: Maybe Int64 -> Bool -> MsgStore s' -> IO (Maybe MessageStats)
|
||||
#if defined(dbServerPostgres)
|
||||
processMessages old_ expire = \case
|
||||
#else
|
||||
processMessages old_ _ = \case
|
||||
#endif
|
||||
StoreMemory ms@STMMsgStore {storeConfig = STMStoreConfig {storePath}} -> case storePath of
|
||||
Just f -> ifM (doesFileExist f) (Just <$> importMessages ms f old_ skipWarnings) (pure Nothing)
|
||||
Just f -> ifM (doesFileExist f) (Just <$> importMessages False ms f old_ skipWarnings) (pure Nothing)
|
||||
Nothing -> pure Nothing
|
||||
StoreJournal ms -> processJournalMessages old_ expire ms
|
||||
#if defined(dbServerPostgres)
|
||||
StoreDatabase ms -> processDbMessages old_ expire ms
|
||||
#endif
|
||||
processJournalMessages :: forall s. Maybe Int64 -> Bool -> JournalMsgStore s -> IO (Maybe MessageStats)
|
||||
processJournalMessages old_ expire ms
|
||||
| expire = Just <$> case old_ of
|
||||
Just old -> do
|
||||
logNote "expiring journal store messages..."
|
||||
run $ processExpireQueue old
|
||||
Nothing -> do
|
||||
logNote "validating journal store messages..."
|
||||
run processValidateQueue
|
||||
| otherwise = logWarn "skipping message expiration" $> Nothing
|
||||
where
|
||||
run a = unsafeWithAllMsgQueues False ms a `catchAny` \_ -> exitFailure
|
||||
processExpireQueue :: Int64 -> JournalQueue s -> IO MessageStats
|
||||
processExpireQueue old q = unsafeRunStore q "processExpireQueue" $ do
|
||||
mq <- getMsgQueue ms q False
|
||||
expiredMsgsCount <- deleteExpireMsgs_ old q mq
|
||||
storedMsgsCount <- getQueueSize_ mq
|
||||
pure MessageStats {storedMsgsCount, expiredMsgsCount, storedQueues = 1}
|
||||
processValidateQueue :: JournalQueue s -> IO MessageStats
|
||||
processValidateQueue q = unsafeRunStore q "processValidateQueue" $ do
|
||||
storedMsgsCount <- getQueueSize_ =<< getMsgQueue ms q False
|
||||
pure newMessageStats {storedMsgsCount, storedQueues = 1}
|
||||
#if defined(dbServerPostgres)
|
||||
processDbMessages old_ expire ms
|
||||
| expire = Just <$> case old_ of
|
||||
Just old -> do
|
||||
@@ -2326,17 +2357,18 @@ processServerMessages StartOptions {skipWarnings} = do
|
||||
| otherwise = logWarn "skipping message expiration" $> Nothing
|
||||
#endif
|
||||
|
||||
importMessages :: STMMsgStore -> FilePath -> Maybe Int64 -> Bool -> IO MessageStats
|
||||
importMessages ms f old_ skipWarnings = do
|
||||
importMessages :: forall s. MsgStoreClass s => Bool -> s -> FilePath -> Maybe Int64 -> Bool -> IO MessageStats
|
||||
importMessages tty ms f old_ skipWarnings = do
|
||||
logNote $ "restoring messages from file " <> T.pack f
|
||||
(_, (storedMsgsCount, expiredMsgsCount, overQuota)) <-
|
||||
foldLogLines False f restoreMsg (Nothing, (0, 0, M.empty))
|
||||
foldLogLines tty f restoreMsg (Nothing, (0, 0, M.empty))
|
||||
renameFile f $ f <> ".bak"
|
||||
mapM_ setOverQuota_ overQuota
|
||||
EntityCounts {queueCount} <- liftIO $ getEntityCounts @STMQueue $ queueStore ms
|
||||
logQueueStates ms
|
||||
EntityCounts {queueCount} <- liftIO $ getEntityCounts @(StoreQueue s) $ queueStore ms
|
||||
pure MessageStats {storedMsgsCount, expiredMsgsCount, storedQueues = queueCount}
|
||||
where
|
||||
restoreMsg :: (Maybe (RecipientId, STMQueue), (Int, Int, Map RecipientId STMQueue)) -> Bool -> ByteString -> IO (Maybe (RecipientId, STMQueue), (Int, Int, Map RecipientId STMQueue))
|
||||
restoreMsg :: (Maybe (RecipientId, StoreQueue s), (Int, Int, Map RecipientId (StoreQueue s))) -> Bool -> ByteString -> IO (Maybe (RecipientId, StoreQueue s), (Int, Int, Map RecipientId (StoreQueue s)))
|
||||
restoreMsg (q_, counts@(!stored, !expired, !overQuota)) eof s = case strDecode s of
|
||||
Right (MLRv3 rId msg) -> runExceptT (addToMsgQueue rId msg) >>= either (exitErr . tshow) pure
|
||||
Left e
|
||||
@@ -2344,6 +2376,7 @@ importMessages ms f old_ skipWarnings = do
|
||||
| otherwise -> exitErr $ parsingErr e
|
||||
where
|
||||
exitErr e = do
|
||||
when tty $ putStrLn ""
|
||||
logError $ "error restoring messages: " <> e
|
||||
liftIO exitFailure
|
||||
parsingErr :: String -> Text
|
||||
@@ -2356,6 +2389,7 @@ importMessages ms f old_ skipWarnings = do
|
||||
case qOrErr of
|
||||
Right q -> addToQueue_ q rId msg
|
||||
Left AUTH -> liftIO $ do
|
||||
when tty $ putStrLn ""
|
||||
warnOrExit $ "queue " <> safeDecodeUtf8 (encode $ unEntityId rId) <> " does not exist"
|
||||
pure (Nothing, counts)
|
||||
Left e -> throwE e
|
||||
@@ -2366,13 +2400,20 @@ importMessages ms f old_ skipWarnings = do
|
||||
writeMsg ms q False msg >>= \case
|
||||
Just _ -> pure (stored + 1, expired, overQuota)
|
||||
Nothing -> liftIO $ do
|
||||
when tty $ putStrLn ""
|
||||
logError $ decodeLatin1 $ "message queue " <> strEncode rId <> " is full, message not restored: " <> strEncode (messageId msg)
|
||||
pure counts
|
||||
| otherwise -> pure (stored, expired + 1, overQuota)
|
||||
MessageQuota {} ->
|
||||
-- queue was over quota at some point,
|
||||
-- it will be set as over quota once fully imported
|
||||
deleteQuotaMsg q >> writeMsg ms q False msg $> (stored, expired, M.insert rId q overQuota)
|
||||
mergeQuotaMsgs >> writeMsg ms q False msg $> (stored, expired, M.insert rId q overQuota)
|
||||
where
|
||||
-- if the first message in queue head is "quota", remove it.
|
||||
mergeQuotaMsgs =
|
||||
withPeekMsgQueue ms q "mergeQuotaMsgs" $ maybe (pure ()) $ \case
|
||||
(mq, MessageQuota {}) -> tryDeleteMsg_ q mq False
|
||||
_ -> pure ()
|
||||
warnOrExit e
|
||||
| skipWarnings = logWarn e'
|
||||
| otherwise = do
|
||||
|
||||
@@ -42,6 +42,7 @@ module Simplex.Messaging.Server.Env.STM
|
||||
VerifiedTransmission,
|
||||
ResponseAndMessage,
|
||||
newEnv,
|
||||
mkJournalStoreConfig,
|
||||
msgStore,
|
||||
fromMsgStore,
|
||||
newClient,
|
||||
@@ -67,6 +68,10 @@ module Simplex.Messaging.Server.Env.STM
|
||||
defaultInactiveClientExpiration,
|
||||
defaultProxyClientConcurrency,
|
||||
defaultNameResolverConcurrency,
|
||||
defaultMaxJournalMsgCount,
|
||||
defaultMaxJournalStateLines,
|
||||
defaultIdleQueueInterval,
|
||||
journalMsgStoreDepth,
|
||||
readWriteQueueStore,
|
||||
noPostgresExitStr,
|
||||
noPostgresExit,
|
||||
@@ -91,7 +96,7 @@ import Data.List.NonEmpty (NonEmpty)
|
||||
import Data.Map.Strict (Map)
|
||||
import Data.Maybe (isJust)
|
||||
import qualified Data.Text as T
|
||||
import Data.Time.Clock (getCurrentTime)
|
||||
import Data.Time.Clock (getCurrentTime, nominalDay)
|
||||
import Data.Time.Clock.System (SystemTime)
|
||||
import qualified Data.X509 as X
|
||||
import Data.X509.Validation (Fingerprint (..))
|
||||
@@ -108,6 +113,7 @@ import qualified Simplex.Messaging.Crypto as C
|
||||
import Simplex.Messaging.Protocol
|
||||
import Simplex.Messaging.Server.Expiration
|
||||
import Simplex.Messaging.Server.Information
|
||||
import Simplex.Messaging.Server.MsgStore.Journal
|
||||
import Simplex.Messaging.Server.MsgStore.STM
|
||||
import Simplex.Messaging.Server.MsgStore.Types
|
||||
import Simplex.Messaging.Server.Names (NamesConfig (..), NamesEnv, newNamesEnv, pingEndpoint)
|
||||
@@ -140,6 +146,8 @@ data ServerConfig s = ServerConfig
|
||||
smpHandshakeTimeout :: Int,
|
||||
tbqSize :: Natural,
|
||||
msgQueueQuota :: Int,
|
||||
maxJournalMsgCount :: Int,
|
||||
maxJournalStateLines :: Int,
|
||||
queueIdBytes :: Int,
|
||||
msgIdBytes :: Int,
|
||||
serverStoreCfg :: ServerStoreCfg s,
|
||||
@@ -156,6 +164,8 @@ data ServerConfig s = ServerConfig
|
||||
messageExpiration :: Maybe ExpirationConfig,
|
||||
expireMessagesOnStart :: Bool,
|
||||
expireMessagesOnSend :: Bool,
|
||||
-- | interval of inactivity after which journal queue is closed
|
||||
idleQueueInterval :: Int64,
|
||||
-- | notification expiration interval (seconds)
|
||||
notificationExpiration :: ExpirationConfig,
|
||||
-- | time after which the socket with inactive client can be disconnected (without any messages or commands, incl. PING),
|
||||
@@ -219,6 +229,9 @@ defaultMessageExpiration =
|
||||
checkInterval = 7200 -- seconds, 2 hours
|
||||
}
|
||||
|
||||
defaultIdleQueueInterval :: Int64
|
||||
defaultIdleQueueInterval = 14400 -- seconds, 4 hours
|
||||
|
||||
defNtfExpirationHours :: Int64
|
||||
defNtfExpirationHours = 24
|
||||
|
||||
@@ -242,9 +255,21 @@ defaultProxyClientConcurrency = 32
|
||||
defaultNameResolverConcurrency :: Int
|
||||
defaultNameResolverConcurrency = 1000
|
||||
|
||||
journalMsgStoreDepth :: Int
|
||||
journalMsgStoreDepth = 5
|
||||
|
||||
defaultMaxJournalStateLines :: Int
|
||||
defaultMaxJournalStateLines = 16
|
||||
|
||||
defaultMaxJournalMsgCount :: Int
|
||||
defaultMaxJournalMsgCount = 256
|
||||
|
||||
defaultMsgQueueQuota :: Int
|
||||
defaultMsgQueueQuota = 128
|
||||
|
||||
defaultStateTailSize :: Int
|
||||
defaultStateTailSize = 512
|
||||
|
||||
data Env s = Env
|
||||
{ config :: ServerConfig s,
|
||||
serverActive :: TVar Bool,
|
||||
@@ -270,6 +295,7 @@ msgStore = fromMsgStore . msgStore_
|
||||
fromMsgStore :: MsgStore s -> s
|
||||
fromMsgStore = \case
|
||||
StoreMemory s -> s
|
||||
StoreJournal s -> s
|
||||
#if defined(dbServerPostgres)
|
||||
StoreDatabase s -> s
|
||||
#endif
|
||||
@@ -277,10 +303,12 @@ fromMsgStore = \case
|
||||
|
||||
type family SupportedStore (qs :: QSType) (ms :: MSType) :: Constraint where
|
||||
SupportedStore 'QSMemory 'MSMemory = ()
|
||||
SupportedStore 'QSMemory 'MSJournal = ()
|
||||
SupportedStore 'QSMemory 'MSPostgres =
|
||||
(Int ~ Bool, TypeError ('TE.Text "Storing messages in Postgres DB with queues in memory is not supported"))
|
||||
SupportedStore 'QSPostgres 'MSMemory =
|
||||
(Int ~ Bool, TypeError ('TE.Text "Storing messages in memory with queues in Postgres DB is not supported"))
|
||||
SupportedStore 'QSPostgres 'MSJournal = ()
|
||||
#if defined(dbServerPostgres)
|
||||
SupportedStore 'QSPostgres 'MSPostgres = ()
|
||||
#else
|
||||
@@ -294,6 +322,8 @@ data AStoreType =
|
||||
|
||||
data ServerStoreCfg s where
|
||||
SSCMemory :: Maybe StorePaths -> ServerStoreCfg STMMsgStore
|
||||
SSCMemoryJournal :: {storeLogFile :: FilePath, storeMsgsPath :: FilePath} -> ServerStoreCfg (JournalMsgStore 'QSMemory)
|
||||
SSCDatabaseJournal :: {storeCfg :: PostgresStoreCfg, storeMsgsPath' :: FilePath} -> ServerStoreCfg (JournalMsgStore 'QSPostgres)
|
||||
#if defined(dbServerPostgres)
|
||||
SSCDatabase :: PostgresStoreCfg -> ServerStoreCfg PostgresMsgStore
|
||||
#endif
|
||||
@@ -301,6 +331,8 @@ data ServerStoreCfg s where
|
||||
dbStoreCfg :: ServerStoreCfg s -> Maybe PostgresStoreCfg
|
||||
dbStoreCfg = \case
|
||||
SSCMemory _ -> Nothing
|
||||
SSCMemoryJournal {} -> Nothing
|
||||
SSCDatabaseJournal {storeCfg} -> Just storeCfg
|
||||
#if defined(dbServerPostgres)
|
||||
SSCDatabase cfg -> Just cfg
|
||||
#endif
|
||||
@@ -308,6 +340,8 @@ dbStoreCfg = \case
|
||||
storeLogFile' :: ServerStoreCfg s -> Maybe FilePath
|
||||
storeLogFile' = \case
|
||||
SSCMemory sp_ -> (\StorePaths {storeLogFile} -> storeLogFile) <$> sp_
|
||||
SSCMemoryJournal {storeLogFile} -> Just storeLogFile
|
||||
SSCDatabaseJournal {storeCfg = PostgresStoreCfg {dbStoreLogPath}} -> dbStoreLogPath
|
||||
#if defined(dbServerPostgres)
|
||||
SSCDatabase (PostgresStoreCfg {dbStoreLogPath}) -> dbStoreLogPath
|
||||
#endif
|
||||
@@ -316,12 +350,14 @@ data StorePaths = StorePaths {storeLogFile :: FilePath, storeMsgsFile :: Maybe F
|
||||
|
||||
type family MsgStoreType (qs :: QSType) (ms :: MSType) where
|
||||
MsgStoreType 'QSMemory 'MSMemory = STMMsgStore
|
||||
MsgStoreType qs 'MSJournal = JournalMsgStore qs
|
||||
#if defined(dbServerPostgres)
|
||||
MsgStoreType 'QSPostgres 'MSPostgres = PostgresMsgStore
|
||||
#endif
|
||||
|
||||
data MsgStore s where
|
||||
StoreMemory :: STMMsgStore -> MsgStore STMMsgStore
|
||||
StoreJournal :: JournalMsgStore qs -> MsgStore (JournalMsgStore qs)
|
||||
#if defined(dbServerPostgres)
|
||||
StoreDatabase :: PostgresMsgStore -> MsgStore PostgresMsgStore
|
||||
#endif
|
||||
@@ -535,21 +571,39 @@ newProhibitedSub = do
|
||||
return Sub {subThread = ProhibitSub, delivered}
|
||||
|
||||
newEnv :: ServerConfig s -> IO (Env s)
|
||||
newEnv config@ServerConfig {smpCredentials, httpCredentials, serverStoreCfg, smpAgentCfg, information, messageExpiration, msgQueueQuota, namesConfig} = do
|
||||
newEnv config@ServerConfig {smpCredentials, httpCredentials, serverStoreCfg, smpAgentCfg, information, messageExpiration, idleQueueInterval, msgQueueQuota, maxJournalMsgCount, maxJournalStateLines, namesConfig} = do
|
||||
serverActive <- newTVarIO True
|
||||
server <- newServer
|
||||
msgStore_ <- case serverStoreCfg of
|
||||
SSCMemory storePaths_ -> do
|
||||
let storePath = storeMsgsFile =<< storePaths_
|
||||
ms <- newMsgStore STMStoreConfig {storePath, quota = msgQueueQuota}
|
||||
forM_ storePaths_ $ \StorePaths {storeLogFile = f} -> loadStoreLog (mkQueue ms) f $ queueStore ms
|
||||
forM_ storePaths_ $ \StorePaths {storeLogFile = f} -> loadStoreLog (mkQueue ms True) f $ queueStore ms
|
||||
pure $ StoreMemory ms
|
||||
SSCMemoryJournal {storeLogFile, storeMsgsPath} -> do
|
||||
logWarn $
|
||||
"Journal message store is deprecated and will be removed soon.\n"
|
||||
<> "Please migrate to in-memory storage using `journal export` command.\n"
|
||||
<> "After that you can migrate to PostgreSQL using `database import` command."
|
||||
let qsCfg = MQStoreCfg
|
||||
cfg = mkJournalStoreConfig qsCfg storeMsgsPath msgQueueQuota maxJournalMsgCount maxJournalStateLines idleQueueInterval
|
||||
ms <- newMsgStore cfg
|
||||
loadStoreLog (mkQueue ms True) storeLogFile $ stmQueueStore ms
|
||||
pure $ StoreJournal ms
|
||||
#if defined(dbServerPostgres)
|
||||
SSCDatabaseJournal {storeCfg, storeMsgsPath'} -> do
|
||||
let StartOptions {compactLog, confirmMigrations} = startOptions config
|
||||
qsCfg = PQStoreCfg (storeCfg {confirmMigrations} :: PostgresStoreCfg)
|
||||
cfg = mkJournalStoreConfig qsCfg storeMsgsPath' msgQueueQuota maxJournalMsgCount maxJournalStateLines idleQueueInterval
|
||||
when compactLog $ compactDbStoreLog $ dbStoreLogPath storeCfg
|
||||
StoreJournal <$> newMsgStore cfg
|
||||
SSCDatabase storeCfg -> do
|
||||
let StartOptions {compactLog, confirmMigrations} = startOptions config
|
||||
cfg = PostgresMsgStoreCfg storeCfg {confirmMigrations} msgQueueQuota
|
||||
when compactLog $ compactDbStoreLog $ dbStoreLogPath storeCfg
|
||||
StoreDatabase <$> newMsgStore cfg
|
||||
#else
|
||||
SSCDatabaseJournal {} -> noPostgresExit
|
||||
#endif
|
||||
ntfStore <- NtfStore <$> TM.emptyIO
|
||||
random <- C.newRandom
|
||||
@@ -601,7 +655,8 @@ newEnv config@ServerConfig {smpCredentials, httpCredentials, serverStoreCfg, smp
|
||||
Just f -> do
|
||||
logNote $ "compacting queues in file " <> T.pack f
|
||||
st <- newMsgStore STMStoreConfig {storePath = Nothing, quota = msgQueueQuota}
|
||||
sl <- readWriteQueueStore False (mkQueue st) f (queueStore st)
|
||||
-- we don't need to have locks in the map
|
||||
sl <- readWriteQueueStore False (mkQueue st False) f (queueStore st)
|
||||
setStoreLog (queueStore st) sl
|
||||
closeMsgStore st
|
||||
Nothing -> do
|
||||
@@ -648,9 +703,7 @@ newEnv config@ServerConfig {smpCredentials, httpCredentials, serverStoreCfg, smp
|
||||
Nothing -> SPMMemoryOnly
|
||||
Just StorePaths {storeMsgsFile = Just _} -> SPMMessages
|
||||
_ -> SPMQueues
|
||||
#if defined(dbServerPostgres)
|
||||
SSCDatabase _ -> SPMMessages
|
||||
#endif
|
||||
_ -> SPMMessages
|
||||
|
||||
noPostgresExit :: IO a
|
||||
noPostgresExit = putStrLn noPostgresExitStr >> exitFailure
|
||||
@@ -660,6 +713,21 @@ noPostgresExitStr =
|
||||
"Error: server binary is compiled without support for PostgreSQL database.\n"
|
||||
<> "Please download `smp-server-postgres` or re-compile with `cabal build -fserver_postgres`."
|
||||
|
||||
mkJournalStoreConfig :: QStoreCfg s -> FilePath -> Int -> Int -> Int -> Int64 -> JournalStoreConfig s
|
||||
mkJournalStoreConfig queueStoreCfg storePath msgQueueQuota maxJournalMsgCount maxJournalStateLines idleQueueInterval =
|
||||
JournalStoreConfig
|
||||
{ storePath,
|
||||
quota = msgQueueQuota,
|
||||
pathParts = journalMsgStoreDepth,
|
||||
queueStoreCfg,
|
||||
maxMsgCount = maxJournalMsgCount,
|
||||
maxStateLines = maxJournalStateLines,
|
||||
stateTailSize = defaultStateTailSize,
|
||||
idleInterval = idleQueueInterval,
|
||||
expireBackupsAfter = 14 * nominalDay,
|
||||
keepMinBackups = 2
|
||||
}
|
||||
|
||||
newSMPProxyAgent :: SMPClientAgentConfig -> TVar ChaChaDRG -> IO ProxyAgent
|
||||
newSMPProxyAgent smpAgentCfg random = do
|
||||
smpAgent <- newSMPClientAgent SSender smpAgentCfg Nothing random
|
||||
|
||||
@@ -28,6 +28,8 @@ module Simplex.Messaging.Server.Main
|
||||
importMessagesToDatabase,
|
||||
exportDatabaseToStoreLog,
|
||||
#endif
|
||||
newJournalMsgStore,
|
||||
storeMsgsJournalDir',
|
||||
getServerSourceCode,
|
||||
simplexmqSource,
|
||||
serverPublicInfo,
|
||||
@@ -59,6 +61,7 @@ import qualified Data.Text.IO as T
|
||||
import Options.Applicative
|
||||
import Simplex.Messaging.Agent.Protocol (ConnectionLink (..), ConnectionMode (..), connReqUriP')
|
||||
import Simplex.Messaging.Agent.Store.Postgres.Options (DBOpts (..))
|
||||
import Simplex.Messaging.Agent.Store.Shared (MigrationConfirmation (..))
|
||||
import Simplex.Messaging.Client (HostMode (..), NetworkConfig (..), ProtocolClientConfig (..), SMPWebPortServers (..), SocksMode (..), defaultNetworkConfig, textToHostMode)
|
||||
import Simplex.Messaging.Client.Agent (SMPClientAgentConfig (..), defaultSMPClientAgentConfig)
|
||||
import qualified Simplex.Messaging.Crypto as C
|
||||
@@ -66,20 +69,23 @@ import Simplex.Messaging.Encoding.String
|
||||
import Simplex.Messaging.Parsers (parseAll)
|
||||
import Network.URI (URI (..), URIAuth (..), parseAbsoluteURI)
|
||||
import Simplex.Messaging.Protocol (BasicAuth (..), ProtoServerWithAuth (ProtoServerWithAuth), pattern SMPServer)
|
||||
import Simplex.Messaging.Server (AttachHTTP, runSMPServer)
|
||||
import Simplex.Messaging.Server (AttachHTTP, exportMessages, importMessages, printMessageStats, runSMPServer)
|
||||
import Simplex.Messaging.Server.CLI
|
||||
import Simplex.Messaging.Server.Env.STM
|
||||
import Simplex.Messaging.Server.Expiration
|
||||
import Simplex.Messaging.Server.Information
|
||||
import Simplex.Messaging.Server.Main.Init
|
||||
import Simplex.Messaging.Server.Web (EmbeddedWebParams (..), WebHttpsParams (..))
|
||||
import Simplex.Messaging.Server.MsgStore.Types (SQSType (..), SMSType (..))
|
||||
import Simplex.Messaging.Server.MsgStore.Journal (JournalMsgStore (..), QStoreCfg (..), stmQueueStore)
|
||||
import Simplex.Messaging.Server.MsgStore.Types (MsgStoreClass (..), SQSType (..), SMSType (..), newMsgStore)
|
||||
import Simplex.Messaging.Server.Names (NamesConfig (..), RpcAuth (..))
|
||||
import Simplex.Messaging.Server.QueueStore.Postgres.Config
|
||||
import Simplex.Messaging.Server.StoreLog.ReadWrite (readQueueStore)
|
||||
import Simplex.Messaging.Transport (supportedProxyClientSMPRelayVRange, alpnSupportedSMPHandshakes, supportedServerSMPRelayVRange)
|
||||
import Simplex.Messaging.Transport.Client (TransportHost (..), defaultSocksProxy)
|
||||
import Simplex.Messaging.Transport.HTTP2 (httpALPN)
|
||||
import Simplex.Messaging.Transport.Server (ServerCredentials (..), mkTransportServerConfig)
|
||||
import Simplex.Messaging.Util (eitherToMaybe, safeDecodeUtf8, whenM)
|
||||
import Simplex.Messaging.Util (eitherToMaybe, ifM, safeDecodeUtf8)
|
||||
import System.Directory (createDirectoryIfMissing, doesDirectoryExist, doesFileExist)
|
||||
import System.Exit (exitFailure)
|
||||
import System.FilePath (combine)
|
||||
@@ -92,16 +98,15 @@ import Data.Int (Int64)
|
||||
import qualified Data.Map.Strict as M
|
||||
import Data.Semigroup (Sum (..))
|
||||
import Simplex.Messaging.Agent.Store.Postgres (checkSchemaExists)
|
||||
import Simplex.Messaging.Agent.Store.Shared (MigrationConfirmation (..))
|
||||
import Simplex.Messaging.Server.MsgStore.Journal (JournalQueue)
|
||||
import Simplex.Messaging.Server.MsgStore.Types (QSType (..))
|
||||
import Simplex.Messaging.Server.MsgStore.Journal (postgresQueueStore)
|
||||
import Simplex.Messaging.Server.MsgStore.Postgres
|
||||
import Simplex.Messaging.Server.MsgStore.STM (STMStoreConfig (..))
|
||||
import Simplex.Messaging.Server.MsgStore.Types (MsgStoreClass (..), newMsgStore)
|
||||
import Simplex.Messaging.Server.QueueStore.Postgres.Config
|
||||
import Simplex.Messaging.Server.QueueStore.Postgres (PostgresQueueStore, batchInsertQueues, batchInsertServices, foldQueueRecs, foldServiceRecs)
|
||||
import Simplex.Messaging.Server.QueueStore.Postgres (batchInsertQueues, batchInsertServices, foldQueueRecs, foldServiceRecs)
|
||||
import Simplex.Messaging.Server.QueueStore.STM (STMQueueStore (..))
|
||||
import Simplex.Messaging.Server.QueueStore.Types
|
||||
import Simplex.Messaging.Server.StoreLog (closeStoreLog, logNewService, logCreateQueue, openWriteStoreLog)
|
||||
import Simplex.Messaging.Util (ifM)
|
||||
import Simplex.Messaging.Util (unlessM)
|
||||
import System.Directory (renameFile)
|
||||
import System.IO (IOMode (..), withFile)
|
||||
#endif
|
||||
@@ -131,6 +136,79 @@ smpServerCLI_ generateSite serveStaticFiles attachStaticFiles cfgPath logPath =
|
||||
deleteDirIfExists cfgPath
|
||||
deleteDirIfExists logPath
|
||||
putStrLn "Deleted configuration and log files"
|
||||
Journal cmd -> withIniFile $ \ini -> do
|
||||
msgsDirExists <- doesDirectoryExist storeMsgsJournalDir
|
||||
msgsFileExists <- doesFileExist storeMsgsFilePath
|
||||
storeLogFile <- getRequiredStoreLogFile ini
|
||||
case cmd of
|
||||
SCImport
|
||||
| msgsFileExists && msgsDirExists -> exitConfigureMsgStorage
|
||||
| msgsDirExists -> do
|
||||
putStrLn $ storeMsgsJournalDir <> " directory already exists."
|
||||
exitFailure
|
||||
| not msgsFileExists -> do
|
||||
putStrLn $ storeMsgsFilePath <> " file does not exist."
|
||||
exitFailure
|
||||
| otherwise -> do
|
||||
confirmOrExit
|
||||
("WARNING: message log file " <> storeMsgsFilePath <> " will be imported to journal directory " <> storeMsgsJournalDir)
|
||||
"Messages not imported"
|
||||
ms <- newJournalMsgStore logPath MQStoreCfg
|
||||
readQueueStore True (mkQueue ms False) storeLogFile $ stmQueueStore ms
|
||||
msgStats <- importMessages True ms storeMsgsFilePath Nothing False -- no expiration
|
||||
putStrLn "Import completed"
|
||||
printMessageStats "Messages" msgStats
|
||||
putStrLn $ case readStoreType ini of
|
||||
Right (ASType SQSMemory SMSMemory) -> "store_messages set to `memory`, update it to `journal` in INI file"
|
||||
Right (ASType SQSPostgres SMSPostgres) -> "store_messages set to `database`, update it to `journal` in INI file"
|
||||
Right (ASType _ SMSJournal) -> "store_messages set to `journal`"
|
||||
Left e -> e <> ", configure storage correctly"
|
||||
SCExport
|
||||
| msgsFileExists && msgsDirExists -> exitConfigureMsgStorage
|
||||
| msgsFileExists -> do
|
||||
putStrLn $ storeMsgsFilePath <> " file already exists."
|
||||
exitFailure
|
||||
| otherwise -> do
|
||||
confirmOrExit
|
||||
("WARNING: journal directory " <> storeMsgsJournalDir <> " will be exported to message log file " <> storeMsgsFilePath)
|
||||
"Journal not exported"
|
||||
case readStoreType ini of
|
||||
Right (ASType SQSMemory msType) -> do
|
||||
ms <- newJournalMsgStore logPath MQStoreCfg
|
||||
readQueueStore True (mkQueue ms False) storeLogFile $ stmQueueStore ms
|
||||
exportMessages True (StoreJournal ms) storeMsgsFilePath False
|
||||
putStrLn "Export completed"
|
||||
putStrLn $ case msType of
|
||||
SMSMemory -> "store_messages set to `memory`, start the server."
|
||||
SMSJournal -> "store_messages set to `journal`, update it to `memory` in INI file"
|
||||
#if defined(dbServerPostgres)
|
||||
Right (ASType SQSPostgres SMSJournal) -> do
|
||||
let dbStoreLogPath = enableDbStoreLog' ini $> storeLogFilePath
|
||||
dbOpts@DBOpts {connstr, schema} = iniDBOptions ini defaultDBOpts
|
||||
unlessM (checkSchemaExists connstr schema) $ do
|
||||
putStrLn $ "Schema " <> B.unpack schema <> " does not exist in PostrgreSQL database: " <> B.unpack connstr
|
||||
exitFailure
|
||||
ms <- newJournalMsgStore logPath $ PQStoreCfg PostgresStoreCfg {dbOpts, dbStoreLogPath, confirmMigrations = MCYesUp, deletedTTL = iniDeletedTTL ini}
|
||||
exportMessages True (StoreJournal ms) storeMsgsFilePath False
|
||||
putStrLn "Export completed"
|
||||
putStrLn "store_messages set to `journal`, store_queues is set to `database`.\nExport queues to store log to use memory storage for messages (`smp-server database export`)."
|
||||
Right (ASType SQSPostgres SMSPostgres) -> do
|
||||
putStrLn $ "Messages can be exported with `dabatase export --table messages`."
|
||||
exitFailure
|
||||
#else
|
||||
Right (ASType SQSPostgres SMSJournal) -> noPostgresExit
|
||||
#endif
|
||||
Left e -> putStrLn $ e <> ", configure storage correctly"
|
||||
SCDelete
|
||||
| not msgsDirExists -> do
|
||||
putStrLn $ storeMsgsJournalDir <> " directory does not exists."
|
||||
exitFailure
|
||||
| otherwise -> do
|
||||
confirmOrExit
|
||||
("WARNING: journal directory " <> storeMsgsJournalDir <> " will be permanently deleted.\nTHIS CANNOT BE UNDONE!")
|
||||
"Messages NOT deleted"
|
||||
deleteDirIfExists storeMsgsJournalDir
|
||||
putStrLn $ "Deleted all messages in journal " <> storeMsgsJournalDir
|
||||
#if defined(dbServerPostgres)
|
||||
Database cmd tables dbOpts@DBOpts {connstr, schema} -> withIniFile $ \ini -> do
|
||||
schemaExists <- checkSchemaExists connstr schema
|
||||
@@ -143,7 +221,7 @@ smpServerCLI_ generateSite serveStaticFiles attachStaticFiles cfgPath logPath =
|
||||
confirmOrExit
|
||||
("WARNING: store log file " <> storeLogFile <> " and message log file " <> storeMsgsFilePath <> " will be imported to PostrgreSQL database: " <> B.unpack connstr <> ", schema: " <> B.unpack schema)
|
||||
"Store logs not imported"
|
||||
(sCnt, qCnt) <- importStoreLogToDatabase storeLogFile dbOpts
|
||||
(sCnt, qCnt) <- importStoreLogToDatabase logPath storeLogFile dbOpts
|
||||
putStrLn $ "Imported: " <> show sCnt <> " services, " <> show qCnt <> " queues"
|
||||
putStrLn "Importing messages..."
|
||||
mCnt <- importMessagesToDatabase storeMsgsFilePath dbOpts
|
||||
@@ -170,10 +248,11 @@ smpServerCLI_ generateSite serveStaticFiles attachStaticFiles cfgPath logPath =
|
||||
confirmOrExit
|
||||
("WARNING: store log file " <> storeLogFile <> " will be compacted and imported to PostrgreSQL database: " <> B.unpack connstr <> ", schema: " <> B.unpack schema)
|
||||
"Queue records not imported"
|
||||
(sCnt, qCnt) <- importStoreLogToDatabase storeLogFile dbOpts
|
||||
(sCnt, qCnt) <- importStoreLogToDatabase logPath storeLogFile dbOpts
|
||||
putStrLn $ "Import completed: " <> show sCnt <> " services, " <> show qCnt <> " queues"
|
||||
putStrLn $ case readStoreType ini of
|
||||
Right (ASType SQSMemory SMSMemory) -> setToDbStr <> "\nstore_messages set to `memory`, update it to `database` and import messages (`smp-server database import --table messages`)"
|
||||
Right (ASType SQSMemory SMSMemory) -> setToDbStr <> "\nstore_messages set to `memory`, import messages to journal to use PostgreSQL database for queues (`smp-server journal import`)"
|
||||
Right (ASType SQSMemory SMSJournal) -> setToDbStr
|
||||
Right (ASType SQSPostgres _) -> "store_queues set to `database`, start the server."
|
||||
Left e -> e <> ", configure storage correctly"
|
||||
where
|
||||
@@ -201,7 +280,7 @@ smpServerCLI_ generateSite serveStaticFiles attachStaticFiles cfgPath logPath =
|
||||
confirmOrExit
|
||||
("WARNING: PostrgreSQL schema " <> B.unpack schema <> " (database: " <> B.unpack connstr <> ") will be exported to store log file " <> storeLogFilePath <> " and to message log file " <> storeMsgsFilePath)
|
||||
"Database store not exported"
|
||||
(sCnt, qCnt) <- exportDatabaseToStoreLog dbOpts storeLogFilePath
|
||||
(sCnt, qCnt) <- exportDatabaseToStoreLog logPath dbOpts storeLogFilePath
|
||||
putStrLn $ "Exported: " <> show sCnt <> " services, " <> show qCnt <> " queues"
|
||||
putStrLn "Exporting messages..."
|
||||
let storeCfg = PostgresStoreCfg {dbOpts, dbStoreLogPath = Nothing, confirmMigrations = MCConsole, deletedTTL = 86400 * defaultDeletedTTL}
|
||||
@@ -227,7 +306,7 @@ smpServerCLI_ generateSite serveStaticFiles attachStaticFiles cfgPath logPath =
|
||||
confirmOrExit
|
||||
("WARNING: PostrgreSQL schema " <> B.unpack schema <> " (database: " <> B.unpack connstr <> ") will be exported to store log file " <> storeLogFilePath)
|
||||
"Queue records not exported"
|
||||
(sCnt, qCnt) <- exportDatabaseToStoreLog dbOpts storeLogFilePath
|
||||
(sCnt, qCnt) <- exportDatabaseToStoreLog logPath dbOpts storeLogFilePath
|
||||
putStrLn $ "Export completed: " <> show sCnt <> " services, " <> show qCnt <> " queues"
|
||||
putStrLn $ case readStoreType ini of
|
||||
Right (ASType SQSPostgres _) -> "store_queues or store_messages set to `database`, update it to `memory` in INI file."
|
||||
@@ -267,7 +346,6 @@ smpServerCLI_ generateSite serveStaticFiles attachStaticFiles cfgPath logPath =
|
||||
doesFileExist iniFile >>= \case
|
||||
True -> readIniFile iniFile >>= either exitError a
|
||||
_ -> exitError $ "Error: server is not initialized (" <> iniFile <> " does not exist).\nRun `" <> executableName <> " init`."
|
||||
#if defined(dbServerPostgres)
|
||||
getRequiredStoreLogFile ini = do
|
||||
case enableStoreLog' ini $> storeLogFilePath of
|
||||
Just storeLogFile -> do
|
||||
@@ -276,21 +354,20 @@ smpServerCLI_ generateSite serveStaticFiles attachStaticFiles cfgPath logPath =
|
||||
(pure storeLogFile)
|
||||
(putStrLn ("Store log file " <> storeLogFile <> " not found") >> exitFailure)
|
||||
Nothing -> putStrLn "Store log disabled, see `[STORE_LOG] enable`" >> exitFailure
|
||||
#endif
|
||||
iniFile = combine cfgPath "smp-server.ini"
|
||||
serverVersion = "SMP server v" <> simplexmqVersionCommit
|
||||
executableName = "smp-server"
|
||||
storeLogFilePath = combine logPath "smp-server-store.log"
|
||||
storeMsgsFilePath = combine logPath "smp-server-messages.log"
|
||||
storeMsgsJournalDir = combine logPath "messages"
|
||||
storeMsgsJournalDir = storeMsgsJournalDir' logPath
|
||||
storeNtfsFilePath = combine logPath "smp-server-ntfs.log"
|
||||
readStoreType :: Ini -> Either String AStoreType
|
||||
readStoreType ini = case (iniStoreQueues, iniStoreMessage) of
|
||||
("memory", "memory") -> Right $ ASType SQSMemory SMSMemory
|
||||
("memory", "journal") -> Left journalRemovedStr
|
||||
("memory", "journal") -> Right $ ASType SQSMemory SMSJournal
|
||||
("memory", "database") -> Left "Database and memory storage are not compatible."
|
||||
("database", "memory") -> Left "Database and memory storage are not compatible."
|
||||
("database", "journal") -> Left journalRemovedStr
|
||||
("database", "journal") -> Right $ ASType SQSPostgres SMSJournal
|
||||
#if defined(dbServerPostgres)
|
||||
("database", "database") -> Right $ ASType SQSPostgres SMSPostgres
|
||||
#else
|
||||
@@ -300,12 +377,10 @@ smpServerCLI_ generateSite serveStaticFiles attachStaticFiles cfgPath logPath =
|
||||
where
|
||||
iniStoreQueues = fromRight "memory" $ lookupValue "STORE_LOG" "store_queues" ini
|
||||
iniStoreMessage = fromRight "memory" $ lookupValue "STORE_LOG" "store_messages" ini
|
||||
#if defined(dbServerPostgres)
|
||||
iniDeletedTTL ini = readIniDefault (86400 * defaultDeletedTTL) "STORE_LOG" "db_deleted_ttl" ini
|
||||
enableDbStoreLog' = settingIsOn "STORE_LOG" "db_store_log"
|
||||
#endif
|
||||
defaultStaticPath = combine logPath "www"
|
||||
enableStoreLog' = settingIsOn "STORE_LOG" "enable"
|
||||
enableDbStoreLog' = settingIsOn "STORE_LOG" "db_store_log"
|
||||
initializeServer opts
|
||||
| scripted opts = initialize opts
|
||||
| otherwise = do
|
||||
@@ -442,13 +517,16 @@ smpServerCLI_ generateSite serveStaticFiles attachStaticFiles cfgPath logPath =
|
||||
iniStoreType = either error id $! readStoreType ini
|
||||
iniStoreCfg :: SupportedStore qs ms => SQSType qs -> SMSType ms -> ServerStoreCfg (MsgStoreType qs ms)
|
||||
iniStoreCfg SQSMemory SMSMemory = SSCMemory $ enableStoreLog' ini $> StorePaths {storeLogFile = storeLogFilePath, storeMsgsFile = restoreMessagesFile storeMsgsFilePath}
|
||||
iniStoreCfg SQSMemory SMSJournal = SSCMemoryJournal {storeLogFile = storeLogFilePath, storeMsgsPath = storeMsgsJournalDir}
|
||||
iniStoreCfg SQSPostgres SMSJournal =
|
||||
let dbStoreLogPath = enableDbStoreLog' ini $> storeLogFilePath
|
||||
storeCfg = PostgresStoreCfg {dbOpts = iniDBOptions ini defaultDBOpts, dbStoreLogPath, confirmMigrations = MCYesUp, deletedTTL = iniDeletedTTL ini}
|
||||
in SSCDatabaseJournal {storeCfg, storeMsgsPath' = storeMsgsJournalDir}
|
||||
#if defined(dbServerPostgres)
|
||||
iniStoreCfg SQSPostgres SMSPostgres =
|
||||
let dbStoreLogPath = enableDbStoreLog' ini $> storeLogFilePath
|
||||
storeCfg = PostgresStoreCfg {dbOpts = iniDBOptions ini defaultDBOpts, dbStoreLogPath, confirmMigrations = MCYesUp, deletedTTL = iniDeletedTTL ini}
|
||||
in SSCDatabase storeCfg
|
||||
#else
|
||||
iniStoreCfg SQSPostgres _ = error noPostgresExitStr
|
||||
#endif
|
||||
serverConfig :: ServerStoreCfg s -> ServerConfig s
|
||||
serverConfig serverStoreCfg =
|
||||
@@ -457,6 +535,8 @@ smpServerCLI_ generateSite serveStaticFiles attachStaticFiles cfgPath logPath =
|
||||
smpHandshakeTimeout = 120000000,
|
||||
tbqSize = 128,
|
||||
msgQueueQuota = defaultMsgQueueQuota,
|
||||
maxJournalMsgCount = defaultMaxJournalMsgCount,
|
||||
maxJournalStateLines = defaultMaxJournalStateLines,
|
||||
queueIdBytes = 24,
|
||||
msgIdBytes = 24, -- must be at least 24 bytes, it is used as 192-bit nonce for XSalsa20
|
||||
smpCredentials =
|
||||
@@ -481,6 +561,7 @@ smpServerCLI_ generateSite serveStaticFiles attachStaticFiles cfgPath logPath =
|
||||
},
|
||||
expireMessagesOnStart = fromMaybe True $ iniOnOff "STORE_LOG" "expire_messages_on_start" ini,
|
||||
expireMessagesOnSend = fromMaybe True $ iniOnOff "STORE_LOG" "expire_messages_on_send" ini,
|
||||
idleQueueInterval = defaultIdleQueueInterval,
|
||||
notificationExpiration =
|
||||
defaultNtfExpiration
|
||||
{ ttl = 3600 * readIniDefault defNtfExpirationHours "STORE_LOG" "expire_ntfs_hours" ini
|
||||
@@ -553,26 +634,47 @@ smpServerCLI_ generateSite serveStaticFiles attachStaticFiles cfgPath logPath =
|
||||
webStaticPath' = eitherToMaybe $ T.unpack <$> lookupValue "WEB" "static_path" ini
|
||||
|
||||
checkMsgStoreMode :: Ini -> AStoreType -> IO ()
|
||||
#if defined(dbServerPostgres)
|
||||
checkMsgStoreMode ini mode = do
|
||||
#else
|
||||
checkMsgStoreMode _ mode = do
|
||||
#endif
|
||||
whenM (doesDirectoryExist storeMsgsJournalDir) $ do
|
||||
putStrLn $ "Error: " <> storeMsgsJournalDir <> " directory is present.\n" <> journalRemovedStr
|
||||
exitFailure
|
||||
msgsDirExists <- doesDirectoryExist storeMsgsJournalDir
|
||||
msgsFileExists <- doesFileExist storeMsgsFilePath
|
||||
storeLogExists <- doesFileExist storeLogFilePath
|
||||
case mode of
|
||||
#if defined(dbServerPostgres)
|
||||
ASType SQSPostgres SMSPostgres -> do
|
||||
whenM (doesFileExist storeMsgsFilePath) $ do
|
||||
putStrLn $ "Error: " <> storeMsgsFilePath <> " file is present."
|
||||
putStrLn "Configure memory storage."
|
||||
exitFailure
|
||||
checkDbStorage ini =<< doesFileExist storeLogFilePath
|
||||
#else
|
||||
ASType SQSPostgres _ -> noPostgresExit
|
||||
ASType SQSPostgres SMSPostgres
|
||||
| msgsFileExists || msgsDirExists -> do
|
||||
putStrLn $ "Error: " <> storeMsgsFilePath <> " file or " <> storeMsgsJournalDir <> " directory are present."
|
||||
putStrLn "Configure memory storage."
|
||||
exitFailure
|
||||
| otherwise -> checkDbStorage ini storeLogExists
|
||||
#endif
|
||||
ASType SQSMemory SMSMemory -> pure ()
|
||||
ASType qs SMSJournal
|
||||
| msgsFileExists && msgsDirExists -> exitConfigureMsgStorage
|
||||
| msgsFileExists -> do
|
||||
putStrLn $ "Error: store_messages is `journal` with " <> storeMsgsFilePath <> " file present."
|
||||
putStrLn "Set store_messages to `memory` or use `smp-server journal export` to migrate."
|
||||
exitFailure
|
||||
| not msgsDirExists ->
|
||||
putStrLn $ "store_messages is `journal`, " <> storeMsgsJournalDir <> " directory will be created."
|
||||
| otherwise -> case qs of
|
||||
SQSMemory ->
|
||||
unless (storeLogExists) $ putStrLn $ "store_queues is `memory`, " <> storeLogFilePath <> " file will be created."
|
||||
#if defined(dbServerPostgres)
|
||||
SQSPostgres -> checkDbStorage ini storeLogExists
|
||||
#else
|
||||
SQSPostgres -> noPostgresExit
|
||||
#endif
|
||||
ASType SQSMemory SMSMemory
|
||||
| msgsFileExists && msgsDirExists -> exitConfigureMsgStorage
|
||||
| msgsDirExists -> do
|
||||
putStrLn $ "Error: store_messages is `memory` with " <> storeMsgsJournalDir <> " directory present."
|
||||
putStrLn "Set store_messages to `journal` or use `smp-server journal import` to migrate."
|
||||
exitFailure
|
||||
| otherwise -> pure ()
|
||||
|
||||
exitConfigureMsgStorage = do
|
||||
putStrLn $ "Error: both " <> storeMsgsFilePath <> " file and " <> storeMsgsJournalDir <> " directory are present."
|
||||
putStrLn "Configure memory storage."
|
||||
exitFailure
|
||||
|
||||
#if defined(dbServerPostgres)
|
||||
checkDbStorage ini storeLogExists = do
|
||||
@@ -603,18 +705,18 @@ smpServerCLI_ generateSite serveStaticFiles attachStaticFiles cfgPath logPath =
|
||||
putStrLn "Configure queue storage."
|
||||
exitFailure
|
||||
|
||||
importStoreLogToDatabase :: FilePath -> DBOpts -> IO (Int64, Int64)
|
||||
importStoreLogToDatabase storeLogFile dbOpts = do
|
||||
ms <- newMsgStore STMStoreConfig {storePath = Nothing, quota = defaultMsgQueueQuota}
|
||||
let st = queueStore ms
|
||||
sl <- readWriteQueueStore True (mkQueue ms) storeLogFile st
|
||||
importStoreLogToDatabase :: FilePath -> FilePath -> DBOpts -> IO (Int64, Int64)
|
||||
importStoreLogToDatabase logPath storeLogFile dbOpts = do
|
||||
ms <- newJournalMsgStore logPath MQStoreCfg
|
||||
let st = stmQueueStore ms
|
||||
sl <- readWriteQueueStore True (mkQueue ms False) storeLogFile st
|
||||
closeStoreLog sl
|
||||
qs <- readTVarIO $ queues st
|
||||
queues <- readTVarIO $ loadedQueues st
|
||||
services' <- M.elems <$> readTVarIO (services st)
|
||||
let storeCfg = PostgresStoreCfg {dbOpts = dbOpts {createSchema = True}, dbStoreLogPath = Nothing, confirmMigrations = MCConsole, deletedTTL = 86400 * defaultDeletedTTL}
|
||||
ps :: PostgresQueueStore PostgresQueue <- newQueueStore @PostgresQueue storeCfg
|
||||
sCnt <- batchInsertServices services' ps
|
||||
qCnt <- batchInsertQueues True qs ps
|
||||
ps <- newJournalMsgStore logPath $ PQStoreCfg storeCfg
|
||||
sCnt <- batchInsertServices services' $ postgresQueueStore ps
|
||||
qCnt <- batchInsertQueues @(JournalQueue 'QSMemory) True queues $ postgresQueueStore ps
|
||||
renameFile storeLogFile $ storeLogFile <> ".bak"
|
||||
pure (sCnt, qCnt)
|
||||
|
||||
@@ -633,22 +735,24 @@ importMessagesToDatabase msgsLogFile dbOpts = do
|
||||
renameFile msgsLogFile $ msgsLogFile <> ".bak"
|
||||
pure mCnt'
|
||||
|
||||
exportDatabaseToStoreLog :: DBOpts -> FilePath -> IO (Int, Int)
|
||||
exportDatabaseToStoreLog dbOpts storeLogFilePath = do
|
||||
exportDatabaseToStoreLog :: FilePath -> DBOpts -> FilePath -> IO (Int, Int)
|
||||
exportDatabaseToStoreLog logPath dbOpts storeLogFilePath = do
|
||||
let storeCfg = PostgresStoreCfg {dbOpts, dbStoreLogPath = Nothing, confirmMigrations = MCConsole, deletedTTL = 86400 * defaultDeletedTTL}
|
||||
ps :: PostgresQueueStore PostgresQueue <- newQueueStore @PostgresQueue storeCfg
|
||||
ps <- newJournalMsgStore logPath $ PQStoreCfg storeCfg
|
||||
sl <- openWriteStoreLog False storeLogFilePath
|
||||
Sum sCnt <- foldServiceRecs ps $ \sr -> logNewService sl sr $> Sum (1 :: Int)
|
||||
Sum qCnt <- foldQueueRecs True True ps $ \(rId, qr) -> logCreateQueue sl rId qr $> Sum (1 :: Int)
|
||||
Sum sCnt <- foldServiceRecs (postgresQueueStore ps) $ \sr -> logNewService sl sr $> Sum (1 :: Int)
|
||||
Sum qCnt <- foldQueueRecs True True (postgresQueueStore ps) $ \(rId, qr) -> logCreateQueue sl rId qr $> Sum (1 :: Int)
|
||||
closeStoreLog sl
|
||||
pure (sCnt, qCnt)
|
||||
#endif
|
||||
|
||||
journalRemovedStr :: String
|
||||
journalRemovedStr =
|
||||
"Journal message storage is removed.\n"
|
||||
<> "Export messages with `smp-server journal export` using smp-server v7.1.0.10 or earlier,\n"
|
||||
<> "then set store_messages to `memory`, or to `database` and run `smp-server database import --table messages`."
|
||||
newJournalMsgStore :: FilePath -> QStoreCfg s -> IO (JournalMsgStore s)
|
||||
newJournalMsgStore logPath qsCfg =
|
||||
let cfg = mkJournalStoreConfig qsCfg (storeMsgsJournalDir' logPath) defaultMsgQueueQuota defaultMaxJournalMsgCount defaultMaxJournalStateLines $ checkInterval defaultMessageExpiration
|
||||
in newMsgStore cfg
|
||||
|
||||
storeMsgsJournalDir' :: FilePath -> FilePath
|
||||
storeMsgsJournalDir' logPath = combine logPath "messages"
|
||||
|
||||
getServerSourceCode :: IO (Maybe String)
|
||||
getServerSourceCode =
|
||||
@@ -769,6 +873,7 @@ data CliCommand
|
||||
| OnlineCert CertOptions
|
||||
| Start StartOptions
|
||||
| Delete
|
||||
| Journal StoreCmd
|
||||
| Database StoreCmd DatabaseTable DBOpts
|
||||
|
||||
data StoreCmd = SCImport | SCExport | SCDelete
|
||||
@@ -794,6 +899,7 @@ cliCommandP cfgPath logPath iniFile =
|
||||
<> command "cert" (info (OnlineCert <$> certOptionsP) (progDesc $ "Generate new online TLS server credentials (configuration: " <> iniFile <> ")"))
|
||||
<> command "start" (info (Start <$> startOptionsP) (progDesc $ "Start server (configuration: " <> iniFile <> ")"))
|
||||
<> command "delete" (info (pure Delete) (progDesc "Delete configuration and log files"))
|
||||
<> command "journal" (info (Journal <$> journalCmdP) (progDesc "Import/export messages to/from journal storage"))
|
||||
<> command "database" (info (Database <$> databaseCmdP <*> dbTableP <*> dbOptsP defaultDBOpts) (progDesc "Import/export queues to/from PostgreSQL database storage"))
|
||||
)
|
||||
where
|
||||
@@ -940,6 +1046,7 @@ cliCommandP cfgPath logPath iniFile =
|
||||
disableWeb,
|
||||
scripted
|
||||
}
|
||||
journalCmdP = storeCmdP "message log file" "journal storage"
|
||||
databaseCmdP = storeCmdP "queue store log file" "PostgreSQL database schema"
|
||||
storeCmdP src dest =
|
||||
hsubparser
|
||||
|
||||
@@ -83,7 +83,7 @@ iniFileContent cfgPath logPath opts host basicAuth controlPortPwds =
|
||||
<> ("enable = " <> onOff enableStoreLog <> "\n\n")
|
||||
<> "# Queue storage mode: `memory` or `database` (to store queue records in PostgreSQL database).\n\
|
||||
\# `memory` - in-memory persistence, with optional append-only log (`enable = on`).\n\
|
||||
\# `database`- PostgreSQL databass (requires `store_messages = database`).\n\
|
||||
\# `database`- PostgreSQL databass (requires `store_messages = journal`).\n\
|
||||
\store_queues = memory\n\n\
|
||||
\# Database connection settings for PostgreSQL database (`store_queues = database`).\n"
|
||||
<> iniDbOpts dbOptions defaultDBOpts
|
||||
@@ -91,7 +91,7 @@ iniFileContent cfgPath logPath opts host basicAuth controlPortPwds =
|
||||
\# db_store_log = off\n\n\
|
||||
\# Time to retain deleted queues in the database, days.\n"
|
||||
<> ("# db_deleted_ttl = " <> tshow defaultDeletedTTL <> "\n\n")
|
||||
<> "# Message storage mode: `memory` or `database` (requires `store_queues = database`).\n\
|
||||
<> "# Message storage mode: `memory` or `journal`.\n\
|
||||
\store_messages = memory\n\n\
|
||||
\# When store_messages is `memory`, undelivered messages are optionally saved and restored\n\
|
||||
\# when the server restarts, they are preserved in the .bak file until the next restart.\n"
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,44 @@
|
||||
module Simplex.Messaging.Server.MsgStore.Journal.SharedLock
|
||||
( withLockWaitShared,
|
||||
withLockMapWaitShared,
|
||||
withSharedWaitLock,
|
||||
)
|
||||
where
|
||||
|
||||
import Control.Concurrent.STM
|
||||
import qualified Control.Exception as E
|
||||
import Control.Monad
|
||||
import Data.Text (Text)
|
||||
import Simplex.Messaging.Agent.Lock
|
||||
import Simplex.Messaging.Agent.Client (getMapLock)
|
||||
import Simplex.Messaging.Protocol (RecipientId)
|
||||
import Simplex.Messaging.TMap (TMap)
|
||||
import qualified Simplex.Messaging.TMap as TM
|
||||
import Simplex.Messaging.Util (($>>), ($>>=))
|
||||
|
||||
-- wait until shared lock with passed ID is released and take lock
|
||||
withLockWaitShared :: RecipientId -> Lock -> TMVar RecipientId -> Text -> IO a -> IO a
|
||||
withLockWaitShared rId lock shared name =
|
||||
E.bracket_
|
||||
(atomically $ waitShared rId shared >> putTMVar lock name)
|
||||
(void $ atomically $ takeTMVar lock)
|
||||
|
||||
-- wait until shared lock with passed ID is released and take lock from Map for this ID
|
||||
withLockMapWaitShared :: RecipientId -> TMap RecipientId Lock -> TMVar RecipientId -> Text -> IO a -> IO a
|
||||
withLockMapWaitShared rId locks shared name a =
|
||||
E.bracket
|
||||
(atomically $ waitShared rId shared >> getPutLock (getMapLock locks) rId name)
|
||||
(atomically . takeTMVar)
|
||||
(const a)
|
||||
|
||||
waitShared :: RecipientId -> TMVar RecipientId -> STM ()
|
||||
waitShared rId shared = tryReadTMVar shared >>= mapM_ (\rId' -> when (rId == rId') retry)
|
||||
|
||||
-- wait until lock with passed ID in Map is released and take shared lock for this ID
|
||||
withSharedWaitLock :: RecipientId -> TMap RecipientId Lock -> TMVar RecipientId -> IO a -> IO a
|
||||
withSharedWaitLock rId locks shared =
|
||||
E.bracket_
|
||||
(atomically $ waitLock >> putTMVar shared rId)
|
||||
(atomically $ takeTMVar shared)
|
||||
where
|
||||
waitLock = TM.lookup rId locks $>>= tryReadTMVar $>> retry
|
||||
@@ -29,6 +29,7 @@ where
|
||||
import Control.Concurrent.STM
|
||||
import qualified Control.Exception as E
|
||||
import Control.Monad
|
||||
import Control.Monad.Reader
|
||||
import Control.Monad.Trans.Except
|
||||
import qualified Data.ByteString as B
|
||||
import qualified Data.ByteString.Builder as BB
|
||||
@@ -38,6 +39,7 @@ import Data.IORef
|
||||
import Data.Int (Int64)
|
||||
import Data.List (intersperse)
|
||||
import qualified Data.Map.Strict as M
|
||||
import Data.Text (Text)
|
||||
import Data.Time.Clock.System (SystemTime (..))
|
||||
import Database.PostgreSQL.Simple (Binary (..), In (..), Only (..), (:.) (..))
|
||||
import qualified Database.PostgreSQL.Simple as DB
|
||||
@@ -54,7 +56,7 @@ import Simplex.Messaging.Server.QueueStore.Postgres
|
||||
import Simplex.Messaging.Server.QueueStore.Types
|
||||
import Simplex.Messaging.Server.StoreLog (foldLogLines)
|
||||
import Simplex.Messaging.Encoding.String
|
||||
import Simplex.Messaging.Util (maybeFirstRow, maybeFirstRow')
|
||||
import Simplex.Messaging.Util (maybeFirstRow, maybeFirstRow', (<$$>))
|
||||
import System.IO (Handle, hFlush, stdout)
|
||||
|
||||
data PostgresMsgStore = PostgresMsgStore
|
||||
@@ -79,30 +81,41 @@ instance StoreQueueClass PostgresQueue where
|
||||
{-# INLINE recipientId #-}
|
||||
queueRec = queueRec'
|
||||
{-# INLINE queueRec #-}
|
||||
withQueueLock PostgresQueue {} _ = id -- TODO [messages] maybe it's just transaction?
|
||||
{-# INLINE withQueueLock #-}
|
||||
|
||||
newtype DBTransaction = DBTransaction {dbConn :: DB.Connection}
|
||||
|
||||
type DBStoreIO a = ReaderT DBTransaction IO a
|
||||
|
||||
instance MsgStoreClass PostgresMsgStore where
|
||||
type StoreMonad PostgresMsgStore = ReaderT DBTransaction IO
|
||||
type MsgQueue PostgresMsgStore = ()
|
||||
type QueueStore PostgresMsgStore = PostgresQueueStore'
|
||||
type StoreQueue PostgresMsgStore = PostgresQueue
|
||||
type MsgStoreConfig PostgresMsgStore = PostgresMsgStoreCfg
|
||||
|
||||
newMsgStore :: PostgresMsgStoreCfg -> IO PostgresMsgStore
|
||||
newMsgStore config = do
|
||||
queueStore_ <- newQueueStore @PostgresQueue (queueStoreCfg config)
|
||||
queueStore_ <- newQueueStore @PostgresQueue (queueStoreCfg config, False)
|
||||
pure PostgresMsgStore {config, queueStore_}
|
||||
|
||||
closeMsgStore :: PostgresMsgStore -> IO ()
|
||||
closeMsgStore = closeQueueStore @PostgresQueue . queueStore_
|
||||
|
||||
withActiveMsgQueues _ _ = error "withActiveMsgQueues not used"
|
||||
|
||||
unsafeWithAllMsgQueues _ _ _ = error "unsafeWithAllMsgQueues not used"
|
||||
|
||||
expireOldMessages :: Bool -> PostgresMsgStore -> Int64 -> Int64 -> IO MessageStats
|
||||
expireOldMessages _tty ms now ttl =
|
||||
maybeFirstRow' newMessageStats toMessageStats $ withConnection st $ \db ->
|
||||
DB.query db "CALL expire_old_messages(?,?,0,0,0)" (oldMsg, batchSize)
|
||||
DB.query db "CALL expire_old_messages(?,?,?,0,0,0)" (oldQueue, oldMsg, batchSize)
|
||||
where
|
||||
st = dbStore $ queueStore_ ms
|
||||
oldQueue = 0 :: Int64 -- expire all queues
|
||||
oldMsg = now - ttl
|
||||
-- expired messages read per page in expire_old_messages, and the page is one
|
||||
-- transaction: queues in it stay row-locked against SEND and ACK until it commits.
|
||||
batchSize = 100 :: Int
|
||||
batchSize = 10000 :: Int
|
||||
toMessageStats (expiredMsgsCount, storedMsgsCount, storedQueues) =
|
||||
MessageStats {expiredMsgsCount, storedMsgsCount, storedQueues}
|
||||
|
||||
@@ -136,13 +149,35 @@ instance MsgStoreClass PostgresMsgStore where
|
||||
msg_ = toMaybeMessage mRow
|
||||
in f a rId $ Right ((qr,) <$> msg_)
|
||||
|
||||
logQueueStates _ = error "logQueueStates not used"
|
||||
|
||||
logQueueState _ = error "logQueueState not used"
|
||||
|
||||
queueStore = queueStore_
|
||||
{-# INLINE queueStore #-}
|
||||
|
||||
mkQueue :: PostgresMsgStore -> RecipientId -> QueueRec -> IO PostgresQueue
|
||||
mkQueue _ rId qr = PostgresQueue rId <$> newTVarIO (Just qr)
|
||||
loadedQueueCounts :: PostgresMsgStore -> IO LoadedQueueCounts
|
||||
loadedQueueCounts ms = do
|
||||
loadedQueueCount <- M.size <$> readTVarIO queues
|
||||
loadedNotifierCount <- M.size <$> readTVarIO notifiers
|
||||
notifierLockCount <- M.size <$> readTVarIO notifierLocks
|
||||
pure LoadedQueueCounts {loadedQueueCount, loadedNotifierCount, openJournalCount = 0, queueLockCount = 0, notifierLockCount}
|
||||
where
|
||||
PostgresQueueStore {queues, notifiers, notifierLocks} = queueStore_ ms
|
||||
|
||||
mkQueue :: PostgresMsgStore -> Bool -> RecipientId -> QueueRec -> IO PostgresQueue
|
||||
mkQueue _ _keepLock rId qr = PostgresQueue rId <$> newTVarIO (Just qr)
|
||||
{-# INLINE mkQueue #-}
|
||||
|
||||
getMsgQueue _ _ _ = pure ()
|
||||
{-# INLINE getMsgQueue #-}
|
||||
|
||||
getPeekMsgQueue :: PostgresMsgStore -> PostgresQueue -> DBStoreIO (Maybe ((), Message))
|
||||
getPeekMsgQueue _ q = ((),) <$$> tryPeekMsg_ q ()
|
||||
|
||||
withIdleMsgQueue :: Int64 -> PostgresMsgStore -> PostgresQueue -> (() -> DBStoreIO a) -> DBStoreIO (Maybe a, Int)
|
||||
withIdleMsgQueue _ _ _ _ = error "withIdleMsgQueue not used"
|
||||
|
||||
deleteQueue :: PostgresMsgStore -> PostgresQueue -> IO (Either ErrorType QueueRec)
|
||||
deleteQueue ms q = deleteStoreQueue (queueStore_ ms) q
|
||||
{-# INLINE deleteQueue #-}
|
||||
@@ -153,6 +188,8 @@ instance MsgStoreClass PostgresMsgStore where
|
||||
qr <- ExceptT $ deleteStoreQueue (queueStore_ ms) q
|
||||
pure (qr, size)
|
||||
|
||||
getQueueMessages_ _ _ _ = error "getQueueMessages_ not used"
|
||||
|
||||
writeMsg :: PostgresMsgStore -> PostgresQueue -> Bool -> Message -> ExceptT ErrorType IO (Maybe (Message, Bool))
|
||||
writeMsg ms q _ msg =
|
||||
uninterruptibleMask_ $
|
||||
@@ -171,26 +208,43 @@ instance MsgStoreClass PostgresMsgStore where
|
||||
[] -> Nothing
|
||||
PostgresMsgStore {config = PostgresMsgStoreCfg {quota}} = ms
|
||||
|
||||
setOverQuota_ :: PostgresQueue -> IO () -- can ONLY be used while restoring messages, not while server running
|
||||
setOverQuota_ _ = error "TODO setOverQuota_" -- TODO [messages]
|
||||
|
||||
getQueueSize_ :: () -> DBStoreIO Int
|
||||
getQueueSize_ _ = error "getQueueSize_ not used"
|
||||
|
||||
getQueueSize :: PostgresMsgStore -> PostgresQueue -> ExceptT ErrorType IO Int
|
||||
getQueueSize ms q =
|
||||
withDB' "getQueueSize" (queueStore_ ms) $ \db ->
|
||||
maybeFirstRow' 0 fromOnly $
|
||||
DB.query db "SELECT msg_queue_size FROM msg_queues WHERE recipient_id = ? AND deleted_at IS NULL" (Only (recipientId' q))
|
||||
|
||||
tryPeekMsg_ :: PostgresQueue -> () -> DBStoreIO (Maybe Message)
|
||||
tryPeekMsg_ q _ = do
|
||||
db <- asks dbConn
|
||||
liftIO $ maybeFirstRow toMessage $
|
||||
DB.query
|
||||
db
|
||||
[sql|
|
||||
SELECT msg_id, msg_ts, msg_quota, msg_ntf_flag, msg_body
|
||||
FROM messages
|
||||
WHERE recipient_id = ?
|
||||
ORDER BY message_id ASC LIMIT 1
|
||||
|]
|
||||
(Only (recipientId' q))
|
||||
|
||||
tryDeleteMsg_ :: PostgresQueue -> () -> Bool -> DBStoreIO ()
|
||||
tryDeleteMsg_ _q _ _ = error "tryDeleteMsg_ not used" -- do
|
||||
|
||||
isolateQueue :: PostgresMsgStore -> PostgresQueue -> Text -> DBStoreIO a -> ExceptT ErrorType IO a
|
||||
isolateQueue ms _q op a = uninterruptibleMask_ $ withDB' op (queueStore_ ms) $ runReaderT a . DBTransaction
|
||||
|
||||
unsafeRunStore _ _ _ = error "unsafeRunStore not used"
|
||||
|
||||
tryPeekMsg :: PostgresMsgStore -> PostgresQueue -> ExceptT ErrorType IO (Maybe Message)
|
||||
tryPeekMsg ms q =
|
||||
uninterruptibleMask_ $
|
||||
withDB' "tryPeekMsg" (queueStore_ ms) $ \db ->
|
||||
maybeFirstRow toMessage $
|
||||
DB.query
|
||||
db
|
||||
[sql|
|
||||
SELECT msg_id, msg_ts, msg_quota, msg_ntf_flag, msg_body
|
||||
FROM messages
|
||||
WHERE recipient_id = ?
|
||||
ORDER BY message_id ASC LIMIT 1
|
||||
|]
|
||||
(Only (recipientId' q))
|
||||
tryPeekMsg ms q = isolateQueue ms q "tryPeekMsg" $ tryPeekMsg_ q ()
|
||||
{-# INLINE tryPeekMsg #-}
|
||||
|
||||
tryPeekMsgs :: PostgresMsgStore -> [PostgresQueue] -> ExceptT ErrorType IO (M.Map RecipientId Message)
|
||||
tryPeekMsgs _ms [] = pure M.empty
|
||||
@@ -310,8 +364,8 @@ deleteAllMessages ms =
|
||||
db
|
||||
[sql|
|
||||
UPDATE msg_queues
|
||||
SET msg_queue_size = 0, msg_can_write = TRUE
|
||||
WHERE msg_queue_size != 0 OR msg_can_write = FALSE
|
||||
SET msg_queue_size = 0, msg_can_write = TRUE, msg_queue_expire = FALSE
|
||||
WHERE msg_queue_size != 0 OR msg_can_write = FALSE OR msg_queue_expire = TRUE
|
||||
|]
|
||||
|
||||
updateQueueCounts :: PostgresMsgStore -> IO ()
|
||||
@@ -331,15 +385,16 @@ updateQueueCounts ms =
|
||||
db
|
||||
[sql|
|
||||
UPDATE msg_queues
|
||||
SET msg_queue_size = 0, msg_can_write = TRUE
|
||||
WHERE msg_queue_size != 0 OR msg_can_write = FALSE
|
||||
SET msg_queue_size = 0, msg_can_write = TRUE, msg_queue_expire = FALSE
|
||||
WHERE msg_queue_size != 0 OR msg_can_write = FALSE OR msg_queue_expire = TRUE
|
||||
|]
|
||||
void $ DB.execute_
|
||||
db
|
||||
[sql|
|
||||
UPDATE msg_queues q
|
||||
SET msg_queue_size = s.size,
|
||||
msg_can_write = s.quota_count = 0
|
||||
msg_can_write = s.quota_count = 0,
|
||||
msg_queue_expire = s.size > s.quota_count
|
||||
FROM queue_stats s
|
||||
WHERE q.recipient_id = s.recipient_id
|
||||
|]
|
||||
|
||||
@@ -15,10 +15,6 @@ module Simplex.Messaging.Server.MsgStore.STM
|
||||
( STMMsgStore (..),
|
||||
STMStoreConfig (..),
|
||||
STMQueue,
|
||||
loadedQueueCounts,
|
||||
getQueueMessages,
|
||||
setOverQuota_,
|
||||
deleteQuotaMsg,
|
||||
)
|
||||
where
|
||||
|
||||
@@ -28,8 +24,7 @@ import Control.Monad.Trans.Except
|
||||
import Data.Functor (($>))
|
||||
import Data.Int (Int64)
|
||||
import qualified Data.Map.Strict as M
|
||||
import Data.Maybe (catMaybes)
|
||||
import Data.Time.Clock.System (SystemTime (systemSeconds))
|
||||
import Data.Text (Text)
|
||||
import Simplex.Messaging.Protocol
|
||||
import Simplex.Messaging.Server.MsgStore.Types
|
||||
import Simplex.Messaging.Server.QueueStore
|
||||
@@ -66,8 +61,12 @@ instance StoreQueueClass STMQueue where
|
||||
{-# INLINE recipientId #-}
|
||||
queueRec = queueRec'
|
||||
{-# INLINE queueRec #-}
|
||||
withQueueLock _ _ = id
|
||||
{-# INLINE withQueueLock #-}
|
||||
|
||||
instance MsgStoreClass STMMsgStore where
|
||||
type StoreMonad STMMsgStore = STM
|
||||
type MsgQueue STMMsgStore = STMMsgQueue
|
||||
type QueueStore STMMsgStore = STMQueueStore STMQueue
|
||||
type StoreQueue STMMsgStore = STMQueue
|
||||
type MsgStoreConfig STMMsgStore = STMStoreConfig
|
||||
@@ -79,22 +78,59 @@ instance MsgStoreClass STMMsgStore where
|
||||
|
||||
closeMsgStore = closeQueueStore @STMQueue . queueStore_
|
||||
{-# INLINE closeMsgStore #-}
|
||||
withActiveMsgQueues = withLoadedQueues . queueStore_
|
||||
{-# INLINE withActiveMsgQueues #-}
|
||||
unsafeWithAllMsgQueues _ = withLoadedQueues . queueStore_
|
||||
{-# INLINE unsafeWithAllMsgQueues #-}
|
||||
|
||||
expireOldMessages :: Bool -> STMMsgStore -> Int64 -> Int64 -> IO MessageStats
|
||||
expireOldMessages _tty ms now ttl =
|
||||
withLoadedQueues (queueStore_ ms) $ atomically . expireQueueMsgs (now - ttl)
|
||||
withLoadedQueues (queueStore_ ms) $ atomically . expireQueueMsgs ms now (now - ttl)
|
||||
|
||||
foldRcvServiceMessages :: STMMsgStore -> ServiceId -> (a -> RecipientId -> Either ErrorType (Maybe (QueueRec, Message)) -> IO a) -> a -> IO (Either ErrorType a)
|
||||
foldRcvServiceMessages ms serviceId f = fmap Right . foldRcvServiceQueues (queueStore_ ms) serviceId f'
|
||||
where
|
||||
f' a (q, qr) = runExceptT (tryPeekMsg ms q) >>= f a (recipientId q) . ((qr,) <$$>)
|
||||
|
||||
logQueueStates _ = pure ()
|
||||
{-# INLINE logQueueStates #-}
|
||||
logQueueState _ = pure ()
|
||||
{-# INLINE logQueueState #-}
|
||||
queueStore = queueStore_
|
||||
{-# INLINE queueStore #-}
|
||||
|
||||
mkQueue _ rId qr = STMQueue rId <$> newTVarIO (Just qr) <*> newTVarIO Nothing
|
||||
loadedQueueCounts :: STMMsgStore -> IO LoadedQueueCounts
|
||||
loadedQueueCounts STMMsgStore {queueStore_ = st} = do
|
||||
loadedQueueCount <- M.size <$> readTVarIO (queues st)
|
||||
loadedNotifierCount <- M.size <$> readTVarIO (notifiers st)
|
||||
pure LoadedQueueCounts {loadedQueueCount, loadedNotifierCount, openJournalCount = 0, queueLockCount = 0, notifierLockCount = 0}
|
||||
|
||||
mkQueue _ _ rId qr = STMQueue rId <$> newTVarIO (Just qr) <*> newTVarIO Nothing
|
||||
{-# INLINE mkQueue #-}
|
||||
|
||||
getMsgQueue :: STMMsgStore -> STMQueue -> Bool -> STM STMMsgQueue
|
||||
getMsgQueue _ STMQueue {msgQueue'} _ = readTVar msgQueue' >>= maybe newQ pure
|
||||
where
|
||||
newQ = do
|
||||
msgTQueue <- newTQueue
|
||||
canWrite <- newTVar True
|
||||
size <- newTVar 0
|
||||
let q = STMMsgQueue {msgTQueue, canWrite, size}
|
||||
writeTVar msgQueue' (Just q)
|
||||
pure q
|
||||
|
||||
getPeekMsgQueue :: STMMsgStore -> STMQueue -> STM (Maybe (STMMsgQueue, Message))
|
||||
getPeekMsgQueue _ q@STMQueue {msgQueue'} = readTVar msgQueue' $>>= \mq -> (mq,) <$$> tryPeekMsg_ q mq
|
||||
|
||||
-- does not create queue if it does not exist, does not delete it if it does (can't just close in-memory queue)
|
||||
withIdleMsgQueue :: Int64 -> STMMsgStore -> STMQueue -> (STMMsgQueue -> STM a) -> STM (Maybe a, Int)
|
||||
withIdleMsgQueue _ _ STMQueue {msgQueue'} action = readTVar msgQueue' >>= \case
|
||||
Just q -> do
|
||||
r <- action q
|
||||
sz <- getQueueSize_ q
|
||||
pure (Just r, sz)
|
||||
Nothing -> pure (Nothing, 0)
|
||||
|
||||
deleteQueue :: STMMsgStore -> STMQueue -> IO (Either ErrorType QueueRec)
|
||||
deleteQueue ms q = fst <$$> deleteQueue_ ms q
|
||||
|
||||
@@ -104,9 +140,17 @@ instance MsgStoreClass STMMsgStore where
|
||||
where
|
||||
getSize = maybe (pure 0) (\STMMsgQueue {size} -> readTVarIO size)
|
||||
|
||||
getQueueMessages_ :: Bool -> STMQueue -> STMMsgQueue -> STM [Message]
|
||||
getQueueMessages_ drainMsgs _ = (if drainMsgs then flushTQueue else snapshotTQueue) . msgTQueue
|
||||
where
|
||||
snapshotTQueue q = do
|
||||
msgs <- flushTQueue q
|
||||
mapM_ (writeTQueue q) msgs
|
||||
pure msgs
|
||||
|
||||
writeMsg :: STMMsgStore -> STMQueue -> Bool -> Message -> ExceptT ErrorType IO (Maybe (Message, Bool))
|
||||
writeMsg ms q' _logState msg = liftIO $ atomically $ do
|
||||
STMMsgQueue {msgTQueue = q, canWrite, size} <- getMsgQueue q'
|
||||
STMMsgQueue {msgTQueue = q, canWrite, size} <- getMsgQueue ms q' True
|
||||
canWrt <- readTVar canWrite
|
||||
empty <- isEmptyTQueue q
|
||||
if canWrt || empty
|
||||
@@ -122,109 +166,29 @@ instance MsgStoreClass STMMsgStore where
|
||||
STMMsgStore {storeConfig = STMStoreConfig {quota}} = ms
|
||||
msgQuota = MessageQuota {msgId = messageId msg, msgTs = messageTs msg}
|
||||
|
||||
tryPeekMsg :: STMMsgStore -> STMQueue -> ExceptT ErrorType IO (Maybe Message)
|
||||
tryPeekMsg _ q = snd <$$> withPeekMsgQueue q pure
|
||||
{-# INLINE tryPeekMsg #-}
|
||||
setOverQuota_ :: STMQueue -> IO ()
|
||||
setOverQuota_ q = readTVarIO (msgQueue' q) >>= mapM_ (\mq -> atomically $ writeTVar (canWrite mq) False)
|
||||
|
||||
tryPeekMsgs :: STMMsgStore -> [STMQueue] -> ExceptT ErrorType IO (M.Map RecipientId Message)
|
||||
tryPeekMsgs st qs = M.fromList . catMaybes <$> mapM (\q -> (recipientId q,) <$$> tryPeekMsg st q) qs
|
||||
getQueueSize_ :: STMMsgQueue -> STM Int
|
||||
getQueueSize_ STMMsgQueue {size} = readTVar size
|
||||
|
||||
tryDelMsg :: STMMsgStore -> STMQueue -> MsgId -> ExceptT ErrorType IO (Maybe Message)
|
||||
tryDelMsg _ q msgId' =
|
||||
withPeekMsgQueue q $
|
||||
maybe (pure Nothing) $ \(mq, msg) ->
|
||||
if messageId msg == msgId'
|
||||
then tryDeleteMsg_ mq $> Just msg
|
||||
else pure Nothing
|
||||
tryPeekMsg_ :: STMQueue -> STMMsgQueue -> STM (Maybe Message)
|
||||
tryPeekMsg_ _ = tryPeekTQueue . msgTQueue
|
||||
{-# INLINE tryPeekMsg_ #-}
|
||||
|
||||
tryDelPeekMsg :: STMMsgStore -> STMQueue -> MsgId -> ExceptT ErrorType IO (Maybe Message, Maybe Message)
|
||||
tryDelPeekMsg _ q msgId' =
|
||||
withPeekMsgQueue q $
|
||||
maybe (pure (Nothing, Nothing)) $ \(mq, msg) ->
|
||||
if messageId msg == msgId'
|
||||
then (Just msg,) <$> (tryDeleteMsg_ mq >> tryPeekMsg_ mq)
|
||||
else pure (Nothing, Just msg)
|
||||
tryDeleteMsg_ :: STMQueue -> STMMsgQueue -> Bool -> STM ()
|
||||
tryDeleteMsg_ _ STMMsgQueue {msgTQueue = q, size} _logState =
|
||||
tryReadTQueue q >>= \case
|
||||
Just _ -> modifyTVar' size (subtract 1)
|
||||
_ -> pure ()
|
||||
|
||||
deleteExpiredMsgs :: STMMsgStore -> STMQueue -> Int64 -> ExceptT ErrorType IO Int
|
||||
deleteExpiredMsgs _ q old = liftIO $ atomically $ getMsgQueue q >>= deleteExpireMsgs_ old
|
||||
isolateQueue :: STMMsgStore -> STMQueue -> Text -> STM a -> ExceptT ErrorType IO a
|
||||
isolateQueue _ _ _ = liftIO . atomically
|
||||
{-# INLINE isolateQueue #-}
|
||||
|
||||
getQueueSize :: STMMsgStore -> STMQueue -> ExceptT ErrorType IO Int
|
||||
getQueueSize _ q = withPeekMsgQueue q $ maybe (pure 0) (getQueueSize_ . fst)
|
||||
{-# INLINE getQueueSize #-}
|
||||
|
||||
loadedQueueCounts :: STMMsgStore -> IO LoadedQueueCounts
|
||||
loadedQueueCounts STMMsgStore {queueStore_ = st} = do
|
||||
loadedQueueCount <- M.size <$> readTVarIO (queues st)
|
||||
loadedNotifierCount <- M.size <$> readTVarIO (notifiers st)
|
||||
pure LoadedQueueCounts {loadedQueueCount, loadedNotifierCount}
|
||||
|
||||
getMsgQueue :: STMQueue -> STM STMMsgQueue
|
||||
getMsgQueue STMQueue {msgQueue'} = readTVar msgQueue' >>= maybe newQ pure
|
||||
where
|
||||
newQ = do
|
||||
msgTQueue <- newTQueue
|
||||
canWrite <- newTVar True
|
||||
size <- newTVar 0
|
||||
let q = STMMsgQueue {msgTQueue, canWrite, size}
|
||||
writeTVar msgQueue' (Just q)
|
||||
pure q
|
||||
|
||||
-- The action is called with Nothing when it is known that the queue is empty
|
||||
withPeekMsgQueue :: STMQueue -> (Maybe (STMMsgQueue, Message) -> STM a) -> ExceptT ErrorType IO a
|
||||
withPeekMsgQueue STMQueue {msgQueue'} a =
|
||||
liftIO $ atomically $ (readTVar msgQueue' $>>= \mq -> (mq,) <$$> tryPeekMsg_ mq) >>= a
|
||||
|
||||
getQueueMessages :: Bool -> STMQueue -> IO [Message]
|
||||
getQueueMessages drainMsgs q = atomically $ (if drainMsgs then flushTQueue else snapshotTQueue) . msgTQueue =<< getMsgQueue q
|
||||
where
|
||||
snapshotTQueue mq = do
|
||||
msgs <- flushTQueue mq
|
||||
mapM_ (writeTQueue mq) msgs
|
||||
pure msgs
|
||||
|
||||
-- can ONLY be used while restoring messages, not while server running
|
||||
setOverQuota_ :: STMQueue -> IO ()
|
||||
setOverQuota_ q = readTVarIO (msgQueue' q) >>= mapM_ (\mq -> atomically $ writeTVar (canWrite mq) False)
|
||||
|
||||
-- if the first message in queue head is "quota", remove it
|
||||
deleteQuotaMsg :: STMQueue -> ExceptT ErrorType IO ()
|
||||
deleteQuotaMsg q =
|
||||
withPeekMsgQueue q $ \case
|
||||
Just (mq, MessageQuota {}) -> tryDeleteMsg_ mq
|
||||
_ -> pure ()
|
||||
|
||||
expireQueueMsgs :: Int64 -> STMQueue -> STM MessageStats
|
||||
expireQueueMsgs old STMQueue {msgQueue'} =
|
||||
readTVar msgQueue' >>= \case
|
||||
Just mq -> do
|
||||
expiredMsgsCount <- deleteExpireMsgs_ old mq
|
||||
storedMsgsCount <- getQueueSize_ mq
|
||||
pure MessageStats {storedMsgsCount, expiredMsgsCount, storedQueues = 1}
|
||||
-- does not create queue if it does not exist
|
||||
Nothing -> pure newMessageStats {storedQueues = 1}
|
||||
|
||||
deleteExpireMsgs_ :: Int64 -> STMMsgQueue -> STM Int
|
||||
deleteExpireMsgs_ old mq = loop 0
|
||||
where
|
||||
loop dc =
|
||||
tryPeekMsg_ mq >>= \case
|
||||
Just Message {msgTs}
|
||||
| systemSeconds msgTs < old ->
|
||||
tryDeleteMsg_ mq >> loop (dc + 1)
|
||||
_ -> pure dc
|
||||
|
||||
getQueueSize_ :: STMMsgQueue -> STM Int
|
||||
getQueueSize_ STMMsgQueue {size} = readTVar size
|
||||
|
||||
tryPeekMsg_ :: STMMsgQueue -> STM (Maybe Message)
|
||||
tryPeekMsg_ = tryPeekTQueue . msgTQueue
|
||||
{-# INLINE tryPeekMsg_ #-}
|
||||
|
||||
tryDeleteMsg_ :: STMMsgQueue -> STM ()
|
||||
tryDeleteMsg_ STMMsgQueue {msgTQueue = q, size} =
|
||||
tryReadTQueue q >>= \case
|
||||
Just _ -> modifyTVar' size (subtract 1)
|
||||
_ -> pure ()
|
||||
unsafeRunStore :: STMQueue -> Text -> STM a -> IO a
|
||||
unsafeRunStore _ _ = atomically
|
||||
{-# INLINE unsafeRunStore #-}
|
||||
|
||||
deleteQueue_ :: STMMsgStore -> STMQueue -> IO (Either ErrorType (QueueRec, Maybe STMMsgQueue))
|
||||
deleteQueue_ ms q = deleteStoreQueue (queueStore_ ms) q >>= mapM remove
|
||||
|
||||
@@ -4,12 +4,17 @@
|
||||
{-# LANGUAGE FlexibleContexts #-}
|
||||
{-# LANGUAGE GADTs #-}
|
||||
{-# LANGUAGE LambdaCase #-}
|
||||
{-# LANGUAGE MultiWayIf #-}
|
||||
{-# LANGUAGE NamedFieldPuns #-}
|
||||
{-# LANGUAGE OverloadedStrings #-}
|
||||
{-# LANGUAGE ScopedTypeVariables #-}
|
||||
{-# LANGUAGE TupleSections #-}
|
||||
{-# LANGUAGE TypeApplications #-}
|
||||
{-# LANGUAGE TypeFamilyDependencies #-}
|
||||
{-# OPTIONS_GHC -Wno-unrecognised-pragmas #-}
|
||||
|
||||
{-# HLINT ignore "Redundant multi-way if" #-}
|
||||
|
||||
module Simplex.Messaging.Server.MsgStore.Types
|
||||
( MsgStoreClass (..),
|
||||
MSType (..),
|
||||
@@ -25,49 +30,107 @@ module Simplex.Messaging.Server.MsgStore.Types
|
||||
getQueues,
|
||||
getQueueRecs,
|
||||
readQueueRec,
|
||||
withPeekMsgQueue,
|
||||
expireQueueMsgs,
|
||||
deleteExpireMsgs_,
|
||||
) where
|
||||
|
||||
import Control.Concurrent.STM
|
||||
import Control.Monad
|
||||
import Control.Monad.Trans.Except
|
||||
import Data.Functor (($>))
|
||||
import Data.Int (Int64)
|
||||
import Data.Kind
|
||||
import Data.Map.Strict (Map)
|
||||
import qualified Data.Map.Strict as M
|
||||
import Data.Maybe (catMaybes, fromMaybe)
|
||||
import Data.Text (Text)
|
||||
import Data.Time.Clock.System (SystemTime (systemSeconds))
|
||||
import Simplex.Messaging.Protocol
|
||||
import Simplex.Messaging.Server.QueueStore
|
||||
import Simplex.Messaging.Server.QueueStore.Types
|
||||
import Simplex.Messaging.Util (($>>=))
|
||||
import Simplex.Messaging.Util ((<$$>), ($>>=))
|
||||
|
||||
class QueueStoreClass (StoreQueue s) (QueueStore s) => MsgStoreClass s where
|
||||
class (Monad (StoreMonad s), QueueStoreClass (StoreQueue s) (QueueStore s)) => MsgStoreClass s where
|
||||
type StoreMonad s = (m :: Type -> Type) | m -> s
|
||||
type MsgStoreConfig s = c | c -> s
|
||||
type MsgQueue s = q | q -> s
|
||||
type StoreQueue s = q | q -> s
|
||||
type QueueStore s = qs | qs -> s
|
||||
newMsgStore :: MsgStoreConfig s -> IO s
|
||||
closeMsgStore :: s -> IO ()
|
||||
withActiveMsgQueues :: Monoid a => s -> (StoreQueue s -> IO a) -> IO a
|
||||
-- This function can only be used in server CLI commands or before server is started.
|
||||
-- tty, store
|
||||
unsafeWithAllMsgQueues :: Monoid a => Bool -> s -> (StoreQueue s -> IO a) -> IO a
|
||||
-- tty, store, now, ttl
|
||||
expireOldMessages :: Bool -> s -> Int64 -> Int64 -> IO MessageStats
|
||||
foldRcvServiceMessages :: s -> ServiceId -> (a -> RecipientId -> Either ErrorType (Maybe (QueueRec, Message)) -> IO a) -> a -> IO (Either ErrorType a)
|
||||
logQueueStates :: s -> IO ()
|
||||
logQueueState :: StoreQueue s -> StoreMonad s ()
|
||||
queueStore :: s -> QueueStore s
|
||||
loadedQueueCounts :: s -> IO LoadedQueueCounts
|
||||
|
||||
-- message store methods
|
||||
mkQueue :: s -> RecipientId -> QueueRec -> IO (StoreQueue s)
|
||||
mkQueue :: s -> Bool -> RecipientId -> QueueRec -> IO (StoreQueue s)
|
||||
getMsgQueue :: s -> StoreQueue s -> Bool -> StoreMonad s (MsgQueue s)
|
||||
getPeekMsgQueue :: s -> StoreQueue s -> StoreMonad s (Maybe (MsgQueue s, Message))
|
||||
|
||||
-- the journal queue will be closed after action if it was initially closed or idle longer than interval in config
|
||||
withIdleMsgQueue :: Int64 -> s -> StoreQueue s -> (MsgQueue s -> StoreMonad s a) -> StoreMonad s (Maybe a, Int)
|
||||
deleteQueue :: s -> StoreQueue s -> IO (Either ErrorType QueueRec)
|
||||
deleteQueueSize :: s -> StoreQueue s -> IO (Either ErrorType (QueueRec, Int))
|
||||
getQueueMessages_ :: Bool -> StoreQueue s -> MsgQueue s -> StoreMonad s [Message]
|
||||
writeMsg :: s -> StoreQueue s -> Bool -> Message -> ExceptT ErrorType IO (Maybe (Message, Bool))
|
||||
setOverQuota_ :: StoreQueue s -> IO () -- can ONLY be used while restoring messages, not while server running
|
||||
getQueueSize_ :: MsgQueue s -> StoreMonad s Int
|
||||
tryPeekMsg_ :: StoreQueue s -> MsgQueue s -> StoreMonad s (Maybe Message)
|
||||
tryDeleteMsg_ :: StoreQueue s -> MsgQueue s -> Bool -> StoreMonad s ()
|
||||
isolateQueue :: s -> StoreQueue s -> Text -> StoreMonad s a -> ExceptT ErrorType IO a
|
||||
unsafeRunStore :: StoreQueue s -> Text -> StoreMonad s a -> IO a
|
||||
|
||||
-- default implementations are overridden for PostgreSQL storage of messages
|
||||
tryPeekMsg :: s -> StoreQueue s -> ExceptT ErrorType IO (Maybe Message)
|
||||
tryPeekMsg st q = snd <$$> withPeekMsgQueue st q "tryPeekMsg" pure
|
||||
{-# INLINE tryPeekMsg #-}
|
||||
|
||||
tryPeekMsgs :: s -> [StoreQueue s] -> ExceptT ErrorType IO (Map RecipientId Message)
|
||||
tryPeekMsgs st qs = M.fromList . catMaybes <$> mapM (\q -> (recipientId q,) <$$> tryPeekMsg st q) qs
|
||||
|
||||
tryDelMsg :: s -> StoreQueue s -> MsgId -> ExceptT ErrorType IO (Maybe Message)
|
||||
tryDelMsg st q msgId' =
|
||||
withPeekMsgQueue st q "tryDelMsg" $
|
||||
maybe (pure Nothing) $ \(mq, msg) ->
|
||||
if
|
||||
| messageId msg == msgId' ->
|
||||
tryDeleteMsg_ q mq True $> Just msg
|
||||
| otherwise -> pure Nothing
|
||||
|
||||
-- atomic delete (== read) last and peek next message if available
|
||||
tryDelPeekMsg :: s -> StoreQueue s -> MsgId -> ExceptT ErrorType IO (Maybe Message, Maybe Message)
|
||||
deleteExpiredMsgs :: s -> StoreQueue s -> Int64 -> ExceptT ErrorType IO Int
|
||||
getQueueSize :: s -> StoreQueue s -> ExceptT ErrorType IO Int
|
||||
tryDelPeekMsg st q msgId' =
|
||||
withPeekMsgQueue st q "tryDelPeekMsg" $
|
||||
maybe (pure (Nothing, Nothing)) $ \(mq, msg) ->
|
||||
if
|
||||
| messageId msg == msgId' -> (Just msg,) <$> (tryDeleteMsg_ q mq True >> tryPeekMsg_ q mq)
|
||||
| otherwise -> pure (Nothing, Just msg)
|
||||
|
||||
data MSType = MSMemory | MSPostgres
|
||||
deleteExpiredMsgs :: s -> StoreQueue s -> Int64 -> ExceptT ErrorType IO Int
|
||||
deleteExpiredMsgs st q old =
|
||||
isolateQueue st q "deleteExpiredMsgs" $
|
||||
getMsgQueue st q False >>= deleteExpireMsgs_ old q
|
||||
|
||||
getQueueSize :: s -> StoreQueue s -> ExceptT ErrorType IO Int
|
||||
getQueueSize st q = withPeekMsgQueue st q "getQueueSize" $ maybe (pure 0) (getQueueSize_ . fst)
|
||||
{-# INLINE getQueueSize #-}
|
||||
|
||||
data MSType = MSMemory | MSJournal | MSPostgres
|
||||
|
||||
data QSType = QSMemory | QSPostgres
|
||||
|
||||
data SMSType :: MSType -> Type where
|
||||
SMSMemory :: SMSType 'MSMemory
|
||||
SMSJournal :: SMSType 'MSJournal
|
||||
SMSPostgres :: SMSType 'MSPostgres
|
||||
|
||||
data SQSType :: QSType -> Type where
|
||||
@@ -91,14 +154,17 @@ instance Semigroup MessageStats where
|
||||
|
||||
data LoadedQueueCounts = LoadedQueueCounts
|
||||
{ loadedQueueCount :: Int,
|
||||
loadedNotifierCount :: Int
|
||||
loadedNotifierCount :: Int,
|
||||
openJournalCount :: Int,
|
||||
queueLockCount :: Int,
|
||||
notifierLockCount :: Int
|
||||
}
|
||||
|
||||
newMessageStats :: MessageStats
|
||||
newMessageStats = MessageStats 0 0 0
|
||||
|
||||
addQueue :: MsgStoreClass s => s -> RecipientId -> QueueRec -> IO (Either ErrorType (StoreQueue s))
|
||||
addQueue st = addQueue_ (queueStore st) (mkQueue st)
|
||||
addQueue st = addQueue_ (queueStore st) (mkQueue st True)
|
||||
{-# INLINE addQueue #-}
|
||||
|
||||
getQueue :: (MsgStoreClass s, QueueParty p) => s -> SParty p -> QueueId -> IO (Either ErrorType (StoreQueue s))
|
||||
@@ -118,3 +184,28 @@ getQueueRecs st party qIds = getQueues st party qIds >>= mapM (fmap join . mapM
|
||||
readQueueRec :: StoreQueueClass q => q -> IO (Either ErrorType (q, QueueRec))
|
||||
readQueueRec q = maybe (Left AUTH) (Right . (q,)) <$> readTVarIO (queueRec q)
|
||||
{-# INLINE readQueueRec #-}
|
||||
|
||||
-- The action is called with Nothing when it is known that the queue is empty
|
||||
withPeekMsgQueue :: MsgStoreClass s => s -> StoreQueue s -> Text -> (Maybe (MsgQueue s, Message) -> StoreMonad s a) -> ExceptT ErrorType IO a
|
||||
withPeekMsgQueue st q op a = isolateQueue st q op $ getPeekMsgQueue st q >>= a
|
||||
{-# INLINE withPeekMsgQueue #-}
|
||||
|
||||
-- not used with PostgreSQL message store
|
||||
expireQueueMsgs :: MsgStoreClass s => s -> Int64 -> Int64 -> StoreQueue s -> StoreMonad s MessageStats
|
||||
expireQueueMsgs st now old q = do
|
||||
(expired_, stored) <- withIdleMsgQueue now st q $ deleteExpireMsgs_ old q
|
||||
pure MessageStats {storedMsgsCount = stored, expiredMsgsCount = fromMaybe 0 expired_, storedQueues = 1}
|
||||
|
||||
-- not used with PostgreSQL message store
|
||||
deleteExpireMsgs_ :: MsgStoreClass s => Int64 -> StoreQueue s -> MsgQueue s -> StoreMonad s Int
|
||||
deleteExpireMsgs_ old q mq = do
|
||||
n <- loop 0
|
||||
logQueueState q
|
||||
pure n
|
||||
where
|
||||
loop dc =
|
||||
tryPeekMsg_ q mq >>= \case
|
||||
Just Message {msgTs}
|
||||
| systemSeconds msgTs < old ->
|
||||
tryDeleteMsg_ q mq False >> loop (dc + 1)
|
||||
_ -> pure dc
|
||||
|
||||
@@ -20,9 +20,7 @@ import Control.Logger.Simple (logError)
|
||||
import Data.Bifunctor (first)
|
||||
import Data.Maybe (fromMaybe)
|
||||
import qualified Data.Text as T
|
||||
import Data.Text.Encoding (decodeLatin1)
|
||||
import Simplex.Messaging.Encoding
|
||||
import Simplex.Messaging.Protocol (NameErrorType (..), NameQuery, NameResponse)
|
||||
import Simplex.Messaging.Protocol (NameErrorType (..), NameRecord)
|
||||
import Simplex.Messaging.Server.Names.HttpResolver
|
||||
( ResolverEnv,
|
||||
ResolverError (..),
|
||||
@@ -32,6 +30,7 @@ import Simplex.Messaging.Server.Names.HttpResolver
|
||||
newResolverEnv,
|
||||
resolveHttp,
|
||||
)
|
||||
import Simplex.Messaging.SimplexName (SimplexDomain, fullDomainName)
|
||||
import System.Timeout (timeout)
|
||||
|
||||
data NamesConfig = NamesConfig
|
||||
@@ -59,9 +58,9 @@ pingEndpoint :: NamesEnv -> IO (Either ResolverError ())
|
||||
pingEndpoint NamesEnv {resolverEnv, config} =
|
||||
fromMaybe (Left ResolverTimeout) <$> timeout (resolverTimeoutMs config * 1000) (healthHttp resolverEnv)
|
||||
|
||||
resolveName :: NamesEnv -> NameQuery -> IO (Either NameErrorType NameResponse)
|
||||
resolveName env q = do
|
||||
r <- E.try (timeout (resolverTimeoutMs (config env) * 1000) (fetch env q))
|
||||
resolveName :: NamesEnv -> SimplexDomain -> IO (Either NameErrorType NameRecord)
|
||||
resolveName env d = do
|
||||
r <- E.try (timeout (resolverTimeoutMs (config env) * 1000) (fetch env d))
|
||||
case r of
|
||||
Right result -> pure (fromMaybe (Left (RESOLVER "timeout")) result)
|
||||
Left e
|
||||
@@ -70,12 +69,14 @@ resolveName env q = do
|
||||
logError $ "[NAMES] resolver fetch raised " <> T.pack (E.displayException e)
|
||||
pure (Left (RESOLVER "resolver error"))
|
||||
|
||||
fetch :: NamesEnv -> NameQuery -> IO (Either NameErrorType NameResponse)
|
||||
fetch NamesEnv {resolverEnv} q =
|
||||
first mapResolverError <$> resolveHttp resolverEnv (decodeLatin1 $ smpEncode q)
|
||||
fetch :: NamesEnv -> SimplexDomain -> IO (Either NameErrorType NameRecord)
|
||||
fetch NamesEnv {resolverEnv} d =
|
||||
first mapResolverError <$> resolveHttp resolverEnv (fullDomainName d)
|
||||
|
||||
mapResolverError :: ResolverError -> NameErrorType
|
||||
mapResolverError = \case
|
||||
HttpStatusErr 404 -> NOT_FOUND
|
||||
HttpStatusErr 400 -> NOT_FOUND
|
||||
HttpStatusErr code -> RESOLVER ("HTTP " <> T.pack (show code))
|
||||
HttpFailure _ -> RESOLVER "transport failure"
|
||||
BodyTooLarge -> RESOLVER "response too large"
|
||||
|
||||
@@ -8,12 +8,10 @@
|
||||
--
|
||||
-- The Python REST resolver (see scripts/resolver/snrc-resolve.py) exposes
|
||||
--
|
||||
-- GET /v2/resolve/<query> -> 200 with a NameRegistration JSON document, for
|
||||
-- all three registration shapes; 400 for unknown
|
||||
-- TLDs, 502 for upstream RPC failures
|
||||
-- GET /v1/resolve/<name> -> 200 with a NameRecord, what relays before SMP
|
||||
-- v22 call as /resolve
|
||||
-- GET /health -> 200 when the resolver process is ready
|
||||
-- GET /resolve/<name> -> 200 with a NameRecord JSON document
|
||||
-- 404 / 400 for unknown names / TLDs
|
||||
-- 502 for upstream RPC failures
|
||||
-- GET /health -> 200 when the resolver process is ready
|
||||
--
|
||||
-- Boundary properties:
|
||||
-- * Response body read with `brReadSome maxResponseBytes` — adversarial
|
||||
@@ -59,7 +57,7 @@ import qualified Network.HTTP.Client as HC
|
||||
import Network.HTTP.Client.TLS (tlsManagerSettings)
|
||||
import qualified Network.HTTP.Types as HT
|
||||
import Network.HTTP.Types.URI (urlEncode)
|
||||
import Simplex.Messaging.Names.Record (NameResponse)
|
||||
import Simplex.Messaging.Names.Record (NameRecord)
|
||||
|
||||
data RpcAuth = AuthBearer Text | AuthBasic Text Text
|
||||
|
||||
@@ -110,12 +108,14 @@ authHeader = \case
|
||||
let encoded = BAE.convertToBase BAE.Base64 (encodeUtf8 u <> ":" <> encodeUtf8 p) :: ByteString
|
||||
in ("Authorization", "Basic " <> encoded)
|
||||
|
||||
-- | The query is a name or a bracketed label hash, percent-encoded (every
|
||||
-- non-unreserved byte per RFC 3986) so it cannot alter the path.
|
||||
resolveHttp :: ResolverEnv -> Text -> IO (Either ResolverError NameResponse)
|
||||
resolveHttp env q =
|
||||
-- | GET <baseUrl>/resolve/<percent-encoded name>, decoding the 200 body
|
||||
-- directly into a NameRecord in one pass (no intermediate Aeson Value). The
|
||||
-- name is percent-encoded (every non-unreserved byte per RFC 3986): the
|
||||
-- resolver expects raw labels, so slashes/punctuation must not alter the path.
|
||||
resolveHttp :: ResolverEnv -> Text -> IO (Either ResolverError NameRecord)
|
||||
resolveHttp env name =
|
||||
(>>= first InvalidJson . J.eitherDecodeStrict . BL.toStrict)
|
||||
<$> httpGet env ("/v2/resolve/" <> B.unpack (urlEncode True (encodeUtf8 q)))
|
||||
<$> httpGet env ("/resolve/" <> B.unpack (urlEncode True (encodeUtf8 name)))
|
||||
|
||||
-- | GET <baseUrl>/health; success = reachable with status < 400. The body is
|
||||
-- size-capped but NOT decoded — the probe only checks reachability.
|
||||
|
||||
@@ -46,7 +46,7 @@ data RealTimeMetrics = RealTimeMetrics
|
||||
deliveredTimes :: TimeBuckets,
|
||||
smpSubs :: RTSubscriberMetrics,
|
||||
ntfSubs :: RTSubscriberMetrics,
|
||||
loadedCounts :: Maybe LoadedQueueCounts
|
||||
loadedCounts :: LoadedQueueCounts
|
||||
}
|
||||
|
||||
data RTSubscriberMetrics = RTSubscriberMetrics
|
||||
@@ -469,11 +469,11 @@ prometheusMetrics sm rtm ts =
|
||||
\# TYPE simplex_smp_names_reqs counter\n\
|
||||
\simplex_smp_names_reqs " <> mshow _rslvReqs <> "\n# rslvReqs\n\
|
||||
\\n\
|
||||
\# HELP simplex_smp_names_success NameRecord resolved, or availability answered.\n\
|
||||
\# HELP simplex_smp_names_success NameRecord successfully resolved and returned.\n\
|
||||
\# TYPE simplex_smp_names_success counter\n\
|
||||
\simplex_smp_names_success " <> mshow _rslvSucc <> "\n# rslvSucc\n\
|
||||
\\n\
|
||||
\# HELP simplex_smp_names_not_found Answers a client below v22 reads as NOT_FOUND.\n\
|
||||
\# HELP simplex_smp_names_not_found Name not registered (resolver returned 404 / 400).\n\
|
||||
\# TYPE simplex_smp_names_not_found counter\n\
|
||||
\simplex_smp_names_not_found " <> mshow _rslvNotFound <> "\n# rslvNotFound\n\
|
||||
\\n\
|
||||
@@ -562,17 +562,27 @@ prometheusMetrics sm rtm ts =
|
||||
\\n\
|
||||
\# HELP simplex_smp_subscription_ntf_service_subs_total Total queues subscribed via NTF services\n\
|
||||
\# TYPE simplex_smp_subscription_ntf_service_subs_total gauge\n\
|
||||
\simplex_smp_subscription_ntf_service_subs_total " <> mshow (subServiceSubsCount ntfSubs) <> "\n# ntf.subServiceSubsCount\n"
|
||||
<> maybe "" loadedQueues loadedCounts
|
||||
loadedQueues LoadedQueueCounts {loadedQueueCount, loadedNotifierCount} =
|
||||
"\n\
|
||||
\# HELP simplex_smp_loaded_queues_queue_count Total loaded queues count (all queues for memory storage)\n\
|
||||
\# TYPE simplex_smp_loaded_queues_queue_count gauge\n\
|
||||
\simplex_smp_loaded_queues_queue_count " <> mshow loadedQueueCount <> "\n# loadedCounts.loadedQueueCount\n\
|
||||
\simplex_smp_subscription_ntf_service_subs_total " <> mshow (subServiceSubsCount ntfSubs) <> "\n# ntf.subServiceSubsCount\n\
|
||||
\\n\
|
||||
\# HELP simplex_smp_loaded_queues_ntf_count Total loaded ntf credential references (all ntf credentials for memory storage)\n\
|
||||
\# HELP simplex_smp_loaded_queues_queue_count Total loaded queues count (all queues for memory/journal storage)\n\
|
||||
\# TYPE simplex_smp_loaded_queues_queue_count gauge\n\
|
||||
\simplex_smp_loaded_queues_queue_count " <> mshow (loadedQueueCount loadedCounts) <> "\n# loadedCounts.loadedQueueCount\n\
|
||||
\\n\
|
||||
\# HELP simplex_smp_loaded_queues_ntf_count Total loaded ntf credential references (all ntf credentials for memory/journal storage)\n\
|
||||
\# TYPE simplex_smp_loaded_queues_ntf_count gauge\n\
|
||||
\simplex_smp_loaded_queues_ntf_count " <> mshow loadedNotifierCount <> "\n# loadedCounts.loadedNotifierCount\n"
|
||||
\simplex_smp_loaded_queues_ntf_count " <> mshow (loadedNotifierCount loadedCounts) <> "\n# loadedCounts.loadedNotifierCount\n\
|
||||
\\n\
|
||||
\# HELP simplex_smp_loaded_queues_open_journal_count Total opened queue journals (0 for memory storage)\n\
|
||||
\# TYPE simplex_smp_loaded_queues_open_journal_count gauge\n\
|
||||
\simplex_smp_loaded_queues_open_journal_count " <> mshow (openJournalCount loadedCounts) <> "\n# loadedCounts.openJournalCount\n\
|
||||
\\n\
|
||||
\# HELP simplex_smp_loaded_queues_queue_lock_count Total queue locks (0 for memory storage)\n\
|
||||
\# TYPE simplex_smp_loaded_queues_queue_lock_count gauge\n\
|
||||
\simplex_smp_loaded_queues_queue_lock_count " <> mshow (queueLockCount loadedCounts) <> "\n# loadedCounts.queueLockCount\n\
|
||||
\\n\
|
||||
\# HELP simplex_smp_loaded_queues_ntf_lock_count Total notifier locks (0 for memory/journal storage)\n\
|
||||
\# TYPE simplex_smp_loaded_queues_ntf_lock_count gauge\n\
|
||||
\simplex_smp_loaded_queues_ntf_lock_count " <> mshow (notifierLockCount loadedCounts) <> "\n# loadedCounts.notifierLockCount\n"
|
||||
|
||||
showTimeBuckets :: Text -> IM.IntMap Int -> Text
|
||||
showTimeBuckets metric = T.concat . snd . mapAccumL accumBucket (0, 0) . IM.assocs
|
||||
|
||||
@@ -13,14 +13,12 @@ module Simplex.Messaging.Server.QueueStore
|
||||
ServiceRec (..),
|
||||
CertFingerprint,
|
||||
ServerEntityStatus (..),
|
||||
isSecuredMsgQueue,
|
||||
) where
|
||||
|
||||
import Control.Applicative (optional, (<|>))
|
||||
import qualified Data.ByteString.Char8 as B
|
||||
import Data.Functor (($>))
|
||||
import Data.List.NonEmpty (NonEmpty)
|
||||
import Data.Maybe (isJust)
|
||||
import qualified Data.X509 as X
|
||||
import qualified Data.X509.Validation as XV
|
||||
import Simplex.Messaging.Encoding
|
||||
@@ -50,11 +48,6 @@ data QueueRec = QueueRec
|
||||
}
|
||||
deriving (Show)
|
||||
|
||||
isSecuredMsgQueue :: QueueRec -> Bool
|
||||
isSecuredMsgQueue QueueRec {queueMode, senderKey} = case queueMode of
|
||||
Just QMContact -> False
|
||||
_ -> isJust senderKey
|
||||
|
||||
data NtfCreds = NtfCreds
|
||||
{ notifierId :: NotifierId,
|
||||
notifierKey :: NtfPublicAuthKey,
|
||||
|
||||
@@ -24,7 +24,9 @@ module Simplex.Messaging.Server.QueueStore.Postgres
|
||||
batchInsertServices,
|
||||
batchInsertQueues,
|
||||
foldServiceRecs,
|
||||
foldRcvServiceQueueRecs,
|
||||
foldQueueRecs,
|
||||
foldRecentQueueRecs,
|
||||
handleDuplicate,
|
||||
rowToQueueRec,
|
||||
withLog_,
|
||||
@@ -41,12 +43,13 @@ import Control.Monad
|
||||
import Control.Monad.Except
|
||||
import Control.Monad.IO.Class
|
||||
import Control.Monad.Trans.Except
|
||||
import Data.Bifunctor (first)
|
||||
import Data.ByteString.Builder (Builder)
|
||||
import qualified Data.ByteString.Builder as BB
|
||||
import Data.ByteString.Char8 (ByteString)
|
||||
import qualified Data.ByteString.Lazy as LB
|
||||
import Data.Bitraversable (bimapM)
|
||||
import Data.Either (fromRight)
|
||||
import Data.Either (fromRight, lefts)
|
||||
import Data.Functor (($>))
|
||||
import Data.Int (Int64)
|
||||
import Data.List (foldl', intersperse, partition)
|
||||
@@ -88,7 +91,7 @@ import Simplex.Messaging.SystemTime
|
||||
import Simplex.Messaging.TMap (TMap)
|
||||
import qualified Simplex.Messaging.TMap as TM
|
||||
import Simplex.Messaging.Transport (SMPServiceRole (..))
|
||||
import Simplex.Messaging.Util (eitherToMaybe, firstRow, maybeFirstRow, maybeFirstRow', tshow, (<$$>), ($>>=))
|
||||
import Simplex.Messaging.Util (eitherToMaybe, firstRow, ifM, maybeFirstRow, maybeFirstRow', tshow, (<$$>), ($>>=))
|
||||
import System.Exit (exitFailure)
|
||||
import System.IO (IOMode (..), hFlush, stdout)
|
||||
import UnliftIO.STM
|
||||
@@ -100,19 +103,35 @@ import Simplex.Messaging.Encoding.String
|
||||
data PostgresQueueStore q = PostgresQueueStore
|
||||
{ dbStore :: DBStore,
|
||||
dbStoreLog :: Maybe (StoreLog 'WriteMode),
|
||||
-- this map caches all created and opened queues
|
||||
queues :: TMap RecipientId q,
|
||||
-- this map only cashes the queues that were attempted to send messages to,
|
||||
senders :: TMap SenderId RecipientId,
|
||||
links :: TMap LinkId RecipientId,
|
||||
-- this map only cashes the queues that were attempted to be subscribed to,
|
||||
notifiers :: TMap NotifierId RecipientId,
|
||||
notifierLocks :: TMap NotifierId Lock,
|
||||
serviceLocks :: TMap CertFingerprint Lock,
|
||||
deletedTTL :: Int64
|
||||
deletedTTL :: Int64,
|
||||
useCache :: Bool
|
||||
}
|
||||
|
||||
instance StoreQueueClass q => QueueStoreClass q (PostgresQueueStore q) where
|
||||
type QueueStoreCfg (PostgresQueueStore q) = PostgresStoreCfg
|
||||
type UseQueueCache = Bool
|
||||
|
||||
newQueueStore :: PostgresStoreCfg -> IO (PostgresQueueStore q)
|
||||
newQueueStore PostgresStoreCfg {dbOpts, dbStoreLogPath, confirmMigrations, deletedTTL} = do
|
||||
instance StoreQueueClass q => QueueStoreClass q (PostgresQueueStore q) where
|
||||
type QueueStoreCfg (PostgresQueueStore q) = (PostgresStoreCfg, UseQueueCache)
|
||||
|
||||
newQueueStore :: (PostgresStoreCfg, UseQueueCache) -> IO (PostgresQueueStore q)
|
||||
newQueueStore (PostgresStoreCfg {dbOpts, dbStoreLogPath, confirmMigrations, deletedTTL}, useCache) = do
|
||||
dbStore <- either err pure =<< createDBStore dbOpts serverMigrations (MigrationConfig confirmMigrations Nothing)
|
||||
dbStoreLog <- mapM (openWriteStoreLog True) dbStoreLogPath
|
||||
queues <- TM.emptyIO
|
||||
senders <- TM.emptyIO
|
||||
links <- TM.emptyIO
|
||||
notifiers <- TM.emptyIO
|
||||
notifierLocks <- TM.emptyIO
|
||||
serviceLocks <- TM.emptyIO
|
||||
pure PostgresQueueStore {dbStore, dbStoreLog, serviceLocks, deletedTTL}
|
||||
pure PostgresQueueStore {dbStore, dbStoreLog, queues, senders, links, notifiers, notifierLocks, serviceLocks, deletedTTL, useCache}
|
||||
where
|
||||
err e = do
|
||||
logError $ "STORE: newQueueStore, error opening PostgreSQL database, " <> tshow e
|
||||
@@ -123,6 +142,9 @@ instance StoreQueueClass q => QueueStoreClass q (PostgresQueueStore q) where
|
||||
closeDBStore dbStore
|
||||
mapM_ closeStoreLog dbStoreLog
|
||||
|
||||
loadedQueues = queues
|
||||
{-# INLINE loadedQueues #-}
|
||||
|
||||
compactQueues :: PostgresQueueStore q -> IO Int64
|
||||
compactQueues st@PostgresQueueStore {deletedTTL} = do
|
||||
old <- subtract deletedTTL . systemSeconds <$> liftIO getSystemTime
|
||||
@@ -137,8 +159,7 @@ instance StoreQueueClass q => QueueStoreClass q (PostgresQueueStore q) where
|
||||
db
|
||||
[sql|
|
||||
SELECT
|
||||
-- estimate via reltuples to avoid a full heap scan on every scrape
|
||||
(SELECT GREATEST(reltuples, 0)::bigint FROM pg_class WHERE oid = 'msg_queues'::regclass) AS queue_count,
|
||||
(SELECT COUNT(1) FROM msg_queues WHERE deleted_at IS NULL) AS queue_count,
|
||||
(SELECT COUNT(1) FROM msg_queues WHERE deleted_at IS NULL AND notifier_id IS NOT NULL) AS notifier_count,
|
||||
(SELECT COUNT(1) FROM services WHERE service_role = ?) AS rcv_service_count,
|
||||
(SELECT COUNT(1) FROM services WHERE service_role = ?) AS ntf_service_count,
|
||||
@@ -148,45 +169,136 @@ instance StoreQueueClass q => QueueStoreClass q (PostgresQueueStore q) where
|
||||
(SRMessaging, SRNotifier)
|
||||
pure EntityCounts {queueCount, notifierCount, rcvServiceCount, ntfServiceCount, rcvServiceQueuesCount, ntfServiceQueuesCount}
|
||||
|
||||
-- this implementation relies on unique constraints in the database to prevent duplicate IDs.
|
||||
-- this implementation assumes that the lock is already taken by addQueue
|
||||
-- and relies on unique constraints in the database to prevent duplicate IDs.
|
||||
addQueue_ :: PostgresQueueStore q -> (RecipientId -> QueueRec -> IO q) -> RecipientId -> QueueRec -> IO (Either ErrorType q)
|
||||
addQueue_ st mkQ rId qr = do
|
||||
sq <- mkQ rId qr
|
||||
E.uninterruptibleMask_ $ runExceptT $ do
|
||||
withQueueLock sq "addQueue_" $ E.uninterruptibleMask_ $ runExceptT $ do
|
||||
void $ withDB "addQueue_" st $ \db ->
|
||||
E.try (DB.execute db insertQueueQuery $ queueRecToRow (rId, qr))
|
||||
>>= bimapM handleDuplicate pure
|
||||
when useCache $ do
|
||||
atomically $ TM.insert rId sq queues
|
||||
atomically $ TM.insert (senderId qr) rId senders
|
||||
forM_ (notifier qr) $ \NtfCreds {notifierId = nId} -> atomically $ TM.insert nId rId notifiers
|
||||
forM_ (queueData qr) $ \(lnkId, _) -> atomically $ TM.insert lnkId rId links
|
||||
withLog "addStoreQueue" st $ \s -> logCreateQueue s rId qr
|
||||
pure sq
|
||||
|
||||
getQueue_ :: QueueParty p => PostgresQueueStore q -> (RecipientId -> QueueRec -> IO q) -> SParty p -> QueueId -> IO (Either ErrorType q)
|
||||
getQueue_ st mkQ party qId = case party of
|
||||
SRecipient -> loadQueue " WHERE recipient_id = ?"
|
||||
SSender -> loadQueue " WHERE sender_id = ?"
|
||||
SSenderLink -> loadQueue " WHERE link_id = ?"
|
||||
SNotifier -> loadQueue " WHERE notifier_id = ?"
|
||||
where
|
||||
loadQueue condition = E.uninterruptibleMask_ $ runExceptT $ do
|
||||
(rId, qRec) <- withDB "getQueue_" st $ \db -> firstRow rowToQueueRec AUTH $
|
||||
DB.query db (queueRecQuery <> condition <> " AND deleted_at IS NULL") (Only qId)
|
||||
liftIO $ mkQ rId qRec
|
||||
PostgresQueueStore {queues, senders, links, notifiers, useCache} = st
|
||||
-- Not doing duplicate checks in maps as the probability of duplicates is very low.
|
||||
-- It needs to be reconsidered when IDs are supplied by the users.
|
||||
-- hasId = anyM [TM.memberIO rId queues, TM.memberIO senderId senders, hasNotifier]
|
||||
-- hasNotifier = maybe (pure False) (\NtfCreds {notifierId} -> TM.memberIO notifierId notifiers) notifier
|
||||
|
||||
getQueues_ :: forall p. BatchParty p => PostgresQueueStore q -> (RecipientId -> QueueRec -> IO q) -> SParty p -> [QueueId] -> IO [Either ErrorType q]
|
||||
getQueue_ :: QueueParty p => PostgresQueueStore q -> (Bool -> RecipientId -> QueueRec -> IO q) -> SParty p -> QueueId -> IO (Either ErrorType q)
|
||||
getQueue_ st mkQ party qId
|
||||
| useCache = case party of
|
||||
SRecipient -> getRcvQueue qId
|
||||
SSender -> TM.lookupIO qId senders >>= maybe (mask loadSndQueue) getRcvQueue
|
||||
SSenderLink -> TM.lookupIO qId links >>= maybe (mask loadLinkQueue) getRcvQueue
|
||||
-- loaded queue is deleted from notifiers map to reduce cache size after queue was subscribed to by ntf server
|
||||
SNotifier -> TM.lookupIO qId notifiers >>= maybe (mask loadNtfQueue) (getRcvQueue >=> (atomically (TM.delete qId notifiers) $>))
|
||||
| otherwise = case party of
|
||||
SRecipient -> loadQueueNoCache " WHERE recipient_id = ?"
|
||||
SSender -> loadQueueNoCache " WHERE sender_id = ?"
|
||||
SSenderLink -> loadQueueNoCache " WHERE link_id = ?"
|
||||
SNotifier -> loadQueueNoCache " WHERE notifier_id = ?"
|
||||
where
|
||||
PostgresQueueStore {queues, senders, links, notifiers, useCache} = st
|
||||
getRcvQueue rId = TM.lookupIO rId queues >>= maybe (mask loadRcvQueue) (pure . Right)
|
||||
loadRcvQueue = do
|
||||
(rId, qRec) <- loadQueue " WHERE recipient_id = ?"
|
||||
liftIO $ cacheQueue rId qRec $ \_ -> pure () -- recipient map already checked, not caching sender ref
|
||||
loadSndQueue = loadSndQueue_ " WHERE sender_id = ?"
|
||||
loadLinkQueue = loadSndQueue_ " WHERE link_id = ?"
|
||||
loadNtfQueue = do
|
||||
(rId, qRec) <- loadQueue " WHERE notifier_id = ?"
|
||||
liftIO $
|
||||
TM.lookupIO rId queues -- checking recipient map first, not creating lock in map, not caching queue
|
||||
>>= maybe (mkQ False rId qRec) pure
|
||||
loadSndQueue_ condition = do
|
||||
(rId, qRec) <- loadQueue condition
|
||||
liftIO $
|
||||
TM.lookupIO rId queues -- checking recipient map first
|
||||
>>= maybe (cacheQueue rId qRec cacheSender) (atomically (cacheSender rId) $>)
|
||||
loadQueueNoCache cond = mask $ loadQueue cond >>= liftIO . uncurry (mkQ True)
|
||||
mask = E.uninterruptibleMask_ . runExceptT
|
||||
cacheSender rId = TM.insert qId rId senders
|
||||
loadQueue condition =
|
||||
withDB "getQueue_" st $ \db -> firstRow rowToQueueRec AUTH $
|
||||
DB.query db (queueRecQuery <> condition <> " AND deleted_at IS NULL") (Only qId)
|
||||
cacheQueue rId qRec insertRef = do
|
||||
sq <- mkQ True rId qRec -- loaded queue
|
||||
-- This lock prevents the scenario when the queue is added to cache,
|
||||
-- while another thread is proccessing the same queue in withAllMsgQueues
|
||||
-- without adding it to cache, possibly trying to open the same files twice.
|
||||
-- Alse see comment in idleDeleteExpiredMsgs.
|
||||
withQueueLock sq "getQueue_" $ atomically $
|
||||
-- checking the cache again for concurrent reads,
|
||||
-- use previously loaded queue if exists.
|
||||
TM.lookup rId queues >>= \case
|
||||
Just sq' -> pure sq'
|
||||
Nothing -> do
|
||||
insertRef rId
|
||||
TM.insert rId sq queues
|
||||
pure sq
|
||||
|
||||
getQueues_ :: forall p. BatchParty p => PostgresQueueStore q -> (Bool -> RecipientId -> QueueRec -> IO q) -> SParty p -> [QueueId] -> IO [Either ErrorType q]
|
||||
getQueues_ st mkQ party qIds
|
||||
| null qIds = pure []
|
||||
| useCache = case party of
|
||||
SRecipient -> do
|
||||
qs <- readTVarIO queues
|
||||
let qs' = map (\qId -> get qs qId qId) qIds
|
||||
E.uninterruptibleMask_ $ loadQueues qs' " WHERE recipient_id IN ?" cacheRcvQueue
|
||||
SNotifier -> do
|
||||
ns <- readTVarIO notifiers
|
||||
qs <- readTVarIO queues
|
||||
let qs' = map (\qId -> get ns qId qId >>= get qs qId) qIds
|
||||
E.uninterruptibleMask_ $ loadQueues qs' " WHERE notifier_id IN ?" $ \(rId, qRec) ->
|
||||
forM (notifier qRec) $ \NtfCreds {notifierId = nId} -> -- it is always Just with this query
|
||||
(nId,) <$> maybe (mkQ False rId qRec) pure (M.lookup rId qs)
|
||||
| otherwise = E.uninterruptibleMask_ $ case party of
|
||||
SRecipient -> loadQueues " WHERE recipient_id IN ?" $ \(rId, qRec) ->
|
||||
Just . (rId,) <$> mkQ rId qRec
|
||||
SNotifier -> loadQueues " WHERE notifier_id IN ?" $ \(rId, qRec) ->
|
||||
forM (notifier qRec) $ \NtfCreds {notifierId = nId} -> (nId,) <$> mkQ rId qRec
|
||||
SRecipient -> loadQueuesNoCache " WHERE recipient_id IN ?" $ \(rId, qRec) ->
|
||||
Just . (rId,) <$> mkQ False rId qRec
|
||||
SNotifier -> loadQueuesNoCache " WHERE notifier_id IN ?" $ \(rId, qRec) ->
|
||||
forM (notifier qRec) $ \NtfCreds {notifierId = nId} -> (nId,) <$> mkQ False rId qRec
|
||||
where
|
||||
loadQueues :: Query -> ((RecipientId, QueueRec) -> IO (Maybe (QueueId, q))) -> IO [Either ErrorType q]
|
||||
loadQueues cond mkQueue' = do
|
||||
qs_ <-
|
||||
runExceptT $ fmap M.fromList $
|
||||
withDB' "getQueues_" st (\db -> DB.query db (queueRecQuery <> cond <> " AND deleted_at IS NULL") (Only (In qIds)))
|
||||
>>= liftIO . fmap catMaybes . mapM (mkQueue' . rowToQueueRec)
|
||||
pure $ map (\qId -> maybe (Left AUTH) Right . M.lookup qId =<< qs_) qIds
|
||||
PostgresQueueStore {queues, notifiers, useCache} = st
|
||||
get :: M.Map QueueId a -> QueueId -> QueueId -> Either QueueId a
|
||||
get m qId = maybe (Left qId) Right . (`M.lookup` m)
|
||||
loadQueues :: [Either QueueId q] -> Query -> ((RecipientId, QueueRec) -> IO (Maybe (QueueId, q))) -> IO [Either ErrorType q]
|
||||
loadQueues qs' cond mkCacheQueue = do
|
||||
let qIds' = lefts qs'
|
||||
if null qIds'
|
||||
then pure $ map (first (const INTERNAL)) qs'
|
||||
else do
|
||||
qs_ <- dbLoadQueues qIds' cond mkCacheQueue
|
||||
pure $ map (result qs_) qs'
|
||||
where
|
||||
result :: Either ErrorType (M.Map QueueId q) -> Either QueueId q -> Either ErrorType q
|
||||
result _ (Right q) = Right q
|
||||
result qs_ (Left qId) = maybe (Left AUTH) Right . M.lookup qId =<< qs_
|
||||
dbLoadQueues qIds' cond mkQueue' =
|
||||
runExceptT $ fmap M.fromList $
|
||||
withDB' "getQueues_" st (\db -> DB.query db (queueRecQuery <> cond <> " AND deleted_at IS NULL") (Only (In qIds')))
|
||||
>>= liftIO . fmap catMaybes . mapM (mkQueue' . rowToQueueRec)
|
||||
cacheRcvQueue (rId, qRec) = do
|
||||
sq <- mkQ True rId qRec
|
||||
sq' <- withQueueLock sq "getQueue_" $ atomically $
|
||||
-- checking the cache again for concurrent reads, use previously loaded queue if exists.
|
||||
TM.lookup rId queues >>= \case
|
||||
Just sq' -> pure sq'
|
||||
Nothing -> sq <$ TM.insert rId sq queues
|
||||
pure $ Just (rId, sq')
|
||||
loadQueuesNoCache cond mkQueue' = do
|
||||
qs_ <- dbLoadQueues qIds cond mkQueue'
|
||||
pure $ map (result qs_) qIds
|
||||
where
|
||||
result :: Either ErrorType (M.Map QueueId q) -> QueueId -> Either ErrorType q
|
||||
result qs_ qId = maybe (Left AUTH) Right . M.lookup qId =<< qs_
|
||||
|
||||
getQueueLinkData :: PostgresQueueStore q -> q -> LinkId -> IO (Either ErrorType QueueLinkData)
|
||||
getQueueLinkData st sq lnkId = runExceptT $ do
|
||||
@@ -199,11 +311,11 @@ instance StoreQueueClass q => QueueStoreClass q (PostgresQueueStore q) where
|
||||
|
||||
addQueueLinkData :: PostgresQueueStore q -> q -> LinkId -> QueueLinkData -> IO (Either ErrorType ())
|
||||
addQueueLinkData st sq lnkId d =
|
||||
withQueueRec sq $ \q -> case queueData q of
|
||||
withQueueRec sq "addQueueLinkData" $ \q -> case queueData q of
|
||||
Nothing ->
|
||||
addLink q $ \db -> DB.execute db qry (d :. (lnkId, rId, QMContact))
|
||||
addLink q $ \db -> DB.execute db qry (d :. (lnkId, rId))
|
||||
Just (lnkId', _) | lnkId' == lnkId ->
|
||||
addLink q $ \db -> DB.execute db (qry <> " AND (fixed_data IS NULL OR fixed_data = ?)") (d :. (lnkId, rId, QMContact, fst d))
|
||||
addLink q $ \db -> DB.execute db (qry <> " AND (fixed_data IS NULL OR fixed_data = ?)") (d :. (lnkId, rId, fst d))
|
||||
_ -> throwE AUTH
|
||||
where
|
||||
rId = recipientId sq
|
||||
@@ -211,12 +323,11 @@ instance StoreQueueClass q => QueueStoreClass q (PostgresQueueStore q) where
|
||||
assertUpdated $ withDB' "addQueueLinkData" st update
|
||||
atomically $ writeTVar (queueRec sq) $ Just q {queueData = Just (lnkId, d)}
|
||||
withLog "addQueueLinkData" st $ \s -> logCreateLink s rId lnkId d
|
||||
-- the sender key condition is checked in SQL because without cache each command reads its own copy of the queue record
|
||||
qry = "UPDATE msg_queues SET fixed_data = ?, user_data = ?, link_id = ? WHERE recipient_id = ? AND deleted_at IS NULL AND (sender_key IS NULL OR queue_mode = ?)"
|
||||
qry = "UPDATE msg_queues SET fixed_data = ?, user_data = ?, link_id = ? WHERE recipient_id = ? AND deleted_at IS NULL"
|
||||
|
||||
deleteQueueLinkData :: PostgresQueueStore q -> q -> IO (Either ErrorType ())
|
||||
deleteQueueLinkData st sq =
|
||||
withQueueRec sq $ \q -> case queueData q of
|
||||
withQueueRec sq "deleteQueueLinkData" $ \q -> case queueData q of
|
||||
Just _ -> do
|
||||
assertUpdated $ withDB' "deleteQueueLinkData" st $ \db ->
|
||||
DB.execute db "UPDATE msg_queues SET link_id = NULL, fixed_data = NULL, user_data = NULL WHERE recipient_id = ? AND deleted_at IS NULL" (Only rId)
|
||||
@@ -228,7 +339,7 @@ instance StoreQueueClass q => QueueStoreClass q (PostgresQueueStore q) where
|
||||
|
||||
secureQueue :: PostgresQueueStore q -> q -> SndPublicAuthKey -> IO (Either ErrorType ())
|
||||
secureQueue st sq sKey =
|
||||
withQueueRec sq $ \q -> do
|
||||
withQueueRec sq "secureQueue" $ \q -> do
|
||||
verify q
|
||||
assertUpdated $ withDB' "secureQueue" st $ \db ->
|
||||
DB.execute db "UPDATE msg_queues SET sender_key = ? WHERE recipient_id = ? AND deleted_at IS NULL AND (sender_key IS NULL OR sender_key = ?)" (sKey, rId, sKey)
|
||||
@@ -242,7 +353,7 @@ instance StoreQueueClass q => QueueStoreClass q (PostgresQueueStore q) where
|
||||
|
||||
updateKeys :: PostgresQueueStore q -> q -> NonEmpty RcvPublicAuthKey -> IO (Either ErrorType ())
|
||||
updateKeys st sq rKeys =
|
||||
withQueueRec sq $ \q -> do
|
||||
withQueueRec sq "updateKeys" $ \q -> do
|
||||
assertUpdated $ withDB' "updateKeys" st $ \db ->
|
||||
DB.execute db "UPDATE msg_queues SET recipient_keys = ? WHERE recipient_id = ? AND deleted_at IS NULL" (rKeys, rId)
|
||||
atomically $ writeTVar (queueRec sq) $ Just q {recipientKeys = rKeys}
|
||||
@@ -252,14 +363,24 @@ instance StoreQueueClass q => QueueStoreClass q (PostgresQueueStore q) where
|
||||
|
||||
addQueueNotifier :: PostgresQueueStore q -> q -> NtfCreds -> IO (Either ErrorType (Maybe NtfCreds))
|
||||
addQueueNotifier st sq ntfCreds@NtfCreds {notifierId = nId, notifierKey, rcvNtfDhSecret} =
|
||||
withQueueRec sq $ \q -> do
|
||||
assertUpdated $ withDB "addQueueNotifier" st $ \db ->
|
||||
E.try (update db) >>= bimapM handleDuplicate pure
|
||||
let !q' = q {notifier = Just ntfCreds}
|
||||
atomically $ writeTVar (queueRec sq) $ Just q'
|
||||
withLog "addQueueNotifier" st $ \s -> logAddNotifier s rId ntfCreds
|
||||
pure $ notifier q
|
||||
withQueueRec sq "addQueueNotifier" $ \q ->
|
||||
checkCachedNotifier $ do
|
||||
assertUpdated $ withDB "addQueueNotifier" st $ \db ->
|
||||
E.try (update db) >>= bimapM handleDuplicate pure
|
||||
nc_ <- forM (notifier q) $ \nc@NtfCreds {notifierId} -> atomically (TM.delete notifierId notifiers) $> nc
|
||||
let !q' = q {notifier = Just ntfCreds}
|
||||
atomically $ writeTVar (queueRec sq) $ Just q'
|
||||
when useCache $ do
|
||||
atomically $ TM.insert nId rId notifiers
|
||||
withLog "addQueueNotifier" st $ \s -> logAddNotifier s rId ntfCreds
|
||||
pure nc_
|
||||
where
|
||||
checkCachedNotifier add
|
||||
| useCache =
|
||||
ExceptT $ withLockMap (notifierLocks st) nId "addQueueNotifier" $
|
||||
ifM (TM.memberIO nId notifiers) (pure $ Left DUPLICATE_) $ runExceptT add
|
||||
| otherwise = add
|
||||
PostgresQueueStore {notifiers, useCache} = st
|
||||
rId = recipientId sq
|
||||
update db =
|
||||
DB.execute
|
||||
@@ -273,13 +394,18 @@ instance StoreQueueClass q => QueueStoreClass q (PostgresQueueStore q) where
|
||||
|
||||
deleteQueueNotifier :: PostgresQueueStore q -> q -> IO (Either ErrorType (Maybe NtfCreds))
|
||||
deleteQueueNotifier st sq =
|
||||
withQueueRec sq $ \q ->
|
||||
forM (notifier q) $ \nc -> do
|
||||
assertUpdated $ withDB' "deleteQueueNotifier" st update
|
||||
atomically $ writeTVar (queueRec sq) $ Just q {notifier = Nothing}
|
||||
withLog "deleteQueueNotifier" st (`logDeleteNotifier` rId)
|
||||
pure nc
|
||||
withQueueRec sq "deleteQueueNotifier" $ \q ->
|
||||
ExceptT $ fmap sequence $ forM (notifier q) $ \nc@NtfCreds {notifierId = nId} ->
|
||||
withNotifierLock nId $ runExceptT $ do
|
||||
assertUpdated $ withDB' "deleteQueueNotifier" st update
|
||||
when (useCache st) $ atomically $ TM.delete nId $ notifiers st
|
||||
atomically $ writeTVar (queueRec sq) $ Just q {notifier = Nothing}
|
||||
withLog "deleteQueueNotifier" st (`logDeleteNotifier` rId)
|
||||
pure nc
|
||||
where
|
||||
withNotifierLock nId
|
||||
| useCache st = withLockMap (notifierLocks st) nId "deleteQueueNotifier"
|
||||
| otherwise = id
|
||||
rId = recipientId sq
|
||||
update db =
|
||||
DB.execute
|
||||
@@ -308,7 +434,7 @@ instance StoreQueueClass q => QueueStoreClass q (PostgresQueueStore q) where
|
||||
|
||||
updateQueueTime :: PostgresQueueStore q -> q -> SystemDate -> IO (Either ErrorType QueueRec)
|
||||
updateQueueTime st sq t =
|
||||
withQueueRec sq $ \q@QueueRec {updatedAt} ->
|
||||
withQueueRec sq "updateQueueTime" $ \q@QueueRec {updatedAt} ->
|
||||
if updatedAt == Just t
|
||||
then pure q
|
||||
else do
|
||||
@@ -321,6 +447,7 @@ instance StoreQueueClass q => QueueStoreClass q (PostgresQueueStore q) where
|
||||
where
|
||||
rId = recipientId sq
|
||||
|
||||
-- this method is called from JournalMsgStore deleteQueue that already locks the queue
|
||||
deleteStoreQueue :: PostgresQueueStore q -> q -> IO (Either ErrorType QueueRec)
|
||||
deleteStoreQueue st sq = E.uninterruptibleMask_ $ runExceptT $ do
|
||||
q <- ExceptT $ readQueueRecIO qr
|
||||
@@ -328,6 +455,11 @@ instance StoreQueueClass q => QueueStoreClass q (PostgresQueueStore q) where
|
||||
assertUpdated $ withDB' "deleteStoreQueue" st $ \db ->
|
||||
DB.execute db "UPDATE msg_queues SET deleted_at = ? WHERE recipient_id = ? AND deleted_at IS NULL" (ts, rId)
|
||||
atomically $ writeTVar qr Nothing
|
||||
when (useCache st) $ do
|
||||
atomically $ TM.delete (senderId q) $ senders st
|
||||
forM_ (notifier q) $ \NtfCreds {notifierId} -> do
|
||||
atomically $ TM.delete notifierId $ notifiers st
|
||||
atomically $ TM.delete notifierId $ notifierLocks st
|
||||
withLog "deleteStoreQueue" st (`logDeleteQueue` rId)
|
||||
pure q
|
||||
where
|
||||
@@ -350,7 +482,7 @@ instance StoreQueueClass q => QueueStoreClass q (PostgresQueueStore q) where
|
||||
pure serviceId
|
||||
|
||||
setQueueService :: (PartyI p, ServiceParty p) => PostgresQueueStore q -> q -> SParty p -> Maybe ServiceId -> IO (Either ErrorType ())
|
||||
setQueueService st sq party serviceId = withQueueRec sq $ \q -> case party of
|
||||
setQueueService st sq party serviceId = withQueueRec sq "setQueueService" $ \q -> case party of
|
||||
SRecipientService
|
||||
| rcvServiceId q == serviceId -> pure ()
|
||||
| otherwise -> do
|
||||
@@ -475,8 +607,33 @@ foldServiceRecs st f =
|
||||
DB.fold_ db "SELECT service_id, service_role, service_cert, service_cert_hash, created_at FROM services" mempty $
|
||||
\ !acc -> fmap (acc <>) . f . rowToServiceRec
|
||||
|
||||
foldRcvServiceQueueRecs :: PostgresQueueStore q -> ServiceId -> (a -> (RecipientId, QueueRec) -> IO a) -> a -> IO (Either ErrorType a)
|
||||
foldRcvServiceQueueRecs st serviceId f acc =
|
||||
runExceptT $ withDB' "foldRcvServiceQueueRecs" st $ \db ->
|
||||
DB.fold db (queueRecQuery <> " WHERE rcv_service_id = ? AND deleted_at IS NULL") (Only serviceId) acc $ \a -> f a . rowToQueueRec
|
||||
|
||||
foldQueueRecs :: Monoid a => Bool -> Bool -> PostgresQueueStore q -> ((RecipientId, QueueRec) -> IO a) -> IO a
|
||||
foldQueueRecs withData tty st f = do
|
||||
foldQueueRecs withData = foldQueueRecs_ foldRecs
|
||||
where
|
||||
foldRecs db acc f'
|
||||
| withData = DB.fold_ db (queueRecQueryWithData <> cond) acc $ \acc' -> f' acc' . rowToQueueRecWithData
|
||||
| otherwise = DB.fold_ db (queueRecQuery <> cond) acc $ \acc' -> f' acc' . rowToQueueRec
|
||||
cond = " WHERE deleted_at IS NULL ORDER BY recipient_id ASC"
|
||||
|
||||
foldRecentQueueRecs :: Monoid a => Int64 -> Bool -> PostgresQueueStore q -> ((RecipientId, QueueRec) -> IO a) -> IO a
|
||||
foldRecentQueueRecs old = foldQueueRecs_ foldRecs
|
||||
where
|
||||
foldRecs db acc f' = DB.fold db (queueRecQuery <> cond) (Only old) acc $ \acc' -> f' acc' . rowToQueueRec
|
||||
cond = " WHERE deleted_at IS NULL AND updated_at > ? ORDER BY recipient_id ASC"
|
||||
|
||||
foldQueueRecs_ ::
|
||||
Monoid a =>
|
||||
(DB.Connection -> (Int, a) -> ((Int, a) -> (RecipientId, QueueRec) -> IO (Int, a)) -> IO (Int, a)) ->
|
||||
Bool ->
|
||||
PostgresQueueStore q ->
|
||||
((RecipientId, QueueRec) -> IO a) ->
|
||||
IO a
|
||||
foldQueueRecs_ foldRecs tty st f = do
|
||||
(n, r) <- withTransaction (dbStore st) $ \db ->
|
||||
foldRecs db (0 :: Int, mempty) $ \(i, acc) qr -> do
|
||||
r <- f qr
|
||||
@@ -487,10 +644,6 @@ foldQueueRecs withData tty st f = do
|
||||
when tty $ putStrLn $ progress n
|
||||
pure r
|
||||
where
|
||||
foldRecs db acc f'
|
||||
| withData = DB.fold_ db (queueRecQueryWithData <> cond) acc $ \acc' -> f' acc' . rowToQueueRecWithData
|
||||
| otherwise = DB.fold_ db (queueRecQuery <> cond) acc $ \acc' -> f' acc' . rowToQueueRec
|
||||
cond = " WHERE deleted_at IS NULL ORDER BY recipient_id ASC"
|
||||
progress i = "Processed: " <> show i <> " records"
|
||||
|
||||
queueRecQuery :: Query
|
||||
@@ -596,15 +749,15 @@ rowToServiceRec (serviceId, serviceRole, serviceCert, Binary fp, serviceCreatedA
|
||||
|
||||
setStatusDB :: StoreQueueClass q => Text -> PostgresQueueStore q -> q -> ServerEntityStatus -> ExceptT ErrorType IO () -> IO (Either ErrorType ())
|
||||
setStatusDB op st sq status writeLog =
|
||||
withQueueRec sq $ \q -> do
|
||||
withQueueRec sq op $ \q -> do
|
||||
assertUpdated $ withDB' op st $ \db ->
|
||||
DB.execute db "UPDATE msg_queues SET status = ? WHERE recipient_id = ? AND deleted_at IS NULL" (status, recipientId sq)
|
||||
atomically $ writeTVar (queueRec sq) $ Just q {status}
|
||||
writeLog
|
||||
|
||||
withQueueRec :: StoreQueueClass q => q -> (QueueRec -> ExceptT ErrorType IO a) -> IO (Either ErrorType a)
|
||||
withQueueRec sq action =
|
||||
E.uninterruptibleMask_ $ runExceptT $ ExceptT (readQueueRecIO $ queueRec sq) >>= action
|
||||
withQueueRec :: StoreQueueClass q => q -> Text -> (QueueRec -> ExceptT ErrorType IO a) -> IO (Either ErrorType a)
|
||||
withQueueRec sq op action =
|
||||
withQueueLock sq op $ E.uninterruptibleMask_ $ runExceptT $ ExceptT (readQueueRecIO $ queueRec sq) >>= action
|
||||
|
||||
assertUpdated :: ExceptT ErrorType IO Int64 -> ExceptT ErrorType IO ()
|
||||
assertUpdated = (>>= \n -> when (n == 0) (throwE AUTH))
|
||||
|
||||
@@ -20,10 +20,7 @@ serverSchemaMigrations =
|
||||
("20250320_short_links", m20250320_short_links, Just down_m20250320_short_links),
|
||||
("20250514_service_certs", m20250514_service_certs, Just down_m20250514_service_certs),
|
||||
("20250903_store_messages", m20250903_store_messages, Just down_m20250903_store_messages),
|
||||
("20250915_queue_ids_hash", m20250915_queue_ids_hash, Just down_m20250915_queue_ids_hash),
|
||||
("20260916_prometheus_indexes", m20260916_prometheus_indexes, Just down_m20260916_prometheus_indexes),
|
||||
("20260917_msg_queues_hot", m20260917_msg_queues_hot, Just down_m20260917_msg_queues_hot),
|
||||
("20260918_expire_messages", m20260918_expire_messages, Just down_m20260918_expire_messages)
|
||||
("20250915_queue_ids_hash", m20250915_queue_ids_hash, Just down_m20250915_queue_ids_hash)
|
||||
]
|
||||
|
||||
-- | The list of migrations in ascending order by date
|
||||
@@ -591,648 +588,3 @@ ALTER TABLE services
|
||||
DROP COLUMN queue_ids_hash;
|
||||
|]
|
||||
<> dropXorHashFuncs
|
||||
|
||||
m20260916_prometheus_indexes :: Text
|
||||
m20260916_prometheus_indexes =
|
||||
[r|
|
||||
CREATE INDEX idx_msg_queues_expire ON msg_queues (recipient_id) WHERE deleted_at IS NULL AND msg_queue_expire;
|
||||
CREATE INDEX idx_msg_queues_notifier_active ON msg_queues (notifier_id) WHERE deleted_at IS NULL AND notifier_id IS NOT NULL;
|
||||
|]
|
||||
|
||||
down_m20260916_prometheus_indexes :: Text
|
||||
down_m20260916_prometheus_indexes =
|
||||
[r|
|
||||
DROP INDEX idx_msg_queues_expire;
|
||||
DROP INDEX idx_msg_queues_notifier_active;
|
||||
|]
|
||||
|
||||
m20260917_msg_queues_hot :: Text
|
||||
m20260917_msg_queues_hot =
|
||||
[r|
|
||||
DROP INDEX idx_msg_queues_updated_at_recipient_id;
|
||||
|
||||
ALTER TABLE msg_queues SET (
|
||||
fillfactor = 80,
|
||||
autovacuum_vacuum_scale_factor = 0.02,
|
||||
autovacuum_analyze_scale_factor = 0.01,
|
||||
autovacuum_vacuum_cost_limit = 1000
|
||||
);
|
||||
|
||||
ALTER TABLE messages SET (
|
||||
autovacuum_vacuum_scale_factor = 0.02,
|
||||
autovacuum_analyze_scale_factor = 0.01,
|
||||
toast.autovacuum_vacuum_scale_factor = 0.02
|
||||
);
|
||||
|
||||
ALTER TABLE services SET (
|
||||
fillfactor = 70,
|
||||
autovacuum_vacuum_threshold = 1000,
|
||||
autovacuum_vacuum_scale_factor = 0
|
||||
);
|
||||
|
||||
DROP PROCEDURE expire_old_messages(bigint, bigint, integer);
|
||||
|
||||
CREATE PROCEDURE expire_old_messages(IN p_old_ts bigint, IN batch_size integer, OUT r_expired_msgs_count bigint, OUT r_stored_msgs_count bigint, OUT r_stored_queues bigint)
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
DECLARE
|
||||
rids BYTEA[];
|
||||
rid BYTEA;
|
||||
last_rid BYTEA := '\x';
|
||||
del_count BIGINT;
|
||||
total_deleted BIGINT := 0;
|
||||
BEGIN
|
||||
LOOP
|
||||
SELECT array_agg(recipient_id)
|
||||
INTO rids
|
||||
FROM (
|
||||
SELECT recipient_id
|
||||
FROM msg_queues
|
||||
WHERE deleted_at IS NULL
|
||||
AND msg_queue_expire = TRUE
|
||||
AND recipient_id > last_rid
|
||||
ORDER BY recipient_id ASC
|
||||
LIMIT batch_size
|
||||
) qs;
|
||||
|
||||
EXIT WHEN rids IS NULL OR cardinality(rids) = 0;
|
||||
|
||||
FOREACH rid IN ARRAY rids
|
||||
LOOP
|
||||
BEGIN
|
||||
del_count := delete_expired_msgs(rid, p_old_ts);
|
||||
total_deleted := total_deleted + del_count;
|
||||
EXCEPTION WHEN OTHERS THEN
|
||||
RAISE WARNING 'STORE, expire_old_messages, error expiring queue %: %', encode(rid, 'base64'), SQLERRM;
|
||||
CONTINUE;
|
||||
END;
|
||||
COMMIT;
|
||||
END LOOP;
|
||||
last_rid := rids[cardinality(rids)];
|
||||
END LOOP;
|
||||
|
||||
r_expired_msgs_count := total_deleted;
|
||||
r_stored_msgs_count := (SELECT COUNT(1) FROM messages);
|
||||
r_stored_queues := (SELECT COUNT(1) FROM msg_queues WHERE deleted_at IS NULL);
|
||||
END;
|
||||
$$;
|
||||
|]
|
||||
|
||||
down_m20260917_msg_queues_hot :: Text
|
||||
down_m20260917_msg_queues_hot =
|
||||
[r|
|
||||
DROP PROCEDURE expire_old_messages(bigint, integer);
|
||||
|
||||
CREATE PROCEDURE expire_old_messages(IN p_old_queue bigint, IN p_old_ts bigint, IN batch_size integer, OUT r_expired_msgs_count bigint, OUT r_stored_msgs_count bigint, OUT r_stored_queues bigint)
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
DECLARE
|
||||
rids BYTEA[];
|
||||
rid BYTEA;
|
||||
last_rid BYTEA := '\x';
|
||||
del_count BIGINT;
|
||||
total_deleted BIGINT := 0;
|
||||
BEGIN
|
||||
LOOP
|
||||
SELECT array_agg(recipient_id)
|
||||
INTO rids
|
||||
FROM (
|
||||
SELECT recipient_id
|
||||
FROM msg_queues
|
||||
WHERE deleted_at IS NULL
|
||||
AND updated_at > p_old_queue
|
||||
AND msg_queue_expire = TRUE
|
||||
AND recipient_id > last_rid
|
||||
ORDER BY recipient_id ASC
|
||||
LIMIT batch_size
|
||||
) qs;
|
||||
|
||||
EXIT WHEN rids IS NULL OR cardinality(rids) = 0;
|
||||
|
||||
FOREACH rid IN ARRAY rids
|
||||
LOOP
|
||||
BEGIN
|
||||
del_count := delete_expired_msgs(rid, p_old_ts);
|
||||
total_deleted := total_deleted + del_count;
|
||||
EXCEPTION WHEN OTHERS THEN
|
||||
RAISE WARNING 'STORE, expire_old_messages, error expiring queue %: %', encode(rid, 'base64'), SQLERRM;
|
||||
CONTINUE;
|
||||
END;
|
||||
COMMIT;
|
||||
END LOOP;
|
||||
last_rid := rids[cardinality(rids)];
|
||||
END LOOP;
|
||||
|
||||
r_expired_msgs_count := total_deleted;
|
||||
r_stored_msgs_count := (SELECT COUNT(1) FROM messages);
|
||||
r_stored_queues := (SELECT COUNT(1) FROM msg_queues WHERE deleted_at IS NULL);
|
||||
END;
|
||||
$$;
|
||||
|
||||
CREATE INDEX idx_msg_queues_updated_at_recipient_id ON msg_queues (deleted_at, updated_at, msg_queue_expire, recipient_id);
|
||||
|
||||
ALTER TABLE msg_queues RESET (fillfactor, autovacuum_vacuum_scale_factor, autovacuum_analyze_scale_factor, autovacuum_vacuum_cost_limit);
|
||||
ALTER TABLE messages RESET (autovacuum_vacuum_scale_factor, autovacuum_analyze_scale_factor, toast.autovacuum_vacuum_scale_factor);
|
||||
ALTER TABLE services RESET (fillfactor, autovacuum_vacuum_threshold, autovacuum_vacuum_scale_factor);
|
||||
|]
|
||||
|
||||
m20260918_expire_messages :: Text
|
||||
m20260918_expire_messages =
|
||||
[r|
|
||||
CREATE INDEX idx_messages_expire ON messages (msg_ts, recipient_id) WHERE NOT msg_quota;
|
||||
|
||||
DROP INDEX idx_messages_recipient_id_msg_ts;
|
||||
|
||||
DROP INDEX idx_msg_queues_expire;
|
||||
|
||||
DROP PROCEDURE expire_old_messages(bigint, integer);
|
||||
|
||||
CREATE PROCEDURE expire_old_messages(IN p_old_ts bigint, IN batch_size integer, OUT r_expired_msgs_count bigint, OUT r_stored_msgs_count bigint, OUT r_stored_queues bigint)
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
DECLARE
|
||||
rids BYTEA[];
|
||||
rid BYTEA;
|
||||
last_ts BIGINT := -1;
|
||||
last_rid BYTEA := '\x';
|
||||
next_ts BIGINT;
|
||||
next_rid BYTEA;
|
||||
del_count BIGINT;
|
||||
total_deleted BIGINT := 0;
|
||||
BEGIN
|
||||
LOOP
|
||||
-- The page is scanned in (msg_ts, recipient_id) order, so its last row is the next
|
||||
-- keyset cursor. Advancing it past every row read, including queues left unexpired
|
||||
-- because delete_expired_msgs skipped a locked row or raised, is what terminates the
|
||||
-- loop; re-reading from the start instead would repeat those queues forever.
|
||||
SELECT array_agg(DISTINCT recipient_id),
|
||||
(array_agg(msg_ts ORDER BY msg_ts DESC, recipient_id DESC))[1],
|
||||
(array_agg(recipient_id ORDER BY msg_ts DESC, recipient_id DESC))[1]
|
||||
INTO rids, next_ts, next_rid
|
||||
FROM (
|
||||
SELECT msg_ts, recipient_id
|
||||
FROM messages
|
||||
WHERE NOT msg_quota
|
||||
AND msg_ts < p_old_ts
|
||||
AND (msg_ts, recipient_id) > (last_ts, last_rid)
|
||||
ORDER BY msg_ts ASC, recipient_id ASC
|
||||
LIMIT batch_size
|
||||
) m;
|
||||
|
||||
EXIT WHEN rids IS NULL;
|
||||
|
||||
FOREACH rid IN ARRAY rids
|
||||
LOOP
|
||||
BEGIN
|
||||
del_count := delete_expired_msgs(rid, p_old_ts);
|
||||
total_deleted := total_deleted + del_count;
|
||||
EXCEPTION WHEN OTHERS THEN
|
||||
RAISE WARNING 'STORE, expire_old_messages, error expiring queue %: %', encode(rid, 'base64'), SQLERRM;
|
||||
CONTINUE;
|
||||
END;
|
||||
END LOOP;
|
||||
COMMIT;
|
||||
|
||||
last_ts := next_ts;
|
||||
last_rid := next_rid;
|
||||
END LOOP;
|
||||
|
||||
r_expired_msgs_count := total_deleted;
|
||||
r_stored_msgs_count := (SELECT COUNT(1) FROM messages);
|
||||
r_stored_queues := (SELECT COUNT(1) FROM msg_queues WHERE deleted_at IS NULL);
|
||||
END;
|
||||
$$;
|
||||
|
||||
CREATE OR REPLACE FUNCTION delete_expired_msgs(p_recipient_id bytea, p_old_ts bigint) RETURNS bigint
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
DECLARE
|
||||
q_size BIGINT;
|
||||
keep_min_id BIGINT;
|
||||
del_count BIGINT;
|
||||
BEGIN
|
||||
SELECT msg_queue_size INTO q_size
|
||||
FROM msg_queues
|
||||
WHERE recipient_id = p_recipient_id AND deleted_at IS NULL
|
||||
FOR UPDATE SKIP LOCKED;
|
||||
|
||||
IF NOT FOUND OR q_size = 0 THEN
|
||||
RETURN 0;
|
||||
END IF;
|
||||
|
||||
SELECT MIN(message_id) INTO keep_min_id
|
||||
FROM messages WHERE recipient_id = p_recipient_id AND msg_ts >= p_old_ts AND msg_quota = FALSE;
|
||||
|
||||
IF keep_min_id IS NULL THEN
|
||||
DELETE FROM messages WHERE recipient_id = p_recipient_id AND msg_quota = FALSE;
|
||||
ELSE
|
||||
DELETE FROM messages WHERE recipient_id = p_recipient_id AND message_id < keep_min_id AND msg_quota = FALSE;
|
||||
END IF;
|
||||
|
||||
GET DIAGNOSTICS del_count = ROW_COUNT;
|
||||
IF del_count > 0 THEN
|
||||
UPDATE msg_queues
|
||||
SET msg_can_write = msg_can_write OR msg_queue_size <= del_count,
|
||||
msg_queue_size = GREATEST(msg_queue_size - del_count, 0)
|
||||
WHERE recipient_id = p_recipient_id;
|
||||
END IF;
|
||||
RETURN del_count;
|
||||
END;
|
||||
$$;
|
||||
|
||||
CREATE OR REPLACE FUNCTION write_message(p_recipient_id bytea, p_msg_id bytea, p_msg_ts bigint, p_msg_quota boolean, p_msg_ntf_flag boolean, p_msg_body bytea, p_quota integer) RETURNS TABLE(quota_written boolean, was_empty boolean)
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
DECLARE
|
||||
q_can_write BOOLEAN;
|
||||
q_size BIGINT;
|
||||
BEGIN
|
||||
SELECT msg_can_write, msg_queue_size INTO q_can_write, q_size
|
||||
FROM msg_queues
|
||||
WHERE recipient_id = p_recipient_id AND deleted_at IS NULL
|
||||
FOR UPDATE;
|
||||
|
||||
IF q_can_write OR q_size = 0 THEN
|
||||
quota_written := p_msg_quota OR q_size >= p_quota;
|
||||
was_empty := q_size = 0;
|
||||
|
||||
INSERT INTO messages(recipient_id, msg_id, msg_ts, msg_quota, msg_ntf_flag, msg_body)
|
||||
VALUES (p_recipient_id, p_msg_id, p_msg_ts, quota_written, p_msg_ntf_flag AND NOT quota_written, CASE WHEN quota_written THEN '' :: BYTEA ELSE p_msg_body END);
|
||||
|
||||
UPDATE msg_queues
|
||||
SET msg_can_write = NOT quota_written,
|
||||
msg_queue_size = msg_queue_size + 1
|
||||
WHERE recipient_id = p_recipient_id;
|
||||
|
||||
RETURN QUERY VALUES (quota_written, was_empty);
|
||||
END IF;
|
||||
END;
|
||||
$$;
|
||||
|
||||
CREATE OR REPLACE FUNCTION try_del_msg(p_recipient_id bytea, p_msg_id bytea) RETURNS TABLE(r_msg_id bytea, r_msg_ts bigint, r_msg_quota boolean, r_msg_ntf_flag boolean, r_msg_body bytea)
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
DECLARE
|
||||
q_size BIGINT;
|
||||
msg RECORD;
|
||||
BEGIN
|
||||
SELECT msg_queue_size INTO q_size
|
||||
FROM msg_queues
|
||||
WHERE recipient_id = p_recipient_id AND deleted_at IS NULL
|
||||
FOR UPDATE;
|
||||
|
||||
IF NOT FOUND THEN
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
SELECT message_id, msg_id, msg_ts, msg_quota, msg_ntf_flag, msg_body
|
||||
INTO msg
|
||||
FROM messages
|
||||
WHERE recipient_id = p_recipient_id
|
||||
ORDER BY message_id ASC LIMIT 1;
|
||||
|
||||
IF NOT FOUND THEN
|
||||
IF q_size != 0 THEN
|
||||
UPDATE msg_queues
|
||||
SET msg_can_write = TRUE,
|
||||
msg_queue_size = 0
|
||||
WHERE recipient_id = p_recipient_id;
|
||||
END IF;
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
IF msg.msg_id = p_msg_id THEN
|
||||
DELETE FROM messages WHERE message_id = msg.message_id;
|
||||
IF FOUND THEN
|
||||
UPDATE msg_queues
|
||||
SET msg_can_write = msg_can_write OR msg_queue_size <= 1,
|
||||
msg_queue_size = GREATEST(msg_queue_size - 1, 0)
|
||||
WHERE recipient_id = p_recipient_id;
|
||||
RETURN QUERY VALUES (msg.msg_id, msg.msg_ts, msg.msg_quota, msg.msg_ntf_flag, msg.msg_body);
|
||||
END IF;
|
||||
END IF;
|
||||
END;
|
||||
$$;
|
||||
|
||||
CREATE OR REPLACE FUNCTION try_del_peek_msg(p_recipient_id bytea, p_msg_id bytea) RETURNS TABLE(r_msg_id bytea, r_msg_ts bigint, r_msg_quota boolean, r_msg_ntf_flag boolean, r_msg_body bytea)
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
DECLARE
|
||||
q_size BIGINT;
|
||||
msg RECORD;
|
||||
msg_deleted BOOLEAN;
|
||||
BEGIN
|
||||
SELECT msg_queue_size INTO q_size
|
||||
FROM msg_queues
|
||||
WHERE recipient_id = p_recipient_id AND deleted_at IS NULL
|
||||
FOR UPDATE;
|
||||
|
||||
IF NOT FOUND THEN
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
SELECT message_id, msg_id, msg_ts, msg_quota, msg_ntf_flag, msg_body
|
||||
INTO msg
|
||||
FROM messages
|
||||
WHERE recipient_id = p_recipient_id
|
||||
ORDER BY message_id ASC LIMIT 1;
|
||||
|
||||
IF NOT FOUND THEN
|
||||
IF q_size != 0 THEN
|
||||
UPDATE msg_queues
|
||||
SET msg_can_write = TRUE,
|
||||
msg_queue_size = 0
|
||||
WHERE recipient_id = p_recipient_id;
|
||||
END IF;
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
IF msg.msg_id = p_msg_id THEN
|
||||
DELETE FROM messages WHERE message_id = msg.message_id;
|
||||
|
||||
msg_deleted := FOUND;
|
||||
IF msg_deleted THEN
|
||||
RETURN QUERY VALUES (msg.msg_id, msg.msg_ts, msg.msg_quota, msg.msg_ntf_flag, msg.msg_body);
|
||||
END IF;
|
||||
|
||||
SELECT msg_id, msg_ts, msg_quota, msg_ntf_flag, msg_body
|
||||
INTO msg
|
||||
FROM messages
|
||||
WHERE recipient_id = p_recipient_id
|
||||
ORDER BY message_id ASC LIMIT 1;
|
||||
|
||||
IF FOUND THEN
|
||||
RETURN QUERY VALUES (msg.msg_id, msg.msg_ts, msg.msg_quota, msg.msg_ntf_flag, msg.msg_body);
|
||||
IF msg_deleted THEN
|
||||
UPDATE msg_queues
|
||||
SET msg_can_write = msg_can_write OR msg_queue_size <= 1,
|
||||
msg_queue_size = GREATEST(msg_queue_size - 1, 0)
|
||||
WHERE recipient_id = p_recipient_id;
|
||||
END IF;
|
||||
ELSIF msg_deleted OR q_size != 0 THEN
|
||||
UPDATE msg_queues
|
||||
SET msg_can_write = TRUE,
|
||||
msg_queue_size = 0
|
||||
WHERE recipient_id = p_recipient_id;
|
||||
END IF;
|
||||
ELSE
|
||||
RETURN QUERY VALUES (msg.msg_id, msg.msg_ts, msg.msg_quota, msg.msg_ntf_flag, msg.msg_body);
|
||||
END IF;
|
||||
END;
|
||||
$$;
|
||||
|
||||
ALTER TABLE msg_queues DROP COLUMN msg_queue_expire;
|
||||
|]
|
||||
|
||||
down_m20260918_expire_messages :: Text
|
||||
down_m20260918_expire_messages =
|
||||
[r|
|
||||
ALTER TABLE msg_queues ADD COLUMN msg_queue_expire boolean NOT NULL DEFAULT FALSE;
|
||||
|
||||
-- ADD COLUMN already defaulted every row to FALSE, so the backfill only needs the rows
|
||||
-- that become TRUE, which avoids rewriting the whole table. The flag is only read to pick
|
||||
-- queues for delete_expired_msgs, and that returns 0 without deleting when msg_queue_size
|
||||
-- is 0, so excluding empty queues here cannot change what expires.
|
||||
UPDATE msg_queues q
|
||||
SET msg_queue_expire = TRUE
|
||||
WHERE msg_queue_size > 0
|
||||
AND EXISTS (SELECT 1 FROM messages m WHERE m.recipient_id = q.recipient_id AND NOT m.msg_quota);
|
||||
|
||||
CREATE INDEX idx_msg_queues_expire ON msg_queues (recipient_id) WHERE deleted_at IS NULL AND msg_queue_expire;
|
||||
|
||||
CREATE INDEX idx_messages_recipient_id_msg_ts ON messages (recipient_id, msg_ts);
|
||||
|
||||
DROP INDEX idx_messages_expire;
|
||||
|
||||
CREATE OR REPLACE FUNCTION delete_expired_msgs(p_recipient_id bytea, p_old_ts bigint) RETURNS bigint
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
DECLARE
|
||||
q_size BIGINT;
|
||||
keep_min_id BIGINT;
|
||||
del_count BIGINT;
|
||||
BEGIN
|
||||
SELECT msg_queue_size INTO q_size
|
||||
FROM msg_queues
|
||||
WHERE recipient_id = p_recipient_id AND deleted_at IS NULL
|
||||
FOR UPDATE SKIP LOCKED;
|
||||
|
||||
IF NOT FOUND OR q_size = 0 THEN
|
||||
RETURN 0;
|
||||
END IF;
|
||||
|
||||
SELECT MIN(message_id) INTO keep_min_id
|
||||
FROM messages WHERE recipient_id = p_recipient_id AND msg_ts >= p_old_ts AND msg_quota = FALSE;
|
||||
|
||||
IF keep_min_id IS NULL THEN
|
||||
DELETE FROM messages WHERE recipient_id = p_recipient_id AND msg_quota = FALSE;
|
||||
ELSE
|
||||
DELETE FROM messages WHERE recipient_id = p_recipient_id AND message_id < keep_min_id AND msg_quota = FALSE;
|
||||
END IF;
|
||||
|
||||
GET DIAGNOSTICS del_count = ROW_COUNT;
|
||||
IF del_count > 0 THEN
|
||||
UPDATE msg_queues
|
||||
SET msg_can_write = msg_can_write OR msg_queue_size <= del_count,
|
||||
msg_queue_expire = msg_queue_size > del_count AND keep_min_id IS NOT NULL,
|
||||
msg_queue_size = GREATEST(msg_queue_size - del_count, 0)
|
||||
WHERE recipient_id = p_recipient_id;
|
||||
END IF;
|
||||
RETURN del_count;
|
||||
END;
|
||||
$$;
|
||||
|
||||
CREATE OR REPLACE FUNCTION write_message(p_recipient_id bytea, p_msg_id bytea, p_msg_ts bigint, p_msg_quota boolean, p_msg_ntf_flag boolean, p_msg_body bytea, p_quota integer) RETURNS TABLE(quota_written boolean, was_empty boolean)
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
DECLARE
|
||||
q_can_write BOOLEAN;
|
||||
q_size BIGINT;
|
||||
BEGIN
|
||||
SELECT msg_can_write, msg_queue_size INTO q_can_write, q_size
|
||||
FROM msg_queues
|
||||
WHERE recipient_id = p_recipient_id AND deleted_at IS NULL
|
||||
FOR UPDATE;
|
||||
|
||||
IF q_can_write OR q_size = 0 THEN
|
||||
quota_written := p_msg_quota OR q_size >= p_quota;
|
||||
was_empty := q_size = 0;
|
||||
|
||||
INSERT INTO messages(recipient_id, msg_id, msg_ts, msg_quota, msg_ntf_flag, msg_body)
|
||||
VALUES (p_recipient_id, p_msg_id, p_msg_ts, quota_written, p_msg_ntf_flag AND NOT quota_written, CASE WHEN quota_written THEN '' :: BYTEA ELSE p_msg_body END);
|
||||
|
||||
UPDATE msg_queues
|
||||
SET msg_can_write = NOT quota_written,
|
||||
msg_queue_expire = TRUE,
|
||||
msg_queue_size = msg_queue_size + 1
|
||||
WHERE recipient_id = p_recipient_id;
|
||||
|
||||
RETURN QUERY VALUES (quota_written, was_empty);
|
||||
END IF;
|
||||
END;
|
||||
$$;
|
||||
|
||||
CREATE OR REPLACE FUNCTION try_del_msg(p_recipient_id bytea, p_msg_id bytea) RETURNS TABLE(r_msg_id bytea, r_msg_ts bigint, r_msg_quota boolean, r_msg_ntf_flag boolean, r_msg_body bytea)
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
DECLARE
|
||||
q_size BIGINT;
|
||||
msg RECORD;
|
||||
BEGIN
|
||||
SELECT msg_queue_size INTO q_size
|
||||
FROM msg_queues
|
||||
WHERE recipient_id = p_recipient_id AND deleted_at IS NULL
|
||||
FOR UPDATE;
|
||||
|
||||
IF NOT FOUND THEN
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
SELECT message_id, msg_id, msg_ts, msg_quota, msg_ntf_flag, msg_body
|
||||
INTO msg
|
||||
FROM messages
|
||||
WHERE recipient_id = p_recipient_id
|
||||
ORDER BY message_id ASC LIMIT 1;
|
||||
|
||||
IF NOT FOUND THEN
|
||||
IF q_size != 0 THEN
|
||||
UPDATE msg_queues
|
||||
SET msg_can_write = TRUE,
|
||||
msg_queue_expire = FALSE,
|
||||
msg_queue_size = 0
|
||||
WHERE recipient_id = p_recipient_id;
|
||||
END IF;
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
IF msg.msg_id = p_msg_id THEN
|
||||
DELETE FROM messages WHERE message_id = msg.message_id;
|
||||
IF FOUND THEN
|
||||
UPDATE msg_queues
|
||||
SET msg_can_write = msg_can_write OR msg_queue_size <= 1,
|
||||
msg_queue_expire = msg_queue_size > 1,
|
||||
msg_queue_size = GREATEST(msg_queue_size - 1, 0)
|
||||
WHERE recipient_id = p_recipient_id;
|
||||
RETURN QUERY VALUES (msg.msg_id, msg.msg_ts, msg.msg_quota, msg.msg_ntf_flag, msg.msg_body);
|
||||
END IF;
|
||||
END IF;
|
||||
END;
|
||||
$$;
|
||||
|
||||
CREATE OR REPLACE FUNCTION try_del_peek_msg(p_recipient_id bytea, p_msg_id bytea) RETURNS TABLE(r_msg_id bytea, r_msg_ts bigint, r_msg_quota boolean, r_msg_ntf_flag boolean, r_msg_body bytea)
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
DECLARE
|
||||
q_size BIGINT;
|
||||
msg RECORD;
|
||||
msg_deleted BOOLEAN;
|
||||
BEGIN
|
||||
SELECT msg_queue_size INTO q_size
|
||||
FROM msg_queues
|
||||
WHERE recipient_id = p_recipient_id AND deleted_at IS NULL
|
||||
FOR UPDATE;
|
||||
|
||||
IF NOT FOUND THEN
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
SELECT message_id, msg_id, msg_ts, msg_quota, msg_ntf_flag, msg_body
|
||||
INTO msg
|
||||
FROM messages
|
||||
WHERE recipient_id = p_recipient_id
|
||||
ORDER BY message_id ASC LIMIT 1;
|
||||
|
||||
IF NOT FOUND THEN
|
||||
IF q_size != 0 THEN
|
||||
UPDATE msg_queues
|
||||
SET msg_can_write = TRUE,
|
||||
msg_queue_expire = FALSE,
|
||||
msg_queue_size = 0
|
||||
WHERE recipient_id = p_recipient_id;
|
||||
END IF;
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
IF msg.msg_id = p_msg_id THEN
|
||||
DELETE FROM messages WHERE message_id = msg.message_id;
|
||||
|
||||
msg_deleted := FOUND;
|
||||
IF msg_deleted THEN
|
||||
RETURN QUERY VALUES (msg.msg_id, msg.msg_ts, msg.msg_quota, msg.msg_ntf_flag, msg.msg_body);
|
||||
END IF;
|
||||
|
||||
SELECT msg_id, msg_ts, msg_quota, msg_ntf_flag, msg_body
|
||||
INTO msg
|
||||
FROM messages
|
||||
WHERE recipient_id = p_recipient_id
|
||||
ORDER BY message_id ASC LIMIT 1;
|
||||
|
||||
IF FOUND THEN
|
||||
RETURN QUERY VALUES (msg.msg_id, msg.msg_ts, msg.msg_quota, msg.msg_ntf_flag, msg.msg_body);
|
||||
IF msg_deleted THEN
|
||||
UPDATE msg_queues
|
||||
SET msg_can_write = msg_can_write OR msg_queue_size <= 1,
|
||||
msg_queue_expire = msg_queue_size > 1,
|
||||
msg_queue_size = GREATEST(msg_queue_size - 1, 0)
|
||||
WHERE recipient_id = p_recipient_id;
|
||||
END IF;
|
||||
ELSIF msg_deleted OR q_size != 0 THEN
|
||||
UPDATE msg_queues
|
||||
SET msg_can_write = TRUE,
|
||||
msg_queue_expire = FALSE,
|
||||
msg_queue_size = 0
|
||||
WHERE recipient_id = p_recipient_id;
|
||||
END IF;
|
||||
ELSE
|
||||
RETURN QUERY VALUES (msg.msg_id, msg.msg_ts, msg.msg_quota, msg.msg_ntf_flag, msg.msg_body);
|
||||
END IF;
|
||||
END;
|
||||
$$;
|
||||
|
||||
DROP PROCEDURE expire_old_messages(bigint, integer);
|
||||
|
||||
CREATE PROCEDURE expire_old_messages(IN p_old_ts bigint, IN batch_size integer, OUT r_expired_msgs_count bigint, OUT r_stored_msgs_count bigint, OUT r_stored_queues bigint)
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
DECLARE
|
||||
rids BYTEA[];
|
||||
rid BYTEA;
|
||||
last_rid BYTEA := '\x';
|
||||
del_count BIGINT;
|
||||
total_deleted BIGINT := 0;
|
||||
BEGIN
|
||||
LOOP
|
||||
SELECT array_agg(recipient_id)
|
||||
INTO rids
|
||||
FROM (
|
||||
SELECT recipient_id
|
||||
FROM msg_queues
|
||||
WHERE deleted_at IS NULL
|
||||
AND msg_queue_expire = TRUE
|
||||
AND recipient_id > last_rid
|
||||
ORDER BY recipient_id ASC
|
||||
LIMIT batch_size
|
||||
) qs;
|
||||
|
||||
EXIT WHEN rids IS NULL OR cardinality(rids) = 0;
|
||||
|
||||
FOREACH rid IN ARRAY rids
|
||||
LOOP
|
||||
BEGIN
|
||||
del_count := delete_expired_msgs(rid, p_old_ts);
|
||||
total_deleted := total_deleted + del_count;
|
||||
EXCEPTION WHEN OTHERS THEN
|
||||
RAISE WARNING 'STORE, expire_old_messages, error expiring queue %: %', encode(rid, 'base64'), SQLERRM;
|
||||
CONTINUE;
|
||||
END;
|
||||
COMMIT;
|
||||
END LOOP;
|
||||
last_rid := rids[cardinality(rids)];
|
||||
END LOOP;
|
||||
|
||||
r_expired_msgs_count := total_deleted;
|
||||
r_stored_msgs_count := (SELECT COUNT(1) FROM messages);
|
||||
r_stored_queues := (SELECT COUNT(1) FROM msg_queues WHERE deleted_at IS NULL);
|
||||
END;
|
||||
$$;
|
||||
|]
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
|
||||
|
||||
|
||||
SET statement_timeout = 0;
|
||||
SET lock_timeout = 0;
|
||||
SET idle_in_transaction_session_timeout = 0;
|
||||
@@ -47,6 +46,7 @@ BEGIN
|
||||
IF del_count > 0 THEN
|
||||
UPDATE msg_queues
|
||||
SET msg_can_write = msg_can_write OR msg_queue_size <= del_count,
|
||||
msg_queue_expire = msg_queue_size > del_count AND keep_min_id IS NOT NULL,
|
||||
msg_queue_size = GREATEST(msg_queue_size - del_count, 0)
|
||||
WHERE recipient_id = p_recipient_id;
|
||||
END IF;
|
||||
@@ -56,39 +56,31 @@ $$;
|
||||
|
||||
|
||||
|
||||
CREATE PROCEDURE smp_server.expire_old_messages(IN p_old_ts bigint, IN batch_size integer, OUT r_expired_msgs_count bigint, OUT r_stored_msgs_count bigint, OUT r_stored_queues bigint)
|
||||
CREATE PROCEDURE smp_server.expire_old_messages(IN p_old_queue bigint, IN p_old_ts bigint, IN batch_size integer, OUT r_expired_msgs_count bigint, OUT r_stored_msgs_count bigint, OUT r_stored_queues bigint)
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
DECLARE
|
||||
rids BYTEA[];
|
||||
rid BYTEA;
|
||||
last_ts BIGINT := -1;
|
||||
last_rid BYTEA := '\x';
|
||||
next_ts BIGINT;
|
||||
next_rid BYTEA;
|
||||
del_count BIGINT;
|
||||
total_deleted BIGINT := 0;
|
||||
BEGIN
|
||||
LOOP
|
||||
-- The page is scanned in (msg_ts, recipient_id) order, so its last row is the next
|
||||
-- keyset cursor. Advancing it past every row read, including queues left unexpired
|
||||
-- because delete_expired_msgs skipped a locked row or raised, is what terminates the
|
||||
-- loop; re-reading from the start instead would repeat those queues forever.
|
||||
SELECT array_agg(DISTINCT recipient_id),
|
||||
(array_agg(msg_ts ORDER BY msg_ts DESC, recipient_id DESC))[1],
|
||||
(array_agg(recipient_id ORDER BY msg_ts DESC, recipient_id DESC))[1]
|
||||
INTO rids, next_ts, next_rid
|
||||
SELECT array_agg(recipient_id)
|
||||
INTO rids
|
||||
FROM (
|
||||
SELECT msg_ts, recipient_id
|
||||
FROM messages
|
||||
WHERE NOT msg_quota
|
||||
AND msg_ts < p_old_ts
|
||||
AND (msg_ts, recipient_id) > (last_ts, last_rid)
|
||||
ORDER BY msg_ts ASC, recipient_id ASC
|
||||
SELECT recipient_id
|
||||
FROM msg_queues
|
||||
WHERE deleted_at IS NULL
|
||||
AND updated_at > p_old_queue
|
||||
AND msg_queue_expire = TRUE
|
||||
AND recipient_id > last_rid
|
||||
ORDER BY recipient_id ASC
|
||||
LIMIT batch_size
|
||||
) m;
|
||||
) qs;
|
||||
|
||||
EXIT WHEN rids IS NULL;
|
||||
EXIT WHEN rids IS NULL OR cardinality(rids) = 0;
|
||||
|
||||
FOREACH rid IN ARRAY rids
|
||||
LOOP
|
||||
@@ -99,11 +91,9 @@ BEGIN
|
||||
RAISE WARNING 'STORE, expire_old_messages, error expiring queue %: %', encode(rid, 'base64'), SQLERRM;
|
||||
CONTINUE;
|
||||
END;
|
||||
COMMIT;
|
||||
END LOOP;
|
||||
COMMIT;
|
||||
|
||||
last_ts := next_ts;
|
||||
last_rid := next_rid;
|
||||
last_rid := rids[cardinality(rids)];
|
||||
END LOOP;
|
||||
|
||||
r_expired_msgs_count := total_deleted;
|
||||
@@ -205,6 +195,7 @@ BEGIN
|
||||
IF q_size != 0 THEN
|
||||
UPDATE msg_queues
|
||||
SET msg_can_write = TRUE,
|
||||
msg_queue_expire = FALSE,
|
||||
msg_queue_size = 0
|
||||
WHERE recipient_id = p_recipient_id;
|
||||
END IF;
|
||||
@@ -216,6 +207,7 @@ BEGIN
|
||||
IF FOUND THEN
|
||||
UPDATE msg_queues
|
||||
SET msg_can_write = msg_can_write OR msg_queue_size <= 1,
|
||||
msg_queue_expire = msg_queue_size > 1,
|
||||
msg_queue_size = GREATEST(msg_queue_size - 1, 0)
|
||||
WHERE recipient_id = p_recipient_id;
|
||||
RETURN QUERY VALUES (msg.msg_id, msg.msg_ts, msg.msg_quota, msg.msg_ntf_flag, msg.msg_body);
|
||||
@@ -253,6 +245,7 @@ BEGIN
|
||||
IF q_size != 0 THEN
|
||||
UPDATE msg_queues
|
||||
SET msg_can_write = TRUE,
|
||||
msg_queue_expire = FALSE,
|
||||
msg_queue_size = 0
|
||||
WHERE recipient_id = p_recipient_id;
|
||||
END IF;
|
||||
@@ -278,12 +271,14 @@ BEGIN
|
||||
IF msg_deleted THEN
|
||||
UPDATE msg_queues
|
||||
SET msg_can_write = msg_can_write OR msg_queue_size <= 1,
|
||||
msg_queue_expire = msg_queue_size > 1,
|
||||
msg_queue_size = GREATEST(msg_queue_size - 1, 0)
|
||||
WHERE recipient_id = p_recipient_id;
|
||||
END IF;
|
||||
ELSIF msg_deleted OR q_size != 0 THEN
|
||||
UPDATE msg_queues
|
||||
SET msg_can_write = TRUE,
|
||||
msg_queue_expire = FALSE,
|
||||
msg_queue_size = 0
|
||||
WHERE recipient_id = p_recipient_id;
|
||||
END IF;
|
||||
@@ -353,6 +348,7 @@ BEGIN
|
||||
|
||||
UPDATE msg_queues
|
||||
SET msg_can_write = NOT quota_written,
|
||||
msg_queue_expire = TRUE,
|
||||
msg_queue_size = msg_queue_size + 1
|
||||
WHERE recipient_id = p_recipient_id;
|
||||
|
||||
@@ -401,8 +397,7 @@ CREATE TABLE smp_server.messages (
|
||||
msg_quota boolean NOT NULL,
|
||||
msg_ntf_flag boolean NOT NULL,
|
||||
msg_body bytea NOT NULL
|
||||
)
|
||||
WITH (autovacuum_vacuum_scale_factor='0.02', autovacuum_analyze_scale_factor='0.01', toast.autovacuum_vacuum_scale_factor='0.02');
|
||||
);
|
||||
|
||||
|
||||
|
||||
@@ -444,9 +439,9 @@ CREATE TABLE smp_server.msg_queues (
|
||||
rcv_service_id bytea,
|
||||
ntf_service_id bytea,
|
||||
msg_can_write boolean DEFAULT true NOT NULL,
|
||||
msg_queue_expire boolean DEFAULT false NOT NULL,
|
||||
msg_queue_size bigint DEFAULT 0 NOT NULL
|
||||
)
|
||||
WITH (fillfactor='80', autovacuum_vacuum_scale_factor='0.02', autovacuum_analyze_scale_factor='0.01', autovacuum_vacuum_cost_limit='1000');
|
||||
);
|
||||
|
||||
|
||||
|
||||
@@ -458,8 +453,7 @@ CREATE TABLE smp_server.services (
|
||||
created_at bigint NOT NULL,
|
||||
queue_count bigint DEFAULT 0 NOT NULL,
|
||||
queue_ids_hash bytea DEFAULT '\x00000000000000000000000000000000'::bytea NOT NULL
|
||||
)
|
||||
WITH (fillfactor='70', autovacuum_vacuum_threshold='1000', autovacuum_vacuum_scale_factor='0');
|
||||
);
|
||||
|
||||
|
||||
|
||||
@@ -488,10 +482,6 @@ ALTER TABLE ONLY smp_server.services
|
||||
|
||||
|
||||
|
||||
CREATE INDEX idx_messages_expire ON smp_server.messages USING btree (msg_ts, recipient_id) WHERE (NOT msg_quota);
|
||||
|
||||
|
||||
|
||||
CREATE INDEX idx_messages_recipient_id_message_id ON smp_server.messages USING btree (recipient_id, message_id);
|
||||
|
||||
|
||||
@@ -500,14 +490,14 @@ CREATE INDEX idx_messages_recipient_id_msg_quota ON smp_server.messages USING bt
|
||||
|
||||
|
||||
|
||||
CREATE INDEX idx_messages_recipient_id_msg_ts ON smp_server.messages USING btree (recipient_id, msg_ts);
|
||||
|
||||
|
||||
|
||||
CREATE UNIQUE INDEX idx_msg_queues_link_id ON smp_server.msg_queues USING btree (link_id);
|
||||
|
||||
|
||||
|
||||
CREATE INDEX idx_msg_queues_notifier_active ON smp_server.msg_queues USING btree (notifier_id) WHERE ((deleted_at IS NULL) AND (notifier_id IS NOT NULL));
|
||||
|
||||
|
||||
|
||||
CREATE UNIQUE INDEX idx_msg_queues_notifier_id ON smp_server.msg_queues USING btree (notifier_id);
|
||||
|
||||
|
||||
@@ -524,6 +514,10 @@ CREATE UNIQUE INDEX idx_msg_queues_sender_id ON smp_server.msg_queues USING btre
|
||||
|
||||
|
||||
|
||||
CREATE INDEX idx_msg_queues_updated_at_recipient_id ON smp_server.msg_queues USING btree (deleted_at, updated_at, msg_queue_expire, recipient_id);
|
||||
|
||||
|
||||
|
||||
CREATE INDEX idx_services_service_role ON smp_server.services USING btree (service_role);
|
||||
|
||||
|
||||
@@ -555,4 +549,3 @@ ALTER TABLE ONLY smp_server.msg_queues
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -18,7 +18,6 @@ module Simplex.Messaging.Server.QueueStore.STM
|
||||
( STMQueueStore (..),
|
||||
STMService (..),
|
||||
foldRcvServiceQueues,
|
||||
withLoadedQueues,
|
||||
setStoreLog,
|
||||
withLog',
|
||||
readQueueRecIO,
|
||||
@@ -92,6 +91,8 @@ instance StoreQueueClass q => QueueStoreClass q (STMQueueStore q) where
|
||||
atomically $ TM.clear senders
|
||||
atomically $ TM.clear notifiers
|
||||
|
||||
loadedQueues = queues
|
||||
{-# INLINE loadedQueues #-}
|
||||
compactQueues _ = pure 0
|
||||
{-# INLINE compactQueues #-}
|
||||
|
||||
@@ -133,7 +134,7 @@ instance StoreQueueClass q => QueueStoreClass q (STMQueueStore q) where
|
||||
hasNotifier = maybe (pure False) (\NtfCreds {notifierId} -> TM.member notifierId notifiers) notifier
|
||||
hasLink = maybe (pure False) (\(lnkId, _) -> TM.member lnkId links) queueData
|
||||
|
||||
getQueue_ :: QueueParty p => STMQueueStore q -> (RecipientId -> QueueRec -> IO q) -> SParty p -> QueueId -> IO (Either ErrorType q)
|
||||
getQueue_ :: QueueParty p => STMQueueStore q -> (Bool -> RecipientId -> QueueRec -> IO q) -> SParty p -> QueueId -> IO (Either ErrorType q)
|
||||
getQueue_ st _ party qId =
|
||||
maybe (Left AUTH) Right <$> case party of
|
||||
SRecipient -> TM.lookupIO qId queues
|
||||
@@ -143,7 +144,7 @@ instance StoreQueueClass q => QueueStoreClass q (STMQueueStore q) where
|
||||
where
|
||||
STMQueueStore {queues, senders, notifiers, links} = st
|
||||
|
||||
getQueues_ :: BatchParty p => STMQueueStore q -> (RecipientId -> QueueRec -> IO q) -> SParty p -> [QueueId] -> IO [Either ErrorType q]
|
||||
getQueues_ :: BatchParty p => STMQueueStore q -> (Bool -> RecipientId -> QueueRec -> IO q) -> SParty p -> [QueueId] -> IO [Either ErrorType q]
|
||||
getQueues_ st _ party qIds = case party of
|
||||
SRecipient -> do
|
||||
qs <- readTVarIO queues
|
||||
@@ -172,7 +173,6 @@ instance StoreQueueClass q => QueueStoreClass q (STMQueueStore q) where
|
||||
rId = recipientId sq
|
||||
qr = queueRec sq
|
||||
add q = case queueData q of
|
||||
_ | isSecuredMsgQueue q -> pure $ Left AUTH
|
||||
Nothing -> addLink
|
||||
Just (lnkId', d') | lnkId' == lnkId && fst d' == fst d -> addLink
|
||||
_ -> pure $ Left AUTH
|
||||
@@ -277,11 +277,9 @@ instance StoreQueueClass q => QueueStoreClass q (STMQueueStore q) where
|
||||
where
|
||||
rId = recipientId sq
|
||||
qr = queueRec sq
|
||||
delete q@QueueRec {senderId, queueData, rcvServiceId} = do
|
||||
delete q@QueueRec {senderId, rcvServiceId} = do
|
||||
writeTVar qr Nothing
|
||||
TM.delete rId $ queues st
|
||||
TM.delete senderId $ senders st
|
||||
forM_ queueData $ \(lnkId, _) -> TM.delete lnkId $ links st
|
||||
mapM_ (removeServiceQueue st serviceRcvQueues rId) rcvServiceId
|
||||
mapM_ (removeNotifier st) $ notifier q
|
||||
pure q
|
||||
@@ -380,11 +378,6 @@ foldRcvServiceQueues st serviceId f acc =
|
||||
where
|
||||
get rId = TM.lookupIO rId (queues st) $>>= \q -> (q,) <$$> readTVarIO (queueRec q)
|
||||
|
||||
withLoadedQueues :: Monoid a => STMQueueStore q -> (q -> IO a) -> IO a
|
||||
withLoadedQueues st f = readTVarIO (queues st) >>= foldM run mempty
|
||||
where
|
||||
run !acc = fmap (acc <>) . f
|
||||
|
||||
withQueueRec :: TVar (Maybe QueueRec) -> (QueueRec -> STM a) -> IO (Either ErrorType a)
|
||||
withQueueRec qr a = atomically $ readQueueRec qr >>= mapM a
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{-# LANGUAGE AllowAmbiguousTypes #-}
|
||||
{-# LANGUAGE BangPatterns #-}
|
||||
{-# LANGUAGE DataKinds #-}
|
||||
{-# LANGUAGE MultiParamTypeClasses #-}
|
||||
{-# LANGUAGE TypeFamilies #-}
|
||||
@@ -8,29 +9,35 @@ module Simplex.Messaging.Server.QueueStore.Types
|
||||
( StoreQueueClass (..),
|
||||
QueueStoreClass (..),
|
||||
EntityCounts (..),
|
||||
withLoadedQueues,
|
||||
) where
|
||||
|
||||
import Control.Concurrent.STM
|
||||
import Control.Monad
|
||||
import Data.Int (Int64)
|
||||
import Data.List.NonEmpty (NonEmpty)
|
||||
import Data.Map.Strict (Map)
|
||||
import Data.Text (Text)
|
||||
import Simplex.Messaging.Protocol
|
||||
import Simplex.Messaging.Server.QueueStore
|
||||
import Simplex.Messaging.SystemTime
|
||||
import Simplex.Messaging.TMap (TMap)
|
||||
|
||||
class StoreQueueClass q where
|
||||
recipientId :: q -> RecipientId
|
||||
queueRec :: q -> TVar (Maybe QueueRec)
|
||||
withQueueLock :: q -> Text -> IO a -> IO a
|
||||
|
||||
class StoreQueueClass q => QueueStoreClass q s where
|
||||
type QueueStoreCfg s
|
||||
newQueueStore :: QueueStoreCfg s -> IO s
|
||||
closeQueueStore :: s -> IO ()
|
||||
getEntityCounts :: s -> IO EntityCounts
|
||||
loadedQueues :: s -> TMap RecipientId q
|
||||
compactQueues :: s -> IO Int64
|
||||
addQueue_ :: s -> (RecipientId -> QueueRec -> IO q) -> RecipientId -> QueueRec -> IO (Either ErrorType q)
|
||||
getQueue_ :: QueueParty p => s -> (RecipientId -> QueueRec -> IO q) -> SParty p -> QueueId -> IO (Either ErrorType q)
|
||||
getQueues_ :: BatchParty p => s -> (RecipientId -> QueueRec -> IO q) -> SParty p -> [QueueId] -> IO [Either ErrorType q]
|
||||
getQueue_ :: QueueParty p => s -> (Bool -> RecipientId -> QueueRec -> IO q) -> SParty p -> QueueId -> IO (Either ErrorType q)
|
||||
getQueues_ :: BatchParty p => s -> (Bool -> RecipientId -> QueueRec -> IO q) -> SParty p -> [QueueId] -> IO [Either ErrorType q]
|
||||
getQueueLinkData :: s -> q -> LinkId -> IO (Either ErrorType QueueLinkData)
|
||||
addQueueLinkData :: s -> q -> LinkId -> QueueLinkData -> IO (Either ErrorType ())
|
||||
deleteQueueLinkData :: s -> q -> IO (Either ErrorType ())
|
||||
@@ -57,3 +64,8 @@ data EntityCounts = EntityCounts
|
||||
rcvServiceQueuesCount :: Int,
|
||||
ntfServiceQueuesCount :: Int
|
||||
}
|
||||
|
||||
withLoadedQueues :: (Monoid a, QueueStoreClass q s) => s -> (q -> IO a) -> IO a
|
||||
withLoadedQueues st f = readTVarIO (loadedQueues st) >>= foldM run mempty
|
||||
where
|
||||
run !acc = fmap (acc <>) . f
|
||||
|
||||
@@ -14,16 +14,19 @@ module Simplex.Messaging.Server.StoreLog.ReadWrite
|
||||
|
||||
import Control.Concurrent.STM
|
||||
import Control.Logger.Simple
|
||||
import Control.Monad
|
||||
import Control.Monad.IO.Class
|
||||
import Control.Monad.Trans.Except
|
||||
import qualified Data.ByteString.Char8 as B
|
||||
import qualified Data.Text as T
|
||||
import Data.Text.Encoding (decodeLatin1)
|
||||
import Simplex.Messaging.Encoding.String
|
||||
import Simplex.Messaging.Protocol (ASubscriberParty (..), ErrorType, RecipientId, SParty (..))
|
||||
import Simplex.Messaging.Server.QueueStore (QueueRec, ServiceRec (..))
|
||||
import Simplex.Messaging.Server.QueueStore.STM (STMQueueStore (..), STMService (..), withLoadedQueues)
|
||||
import Simplex.Messaging.Server.QueueStore.STM (STMQueueStore (..), STMService (..))
|
||||
import Simplex.Messaging.Server.QueueStore.Types
|
||||
import Simplex.Messaging.Server.StoreLog
|
||||
import Simplex.Messaging.Util (tshow, ($>>=))
|
||||
import Simplex.Messaging.Util (tshow)
|
||||
import System.IO
|
||||
|
||||
writeQueueStore :: forall q. StoreQueueClass q => StoreLog 'WriteMode -> STMQueueStore q -> IO ()
|
||||
@@ -65,7 +68,13 @@ readQueueStore tty mkQ f st = readLogLines tty f $ \_ -> processLine
|
||||
printError :: String -> IO ()
|
||||
printError e = B.putStrLn $ "Error parsing log: " <> B.pack e <> " - " <> s
|
||||
withQueue :: forall a. RecipientId -> T.Text -> (q -> IO (Either ErrorType a)) -> IO ()
|
||||
withQueue qId op a = (getQueue_ st mkQ SRecipient qId $>>= a) >>= qError qId op
|
||||
withQueue qId op a = runExceptT go >>= qError qId op
|
||||
where
|
||||
go = do
|
||||
q <- ExceptT $ getQueue_ st (\_ -> mkQ) SRecipient qId
|
||||
liftIO (readTVarIO $ queueRec q) >>= \case
|
||||
Nothing -> logWarn $ logPfx qId op <> "already deleted"
|
||||
Just _ -> void $ ExceptT $ a q
|
||||
qError qId op = \case
|
||||
Left e -> logError $ logPfx qId op <> tshow e
|
||||
Right _ -> pure ()
|
||||
|
||||
@@ -10,21 +10,14 @@ module Simplex.Messaging.SimplexName
|
||||
SimplexTLD (..),
|
||||
SimplexNameType (..),
|
||||
fullDomainName,
|
||||
LabelHash (..),
|
||||
labelHash,
|
||||
boundedNonSpace,
|
||||
shortNameInfoStr,
|
||||
)
|
||||
where
|
||||
|
||||
import Control.Applicative (optional, (<|>))
|
||||
import Crypto.Hash (Digest, hash)
|
||||
import Crypto.Hash.Algorithms (Keccak_256)
|
||||
import qualified Data.Aeson.TH as J
|
||||
import qualified Data.Attoparsec.ByteString.Char8 as A
|
||||
import qualified Data.Attoparsec.Text as AT
|
||||
import qualified Data.ByteArray as BA
|
||||
import qualified Data.ByteArray.Encoding as BAE
|
||||
import Data.ByteString.Char8 (ByteString)
|
||||
import qualified Data.ByteString.Char8 as B
|
||||
import Data.Char (isDigit)
|
||||
@@ -77,20 +70,6 @@ nameLabelP = do
|
||||
-- (Cyrillic а vs ASCII a hash to different on-chain records).
|
||||
isNameLetter c = c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z'
|
||||
|
||||
-- | The registry's key for a label: 32 bytes, as labelOf takes it.
|
||||
newtype LabelHash = LabelHash ByteString
|
||||
deriving (Eq, Show)
|
||||
|
||||
-- | keccak-256 of the lowercased label, as the registry keys it.
|
||||
labelHash :: Text -> LabelHash
|
||||
labelHash label = LabelHash $ BA.convert (hash (encodeUtf8 (T.toLower label)) :: Digest Keccak_256)
|
||||
|
||||
instance StrEncoding LabelHash where
|
||||
strEncode (LabelHash h) = '[' `B.cons` (BAE.convertToBase BAE.Base16 h `B.snoc` ']')
|
||||
strP = do
|
||||
h <- BAE.convertFromBase BAE.Base16 <$?> (A.char '[' *> A.takeWhile (/= ']') <* A.char ']')
|
||||
if B.length h == 32 then pure $ LabelHash h else fail "bad LabelHash"
|
||||
|
||||
-- | Cap the name at 253 bytes (DNS full-domain limit)
|
||||
boundedNonSpace :: A.Parser ByteString
|
||||
boundedNonSpace = do
|
||||
@@ -129,15 +108,12 @@ instance Encoding SimplexDomain where
|
||||
smpP = strP
|
||||
|
||||
fullDomainName :: SimplexDomain -> Text
|
||||
fullDomainName SimplexDomain {nameTLD, domain, subDomain} = T.intercalate "." (reverse subDomain ++ [domain]) <> decodeLatin1 (strEncode nameTLD)
|
||||
|
||||
instance StrEncoding SimplexTLD where
|
||||
strEncode = \case
|
||||
TLDSimplex -> ".simplex"
|
||||
TLDTesting -> ".testing"
|
||||
TLDWeb -> ""
|
||||
strP =
|
||||
".simplex" $> TLDSimplex <|> ".testing" $> TLDTesting <|> pure TLDWeb
|
||||
fullDomainName SimplexDomain {nameTLD, domain, subDomain} = T.intercalate "." (reverse subDomain ++ [domain] ++ tld')
|
||||
where
|
||||
tld' = case nameTLD of
|
||||
TLDSimplex -> ["simplex"]
|
||||
TLDTesting -> ["testing"]
|
||||
TLDWeb -> []
|
||||
|
||||
shortNameInfoStr :: SimplexNameInfo -> Text
|
||||
shortNameInfoStr = \case
|
||||
|
||||
@@ -46,14 +46,13 @@ module Simplex.Messaging.Transport
|
||||
minServerSMPRelayVersion,
|
||||
currentClientSMPRelayVersion,
|
||||
currentServerSMPRelayVersion,
|
||||
shortLinksSMPVersion,
|
||||
serviceCertsSMPVersion,
|
||||
newNtfCredsSMPVersion,
|
||||
clientNoticesSMPVersion,
|
||||
rcvServiceSMPVersion,
|
||||
namesSMPVersion,
|
||||
serverInfoSMPVersion,
|
||||
nameAvailSMPVersion,
|
||||
fwdNoncesSMPVersion,
|
||||
simplexMQVersion,
|
||||
smpBlockSize,
|
||||
TransportConfig (..),
|
||||
@@ -176,8 +175,6 @@ smpBlockSize = 16384
|
||||
-- 19 - service subscriptions to messages (10/20/2025)
|
||||
-- 20 - public namespaces resolver, RSLV command (6/20/2026)
|
||||
-- 21 - server public information in handshake (7/5/2026)
|
||||
-- 22 - RNAME answers name availability as well as the record (7/25/2026)
|
||||
-- 23 - version in forwarded command nonce, random nonce in forwarded responses (10/2/2026)
|
||||
|
||||
data SMPVersion
|
||||
|
||||
@@ -190,8 +187,11 @@ type VersionRangeSMP = VersionRange SMPVersion
|
||||
pattern VersionSMP :: Word16 -> VersionSMP
|
||||
pattern VersionSMP v = Version v
|
||||
|
||||
_shortLinksSMPVersion :: VersionSMP
|
||||
_shortLinksSMPVersion = VersionSMP 15
|
||||
_proxyServerHandshakeSMPVersion :: VersionSMP
|
||||
_proxyServerHandshakeSMPVersion = VersionSMP 14
|
||||
|
||||
shortLinksSMPVersion :: VersionSMP
|
||||
shortLinksSMPVersion = VersionSMP 15
|
||||
|
||||
serviceCertsSMPVersion :: VersionSMP
|
||||
serviceCertsSMPVersion = VersionSMP 16
|
||||
@@ -211,37 +211,29 @@ namesSMPVersion = VersionSMP 20
|
||||
serverInfoSMPVersion :: VersionSMP
|
||||
serverInfoSMPVersion = VersionSMP 21
|
||||
|
||||
-- | RNAME carries availability. A server below this answers RSLV with the
|
||||
-- record alone, and ERR NAME NOT_FOUND for a name that does not resolve.
|
||||
nameAvailSMPVersion :: VersionSMP
|
||||
nameAvailSMPVersion = VersionSMP 22
|
||||
|
||||
fwdNoncesSMPVersion :: VersionSMP
|
||||
fwdNoncesSMPVersion = VersionSMP 23
|
||||
|
||||
minClientSMPRelayVersion :: VersionSMP
|
||||
minClientSMPRelayVersion = VersionSMP 15
|
||||
minClientSMPRelayVersion = VersionSMP 14
|
||||
|
||||
minServerSMPRelayVersion :: VersionSMP
|
||||
minServerSMPRelayVersion = VersionSMP 15
|
||||
minServerSMPRelayVersion = VersionSMP 14
|
||||
|
||||
currentClientSMPRelayVersion :: VersionSMP
|
||||
currentClientSMPRelayVersion = VersionSMP 23
|
||||
currentClientSMPRelayVersion = VersionSMP 21
|
||||
|
||||
currentServerSMPRelayVersion :: VersionSMP
|
||||
currentServerSMPRelayVersion = VersionSMP 23
|
||||
currentServerSMPRelayVersion = VersionSMP 21
|
||||
|
||||
-- Max SMP protocol version to be used in e2e encrypted connection between
|
||||
-- client and server, as defined by SMP proxy. Normally set below the current
|
||||
-- version to prevent client version fingerprinting by the destination relays
|
||||
-- when clients upgrade at different times. Pinned to the current version (23)
|
||||
-- for this release because forwarded commands use the nonces from
|
||||
-- fwdNoncesSMPVersion (23), so the one-version anti-fingerprinting buffer does
|
||||
-- not apply yet; it reappears once the current version advances past 23.
|
||||
-- when clients upgrade at different times. Pinned to the current version (20)
|
||||
-- for this release because proxied name resolution is gated on namesSMPVersion
|
||||
-- (20), so the one-version anti-fingerprinting buffer does not apply yet; it
|
||||
-- reappears once the current version advances past 20.
|
||||
proxiedSMPRelayVersion :: VersionSMP
|
||||
proxiedSMPRelayVersion = VersionSMP 23
|
||||
proxiedSMPRelayVersion = VersionSMP 20
|
||||
|
||||
-- minimal supported protocol version is 15
|
||||
-- minimal supported protocol version is 14
|
||||
supportedClientSMPRelayVRange :: VersionRangeSMP
|
||||
supportedClientSMPRelayVRange = mkVersionRange minClientSMPRelayVersion currentClientSMPRelayVersion
|
||||
|
||||
@@ -866,6 +858,7 @@ forceCertChain :: CertChainPubKey -> CertChainPubKey
|
||||
forceCertChain cert@(CertChainPubKey (X.CertificateChain cc) signedKey) = length (show cc) `seq` show signedKey `seq` cert
|
||||
{-# INLINE forceCertChain #-}
|
||||
|
||||
-- This function is only used with v >= 8, so currently it's a simple record update.
|
||||
-- * Note: it requires updating version-based parameters, to be consistent with smpTHandle_.
|
||||
smpTHParamsSetVersion :: VersionSMP -> THandleParams SMPVersion p -> THandleParams SMPVersion p
|
||||
smpTHParamsSetVersion v params =
|
||||
|
||||
@@ -26,7 +26,6 @@ module Simplex.RemoteControl.Client
|
||||
-- for tests only
|
||||
sendRCPacket,
|
||||
receiveRCPacket,
|
||||
findRCCtrlPairing,
|
||||
) where
|
||||
|
||||
import Control.Applicative ((<|>))
|
||||
@@ -47,7 +46,7 @@ import Data.List.NonEmpty (NonEmpty (..))
|
||||
import qualified Data.List.NonEmpty as L
|
||||
import Data.Maybe (isNothing)
|
||||
import qualified Data.Text as T
|
||||
import Data.Time.Clock.System (SystemTime (..), getSystemTime)
|
||||
import Data.Time.Clock.System (getSystemTime)
|
||||
import Data.Tuple (swap)
|
||||
import Data.Word (Word16)
|
||||
import qualified Data.X509 as X
|
||||
@@ -396,10 +395,8 @@ findRCCtrlPairing :: NonEmpty RCCtrlPairing -> RCEncInvitation -> ExceptT RCErro
|
||||
findRCCtrlPairing pairings RCEncInvitation {dhPubKey, nonce, encInvitation} = do
|
||||
(pairing, signedInvStr) <- liftEither $ decrypt (L.toList pairings)
|
||||
signedInv <- liftEitherWith RCESyntax $ strDecode signedInvStr
|
||||
inv@(RCVerifiedInvitation RCInvitation {dh = invDh, ts}) <- maybe (throwE RCEInvitation) pure $ verifySignedInvitation signedInv
|
||||
inv@(RCVerifiedInvitation RCInvitation {dh = invDh}) <- maybe (throwE RCEInvitation) pure $ verifySignedInvitation signedInv
|
||||
unless (invDh == dhPubKey) $ throwE RCEInvitation
|
||||
now <- systemSeconds <$> liftIO getSystemTime
|
||||
unless (now - 3660 <= systemSeconds ts && systemSeconds ts <= now + 3600) $ throwE RCEInvitation
|
||||
pure (pairing, inv)
|
||||
where
|
||||
decrypt :: [RCCtrlPairing] -> Either RCErrorType (RCCtrlPairing, ByteString)
|
||||
|
||||
@@ -122,22 +122,18 @@ closeListener subscribers sock =
|
||||
joinMulticast :: TMVar Int -> N.Socket -> N.HostAddress -> IO ()
|
||||
joinMulticast subscribers sock group = do
|
||||
now <- atomically $ takeTMVar subscribers
|
||||
if now == 0
|
||||
then
|
||||
setMembership sock group True >>= \case
|
||||
Left e -> atomically (putTMVar subscribers now) >> logError ("setMembership failed " <> tshow e)
|
||||
Right () -> atomically $ putTMVar subscribers (now + 1)
|
||||
else atomically $ putTMVar subscribers (now + 1)
|
||||
when (now == 0) $ do
|
||||
setMembership sock group True >>= \case
|
||||
Left e -> atomically (putTMVar subscribers now) >> logError ("setMembership failed " <> tshow e)
|
||||
Right () -> atomically $ putTMVar subscribers (now + 1)
|
||||
|
||||
partMulticast :: TMVar Int -> N.Socket -> N.HostAddress -> IO ()
|
||||
partMulticast subscribers sock group = do
|
||||
now <- atomically $ takeTMVar subscribers
|
||||
if now == 1
|
||||
then
|
||||
setMembership sock group False >>= \case
|
||||
Left e -> atomically (putTMVar subscribers (now - 1)) >> logError ("setMembership failed " <> tshow e)
|
||||
Right () -> atomically $ putTMVar subscribers (now - 1)
|
||||
else atomically $ putTMVar subscribers (max 0 (now - 1))
|
||||
when (now == 1) $
|
||||
setMembership sock group False >>= \case
|
||||
Left e -> atomically (putTMVar subscribers now) >> logError ("setMembership failed " <> tshow e)
|
||||
Right () -> atomically $ putTMVar subscribers (now - 1)
|
||||
|
||||
listenerHostAddr4 :: UDP.ListenSocket -> N.HostAddress
|
||||
listenerHostAddr4 sock = case UDP.mySockAddr sock of
|
||||
|
||||
@@ -20,8 +20,6 @@ module AgentTests.ConnectionRequestTests
|
||||
|
||||
import AgentTests.EqInstances ()
|
||||
import Data.ByteString (ByteString)
|
||||
import qualified Data.ByteString.Char8 as B
|
||||
import Data.Either (isLeft)
|
||||
import Network.HTTP.Types (urlEncode)
|
||||
import Simplex.Messaging.Agent.Protocol
|
||||
import qualified Simplex.Messaging.Crypto as C
|
||||
@@ -287,9 +285,6 @@ connectionRequestTests =
|
||||
contactAddressV6 #== ("https://simplex.chat/contact#/?v=1-2&smp=" <> url queueStr) -- adjusted to v6
|
||||
contactAddressV6 #== ("https://simplex.chat/contact#/?v=2-2&smp=" <> url queueStr)
|
||||
contactAddressClientData #==# ("simplex:/contact#/?v=6-8&smp=" <> url queueStr <> "&data=" <> url "{\"type\":\"group_link\", \"group_link_id\":\"abc\"}")
|
||||
it "should reject KEM ciphertext without KEM key in e2e params" $
|
||||
strDecode @(RcvE2ERatchetParamsUri 'C.X448) (strEncode testE2ERatchetParams <> "&kem_ct=" <> strEncode (B.replicate 1039 '\0'))
|
||||
`shouldSatisfy` isLeft
|
||||
it "should serialize / parse queue address, connection invitations and contact addresses as binary" $ do
|
||||
smpEncodingTest queue
|
||||
smpEncodingTest queueNoQM -- this passes, no queue mode patch in SMPQueueUri encoding
|
||||
@@ -362,10 +357,6 @@ connectionRequestTests =
|
||||
smpEncodingTest $ AgentServiceRequest [qInfo] Nothing "service request payload"
|
||||
smpEncodingTest $ AgentServiceResponse "service response payload"
|
||||
smpEncodingTest $ AgentRejection "rejected: not allowed"
|
||||
it "should serialize and parse ratchet key info" $ do
|
||||
smpDecode "R" `shouldBe` Right (AgentRatchetInfo RatchetInfo {answeredKeyHash = Nothing})
|
||||
smpDecode "R1\3abcdef" `shouldBe` Right (AgentRatchetInfo RatchetInfo {answeredKeyHash = Just "abc"})
|
||||
smpEncodingTest $ AgentRatchetInfo RatchetInfo {answeredKeyHash = Just "0123456789abcdef0123456789abcdef"}
|
||||
where
|
||||
smpEncodingTest :: (Encoding a, Eq a, Show a, HasCallStack) => a -> Expectation
|
||||
smpEncodingTest a = smpDecode (smpEncode a) `shouldBe` Right a
|
||||
|
||||
@@ -53,7 +53,6 @@ doubleRatchetTests = do
|
||||
it "should propose KEM during agreement, but no shared secret" $ testAlgs testPqX3dhProposeInReply
|
||||
it "should agree shared secret using KEM" $ testAlgs testPqX3dhProposeAccept
|
||||
it "should reject proposed KEM in reply" $ testAlgs testPqX3dhProposeReject
|
||||
it "should agree different PQ associated data with substituted KEM key" $ testAlgs testPqX3dhSubstitutedKem
|
||||
it "should allow second proposal in reply" $ testAlgs testPqX3dhProposeAgain
|
||||
describe "hybrid KEM key agreement errors" $ do
|
||||
it "should fail if reply contains acceptance without proposal" $ testAlgs testPqX3dhAcceptWithoutProposalError
|
||||
@@ -363,7 +362,6 @@ testDecodeV2RatchetJSON :: IO ()
|
||||
testDecodeV2RatchetJSON = do
|
||||
let v2RatchetJSON = "{\"rcVersion\":[2,2],\"rcAD\":\"2GEJrq48TmQse6NR16I-hrI0tSySZQ57E_g46nDceAPRAiF6j0drq26RTE7be6X7uiB4RaGJGf4QRXzcYuVtWw==\",\"rcDHRs\":\"TUM0Q0FRQXdCUVlESzJWdUJDSUVJRkNYbUxtSHQ3SUNfeHpGTi1Qb3ZqTVQ3S2p6XzZlZlBjOG9fRFY2RWxKOQ==\",\"rcRK\":\"BOX2X7YW5qDSp2XknY_lqacSrtDqQNPvS6iJlZIs3G0=\",\"rcNs\":0,\"rcNr\":0,\"rcPN\":0,\"rcNHKs\":\"IMouSkXUvzT_mo0WM-pqEUK09-HTLk9WOTCFQglyQxU=\",\"rcNHKr\":\"g-tus1clYPV0rGlzkf5a959tUqDYQVZ1FpcPeXdKwxI=\"}"
|
||||
Right (r :: Ratchet X25519) <- pure $ J.eitherDecodeStrict' v2RatchetJSON
|
||||
rcVCPQ r `shouldBe` Nothing
|
||||
rcSupportKEM r `shouldBe` PQSupportOff
|
||||
rcEnableKEM r `shouldBe` PQEncOff
|
||||
rcSndKEM r `shouldBe` PQEncOff
|
||||
@@ -419,18 +417,6 @@ testPqX3dhProposeAccept _ = do
|
||||
Right paramsAlice <- runExceptT $ pqX3dhRcv pksAlice e2eBob
|
||||
paramsAlice `compatibleRatchets` paramsBob
|
||||
|
||||
testPqX3dhSubstitutedKem :: forall a. (AlgorithmI a, DhAlgorithm a) => C.SAlgorithm a -> IO ()
|
||||
testPqX3dhSubstitutedKem _ = do
|
||||
g <- C.newRandom
|
||||
let v = currentE2EEncryptVersion
|
||||
(pksAlice@(_, _, Just _), e2eAlice) <- liftIO $ generateRcvE2EParams @a g v PQSupportOn
|
||||
(_, E2ERatchetParams _ _ _ (Just (RKParamsProposed mallorysKem))) <- liftIO $ generateRcvE2EParams @a g v PQSupportOn
|
||||
(pksBob@(_, _, Just _), AE2ERatchetParams _ e2eBob) <- liftIO $ generateSndE2EParams @a g v (Just $ AUseKEM SRKSAccepted $ AcceptKEM mallorysKem)
|
||||
Right (paramsBob, _) <- pure $ pqX3dhSnd pksBob e2eAlice
|
||||
Right (paramsAlice, _) <- runExceptT $ pqX3dhRcv pksAlice e2eBob
|
||||
assocData paramsAlice `shouldBe` assocData paramsBob
|
||||
rcVerifyCodePQ paramsAlice `shouldNotBe` rcVerifyCodePQ paramsBob
|
||||
|
||||
testPqX3dhProposeReject :: forall a. (AlgorithmI a, DhAlgorithm a) => C.SAlgorithm a -> IO ()
|
||||
testPqX3dhProposeReject _ = do
|
||||
g <- C.newRandom
|
||||
@@ -473,9 +459,9 @@ testPqX3dhProposeAgain _ = do
|
||||
|
||||
compatibleRatchets :: (RatchetInitParams, x) -> (RatchetInitParams, x) -> Expectation
|
||||
compatibleRatchets
|
||||
(RatchetInitParams {assocData, rcVerifyCodePQ, ratchetKey, sndHK, rcvNextHK, kemAccepted}, _)
|
||||
(RatchetInitParams {assocData = ad, rcVerifyCodePQ = vcPQ, ratchetKey = rk, sndHK = shk, rcvNextHK = rnhk, kemAccepted = ka}, _) = do
|
||||
assocData == ad && rcVerifyCodePQ == vcPQ && ratchetKey == rk && sndHK == shk && rcvNextHK == rnhk `shouldBe` True
|
||||
(RatchetInitParams {assocData, ratchetKey, sndHK, rcvNextHK, kemAccepted}, _)
|
||||
(RatchetInitParams {assocData = ad, ratchetKey = rk, sndHK = shk, rcvNextHK = rnhk, kemAccepted = ka}, _) = do
|
||||
assocData == ad && ratchetKey == rk && sndHK == shk && rcvNextHK == rnhk `shouldBe` True
|
||||
case (kemAccepted, ka) of
|
||||
(Just RatchetKEMAccepted {rcPQRr, rcPQRss, rcPQRct}, Just RatchetKEMAccepted {rcPQRr = pqk, rcPQRss = pqss, rcPQRct = pqct}) ->
|
||||
pqk /= rcPQRr && pqss == rcPQRss && pqct == rcPQRct `shouldBe` True
|
||||
|
||||
@@ -70,8 +70,8 @@ import Data.ByteString.Char8 (ByteString)
|
||||
import qualified Data.ByteString.Char8 as B
|
||||
import Data.Either (isRight)
|
||||
import Data.Int (Int64)
|
||||
import Data.List (find, isInfixOf, isPrefixOf, isSuffixOf)
|
||||
import Data.List.NonEmpty (NonEmpty (..))
|
||||
import Data.List (find, isPrefixOf, isSuffixOf)
|
||||
import Data.List.NonEmpty (NonEmpty)
|
||||
import qualified Data.Map as M
|
||||
import Data.Maybe (isJust, isNothing)
|
||||
import qualified Data.Set as S
|
||||
@@ -87,12 +87,12 @@ import SMPAgentClient
|
||||
import SMPClient
|
||||
import Simplex.Messaging.Agent hiding (acceptContact, createConnection, deleteConnection, deleteConnections, getConnShortLink, joinConnection, sendMessage, setConnShortLink, subscribeConnection, suspendConnection)
|
||||
import qualified Simplex.Messaging.Agent as A
|
||||
import Simplex.Messaging.Agent.Client (ProtocolTestFailure (..), ProtocolTestStep (..), ServerQueueInfo (..), UserNetworkInfo (..), UserNetworkType (..), sendAgentMessage, waitForUserNetwork)
|
||||
import Simplex.Messaging.Agent.Client (ProtocolTestFailure (..), ProtocolTestStep (..), ServerQueueInfo (..), UserNetworkInfo (..), UserNetworkType (..), waitForUserNetwork)
|
||||
import Simplex.Messaging.Agent.Env.SQLite (AgentConfig (..), Env (..), InitialAgentServers (..), createAgentStore)
|
||||
import Simplex.Messaging.Agent.Protocol hiding (CON, CONF, INFO, REQ, SENT)
|
||||
import qualified Simplex.Messaging.Agent.Protocol as A
|
||||
import Simplex.Messaging.Agent.Store (Connection' (..), SomeConn' (..), StoredRcvQueue (..))
|
||||
import Simplex.Messaging.Agent.Store.AgentStore (deleteRatchetKeyHashesExpired, getConn, getRatchetX3dhKeys)
|
||||
import Simplex.Messaging.Agent.Store.AgentStore (getConn)
|
||||
import Simplex.Messaging.Agent.Store.Common (DBStore (..), withTransaction)
|
||||
import Simplex.Messaging.Agent.Store.Interface
|
||||
import qualified Simplex.Messaging.Agent.Store.DB as DB
|
||||
@@ -112,7 +112,7 @@ import Simplex.Messaging.Server.Information (ServerPublicInfo (..))
|
||||
import Simplex.Messaging.Server.MsgStore.Types (SMSType (..), SQSType (..))
|
||||
import Simplex.Messaging.Server.QueueStore.QueueInfo
|
||||
import Simplex.Messaging.Server.StoreLog (StoreLogRecord (..))
|
||||
import Simplex.Messaging.Transport (ASrvTransport, SMPVersion, VersionSMP, currentServerSMPRelayVersion, minClientSMPRelayVersion, minServerSMPRelayVersion, alpnSupportedSMPHandshakes)
|
||||
import Simplex.Messaging.Transport (ASrvTransport, SMPVersion, VersionSMP, currentServerSMPRelayVersion, minClientSMPRelayVersion, minServerSMPRelayVersion, alpnSupportedSMPHandshakes, supportedServerSMPRelayVRange)
|
||||
import Simplex.Messaging.Transport.Server (TransportServerConfig (..))
|
||||
import Simplex.Messaging.Util (bshow, diffToMicroseconds)
|
||||
import Simplex.Messaging.Version (VersionRange (..))
|
||||
@@ -133,7 +133,9 @@ import Simplex.Messaging.Agent.Store.AgentStore (deleteClientService, getSubscri
|
||||
import qualified Database.PostgreSQL.Simple as PSQL
|
||||
import qualified Simplex.Messaging.Agent.Store.Postgres as Postgres
|
||||
import qualified Simplex.Messaging.Agent.Store.Postgres.Common as Postgres
|
||||
import Simplex.Messaging.Server.MsgStore.Journal (JournalQueue)
|
||||
import Simplex.Messaging.Server.MsgStore.Postgres (PostgresQueue)
|
||||
import Simplex.Messaging.Server.MsgStore.Types (QSType (..))
|
||||
import Simplex.Messaging.Server.QueueStore.Postgres
|
||||
import Simplex.Messaging.Server.QueueStore.Postgres.Migrations
|
||||
import Simplex.Messaging.Server.QueueStore.Types (QueueStoreClass (..))
|
||||
@@ -202,7 +204,7 @@ pattern INFO :: ConnInfo -> AEvent 'AEConn
|
||||
pattern INFO connInfo = A.INFO PQSupportOn connInfo
|
||||
|
||||
pattern REQ :: InvitationId -> NonEmpty SMPServer -> ConnInfo -> AEvent e
|
||||
pattern REQ invId srvs connInfo <- A.REQ invId PQSupportOn srvs connInfo _ _
|
||||
pattern REQ invId srvs connInfo <- A.REQ invId PQSupportOn srvs connInfo _
|
||||
|
||||
pattern CON :: AEvent 'AEConn
|
||||
pattern CON = A.CON PQEncOn
|
||||
@@ -229,11 +231,7 @@ pattern Rcvd' :: AgentMsgId -> AgentMsgId -> AEvent 'AEConn
|
||||
pattern Rcvd' aMsgId rcvdMsgId <- RCVD MsgMeta {integrity = MsgOk, recipient = (aMsgId, _)} [MsgReceipt {agentMsgId = rcvdMsgId, msgRcptStatus = MROk}]
|
||||
|
||||
smpCfgVPrev :: ProtocolClientConfig SMPVersion
|
||||
smpCfgVPrev =
|
||||
(smpCfg agentCfg)
|
||||
{ serverVRange = prevRange $ serverVRange $ smpCfg agentCfg,
|
||||
proxiedRelayVRange = prevRange $ proxiedRelayVRange $ smpCfg agentCfg
|
||||
}
|
||||
smpCfgVPrev = (smpCfg agentCfg) {serverVRange = prevRange $ serverVRange $ smpCfg agentCfg}
|
||||
|
||||
-- ntfCfgVPrev :: ProtocolClientConfig NTFVersion
|
||||
-- ntfCfgVPrev = (ntfCfg agentCfg) {clientALPN = Nothing, serverVRange = V.mkVersionRange (VersionNTF 1) (VersionNTF 1)}
|
||||
@@ -295,7 +293,7 @@ createConnection c userId enableNtfs cMode clientData subMode = do
|
||||
|
||||
joinConnection :: AgentClient -> UserId -> Bool -> ConnectionRequestUri c -> ConnInfo -> SubscriptionMode -> AE (ConnId, SndQueueSecured)
|
||||
joinConnection c userId enableNtfs cReq connInfo subMode = do
|
||||
(connId, _) <- A.prepareConnectionToJoin c userId enableNtfs cReq PQSupportOn
|
||||
connId <- A.prepareConnectionToJoin c userId enableNtfs cReq PQSupportOn
|
||||
sndSecure <- A.joinConnection c NRMInteractive userId connId enableNtfs cReq connInfo PQSupportOn subMode
|
||||
pure (connId, sndSecure)
|
||||
|
||||
@@ -413,6 +411,7 @@ functionalAPITests ps = do
|
||||
describe "should connect via 1-time short link with async join" $ testProxyMatrix ps testInvitationShortLinkAsync
|
||||
describe "should connect via contact short link" $ testProxyMatrix ps testContactShortLink
|
||||
describe "should add short link to existing contact and connect" $ testProxyMatrix ps testAddContactShortLink
|
||||
xdescribe "try to create 1-time short link with prev versions" $ testProxyMatrixWithPrev ps testInvitationShortLinkPrev
|
||||
describe "server restart" $ do
|
||||
it "should get 1-time link data after restart" $ testInvitationShortLinkRestart ps
|
||||
it "should connect via contact short link after restart" $ testContactShortLinkRestart ps
|
||||
@@ -443,9 +442,6 @@ functionalAPITests ps = do
|
||||
it "should expire multiple messages" $ testExpireManyMessages ps
|
||||
it "should expire one message if quota is exceeded" $ testExpireMessageQuota ps
|
||||
it "should expire multiple messages if quota is exceeded" $ testExpireManyMessagesQuota ps
|
||||
describe "joining a full contact address" $ do
|
||||
it "should retry joining until the address has space" $ testJoinFullContactAsync ps
|
||||
it "should fail joining after quota exceeded timeout" $ testJoinFullContactAsyncExpire ps
|
||||
it "should drop message after too many receive attempts" $ testDropMsgAfterRcvAttempts ps
|
||||
#if !defined(dbPostgres)
|
||||
-- TODO [postgres] restore from outdated db backup (we use copyFile/renameFile for sqlite)
|
||||
@@ -460,16 +456,6 @@ functionalAPITests ps = do
|
||||
testRatchetSyncSuspendForeground ps
|
||||
it "should synchronize ratchets when clients start synchronization simultaneously" $
|
||||
testRatchetSyncSimultaneous ps
|
||||
it "should ignore replayed ratchet key after expired hashes are deleted" $
|
||||
testRatchetSyncReplayedKey ps
|
||||
it "should synchronize ratchets when synchronization is forced again" $
|
||||
testRatchetSyncRepeated ps
|
||||
it "should not mark ratchet key as processed when ratchet recreation fails" $
|
||||
testRatchetSyncFailedKeyNotProcessed ps
|
||||
it "should not store reply ratchet key when ratchet recreation fails" $
|
||||
testRatchetSyncFailedRecreationNoReply ps
|
||||
it "should not store ratchet key when starting synchronization fails" $
|
||||
testRatchetSyncStartFailedNoKey ps
|
||||
#endif
|
||||
describe "Subscription mode OnlyCreate" $ do
|
||||
it "messages delivered only when polled" $
|
||||
@@ -595,9 +581,9 @@ functionalAPITests ps = do
|
||||
it "should pass with correct password" $ testSMPServerConnectionTest ps auth (srv auth) `shouldReturn` Right (Just (Right testServerInformation))
|
||||
it "should fail without password" $ testSMPServerConnectionTest ps auth (srv Nothing) `shouldReturn` Left authErr
|
||||
it "should fail with incorrect password" $ testSMPServerConnectionTest ps auth (srv $ Just "wrong") `shouldReturn` Left authErr
|
||||
describe "getConnectionVerifyCodes" $
|
||||
it "should return the same codes for both peers" $
|
||||
withSmpServer ps testConnectionVerifyCodes
|
||||
describe "getRatchetAdHash" $
|
||||
it "should return the same data for both peers" $
|
||||
withSmpServer ps testRatchetAdHash
|
||||
describe "Delivery receipts" $ do
|
||||
it "should send and receive delivery receipt" $ withSmpServer ps testDeliveryReceipts
|
||||
it "send delivery receipts concurrently with messages" $ testDeliveryReceiptsConcurrent ps
|
||||
@@ -674,6 +660,19 @@ testProxyMatrix ps runTest = do
|
||||
it "2 servers, directly" $ withSmpServers2 ps $ withAgentClientsServers2 (agentCfg, initAgentServers) (agentCfg, initAgentServers2) $ runTest False
|
||||
it "2 servers, via proxy" $ withSmpServersProxy2 ps $ withAgentClientsServers2 (agentCfg, initAgentServersProxy) (agentCfg, initAgentServersProxy2) $ runTest True
|
||||
|
||||
testProxyMatrixWithPrev :: HasCallStack => (ASrvTransport, AStoreType) -> (Bool -> Bool -> AgentClient -> AgentClient -> IO ()) -> Spec
|
||||
testProxyMatrixWithPrev ps@(t, msType@(ASType qs _ms)) runTest = do
|
||||
it "2 servers, directly, curr clients, prev servers" $ withSmpServers2Prev $ withAgentClientsServers2 (agentCfg, initAgentServers) (agentCfg, initAgentServers2) $ runTest False True
|
||||
it "2 servers, via proxy, curr clients, prev servers" $ withSmpServersProxy2Prev $ withAgentClientsServers2 (agentCfg, initAgentServersProxy) (agentCfg, initAgentServersProxy2) $ runTest True True
|
||||
it "2 servers, directly, prev clients, curr servers" $ withSmpServers2 ps $ withAgentClientsServers2 (agentCfgVPrevPQ, initAgentServers) (agentCfgVPrevPQ, initAgentServers2) $ runTest False False
|
||||
it "2 servers, via proxy, prev clients, curr servers" $ withSmpServersProxy2 ps $ withAgentClientsServers2 (agentCfgVPrevPQ, initAgentServersProxy) (agentCfgVPrevPQ, initAgentServersProxy2) $ runTest True False
|
||||
where
|
||||
prev cfg' = updateCfg cfg' $ \cfg_ -> cfg_ {smpServerVRange = prevRange supportedServerSMPRelayVRange}
|
||||
withSmpServers2Prev a = withServers2 (prev $ cfgMS msType) (prev $ cfgJ2QS qs) a
|
||||
withSmpServersProxy2Prev a = withServers2 (prev $ proxyCfgMS msType) (prev $ proxyCfgJ2QS qs) a
|
||||
withServers2 cfg1 cfg2 a =
|
||||
withSmpServerConfigOn t cfg1 testPort $ \_ -> withSmpServerConfigOn t cfg2 testPort2 $ \_ -> a
|
||||
|
||||
testPQMatrix2 :: HasCallStack => (ASrvTransport, AStoreType) -> (HasCallStack => (AgentClient, InitialKeys) -> (AgentClient, PQSupport) -> AgentMsgId -> IO ()) -> Spec
|
||||
testPQMatrix2 = pqMatrix2_ True
|
||||
|
||||
@@ -756,7 +755,7 @@ runAgentClientTestPQ :: HasCallStack => Bool -> (AgentClient, InitialKeys) -> (A
|
||||
runAgentClientTestPQ viaProxy (alice, aPQ) (bob, bPQ) baseId =
|
||||
runRight_ $ do
|
||||
(bobId, CCLink qInfo Nothing) <- A.createConnection alice NRMInteractive 1 True True SCMInvitation Nothing Nothing aPQ False SMSubscribe
|
||||
(aliceId, _) <- A.prepareConnectionToJoin bob 1 True qInfo bPQ
|
||||
aliceId <- A.prepareConnectionToJoin bob 1 True qInfo bPQ
|
||||
sqSecured' <- A.joinConnection bob NRMInteractive 1 aliceId True qInfo "bob's connInfo" bPQ SMSubscribe
|
||||
liftIO $ sqSecured' `shouldBe` True
|
||||
("", _, A.CONF confId pqSup' _ "bob's connInfo") <- get alice
|
||||
@@ -985,12 +984,12 @@ runAgentClientContactTestPQ :: HasCallStack => SndQueueSecured -> Bool -> (Agent
|
||||
runAgentClientContactTestPQ sqSecured viaProxy (alice, aPQ) (bob, bPQ) baseId =
|
||||
runRight_ $ do
|
||||
(_, CCLink qInfo Nothing) <- A.createConnection alice NRMInteractive 1 True True SCMContact Nothing Nothing aPQ False SMSubscribe
|
||||
(aliceId, _) <- A.prepareConnectionToJoin bob 1 True qInfo bPQ
|
||||
aliceId <- A.prepareConnectionToJoin bob 1 True qInfo bPQ
|
||||
sqSecuredJoin <- A.joinConnection bob NRMInteractive 1 aliceId True qInfo "bob's connInfo" bPQ SMSubscribe
|
||||
liftIO $ sqSecuredJoin `shouldBe` False -- joining via contact address connection
|
||||
("", _, A.REQ invId pqSup' _ "bob's connInfo" _ _) <- get alice
|
||||
("", _, A.REQ invId pqSup' _ "bob's connInfo" _) <- get alice
|
||||
liftIO $ pqSup' `shouldBe` PQSupportOn
|
||||
(bobId, _) <- A.prepareConnectionToAccept alice 1 True invId (CR.connPQEncryption aPQ)
|
||||
bobId <- A.prepareConnectionToAccept alice 1 True invId (CR.connPQEncryption aPQ)
|
||||
sqSecured' <- acceptContact alice 1 bobId True invId "alice's connInfo" (CR.connPQEncryption aPQ) SMSubscribe
|
||||
liftIO $ sqSecured' `shouldBe` sqSecured
|
||||
("", _, A.CONF confId pqSup'' _ "alice's connInfo") <- get bob
|
||||
@@ -1054,7 +1053,7 @@ runAgentClientContactDRTest_ asyncAccept asyncJoin addrIK useDR bPQ ps = withSmp
|
||||
Nothing -> expectationFailure "address must advertise DR ratchet keys"
|
||||
-- classic (non-DR) join drops the address keys from the request
|
||||
let connReqJoin = if useDR then connReq' else case connReq' of CRContactUri d _ -> CRContactUri d Nothing
|
||||
(aliceId, _) <- A.prepareConnectionToJoin bob 1 True connReqJoin bPQ
|
||||
aliceId <- A.prepareConnectionToJoin bob 1 True connReqJoin bPQ
|
||||
if asyncJoin
|
||||
then do
|
||||
A.joinConnectionAsync bob "join" False aliceId True connReqJoin "bob's connInfo" bPQ SMSubscribe
|
||||
@@ -1062,9 +1061,9 @@ runAgentClientContactDRTest_ asyncAccept asyncJoin addrIK useDR bPQ ps = withSmp
|
||||
else do
|
||||
sqSecuredJoin <- A.joinConnection bob NRMInteractive 1 aliceId True connReqJoin "bob's connInfo" bPQ SMSubscribe
|
||||
liftIO $ sqSecuredJoin `shouldBe` False
|
||||
("", _, A.REQ invId reqPQ _ "bob's connInfo" _ _) <- get alice
|
||||
("", _, A.REQ invId reqPQ _ "bob's connInfo" _) <- get alice
|
||||
liftIO $ reqPQ `shouldBe` PQSupportOn
|
||||
(bobId, _) <- A.prepareConnectionToAccept alice 1 True invId (CR.connPQEncryption addrIK)
|
||||
bobId <- A.prepareConnectionToAccept alice 1 True invId (CR.connPQEncryption addrIK)
|
||||
if asyncAccept
|
||||
then do
|
||||
acceptContactAsync alice "accept" bobId True invId "alice's connInfo" (CR.connPQEncryption addrIK) SMSubscribe
|
||||
@@ -1094,10 +1093,10 @@ runCreateConnectionDRTest_ asyncNew addrIK bPQ ps = withSmpServer ps $ withAgent
|
||||
liftIO $ case addrKeys_ of
|
||||
Just (_, CR.E2ERatchetParamsUri _ _ _ kem_) -> isJust kem_ `shouldBe` supportPQ (CR.initialPQEncryption False addrIK)
|
||||
Nothing -> expectationFailure "createConnection must advertise DR ratchet keys"
|
||||
(aliceId, _) <- A.prepareConnectionToJoin bob 1 True connReq bPQ
|
||||
aliceId <- A.prepareConnectionToJoin bob 1 True connReq bPQ
|
||||
void $ A.joinConnection bob NRMInteractive 1 aliceId True connReq "bob's connInfo" bPQ SMSubscribe
|
||||
("", _, A.REQ invId _ _ "bob's connInfo" _ _) <- get alice
|
||||
(bobId, _) <- A.prepareConnectionToAccept alice 1 True invId (CR.connPQEncryption addrIK)
|
||||
("", _, A.REQ invId _ _ "bob's connInfo" _) <- get alice
|
||||
bobId <- A.prepareConnectionToAccept alice 1 True invId (CR.connPQEncryption addrIK)
|
||||
void $ acceptContact alice 1 bobId True invId "alice's connInfo" (CR.connPQEncryption addrIK) SMSubscribe
|
||||
("", _, A.CONF confId _ _ "alice's connInfo") <- get bob
|
||||
allowConfirmGreet alice bobId bob aliceId confId addrIK pqEnc
|
||||
@@ -1119,10 +1118,10 @@ testContactDRMatrix ps = do
|
||||
|
||||
joinContactDR :: HasCallStack => AgentClient -> AgentClient -> ConnectionRequestUri 'CMContact -> InitialKeys -> PQEncryption -> ExceptT AgentErrorType IO ()
|
||||
joinContactDR alice requester connReq addrIK pqEnc = do
|
||||
(aliceId, _) <- A.prepareConnectionToJoin requester 1 True connReq PQSupportOn
|
||||
aliceId <- A.prepareConnectionToJoin requester 1 True connReq PQSupportOn
|
||||
void $ A.joinConnection requester NRMInteractive 1 aliceId True connReq "bob's connInfo" PQSupportOn SMSubscribe
|
||||
("", _, A.REQ invId _ _ "bob's connInfo" _ _) <- get alice
|
||||
(reqId, _) <- A.prepareConnectionToAccept alice 1 True invId (CR.connPQEncryption addrIK)
|
||||
("", _, A.REQ invId _ _ "bob's connInfo" _) <- get alice
|
||||
reqId <- A.prepareConnectionToAccept alice 1 True invId (CR.connPQEncryption addrIK)
|
||||
void $ acceptContact alice 1 reqId True invId "alice's connInfo" (CR.connPQEncryption addrIK) SMSubscribe
|
||||
("", _, A.CONF confId _ _ "alice's connInfo") <- get requester
|
||||
allowConfirmGreet alice reqId requester aliceId confId addrIK pqEnc
|
||||
@@ -1150,7 +1149,7 @@ testAddressKeyRotation ps = withSmpServer ps $ withAgentClients3 $ \alice bob ca
|
||||
joinContactDR alice carol connReq2 addrIK pqEnc
|
||||
void $ A.setConnShortLink alice NRMInteractive addrConnId SCMContact userLinkData Nothing True (Just addrIK)
|
||||
void $ A.setConnShortLink alice NRMInteractive addrConnId SCMContact userLinkData Nothing True (Just addrIK)
|
||||
(aId, _) <- A.prepareConnectionToJoin bob 1 True connReq1 PQSupportOn
|
||||
aId <- A.prepareConnectionToJoin bob 1 True connReq1 PQSupportOn
|
||||
void $ A.joinConnection bob NRMInteractive 1 aId True connReq1 "bob's connInfo" PQSupportOn SMSubscribe
|
||||
get alice =##> \case ("", _, A.ERR _) -> True; _ -> False
|
||||
|
||||
@@ -1193,11 +1192,11 @@ testAddressUpdatePreservesDRKeys ps = withSmpServer ps $ withAgentClients2 $ \al
|
||||
liftIO $ shortLink' `shouldBe` shortLink
|
||||
(_, ContactLinkData _ updated, connReq') <- getConnShortLink bob 1 shortLink
|
||||
liftIO $ ratchetKeys updated `shouldBe` ratchetKeys published
|
||||
(aliceId, _) <- A.prepareConnectionToJoin bob 1 True connReq' bPQ
|
||||
aliceId <- A.prepareConnectionToJoin bob 1 True connReq' bPQ
|
||||
sqSecuredJoin <- A.joinConnection bob NRMInteractive 1 aliceId True connReq' "bob's connInfo" bPQ SMSubscribe
|
||||
liftIO $ sqSecuredJoin `shouldBe` False
|
||||
("", _, A.REQ invId _ _ "bob's connInfo" _ _) <- get alice
|
||||
(bobId, _) <- A.prepareConnectionToAccept alice 1 True invId (CR.connPQEncryption addrIK)
|
||||
("", _, A.REQ invId _ _ "bob's connInfo" _) <- get alice
|
||||
bobId <- A.prepareConnectionToAccept alice 1 True invId (CR.connPQEncryption addrIK)
|
||||
_ <- acceptContact alice 1 bobId True invId "alice's connInfo" (CR.connPQEncryption addrIK) SMSubscribe
|
||||
("", _, A.CONF confId _ _ "alice's connInfo") <- get bob
|
||||
allowConfirmGreet alice bobId bob aliceId confId addrIK pqEnc
|
||||
@@ -1217,10 +1216,10 @@ testAcceptContactDRResumeAfterOffline ps = withAgentClients2 $ \alice bob -> do
|
||||
(ccLink@(CCLink _ (Just shortLink)), preparedParams) <- A.prepareConnectionLink alice 1 rootKey linkEntId True Nothing connIK True Nothing
|
||||
_ <- A.createConnectionForLink alice NRMInteractive 1 True ccLink preparedParams (UserContactLinkData userCtData) SMSubscribe
|
||||
(_, _, connReq') <- getConnShortLink bob 1 shortLink
|
||||
(aId, _) <- A.prepareConnectionToJoin bob 1 True connReq' PQSupportOn
|
||||
aId <- A.prepareConnectionToJoin bob 1 True connReq' PQSupportOn
|
||||
_ <- A.joinConnection bob NRMInteractive 1 aId True connReq' "bob's connInfo" PQSupportOn SMSubscribe
|
||||
("", _, A.REQ invId _ _ "bob's connInfo" _ _) <- get alice
|
||||
(bId, _) <- A.prepareConnectionToAccept alice 1 True invId (CR.connPQEncryption addrIK)
|
||||
("", _, A.REQ invId _ _ "bob's connInfo" _) <- get alice
|
||||
bId <- A.prepareConnectionToAccept alice 1 True invId (CR.connPQEncryption addrIK)
|
||||
pure (bId, invId, aId)
|
||||
("", "", DOWN _ _) <- nGet alice
|
||||
("", "", DOWN _ _) <- nGet bob
|
||||
@@ -1245,12 +1244,12 @@ runAgentClientContactTestPQ3 viaProxy (alice, aPQ) (bob, bPQ) (tom, tPQ) baseId
|
||||
where
|
||||
msgId = subtract baseId . fst
|
||||
connectViaContact b pq qInfo = do
|
||||
(aId, _) <- A.prepareConnectionToJoin b 1 True qInfo pq
|
||||
aId <- A.prepareConnectionToJoin b 1 True qInfo pq
|
||||
sqSecuredJoin <- A.joinConnection b NRMInteractive 1 aId True qInfo "bob's connInfo" pq SMSubscribe
|
||||
liftIO $ sqSecuredJoin `shouldBe` False -- joining via contact address connection
|
||||
("", _, A.REQ invId pqSup' _ "bob's connInfo" _ _) <- get alice
|
||||
("", _, A.REQ invId pqSup' _ "bob's connInfo" _) <- get alice
|
||||
liftIO $ pqSup' `shouldBe` PQSupportOn
|
||||
(bId, _) <- A.prepareConnectionToAccept alice 1 True invId (CR.connPQEncryption aPQ)
|
||||
bId <- A.prepareConnectionToAccept alice 1 True invId (CR.connPQEncryption aPQ)
|
||||
sqSecuredAccept <- acceptContact alice 1 bId True invId "alice's connInfo" (CR.connPQEncryption aPQ) SMSubscribe
|
||||
liftIO $ sqSecuredAccept `shouldBe` True
|
||||
("", _, A.CONF confId pqSup'' _ "alice's connInfo") <- get b
|
||||
@@ -1291,10 +1290,10 @@ testRejectContactRequest :: HasCallStack => IO ()
|
||||
testRejectContactRequest =
|
||||
withAgentClients2 $ \alice bob -> runRight_ $ do
|
||||
(_addrConnId, CCLink qInfo Nothing) <- A.createConnection alice NRMInteractive 1 True True SCMContact Nothing Nothing IKPQOn False SMSubscribe
|
||||
(aliceId, _) <- A.prepareConnectionToJoin bob 1 True qInfo PQSupportOn
|
||||
aliceId <- A.prepareConnectionToJoin bob 1 True qInfo PQSupportOn
|
||||
sqSecured <- A.joinConnection bob NRMInteractive 1 aliceId True qInfo "bob's connInfo" PQSupportOn SMSubscribe
|
||||
liftIO $ sqSecured `shouldBe` False -- joining via contact address connection
|
||||
("", _, A.REQ invId PQSupportOn _ "bob's connInfo" _ _) <- get alice
|
||||
("", _, A.REQ invId PQSupportOn _ "bob's connInfo" _) <- get alice
|
||||
Left (A.CMD PROHIBITED _) <- tryError $ rejectContact alice NRMInteractive 1 invId (Just "no rejection without double ratchet")
|
||||
rejectContact alice NRMInteractive 1 invId Nothing
|
||||
liftIO $ noMessages bob "nothing delivered to bob"
|
||||
@@ -1304,9 +1303,9 @@ testRejectContactRequestDR =
|
||||
withAgentClients2 $ \alice bob -> runRight_ $ do
|
||||
let userLinkData = UserContactLinkData UserContactData {direct = True, owners = [], relays = [], userData = UserLinkData "test user data", ratchetKeys = Nothing}
|
||||
(_addrConnId, CCLink connReq _) <- A.createConnection alice NRMInteractive 1 True True SCMContact (Just userLinkData) Nothing IKPQOn True SMSubscribe
|
||||
(aliceId, _) <- A.prepareConnectionToJoin bob 1 True connReq PQSupportOn
|
||||
aliceId <- A.prepareConnectionToJoin bob 1 True connReq PQSupportOn
|
||||
void $ A.joinConnection bob NRMInteractive 1 aliceId True connReq "bob's connInfo" PQSupportOn SMSubscribe
|
||||
("", _, A.REQ invId _ _ "bob's connInfo" _ _) <- get alice
|
||||
("", _, A.REQ invId _ _ "bob's connInfo" _) <- get alice
|
||||
rejectContact alice NRMInteractive 1 invId (Just "not now")
|
||||
("", _, A.RJCT "not now") <- get bob
|
||||
pure ()
|
||||
@@ -1316,9 +1315,9 @@ testRejectContactRequestDRAsync =
|
||||
withAgentClients2 $ \alice bob -> runRight_ $ do
|
||||
let userLinkData = UserContactLinkData UserContactData {direct = True, owners = [], relays = [], userData = UserLinkData "test user data", ratchetKeys = Nothing}
|
||||
(_addrConnId, CCLink connReq _) <- A.createConnection alice NRMInteractive 1 True True SCMContact (Just userLinkData) Nothing IKPQOn True SMSubscribe
|
||||
(aliceId, _) <- A.prepareConnectionToJoin bob 1 True connReq PQSupportOn
|
||||
aliceId <- A.prepareConnectionToJoin bob 1 True connReq PQSupportOn
|
||||
void $ A.joinConnection bob NRMInteractive 1 aliceId True connReq "bob's connInfo" PQSupportOn SMSubscribe
|
||||
("", _, A.REQ invId _ _ "bob's connInfo" _ _) <- get alice
|
||||
("", _, A.REQ invId _ _ "bob's connInfo" _) <- get alice
|
||||
rejectContactAsync alice "1" 1 invId (Just "not now")
|
||||
("", _, A.RJCT "not now") <- get bob
|
||||
pure ()
|
||||
@@ -1431,7 +1430,7 @@ testUpdateConnectionUserId =
|
||||
(connId, qInfo) <- createConnection alice 1 True SCMInvitation Nothing SMSubscribe
|
||||
newUserId <- createUser alice False [noAuthSrvCfg testSMPServer] [noAuthSrvCfg testXFTPServer]
|
||||
_ <- changeConnectionUser alice 1 connId newUserId
|
||||
(aliceId, _) <- A.prepareConnectionToJoin bob 1 True qInfo PQSupportOn
|
||||
aliceId <- A.prepareConnectionToJoin bob 1 True qInfo PQSupportOn
|
||||
sqSecured' <- A.joinConnection bob NRMInteractive 1 aliceId True qInfo "bob's connInfo" PQSupportOn SMSubscribe
|
||||
liftIO $ sqSecured' `shouldBe` True
|
||||
("", _, A.CONF confId pqSup' _ "bob's connInfo") <- get alice
|
||||
@@ -1552,7 +1551,7 @@ testAllowConnectionClientRestart ps@(t, ASType qsType _) = do
|
||||
bob <- getSMPAgentClient' 2 agentCfg initAgentServersSrv2 testDB2
|
||||
withSmpServerStoreLogOn ps testPort $ \_ -> do
|
||||
(aliceId, bobId, confId) <-
|
||||
withSmpServerConfigOn t (cfgS2QS qsType) testPort2 $ \_ -> do
|
||||
withSmpServerConfigOn t (cfgJ2QS qsType) testPort2 $ \_ -> do
|
||||
runRight $ do
|
||||
(bobId, qInfo) <- createConnection alice 1 True SCMInvitation Nothing SMSubscribe
|
||||
(aliceId, sqSecured) <- joinConnection bob 1 True qInfo "bob's connInfo" SMSubscribe
|
||||
@@ -1574,7 +1573,7 @@ testAllowConnectionClientRestart ps@(t, ASType qsType _) = do
|
||||
alice2 <- getSMPAgentClient' 3 agentCfg initAgentServers testDB
|
||||
runRight_ $ subscribeConnection alice2 bobId
|
||||
threadDelay 500000
|
||||
withSmpServerConfigOn t (cfgS2QS qsType) testPort2 $ \_ -> do
|
||||
withSmpServerConfigOn t (cfgJ2QS qsType) testPort2 $ \_ -> do
|
||||
runRight $ do
|
||||
("", "", UP _ _) <- nGet bob
|
||||
get alice2 ##> ("", bobId, CON)
|
||||
@@ -1590,7 +1589,7 @@ testInvitationErrors ps restart = do
|
||||
b <- getAgentB
|
||||
(bId, cReq) <- withServer1 ps $ runRight $ createConnection a 1 True SCMInvitation Nothing SMSubscribe
|
||||
("", "", DOWN _ [_]) <- nGet a
|
||||
(aId, _) <- runRight $ A.prepareConnectionToJoin b 1 True cReq PQSupportOn
|
||||
aId <- runRight $ A.prepareConnectionToJoin b 1 True cReq PQSupportOn
|
||||
-- fails to secure the queue on testPort
|
||||
BROKER srv (NETWORK _) <- runLeft $ A.joinConnection b NRMInteractive 1 aId True cReq "bob's connInfo" PQSupportOn SMSubscribe
|
||||
(testPort `isSuffixOf` srv) `shouldBe` True
|
||||
@@ -1660,7 +1659,7 @@ testContactErrors ps restart = do
|
||||
b <- getAgentB
|
||||
(contactId, cReq) <- withServer1 ps $ runRight $ createConnection a 1 True SCMContact Nothing SMSubscribe
|
||||
("", "", DOWN _ [_]) <- nGet a
|
||||
(aId, _) <- runRight $ A.prepareConnectionToJoin b 1 True cReq PQSupportOn
|
||||
aId <- runRight $ A.prepareConnectionToJoin b 1 True cReq PQSupportOn
|
||||
-- fails to create queue on testPort2
|
||||
BROKER srv2 (NETWORK _) <- runLeft $ A.joinConnection b NRMInteractive 1 aId True cReq "bob's connInfo" PQSupportOn SMSubscribe
|
||||
(testPort2 `isSuffixOf` srv2) `shouldBe` True
|
||||
@@ -1682,10 +1681,10 @@ testContactErrors ps restart = do
|
||||
Right r -> error $ "unexpected result " <> show r
|
||||
Left _ -> putStrLn "retrying send" >> threadDelay 200000 >> loopSend
|
||||
loopSend
|
||||
("", _, A.REQ invId PQSupportOn _ "bob's connInfo" _ _) <- get a
|
||||
("", _, A.REQ invId PQSupportOn _ "bob's connInfo" _) <- get a
|
||||
pure invId
|
||||
("", "", DOWN _ [_]) <- nGet a
|
||||
(bId, _) <- runRight $ A.prepareConnectionToAccept a 1 True invId PQSupportOn
|
||||
bId <- runRight $ A.prepareConnectionToAccept a 1 True invId PQSupportOn
|
||||
withServer2 ps $ do
|
||||
("", "", UP _ [_]) <- nGet b''
|
||||
let loopSecure = do
|
||||
@@ -1737,7 +1736,7 @@ withServer1 :: (ASrvTransport, AStoreType) -> IO a -> IO a
|
||||
withServer1 ps = withSmpServerStoreLogOn ps testPort . const
|
||||
|
||||
withServer2 :: (ASrvTransport, AStoreType) -> IO a -> IO a
|
||||
withServer2 (t, ASType qsType _) = withSmpServerConfigOn t (cfgS2QS qsType) testPort2 . const
|
||||
withServer2 (t, ASType qsType _) = withSmpServerConfigOn t (cfgJ2QS qsType) testPort2 . const
|
||||
|
||||
testInvitationShortLink :: HasCallStack => Bool -> AgentClient -> AgentClient -> IO ()
|
||||
testInvitationShortLink viaProxy a b =
|
||||
@@ -1765,7 +1764,7 @@ testInvitationShortLink viaProxy a b =
|
||||
|
||||
testJoinConn_ :: Bool -> Bool -> AgentClient -> ConnId -> AgentClient -> ConnectionRequestUri c -> ExceptT AgentErrorType IO ()
|
||||
testJoinConn_ viaProxy sndSecure a bId b connReq = do
|
||||
(aId, _) <- A.prepareConnectionToJoin b 1 True connReq PQSupportOn
|
||||
aId <- A.prepareConnectionToJoin b 1 True connReq PQSupportOn
|
||||
sndSecure' <- A.joinConnection b NRMInteractive 1 aId True connReq "bob's connInfo" PQSupportOn SMSubscribe
|
||||
liftIO $ sndSecure' `shouldBe` sndSecure
|
||||
("", _, CONF confId _ "bob's connInfo") <- get a
|
||||
@@ -1775,6 +1774,14 @@ testJoinConn_ viaProxy sndSecure a bId b connReq = do
|
||||
get b ##> ("", aId, CON)
|
||||
exchangeGreetingsViaProxy viaProxy a bId b aId
|
||||
|
||||
testInvitationShortLinkPrev :: HasCallStack => Bool -> Bool -> AgentClient -> AgentClient -> IO ()
|
||||
testInvitationShortLinkPrev viaProxy sndSecure a b = runRight_ $ do
|
||||
let userData = UserLinkData "some user data"
|
||||
newLinkData = UserInvLinkData userData
|
||||
-- can't create short link with previous version
|
||||
(bId, CCLink connReq Nothing) <- A.createConnection a NRMInteractive 1 True True SCMInvitation (Just newLinkData) Nothing CR.IKPQOn False SMSubscribe
|
||||
testJoinConn_ viaProxy sndSecure a bId b connReq
|
||||
|
||||
testInvitationShortLinkAsync :: HasCallStack => Bool -> AgentClient -> AgentClient -> IO ()
|
||||
testInvitationShortLinkAsync viaProxy a b = do
|
||||
let userData = UserLinkData "some user data"
|
||||
@@ -1785,7 +1792,7 @@ testInvitationShortLinkAsync viaProxy a b = do
|
||||
connReq' `shouldBe` connReq
|
||||
linkUserData connData' `shouldBe` userData
|
||||
runRight $ do
|
||||
(aId, _) <- A.prepareConnectionToJoin b 1 True connReq PQSupportOn
|
||||
aId <- A.prepareConnectionToJoin b 1 True connReq PQSupportOn
|
||||
A.joinConnectionAsync b "123" False aId True connReq "bob's connInfo" PQSupportOn SMSubscribe
|
||||
get b =##> \case ("123", c, JOINED sndSecure) -> c == aId && sndSecure; _ -> False
|
||||
("", _, CONF confId _ "bob's connInfo") <- get a
|
||||
@@ -1827,7 +1834,7 @@ testContactShortLink viaProxy a b =
|
||||
(aId, sndSecure) <- joinConnection b 1 True connReq "bob's connInfo" SMSubscribe
|
||||
liftIO $ sndSecure `shouldBe` False
|
||||
("", _, REQ invId _ "bob's connInfo") <- get a
|
||||
(bId, _) <- A.prepareConnectionToAccept a 1 True invId PQSupportOn
|
||||
bId <- A.prepareConnectionToAccept a 1 True invId PQSupportOn
|
||||
sndSecure' <- acceptContact a 1 bId True invId "alice's connInfo" PQSupportOn SMSubscribe
|
||||
liftIO $ sndSecure' `shouldBe` True
|
||||
("", _, CONF confId _ "alice's connInfo") <- get b
|
||||
@@ -1880,7 +1887,7 @@ testAddContactShortLink viaProxy a b =
|
||||
(aId, sndSecure) <- joinConnection b 1 True connReq "bob's connInfo" SMSubscribe
|
||||
liftIO $ sndSecure `shouldBe` False
|
||||
("", _, REQ invId _ "bob's connInfo") <- get a
|
||||
(bId, _) <- A.prepareConnectionToAccept a 1 True invId PQSupportOn
|
||||
bId <- A.prepareConnectionToAccept a 1 True invId PQSupportOn
|
||||
sndSecure' <- acceptContact a 1 bId True invId "alice's connInfo" PQSupportOn SMSubscribe
|
||||
liftIO $ sndSecure' `shouldBe` True
|
||||
("", _, CONF confId _ "alice's connInfo") <- get b
|
||||
@@ -1977,11 +1984,18 @@ testOldContactQueueShortLink ps@(_, msType) = withAgentClients2 $ \a b -> do
|
||||
#endif
|
||||
() <- case testServerStoreConfig msType of
|
||||
ASSCfg _ _ (SSCMemory sp_) -> mapM_ (\StorePaths {storeLogFile} -> updateStoreLog storeLogFile) sp_
|
||||
ASSCfg _ _ SSCMemoryJournal {storeLogFile} -> updateStoreLog storeLogFile
|
||||
#if defined(dbServerPostgres)
|
||||
ASSCfg _ _ SSCDatabaseJournal {storeCfg} -> do
|
||||
st :: PostgresQueueStore (JournalQueue 'QSPostgres) <- newQueueStore @(JournalQueue 'QSPostgres) (storeCfg, True)
|
||||
updateDbStore st
|
||||
closeQueueStore @(JournalQueue 'QSPostgres) st
|
||||
ASSCfg _ _ (SSCDatabase storeCfg) -> do
|
||||
st :: PostgresQueueStore PostgresQueue <- newQueueStore @PostgresQueue storeCfg
|
||||
st :: PostgresQueueStore PostgresQueue <- newQueueStore @PostgresQueue (storeCfg, False)
|
||||
updateDbStore st
|
||||
closeQueueStore @PostgresQueue st
|
||||
#else
|
||||
ASSCfg _ _ SSCDatabaseJournal {} -> error "no dbServerPostgres flag"
|
||||
#endif
|
||||
|
||||
withSmpServer ps $ do
|
||||
@@ -2032,7 +2046,7 @@ testPrepareCreateConnectionLink ps = withSmpServer ps $ withAgentClients2 $ \a b
|
||||
(bId, sndSecure) <- joinConnection b 1 True connReq' "bob's connInfo" SMSubscribe
|
||||
liftIO $ sndSecure `shouldBe` False
|
||||
("", _, REQ invId _ "bob's connInfo") <- get a
|
||||
(aId, _) <- A.prepareConnectionToAccept a 1 True invId PQSupportOn
|
||||
aId <- A.prepareConnectionToAccept a 1 True invId PQSupportOn
|
||||
sndSecure' <- acceptContact a 1 aId True invId "alice's connInfo" PQSupportOn SMSubscribe
|
||||
liftIO $ sndSecure' `shouldBe` True
|
||||
("", _, CONF confId _ "alice's connInfo") <- get b
|
||||
@@ -2429,7 +2443,7 @@ testExpireMessageQuota (t, msType) = withSmpServerConfigOn t cfg' testPort $ \_
|
||||
ackMessage b' aId 4 Nothing
|
||||
disposeAgentClient a
|
||||
where
|
||||
cfg' = updateCfg (cfgMS msType) $ \cfg_ -> cfg_ {msgQueueQuota = 1}
|
||||
cfg' = updateCfg (cfgMS msType) $ \cfg_ -> cfg_ {msgQueueQuota = 1, maxJournalMsgCount = 2}
|
||||
|
||||
testExpireManyMessagesQuota :: (ASrvTransport, AStoreType) -> IO ()
|
||||
testExpireManyMessagesQuota (t, msType) = withSmpServerConfigOn t cfg' testPort $ \_ -> do
|
||||
@@ -2468,50 +2482,7 @@ testExpireManyMessagesQuota (t, msType) = withSmpServerConfigOn t cfg' testPort
|
||||
ackMessage b' aId 4 Nothing
|
||||
disposeAgentClient a
|
||||
where
|
||||
cfg' = updateCfg (cfgMS msType) $ \cfg_ -> cfg_ {msgQueueQuota = 1}
|
||||
|
||||
testJoinFullContactAsync :: HasCallStack => (ASrvTransport, AStoreType) -> IO ()
|
||||
testJoinFullContactAsync (t, msType) = withSmpServerConfigOn t cfg' testPort $ \_ -> do
|
||||
(contactId, qInfo) <- fillContactAddress
|
||||
withAgent 2 agentCfg {commandQuotaRetryInterval = fastRetryInterval} initAgentServers testDB2 $ \bob -> do
|
||||
aliceId <- runRight $ do
|
||||
(aliceId, _) <- A.prepareConnectionToJoin bob 1 True qInfo PQSupportOn
|
||||
A.joinConnectionAsync bob "2" False aliceId True qInfo "bob's connInfo" PQSupportOn SMSubscribe
|
||||
pure aliceId
|
||||
threadDelay 500000
|
||||
noMessages bob "joining should be retried while the address is full"
|
||||
withAgent 1 agentCfg initAgentServers testDB $ \alice -> runRight_ $ do
|
||||
subscribeConnection alice contactId
|
||||
("", _, A.REQ _ _ _ "carol's connInfo" _ _) <- get alice
|
||||
get bob =##> \case ("2", c, JOINED False) -> c == aliceId; _ -> False
|
||||
("", _, A.REQ _ _ _ "bob's connInfo" _ _) <- get alice
|
||||
pure ()
|
||||
where
|
||||
cfg' = updateCfg (cfgMS msType) $ \cfg_ -> cfg_ {msgQueueQuota = 1}
|
||||
|
||||
testJoinFullContactAsyncExpire :: HasCallStack => (ASrvTransport, AStoreType) -> IO ()
|
||||
testJoinFullContactAsyncExpire (t, msType) = withSmpServerConfigOn t cfg' testPort $ \_ -> do
|
||||
(_, qInfo) <- fillContactAddress
|
||||
withAgent 2 agentCfg {quotaExceededTimeout = 2, commandQuotaRetryInterval = fastRetryInterval} initAgentServers testDB2 $ \bob -> do
|
||||
aliceId <- runRight $ do
|
||||
(aliceId, _) <- A.prepareConnectionToJoin bob 1 True qInfo PQSupportOn
|
||||
A.joinConnectionAsync bob "2" False aliceId True qInfo "bob's connInfo" PQSupportOn SMSubscribe
|
||||
pure aliceId
|
||||
threadDelay 500000
|
||||
noMessages bob "joining should be retried until quota exceeded timeout"
|
||||
get bob =##> \case ("2", c, ERR (SMP _ QUOTA)) -> c == aliceId; _ -> False
|
||||
where
|
||||
cfg' = updateCfg (cfgMS msType) $ \cfg_ -> cfg_ {msgQueueQuota = 1}
|
||||
|
||||
fillContactAddress :: HasCallStack => IO (ConnId, ConnectionRequestUri 'CMContact)
|
||||
fillContactAddress = do
|
||||
(contactId, qInfo) <- withAgent 1 agentCfg initAgentServers testDB $ \alice -> runRight $ do
|
||||
(contactId, CCLink qInfo Nothing) <- A.createConnection alice NRMInteractive 1 True True SCMContact Nothing Nothing IKPQOn False SMSubscribe
|
||||
pure (contactId, qInfo)
|
||||
withAgent 3 agentCfg initAgentServers testDB3 $ \carol -> runRight_ $ do
|
||||
(aliceId, _) <- A.prepareConnectionToJoin carol 1 True qInfo PQSupportOn
|
||||
void $ A.joinConnection carol NRMInteractive 1 aliceId True qInfo "carol's connInfo" PQSupportOn SMSubscribe
|
||||
pure (contactId, qInfo)
|
||||
cfg' = updateCfg (cfgMS msType) $ \cfg_ -> cfg_ {msgQueueQuota = 1, maxJournalMsgCount = 2}
|
||||
|
||||
testDropMsgAfterRcvAttempts :: HasCallStack => (ASrvTransport, AStoreType) -> IO ()
|
||||
testDropMsgAfterRcvAttempts ps =
|
||||
@@ -2727,117 +2698,6 @@ testRatchetSyncSimultaneous ps = do
|
||||
disposeAgentClient bob
|
||||
disposeAgentClient bob2
|
||||
|
||||
testRatchetSyncReplayedKey :: HasCallStack => (ASrvTransport, AStoreType) -> IO ()
|
||||
testRatchetSyncReplayedKey ps = withAgentClients2 $ \alice bob -> do
|
||||
(aliceId, bobId, bob2) <- withSmpServerStoreMsgLogOn ps testPort $ \_ ->
|
||||
setupDesynchronizedRatchet alice bob
|
||||
("", "", DOWN _ _) <- nGet alice
|
||||
("", "", DOWN _ _) <- nGet bob2
|
||||
_ <- runRight $ synchronizeRatchet bob2 aliceId PQSupportOn False
|
||||
Right pks <- withTransaction (store $ agentEnv bob2) (`getRatchetX3dhKeys` aliceId)
|
||||
Right (SomeConn _ (DuplexConnection _ _ (sq :| _))) <- withTransaction (store $ agentEnv bob2) (`getConn` aliceId)
|
||||
withSmpServerStoreMsgLogOn ps testPort $ \_ -> do
|
||||
concurrently_
|
||||
(getInAnyOrder alice [ratchetSyncP' bobId RSAgreed, serverUpP])
|
||||
(getInAnyOrder bob2 [ratchetSyncP' aliceId RSAgreed, serverUpP])
|
||||
get alice =##> ratchetSyncP bobId RSOk
|
||||
get bob2 =##> ratchetSyncP aliceId RSOk
|
||||
withTransaction (store $ agentEnv alice) $ \db -> deleteRatchetKeyHashesExpired db 0 100
|
||||
let keyMsg = AgentRatchetKey {agentVersion = currentSMPAgentVersion, e2eEncryption = CR.mkRcvE2ERatchetParams CR.currentE2EEncryptVersion pks, info = ""}
|
||||
Right _ <- runReaderT (runExceptT $ sendAgentMessage bob2 sq SMP.noMsgFlags $ smpEncode keyMsg) (agentEnv bob2)
|
||||
runRight_ $ exchangeGreetingsMsgIds alice bobId 10 bob2 aliceId 7
|
||||
disposeAgentClient bob2
|
||||
|
||||
testRatchetSyncRepeated :: HasCallStack => (ASrvTransport, AStoreType) -> IO ()
|
||||
testRatchetSyncRepeated ps = withAgentClients2 $ \alice bob -> do
|
||||
(aliceId, bobId, bob2) <- startRatchetSyncOffline ps alice bob
|
||||
ConnectionStats {ratchetSyncState = rss2} <- runRight $ synchronizeRatchet bob2 aliceId PQSupportOn True
|
||||
rss2 `shouldBe` RSStarted
|
||||
|
||||
withSmpServerStoreMsgLogOn ps testPort $ \_ -> do
|
||||
concurrently_
|
||||
(getInAnyOrder alice [ratchetSyncP' bobId RSAgreed, serverUpP])
|
||||
(getInAnyOrder bob2 [ratchetSyncP' aliceId RSAgreed, serverUpP])
|
||||
runRight_ $ do
|
||||
get alice =##> ratchetSyncP bobId RSAgreed
|
||||
get alice =##> ratchetSyncP bobId RSOk
|
||||
get bob2 =##> ratchetSyncP aliceId RSOk
|
||||
msgId <- sendMessage alice bobId SMP.noMsgFlags "hello"
|
||||
get alice ##> ("", bobId, SENT msgId)
|
||||
get bob2 =##> \case ("", c, Msg "hello") -> c == aliceId; _ -> False
|
||||
ackMessage bob2 aliceId 8 Nothing
|
||||
map fst <$> processedRatchetKeyHashes bob2 `shouldReturn` [aliceId, aliceId]
|
||||
disposeAgentClient bob2
|
||||
|
||||
testRatchetSyncFailedKeyNotProcessed :: HasCallStack => (ASrvTransport, AStoreType) -> IO ()
|
||||
testRatchetSyncFailedKeyNotProcessed ps = withAgentClients2 $ \alice bob -> do
|
||||
(aliceId, bobId, bob2) <- startRatchetSyncOffline ps alice bob
|
||||
withTransaction (store $ agentEnv bob2) $ \db ->
|
||||
DB.execute_ db "UPDATE ratchets SET x3dh_priv_key_1 = NULL"
|
||||
|
||||
withSmpServerStoreMsgLogOn ps testPort $ \_ ->
|
||||
concurrently_
|
||||
(getInAnyOrder alice [ratchetSyncP' bobId RSAgreed, serverUpP])
|
||||
(getInAnyOrder bob2 [x3dhKeysNotFoundP aliceId, serverUpP])
|
||||
map fst <$> processedRatchetKeyHashes alice `shouldReturn` [bobId]
|
||||
processedRatchetKeyHashes bob2 `shouldReturn` []
|
||||
disposeAgentClient bob2
|
||||
where
|
||||
x3dhKeysNotFoundP :: ConnId -> ATransmission -> Bool
|
||||
x3dhKeysNotFoundP cId = \case
|
||||
(_, cId', AEvt SAEConn (ERR (A.INTERNAL e))) -> cId' == cId && "SEX3dhKeysNotFound" `isPrefixOf` e
|
||||
_ -> False
|
||||
|
||||
testRatchetSyncFailedRecreationNoReply :: HasCallStack => (ASrvTransport, AStoreType) -> IO ()
|
||||
testRatchetSyncFailedRecreationNoReply ps = withAgentClients2 $ \alice bob -> do
|
||||
(_, bobId, bob2) <- startRatchetSyncOffline ps alice bob
|
||||
aliceSndMsgs <- sndMessages alice
|
||||
withTransaction (store $ agentEnv alice) $ \db ->
|
||||
DB.execute_ db "CREATE TRIGGER fail_ratchet_insert BEFORE INSERT ON ratchets BEGIN SELECT RAISE(ABORT, 'ratchet insert failed'); END"
|
||||
withSmpServerStoreMsgLogOn ps testPort $ \_ ->
|
||||
concurrently_
|
||||
(getInAnyOrder alice [ratchetInsertFailedP bobId, serverUpP])
|
||||
(getInAnyOrder bob2 [serverUpP])
|
||||
processedRatchetKeyHashes alice `shouldReturn` []
|
||||
sndMessages alice `shouldReturn` aliceSndMsgs
|
||||
disposeAgentClient bob2
|
||||
where
|
||||
ratchetInsertFailedP :: ConnId -> ATransmission -> Bool
|
||||
ratchetInsertFailedP cId = \case
|
||||
(_, cId', AEvt SAEConn (ERR (A.INTERNAL e))) -> cId' == cId && "ratchet insert failed" `isInfixOf` e
|
||||
_ -> False
|
||||
|
||||
testRatchetSyncStartFailedNoKey :: HasCallStack => (ASrvTransport, AStoreType) -> IO ()
|
||||
testRatchetSyncStartFailedNoKey ps = withAgentClients2 $ \alice bob -> do
|
||||
(aliceId, _, bob2) <- withSmpServerStoreMsgLogOn ps testPort $ \_ ->
|
||||
setupDesynchronizedRatchet alice bob
|
||||
bobSndMsgs <- sndMessages bob2
|
||||
withTransaction (store $ agentEnv bob2) $ \db ->
|
||||
DB.execute_ db "CREATE TRIGGER fail_ratchet_update BEFORE UPDATE ON ratchets BEGIN SELECT RAISE(ABORT, 'ratchet update failed'); END"
|
||||
Left (A.INTERNAL e) <- runExceptT $ synchronizeRatchet bob2 aliceId PQSupportOff False
|
||||
e `shouldContain` "ratchet update failed"
|
||||
ConnectionStats {ratchetSyncState} <- runRight $ getConnectionServers bob2 aliceId
|
||||
ratchetSyncState `shouldBe` RSRequired
|
||||
withTransaction (store $ agentEnv bob2) (`DB.query_` "SELECT conn_id, pq_support FROM connections") `shouldReturn` [(aliceId, PQSupportOn)]
|
||||
sndMessages bob2 `shouldReturn` bobSndMsgs
|
||||
disposeAgentClient bob2
|
||||
|
||||
startRatchetSyncOffline :: HasCallStack => (ASrvTransport, AStoreType) -> AgentClient -> AgentClient -> IO (ConnId, ConnId, AgentClient)
|
||||
startRatchetSyncOffline ps alice bob = do
|
||||
(aliceId, bobId, bob2) <- withSmpServerStoreMsgLogOn ps testPort $ \_ ->
|
||||
setupDesynchronizedRatchet alice bob
|
||||
("", "", DOWN _ _) <- nGet alice
|
||||
("", "", DOWN _ _) <- nGet bob2
|
||||
ConnectionStats {ratchetSyncState} <- runRight $ synchronizeRatchet bob2 aliceId PQSupportOn False
|
||||
ratchetSyncState `shouldBe` RSStarted
|
||||
pure (aliceId, bobId, bob2)
|
||||
|
||||
processedRatchetKeyHashes :: AgentClient -> IO [(ConnId, ByteString)]
|
||||
processedRatchetKeyHashes c = withTransaction (store $ agentEnv c) (`DB.query_` "SELECT conn_id, hash FROM processed_ratchet_key_hashes")
|
||||
|
||||
sndMessages :: AgentClient -> IO [(ConnId, Int64)]
|
||||
sndMessages c = withTransaction (store $ agentEnv c) (`DB.query_` "SELECT conn_id, internal_id FROM snd_messages ORDER BY conn_id, internal_id")
|
||||
|
||||
getMsg :: AgentClient -> ConnId -> ExceptT AgentErrorType IO a -> ExceptT AgentErrorType IO a
|
||||
getMsg c cId action = do
|
||||
liftIO $ noMessages c "nothing should be delivered before GET"
|
||||
@@ -2889,7 +2749,7 @@ makeConnectionForUsers = makeConnectionForUsers_ PQSupportOn
|
||||
makeConnectionForUsers_ :: HasCallStack => PQSupport -> AgentClient -> UserId -> AgentClient -> UserId -> ExceptT AgentErrorType IO (ConnId, ConnId)
|
||||
makeConnectionForUsers_ pqSupport alice aliceUserId bob bobUserId = do
|
||||
(bobId, CCLink qInfo Nothing) <- A.createConnection alice NRMInteractive aliceUserId True True SCMInvitation Nothing Nothing (IKLinkPQ pqSupport) False SMSubscribe
|
||||
(aliceId, _) <- A.prepareConnectionToJoin bob bobUserId True qInfo pqSupport
|
||||
aliceId <- A.prepareConnectionToJoin bob bobUserId True qInfo pqSupport
|
||||
sqSecured' <- A.joinConnection bob NRMInteractive bobUserId aliceId True qInfo "bob's connInfo" pqSupport SMSubscribe
|
||||
liftIO $ sqSecured' `shouldBe` True
|
||||
("", _, A.CONF confId pqSup' _ "bob's connInfo") <- get alice
|
||||
@@ -3079,7 +2939,7 @@ testBatchedSubscriptions nCreate nDel ps@(t, ASType qsType _) = do
|
||||
runServers :: ExceptT AgentErrorType IO a -> IO a
|
||||
runServers a = do
|
||||
withSmpServerStoreLogOn ps testPort $ \t1 -> do
|
||||
res <- withSmpServerConfigOn t (cfgS2QS qsType) testPort2 $ \t2 ->
|
||||
res <- withSmpServerConfigOn t (cfgJ2QS qsType) testPort2 $ \t2 ->
|
||||
runRight a `finally` killThread t2
|
||||
killThread t1
|
||||
pure res
|
||||
@@ -3171,7 +3031,7 @@ testAsyncCommands alice bob baseId =
|
||||
createConnectionAsync alice "1" bobId True SCMInvitation IKPQOn False SMSubscribe
|
||||
("1", bobId', INV (ACR _ qInfo)) <- get alice
|
||||
liftIO $ bobId' `shouldBe` bobId
|
||||
(aliceId, _) <- prepareConnectionToJoin bob 1 True qInfo PQSupportOn
|
||||
aliceId <- prepareConnectionToJoin bob 1 True qInfo PQSupportOn
|
||||
joinConnectionAsync bob "2" False aliceId True qInfo "bob's connInfo" PQSupportOn SMSubscribe
|
||||
("2", aliceId', JOINED sqSecured') <- get bob
|
||||
liftIO $ do
|
||||
@@ -3241,7 +3101,7 @@ testSetConnShortLinkAsync ps = withAgentClients2 $ \alice bob ->
|
||||
-- complete connection via contact address
|
||||
(aliceId, _) <- joinConnection bob 1 True qInfo "bob's connInfo" SMSubscribe
|
||||
("", _, REQ invId _ "bob's connInfo") <- get alice
|
||||
(bobId, _) <- A.prepareConnectionToAccept alice 1 True invId PQSupportOn
|
||||
bobId <- A.prepareConnectionToAccept alice 1 True invId PQSupportOn
|
||||
_ <- acceptContact alice 1 bobId True invId "alice's connInfo" PQSupportOn SMSubscribe
|
||||
("", _, CONF confId _ "alice's connInfo") <- get bob
|
||||
allowConnection bob aliceId confId "bob's connInfo"
|
||||
@@ -3269,7 +3129,7 @@ testGetConnShortLinkAsync ps = withAgentClients2 $ \alice bob ->
|
||||
liftIO $ aliceId' `shouldBe` aliceId
|
||||
-- complete connection
|
||||
("", _, REQ invId _ "bob's connInfo") <- get alice
|
||||
(bobId, _) <- A.prepareConnectionToAccept alice 1 True invId PQSupportOn
|
||||
bobId <- A.prepareConnectionToAccept alice 1 True invId PQSupportOn
|
||||
_ <- acceptContact alice 1 bobId True invId "alice's connInfo" PQSupportOn SMSubscribe
|
||||
("", _, CONF confId _ "alice's connInfo") <- get bob
|
||||
allowConnection bob aliceId confId "bob's connInfo"
|
||||
@@ -3299,7 +3159,7 @@ testAcceptContactAsync alice bob baseId =
|
||||
(aliceId, sqSecuredJoin) <- joinConnection bob 1 True qInfo "bob's connInfo" SMSubscribe
|
||||
liftIO $ sqSecuredJoin `shouldBe` False -- joining via contact address connection
|
||||
("", _, REQ invId _ "bob's connInfo") <- get alice
|
||||
(bobId, _) <- prepareConnectionToAccept alice 1 True invId PQSupportOn
|
||||
bobId <- prepareConnectionToAccept alice 1 True invId PQSupportOn
|
||||
acceptContactAsync alice "1" bobId True invId "alice's connInfo" PQSupportOn SMSubscribe
|
||||
get alice =##> \case ("1", c, JOINED sqSecured') -> c == bobId && sqSecured' == True; _ -> False
|
||||
("", _, CONF confId _ "alice's connInfo") <- get bob
|
||||
@@ -3580,13 +3440,13 @@ testJoinConnectionAsyncReplyError ps@(t, ASType qsType _) = do
|
||||
createConnectionAsync a "1" bId True SCMInvitation IKPQOn False SMSubscribe
|
||||
("1", bId', INV (ACR _ qInfo)) <- get a
|
||||
liftIO $ bId' `shouldBe` bId
|
||||
(aId, _) <- prepareConnectionToJoin b 1 True qInfo PQSupportOn
|
||||
aId <- prepareConnectionToJoin b 1 True qInfo PQSupportOn
|
||||
joinConnectionAsync b "2" False aId True qInfo "bob's connInfo" PQSupportOn SMSubscribe
|
||||
liftIO $ threadDelay 500000
|
||||
ConnectionStats {rcvQueuesInfo = [], sndQueuesInfo = [SndQueueInfo {}]} <- getConnectionServers b aId
|
||||
pure (aId, bId)
|
||||
nGet a =##> \case ("", "", DOWN _ [c]) -> c == bId; _ -> False
|
||||
withSmpServerConfigOn t (cfgS2QS qsType) testPort2 $ \_ -> do
|
||||
withSmpServerConfigOn t (cfgJ2QS qsType) testPort2 $ \_ -> do
|
||||
confId <- withSmpServerStoreLogOn ps testPort $ \_ -> do
|
||||
-- both servers need to be online for connection to progress because of SKEY
|
||||
get b =##> \case ("2", c, JOINED sqSecured) -> c == aId && sqSecured; _ -> False
|
||||
@@ -3719,7 +3579,7 @@ fastSwitchComplete a bId b aId = do
|
||||
testFastSwitchDeadOldServer :: HasCallStack => (ASrvTransport, AStoreType) -> IO ()
|
||||
testFastSwitchDeadOldServer ps@(t, ASType qsType _) = do
|
||||
let bServers = initAgentServers {smp = userServers [testSMPServer2]}
|
||||
withSmpServerConfigOn t (cfgS2QS qsType) testPort2 $ \_ ->
|
||||
withSmpServerConfigOn t (cfgJ2QS qsType) testPort2 $ \_ ->
|
||||
withAgent 1 agentCfg initAgentServers testDB $ \a ->
|
||||
withAgent 2 agentCfg bServers testDB2 $ \b -> do
|
||||
(aId, bId) <- withSmpServerStoreLogOn ps testPort $ \_ -> runRight $ do
|
||||
@@ -4163,23 +4023,13 @@ testServerInformation =
|
||||
serverCountry = Nothing
|
||||
}
|
||||
|
||||
testConnectionVerifyCodes :: HasCallStack => IO ()
|
||||
testConnectionVerifyCodes =
|
||||
testRatchetAdHash :: HasCallStack => IO ()
|
||||
testRatchetAdHash =
|
||||
withAgentClients2 $ \a b -> runRight_ $ do
|
||||
(aId, bId) <- makeConnection a b
|
||||
codes1 <- getConnectionVerifyCodes a bId
|
||||
codes2 <- getConnectionVerifyCodes b aId
|
||||
liftIO $ do
|
||||
codes1 `shouldBe` codes2
|
||||
codePQ codes1 `shouldNotBe` Nothing
|
||||
liftIO $ withTransaction (store $ agentEnv a) $ \db ->
|
||||
DB.execute_ db "UPDATE ratchets SET rc_verify_code_ad = NULL, rc_verify_code_pq = NULL"
|
||||
codes1' <- getConnectionVerifyCodes a bId
|
||||
liftIO $ codes1' `shouldBe` codes1
|
||||
liftIO $ withTransaction (store $ agentEnv a) $ \db ->
|
||||
DB.execute_ db "UPDATE ratchets SET ratchet_state = NULL"
|
||||
codes1'' <- getConnectionVerifyCodes a bId
|
||||
liftIO $ codes1'' `shouldBe` codes1
|
||||
ad1 <- getConnectionRatchetAdHash a bId
|
||||
ad2 <- getConnectionRatchetAdHash b aId
|
||||
liftIO $ ad1 `shouldBe` ad2
|
||||
|
||||
testDeliveryReceipts :: HasCallStack => IO ()
|
||||
testDeliveryReceipts =
|
||||
@@ -4213,7 +4063,7 @@ testDeliveryReceiptsConcurrent (t, msType) =
|
||||
liftIO $ noMessages a "nothing else should be delivered to alice"
|
||||
liftIO $ noMessages b "nothing else should be delivered to bob"
|
||||
where
|
||||
cfg' = updateCfg (cfgMS msType) $ \cfg_ -> cfg_ {msgQueueQuota = 256}
|
||||
cfg' = updateCfg (cfgMS msType) $ \cfg_ -> cfg_ {msgQueueQuota = 256, maxJournalMsgCount = 512}
|
||||
runClient :: String -> AgentClient -> ConnId -> IO ()
|
||||
runClient _cName client connId = do
|
||||
concurrently_ send receive
|
||||
@@ -4768,7 +4618,8 @@ testServerMultipleIdentities =
|
||||
testE2ERatchetParams12
|
||||
|
||||
testWaitForUserNetwork :: IO ()
|
||||
testWaitForUserNetwork = withAgent 1 aCfg initAgentServers testDB $ \a -> do
|
||||
testWaitForUserNetwork = do
|
||||
a <- getSMPAgentClient' 1 aCfg initAgentServers testDB
|
||||
noNetworkDelay a
|
||||
setUserNetworkInfo a $ UserNetworkInfo UNNone False
|
||||
networkDelay a 100000
|
||||
@@ -4785,7 +4636,8 @@ testWaitForUserNetwork = withAgent 1 aCfg initAgentServers testDB $ \a -> do
|
||||
aCfg = agentCfg {userNetworkInterval = 100000, userOfflineDelay = 0}
|
||||
|
||||
testDoNotResetOnlineToOffline :: IO ()
|
||||
testDoNotResetOnlineToOffline = withAgent 1 aCfg initAgentServers testDB $ \a -> do
|
||||
testDoNotResetOnlineToOffline = do
|
||||
a <- getSMPAgentClient' 1 aCfg initAgentServers testDB
|
||||
noNetworkDelay a
|
||||
setUserNetworkInfo a $ UserNetworkInfo UNWifi False
|
||||
networkDelay a 100000
|
||||
@@ -4806,7 +4658,8 @@ testDoNotResetOnlineToOffline = withAgent 1 aCfg initAgentServers testDB $ \a ->
|
||||
aCfg = agentCfg {userNetworkInterval = 100000, userOfflineDelay = 0.1}
|
||||
|
||||
testResumeMultipleThreads :: IO ()
|
||||
testResumeMultipleThreads = withAgent 1 aCfg initAgentServers testDB $ \a -> do
|
||||
testResumeMultipleThreads = do
|
||||
a <- getSMPAgentClient' 1 aCfg initAgentServers testDB
|
||||
noNetworkDelay a
|
||||
setUserNetworkInfo a $ UserNetworkInfo UNNone False
|
||||
vs <-
|
||||
|
||||
@@ -869,7 +869,7 @@ testNotificationsSMPRestartBatch n ps@(t, ASType qsType _) apns =
|
||||
runServers :: ExceptT AgentErrorType IO a -> IO a
|
||||
runServers a = do
|
||||
withSmpServerStoreLogOn ps testPort $ \t1 -> do
|
||||
res <- withSmpServerConfigOn t (cfgS2QS qsType) testPort2 $ \t2 ->
|
||||
res <- withSmpServerConfigOn t (cfgJ2QS qsType) testPort2 $ \t2 ->
|
||||
runRight a `finally` killThread t2
|
||||
killThread t1
|
||||
pure res
|
||||
|
||||
@@ -14,6 +14,7 @@ module AgentTests.ResolveNameTests (resolveNameTests) where
|
||||
|
||||
import AgentTests.FunctionalAPITests (withAgent)
|
||||
import Control.Monad.Except (runExceptT)
|
||||
import qualified Data.Aeson as J
|
||||
import qualified Data.ByteString.Lazy as LB
|
||||
import Data.List (isInfixOf)
|
||||
import Network.HTTP.Types (Status, status200, status404, status502)
|
||||
@@ -21,7 +22,7 @@ import NamesResolverServer (memCfg, memCfg2, memProxyCfg, withNames)
|
||||
import qualified NamesResolverServer as NRS
|
||||
import SMPAgentClient
|
||||
import SMPClient
|
||||
import SMPNamesTests (availableBody, registeredBody, testNameRecord)
|
||||
import SMPNamesTests (testNameRecord)
|
||||
import Simplex.Messaging.Agent (resolveSimplexName)
|
||||
import Simplex.Messaging.Agent.Client (AgentClient)
|
||||
import Simplex.Messaging.Agent.Env.SQLite (InitialAgentServers (..), ServerCfg, ServerRoles (..), presetServerCfg)
|
||||
@@ -70,13 +71,13 @@ withNoNameServers k = withAgent 1 agentCfg (oneSrv (proxySrvCfg testSMPServer))
|
||||
resolveNameTests :: Spec
|
||||
resolveNameTests = do
|
||||
describe "direct path (SPMNever)" $
|
||||
it "a resolver error propagates as SMP host (NAME RESOLVER)" testDirectResolverErr
|
||||
it "404 propagates as SMP host (NAME NOT_FOUND)" testDirectNotFound
|
||||
describe "proxy path (SPMAlways)" $
|
||||
it "a resolver error propagates via proxy as SMP <proxyHost> (NAME RESOLVER)" testProxyResolverErr
|
||||
it "404 from resolver propagates via proxy as SMP <proxyHost> (NAME NOT_FOUND)" testProxyNotFound
|
||||
describe "TLDTesting path" $
|
||||
it "NAME RESOLVER for TLDTesting too" testTestingTldResolverErr
|
||||
it "NAME NOT_FOUND for TLDTesting too" testTestingTldNotFound
|
||||
describe "TLDWeb path" $
|
||||
it "NAME RESOLVER for TLDWeb too" testWebTldResolverErr
|
||||
it "NAME NOT_FOUND for TLDWeb too" testWebTldNotFound
|
||||
describe "no resolver configured" $
|
||||
it "answers NAME NO_RESOLVER" testNoResolver
|
||||
describe "no names servers (names role off everywhere)" $
|
||||
@@ -85,50 +86,38 @@ resolveNameTests = do
|
||||
it "surfaces as SMP host (NAME (RESOLVER ..))" testBackendError
|
||||
describe "success path" $
|
||||
it "returns NameRecord" testDirectSuccess
|
||||
describe "name availability" $
|
||||
it "an unregistered name answers as available" testAvailSuccess
|
||||
|
||||
testAvailSuccess :: HasCallStack => IO ()
|
||||
testAvailSuccess =
|
||||
withDirectResolver (status200, availableBody) $ \c -> do
|
||||
r <- runExceptT $ resolveSimplexName c NRMInteractive 1 (SimplexDomain TLDSimplex "alice" [])
|
||||
case r of
|
||||
Right SMP.NameResponse {registration = SMP.NRAvailable {}} -> pure ()
|
||||
_ -> expectationFailure $ "expected Right NRAvailable, got: " <> show r
|
||||
|
||||
-- | 404 is a resolver that predates /v2/resolve: no status from that endpoint
|
||||
-- means "not registered", since an unregistered name answers NRAvailable.
|
||||
testDirectResolverErr :: HasCallStack => IO ()
|
||||
testDirectResolverErr =
|
||||
testDirectNotFound :: HasCallStack => IO ()
|
||||
testDirectNotFound =
|
||||
withDirectResolver (status404, "{}") $ \c -> do
|
||||
r <- runExceptT $ resolveSimplexName c NRMInteractive 1 (SimplexDomain TLDSimplex "alice" [])
|
||||
case r of
|
||||
Left (SMP _ (SMP.NAME (SMP.RESOLVER _))) -> pure ()
|
||||
_ -> expectationFailure $ "expected Left (SMP _ (NAME (RESOLVER _))), got: " <> show r
|
||||
Left (SMP _ (SMP.NAME SMP.NOT_FOUND)) -> pure ()
|
||||
_ -> expectationFailure $ "expected Left (SMP _ (NAME NOT_FOUND)), got: " <> show r
|
||||
|
||||
testProxyResolverErr :: HasCallStack => IO ()
|
||||
testProxyResolverErr =
|
||||
testProxyNotFound :: HasCallStack => IO ()
|
||||
testProxyNotFound =
|
||||
withProxyAndResolver (status404, "{}") $ \c -> do
|
||||
r <- runExceptT $ resolveSimplexName c NRMInteractive 1 (SimplexDomain TLDSimplex "alice" [])
|
||||
case r of
|
||||
Left (SMP host (SMP.NAME (SMP.RESOLVER _))) | testPort `isInfixOf` host -> pure ()
|
||||
_ -> expectationFailure $ "expected Left (SMP <proxyHost:" <> testPort <> "> (NAME (RESOLVER _))), got: " <> show r
|
||||
Left (SMP host (SMP.NAME SMP.NOT_FOUND)) | testPort `isInfixOf` host -> pure ()
|
||||
_ -> expectationFailure $ "expected Left (SMP <proxyHost:" <> testPort <> "> (NAME NOT_FOUND)), got: " <> show r
|
||||
|
||||
testTestingTldResolverErr :: HasCallStack => IO ()
|
||||
testTestingTldResolverErr =
|
||||
testTestingTldNotFound :: HasCallStack => IO ()
|
||||
testTestingTldNotFound =
|
||||
withDirectResolver (status404, "{}") $ \c -> do
|
||||
r <- runExceptT $ resolveSimplexName c NRMInteractive 1 (SimplexDomain TLDTesting "bob" [])
|
||||
case r of
|
||||
Left (SMP _ (SMP.NAME (SMP.RESOLVER _))) -> pure ()
|
||||
_ -> expectationFailure $ "expected Left (SMP _ (NAME (RESOLVER _))), got: " <> show r
|
||||
Left (SMP _ (SMP.NAME SMP.NOT_FOUND)) -> pure ()
|
||||
_ -> expectationFailure $ "expected Left (SMP _ (NAME NOT_FOUND)), got: " <> show r
|
||||
|
||||
testWebTldResolverErr :: HasCallStack => IO ()
|
||||
testWebTldResolverErr =
|
||||
testWebTldNotFound :: HasCallStack => IO ()
|
||||
testWebTldNotFound =
|
||||
withDirectResolver (status404, "{}") $ \c -> do
|
||||
r <- runExceptT $ resolveSimplexName c NRMInteractive 1 (SimplexDomain TLDWeb "example.com" [])
|
||||
case r of
|
||||
Left (SMP _ (SMP.NAME (SMP.RESOLVER _))) -> pure ()
|
||||
_ -> expectationFailure $ "expected Left (SMP _ (NAME (RESOLVER _))), got: " <> show r
|
||||
Left (SMP _ (SMP.NAME SMP.NOT_FOUND)) -> pure ()
|
||||
_ -> expectationFailure $ "expected Left (SMP _ (NAME NOT_FOUND)), got: " <> show r
|
||||
|
||||
testNoResolver :: HasCallStack => IO ()
|
||||
testNoResolver =
|
||||
@@ -156,8 +145,8 @@ testBackendError =
|
||||
|
||||
testDirectSuccess :: HasCallStack => IO ()
|
||||
testDirectSuccess =
|
||||
withDirectResolver (status200, registeredBody testNameRecord) $ \c -> do
|
||||
withDirectResolver (status200, J.encode testNameRecord) $ \c -> do
|
||||
r <- runExceptT $ resolveSimplexName c NRMInteractive 1 (SimplexDomain TLDSimplex "alice" [])
|
||||
case r of
|
||||
Right SMP.NameResponse {registration = SMP.NRRegistered {nameRecord}} -> nameRecord `shouldBe` testNameRecord
|
||||
_ -> expectationFailure $ "expected Right NRRegistered, got: " <> show r
|
||||
Right nr -> nr `shouldBe` testNameRecord
|
||||
_ -> expectationFailure $ "expected Right NameRecord, got: " <> show r
|
||||
|
||||
@@ -20,13 +20,12 @@ import Control.Concurrent.Async (concurrently_)
|
||||
import Control.Concurrent.MVar
|
||||
import Control.Concurrent.STM
|
||||
import Control.Exception (SomeException)
|
||||
import Control.Monad (forM_, replicateM_)
|
||||
import Control.Monad (replicateM_)
|
||||
import Control.Monad.Trans.Except
|
||||
import Crypto.Random (ChaChaDRG)
|
||||
import Data.ByteArray (ScrubbedBytes)
|
||||
import Data.ByteString.Char8 (ByteString)
|
||||
import Data.List (isInfixOf)
|
||||
import qualified Data.Map.Strict as M
|
||||
import qualified Data.Text as T
|
||||
import Data.Text.Encoding (encodeUtf8)
|
||||
import Data.Time
|
||||
@@ -52,7 +51,6 @@ import qualified Simplex.Messaging.Crypto as C
|
||||
import Simplex.Messaging.Crypto.File (CryptoFile (..))
|
||||
import Simplex.Messaging.Crypto.Ratchet (pattern IKPQOn)
|
||||
import qualified Simplex.Messaging.Crypto.Ratchet as CR
|
||||
import Simplex.Messaging.Encoding (Encoding (..))
|
||||
import Simplex.Messaging.Encoding.String (StrEncoding (..))
|
||||
import Simplex.Messaging.Protocol (EntityId (..), QueueMode (..), SubscriptionMode (..), pattern VersionSMPC)
|
||||
import qualified Simplex.Messaging.Protocol as SMP
|
||||
@@ -137,8 +135,6 @@ storeTests = do
|
||||
testCreateRcvMsg
|
||||
testCreateSndMsg
|
||||
testCreateRcvAndSndMsgs
|
||||
describe "deleteRatchetKeyHashesExpired" testDeleteRatchetKeyHashesExpired
|
||||
it "should keep only the newest skipped message keys" testGetSkippedMsgKeys
|
||||
describe "Work items" $ do
|
||||
it "should getPendingQueueMsg" testGetPendingQueueMsg
|
||||
it "should getPendingServerCommand" testGetPendingServerCommand
|
||||
@@ -603,42 +599,6 @@ testCreateRcvAndSndMsgs =
|
||||
testCreateSndMsg_ db "snd_hash_1" connId sq $ mkSndMsgData (InternalId 5) (InternalSndId 2) "snd_hash_2"
|
||||
testCreateSndMsg_ db "snd_hash_2" connId sq $ mkSndMsgData (InternalId 6) (InternalSndId 3) "snd_hash_3"
|
||||
|
||||
testDeleteRatchetKeyHashesExpired :: SpecWith DBStore
|
||||
testDeleteRatchetKeyHashesExpired =
|
||||
it "should delete expired ratchet key hashes except the newest in each connection" . withStoreTransaction $ \db -> do
|
||||
g <- C.newRandom
|
||||
Right connId <- createNewConn db g cData1 {connId = ""} SCMInvitation
|
||||
Right connId' <- createNewConn db g cData1 {connId = ""} SCMContact
|
||||
let hashes = ["h1", "h2", "h3", "h4", "h5", "h6"]
|
||||
forM_ hashes $ addProcessedRatchetKeyHash db connId
|
||||
forM_ (take 4 hashes) $ addProcessedRatchetKeyHash db connId'
|
||||
deleteRatchetKeyHashesExpired db 86400 4
|
||||
mapM (checkRatchetKeyHashExists db connId) hashes `shouldReturn` replicate 6 True
|
||||
deleteRatchetKeyHashesExpired db 0 4
|
||||
mapM (checkRatchetKeyHashExists db connId) hashes `shouldReturn` [False, False, True, True, True, True]
|
||||
mapM (checkRatchetKeyHashExists db connId') (take 4 hashes) `shouldReturn` replicate 4 True
|
||||
|
||||
testGetSkippedMsgKeys :: DBStore -> Expectation
|
||||
testGetSkippedMsgKeys st = do
|
||||
g <- C.newRandom
|
||||
withTransaction st $ \db -> do
|
||||
Right connId <- createNewConn db g cData1 {connId = ""} SCMInvitation
|
||||
Right connId' <- createNewConn db g cData1 {connId = ""} SCMInvitation
|
||||
createSkippedKeys db connId'
|
||||
createSkippedKeys db connId
|
||||
M.map M.keys <$> getSkippedMsgKeys db connId 4
|
||||
`shouldReturn` M.singleton (C.Key "header_key") [1, 2, 3, 4]
|
||||
getMsgNs db connId `shouldReturn` [1 .. 4]
|
||||
getMsgNs db connId' `shouldReturn` [1 .. 10]
|
||||
where
|
||||
createSkippedKeys :: DB.Connection -> ConnId -> IO ()
|
||||
createSkippedKeys db connId = do
|
||||
DB.execute db "INSERT INTO ratchets (conn_id) VALUES (?)" (Only connId)
|
||||
forM_ ([10, 9 .. 1] :: [Int]) $ \msgN ->
|
||||
DB.execute db "INSERT INTO skipped_messages (conn_id, header_key, msg_n, msg_key) VALUES (?, ?, ?, ?)" (connId, "header_key" :: ByteString, msgN, smpEncode ("key" :: ByteString, "iv" :: ByteString))
|
||||
getMsgNs :: DB.Connection -> ConnId -> IO [Int]
|
||||
getMsgNs db connId = map fromOnly <$> DB.query db "SELECT msg_n FROM skipped_messages WHERE conn_id = ? ORDER BY msg_n" (Only connId)
|
||||
|
||||
testCloseReopenStore :: IO ()
|
||||
testCloseReopenStore = do
|
||||
st <- createStore'
|
||||
@@ -783,11 +743,9 @@ testGetNextRcvChunkToDownload st = do
|
||||
withTransaction st $ \db -> do
|
||||
Right Nothing <- getNextRcvChunkToDownload db xftpServer1 86400
|
||||
|
||||
Right fId1 <- createRcvFile db g 1 rcvFileDescr1 "filepath" "filepath" (CryptoFile "filepath" Nothing) True
|
||||
Right _ <- startPreparedRcvFile db fId1
|
||||
Right _ <- createRcvFile db g 1 rcvFileDescr1 "filepath" "filepath" (CryptoFile "filepath" Nothing) True
|
||||
DB.execute_ db "UPDATE rcv_file_chunk_replicas SET replica_key = cast('bad' as blob) WHERE rcv_file_chunk_replica_id = 1"
|
||||
Right fId2 <- createRcvFile db g 1 rcvFileDescr1 "filepath" "filepath" (CryptoFile "filepath" Nothing) True
|
||||
Right _ <- startPreparedRcvFile db fId2
|
||||
|
||||
Left e <- getNextRcvChunkToDownload db xftpServer1 86400
|
||||
show e `shouldContain` "ConversionFailed"
|
||||
@@ -825,7 +783,6 @@ testGetNextSndFileToPrepare st = do
|
||||
-- Right _ <- createSndFile db g 1 (CryptoFile "filepath" Nothing) 1 "filepath" testFileSbKey testFileCbNonce Nothing
|
||||
-- DB.execute_ db "UPDATE snd_files SET status = 'new', num_recipients = 'bad' WHERE snd_file_id = 1"
|
||||
Right fId2 <- createSndFile db g 1 (CryptoFile "filepath" Nothing) 1 "filepath" testFileSbKey testFileCbNonce Nothing Nothing
|
||||
Right _ <- startPreparedSndFile db fId2
|
||||
DB.execute_ db "UPDATE snd_files SET status = 'new' WHERE snd_file_id = 2"
|
||||
|
||||
-- Left e <- getNextSndFileToPrepare db 86400
|
||||
|
||||
@@ -48,7 +48,6 @@ schemaDumpTest = do
|
||||
it "verify strict tables" testVerifyStrict
|
||||
it "should NOT create user record for new database" testUsersMigrationNew
|
||||
it "should create user record for old database" testUsersMigrationOld
|
||||
it "should remove duplicate ratchet key hashes before adding unique index" testRatchetKeyHashesUniqueMigration
|
||||
|
||||
testVerifySchemaDump :: IO ()
|
||||
testVerifySchemaDump = do
|
||||
@@ -115,28 +114,12 @@ testUsersMigrationOld = do
|
||||
`shouldReturn` ([Only (1 :: Int)])
|
||||
closeDBStore st'
|
||||
|
||||
testRatchetKeyHashesUniqueMigration :: IO ()
|
||||
testRatchetKeyHashesUniqueMigration = do
|
||||
let beforeUnique = takeWhile (("m20260929_ratchet_indexes" /=) . name) appMigrations
|
||||
Right st <- createDBStore (DBOpts testDB [] "" False True TQOff) beforeUnique (MigrationConfig MCError Nothing)
|
||||
withTransaction' st $ \db -> do
|
||||
SQL.execute_ db "INSERT INTO users (user_id) VALUES (1)"
|
||||
SQL.execute_ db "INSERT INTO connections (conn_id, conn_mode, user_id) VALUES (x'01', 'INV', 1), (x'02', 'INV', 1)"
|
||||
SQL.execute_ db "INSERT INTO processed_ratchet_key_hashes (conn_id, hash) VALUES (x'01', x'aa'), (x'01', x'aa'), (x'01', x'bb'), (x'02', x'aa'), (x'01', x'aa')"
|
||||
closeDBStore st
|
||||
Right st' <- createDBStore (DBOpts testDB [] "" False True TQOff) appMigrations (MigrationConfig MCYesUp Nothing)
|
||||
withTransaction' st' (`SQL.query_` "SELECT processed_ratchet_key_hash_id FROM processed_ratchet_key_hashes ORDER BY processed_ratchet_key_hash_id")
|
||||
`shouldReturn` [Only (1 :: Int), Only 3, Only 4]
|
||||
closeDBStore st'
|
||||
|
||||
skipComparisonForDownMigrations :: [String]
|
||||
skipComparisonForDownMigrations =
|
||||
[ -- on down migration idx_messages_internal_snd_id_ts index moves down to the end of the file
|
||||
"m20230814_indexes",
|
||||
-- snd_secure and last_broker_ts columns swap order on down migration
|
||||
"m20250322_short_links",
|
||||
-- on down migration idx_processed_ratchet_key_hashes_hash index moves down to the end of the file
|
||||
"m20260929_ratchet_indexes"
|
||||
"m20250322_short_links"
|
||||
]
|
||||
|
||||
getSchema :: FilePath -> FilePath -> IO String
|
||||
|
||||
@@ -71,7 +71,9 @@ initServers =
|
||||
}
|
||||
|
||||
testChooseDifferentOperator :: IO ()
|
||||
testChooseDifferentOperator = withAgent 1 agentCfg initServers testDB $ \c -> runRight_ $ do
|
||||
testChooseDifferentOperator = do
|
||||
c <- getSMPAgentClient' 1 agentCfg initServers testDB
|
||||
runRight_ $ do
|
||||
-- chooses the only operator with storage role
|
||||
srv1 <- withAgentEnv c $ getNextServer c 1 storageSrvs []
|
||||
liftIO $ srv1 == testOp1Srv1 || srv1 == testOp1Srv2 `shouldBe` True
|
||||
|
||||
@@ -6,9 +6,8 @@
|
||||
|
||||
module CoreTests.CryptoTests (cryptoTests) where
|
||||
|
||||
import Control.Concurrent (forkIO, newEmptyMVar, putMVar, takeMVar)
|
||||
import Control.Concurrent.STM
|
||||
import Control.Exception (bracket, evaluate)
|
||||
import Control.Exception (evaluate)
|
||||
import Control.Monad.Except
|
||||
import qualified Data.Aeson as J
|
||||
import qualified Data.ByteString.Char8 as B
|
||||
@@ -23,12 +22,9 @@ import Data.Time.Clock (UTCTime (..))
|
||||
import qualified Data.Text.Lazy as LT
|
||||
import qualified Data.Text.Lazy.Encoding as LE
|
||||
import Data.Type.Equality
|
||||
import Data.Word (Word8)
|
||||
import qualified Data.X509 as X
|
||||
import qualified Data.X509.CertificateStore as XS
|
||||
import qualified Data.X509.Validation as XV
|
||||
import Foreign (FunPtr, allocaBytes, fillBytes, freeHaskellFunPtr, nullPtr)
|
||||
import Foreign.C.Types (CInt, CSize (..))
|
||||
import qualified SMPClient
|
||||
import qualified Simplex.Messaging.Crypto as C
|
||||
import qualified Simplex.Messaging.Crypto.Lazy as LC
|
||||
@@ -36,12 +32,9 @@ import Simplex.Messaging.Crypto.BBS
|
||||
import Simplex.Messaging.Crypto.Entitlement
|
||||
import Simplex.Messaging.Crypto.SNTRUP761.Bindings
|
||||
import Simplex.Messaging.Crypto.SNTRUP761.Bindings.Defines
|
||||
import Simplex.Messaging.Crypto.SNTRUP761.Bindings.FFI (c_sntrup761_keypair)
|
||||
import Simplex.Messaging.Crypto.SNTRUP761.Bindings.RNG (RNGFunc)
|
||||
import Simplex.Messaging.Encoding (Large (..), smpDecode, smpEncode)
|
||||
import Simplex.Messaging.Encoding.String (strDecode, strEncode)
|
||||
import Simplex.Messaging.Transport.Client
|
||||
import System.Timeout (timeout)
|
||||
import Test.Hspec hiding (fit, it)
|
||||
import Test.Hspec.QuickCheck (modifyMaxSuccess)
|
||||
import Test.QuickCheck hiding (Large)
|
||||
@@ -120,7 +113,6 @@ cryptoTests = do
|
||||
describe "sntrup761" $ do
|
||||
it "should enc/dec key" testSNTRUP761
|
||||
it "should reject malformed KEM encodings" testSNTRUP761RejectsMalformedEncodings
|
||||
it "should fail key generation with degenerate RNG" testSNTRUP761KeypairDegenerateRNG
|
||||
describe "BBS+" $ do
|
||||
it "should sign and verify" testBBSSignVerify
|
||||
it "should derive public key from secret key" testBBSPublicKeyDerivation
|
||||
@@ -306,26 +298,6 @@ testSNTRUP761 = do
|
||||
KEMSharedKey k' <- sntrup761Dec c sk
|
||||
k' `shouldBe` k
|
||||
|
||||
foreign import ccall "wrapper"
|
||||
mkRNGFunc :: RNGFunc -> IO (FunPtr RNGFunc)
|
||||
|
||||
testSNTRUP761KeypairDegenerateRNG :: IO ()
|
||||
testSNTRUP761KeypairDegenerateRNG = do
|
||||
-- constant byte 0 draws invertible g = -(1 + x + ... + x^760), byte 0x20 draws g = 0
|
||||
keypairWithConstantRNG 0 `shouldReturn` Just 0
|
||||
keypairWithConstantRNG 0x20 `shouldReturn` Just (-1)
|
||||
where
|
||||
keypairWithConstantRNG :: Word8 -> IO (Maybe CInt)
|
||||
keypairWithConstantRNG b = do
|
||||
result <- newEmptyMVar
|
||||
-- timeout cannot interrupt a foreign call, so the call runs in another thread
|
||||
_ <- forkIO $
|
||||
bracket (mkRNGFunc $ \_ sz buf -> fillBytes buf b (fromIntegral sz)) freeHaskellFunPtr $ \rng ->
|
||||
allocaBytes c_SNTRUP761_PUBLICKEY_SIZE $ \pkPtr ->
|
||||
allocaBytes c_SNTRUP761_SECRETKEY_SIZE $ \skPtr ->
|
||||
c_sntrup761_keypair pkPtr skPtr nullPtr rng >>= putMVar result
|
||||
timeout 10000000 $ takeMVar result
|
||||
|
||||
testSNTRUP761RejectsMalformedEncodings :: IO ()
|
||||
testSNTRUP761RejectsMalformedEncodings = do
|
||||
smpDecode @KEMPublicKey (smpEncode $ Large shortPublicKey) `shouldSatisfy` isLeft
|
||||
|
||||
+461
-167
@@ -16,7 +16,8 @@
|
||||
|
||||
module CoreTests.MsgStoreTests where
|
||||
|
||||
import AgentTests.FunctionalAPITests (runRight_)
|
||||
import AgentTests.FunctionalAPITests (runRight, runRight_)
|
||||
import Control.Concurrent (threadDelay)
|
||||
import Control.Concurrent.STM
|
||||
import Control.Exception (bracket)
|
||||
import Control.Monad
|
||||
@@ -24,56 +25,81 @@ import Control.Monad.IO.Class
|
||||
import Control.Monad.Trans.Except
|
||||
import Crypto.Random (ChaChaDRG)
|
||||
import Data.ByteString.Char8 (ByteString)
|
||||
import qualified Data.Map.Strict as M
|
||||
import Data.Time.Clock.System (getSystemTime)
|
||||
import qualified Data.ByteString.Char8 as B
|
||||
import Data.Int (Int64)
|
||||
import Data.List (isPrefixOf, isSuffixOf)
|
||||
import Data.Maybe (fromJust)
|
||||
import Data.Time.Clock (addUTCTime)
|
||||
import Data.Time.Clock.System (SystemTime (..), getSystemTime)
|
||||
import SMPClient (testStoreLogFile, testStoreMsgsDir, testStoreMsgsDir2, testStoreMsgsFile, testStoreMsgsFile2)
|
||||
import Simplex.Messaging.Crypto (pattern MaxLenBS)
|
||||
import qualified Simplex.Messaging.Crypto as C
|
||||
import Simplex.Messaging.Protocol (EncDataBytes (..), EntityId (..), ErrorType (..), LinkId, Message (..), QueueLinkData, RecipientId, SParty (..), noMsgFlags)
|
||||
import Simplex.Messaging.Protocol (EntityId (..), ErrorType, LinkId, Message (..), QueueLinkData, RecipientId, SParty (..), noMsgFlags)
|
||||
import Simplex.Messaging.Server (exportMessages, importMessages, printMessageStats)
|
||||
import Simplex.Messaging.Server.Env.STM (MsgStore (..), journalMsgStoreDepth, readWriteQueueStore)
|
||||
import Simplex.Messaging.Server.Expiration (ExpirationConfig (..), expireBeforeEpoch)
|
||||
import Simplex.Messaging.Server.MsgStore.Journal
|
||||
import Simplex.Messaging.Server.MsgStore.STM
|
||||
import Simplex.Messaging.Server.MsgStore.Types
|
||||
import Simplex.Messaging.Server.QueueStore
|
||||
import Simplex.Messaging.Server.QueueStore.QueueInfo
|
||||
import Simplex.Messaging.Server.QueueStore.STM (STMQueueStore (..))
|
||||
import Simplex.Messaging.Server.QueueStore.Types
|
||||
import Simplex.Messaging.TMap (TMap)
|
||||
import Simplex.Messaging.Server.StoreLog (closeStoreLog, logCreateQueue)
|
||||
import System.Directory (copyFile, createDirectoryIfMissing, listDirectory, removeFile, renameFile)
|
||||
import System.FilePath ((</>))
|
||||
import System.IO (IOMode (..), withFile)
|
||||
import Test.Hspec hiding (fit, it)
|
||||
import Util
|
||||
|
||||
#if defined(dbServerPostgres)
|
||||
import AgentTests.FunctionalAPITests (runRight)
|
||||
import Control.Concurrent (threadDelay)
|
||||
import Data.Int (Int64)
|
||||
import Data.Time.Clock.System (SystemTime (..))
|
||||
import Database.PostgreSQL.Simple (Only (..))
|
||||
import qualified Database.PostgreSQL.Simple as DB
|
||||
import Simplex.Messaging.Agent.Store.Postgres.Common
|
||||
import Simplex.Messaging.Agent.Store.Shared (MigrationConfirmation (..))
|
||||
import Simplex.Messaging.Server.MsgStore.Postgres
|
||||
import Simplex.Messaging.Server.QueueStore.Postgres
|
||||
import Simplex.Messaging.Server.QueueStore.Types
|
||||
import SMPClient (postgressBracket, testServerDBConnectInfo, testStoreDBOpts)
|
||||
#endif
|
||||
|
||||
msgStoreTests :: Spec
|
||||
msgStoreTests = do
|
||||
around (withMsgStore testSMTStoreConfig) $ describe "STM message store" $ do
|
||||
around (withMsgStore testSMTStoreConfig) $ describe "STM message store" someMsgStoreTests
|
||||
around (withMsgStore $ testJournalStoreCfg MQStoreCfg) $ describe "Journal message store" $ do
|
||||
someMsgStoreTests
|
||||
it "should remove deleted queues from queue store maps" testDeleteQueueMaps
|
||||
journalMsgStoreTests
|
||||
it "should export and import journal store" testExportImportStore
|
||||
#if defined(dbServerPostgres)
|
||||
around_ (postgressBracket testServerDBConnectInfo) $ do
|
||||
around (withMsgStore $ testJournalStoreCfg $ PQStoreCfg testPostgresStoreCfg) $
|
||||
describe "Postgres+journal message store" $ do
|
||||
someMsgStoreTests
|
||||
journalMsgStoreTests
|
||||
around (withMsgStore testPostgresStoreConfig) $
|
||||
describe "Postgres-only message store" $ do
|
||||
someMsgStoreTests
|
||||
it "should correctly update message counts and canWrite flag" testUpdateMessageCounts
|
||||
it "tryDelPeekMsg (ACK not from NSE) should reset message counts when queue is empty" testResetMessageCounts
|
||||
it "should expire messages across commit batches" testExpireMessagesInBatches
|
||||
#endif
|
||||
describe "Journal message store: queue state backup expiration" $ do
|
||||
it "should remove old queue state backups" testRemoveQueueStateBackups
|
||||
it "should expire messages in idle queues" testExpireIdleQueues
|
||||
where
|
||||
journalMsgStoreTests :: SpecWith (JournalMsgStore s)
|
||||
journalMsgStoreTests = do
|
||||
describe "queue state" $ do
|
||||
it "should restore queue state from the last line" testQueueState
|
||||
it "should recover when message is written and state is not" testMessageState
|
||||
it "should remove journal files when queue is empty" testRemoveJournals
|
||||
describe "missing files" $ do
|
||||
it "should create read file when missing" testReadFileMissing
|
||||
it "should switch to write file when read file missing" testReadFileMissingSwitch
|
||||
it "should create write file when missing" testWriteFileMissing
|
||||
it "should create read file when read and write files are missing" testReadAndWriteFilesMissing
|
||||
someMsgStoreTests :: MsgStoreClass s => SpecWith s
|
||||
someMsgStoreTests = do
|
||||
it "should get queue and store/read messages" testGetQueue
|
||||
it "should write/ack messages" testWriteAckMessages
|
||||
it "should resolve sender ID equal to link ID of another queue" testLinkIdSenderIdCollision
|
||||
it "should not add link data to secured messaging queue" testLinkDataSecuredQueue
|
||||
it "should not fail on EOF when changing read journal" testChangeReadJournal
|
||||
|
||||
-- TODO constrain to STM stores?
|
||||
withMsgStore :: MsgStoreClass s => MsgStoreConfig s -> (s -> IO ()) -> IO ()
|
||||
@@ -82,6 +108,21 @@ withMsgStore cfg = bracket (newMsgStore cfg) closeMsgStore
|
||||
testSMTStoreConfig :: STMStoreConfig
|
||||
testSMTStoreConfig = STMStoreConfig {storePath = Nothing, quota = 3}
|
||||
|
||||
testJournalStoreCfg :: QStoreCfg s -> JournalStoreConfig s
|
||||
testJournalStoreCfg queueStoreCfg =
|
||||
JournalStoreConfig
|
||||
{ storePath = testStoreMsgsDir,
|
||||
pathParts = journalMsgStoreDepth,
|
||||
queueStoreCfg,
|
||||
quota = 3,
|
||||
maxMsgCount = 4,
|
||||
maxStateLines = 2,
|
||||
stateTailSize = 256,
|
||||
idleInterval = 21600,
|
||||
expireBackupsAfter = 0,
|
||||
keepMinBackups = 1
|
||||
}
|
||||
|
||||
#if defined(dbServerPostgres)
|
||||
testPostgresStoreConfig :: PostgresMsgStoreCfg
|
||||
testPostgresStoreConfig =
|
||||
@@ -110,6 +151,12 @@ mkMessage body = liftIO $ do
|
||||
pattern Msg :: ByteString -> Maybe Message
|
||||
pattern Msg s <- Just Message {msgBody = MaxLenBS s}
|
||||
|
||||
deriving instance Eq MsgQueueState
|
||||
|
||||
deriving instance Eq (JournalState t)
|
||||
|
||||
deriving instance Eq (SJournalType t)
|
||||
|
||||
testNewQueueRec :: TVar ChaChaDRG -> QueueMode -> IO (RecipientId, QueueRec)
|
||||
testNewQueueRec g qm = testNewQueueRecData g qm Nothing
|
||||
|
||||
@@ -136,18 +183,6 @@ testNewQueueRecData g qm queueData = do
|
||||
where
|
||||
rndId = atomically $ EntityId <$> C.randomBytes 24 g
|
||||
|
||||
testNtfCreds :: TVar ChaChaDRG -> IO NtfCreds
|
||||
testNtfCreds g = do
|
||||
(notifierKey, _) <- atomically $ C.generateAuthKeyPair C.SX25519 g
|
||||
(k, pk) <- atomically $ C.generateKeyPair @'C.X25519 g
|
||||
pure
|
||||
NtfCreds
|
||||
{ notifierId = EntityId "ijkl",
|
||||
notifierKey,
|
||||
rcvNtfDhSecret = C.dh' k pk,
|
||||
ntfServiceId = Nothing
|
||||
}
|
||||
|
||||
testGetQueue :: MsgStoreClass s => s -> IO ()
|
||||
testGetQueue ms = do
|
||||
g <- C.newRandom
|
||||
@@ -212,97 +247,76 @@ testWriteAckMessages ms = do
|
||||
void $ ExceptT $ deleteQueue ms q1
|
||||
void $ ExceptT $ deleteQueue ms q2
|
||||
|
||||
testLinkDataSecuredQueue :: MsgStoreClass s => s -> IO ()
|
||||
testLinkDataSecuredQueue ms = do
|
||||
testChangeReadJournal :: MsgStoreClass s => s -> IO ()
|
||||
testChangeReadJournal ms = do
|
||||
g <- C.newRandom
|
||||
(sKey, _) <- atomically $ C.generateAuthKeyPair C.SEd25519 g
|
||||
let st = queueStore ms
|
||||
ld = (EncDataBytes "fixed data", EncDataBytes "user data")
|
||||
rndId = atomically $ EntityId <$> C.randomBytes 24 g
|
||||
(rId, qr) <- testNewQueueRec g QMMessaging
|
||||
(cId, cqr) <- testNewQueueRec g QMContact
|
||||
lnkId <- rndId
|
||||
cLnkId <- rndId
|
||||
runRight_ $ do
|
||||
q <- ExceptT $ addQueue ms rId qr
|
||||
-- the handle is read before SKEY, as in a command that raced with it
|
||||
staleQ <- ExceptT $ getQueue ms SRecipient rId
|
||||
ExceptT $ secureQueue st q sKey
|
||||
liftIO $ addQueueLinkData st staleQ lnkId ld `shouldReturn` Left AUTH
|
||||
freshQ <- ExceptT $ getQueue ms SRecipient rId
|
||||
liftIO $ getQueueLinkData st freshQ lnkId `shouldReturn` Left AUTH
|
||||
cq <- ExceptT $ addQueue ms cId cqr
|
||||
ExceptT $ secureQueue st cq sKey
|
||||
ExceptT $ addQueueLinkData st cq cLnkId ld
|
||||
ld' <- ExceptT $ getQueueLinkData st cq cLnkId
|
||||
liftIO $ ld' `shouldBe` ld
|
||||
let write s = writeMsg ms q True =<< mkMessage s
|
||||
Just (Message {msgId = mId1}, True) <- write "message 1"
|
||||
(Msg "message 1", Nothing) <- tryDelPeekMsg ms q mId1
|
||||
Just (Message {msgId = mId2}, True) <- write "message 2"
|
||||
(Msg "message 2", Nothing) <- tryDelPeekMsg ms q mId2
|
||||
Just (Message {msgId = mId3}, True) <- write "message 3"
|
||||
(Msg "message 3", Nothing) <- tryDelPeekMsg ms q mId3
|
||||
Just (Message {msgId = mId4}, True) <- write "message 4"
|
||||
(Msg "message 4", Nothing) <- tryDelPeekMsg ms q mId4
|
||||
Just (Message {msgId = mId5}, True) <- write "message 5"
|
||||
(Msg "message 5", Nothing) <- tryDelPeekMsg ms q mId5
|
||||
void $ ExceptT $ deleteQueue ms q
|
||||
void $ ExceptT $ deleteQueue ms cq
|
||||
|
||||
-- sizes of queues, senders, links and notifiers maps
|
||||
type QueueMapSizes = (Int, Int, Int, Int)
|
||||
|
||||
queueMapSizes :: STMMsgStore -> IO QueueMapSizes
|
||||
queueMapSizes ms = (,,,) <$> size queues <*> size senders <*> size links <*> size notifiers
|
||||
where
|
||||
STMQueueStore {queues, senders, links, notifiers} = queueStore ms
|
||||
size :: TMap k v -> IO Int
|
||||
size = fmap M.size . readTVarIO
|
||||
|
||||
testDeleteQueueMaps :: STMMsgStore -> IO ()
|
||||
testDeleteQueueMaps ms = do
|
||||
testExportImportStore :: JournalMsgStore 'QSMemory -> IO ()
|
||||
testExportImportStore ms = do
|
||||
g <- C.newRandom
|
||||
ntfCreds <- testNtfCreds g
|
||||
let newLinkId = atomically $ EntityId <$> C.randomBytes 24 g
|
||||
lnkId1 <- newLinkId
|
||||
lnkId2 <- newLinkId
|
||||
lnkId3 <- newLinkId
|
||||
(rId1, qr1) <- testNewQueueRec g QMMessaging
|
||||
(rId2, qr2) <- testNewQueueRecData g QMContact (Just (lnkId1, testLinkData))
|
||||
(rId3, qr3) <- testNewQueueRec g QMMessaging
|
||||
(rId4, qr4) <- testNewQueueRec g QMMessaging
|
||||
let rIds = [rId1, rId2, rId3, rId4] :: [RecipientId]
|
||||
sIds = map senderId [qr1, qr2, qr3, qr4]
|
||||
lnkIds = [lnkId1, lnkId2, lnkId3] :: [LinkId]
|
||||
queueMapSizes ms `shouldReturn` (0, 0, 0, 0)
|
||||
(rId2, qr2) <- testNewQueueRec g QMMessaging
|
||||
sl <- readWriteQueueStore True (mkQueue ms True) testStoreLogFile $ stmQueueStore ms
|
||||
runRight_ $ do
|
||||
q1 <- ExceptT $ addQueue ms rId1 qr1 {notifier = Just ntfCreds}
|
||||
let write q s = writeMsg ms q True =<< mkMessage s
|
||||
q1 <- ExceptT $ addQueue ms rId1 qr1
|
||||
liftIO $ logCreateQueue sl rId1 qr1
|
||||
Just (Message {}, True) <- write q1 "message 1"
|
||||
Just (Message {}, False) <- write q1 "message 2"
|
||||
q2 <- ExceptT $ addQueue ms rId2 qr2
|
||||
q3 <- ExceptT $ addQueue ms rId3 qr3
|
||||
q4 <- ExceptT $ addQueue ms rId4 qr4
|
||||
ExceptT $ addQueueLinkData (queueStore ms) q3 lnkId2 testLinkData
|
||||
ExceptT $ addQueueLinkData (queueStore ms) q4 lnkId3 testLinkData
|
||||
forM_ sIds $ void . ExceptT . getQueue ms SSender
|
||||
forM_ lnkIds $ void . ExceptT . getQueue ms SSenderLink
|
||||
liftIO $ queueMapSizes ms `shouldReturn` (4, 4, 3, 1)
|
||||
ExceptT $ deleteQueueLinkData (queueStore ms) q3
|
||||
liftIO $ queueMapSizes ms `shouldReturn` (4, 4, 2, 1)
|
||||
forM_ ([q1, q2, q3, q4] :: [STMQueue]) $ void . ExceptT . deleteQueue ms
|
||||
queueMapSizes ms `shouldReturn` (0, 0, 0, 0)
|
||||
forM_ rIds $ \rId -> getQueue ms SRecipient rId >>= expectAuth
|
||||
forM_ sIds $ \sId -> getQueue ms SSender sId >>= expectAuth
|
||||
forM_ lnkIds $ \lnkId -> getQueue ms SSenderLink lnkId >>= expectAuth
|
||||
queueMapSizes ms `shouldReturn` (0, 0, 0, 0)
|
||||
where
|
||||
expectAuth = either (`shouldBe` AUTH) (\_ -> expectationFailure "deleted queue is still found")
|
||||
liftIO $ logCreateQueue sl rId2 qr2
|
||||
Just (Message {msgId = mId3}, True) <- write q2 "message 3"
|
||||
Just (Message {msgId = mId4}, False) <- write q2 "message 4"
|
||||
(Msg "message 3", Msg "message 4") <- tryDelPeekMsg ms q2 mId3
|
||||
(Msg "message 4", Nothing) <- tryDelPeekMsg ms q2 mId4
|
||||
Just (Message {}, True) <- write q2 "message 5"
|
||||
Just (Message {}, False) <- write q2 "message 6"
|
||||
Just (Message {}, False) <- write q2 "message 7"
|
||||
Nothing <- write q2 "message 8"
|
||||
pure ()
|
||||
length <$> listDirectory (msgQueueDirectory ms rId1) `shouldReturn` 2
|
||||
length <$> listDirectory (msgQueueDirectory ms rId2) `shouldReturn` 3
|
||||
exportMessages False (StoreJournal ms) testStoreMsgsFile False
|
||||
closeMsgStore ms
|
||||
closeStoreLog sl
|
||||
-- export with closed queues and compare
|
||||
ms2 <- newMsgStore $ testJournalStoreCfg MQStoreCfg
|
||||
readWriteQueueStore True (mkQueue ms2 True) testStoreLogFile (stmQueueStore ms2) >>= closeStoreLog
|
||||
exportMessages False (StoreJournal ms2) (testStoreMsgsFile <> ".copy") False
|
||||
s <- B.readFile testStoreMsgsFile
|
||||
B.readFile (testStoreMsgsFile <> ".copy") `shouldReturn` s
|
||||
|
||||
testLinkData :: QueueLinkData
|
||||
testLinkData = (EncDataBytes "fixed data", EncDataBytes "user data")
|
||||
|
||||
testLinkIdSenderIdCollision :: MsgStoreClass s => s -> IO ()
|
||||
testLinkIdSenderIdCollision ms = do
|
||||
g <- C.newRandom
|
||||
(rIdV, qrV) <- testNewQueueRec g QMContact
|
||||
(rIdA, qrA) <- testNewQueueRec g QMContact
|
||||
let sIdV = senderId qrV
|
||||
runRight_ $ do
|
||||
void $ ExceptT $ addQueue ms rIdV qrV
|
||||
qA <- ExceptT $ addQueue ms rIdA qrA
|
||||
ExceptT $ addQueueLinkData (queueStore ms) qA sIdV testLinkData
|
||||
qA' <- ExceptT $ getQueue ms SSenderLink sIdV
|
||||
liftIO $ recipientId qA' `shouldBe` rIdA
|
||||
qV <- ExceptT $ getQueue ms SSender sIdV
|
||||
liftIO $ recipientId qV `shouldBe` rIdV
|
||||
let cfg = (testJournalStoreCfg MQStoreCfg :: JournalStoreConfig 'QSMemory) {storePath = testStoreMsgsDir2}
|
||||
ms' <- newMsgStore cfg
|
||||
readWriteQueueStore True (mkQueue ms' True) testStoreLogFile (stmQueueStore ms') >>= closeStoreLog
|
||||
stats@MessageStats {storedMsgsCount = 5, expiredMsgsCount = 0, storedQueues = 2} <-
|
||||
importMessages False ms' testStoreMsgsFile Nothing False
|
||||
printMessageStats "Messages" stats
|
||||
length <$> listDirectory (msgQueueDirectory ms rId1) `shouldReturn` 2
|
||||
length <$> listDirectory (msgQueueDirectory ms rId2) `shouldReturn` 3 -- 2 message files
|
||||
exportMessages False (StoreJournal ms') testStoreMsgsFile2 False
|
||||
(B.readFile testStoreMsgsFile2 `shouldReturn`) =<< B.readFile (testStoreMsgsFile <> ".bak")
|
||||
stmStore <- newMsgStore testSMTStoreConfig
|
||||
readWriteQueueStore True (mkQueue stmStore True) testStoreLogFile (queueStore stmStore) >>= closeStoreLog
|
||||
MessageStats {storedMsgsCount = 5, expiredMsgsCount = 0, storedQueues = 2} <-
|
||||
importMessages False stmStore testStoreMsgsFile2 Nothing False
|
||||
exportMessages False (StoreMemory stmStore) testStoreMsgsFile False
|
||||
(B.sort <$> B.readFile testStoreMsgsFile `shouldReturn`) =<< (B.sort <$> B.readFile (testStoreMsgsFile2 <> ".bak"))
|
||||
|
||||
#if defined(dbServerPostgres)
|
||||
testUpdateMessageCounts :: PostgresMsgStore -> IO ()
|
||||
@@ -313,34 +327,35 @@ testUpdateMessageCounts ms = do
|
||||
q <- ExceptT $ addQueue ms rId qr
|
||||
let write s = writeMsg ms q True =<< mkMessage s
|
||||
hasSize = checkQueueSize ms
|
||||
q `hasSize` (0, True)
|
||||
q `hasSize` (0, True, False)
|
||||
Just (Message {msgId = mId1}, True) <- write "message 1"
|
||||
q `hasSize` (1, True)
|
||||
q `hasSize` (1, True, True)
|
||||
Just (Message {msgId = mId2}, False) <- write "message 2"
|
||||
q `hasSize` (2, True)
|
||||
q `hasSize` (2, True, True)
|
||||
Just (Message {msgId = mId3}, False) <- write "message 3"
|
||||
q `hasSize` (3, True)
|
||||
q `hasSize` (3, True, True)
|
||||
Nothing <- write "message 4"
|
||||
q `hasSize` (4, False)
|
||||
q `hasSize` (4, False, True)
|
||||
Msg "message 1" <- tryPeekMsg ms q
|
||||
q `hasSize` (4, False)
|
||||
q `hasSize` (4, False, True)
|
||||
Msg "message 1" <- tryDelMsg ms q mId1
|
||||
q `hasSize` (3, False)
|
||||
q `hasSize` (3, False, True)
|
||||
Msg "message 2" <- tryPeekMsg ms q
|
||||
(Msg "message 2", Msg "message 3") <- tryDelPeekMsg ms q mId2
|
||||
q `hasSize` (2, False)
|
||||
q `hasSize` (2, False, True)
|
||||
(Msg "message 3", Just MessageQuota {msgId = mId4}) <- tryDelPeekMsg ms q mId3
|
||||
q `hasSize` (1, False)
|
||||
q `hasSize` (1, False, True)
|
||||
(Just MessageQuota {}, Nothing) <- tryDelPeekMsg ms q mId4
|
||||
q `hasSize` (0, True)
|
||||
q `hasSize` (0, True, False)
|
||||
|
||||
checkQueueSize :: PostgresMsgStore -> PostgresQueue -> (Int64, Bool) -> ExceptT ErrorType IO ()
|
||||
checkQueueSize ms q (size, canWrt) = liftIO $ do
|
||||
[(size', canWrt')] <-
|
||||
checkQueueSize :: PostgresMsgStore -> PostgresQueue -> (Int64, Bool, Bool) -> ExceptT ErrorType IO ()
|
||||
checkQueueSize ms q (size, canWrt, expire) = liftIO $ do
|
||||
[(size', canWrt', expire')] <-
|
||||
withTransaction (dbStore $ queueStore ms) $ \db ->
|
||||
DB.query db "SELECT msg_queue_size, msg_can_write FROM msg_queues WHERE recipient_id = ?" (Only (recipientId q))
|
||||
DB.query db "SELECT msg_queue_size, msg_can_write, msg_queue_expire FROM msg_queues WHERE recipient_id = ?" (Only (recipientId q))
|
||||
size' `shouldBe` size
|
||||
canWrt' `shouldBe` canWrt
|
||||
expire' `shouldBe` expire
|
||||
|
||||
testResetMessageCounts :: PostgresMsgStore -> IO ()
|
||||
testResetMessageCounts ms = do
|
||||
@@ -354,62 +369,341 @@ testResetMessageCounts ms = do
|
||||
Just (Message {msgId = mId2}, False) <- write "message 2"
|
||||
Just (Message {msgId = mId3}, False) <- write "message 3"
|
||||
Nothing <- write "message 4"
|
||||
q `hasSize` (4, False)
|
||||
q `hasSize` (4, False, True)
|
||||
liftIO $ setIncorrectSize q (10, True)
|
||||
Nothing <- write "message 5"
|
||||
q `hasSize` (11, False)
|
||||
q `hasSize` (11, False, True)
|
||||
(Msg "message 1", Msg "message 2") <- tryDelPeekMsg ms q mId1
|
||||
q `hasSize` (10, False)
|
||||
q `hasSize` (10, False, True)
|
||||
(Msg "message 2", Msg "message 3") <- tryDelPeekMsg ms q mId2
|
||||
q `hasSize` (9, False)
|
||||
q `hasSize` (9, False, True)
|
||||
(Msg "message 3", Just MessageQuota {msgId = mId4}) <- tryDelPeekMsg ms q mId3
|
||||
q `hasSize` (8, False)
|
||||
q `hasSize` (8, False, True)
|
||||
(Just MessageQuota {}, Just MessageQuota {msgId = mId5}) <- tryDelPeekMsg ms q mId4
|
||||
q `hasSize` (7, False)
|
||||
q `hasSize` (7, False, True)
|
||||
(Just MessageQuota {}, Nothing) <- tryDelPeekMsg ms q mId5
|
||||
q `hasSize` (0, True) -- reset
|
||||
q `hasSize` (0, True, False) -- reset
|
||||
where
|
||||
setIncorrectSize :: PostgresQueue -> (Int64, Bool) -> IO ()
|
||||
setIncorrectSize q (size, canWrt) =
|
||||
void $ withTransaction (dbStore $ queueStore ms) $ \db ->
|
||||
DB.execute db "UPDATE msg_queues SET msg_queue_size = ?, msg_can_write = ? WHERE recipient_id = ?" (size, canWrt, recipientId q)
|
||||
|
||||
testExpireMessagesInBatches :: PostgresMsgStore -> IO ()
|
||||
testExpireMessagesInBatches ms = do
|
||||
g <- C.newRandom
|
||||
emptiedQs <- replicateM emptiedCount $ newQueue g
|
||||
partialQs <- replicateM partialCount $ newQueue g
|
||||
overQuotaQs <- replicateM overQuotaCount $ newQueue g
|
||||
quotaMsgs <- runRight $ do
|
||||
forM_ (emptiedQs <> partialQs) $ \q -> void $ write q "old 1"
|
||||
forM_ emptiedQs $ \q -> void $ write q "old 2"
|
||||
mapM fillPastQuota overQuotaQs
|
||||
-- msg_ts has second granularity, so the recent messages need a new second to be kept
|
||||
threadDelay 1100000
|
||||
boundary <- systemSeconds <$> getSystemTime
|
||||
runRight_ $ forM_ partialQs $ \q -> void $ write q "recent"
|
||||
|
||||
MessageStats {expiredMsgsCount, storedMsgsCount, storedQueues} <- expireOldMessages False ms boundary 0
|
||||
expiredMsgsCount `shouldBe` (emptiedCount * 2 + partialCount + sum quotaMsgs)
|
||||
storedMsgsCount `shouldBe` (partialCount + overQuotaCount) -- recent messages and quota markers
|
||||
storedQueues `shouldBe` (emptiedCount + partialCount + overQuotaCount)
|
||||
runRight_ $ do
|
||||
forM_ emptiedQs $ \q -> checkQueueSize ms q (0, True)
|
||||
forM_ partialQs $ \q -> checkQueueSize ms q (1, True)
|
||||
-- the quota marker is never expired, and the queue stays blocked until it is acked
|
||||
forM_ overQuotaQs $ \q -> checkQueueSize ms q (1, False)
|
||||
where
|
||||
-- expire_old_messages pages through expired messages and commits per page, so these
|
||||
-- counts put a page boundary inside each group of queues.
|
||||
emptiedCount = 120 :: Int
|
||||
partialCount = 40 :: Int
|
||||
overQuotaCount = 10 :: Int
|
||||
newQueue g = do
|
||||
(rId, qr) <- testNewQueueRec g QMMessaging
|
||||
runRight $ ExceptT $ addQueue ms rId qr
|
||||
write q s = writeMsg ms q True =<< mkMessage s
|
||||
fillPastQuota q = go 0
|
||||
where
|
||||
go n = write q "fill" >>= maybe (pure n) (const $ go (n + 1))
|
||||
#endif
|
||||
|
||||
testQueueState :: JournalMsgStore s -> IO ()
|
||||
testQueueState ms = do
|
||||
g <- C.newRandom
|
||||
rId <- EntityId <$> atomically (C.randomBytes 24 g)
|
||||
let dir = msgQueueDirectory ms rId
|
||||
statePath = msgQueueStatePath dir rId
|
||||
createDirectoryIfMissing True dir
|
||||
state <- newMsgQueueState <$> newJournalId (random ms)
|
||||
withFile statePath WriteMode (`appendState` state)
|
||||
length . lines <$> readFile statePath `shouldReturn` 1
|
||||
readQueueState ms statePath `shouldReturn` (Just state, False)
|
||||
length <$> listDirectory dir `shouldReturn` 1 -- no backup
|
||||
let state1 =
|
||||
state
|
||||
{ size = 1,
|
||||
readState = (readState state) {msgCount = 1, byteCount = 100},
|
||||
writeState = (writeState state) {msgPos = 1, msgCount = 1, bytePos = 100, byteCount = 100}
|
||||
}
|
||||
withFile statePath AppendMode (`appendState` state1)
|
||||
length . lines <$> readFile statePath `shouldReturn` 2
|
||||
readQueueState ms statePath `shouldReturn` (Just state1, False)
|
||||
length <$> listDirectory dir `shouldReturn` 1 -- no backup
|
||||
let state2 =
|
||||
state
|
||||
{ size = 2,
|
||||
readState = (readState state) {msgCount = 2, byteCount = 200},
|
||||
writeState = (writeState state) {msgPos = 2, msgCount = 2, bytePos = 200, byteCount = 200}
|
||||
}
|
||||
withFile statePath AppendMode (`appendState` state2)
|
||||
length . lines <$> readFile statePath `shouldReturn` 3
|
||||
copyFile statePath (statePath <> ".2")
|
||||
readQueueState ms statePath `shouldReturn` (Just state2, True)
|
||||
length <$> listDirectory dir `shouldReturn` 2 -- new state + copy
|
||||
ls <- lines <$> readFile statePath
|
||||
length ls `shouldBe` 3
|
||||
-- mock compacting file
|
||||
writeFile statePath $ last ls
|
||||
|
||||
-- corrupt the only line
|
||||
corruptFile statePath
|
||||
(Nothing, True) <- readQueueState ms statePath
|
||||
|
||||
-- corrupt the last line
|
||||
renameFile (statePath <> ".2") statePath
|
||||
removeOtherFiles dir statePath
|
||||
length . lines <$> readFile statePath `shouldReturn` 3
|
||||
corruptFile statePath
|
||||
readQueueState ms statePath `shouldReturn` (Just state1, True)
|
||||
length <$> listDirectory dir `shouldReturn` 1
|
||||
length . lines <$> readFile statePath `shouldReturn` 3
|
||||
where
|
||||
corruptFile f = do
|
||||
s <- readFile f
|
||||
removeFile f
|
||||
writeFile f $ take (length s - 4) s
|
||||
removeOtherFiles dir keep = do
|
||||
names <- listDirectory dir
|
||||
forM_ names $ \name ->
|
||||
let f = dir </> name
|
||||
in unless (f == keep) $ removeFile f
|
||||
|
||||
testMessageState :: JournalMsgStore s -> IO ()
|
||||
testMessageState ms = do
|
||||
g <- C.newRandom
|
||||
(rId, qr) <- testNewQueueRec g QMMessaging
|
||||
let dir = msgQueueDirectory ms rId
|
||||
statePath = msgQueueStatePath dir rId
|
||||
write q s = writeMsg ms q True =<< mkMessage s
|
||||
|
||||
mId1 <- runRight $ do
|
||||
q <- ExceptT $ addQueue ms rId qr
|
||||
Just (Message {msgId = mId1}, True) <- write q "message 1"
|
||||
Just (Message {}, False) <- write q "message 2"
|
||||
liftIO $ closeMsgQueue ms q
|
||||
pure mId1
|
||||
|
||||
ls <- B.lines <$> B.readFile statePath
|
||||
B.writeFile statePath $ B.unlines $ take (length ls - 1) ls
|
||||
|
||||
runRight_ $ do
|
||||
q <- ExceptT $ getQueue ms SRecipient rId
|
||||
Just (Message {msgId = mId3}, False) <- write q "message 3"
|
||||
(Msg "message 1", Msg "message 3") <- tryDelPeekMsg ms q mId1
|
||||
(Msg "message 3", Nothing) <- tryDelPeekMsg ms q mId3
|
||||
liftIO $ closeMsgQueue ms q
|
||||
|
||||
testRemoveJournals :: JournalMsgStore s -> IO ()
|
||||
testRemoveJournals ms = do
|
||||
g <- C.newRandom
|
||||
(rId, qr) <- testNewQueueRec g QMMessaging
|
||||
let dir = msgQueueDirectory ms rId
|
||||
statePath = msgQueueStatePath dir rId
|
||||
write q s = writeMsg ms q True =<< mkMessage s
|
||||
|
||||
runRight $ do
|
||||
q <- ExceptT $ addQueue ms rId qr
|
||||
Just (Message {msgId = mId1}, True) <- write q "message 1"
|
||||
Just (Message {msgId = mId2}, False) <- write q "message 2"
|
||||
(Msg "message 1", Msg "message 2") <- tryDelPeekMsg ms q mId1
|
||||
(Msg "message 2", Nothing) <- tryDelPeekMsg ms q mId2
|
||||
liftIO $ closeMsgQueue ms q
|
||||
|
||||
ls <- B.lines <$> B.readFile statePath
|
||||
length ls `shouldBe` 4
|
||||
journalFilesCount dir `shouldReturn` 1
|
||||
stateBackupCount dir `shouldReturn` 0
|
||||
|
||||
runRight $ do
|
||||
q <- ExceptT $ getQueue ms SRecipient rId
|
||||
-- not removed yet
|
||||
liftIO $ journalFilesCount dir `shouldReturn` 1
|
||||
liftIO $ stateBackupCount dir `shouldReturn` 0
|
||||
Nothing <- tryPeekMsg ms q
|
||||
-- still not removed, queue is empty and not opened
|
||||
liftIO $ journalFilesCount dir `shouldReturn` 1
|
||||
_mq <- isolateQueue ms q "test" $ getMsgQueue ms q False
|
||||
-- journal is removed
|
||||
liftIO $ journalFilesCount dir `shouldReturn` 0
|
||||
liftIO $ stateBackupCount dir `shouldReturn` 1
|
||||
Just (Message {msgId = mId3}, True) <- write q "message 3"
|
||||
-- journal is created
|
||||
liftIO $ journalFilesCount dir `shouldReturn` 1
|
||||
Just (Message {msgId = mId4}, False) <- write q "message 4"
|
||||
(Msg "message 3", Msg "message 4") <- tryDelPeekMsg ms q mId3
|
||||
(Msg "message 4", Nothing) <- tryDelPeekMsg ms q mId4
|
||||
Just (Message {msgId = mId5}, True) <- write q "message 5"
|
||||
Just (Message {msgId = mId6}, False) <- write q "message 6"
|
||||
liftIO $ journalFilesCount dir `shouldReturn` 1
|
||||
Just (Message {msgId = mId7}, False) <- write q "message 7"
|
||||
-- separate write journal is created
|
||||
liftIO $ journalFilesCount dir `shouldReturn` 2
|
||||
Nothing <- write q "message 8"
|
||||
(Msg "message 5", Msg "message 6") <- tryDelPeekMsg ms q mId5
|
||||
liftIO $ journalFilesCount dir `shouldReturn` 2
|
||||
(Msg "message 6", Msg "message 7") <- tryDelPeekMsg ms q mId6
|
||||
-- read journal is removed
|
||||
liftIO $ journalFilesCount dir `shouldReturn` 1
|
||||
(Msg "message 7", Just MessageQuota {msgId = mId8}) <- tryDelPeekMsg ms q mId7
|
||||
(Just MessageQuota {}, Nothing) <- tryDelPeekMsg ms q mId8
|
||||
liftIO $ closeMsgQueue ms q
|
||||
|
||||
journalFilesCount dir `shouldReturn` 1
|
||||
runRight $ do
|
||||
q <- ExceptT $ getQueue ms SRecipient rId
|
||||
Just (Message {}, True) <- write q "message 8"
|
||||
liftIO $ journalFilesCount dir `shouldReturn` 1
|
||||
liftIO $ stateBackupCount dir `shouldReturn` 2
|
||||
liftIO $ closeMsgQueue ms q
|
||||
where
|
||||
journalFilesCount dir = length . filter ("messages." `isPrefixOf`) <$> listDirectory dir
|
||||
stateBackupCount dir = length . filter (".bak" `isSuffixOf`) <$> listDirectory dir
|
||||
|
||||
testRemoveQueueStateBackups :: IO ()
|
||||
testRemoveQueueStateBackups = do
|
||||
g <- C.newRandom
|
||||
(rId, qr) <- testNewQueueRec g QMMessaging
|
||||
|
||||
ms' <- newMsgStore (testJournalStoreCfg MQStoreCfg) {maxStateLines = 1, expireBackupsAfter = 0, keepMinBackups = 0}
|
||||
-- set expiration time 1 second ahead
|
||||
let ms = ms' {expireBackupsBefore = addUTCTime 1 $ expireBackupsBefore ms'}
|
||||
|
||||
let dir = msgQueueDirectory ms rId
|
||||
write q s = writeMsg ms q True =<< mkMessage s
|
||||
|
||||
runRight $ do
|
||||
q <- ExceptT $ addQueue ms rId qr
|
||||
Just (Message {msgId = mId1}, True) <- write q "message 1"
|
||||
Just (Message {msgId = mId2}, False) <- write q "message 2"
|
||||
(Msg "message 1", Msg "message 2") <- tryDelPeekMsg ms q mId1
|
||||
(Msg "message 2", Nothing) <- tryDelPeekMsg ms q mId2
|
||||
liftIO $ closeMsgQueue ms q
|
||||
liftIO $ stateBackupCount dir `shouldReturn` 0
|
||||
|
||||
q1 <- ExceptT $ getQueue ms SRecipient rId
|
||||
Just (Message {}, True) <- write q1 "message 3"
|
||||
Just (Message {}, False) <- write q1 "message 4"
|
||||
liftIO $ closeMsgQueue ms q1
|
||||
liftIO $ stateBackupCount dir `shouldReturn` 0
|
||||
|
||||
liftIO $ threadDelay 1000000
|
||||
q2 <- ExceptT $ getQueue ms SRecipient rId
|
||||
Just (Message {}, False) <- write q2 "message 5"
|
||||
Nothing <- write q2 "message 5"
|
||||
liftIO $ closeMsgQueue ms q2
|
||||
liftIO $ stateBackupCount dir `shouldReturn` 1
|
||||
where
|
||||
stateBackupCount dir = length . filter (".bak" `isSuffixOf`) <$> listDirectory dir
|
||||
|
||||
testExpireIdleQueues :: IO ()
|
||||
testExpireIdleQueues = do
|
||||
g <- C.newRandom
|
||||
(rId, qr) <- testNewQueueRec g QMMessaging
|
||||
|
||||
ms <- newMsgStore (testJournalStoreCfg MQStoreCfg) {idleInterval = 0}
|
||||
|
||||
let dir = msgQueueDirectory ms rId
|
||||
statePath = msgQueueStatePath dir rId
|
||||
write q s = writeMsg ms q True =<< mkMessage s
|
||||
|
||||
q <- runRight $ do
|
||||
q <- ExceptT $ addQueue ms rId qr
|
||||
Just (Message {msgId = mId1}, True) <- write q "message 1"
|
||||
Just (Message {msgId = mId2}, False) <- write q "message 2"
|
||||
(Msg "message 1", Msg "message 2") <- tryDelPeekMsg ms q mId1
|
||||
(Msg "message 2", Nothing) <- tryDelPeekMsg ms q mId2
|
||||
liftIO $ closeMsgQueue ms q
|
||||
pure q
|
||||
|
||||
(Just MsgQueueState {size = 0, readState = rs, writeState = ws}, True) <- readQueueState ms statePath
|
||||
msgCount rs `shouldBe` 2
|
||||
msgCount ws `shouldBe` 2
|
||||
|
||||
old <- expireBeforeEpoch ExpirationConfig {ttl = 1, checkInterval = 1} -- no old messages
|
||||
now <- systemSeconds <$> getSystemTime
|
||||
|
||||
(expired_, stored) <- runRight $ isolateQueue ms q "" $ withIdleMsgQueue now ms q $ deleteExpireMsgs_ old q
|
||||
expired_ `shouldBe` Just 0
|
||||
stored `shouldBe` 0
|
||||
(Nothing, False) <- readQueueState ms statePath
|
||||
pure ()
|
||||
|
||||
testReadFileMissing :: JournalMsgStore s -> IO ()
|
||||
testReadFileMissing ms = do
|
||||
g <- C.newRandom
|
||||
(rId, qr) <- testNewQueueRec g QMMessaging
|
||||
let write q s = writeMsg ms q True =<< mkMessage s
|
||||
q <- runRight $ do
|
||||
q <- ExceptT $ addQueue ms rId qr
|
||||
Just (Message {}, True) <- write q "message 1"
|
||||
Msg "message 1" <- tryPeekMsg ms q
|
||||
pure q
|
||||
|
||||
mq <- fromJust <$> readTVarIO (msgQueue' q)
|
||||
MsgQueueState {readState = rs} <- readTVarIO $ state mq
|
||||
closeMsgQueue ms q
|
||||
let path = journalFilePath (queueDirectory $ queue mq) $ journalId rs
|
||||
removeFile path
|
||||
|
||||
runRight_ $ do
|
||||
q' <- ExceptT $ getQueue ms SRecipient rId
|
||||
Nothing <- tryPeekMsg ms q'
|
||||
Just (Message {}, True) <- write q' "message 2"
|
||||
Msg "message 2" <- tryPeekMsg ms q'
|
||||
pure ()
|
||||
|
||||
testReadFileMissingSwitch :: JournalMsgStore s -> IO ()
|
||||
testReadFileMissingSwitch ms = do
|
||||
g <- C.newRandom
|
||||
(rId, qr) <- testNewQueueRec g QMMessaging
|
||||
q <- writeMessages ms rId qr
|
||||
|
||||
mq <- fromJust <$> readTVarIO (msgQueue' q)
|
||||
MsgQueueState {readState = rs} <- readTVarIO $ state mq
|
||||
closeMsgQueue ms q
|
||||
let path = journalFilePath (queueDirectory $ queue mq) $ journalId rs
|
||||
removeFile path
|
||||
|
||||
runRight_ $ do
|
||||
q' <- ExceptT $ getQueue ms SRecipient rId
|
||||
Just (Message {}, False) <- writeMsg ms q' True =<< mkMessage "message 6"
|
||||
Msg "message 5" <- tryPeekMsg ms q'
|
||||
pure ()
|
||||
|
||||
testWriteFileMissing :: JournalMsgStore s -> IO ()
|
||||
testWriteFileMissing ms = do
|
||||
g <- C.newRandom
|
||||
(rId, qr) <- testNewQueueRec g QMMessaging
|
||||
q <- writeMessages ms rId qr
|
||||
|
||||
mq <- fromJust <$> readTVarIO (msgQueue' q)
|
||||
MsgQueueState {writeState = ws} <- readTVarIO $ state mq
|
||||
closeMsgQueue ms q
|
||||
let path = journalFilePath (queueDirectory $ queue mq) $ journalId ws
|
||||
print path
|
||||
removeFile path
|
||||
|
||||
runRight_ $ do
|
||||
q' <- ExceptT $ getQueue ms SRecipient rId
|
||||
Just Message {msgId = mId3} <- tryPeekMsg ms q'
|
||||
(Msg "message 3", Msg "message 4") <- tryDelPeekMsg ms q' mId3
|
||||
Just Message {msgId = mId4} <- tryPeekMsg ms q'
|
||||
(Msg "message 4", Nothing) <- tryDelPeekMsg ms q' mId4
|
||||
Just (Message {}, True) <- writeMsg ms q' True =<< mkMessage "message 6"
|
||||
Msg "message 6" <- tryPeekMsg ms q'
|
||||
pure ()
|
||||
|
||||
testReadAndWriteFilesMissing :: JournalMsgStore s -> IO ()
|
||||
testReadAndWriteFilesMissing ms = do
|
||||
g <- C.newRandom
|
||||
(rId, qr) <- testNewQueueRec g QMMessaging
|
||||
q <- writeMessages ms rId qr
|
||||
|
||||
mq <- fromJust <$> readTVarIO (msgQueue' q)
|
||||
MsgQueueState {readState = rs, writeState = ws} <- readTVarIO $ state mq
|
||||
closeMsgQueue ms q
|
||||
removeFile $ journalFilePath (queueDirectory $ queue mq) $ journalId rs
|
||||
removeFile $ journalFilePath (queueDirectory $ queue mq) $ journalId ws
|
||||
|
||||
runRight_ $ do
|
||||
q' <- ExceptT $ getQueue ms SRecipient rId
|
||||
Nothing <- tryPeekMsg ms q'
|
||||
Just (Message {}, True) <- writeMsg ms q' True =<< mkMessage "message 6"
|
||||
Msg "message 6" <- tryPeekMsg ms q'
|
||||
pure ()
|
||||
|
||||
writeMessages :: JournalMsgStore s -> RecipientId -> QueueRec -> IO (JournalQueue s)
|
||||
writeMessages ms rId qr = runRight $ do
|
||||
q <- ExceptT $ addQueue ms rId qr
|
||||
let write s = writeMsg ms q True =<< mkMessage s
|
||||
Just (Message {msgId = mId1}, True) <- write "message 1"
|
||||
Just (Message {msgId = mId2}, False) <- write "message 2"
|
||||
Just (Message {}, False) <- write "message 3"
|
||||
(Msg "message 1", Msg "message 2") <- tryDelPeekMsg ms q mId1
|
||||
(Msg "message 2", Msg "message 3") <- tryDelPeekMsg ms q mId2
|
||||
Just (Message {}, False) <- write "message 4"
|
||||
Just (Message {}, False) <- write "message 5"
|
||||
pure q
|
||||
|
||||
@@ -14,11 +14,8 @@ import Util
|
||||
retryIntervalTests :: Spec
|
||||
retryIntervalTests = do
|
||||
describe "Retry interval with 2 modes and lock" $ do
|
||||
testRetryIntervalSameMode withRetryNewLock2
|
||||
testRetryIntervalSwitchMode withRetryNewLock2
|
||||
describe "Retry interval with 2 modes" $ do
|
||||
testRetryIntervalSameMode withRetryInterval2
|
||||
testRetryIntervalSwitchMode withRetryInterval2
|
||||
testRetryIntervalSameMode
|
||||
testRetryIntervalSwitchMode
|
||||
describe "Foreground retry interval" $ do
|
||||
testRetryForeground
|
||||
testRetryToBackground
|
||||
@@ -44,18 +41,14 @@ testFastRI =
|
||||
maxInterval = 40000
|
||||
}
|
||||
|
||||
type WithRetry2 = RetryInterval2 -> (RI2State -> (RetryIntervalMode -> IO ()) -> IO ()) -> IO ()
|
||||
|
||||
withRetryNewLock2 :: WithRetry2
|
||||
withRetryNewLock2 ri action = newEmptyTMVarIO >>= \lock -> withRetryLock2 ri lock action
|
||||
|
||||
testRetryIntervalSameMode :: WithRetry2 -> Spec
|
||||
testRetryIntervalSameMode withRetry =
|
||||
testRetryIntervalSameMode :: Spec
|
||||
testRetryIntervalSameMode =
|
||||
it "should increase elapased time and interval when the mode stays the same" $ do
|
||||
lock <- newEmptyTMVarIO
|
||||
intervals <- newTVarIO []
|
||||
reportedIntervals <- newTVarIO []
|
||||
ts <- newTVarIO =<< getCurrentTime
|
||||
withRetry testRI $ \(RI2State slow fast) loop -> do
|
||||
withRetryLock2 testRI lock $ \(RI2State slow fast) loop -> do
|
||||
ints <- addInterval intervals ts
|
||||
atomically $ modifyTVar' reportedIntervals ((slow, fast) :)
|
||||
when (length ints < 9) $ loop RIFast
|
||||
@@ -72,13 +65,14 @@ testRetryIntervalSameMode withRetry =
|
||||
(20000, 40000)
|
||||
]
|
||||
|
||||
testRetryIntervalSwitchMode :: WithRetry2 -> Spec
|
||||
testRetryIntervalSwitchMode withRetry =
|
||||
testRetryIntervalSwitchMode :: Spec
|
||||
testRetryIntervalSwitchMode =
|
||||
it "should increase elapased time and interval when the mode switches" $ do
|
||||
lock <- newEmptyTMVarIO
|
||||
intervals <- newTVarIO []
|
||||
reportedIntervals <- newTVarIO []
|
||||
ts <- newTVarIO =<< getCurrentTime
|
||||
withRetry testRI $ \(RI2State slow fast) loop -> do
|
||||
withRetryLock2 testRI lock $ \(RI2State slow fast) loop -> do
|
||||
ints <- addInterval intervals ts
|
||||
atomically $ modifyTVar' reportedIntervals ((slow, fast) :)
|
||||
when (length ints < 11) $ loop $ if length ints <= 5 then RIFast else RISlow
|
||||
|
||||
@@ -2,18 +2,15 @@
|
||||
{-# LANGUAGE NamedFieldPuns #-}
|
||||
{-# LANGUAGE OverloadedLists #-}
|
||||
{-# LANGUAGE OverloadedStrings #-}
|
||||
{-# LANGUAGE PatternSynonyms #-}
|
||||
{-# LANGUAGE TypeApplications #-}
|
||||
{-# OPTIONS_GHC -fno-warn-ambiguous-fields #-}
|
||||
|
||||
module CoreTests.SOCKSSettings where
|
||||
|
||||
import Network.Socket (SockAddr (..), tupleToHostAddress)
|
||||
import Simplex.Messaging.Agent.Client (ipAddressProtected)
|
||||
import Simplex.Messaging.Client
|
||||
import qualified Simplex.Messaging.Crypto as C
|
||||
import Simplex.Messaging.Encoding.String
|
||||
import Simplex.Messaging.Protocol (ErrorType, pattern SMPServer)
|
||||
import Simplex.Messaging.Protocol (ErrorType)
|
||||
import Simplex.Messaging.Transport.Client
|
||||
import Test.Hspec hiding (fit, it)
|
||||
import Util
|
||||
@@ -22,7 +19,6 @@ socksSettingsTests :: Spec
|
||||
socksSettingsTests = do
|
||||
describe "hostMode and requiredHostMode settings" testHostMode
|
||||
describe "socksMode setting, independent of hostMode setting" testSocksMode
|
||||
describe "ipAddressProtected, consistent with chosen host and socksMode" testIPAddressProtected
|
||||
describe "socks proxy address encoding" testSocksProxyEncoding
|
||||
|
||||
testPublicHost :: TransportHost
|
||||
@@ -98,19 +94,6 @@ testSocksMode = do
|
||||
let TransportClientConfig {socksProxy} = transportClientConfig cfg NRMInteractive host False Nothing
|
||||
in socksProxy
|
||||
|
||||
testIPAddressProtected :: Spec
|
||||
testIPAddressProtected = do
|
||||
it "should be protected if SOCKS proxy is used for the chosen host" $ do
|
||||
protected SMAlways HMOnionViaSocks [testPublicHost] `shouldBe` True
|
||||
protected SMOnion HMOnionViaSocks [testPublicHost, testOnionHost] `shouldBe` True
|
||||
protected SMOnion HMPublic [testOnionHost] `shouldBe` True
|
||||
it "should not be protected if SOCKS proxy is not used for the chosen host" $ do
|
||||
protected SMOnion HMOnionViaSocks [testPublicHost] `shouldBe` False
|
||||
protected SMOnion HMPublic [testPublicHost, testOnionHost] `shouldBe` False
|
||||
where
|
||||
protected socksMode hostMode hosts =
|
||||
ipAddressProtected defaultNetworkConfig {socksProxy = Just defaultSocksProxyWithAuth, socksMode, hostMode} (SMPServer hosts "" (C.KeyHash ""))
|
||||
|
||||
testSocksProxyEncoding :: Spec
|
||||
testSocksProxyEncoding = do
|
||||
it "should decode SOCKS proxy with isolate-by-auth mode" $ do
|
||||
|
||||
@@ -22,13 +22,13 @@ import qualified Data.X509.Validation as XV
|
||||
import SMPClient
|
||||
import Simplex.FileTransfer.Protocol (FileInfo (..))
|
||||
import Simplex.FileTransfer.Server.Store (FileRec (..), FileRecipient (..), FileStoreClass (..), RoundedFileTime, STMFileStore (..))
|
||||
import Simplex.FileTransfer.Server.StoreLog (FileStoreLogRecord (..), readWriteFileStore, serverFileName)
|
||||
import Simplex.FileTransfer.Server.StoreLog (FileStoreLogRecord (..), readWriteFileStore)
|
||||
import qualified Simplex.Messaging.Crypto as C
|
||||
import Simplex.Messaging.Encoding.String
|
||||
import Simplex.Messaging.Protocol
|
||||
import Simplex.Messaging.Protocol.Types (ClientNotice (..))
|
||||
import Simplex.Messaging.Server.Env.STM (readWriteQueueStore)
|
||||
import Simplex.Messaging.Server.MsgStore.STM (STMMsgStore (..), STMStoreConfig (..))
|
||||
import Simplex.Messaging.Server.MsgStore.Journal
|
||||
import Simplex.Messaging.Server.MsgStore.Types
|
||||
import Simplex.Messaging.Server.QueueStore
|
||||
import Simplex.Messaging.Server.QueueStore.STM (STMQueueStore (..))
|
||||
@@ -37,7 +37,6 @@ import Simplex.Messaging.Server.StoreLog
|
||||
import Simplex.Messaging.SystemTime
|
||||
import Simplex.Messaging.Transport (SMPServiceRole (..))
|
||||
import Simplex.Messaging.Transport.Credentials (genCredentials)
|
||||
import System.FilePath ((</>))
|
||||
import Test.Hspec hiding (fit, it)
|
||||
import Util
|
||||
|
||||
@@ -48,6 +47,18 @@ import Simplex.Messaging.Server.Main
|
||||
testPublicAuthKey :: C.APublicAuthKey
|
||||
testPublicAuthKey = C.APublicAuthKey C.SEd25519 (C.publicKey "MC4CAQAwBQYDK2VwBCIEIDfEfevydXXfKajz3sRkcQ7RPvfWUPoq6pu1TYHV1DEe")
|
||||
|
||||
testNtfCreds :: TVar ChaChaDRG -> IO NtfCreds
|
||||
testNtfCreds g = do
|
||||
(notifierKey, _) <- atomically $ C.generateAuthKeyPair C.SX25519 g
|
||||
(k, pk) <- atomically $ C.generateKeyPair @'C.X25519 g
|
||||
pure
|
||||
NtfCreds
|
||||
{ notifierId = EntityId "ijkl",
|
||||
notifierKey,
|
||||
rcvNtfDhSecret = C.dh' k pk,
|
||||
ntfServiceId = Nothing
|
||||
}
|
||||
|
||||
data StoreLogTestCase r s = SLTC {name :: String, saved :: [r], state :: s, compacted :: [r]}
|
||||
|
||||
type SMPStoreLogTestCase = StoreLogTestCase StoreLogRecord (M.Map RecipientId QueueRec)
|
||||
@@ -165,10 +176,10 @@ testSMPStoreLog testSuite tests =
|
||||
closeStoreLog l
|
||||
replicateM_ 3 $ testReadWrite t
|
||||
#if defined(dbServerPostgres)
|
||||
(sCnt, qCnt) <- importStoreLogToDatabase testStoreLogFile testStoreDBOpts
|
||||
(sCnt, qCnt) <- importStoreLogToDatabase "tests/tmp/" testStoreLogFile testStoreDBOpts
|
||||
fromIntegral (sCnt + qCnt) `shouldBe` length (compacted t)
|
||||
imported <- B.readFile $ testStoreLogFile <> ".bak"
|
||||
(sCnt', qCnt') <- exportDatabaseToStoreLog testStoreDBOpts testStoreLogFile
|
||||
(sCnt', qCnt') <- exportDatabaseToStoreLog "tests/tmp/" testStoreDBOpts testStoreLogFile
|
||||
sCnt' `shouldBe` fromIntegral sCnt
|
||||
qCnt' `shouldBe` fromIntegral qCnt
|
||||
exported <- B.readFile testStoreLogFile
|
||||
@@ -176,14 +187,14 @@ testSMPStoreLog testSuite tests =
|
||||
#endif
|
||||
where
|
||||
testReadWrite SLTC {compacted, state} = do
|
||||
st <- newMsgStore STMStoreConfig {storePath = Nothing, quota = 3}
|
||||
l <- readWriteQueueStore True (mkQueue st) testStoreLogFile $ queueStore st
|
||||
st <- newMsgStore $ testJournalStoreCfg MQStoreCfg
|
||||
l <- readWriteQueueStore True (mkQueue st True) testStoreLogFile $ stmQueueStore st
|
||||
storeState st `shouldReturn` state
|
||||
closeStoreLog l
|
||||
([], compacted') <- partitionEithers . map strDecode . B.lines <$> B.readFile testStoreLogFile
|
||||
compacted' `shouldBe` compacted
|
||||
storeState :: STMMsgStore -> IO (M.Map RecipientId QueueRec)
|
||||
storeState st = M.mapMaybe id <$> (readTVarIO (queues $ queueStore st) >>= mapM (readTVarIO . queueRec))
|
||||
storeState :: JournalMsgStore 'QSMemory -> IO (M.Map RecipientId QueueRec)
|
||||
storeState st = M.mapMaybe id <$> (readTVarIO (queues $ stmQueueStore st) >>= mapM (readTVarIO . queueRec))
|
||||
|
||||
type FileRecState = (FileInfo, RoundedFileTime, Maybe RoundedFileTime, ServerEntityStatus)
|
||||
|
||||
@@ -258,18 +269,6 @@ fileStoreLogTests = do
|
||||
saved = [AddFile sId file createdAt Nothing EntityActive, BlockFile sId blockedWithNotice],
|
||||
compacted = [AddFile sId file createdAt Nothing (EntityBlocked blockedWithNotice)],
|
||||
state = M.fromList [(sId, (file, createdAt, Nothing, EntityBlocked blockedWithNotice))]
|
||||
},
|
||||
SLTC
|
||||
{ name = "put file with wrong file name",
|
||||
saved = [AddFile sId file createdAt (Just expiresAt) EntityActive, PutFile sId "/etc/passwd"],
|
||||
compacted = [AddFile sId file createdAt (Just expiresAt) EntityActive],
|
||||
state = M.fromList [(sId, (file, createdAt, Just expiresAt, EntityActive))]
|
||||
},
|
||||
SLTC
|
||||
{ name = "put file in another directory",
|
||||
saved = [AddFile sId file createdAt (Just expiresAt) EntityActive, PutFile sId ("/srv/xftp" </> serverFileName sId)],
|
||||
compacted = [AddFile sId file createdAt (Just expiresAt) EntityActive, PutFile sId ("/srv/xftp" </> serverFileName sId)],
|
||||
state = M.fromList [(sId, (file, createdAt, Just expiresAt, EntityActive))]
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ import Data.Word (Word32)
|
||||
import Simplex.FileTransfer.Protocol (FileInfo (..), SFileParty (..))
|
||||
import Simplex.FileTransfer.Server.Store
|
||||
import Simplex.FileTransfer.Server.Store.Postgres (PostgresFileStore, importFileStore, exportFileStore)
|
||||
import Simplex.FileTransfer.Server.StoreLog (closeStoreLog, readWriteFileStore, serverFileName, writeFileStore)
|
||||
import Simplex.FileTransfer.Server.StoreLog (closeStoreLog, readWriteFileStore, writeFileStore)
|
||||
import Simplex.FileTransfer.Transport (XFTPErrorType (..))
|
||||
import qualified Simplex.Messaging.Crypto as C
|
||||
import Simplex.Messaging.Protocol (BlockingInfo (..), BlockingReason (..), EntityId (..))
|
||||
@@ -17,7 +17,6 @@ import Simplex.Messaging.Server.QueueStore (ServerEntityStatus (..))
|
||||
import Simplex.Messaging.Server.StoreLog (openWriteStoreLog)
|
||||
import Simplex.Messaging.SystemTime (RoundedSystemTime (..))
|
||||
import System.Directory (doesFileExist, removeFile)
|
||||
import System.FilePath ((</>))
|
||||
import Test.Hspec hiding (fit, it)
|
||||
import UnliftIO.STM
|
||||
import Util
|
||||
@@ -266,7 +265,7 @@ testMigrationRoundTrip = do
|
||||
sId2 = EntityId "migration_file_2"
|
||||
rId1 = EntityId "migration_rcp_1_"
|
||||
addFile stmStore sId1 fileInfo1 testCreatedAt (Just testExpiresAt) EntityActive `shouldReturn` Right ()
|
||||
void $ setFilePath stmStore sId1 ("/tmp" </> serverFileName sId1)
|
||||
void $ setFilePath stmStore sId1 "/tmp/file1"
|
||||
addRecipient stmStore sId1 (FileRecipient rId1 rcpKey1) `shouldReturn` Right ()
|
||||
let testBlockInfo = BlockingInfo {reason = BRSpam, notice = Nothing}
|
||||
addFile stmStore sId2 fileInfo2 testCreatedAt Nothing (EntityBlocked testBlockInfo) `shouldReturn` Right ()
|
||||
@@ -290,7 +289,7 @@ testMigrationRoundTrip = do
|
||||
case result1 of
|
||||
Right (FileRec {fileInfo = fi, filePath, expiresAt, fileStatus}, _) -> do
|
||||
size fi `shouldBe` 128000
|
||||
readTVarIO filePath `shouldReturn` Just ("/tmp" </> serverFileName sId1)
|
||||
readTVarIO filePath `shouldReturn` Just "/tmp/file1"
|
||||
expiresAt `shouldBe` Just testExpiresAt
|
||||
readTVarIO fileStatus `shouldReturn` EntityActive
|
||||
Left e -> expectationFailure $ "getFile sId1 failed: " <> show e
|
||||
|
||||
@@ -47,12 +47,10 @@ withResolverServerDelayed delayMs handler action = do
|
||||
let (st, body) = handler (pathInfo req)
|
||||
send $ responseLBS st [(hContentType, "application/json")] body
|
||||
|
||||
-- | The resolver API is versioned on its own: v2 answers with NameRegistration
|
||||
-- JSON, which is the only shape the server asks for.
|
||||
resolveResp :: Status -> LB.ByteString -> [Text] -> (Status, LB.ByteString)
|
||||
resolveResp st body = \case
|
||||
["health"] -> (ok200, "{}")
|
||||
("v2" : "resolve" : _) -> (st, body)
|
||||
("resolve" : _) -> (st, body)
|
||||
_ -> (notFound404, "{}")
|
||||
|
||||
testNamesConfig :: Int -> NamesConfig
|
||||
|
||||
+36
-136
@@ -11,10 +11,13 @@
|
||||
|
||||
module RSLVTests (rslvTests) where
|
||||
|
||||
import Control.Concurrent (threadDelay)
|
||||
import Control.Monad (forM_)
|
||||
import Control.Monad.Trans.Except (ExceptT, runExceptT)
|
||||
import Data.IORef (readIORef)
|
||||
import qualified Data.Aeson as J
|
||||
import qualified Data.ByteString.Char8 as B
|
||||
import qualified Data.ByteString.Lazy as LB
|
||||
import Data.IORef (IORef, readIORef)
|
||||
import Data.List.NonEmpty (NonEmpty (..))
|
||||
import Data.Text (Text)
|
||||
import Data.Text.Encoding (encodeUtf8)
|
||||
@@ -26,18 +29,14 @@ import SMPClient
|
||||
import Simplex.Messaging.Client
|
||||
import qualified Simplex.Messaging.Crypto as C
|
||||
import Simplex.Messaging.Encoding.String (strDecode)
|
||||
import SMPNamesTests (availableBody, registeredBody, reservedBody, resolved, testNameRecord, testPricing)
|
||||
import SMPNamesTests (testNameRecord)
|
||||
import Simplex.Messaging.Protocol
|
||||
( BrokerMsg (..),
|
||||
Cmd (..),
|
||||
Command (..),
|
||||
CorrId (..),
|
||||
ErrorType (..),
|
||||
NameQuery (..),
|
||||
NameRegistration (..),
|
||||
NameResponse (..),
|
||||
NameErrorType (..),
|
||||
NameReservedReason (..),
|
||||
SParty (..),
|
||||
Transmission,
|
||||
TransmissionForAuth (..),
|
||||
@@ -61,11 +60,6 @@ withResolverServer (st, body) runTest =
|
||||
NRS.withResolverServer (NRS.resolveResp st body) $ \port _ ->
|
||||
withSmpServerConfigOn (transport @TLS) (withNames port memCfg) testPort (const runTest)
|
||||
|
||||
withResolverServerReqs :: (Status, LB.ByteString) -> (IORef [[Text]] -> IO a) -> IO a
|
||||
withResolverServerReqs (st, body) runTest =
|
||||
NRS.withResolverServer (NRS.resolveResp st body) $ \port reqs ->
|
||||
withSmpServerConfigOn (transport @TLS) (withNames port memCfg) testPort (const (runTest reqs))
|
||||
|
||||
withProxyAndResolver :: (Status, LB.ByteString) -> IO a -> IO a
|
||||
withProxyAndResolver (st, body) runTest =
|
||||
NRS.withResolverServer (NRS.resolveResp st body) $ \port _ ->
|
||||
@@ -74,7 +68,7 @@ withProxyAndResolver (st, body) runTest =
|
||||
|
||||
sendRslv :: Transport c => THandleSMP c 'TClient -> B.ByteString -> SimplexDomain -> IO (Transmission (Either ErrorType BrokerMsg))
|
||||
sendRslv h@THandle {params} corrId d = do
|
||||
let TransmissionForAuth {tToSend} = encodeTransmissionForAuth params (CorrId corrId, NoEntity, Cmd SResolver (RSLV (NQDomain d)))
|
||||
let TransmissionForAuth {tToSend} = encodeTransmissionForAuth params (CorrId corrId, NoEntity, Cmd SResolver (RSLV d))
|
||||
[Right ()] <- tPut h (Right (Nothing, tToSend) :| [])
|
||||
r :| _ <- tGetClient h
|
||||
pure r
|
||||
@@ -82,36 +76,40 @@ sendRslv h@THandle {params} corrId d = do
|
||||
rslvTests :: Spec
|
||||
rslvTests = do
|
||||
describe "RSLV direct (non-forwarded)" $ do
|
||||
it "resolver without the v2 route (404) -> NAME RESOLVER, not NOT_FOUND" testRslvBackendNotFound
|
||||
it "resolver replies 404 -> NAME NOT_FOUND (reached, not CMD PROHIBITED)" testRslvBackendNotFound
|
||||
it "resolver replies 502 -> NAME (RESOLVER ..)" testRslvBackendHttpErr
|
||||
it "no names config -> NAME NO_RESOLVER" testRslvDisabled
|
||||
it "refuses to send RSLV on a session below namesSMPVersion" testRslvVersion
|
||||
describe "RSLV forwarded (PFWD)" $ do
|
||||
it "PFWD-wrapped RSLV reaches resolver via proxy (PCEProtocolError (NAME RESOLVER))" testRslvForwarded
|
||||
it "PFWD-wrapped RSLV reaches resolver via proxy (PCEProtocolError (NAME NOT_FOUND))" testRslvForwarded
|
||||
it "PFWD-wrapped RSLV success returns RNAME (record JSON frames over the proxy)" testRslvForwardedSuccess
|
||||
describe "RSLV success path (RNAME response)" $ do
|
||||
it "returns RNAME with NameRecord" testRslvSuccess
|
||||
describe "RSLV availability (RNAME response)" $ do
|
||||
it "unregistered comes back AVAILABLE" testRslvAvailable
|
||||
it "reserved comes back with the reason" testRslvReserved
|
||||
it "PFWD-wrapped availability reaches the resolver" testRslvForwardedAvailable
|
||||
describe "RSLV below v22" $ do
|
||||
it "still resolves a name to its record" testRslvOldClientRecord
|
||||
it "still answers NAME NOT_FOUND for a name that does not resolve" testRslvOldClientNotFound
|
||||
describe "hashed lookups" $ do
|
||||
it "RSLV sends the 2LD as its hash" testRslvSendsTheHash
|
||||
it "a name with subnames is sent as text" testSubnameKeepsItsLabels
|
||||
it "a record naming a different name is rejected" testRslvWrongName
|
||||
describe "RSLV resource use" $
|
||||
xit "one connection must not fan out to many concurrent resolver requests" testRslvFanOut
|
||||
|
||||
testRslvFanOut :: IO ()
|
||||
testRslvFanOut =
|
||||
NRS.withResolverServerDelayed 3000 (NRS.resolveResp status200 "{}") $ \port reqs ->
|
||||
withSmpServerConfigOn (transport @TLS) (withNames port memCfg) testPort $ const $
|
||||
testSMPClient @TLS $ \(h@THandle {params} :: THandleSMP TLS 'TClient) -> do
|
||||
let k = 64 :: Int
|
||||
globalCap = 8 :: Int
|
||||
forM_ [1 .. k] $ \i -> do
|
||||
let TransmissionForAuth {tToSend} = encodeTransmissionForAuth params (CorrId (B.pack $ "fan" <> show i), NoEntity, Cmd SResolver (RSLV (domain "alice.simplex")))
|
||||
[Right ()] <- tPut h (Right (Nothing, tToSend) :| [])
|
||||
pure ()
|
||||
threadDelay 800000
|
||||
inFlight <- length . filter ((== ["resolve"]) . take 1) <$> readIORef reqs
|
||||
inFlight `shouldSatisfy` (<= globalCap)
|
||||
|
||||
-- | /v2/resolve answers 200, 400 or 502, so a 404 is a resolver that predates
|
||||
-- the route, not a name that does not exist.
|
||||
testRslvBackendNotFound :: IO ()
|
||||
testRslvBackendNotFound =
|
||||
withResolverServer (status404, "{}") $
|
||||
testSMPClient @TLS $ \h -> do
|
||||
(corrId, _entId, resp) <- sendRslv h "rs01" (domain "ghost.simplex")
|
||||
corrId `shouldBe` CorrId "rs01"
|
||||
resp `shouldBe` Right (ERR (NAME (RESOLVER "HTTP 404")))
|
||||
resp `shouldBe` Right (ERR (NAME NOT_FOUND))
|
||||
|
||||
testRslvBackendHttpErr :: IO ()
|
||||
testRslvBackendHttpErr =
|
||||
@@ -129,7 +127,7 @@ testRslvDisabled =
|
||||
|
||||
testRslvVersion :: IO ()
|
||||
testRslvVersion =
|
||||
withResolverServer (status200, registeredBody testNameRecord) $ do
|
||||
withResolverServer (status200, J.encode testNameRecord) $ do
|
||||
g <- C.newRandom
|
||||
ts <- getCurrentTime
|
||||
let srv = SMPServer testHost testPort testKeyHash
|
||||
@@ -141,7 +139,7 @@ testRslvVersion =
|
||||
Left (PCETransportError TEVersion) -> pure ()
|
||||
_ -> expectationFailure $ "expected Left (PCETransportError TEVersion), got: " <> show r
|
||||
|
||||
forwardedResolveAlice :: IO (Either SMPClientError (Either ProxyClientError SMP.NameResponse))
|
||||
forwardedResolveAlice :: IO (Either SMPClientError (Either ProxyClientError SMP.NameRecord))
|
||||
forwardedResolveAlice = do
|
||||
g <- C.newRandom
|
||||
ts <- getCurrentTime
|
||||
@@ -157,123 +155,25 @@ testRslvForwarded :: IO ()
|
||||
testRslvForwarded =
|
||||
withProxyAndResolver (status404, "{}") $
|
||||
forwardedResolveAlice >>= \r -> case r of
|
||||
Left (PCEProtocolError (SMP.NAME (SMP.RESOLVER _))) -> pure ()
|
||||
_ -> expectationFailure $ "expected Left (PCEProtocolError (NAME (RESOLVER _))), got: " <> show r
|
||||
Left (PCEProtocolError (SMP.NAME SMP.NOT_FOUND)) -> pure ()
|
||||
_ -> expectationFailure $ "expected Left (PCEProtocolError (NAME NOT_FOUND)), got: " <> show r
|
||||
|
||||
testRslvForwardedSuccess :: IO ()
|
||||
testRslvForwardedSuccess =
|
||||
withProxyAndResolver (status200, registeredBody testNameRecord) $
|
||||
withProxyAndResolver (status200, J.encode testNameRecord) $
|
||||
forwardedResolveAlice >>= \r -> case r of
|
||||
Right (Right NameResponse {registration = NRRegistered {nameRecord}}) -> nameRecord `shouldBe` testNameRecord
|
||||
_ -> expectationFailure $ "expected Right (Right NRRegistered), got: " <> show r
|
||||
Right (Right nr) -> nr `shouldBe` testNameRecord
|
||||
_ -> expectationFailure $ "expected Right (Right NameRecord), got: " <> show r
|
||||
|
||||
testRslvSuccess :: IO ()
|
||||
testRslvSuccess =
|
||||
withResolverServer (status200, registeredBody testNameRecord) $
|
||||
withResolverServer (status200, J.encode testNameRecord) $
|
||||
testSMPClient @TLS $ \h -> do
|
||||
(corrId, _entId, resp) <- sendRslv h "rs07" (domain "alice.simplex")
|
||||
corrId `shouldBe` CorrId "rs07"
|
||||
case resp of
|
||||
Right (RNAME NameResponse {registration = NRRegistered {nameRecord}}) -> nameRecord `shouldBe` testNameRecord
|
||||
_ -> expectationFailure $ "expected Right (RNAME NRRegistered), got: " <> show resp
|
||||
|
||||
testRslvAvailable :: IO ()
|
||||
testRslvAvailable =
|
||||
withResolverServer (status200, availableBody) $
|
||||
testSMPClient @TLS $ \h -> do
|
||||
(corrId, _entId, resp) <- sendRslv h "na01" (domain "ghost.simplex")
|
||||
corrId `shouldBe` CorrId "na01"
|
||||
resp `shouldBe` Right (RNAME (resolved (NRAvailable testPricing)))
|
||||
|
||||
testRslvReserved :: IO ()
|
||||
testRslvReserved =
|
||||
withResolverServer (status200, reservedBody) $
|
||||
testSMPClient @TLS $ \h -> do
|
||||
(_, _, resp) <- sendRslv h "na03" (domain "acme.simplex")
|
||||
resp `shouldBe` Right (RNAME (resolved (NRReserved NRRTrademark)))
|
||||
|
||||
-- | A client that predates v22 must see exactly what it saw before: the record
|
||||
-- for a name that resolves, and NOT_FOUND for one that does not.
|
||||
oldClient :: IO SMPClient
|
||||
oldClient = do
|
||||
g <- C.newRandom
|
||||
ts <- getCurrentTime
|
||||
let srv = SMPServer testHost testPort testKeyHash
|
||||
-- the version just below the gate: a lower ceiling would pass even if
|
||||
-- the gate were at 20 or 21
|
||||
oldCfg = defaultSMPClientConfig {serverVRange = mkVersionRange minServerSMPRelayVersion serverInfoSMPVersion}
|
||||
pcE <- getProtocolClient g NRMInteractive (1, srv, Nothing) oldCfg [] Nothing ts (\_ -> pure ())
|
||||
either (fail . show) pure pcE
|
||||
|
||||
testRslvOldClientRecord :: IO ()
|
||||
testRslvOldClientRecord =
|
||||
withResolverServer (status200, registeredBody testNameRecord) $ do
|
||||
pc <- oldClient
|
||||
r <- runExceptT' (directResolveName pc NRMInteractive (domain "alice.simplex"))
|
||||
r `shouldBe` NameResponse Nothing (NRRegistered Nothing Nothing Nothing testNameRecord)
|
||||
|
||||
testRslvOldClientNotFound :: IO ()
|
||||
testRslvOldClientNotFound =
|
||||
withResolverServer (status200, availableBody) $ do
|
||||
pc <- oldClient
|
||||
r <- runExceptT (directResolveName pc NRMInteractive (domain "alice.simplex"))
|
||||
case r of
|
||||
Left (PCEProtocolError (SMP.NAME SMP.NOT_FOUND)) -> pure ()
|
||||
_ -> expectationFailure $ "expected Left (PCEProtocolError (NAME NOT_FOUND)), got: " <> show r
|
||||
|
||||
testRslvForwardedAvailable :: IO ()
|
||||
testRslvForwardedAvailable =
|
||||
withProxyAndResolver (status200, availableBody) $
|
||||
forwardedResolveAlice >>= \r -> case r of
|
||||
Right (Right NameResponse {registration = NRAvailable {pricing}}) -> pricing `shouldBe` testPricing
|
||||
_ -> expectationFailure $ "expected Right (Right NRAvailable), got: " <> show r
|
||||
|
||||
-- keccak-256("alice"), the registry key
|
||||
aliceHash :: Text
|
||||
aliceHash = "[9c0257114eb9399a2985f8e75dad7600c5d89fe3824ffa99ec1c3eb8bf3b0501]"
|
||||
|
||||
-- | The paths the client asked the resolver for.
|
||||
resolvePaths :: IORef [[Text]] -> IO [[Text]]
|
||||
resolvePaths reqs = filter isResolve <$> readIORef reqs
|
||||
where
|
||||
isResolve = \case ("v2" : "resolve" : _) -> True; _ -> False
|
||||
|
||||
currentClient :: IO SMPClient
|
||||
currentClient = do
|
||||
g <- C.newRandom
|
||||
ts <- getCurrentTime
|
||||
let srv = SMPServer testHost testPort testKeyHash
|
||||
pcE <- getProtocolClient g NRMInteractive (1, srv, Nothing) defaultSMPClientConfig [] Nothing ts (\_ -> pure ())
|
||||
either (fail . show) pure pcE
|
||||
|
||||
testRslvSendsTheHash :: IO ()
|
||||
testRslvSendsTheHash =
|
||||
withResolverServerReqs (status200, registeredBody testNameRecord) $ \reqs -> do
|
||||
pc <- currentClient
|
||||
r <- runExceptT' (directResolveName pc NRMInteractive (domain "alice.simplex"))
|
||||
resolvePaths reqs `shouldReturn` [["v2", "resolve", aliceHash <> ".simplex"]]
|
||||
-- the client never sent the name, and the record still names it
|
||||
case r of
|
||||
NameResponse {registration = NRRegistered {nameRecord}} -> SMP.nrName nameRecord `shouldBe` "alice.simplex"
|
||||
_ -> expectationFailure $ "expected NRRegistered, got: " <> show r
|
||||
|
||||
testSubnameKeepsItsLabels :: IO ()
|
||||
testSubnameKeepsItsLabels =
|
||||
withResolverServerReqs (status200, availableBody) $ \reqs -> do
|
||||
pc <- currentClient
|
||||
_ <- runExceptT' (directResolveName pc NRMInteractive (domain "x.alice.simplex"))
|
||||
resolvePaths reqs `shouldReturn` [["v2", "resolve", "x.alice.simplex"]]
|
||||
|
||||
-- a hashed query does not tell the router the name, so the record's own name is
|
||||
-- checked against the one that was asked for
|
||||
testRslvWrongName :: IO ()
|
||||
testRslvWrongName =
|
||||
withResolverServer (status200, registeredBody testNameRecord {SMP.nrName = "mallory.simplex"}) $ do
|
||||
pc <- currentClient
|
||||
r <- runExceptT (directResolveName pc NRMInteractive (domain "alice.simplex"))
|
||||
case r of
|
||||
Left (PCEUnexpectedResponse _) -> pure ()
|
||||
_ -> expectationFailure $ "expected Left (PCEUnexpectedResponse ..), got: " <> show r
|
||||
Right (RNAME nr) -> nr `shouldBe` testNameRecord
|
||||
_ -> expectationFailure $ "expected Right (RNAME ..), got: " <> show resp
|
||||
|
||||
runExceptT' :: Show e => ExceptT e IO a -> IO a
|
||||
runExceptT' a = runExceptT a >>= either (fail . show) pure
|
||||
|
||||
+2
-27
@@ -9,15 +9,13 @@ module RemoteControl where
|
||||
|
||||
import AgentTests.FunctionalAPITests (runRight)
|
||||
import Control.Logger.Simple
|
||||
import Control.Monad (void)
|
||||
import Control.Monad.Trans.Except (runExceptT)
|
||||
import Crypto.Random (ChaChaDRG)
|
||||
import qualified Data.Aeson as J
|
||||
import qualified Data.ByteString.Char8 as B
|
||||
import qualified Data.ByteString.Lazy.Char8 as LB
|
||||
import Data.List (stripPrefix)
|
||||
import Data.List.NonEmpty (NonEmpty (..))
|
||||
import Data.Time.Clock.System (SystemTime (..), getSystemTime)
|
||||
import Data.Time.Clock.System (SystemTime (..))
|
||||
import qualified Simplex.Messaging.Crypto as C
|
||||
import Simplex.Messaging.Encoding.String (StrEncoding (..))
|
||||
import Simplex.Messaging.Transport (TSbChainKeys (..))
|
||||
@@ -26,10 +24,8 @@ import qualified Simplex.RemoteControl.Client as HC (RCHostClient (action))
|
||||
import qualified Simplex.RemoteControl.Client as RC
|
||||
import Simplex.RemoteControl.Discovery (mkLastLocalHost, preferAddress)
|
||||
import Simplex.RemoteControl.Invitation
|
||||
( RCEncInvitation (..),
|
||||
RCInvitation (..),
|
||||
( RCInvitation (..),
|
||||
RCSignedInvitation,
|
||||
signInvitation,
|
||||
verifySignedInvitation,
|
||||
)
|
||||
import Simplex.RemoteControl.Types
|
||||
@@ -49,7 +45,6 @@ remoteControlTests = do
|
||||
it "should connect to existing pairing" testExistingPairing
|
||||
describe "Multicast discovery" $ do
|
||||
it "should find paired host and connect" testMulticast
|
||||
it "should accept announcement only within timestamp window" testAnnouncementTimestamp
|
||||
|
||||
testPreferAddress :: Spec
|
||||
testPreferAddress = do
|
||||
@@ -249,26 +244,6 @@ testMulticast = do
|
||||
Nothing -> fail "timeout"
|
||||
Just _ -> pure ()
|
||||
|
||||
testAnnouncementTimestamp :: IO ()
|
||||
testAnnouncementTimestamp = do
|
||||
drg <- C.newRandom
|
||||
RCHostPairing {caKey, caCert, idPrivKey} <- RC.newRCHostPairing drg
|
||||
(hostDhPubKey, dhPrivKey) <- atomically $ C.generateKeyPair @'C.X25519 drg
|
||||
(skey, sessPrivKey) <- atomically $ C.generateKeyPair @'C.Ed25519 drg
|
||||
(dh, ctrlDhPrivKey) <- atomically $ C.generateKeyPair @'C.X25519 drg
|
||||
nonce <- atomically $ C.randomCbNonce drg
|
||||
now <- systemSeconds <$> getSystemTime
|
||||
let pairing = RCCtrlPairing {caKey, caCert, ctrlFingerprint = C.KeyHash "test-ca", idPubKey = C.publicKey idPrivKey, dhPrivKey, prevDhPrivKey = Nothing}
|
||||
announce offset = do
|
||||
let inv = RCInvitation {ca = C.KeyHash "test-ca", host = "127.0.0.1", port = 5223, v = supportedRCPVRange, app = J.String "app", ts = MkSystemTime (now + offset) 0, skey, idkey = C.publicKey idPrivKey, dh}
|
||||
encInvitation <- either (fail . show) pure $ C.cbEncrypt (C.dh' hostDhPubKey ctrlDhPrivKey) nonce (strEncode $ signInvitation sessPrivKey idPrivKey inv) 900
|
||||
runExceptT . void $ RC.findRCCtrlPairing (pairing :| []) RCEncInvitation {dhPubKey = dh, nonce, encInvitation}
|
||||
announce 0 `shouldReturn` Right ()
|
||||
announce (-3600) `shouldReturn` Right ()
|
||||
announce 3500 `shouldReturn` Right ()
|
||||
announce (-3700) `shouldReturn` Left RCEInvitation
|
||||
announce 3700 `shouldReturn` Left RCEInvitation
|
||||
|
||||
runCtrl :: TVar ChaChaDRG -> Bool -> RCHostPairing -> MVar RCSignedInvitation -> IO (Async RCHostPairing)
|
||||
runCtrl drg multicast hp invVar = async . runRight $ do
|
||||
(_found, inv, hc, r) <- RC.connectRCHost drg hp (J.String "app") multicast Nothing Nothing
|
||||
|
||||
+49
-54
@@ -33,6 +33,7 @@ import Simplex.Messaging.Protocol
|
||||
import Simplex.Messaging.Server (runSMPServerBlocking)
|
||||
import Simplex.Messaging.Server.Env.STM
|
||||
import Simplex.Messaging.Server.MsgStore.Types (MsgStoreClass (..), SMSType (..), SQSType (..))
|
||||
import Simplex.Messaging.Server.QueueStore.Postgres.Config (PostgresStoreCfg (..))
|
||||
import Simplex.Messaging.Transport
|
||||
import Simplex.Messaging.Transport.Client
|
||||
import Simplex.Messaging.Transport.Server
|
||||
@@ -50,7 +51,6 @@ import Util
|
||||
|
||||
#if defined(dbServerPostgres)
|
||||
import Database.PostgreSQL.Simple (defaultConnectInfo)
|
||||
import Simplex.Messaging.Server.QueueStore.Postgres.Config (PostgresStoreCfg (..))
|
||||
#endif
|
||||
|
||||
#if defined(dbPostgres) || defined(dbServerPostgres)
|
||||
@@ -123,6 +123,12 @@ testStoreMsgsFile = "tests/tmp/smp-server-messages.log"
|
||||
testStoreMsgsFile2 :: FilePath
|
||||
testStoreMsgsFile2 = "tests/tmp/smp-server-messages.log.2"
|
||||
|
||||
testStoreMsgsDir :: FilePath
|
||||
testStoreMsgsDir = "tests/tmp/messages"
|
||||
|
||||
testStoreMsgsDir2 :: FilePath
|
||||
testStoreMsgsDir2 = "tests/tmp/messages.2"
|
||||
|
||||
testStoreNtfsFile :: FilePath
|
||||
testStoreNtfsFile = "tests/tmp/smp-server-ntfs.log"
|
||||
|
||||
@@ -206,35 +212,24 @@ ntfTestServerCredentials =
|
||||
}
|
||||
|
||||
cfg :: AServerConfig
|
||||
cfg = cfgMS (ASType SQSMemory SMSMemory)
|
||||
cfg = cfgMS (ASType SQSMemory SMSJournal)
|
||||
|
||||
cfgS2 :: AServerConfig
|
||||
cfgS2 = memoryCfg cfg testStoreLogFile2 testStoreMsgsFile2
|
||||
cfgJ2 :: AServerConfig
|
||||
cfgJ2 = journalCfg cfg testStoreLogFile2 testStoreMsgsDir2
|
||||
|
||||
cfgS2QS :: SQSType s -> AServerConfig
|
||||
cfgS2QS = \case
|
||||
SQSMemory -> memoryCfg (cfgMS $ ASType SQSMemory SMSMemory) testStoreLogFile2 testStoreMsgsFile2
|
||||
SQSPostgres -> databaseCfg (cfgMS postgresStoreType) testStoreDBOpts2
|
||||
cfgJ2QS :: SQSType s -> AServerConfig
|
||||
cfgJ2QS = \case
|
||||
SQSMemory -> journalCfg (cfgMS $ ASType SQSMemory SMSJournal) testStoreLogFile2 testStoreMsgsDir2
|
||||
SQSPostgres -> journalCfgDB (cfgMS $ ASType SQSPostgres SMSJournal) testStoreDBOpts2 testStoreMsgsDir2
|
||||
|
||||
memoryCfg :: AServerConfig -> FilePath -> FilePath -> AServerConfig
|
||||
memoryCfg (ASrvCfg _ _ cfg') storeLogFile storeMsgsFile =
|
||||
ASrvCfg SQSMemory SMSMemory cfg' {serverStoreCfg = SSCMemory $ Just StorePaths {storeLogFile, storeMsgsFile = Just storeMsgsFile}}
|
||||
journalCfg :: AServerConfig -> FilePath -> FilePath -> AServerConfig
|
||||
journalCfg (ASrvCfg _ _ cfg') storeLogFile storeMsgsPath =
|
||||
ASrvCfg SQSMemory SMSJournal cfg' {serverStoreCfg = SSCMemoryJournal {storeLogFile, storeMsgsPath}}
|
||||
|
||||
databaseCfg :: AServerConfig -> DBOpts -> AServerConfig
|
||||
#if defined(dbServerPostgres)
|
||||
databaseCfg (ASrvCfg _ _ cfg') dbOpts =
|
||||
journalCfgDB :: AServerConfig -> DBOpts -> FilePath -> AServerConfig
|
||||
journalCfgDB (ASrvCfg _ _ cfg') dbOpts storeMsgsPath' =
|
||||
let storeCfg = PostgresStoreCfg {dbOpts, dbStoreLogPath = Nothing, confirmMigrations = MCYesUp, deletedTTL = 86400}
|
||||
in ASrvCfg SQSPostgres SMSPostgres cfg' {serverStoreCfg = SSCDatabase storeCfg}
|
||||
#else
|
||||
databaseCfg _ _ = error "no dbServerPostgres flag"
|
||||
#endif
|
||||
|
||||
postgresStoreType :: AStoreType
|
||||
#if defined(dbServerPostgres)
|
||||
postgresStoreType = ASType SQSPostgres SMSPostgres
|
||||
#else
|
||||
postgresStoreType = error "no dbServerPostgres flag"
|
||||
#endif
|
||||
in ASrvCfg SQSPostgres SMSJournal cfg' {serverStoreCfg = SSCDatabaseJournal {storeCfg, storeMsgsPath'}}
|
||||
|
||||
cfgMS :: AStoreType -> AServerConfig
|
||||
cfgMS msType = withStoreCfg (testServerStoreConfig msType) $ \serverStoreCfg ->
|
||||
@@ -243,6 +238,8 @@ cfgMS msType = withStoreCfg (testServerStoreConfig msType) $ \serverStoreCfg ->
|
||||
smpHandshakeTimeout = 60000000,
|
||||
tbqSize = 4,
|
||||
msgQueueQuota = 4,
|
||||
maxJournalMsgCount = 5,
|
||||
maxJournalStateLines = 2,
|
||||
queueIdBytes = 24,
|
||||
msgIdBytes = 24,
|
||||
serverStoreCfg,
|
||||
@@ -255,6 +252,7 @@ cfgMS msType = withStoreCfg (testServerStoreConfig msType) $ \serverStoreCfg ->
|
||||
messageExpiration = Just defaultMessageExpiration,
|
||||
expireMessagesOnStart = True,
|
||||
expireMessagesOnSend = False,
|
||||
idleQueueInterval = defaultIdleQueueInterval,
|
||||
notificationExpiration = defaultNtfExpiration,
|
||||
inactiveClientExpiration = Just defaultInactiveClientExpiration,
|
||||
logStatsInterval = Nothing,
|
||||
@@ -294,21 +292,20 @@ testServerStoreConfig :: AStoreType -> AServerStoreCfg
|
||||
testServerStoreConfig = serverStoreConfig_ False
|
||||
|
||||
serverStoreConfig_ :: Bool -> AStoreType -> AServerStoreCfg
|
||||
#if defined(dbServerPostgres)
|
||||
serverStoreConfig_ useDbStoreLog = \case
|
||||
#else
|
||||
serverStoreConfig_ _ = \case
|
||||
#endif
|
||||
ASType SQSMemory SMSMemory ->
|
||||
ASSCfg SQSMemory SMSMemory $ SSCMemory $ Just StorePaths {storeLogFile = testStoreLogFile, storeMsgsFile = Just testStoreMsgsFile}
|
||||
ASType SQSMemory SMSJournal ->
|
||||
ASSCfg SQSMemory SMSJournal $ SSCMemoryJournal {storeLogFile = testStoreLogFile, storeMsgsPath = testStoreMsgsDir}
|
||||
ASType SQSPostgres SMSJournal ->
|
||||
ASSCfg SQSPostgres SMSJournal SSCDatabaseJournal {storeCfg, storeMsgsPath' = testStoreMsgsDir}
|
||||
#if defined(dbServerPostgres)
|
||||
ASType SQSPostgres SMSPostgres ->
|
||||
let dbStoreLogPath = if useDbStoreLog then Just testStoreLogFile else Nothing
|
||||
storeCfg = PostgresStoreCfg {dbOpts = testStoreDBOpts, dbStoreLogPath, confirmMigrations = MCYesUp, deletedTTL = 86400}
|
||||
in ASSCfg SQSPostgres SMSPostgres $ SSCDatabase storeCfg
|
||||
#else
|
||||
ASType SQSPostgres _ -> error "no dbServerPostgres flag"
|
||||
ASSCfg SQSPostgres SMSPostgres $ SSCDatabase storeCfg
|
||||
#endif
|
||||
where
|
||||
dbStoreLogPath = if useDbStoreLog then Just testStoreLogFile else Nothing
|
||||
storeCfg = PostgresStoreCfg {dbOpts = testStoreDBOpts, dbStoreLogPath, confirmMigrations = MCYesUp, deletedTTL = 86400}
|
||||
|
||||
cfgVPrev :: AStoreType -> AServerConfig
|
||||
cfgVPrev msType = updateCfg (cfgMS msType) $ \cfg' -> cfg' {smpServerVRange = prevRange $ smpServerVRange cfg'}
|
||||
@@ -319,11 +316,8 @@ prevRange vr = vr {maxVersion = max (minVersion vr) (prevVersion $ maxVersion vr
|
||||
prevVersion :: Version v -> Version v
|
||||
prevVersion (Version v) = Version (v - 1)
|
||||
|
||||
nextVersion :: Version v -> Version v
|
||||
nextVersion (Version v) = Version (v + 1)
|
||||
|
||||
proxyCfg :: AServerConfig
|
||||
proxyCfg = proxyCfgMS (ASType SQSMemory SMSMemory)
|
||||
proxyCfg = proxyCfgMS (ASType SQSMemory SMSJournal)
|
||||
|
||||
proxyCfgMS :: AStoreType -> AServerConfig
|
||||
proxyCfgMS msType =
|
||||
@@ -334,13 +328,13 @@ proxyCfgMS msType =
|
||||
smpAgentCfg = smpAgentCfg' {smpCfg = (smpCfg smpAgentCfg') {agreeSecret = True, proxyServer = True, serverVRange = supportedProxyClientSMPRelayVRange}}
|
||||
}
|
||||
|
||||
proxyCfgS2 :: AServerConfig
|
||||
proxyCfgS2 = memoryCfg proxyCfg testStoreLogFile2 testStoreMsgsFile2
|
||||
proxyCfgJ2 :: AServerConfig
|
||||
proxyCfgJ2 = journalCfg proxyCfg testStoreLogFile2 testStoreMsgsDir2
|
||||
|
||||
proxyCfgS2QS :: SQSType qs -> AServerConfig
|
||||
proxyCfgS2QS = \case
|
||||
SQSMemory -> memoryCfg (proxyCfgMS $ ASType SQSMemory SMSMemory) testStoreLogFile2 testStoreMsgsFile2
|
||||
SQSPostgres -> databaseCfg (proxyCfgMS postgresStoreType) testStoreDBOpts2
|
||||
proxyCfgJ2QS :: SQSType qs -> AServerConfig
|
||||
proxyCfgJ2QS = \case
|
||||
SQSMemory -> journalCfg (proxyCfgMS $ ASType SQSMemory SMSJournal) testStoreLogFile2 testStoreMsgsDir2
|
||||
SQSPostgres -> journalCfgDB (proxyCfgMS $ ASType SQSPostgres SMSJournal) testStoreDBOpts2 testStoreMsgsDir2
|
||||
|
||||
-- Proxy config with a short relay-connection timeout, to bound how long a failing
|
||||
-- proxy->relay connection attempt blocks in the relay reconnection tests.
|
||||
@@ -352,15 +346,16 @@ proxyCfgShortTimeout =
|
||||
nt = NetworkTimeout {backgroundTimeout = 4_000000, interactiveTimeout = 4_000000}
|
||||
in cfg' {smpAgentCfg = aCfg {smpCfg = cCfg {networkConfig = (networkConfig cCfg) {tcpConnectTimeout = nt}}}}
|
||||
|
||||
proxyCfgVPrev :: AStoreType -> AServerConfig
|
||||
proxyCfgVPrev msType =
|
||||
updateCfg (proxyCfgMS msType) $ \cfg' ->
|
||||
-- Proxy whose forward (RFWD) response deadline is 1us: every forward to the relay times out
|
||||
-- while the initial relay connect (tcpConnectTimeout) is unaffected, so a session is established
|
||||
-- and then every forward on it fails with a response timeout.
|
||||
proxyCfgForwardTimeout :: AServerConfig
|
||||
proxyCfgForwardTimeout =
|
||||
updateCfg proxyCfg $ \cfg' ->
|
||||
let aCfg = smpAgentCfg cfg'
|
||||
cCfg = smpCfg aCfg
|
||||
in cfg'
|
||||
{ smpServerVRange = prevRange $ smpServerVRange cfg',
|
||||
smpAgentCfg = aCfg {smpCfg = cCfg {serverVRange = prevRange $ serverVRange cCfg}}
|
||||
}
|
||||
nt = NetworkTimeout {backgroundTimeout = 1, interactiveTimeout = 1}
|
||||
in cfg' {smpAgentCfg = aCfg {smpCfg = cCfg {networkConfig = (networkConfig cCfg) {tcpTimeout = nt}}}}
|
||||
|
||||
withSmpServerStoreMsgLogOn :: HasCallStack => (ASrvTransport, AStoreType) -> ServiceName -> (HasCallStack => ThreadId -> IO a) -> IO a
|
||||
withSmpServerStoreMsgLogOn (t, msType) =
|
||||
@@ -422,10 +417,10 @@ withSmpServerProxy :: HasCallStack => (ASrvTransport, AStoreType) -> IO a -> IO
|
||||
withSmpServerProxy (t, msType) = withSmpServerConfigOn t (proxyCfgMS msType) testPort . const
|
||||
|
||||
withSmpServers2 :: HasCallStack => (ASrvTransport, AStoreType) -> IO a -> IO a
|
||||
withSmpServers2 ps@(t, ASType qs _ms) = withSmpServer ps . withSmpServerConfigOn t (cfgS2QS qs) testPort2 . const
|
||||
withSmpServers2 ps@(t, ASType qs _ms) = withSmpServer ps . withSmpServerConfigOn t (cfgJ2QS qs) testPort2 . const
|
||||
|
||||
withSmpServersProxy2 :: HasCallStack => (ASrvTransport, AStoreType) -> IO a -> IO a
|
||||
withSmpServersProxy2 ps@(t, ASType qs _ms) = withSmpServerProxy ps . withSmpServerConfigOn t (proxyCfgS2QS qs) testPort2 . const
|
||||
withSmpServersProxy2 ps@(t, ASType qs _ms) = withSmpServerProxy ps . withSmpServerConfigOn t (proxyCfgJ2QS qs) testPort2 . const
|
||||
|
||||
runSmpTest :: forall c a. (HasCallStack, Transport c) => AStoreType -> (HasCallStack => THandleSMP c 'TClient -> IO a) -> IO a
|
||||
runSmpTest msType test = withSmpServerConfigOn (transport @c) (cfgMS msType) testPort $ \_ -> testSMPClient test
|
||||
@@ -446,7 +441,7 @@ smpServerTest ::
|
||||
TProxy c 'TServer ->
|
||||
(Maybe TAuthorizations, ByteString, ByteString, smp) ->
|
||||
IO (Maybe TAuthorizations, ByteString, ByteString, BrokerMsg)
|
||||
smpServerTest _ t = runSmpTest (ASType SQSMemory SMSMemory) $ \h -> tPut' h t >> tGet' h
|
||||
smpServerTest _ t = runSmpTest (ASType SQSMemory SMSJournal) $ \h -> tPut' h t >> tGet' h
|
||||
where
|
||||
tPut' :: THandleSMP c 'TClient -> (Maybe TAuthorizations, ByteString, ByteString, smp) -> IO ()
|
||||
tPut' h@THandle {params = THandleParams {sessionId, implySessId}} (sig, corrId, queueId, smp) = do
|
||||
|
||||
+16
-130
@@ -3,7 +3,7 @@
|
||||
{-# LANGUAGE OverloadedStrings #-}
|
||||
{-# LANGUAGE ScopedTypeVariables #-}
|
||||
|
||||
module SMPNamesTests (smpNamesTests, testNameRecord, testPricing, registeredBody, availableBody, reservedBody, responseBody, resolved) where
|
||||
module SMPNamesTests (smpNamesTests, testNameRecord) where
|
||||
|
||||
import qualified Data.Aeson as J
|
||||
import qualified Data.ByteString.Char8 as B
|
||||
@@ -11,14 +11,13 @@ import qualified Data.ByteString.Lazy as LB
|
||||
import Data.Either (isLeft, isRight)
|
||||
import Data.IORef (readIORef)
|
||||
import Data.List (sort)
|
||||
import qualified Data.Map.Strict as M
|
||||
import qualified Data.Text as T
|
||||
import Data.Text.Encoding (encodeUtf8)
|
||||
import Network.HTTP.Types (status200, status400, status404, status500, status502)
|
||||
import NamesResolverServer (resolveResp, testNamesConfig, withResolverServer, withResolverServerDelayed)
|
||||
import Simplex.Messaging.Encoding (smpDecode, smpEncode)
|
||||
import Simplex.Messaging.Encoding.String (strDecode)
|
||||
import Simplex.Messaging.Protocol (Command (..), ErrorType (..), NameErrorType (..), NamePricing (..), NameQuery (..), NameRecord (..), NameRegistration (..), NameResponse (..), NameReservedReason (..), ProtocolEncoding (..), USDCents (..))
|
||||
import Simplex.Messaging.Protocol (ErrorType (..), NameErrorType (..), NameRecord (..))
|
||||
import Simplex.Messaging.Server.Main (validateUrl)
|
||||
import Simplex.Messaging.Server.Names
|
||||
( NamesConfig (..),
|
||||
@@ -28,9 +27,7 @@ import Simplex.Messaging.Server.Names
|
||||
resolveName,
|
||||
)
|
||||
import Simplex.Messaging.Server.Names.HttpResolver (ResolverError (..))
|
||||
import Simplex.Messaging.SimplexName (SimplexDomain (..), SimplexTLD (..), labelHash)
|
||||
import Simplex.Messaging.SystemTime (RoundedSystemTime (..))
|
||||
import Simplex.Messaging.Transport (nameAvailSMPVersion, serverInfoSMPVersion)
|
||||
import Simplex.Messaging.SimplexName (SimplexDomain (..), SimplexTLD (..))
|
||||
import Test.Hspec
|
||||
|
||||
testNameRecord :: NameRecord
|
||||
@@ -50,41 +47,12 @@ testNameRecord =
|
||||
nrResolver = "0x0202020202020202020202020202020202020202"
|
||||
}
|
||||
|
||||
-- | What the resolver serves on /v2/resolve. Spelled out rather than encoded
|
||||
-- from the Haskell value: the literal JSON is the contract with the resolver.
|
||||
registeredBody :: NameRecord -> LB.ByteString
|
||||
registeredBody nameRec =
|
||||
responseBody $ "{\"type\":\"registered\",\"expires\":1813853483,\"graceUntil\":1821629483,\"reservedReason_\":null,\"nameRecord\":" <> J.encode nameRec <> "}"
|
||||
|
||||
availableBody :: LB.ByteString
|
||||
availableBody = responseBody "{\"type\":\"available\",\"pricing\":{\"registrationPrices\":{\"3\":12793,\"4\":3198},\"basePrice\":100,\"minLabelLength\":3}}"
|
||||
|
||||
reservedBody :: LB.ByteString
|
||||
reservedBody = responseBody "{\"type\":\"reserved\",\"reservedReason\":\"trademark\"}"
|
||||
|
||||
-- | The registration, and the block the resolver read it at.
|
||||
responseBody :: LB.ByteString -> LB.ByteString
|
||||
responseBody reg = "{\"lastBlockTs\":" <> testReadAt <> ",\"registration\":" <> reg <> "}"
|
||||
|
||||
testReadAt :: LB.ByteString
|
||||
testReadAt = "1813000000"
|
||||
|
||||
resolved :: NameRegistration -> NameResponse
|
||||
resolved registration = NameResponse {lastBlockTs = Just (RoundedSystemTime 1813000000), registration}
|
||||
|
||||
-- | What `registeredBody testNameRecord` resolves to.
|
||||
registeredAlice :: NameRegistration
|
||||
registeredAlice =
|
||||
NRRegistered {expires = Just (RoundedSystemTime 1813853483), graceUntil = Just (RoundedSystemTime 1821629483), reservedReason_ = Nothing, nameRecord = testNameRecord}
|
||||
|
||||
smpNamesTests :: Spec
|
||||
smpNamesTests = do
|
||||
describe "NameRecord JSON (Protocol)" nameRecordEncodingSpec
|
||||
describe "ErrorType NAME wire encoding" errorWireSpec
|
||||
describe "RSLV wire encoding" rslvWireSpec
|
||||
describe "Name parsing (SimplexDomain)" parseNameSpec
|
||||
describe "HTTP resolver" resolverSpec
|
||||
describe "name availability" availabilitySpec
|
||||
describe "Resolver health probe" healthSpec
|
||||
describe "resolver_endpoint validation" validateUrlSpec
|
||||
|
||||
@@ -133,87 +101,8 @@ errorWireSpec =
|
||||
-- RESOLVER detail may contain spaces - must survive the round-trip
|
||||
smpDecode (smpEncode (NAME (RESOLVER "HTTP 502"))) `shouldBe` Right (NAME (RESOLVER "HTTP 502"))
|
||||
|
||||
-- the query format changed at v22, so an older session must still get the name
|
||||
rslvWireSpec :: Spec
|
||||
rslvWireSpec = do
|
||||
it "below v22 carries the name, as it did before" $
|
||||
encodeProtocol v20 (RSLV (NQDomain aliceDomain')) `shouldBe` "RSLV " <> smpEncode aliceDomain'
|
||||
-- keccak-256("alice"), the same constant the resolver's own tests use
|
||||
it "from v22 carries the 2LD as its hash" $
|
||||
encodeProtocol v22 (RSLV (NQDomain aliceDomain'))
|
||||
`shouldBe` "RSLV [9c0257114eb9399a2985f8e75dad7600c5d89fe3824ffa99ec1c3eb8bf3b0501].simplex"
|
||||
-- the hashed query has no room for subname labels, so such a name goes as text
|
||||
it "a name with subnames is not hashed" $
|
||||
encodeProtocol v22 (RSLV (NQDomain aliceDomain' {subDomain = ["x"]})) `shouldBe` "RSLV x.alice.simplex"
|
||||
it "leaves a web name alone: no registry, nothing to key on" $
|
||||
encodeProtocol v22 (RSLV (NQDomain webDomain')) `shouldBe` "RSLV example.com"
|
||||
where
|
||||
v20 = serverInfoSMPVersion
|
||||
v22 = nameAvailSMPVersion
|
||||
aliceDomain' = SimplexDomain {nameTLD = TLDSimplex, domain = "alice", subDomain = []}
|
||||
webDomain' = SimplexDomain {nameTLD = TLDWeb, domain = "example.com", subDomain = []}
|
||||
|
||||
availabilitySpec :: Spec
|
||||
availabilitySpec = do
|
||||
-- one lookup answers what the name points to, whether it can be taken, and
|
||||
-- whether it is held back
|
||||
it "a registered name answers with its record and dates" $
|
||||
answers (registeredBody testNameRecord) (resolved registeredAlice)
|
||||
it "a registered name can be held back too" $
|
||||
answers heldBackBody . resolved $
|
||||
NRRegistered {expires = Just (RoundedSystemTime 1813853483), graceUntil = Just (RoundedSystemTime 1821629483), reservedReason_ = Just NRRInternal, nameRecord = testNameRecord}
|
||||
it "an unregistered name answers with the price" $
|
||||
answers availableBody (resolved NRAvailable {pricing = testPricing})
|
||||
it "reserved carries the reason and no price" $
|
||||
answers reservedBody (resolved $ NRReserved NRRTrademark)
|
||||
-- losing the reservation would offer a name that cannot be registered
|
||||
it "a reason from a later version still reserves the name" $
|
||||
answers (responseBody "{\"type\":\"reserved\",\"reservedReason\":\"seasonal\"}") (resolved $ NRReserved (NRRUnknown "seasonal"))
|
||||
-- RNAME carries the registration as JSON, so that is the encoding to hold
|
||||
it "every registration survives the wire" $
|
||||
mapM_
|
||||
(\a -> J.eitherDecodeStrict (LB.toStrict (J.encode a)) `shouldBe` Right a)
|
||||
[ registeredAlice,
|
||||
NRRegistered {expires = Nothing, graceUntil = Nothing, reservedReason_ = Just NRRInternal, nameRecord = testNameRecord},
|
||||
NRAvailable {pricing = testPricing},
|
||||
NRReserved NRRInternal,
|
||||
NRReserved NRRTrademark,
|
||||
NRReserved NRRCommunity,
|
||||
NRReserved (NRRUnknown "seasonal")
|
||||
]
|
||||
-- one vocabulary: the same word from the resolver and in JSON
|
||||
it "a reason reads the same in JSON as from the resolver" $ do
|
||||
J.encode (NRRUnknown "seasonal") `shouldBe` "\"seasonal\""
|
||||
J.encode NRRTrademark `shouldBe` "\"trademark\""
|
||||
where
|
||||
heldBackBody =
|
||||
responseBody $ "{\"type\":\"registered\",\"expires\":1813853483,\"graceUntil\":1821629483,\"reservedReason_\":\"internal\",\"nameRecord\":" <> J.encode testNameRecord <> "}"
|
||||
answers body a =
|
||||
withResolverServer (resolveResp status200 body) $ \port _ -> do
|
||||
env <- newNamesEnv (testNamesConfig port)
|
||||
resolveName env aliceQuery `shouldReturn` Right a
|
||||
aliceQuery = NQDomain SimplexDomain {nameTLD = TLDSimplex, domain = "alice", subDomain = []}
|
||||
|
||||
-- | The .testing oracle: US cents per year by label length.
|
||||
testPricing :: NamePricing
|
||||
testPricing =
|
||||
NamePricing
|
||||
{ registrationPrices = M.fromList [(3, USDCents 12793), (4, USDCents 3198)],
|
||||
basePrice = USDCents 100,
|
||||
minLabelLength = 3
|
||||
}
|
||||
|
||||
parseNameSpec :: Spec
|
||||
parseNameSpec = do
|
||||
-- the hashed form is a query, not a name: it has its own type
|
||||
it "a name is never a hash" $
|
||||
parseN ("[" <> T.replicate 64 "b" <> "].simplex") `shouldSatisfy` isLeft
|
||||
it "a query survives the wire" $
|
||||
mapM_
|
||||
(\q -> smpDecode (smpEncode q) `shouldBe` Right q)
|
||||
[ NQDomain d,
|
||||
NQHash (labelHash "alice") TLDSimplex
|
||||
]
|
||||
it "accepts a valid simplex-TLD name" $
|
||||
case parseN "privacy.simplex" of
|
||||
Right d -> do
|
||||
@@ -249,26 +138,23 @@ parseNameSpec = do
|
||||
where
|
||||
parseN :: T.Text -> Either String SimplexDomain
|
||||
parseN = strDecode . encodeUtf8
|
||||
d = SimplexDomain {nameTLD = TLDSimplex, domain = "alice", subDomain = ["x"]}
|
||||
|
||||
resolverSpec :: Spec
|
||||
resolverSpec = do
|
||||
it "returns the registration on 200 OK" $
|
||||
withResolverServer (resolveResp status200 (registeredBody testNameRecord)) $ \port _ -> do
|
||||
it "returns NameRecord on 200 OK" $
|
||||
withResolverServer (resolveResp status200 (J.encode testNameRecord)) $ \port _ -> do
|
||||
env <- newNamesEnv (testNamesConfig port)
|
||||
resolveName env aliceDomain `shouldReturn` Right (resolved registeredAlice)
|
||||
resolveName env aliceDomain `shouldReturn` Right testNameRecord
|
||||
|
||||
-- /v2/resolve answers 200, 400 or 502 and never says "no such name": an
|
||||
-- unregistered name is NRAvailable. So no status maps to NOT_FOUND.
|
||||
it "returns RESOLVER on 404 (a resolver without the v2 route)" $
|
||||
it "returns NOT_FOUND on 404" $
|
||||
withResolverServer (resolveResp status404 "{}") $ \port _ -> do
|
||||
env <- newNamesEnv (testNamesConfig port)
|
||||
resolveName env aliceDomain `shouldReturn` Left (RESOLVER "HTTP 404")
|
||||
resolveName env aliceDomain `shouldReturn` Left NOT_FOUND
|
||||
|
||||
it "returns RESOLVER on 400 (TLD not configured)" $
|
||||
it "returns NOT_FOUND on 400 (unknown TLD)" $
|
||||
withResolverServer (resolveResp status400 "{}") $ \port _ -> do
|
||||
env <- newNamesEnv (testNamesConfig port)
|
||||
resolveName env aliceDomain `shouldReturn` Left (RESOLVER "HTTP 400")
|
||||
resolveName env aliceDomain `shouldReturn` Left NOT_FOUND
|
||||
|
||||
it "returns RESOLVER on 502 (upstream failure)" $
|
||||
withResolverServer (resolveResp status502 "{}") $ \port _ -> do
|
||||
@@ -285,31 +171,31 @@ resolverSpec = do
|
||||
env <- newNamesEnv (testNamesConfig port)
|
||||
resolveName env aliceDomain `shouldReturn` Left (RESOLVER "invalid response")
|
||||
|
||||
it "returns RESOLVER when JSON parses but isn't a NameRegistration shape" $
|
||||
it "returns RESOLVER when JSON parses but isn't a NameRecord shape" $
|
||||
withResolverServer (resolveResp status200 "{}") $ \port _ -> do
|
||||
env <- newNamesEnv (testNamesConfig port)
|
||||
resolveName env aliceDomain `shouldReturn` Left (RESOLVER "invalid response")
|
||||
|
||||
it "returns RESOLVER (timeout) when the resolver is slower than resolverTimeoutMs" $
|
||||
withResolverServerDelayed 1500 (resolveResp status200 (registeredBody testNameRecord)) $ \port _ -> do
|
||||
withResolverServerDelayed 1500 (resolveResp status200 (J.encode testNameRecord)) $ \port _ -> do
|
||||
env <- newNamesEnv (testNamesConfig port) {resolverTimeoutMs = 300}
|
||||
resolveName env aliceDomain `shouldReturn` Left (RESOLVER "timeout")
|
||||
|
||||
it "sends one HTTP request per lookup (no cache)" $
|
||||
withResolverServer (resolveResp status200 (registeredBody testNameRecord)) $ \port reqs -> do
|
||||
withResolverServer (resolveResp status200 (J.encode testNameRecord)) $ \port reqs -> do
|
||||
env <- newNamesEnv (testNamesConfig port)
|
||||
_ <- resolveName env aliceDomain
|
||||
_ <- resolveName env aliceDomain
|
||||
readIORef reqs >>= \rs -> length rs `shouldBe` 2
|
||||
|
||||
it "addresses the resolver with the full canonical domain name" $
|
||||
withResolverServer (resolveResp status200 (registeredBody testNameRecord)) $ \port reqs -> do
|
||||
withResolverServer (resolveResp status200 (J.encode testNameRecord)) $ \port reqs -> do
|
||||
env <- newNamesEnv (testNamesConfig port)
|
||||
_ <- resolveName env aliceDomain
|
||||
readIORef reqs `shouldReturn` [["v2", "resolve", "alice.simplex"]]
|
||||
readIORef reqs `shouldReturn` [["resolve", "alice.simplex"]]
|
||||
|
||||
where
|
||||
aliceDomain = NQDomain SimplexDomain {nameTLD = TLDSimplex, domain = "alice", subDomain = []}
|
||||
aliceDomain = SimplexDomain {nameTLD = TLDSimplex, domain = "alice", subDomain = []}
|
||||
|
||||
healthSpec :: Spec
|
||||
healthSpec = do
|
||||
|
||||
+34
-49
@@ -63,6 +63,8 @@ smpProxyTests = do
|
||||
testProxyRecoversWithoutDisconnect
|
||||
it "reconnects to relay after sender disconnects mid-connection" $ \_ ->
|
||||
testProxyReconnectAfterRelayRestart
|
||||
xit "must drop a stuck relay session after forward timeouts" $ \_ ->
|
||||
testProxyForwardTimeoutStuckSession
|
||||
describe "agent client reconnection" $ do
|
||||
it "reconnects after a connect is cancelled mid-flight" $ \_ ->
|
||||
testAgentClientReconnectAfterCancel
|
||||
@@ -73,8 +75,6 @@ smpProxyTests = do
|
||||
xit "no SMP service at host/port" todo
|
||||
xit "bad SMP fingerprint" todo
|
||||
xit "batching proxy requests" todo
|
||||
it "relay rejects forwarded command with changed version" $ \_ ->
|
||||
testChangedFwdVersion
|
||||
describe "deliver message via SMP proxy" $ do
|
||||
let srv1 = SMPServer testHost testPort testKeyHash
|
||||
srv2 = SMPServer testHost2 testPort2 testKeyHash
|
||||
@@ -97,11 +97,6 @@ smpProxyTests = do
|
||||
deliverMessageViaProxy proxyServ relayServ C.SEd25519 msg1 msg2
|
||||
it "max message size, X25519 keys" . twoServersFirstProxy $
|
||||
deliverMessageViaProxy proxyServ relayServ C.SX25519 msg1 msg2
|
||||
describe "version compatibility" $ do
|
||||
let deliver clientVR = deliverMessagesViaProxyVR clientVR srv1 srv2 C.SEd448 ["hello 1"] ["hello 2"]
|
||||
it "prev client" . twoServersFirstProxy $ deliver (prevRange supportedClientSMPRelayVRange)
|
||||
it "prev proxy" . twoServersPrevProxy $ deliver supportedClientSMPRelayVRange
|
||||
it "prev relay" . twoServersPrevRelay $ deliver supportedClientSMPRelayVRange
|
||||
describe "stress test 1k" $ do
|
||||
let deliver n = deliverMessagesViaProxy srv1 srv2 C.SEd448 [] (map bshow [1 :: Int .. n])
|
||||
it "1x1000" . twoServersFirstProxy $ deliver 1000
|
||||
@@ -157,34 +152,28 @@ smpProxyTests = do
|
||||
let deliver nAgents nMsgs = agentDeliverMessagesViaProxyConc (replicate nAgents [srv1]) (map bshow [1 :: Int .. nMsgs])
|
||||
it "25 agents, 300 pairs, 17 messages" . oneServer . withNumCapabilities 4 $ deliver 25 17
|
||||
where
|
||||
oneServer test msType = withSmpServerConfigOn (transport @TLS) (updateCfg (proxyCfgMS msType) $ \cfg_ -> cfg_ {msgQueueQuota = 128}) testPort $ const test
|
||||
oneServer test msType = withSmpServerConfigOn (transport @TLS) (updateCfg (proxyCfgMS msType) $ \cfg_ -> cfg_ {msgQueueQuota = 128, maxJournalMsgCount = 256}) testPort $ const test
|
||||
twoServers test msType = twoServers_ (proxyCfgMS msType) (proxyCfgMS msType) test msType
|
||||
twoServersFirstProxy test msType = twoServers_ (proxyCfgMS msType) (updateCfg (cfgMS msType) $ \cfg_ -> cfg_ {msgQueueQuota = 128}) test msType
|
||||
twoServersMoreConc test msType = twoServers_ (updateCfg (proxyCfgMS msType) $ \cfg_ -> cfg_ {serverClientConcurrency = 128}) (updateCfg (cfgMS msType) $ \cfg_ -> cfg_ {msgQueueQuota = 128}) test msType
|
||||
twoServersNoConc test msType = twoServers_ (updateCfg (proxyCfgMS msType) $ \cfg_ -> cfg_ {serverClientConcurrency = 1}) (updateCfg (cfgMS msType) $ \cfg_ -> cfg_ {msgQueueQuota = 128}) test msType
|
||||
twoServersPrevProxy test msType = twoServers_ (proxyCfgVPrev msType) (cfgMS msType) test msType
|
||||
twoServersPrevRelay test msType = twoServers_ (proxyCfgMS msType) (prevServerVRange $ cfgMS msType) test msType
|
||||
prevServerVRange cfg' = updateCfg cfg' $ \cfg_ -> cfg_ {smpServerVRange = prevRange $ smpServerVRange cfg_}
|
||||
twoServersFirstProxy test msType = twoServers_ (proxyCfgMS msType) (updateCfg (cfgMS msType) $ \cfg_ -> cfg_ {msgQueueQuota = 128, maxJournalMsgCount = 256}) test msType
|
||||
twoServersMoreConc test msType = twoServers_ (updateCfg (proxyCfgMS msType) $ \cfg_ -> cfg_ {serverClientConcurrency = 128}) (updateCfg (cfgMS msType) $ \cfg_ -> cfg_ {msgQueueQuota = 128, maxJournalMsgCount = 256}) test msType
|
||||
twoServersNoConc test msType = twoServers_ (updateCfg (proxyCfgMS msType) $ \cfg_ -> cfg_ {serverClientConcurrency = 1}) (updateCfg (cfgMS msType) $ \cfg_ -> cfg_ {msgQueueQuota = 128, maxJournalMsgCount = 256}) test msType
|
||||
twoServers_ :: AServerConfig -> AServerConfig -> IO () -> AStoreType -> IO ()
|
||||
twoServers_ cfg1 cfg2 runTest (ASType qsType _) =
|
||||
withSmpServerConfigOn (transport @TLS) cfg1 testPort $ \_ ->
|
||||
let cfg2' = case qsType of
|
||||
SQSMemory -> memoryCfg cfg2 testStoreLogFile2 testStoreMsgsFile2
|
||||
SQSPostgres -> databaseCfg cfg2 testStoreDBOpts2
|
||||
SQSMemory -> journalCfg cfg2 testStoreLogFile2 testStoreMsgsDir2
|
||||
SQSPostgres -> journalCfgDB cfg2 testStoreDBOpts2 testStoreMsgsDir2
|
||||
in withSmpServerConfigOn (transport @TLS) cfg2' testPort2 $ const runTest
|
||||
|
||||
deliverMessageViaProxy :: (C.AlgorithmI a, C.AuthAlgorithm a) => SMPServer -> SMPServer -> C.SAlgorithm a -> ByteString -> ByteString -> IO ()
|
||||
deliverMessageViaProxy proxyServ relayServ alg msg msg' = deliverMessagesViaProxy proxyServ relayServ alg [msg] [msg']
|
||||
|
||||
deliverMessagesViaProxy :: (C.AlgorithmI a, C.AuthAlgorithm a) => SMPServer -> SMPServer -> C.SAlgorithm a -> [ByteString] -> [ByteString] -> IO ()
|
||||
deliverMessagesViaProxy = deliverMessagesViaProxyVR $ mkVersionRange minServerSMPRelayVersion currentClientSMPRelayVersion
|
||||
|
||||
deliverMessagesViaProxyVR :: (C.AlgorithmI a, C.AuthAlgorithm a) => VersionRangeSMP -> SMPServer -> SMPServer -> C.SAlgorithm a -> [ByteString] -> [ByteString] -> IO ()
|
||||
deliverMessagesViaProxyVR clientVR proxyServ relayServ alg unsecuredMsgs securedMsgs = do
|
||||
deliverMessagesViaProxy proxyServ relayServ alg unsecuredMsgs securedMsgs = do
|
||||
g <- C.newRandom
|
||||
-- set up proxy
|
||||
ts <- getCurrentTime
|
||||
pc' <- getProtocolClient g NRMInteractive (1, proxyServ, Nothing) defaultSMPClientConfig {serverVRange = clientVR, proxiedRelayVRange = clientVR} [] Nothing ts (\_ -> pure ())
|
||||
pc' <- getProtocolClient g NRMInteractive (1, proxyServ, Nothing) defaultSMPClientConfig {serverVRange = mkVersionRange minServerSMPRelayVersion currentClientSMPRelayVersion} [] Nothing ts (\_ -> pure ())
|
||||
pc <- either (fail . show) pure pc'
|
||||
THAuthClient {} <- maybe (fail "getProtocolClient returned no thAuth") pure $ thAuth $ thParams pc
|
||||
-- set up relay
|
||||
@@ -204,8 +193,6 @@ deliverMessagesViaProxyVR clientVR proxyServ relayServ alg unsecuredMsgs secured
|
||||
forM_ unsecuredMsgs $ \msg -> do
|
||||
runExceptT' (proxySMPMessage pc NRMInteractive sess Nothing sndId noMsgFlags msg) `shouldReturn` Right ()
|
||||
runExceptT' (proxySMPMessage pc NRMInteractive sess {prSessionId = "bad session"} Nothing sndId noMsgFlags msg) `shouldReturn` Left (ProxyProtocolError $ SMP.PROXY SMP.NO_SESSION)
|
||||
forM_ ([prevVersion minServerSMPRelayVersion, nextVersion currentServerSMPRelayVersion] :: [VersionSMP]) $ \v ->
|
||||
runExceptT' (proxySMPMessage pc NRMInteractive sess {prVersion = v} Nothing sndId noMsgFlags msg) `shouldReturn` Left (ProxyProtocolError $ SMP.PROXY $ SMP.PROTOCOL $ SMP.PROXY $ SMP.BROKER $ SMP.TRANSPORT TEVersion)
|
||||
-- receive 1
|
||||
(_tSess, _, [(_entId, STEvent (Right (SMP.MSG RcvMessage {msgId, msgBody = EncRcvMsgBody encBody})))]) <- atomically $ readTBQueue msgQ
|
||||
dec msgId encBody `shouldBe` Right msg
|
||||
@@ -244,7 +231,7 @@ agentDeliverMessageViaProxy aTestCfg@(aSrvs, _, aViaProxy) bTestCfg@(bSrvs, _, b
|
||||
withAgent 1 aCfg (servers aTestCfg) testDB $ \alice ->
|
||||
withAgent 2 aCfg (servers bTestCfg) testDB2 $ \bob -> runRight_ $ do
|
||||
(bobId, CCLink qInfo Nothing) <- A.createConnection alice NRMInteractive 1 True True SCMInvitation Nothing Nothing CR.IKPQOn False SMSubscribe
|
||||
(aliceId, _) <- A.prepareConnectionToJoin bob 1 True qInfo PQSupportOn
|
||||
aliceId <- A.prepareConnectionToJoin bob 1 True qInfo PQSupportOn
|
||||
sqSecured <- A.joinConnection bob NRMInteractive 1 aliceId True qInfo "bob's connInfo" PQSupportOn SMSubscribe
|
||||
liftIO $ sqSecured `shouldBe` True
|
||||
("", _, A.CONF confId pqSup' _ "bob's connInfo") <- get alice
|
||||
@@ -300,7 +287,7 @@ agentDeliverMessagesViaProxyConc agentServers msgs =
|
||||
-- otherwise the CONF messages would get mixed with MSG
|
||||
prePair alice bob = do
|
||||
(bobId, CCLink qInfo Nothing) <- runExceptT' $ A.createConnection alice NRMInteractive 1 True True SCMInvitation Nothing Nothing CR.IKPQOn False SMSubscribe
|
||||
(aliceId, _) <- runExceptT' $ A.prepareConnectionToJoin bob 1 True qInfo PQSupportOn
|
||||
aliceId <- runExceptT' $ A.prepareConnectionToJoin bob 1 True qInfo PQSupportOn
|
||||
sqSecured <- runExceptT' $ A.joinConnection bob NRMInteractive 1 aliceId True qInfo "bob's connInfo" PQSupportOn SMSubscribe
|
||||
liftIO $ sqSecured `shouldBe` True
|
||||
confId <-
|
||||
@@ -359,7 +346,7 @@ agentViaProxyRetryOffline = do
|
||||
withServer $ \_ -> do
|
||||
(aliceId, bobId) <- withServer2 $ \_ -> runRight $ do
|
||||
(bobId, CCLink qInfo Nothing) <- A.createConnection alice NRMInteractive 1 True True SCMInvitation Nothing Nothing CR.IKPQOn False SMSubscribe
|
||||
(aliceId, _) <- A.prepareConnectionToJoin bob 1 True qInfo PQSupportOn
|
||||
aliceId <- A.prepareConnectionToJoin bob 1 True qInfo PQSupportOn
|
||||
sqSecured <- A.joinConnection bob NRMInteractive 1 aliceId True qInfo "bob's connInfo" PQSupportOn SMSubscribe
|
||||
liftIO $ sqSecured `shouldBe` True
|
||||
("", _, A.CONF confId pqSup' _ "bob's connInfo") <- get alice
|
||||
@@ -401,15 +388,15 @@ agentViaProxyRetryOffline = do
|
||||
ackMessage alice bobId (baseId + 4) Nothing
|
||||
where
|
||||
withServer :: (ThreadId -> IO a) -> IO a
|
||||
withServer = withServer_ testStoreLogFile testStoreMsgsFile testStoreNtfsFile testPort
|
||||
withServer = withServer_ testStoreLogFile testStoreMsgsDir testStoreNtfsFile testPort
|
||||
-- TODO [postgres]
|
||||
-- withServer = withServer_ testStoreDBOpts testStoreNtfsFile testPort
|
||||
-- withServer = withServer_ testStoreDBOpts testStoreMsgsDir testStoreNtfsFile testPort
|
||||
withServer2 :: (ThreadId -> IO a) -> IO a
|
||||
withServer2 = withServer_ testStoreLogFile2 testStoreMsgsFile2 testStoreNtfsFile2 testPort2
|
||||
withServer2 = withServer_ testStoreLogFile2 testStoreMsgsDir2 testStoreNtfsFile2 testPort2
|
||||
-- TODO [postgres]
|
||||
-- withServer2 = withServer_ testStoreDBOpts2 testStoreNtfsFile2 testPort2
|
||||
-- withServer2 = withServer_ testStoreDBOpts2 testStoreMsgsDir2 testStoreNtfsFile2 testPort2
|
||||
withServer_ storeLog storeMsgs storeNtfs =
|
||||
let cfg' = updateCfg (memoryCfg proxyCfg storeLog storeMsgs) $ \cfg_ -> cfg_ {storeNtfsFile = Just storeNtfs}
|
||||
let cfg' = updateCfg (journalCfg proxyCfg storeLog storeMsgs) $ \cfg_ -> cfg_ {storeNtfsFile = Just storeNtfs}
|
||||
in withSmpServerConfigOn (transport @TLS) cfg'
|
||||
a `up` cId = nGet a =##> \case ("", "", UP _ [c]) -> c == cId; _ -> False
|
||||
a `down` cId = nGet a =##> \case ("", "", DOWN _ [c]) -> c == cId; _ -> False
|
||||
@@ -435,7 +422,7 @@ agentViaProxyRetryNoSession = do
|
||||
_ <- runRight $ makeConnection b a
|
||||
pure ()
|
||||
where
|
||||
withServer2 = withSmpServerConfigOn (transport @TLS) proxyCfgS2 testPort2
|
||||
withServer2 = withSmpServerConfigOn (transport @TLS) proxyCfgJ2 testPort2
|
||||
servers srv = initAgentServersProxy {smp = userServers [srv]}
|
||||
|
||||
testNoProxy :: AStoreType -> IO ()
|
||||
@@ -461,27 +448,12 @@ requestRelaySession =
|
||||
testSMPClient_ "localhost" testPort supportedServerSMPRelayVRange Nothing $ \(th :: THandleSMP TLS 'TClient) ->
|
||||
(\(_, _, reply) -> reply) <$> sendRecv th (Nothing, "1", NoEntity, SMP.PRXY testSMPServer2 Nothing)
|
||||
|
||||
testChangedFwdVersion :: IO ()
|
||||
testChangedFwdVersion =
|
||||
withSmpServerConfigOn (transport @TLS) cfg testPort $ \_ -> do
|
||||
g <- C.newRandom
|
||||
ts <- getCurrentTime
|
||||
rc <- either (fail . show) pure =<< getProtocolClient g NRMInteractive (1, testSMPServer, Nothing) defaultSMPClientConfig [] Nothing ts (\_ -> pure ())
|
||||
THAuthClient {peerServerPubKey} <- maybe (fail "getProtocolClient returned no thAuth") pure $ thAuth $ thParams rc
|
||||
(cmdPubKey, cmdPrivKey) <- atomically $ C.generateKeyPair g
|
||||
nonce@(C.CbNonce corrId) <- atomically $ C.randomCbNonce g
|
||||
let v = currentClientSMPRelayVersion
|
||||
et <- either (fail . show) (pure . SMP.EncTransmission) $ C.cbEncrypt (C.dh' peerServerPubKey cmdPrivKey) (SMP.encTransmissionNonce v nonce) "" SMP.paddedProxiedTLength
|
||||
let forward fwdVersion = forwardSMPTransmission rc (SMP.CorrId corrId) fwdVersion cmdPubKey et
|
||||
_ <- runExceptT' $ forward v
|
||||
runExceptT (forward $ prevVersion v) `shouldReturn` Left (PCEProtocolError SMP.CRYPTO)
|
||||
|
||||
-- Shared "phase 2" of the reconnection tests: start a healthy relay, confirm it is reachable
|
||||
-- directly (PING, not via the proxy) so a proxy failure can only mean the proxy didn't reconnect,
|
||||
-- let any stored connection error expire, then require the proxy to establish the session (PKEY).
|
||||
requireProxyReconnect :: IO ()
|
||||
requireProxyReconnect =
|
||||
withSmpServerConfigOn (transport @TLS) proxyCfgS2 testPort2 $ \_ -> do
|
||||
withSmpServerConfigOn (transport @TLS) proxyCfgJ2 testPort2 $ \_ -> do
|
||||
testSMPClient_ "127.0.0.1" testPort2 supportedServerSMPRelayVRange Nothing $ \(th :: THandleSMP TLS 'TClient) -> do
|
||||
(_, _, reply) <- sendRecv th (Nothing, "0", NoEntity, SMP.PING)
|
||||
reply `shouldBe` Right SMP.PONG
|
||||
@@ -524,6 +496,19 @@ testProxyReconnectAfterRelayRestart =
|
||||
race_ (threadDelay 1000000) requestRelaySession
|
||||
requireProxyReconnect
|
||||
|
||||
testProxyForwardTimeoutStuckSession :: IO ()
|
||||
testProxyForwardTimeoutStuckSession =
|
||||
withSmpServerConfigOn (transport @TLS) proxyCfgForwardTimeout testPort $ \_ -> do
|
||||
g <- C.newRandom
|
||||
ts <- getCurrentTime
|
||||
let srv = SMPServer testHost testPort testKeyHash
|
||||
vr = mkVersionRange minServerSMPRelayVersion currentClientSMPRelayVersion
|
||||
pc <- either (fail . show) pure =<< getProtocolClient g NRMInteractive (1, srv, Nothing) defaultSMPClientConfig {serverVRange = vr} [] Nothing ts (\_ -> pure ())
|
||||
sess <- runExceptT' $ connectSMPProxiedRelay pc NRMInteractive srv (Just "correct")
|
||||
sId <- atomically $ SMP.EntityId <$> C.randomBytes 24 g
|
||||
rs <- forM ([1 .. 10] :: [Int]) $ \_ -> runExceptT' (proxySMPMessage pc NRMInteractive sess Nothing sId noMsgFlags "hi")
|
||||
rs `shouldSatisfy` elem (Left (ProxyProtocolError (SMP.PROXY SMP.NO_SESSION)))
|
||||
|
||||
-- Bug B (same root cause as the proxy, in the messaging agent): getSMPServerClient inserts an
|
||||
-- empty SessionVar into smpClients, then connects inside newProtocolClient's tryAllErrors, which
|
||||
-- rethrows async exceptions. If the connecting thread is cancelled mid-connect, putTMVar is
|
||||
@@ -537,7 +522,7 @@ testAgentClientReconnectAfterCancel =
|
||||
t <- async $ runExceptT $ A.createConnection a NRMInteractive 1 True True SCMInvitation Nothing Nothing CR.IKPQOn False SMSubscribe
|
||||
threadDelay 1000000 -- let the connect to the stalling relay start, then kill it mid-flight
|
||||
cancel t
|
||||
withSmpServerConfigOn (transport @TLS) cfgS2 testPort2 $ \_ -> do
|
||||
withSmpServerConfigOn (transport @TLS) cfgJ2 testPort2 $ \_ -> do
|
||||
testSMPClient_ "127.0.0.1" testPort2 supportedServerSMPRelayVRange Nothing $ \(th :: THandleSMP TLS 'TClient) -> do
|
||||
(_, _, reply) <- sendRecv th (Nothing, "0", NoEntity, SMP.PING)
|
||||
reply `shouldBe` Right SMP.PONG -- the relay is up and reachable, so a timeout can only be the poisoned var
|
||||
|
||||
+22
-13
@@ -24,6 +24,7 @@ import Control.Concurrent.STM
|
||||
import Control.Exception (SomeException, throwIO, try)
|
||||
import Control.Monad
|
||||
import Control.Monad.IO.Class
|
||||
import CoreTests.MsgStoreTests (testJournalStoreCfg)
|
||||
import Data.Bifunctor (first)
|
||||
import qualified Data.ByteString.Base64 as B64
|
||||
import Data.ByteString.Char8 (ByteString)
|
||||
@@ -45,14 +46,17 @@ import Simplex.Messaging.Encoding
|
||||
import Simplex.Messaging.Encoding.String
|
||||
import Simplex.Messaging.Parsers (parseAll, parseString)
|
||||
import Simplex.Messaging.Protocol
|
||||
import Simplex.Messaging.Server.Env.STM (AStoreType (..), ServerConfig (..), ServerStoreCfg (..))
|
||||
import Simplex.Messaging.Server (exportMessages)
|
||||
import Simplex.Messaging.Server.Env.STM (AStoreType (..), MsgStore (..), ServerConfig (..), ServerStoreCfg (..), readWriteQueueStore)
|
||||
import Simplex.Messaging.Server.Expiration
|
||||
import Simplex.Messaging.Server.MsgStore.Types (SMSType (..), SQSType (..))
|
||||
import Simplex.Messaging.Server.MsgStore.Journal (JournalStoreConfig (..), QStoreCfg (..), stmQueueStore)
|
||||
import Simplex.Messaging.Server.MsgStore.Types (MsgStoreClass (..), QSType (..), SMSType (..), SQSType (..), newMsgStore)
|
||||
import Simplex.Messaging.Server.Stats (PeriodStatsData (..), ServerStatsData (..))
|
||||
import Simplex.Messaging.Server.StoreLog (StoreLogRecord (..))
|
||||
import Simplex.Messaging.Server.StoreLog (StoreLogRecord (..), closeStoreLog)
|
||||
import Simplex.Messaging.Transport
|
||||
import Simplex.Messaging.Transport.Credentials
|
||||
import System.Directory (doesFileExist, removeFile)
|
||||
import Simplex.Messaging.Util (whenM)
|
||||
import System.Directory (doesDirectoryExist, doesFileExist, removeDirectoryRecursive, removeFile)
|
||||
import System.IO (IOMode (..), withFile)
|
||||
import System.TimeIt (timeItT)
|
||||
import System.Timeout
|
||||
@@ -62,11 +66,7 @@ import Util
|
||||
|
||||
#if defined(dbServerPostgres)
|
||||
import CoreTests.MsgStoreTests (testPostgresStoreConfig)
|
||||
import Simplex.Messaging.Server.Env.STM (readWriteQueueStore)
|
||||
import Simplex.Messaging.Server.MsgStore.Postgres (PostgresMsgStoreCfg (..), exportDbMessages)
|
||||
import Simplex.Messaging.Server.MsgStore.STM (STMStoreConfig (..))
|
||||
import Simplex.Messaging.Server.MsgStore.Types (MsgStoreClass (..), newMsgStore)
|
||||
import Simplex.Messaging.Server.StoreLog (closeStoreLog)
|
||||
#endif
|
||||
|
||||
serverTests :: SpecWith (ASrvTransport, AStoreType)
|
||||
@@ -1068,6 +1068,7 @@ testRestoreMessages =
|
||||
it "should store messages on exit and restore on start" $ \(at@(ATransport t), msType) -> do
|
||||
removeFileIfExists testStoreLogFile
|
||||
removeFileIfExists testStoreMsgsFile
|
||||
whenM (doesDirectoryExist testStoreMsgsDir) $ removeDirectoryRecursive testStoreMsgsDir
|
||||
removeFileIfExists testServerStatsBackupFile
|
||||
|
||||
g <- C.newRandom
|
||||
@@ -1137,6 +1138,7 @@ testRestoreMessages =
|
||||
Right stats3 <- strDecode <$> B.readFile testServerStatsBackupFile
|
||||
checkStats stats3 [rId] 5 5
|
||||
removeFileIfExists testStoreMsgsFile
|
||||
whenM (doesDirectoryExist testStoreMsgsDir) $ removeDirectoryRecursive testStoreMsgsDir
|
||||
removeFile testServerStatsBackupFile
|
||||
where
|
||||
runTest :: Transport c => TProxy c 'TServer -> (THandleSMP c 'TClient -> IO ()) -> ThreadId -> Expectation
|
||||
@@ -1216,21 +1218,28 @@ testRestoreExpireMessages =
|
||||
where
|
||||
exportStoreMessages :: AStoreType -> IO ()
|
||||
exportStoreMessages = \case
|
||||
ASType _ SMSJournal -> export
|
||||
ASType _ SMSPostgres -> exportDB
|
||||
ASType _ SMSMemory -> pure ()
|
||||
where
|
||||
export = do
|
||||
ms <- readWriteQueues
|
||||
exportMessages False (StoreJournal ms) testStoreMsgsFile False
|
||||
closeMsgStore ms
|
||||
#if defined(dbServerPostgres)
|
||||
exportDB = do
|
||||
ms <- newMsgStore STMStoreConfig {storePath = Nothing, quota = 4}
|
||||
readWriteQueueStore True (mkQueue ms) testStoreLogFile (queueStore ms) >>= closeStoreLog
|
||||
removeFileIfExists testStoreMsgsFile
|
||||
closeMsgStore ms
|
||||
readWriteQueues >>= closeMsgStore
|
||||
ms' <- newMsgStore (testPostgresStoreConfig {quota = 4} :: PostgresMsgStoreCfg)
|
||||
_n <- withFile testStoreMsgsFile WriteMode $ exportDbMessages False ms'
|
||||
closeMsgStore ms'
|
||||
#else
|
||||
exportDB = error "compiled without server_postgres flag"
|
||||
#endif
|
||||
readWriteQueues = do
|
||||
ms <- newMsgStore ((testJournalStoreCfg MQStoreCfg) {quota = 4} :: JournalStoreConfig 'QSMemory)
|
||||
readWriteQueueStore True (mkQueue ms True) testStoreLogFile (stmQueueStore ms) >>= closeStoreLog
|
||||
removeFileIfExists testStoreMsgsFile
|
||||
pure ms
|
||||
runTest :: Transport c => TProxy c 'TServer -> (THandleSMP c 'TClient -> IO ()) -> ThreadId -> Expectation
|
||||
runTest _ test' server = do
|
||||
testSMPClient test' `shouldReturn` ()
|
||||
@@ -1540,7 +1549,7 @@ testMsgExpireOnInterval =
|
||||
it "should expire messages that are not received before messageTTL after expiry interval" $ \(ATransport (t :: TProxy c 'TServer), msType) -> do
|
||||
g <- C.newRandom
|
||||
(sPub, sKey) <- atomically $ C.generateAuthKeyPair C.SEd25519 g
|
||||
let cfg' = updateCfg (cfgMS msType) $ \cfg_ -> cfg_ {messageExpiration = Just ExpirationConfig {ttl = 1, checkInterval = 1}}
|
||||
let cfg' = updateCfg (cfgMS msType) $ \cfg_ -> cfg_ {messageExpiration = Just ExpirationConfig {ttl = 1, checkInterval = 1}, idleQueueInterval = 1}
|
||||
withSmpServerConfigOn (ATransport t) cfg' testPort $ \_ ->
|
||||
testSMPClient @c $ \sh -> do
|
||||
(sId, rId, rKey, _) <- testSMPClient @c $ \rh -> createAndSecureQueue rh sPub
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user