mirror of
https://github.com/azukaar/Cosmos-Server.git
synced 2026-10-12 02:11:26 +00:00
[release] v0.23.00-unstable001
This commit is contained in:
Yann Stepienik
+57
-1
@@ -4,7 +4,7 @@
|
||||
"description": "REST API for Cosmos Cloud server management",
|
||||
"title": "Cosmos Server API",
|
||||
"contact": {},
|
||||
"version": "0.22.35"
|
||||
"version": "0.23.00-unstable001"
|
||||
},
|
||||
"paths": {
|
||||
"/_logs": {
|
||||
@@ -3113,6 +3113,62 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/constellation/force-reform": {
|
||||
"post": {
|
||||
"security": [
|
||||
{
|
||||
"BearerAuth": []
|
||||
}
|
||||
],
|
||||
"description": "HA only. Bumps the op-log epoch, drops local JetStream state, blocks the other managers and restarts NATS, leaving this node writable in formation mode so replacements can be enrolled. Does not touch the CA, keys, config or device data.",
|
||||
"tags": [
|
||||
"constellation"
|
||||
],
|
||||
"summary": "Force the constellation to re-form around this manager",
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "OK",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/utils.APIResponse"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "Bad Request",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/utils.HTTPErrorResult"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"403": {
|
||||
"description": "Forbidden",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/utils.HTTPErrorResult"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Internal Server Error",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/utils.HTTPErrorResult"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/constellation/get-next-ip": {
|
||||
"get": {
|
||||
"security": [
|
||||
|
||||
+44
-1
@@ -4,7 +4,7 @@
|
||||
"description": "REST API for Cosmos Cloud server management",
|
||||
"title": "Cosmos Server API",
|
||||
"contact": {},
|
||||
"version": "0.22.35"
|
||||
"version": "0.23.00-unstable001"
|
||||
},
|
||||
"basePath": "/cosmos",
|
||||
"paths": {
|
||||
@@ -2563,6 +2563,49 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/constellation/force-reform": {
|
||||
"post": {
|
||||
"security": [
|
||||
{
|
||||
"BearerAuth": []
|
||||
}
|
||||
],
|
||||
"description": "HA only. Bumps the op-log epoch, drops local JetStream state, blocks the other managers and restarts NATS, leaving this node writable in formation mode so replacements can be enrolled. Does not touch the CA, keys, config or device data.",
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"constellation"
|
||||
],
|
||||
"summary": "Force the constellation to re-form around this manager",
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "OK",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/utils.APIResponse"
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "Bad Request",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/utils.HTTPErrorResult"
|
||||
}
|
||||
},
|
||||
"403": {
|
||||
"description": "Forbidden",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/utils.HTTPErrorResult"
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Internal Server Error",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/utils.HTTPErrorResult"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/constellation/get-next-ip": {
|
||||
"get": {
|
||||
"security": [
|
||||
|
||||
+31
-1
@@ -2061,7 +2061,7 @@ info:
|
||||
contact: {}
|
||||
description: REST API for Cosmos Cloud server management
|
||||
title: Cosmos Server API
|
||||
version: 0.22.35
|
||||
version: 0.23.00-unstable001
|
||||
paths:
|
||||
/_logs:
|
||||
get:
|
||||
@@ -3694,6 +3694,36 @@ paths:
|
||||
summary: Edit the current Constellation device properties
|
||||
tags:
|
||||
- constellation
|
||||
/api/constellation/force-reform:
|
||||
post:
|
||||
description: HA only. Bumps the op-log epoch, drops local JetStream state, blocks
|
||||
the other managers and restarts NATS, leaving this node writable in formation
|
||||
mode so replacements can be enrolled. Does not touch the CA, keys, config
|
||||
or device data.
|
||||
produces:
|
||||
- application/json
|
||||
responses:
|
||||
"200":
|
||||
description: OK
|
||||
schema:
|
||||
$ref: '#/definitions/utils.APIResponse'
|
||||
"400":
|
||||
description: Bad Request
|
||||
schema:
|
||||
$ref: '#/definitions/utils.HTTPErrorResult'
|
||||
"403":
|
||||
description: Forbidden
|
||||
schema:
|
||||
$ref: '#/definitions/utils.HTTPErrorResult'
|
||||
"500":
|
||||
description: Internal Server Error
|
||||
schema:
|
||||
$ref: '#/definitions/utils.HTTPErrorResult'
|
||||
security:
|
||||
- BearerAuth: []
|
||||
summary: Force the constellation to re-form around this manager
|
||||
tags:
|
||||
- constellation
|
||||
/api/constellation/get-next-ip:
|
||||
get:
|
||||
produces:
|
||||
|
||||
@@ -1,3 +1,9 @@
|
||||
## Version 0.23.00
|
||||
- Remove Lungo, use SQLite instead
|
||||
- Remove date base sync - use message based sync
|
||||
- Fix a couple of Constellation DNS resolution issues
|
||||
- Fix quirks with inverted setups (agent is private)
|
||||
|
||||
## Version 0.22.35
|
||||
- Fix "JetStream Unavailable" once and for all (hopefully :p )
|
||||
- Fix rare race condition breaking config file
|
||||
|
||||
@@ -98,6 +98,16 @@ export default function createConstellationAPI(apiFetch: ApiFetch) {
|
||||
}))
|
||||
}
|
||||
|
||||
// HA only: re-form the constellation around this manager after a lost quorum
|
||||
function forceReform() {
|
||||
return wrap(apiFetch('/cosmos/api/constellation/force-reform', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json'
|
||||
}
|
||||
}))
|
||||
}
|
||||
|
||||
function getConfig() {
|
||||
return wrap(apiFetch('/cosmos/api/constellation/config', {
|
||||
method: 'GET',
|
||||
@@ -230,6 +240,7 @@ export default function createConstellationAPI(apiFetch: ApiFetch) {
|
||||
getConfig,
|
||||
getLogs,
|
||||
reset,
|
||||
forceReform,
|
||||
connect,
|
||||
block,
|
||||
ping,
|
||||
|
||||
@@ -10,7 +10,9 @@ import * as React from 'react';
|
||||
import { useEffect, useState } from 'react';
|
||||
import { useTranslation } from 'react-i18next';
|
||||
|
||||
const ConfirmModal = ({ callback, label, content, startIcon }) => {
|
||||
// variant/color were already being passed by call sites but ignored; they now
|
||||
// apply, with the previous hardcoded values as the defaults.
|
||||
const ConfirmModal = ({ callback, label, content, startIcon, variant = 'outlined', color = 'warning' }) => {
|
||||
const { t } = useTranslation();
|
||||
const [openModal, setOpenModal] = useState(false);
|
||||
|
||||
@@ -36,8 +38,8 @@ const ConfirmModal = ({ callback, label, content, startIcon }) => {
|
||||
|
||||
<Button
|
||||
disableElevation
|
||||
variant="outlined"
|
||||
color="warning"
|
||||
variant={variant}
|
||||
color={color}
|
||||
startIcon={startIcon}
|
||||
onClick={() => {
|
||||
setOpenModal(true);
|
||||
|
||||
@@ -16,7 +16,7 @@ import { isDomain } from "../../utils/indexs";
|
||||
import ConfirmModal from "../../components/confirmModal";
|
||||
import UploadButtons from "../../components/fileUpload";
|
||||
import { useClientInfos } from "../../utils/hooks";
|
||||
import { PERM_RESOURCES } from "../../utils/permissions";
|
||||
import { PERM_ADMIN, PERM_RESOURCES } from "../../utils/permissions";
|
||||
import PermissionGuard from "../../components/permissionGuard";
|
||||
import { Trans, useTranslation } from 'react-i18next';
|
||||
import ResyncDeviceModal from "./resyncDevice";
|
||||
@@ -505,6 +505,23 @@ export const ConstellationVPN = ({ freeVersion }) => {
|
||||
refreshConfig();
|
||||
}}
|
||||
/>
|
||||
{/* HA only: the way out of a quorum this network can no longer
|
||||
recover on its own. Guarded on PERM_ADMIN to match the
|
||||
endpoint — the surrounding block is PERM_RESOURCES, which
|
||||
would show a resources operator a destructive button that
|
||||
only ever 403s. */}
|
||||
{natsInfo?.haMode && <PermissionGuard permission={PERM_ADMIN}>
|
||||
<ConfirmModal
|
||||
variant="outlined"
|
||||
color="error"
|
||||
label={t('mgmt.constellation.forceReformLabel')}
|
||||
content={t('mgmt.constellation.forceReformText')}
|
||||
callback={async () => {
|
||||
await API.constellation.forceReform();
|
||||
refreshConfig();
|
||||
}}
|
||||
/>
|
||||
</PermissionGuard>}
|
||||
</Stack>
|
||||
</Box>
|
||||
</PermissionGuard>}
|
||||
|
||||
@@ -311,6 +311,8 @@
|
||||
"mgmt.constellation.isRelay.label": "Relay traffic via this Lighthouse",
|
||||
"mgmt.constellation.isExitNode.label": "Use this Lighthouse as an exit node",
|
||||
"mgmt.constellation.isLoadBalancer.label": "Load Balancer / Tunnel Input Point",
|
||||
"mgmt.constellation.forceReformLabel": "Force Re-form Cluster",
|
||||
"mgmt.constellation.forceReformText": "Use this only when this network has permanently lost the majority of its manager nodes and configuration has become read-only. It rebuilds the cluster around this server alone: the other managers are removed and must be re-enrolled from scratch, and until they are, they can no longer change any configuration. Your devices, users, certificates and settings are kept. Note that a removed server keeps the configuration it already holds, so if one of them was lost or compromised, rotate your certificates, API tokens and stored credentials afterwards. This cannot be undone.",
|
||||
"mgmt.constellation.resetLabel": "Reset Network",
|
||||
"mgmt.constellation.resetText": "This will completely reset the network, and disconnect all the clients. You will need to reconnect them. This cannot be undone.",
|
||||
"mgmt.constellation.restartButton": "Restart VPN Service",
|
||||
|
||||
+499
-312
@@ -2350,6 +2350,13 @@ type ClientInterface interface {
|
||||
// Corresponds with POST /api/constellation/edit-device (the `PostApiConstellationEditDevice` operationId).
|
||||
PostApiConstellationEditDevice(ctx context.Context, body PostApiConstellationEditDeviceJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error)
|
||||
|
||||
// PostApiConstellationForceReform Force the constellation to re-form around this manager
|
||||
//
|
||||
// HA only. Bumps the op-log epoch, drops local JetStream state, blocks the other managers and restarts NATS, leaving this node writable in formation mode so replacements can be enrolled. Does not touch the CA, keys, config or device data.
|
||||
//
|
||||
// Corresponds with POST /api/constellation/force-reform (the `PostApiConstellationForceReform` operationId).
|
||||
PostApiConstellationForceReform(ctx context.Context, reqEditors ...RequestEditorFn) (*http.Response, error)
|
||||
|
||||
// GetApiConstellationGetNextIp Get the next available IP address in the Constellation CIDR range
|
||||
//
|
||||
// Corresponds with GET /api/constellation/get-next-ip (the `GetApiConstellationGetNextIp` operationId).
|
||||
@@ -4708,6 +4715,23 @@ func (c *Client) PostApiConstellationEditDevice(ctx context.Context, body PostAp
|
||||
return c.Client.Do(req)
|
||||
}
|
||||
|
||||
// PostApiConstellationForceReform Force the constellation to re-form around this manager
|
||||
//
|
||||
// HA only. Bumps the op-log epoch, drops local JetStream state, blocks the other managers and restarts NATS, leaving this node writable in formation mode so replacements can be enrolled. Does not touch the CA, keys, config or device data.
|
||||
//
|
||||
// Corresponds with POST /api/constellation/force-reform (the `PostApiConstellationForceReform` operationId).
|
||||
func (c *Client) PostApiConstellationForceReform(ctx context.Context, reqEditors ...RequestEditorFn) (*http.Response, error) {
|
||||
req, err := NewPostApiConstellationForceReformRequest(c.Server)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req = req.WithContext(ctx)
|
||||
if err := c.applyEditors(ctx, req, reqEditors); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return c.Client.Do(req)
|
||||
}
|
||||
|
||||
// GetApiConstellationGetNextIp Get the next available IP address in the Constellation CIDR range
|
||||
//
|
||||
// Corresponds with GET /api/constellation/get-next-ip (the `GetApiConstellationGetNextIp` operationId).
|
||||
@@ -9319,6 +9343,33 @@ func NewPostApiConstellationEditDeviceRequestWithBody(server string, contentType
|
||||
return req, nil
|
||||
}
|
||||
|
||||
// NewPostApiConstellationForceReformRequest constructs an http.Request for the PostApiConstellationForceReform method
|
||||
func NewPostApiConstellationForceReformRequest(server string) (*http.Request, error) {
|
||||
var err error
|
||||
|
||||
serverURL, err := url.Parse(server)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
operationPath := fmt.Sprintf("/api/constellation/force-reform")
|
||||
if operationPath[0] == '/' {
|
||||
operationPath = "." + operationPath
|
||||
}
|
||||
|
||||
queryURL, err := serverURL.Parse(operationPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(http.MethodPost, queryURL.String(), nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return req, nil
|
||||
}
|
||||
|
||||
// NewGetApiConstellationGetNextIpRequest constructs an http.Request for the GetApiConstellationGetNextIp method
|
||||
func NewGetApiConstellationGetNextIpRequest(server string) (*http.Request, error) {
|
||||
var err error
|
||||
@@ -14515,6 +14566,15 @@ type ClientWithResponsesInterface interface {
|
||||
// Corresponds with POST /api/constellation/edit-device (the `PostApiConstellationEditDevice` operationId).
|
||||
PostApiConstellationEditDeviceWithResponse(ctx context.Context, body PostApiConstellationEditDeviceJSONRequestBody, reqEditors ...RequestEditorFn) (*PostApiConstellationEditDeviceResponse, error)
|
||||
|
||||
// PostApiConstellationForceReformWithResponse Force the constellation to re-form around this manager
|
||||
//
|
||||
// HA only. Bumps the op-log epoch, drops local JetStream state, blocks the other managers and restarts NATS, leaving this node writable in formation mode so replacements can be enrolled. Does not touch the CA, keys, config or device data.
|
||||
//
|
||||
// Returns a wrapper object for the known response body format(s).
|
||||
//
|
||||
// Corresponds with POST /api/constellation/force-reform (the `PostApiConstellationForceReform` operationId).
|
||||
PostApiConstellationForceReformWithResponse(ctx context.Context, reqEditors ...RequestEditorFn) (*PostApiConstellationForceReformResponse, error)
|
||||
|
||||
// GetApiConstellationGetNextIpWithResponse Get the next available IP address in the Constellation CIDR range
|
||||
//
|
||||
// Returns a wrapper object for the known response body format(s).
|
||||
@@ -18756,6 +18816,68 @@ func (r PostApiConstellationEditDeviceResponse) ContentType() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
type PostApiConstellationForceReformResponse struct {
|
||||
Body []byte
|
||||
HTTPResponse *http.Response
|
||||
// JSON200 the response for an HTTP 200 `application/json` response
|
||||
JSON200 *UtilsAPIResponse
|
||||
// JSON400 the response for an HTTP 400 `application/json` response
|
||||
JSON400 *UtilsHTTPErrorResult
|
||||
// JSON403 the response for an HTTP 403 `application/json` response
|
||||
JSON403 *UtilsHTTPErrorResult
|
||||
// JSON500 the response for an HTTP 500 `application/json` response
|
||||
JSON500 *UtilsHTTPErrorResult
|
||||
}
|
||||
|
||||
// GetJSON200 returns the response for an HTTP 200 `application/json` response
|
||||
func (r PostApiConstellationForceReformResponse) GetJSON200() *UtilsAPIResponse {
|
||||
return r.JSON200
|
||||
}
|
||||
|
||||
// GetJSON400 returns the response for an HTTP 400 `application/json` response
|
||||
func (r PostApiConstellationForceReformResponse) GetJSON400() *UtilsHTTPErrorResult {
|
||||
return r.JSON400
|
||||
}
|
||||
|
||||
// GetJSON403 returns the response for an HTTP 403 `application/json` response
|
||||
func (r PostApiConstellationForceReformResponse) GetJSON403() *UtilsHTTPErrorResult {
|
||||
return r.JSON403
|
||||
}
|
||||
|
||||
// GetJSON500 returns the response for an HTTP 500 `application/json` response
|
||||
func (r PostApiConstellationForceReformResponse) GetJSON500() *UtilsHTTPErrorResult {
|
||||
return r.JSON500
|
||||
}
|
||||
|
||||
// GetBody returns the raw response body bytes
|
||||
func (r PostApiConstellationForceReformResponse) GetBody() []byte {
|
||||
return r.Body
|
||||
}
|
||||
|
||||
// Status returns HTTPResponse.Status
|
||||
func (r PostApiConstellationForceReformResponse) Status() string {
|
||||
if r.HTTPResponse != nil {
|
||||
return r.HTTPResponse.Status
|
||||
}
|
||||
return http.StatusText(0)
|
||||
}
|
||||
|
||||
// StatusCode returns HTTPResponse.StatusCode
|
||||
func (r PostApiConstellationForceReformResponse) StatusCode() int {
|
||||
if r.HTTPResponse != nil {
|
||||
return r.HTTPResponse.StatusCode
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// ContentType is a convenience method to retrieve the Content-Type value from the HTTP response headers
|
||||
func (r PostApiConstellationForceReformResponse) ContentType() string {
|
||||
if r.HTTPResponse != nil {
|
||||
return r.HTTPResponse.Header.Get("Content-Type")
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
type GetApiConstellationGetNextIpResponse struct {
|
||||
Body []byte
|
||||
HTTPResponse *http.Response
|
||||
@@ -26704,6 +26826,21 @@ func (c *ClientWithResponses) PostApiConstellationEditDeviceWithResponse(ctx con
|
||||
return ParsePostApiConstellationEditDeviceResponse(rsp)
|
||||
}
|
||||
|
||||
// PostApiConstellationForceReformWithResponse Force the constellation to re-form around this manager
|
||||
//
|
||||
// HA only. Bumps the op-log epoch, drops local JetStream state, blocks the other managers and restarts NATS, leaving this node writable in formation mode so replacements can be enrolled. Does not touch the CA, keys, config or device data.
|
||||
//
|
||||
// Returns a wrapper object for the known response body format(s).
|
||||
//
|
||||
// Corresponds with POST /api/constellation/force-reform (the `PostApiConstellationForceReform` operationId).
|
||||
func (c *ClientWithResponses) PostApiConstellationForceReformWithResponse(ctx context.Context, reqEditors ...RequestEditorFn) (*PostApiConstellationForceReformResponse, error) {
|
||||
rsp, err := c.PostApiConstellationForceReform(ctx, reqEditors...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ParsePostApiConstellationForceReformResponse(rsp)
|
||||
}
|
||||
|
||||
// GetApiConstellationGetNextIpWithResponse Get the next available IP address in the Constellation CIDR range
|
||||
//
|
||||
// Returns a wrapper object for the known response body format(s).
|
||||
@@ -31158,6 +31295,53 @@ func ParsePostApiConstellationEditDeviceResponse(rsp *http.Response) (*PostApiCo
|
||||
return response, nil
|
||||
}
|
||||
|
||||
// ParsePostApiConstellationForceReformResponse parses an HTTP response from a PostApiConstellationForceReformWithResponse call
|
||||
func ParsePostApiConstellationForceReformResponse(rsp *http.Response) (*PostApiConstellationForceReformResponse, error) {
|
||||
bodyBytes, err := io.ReadAll(rsp.Body)
|
||||
defer func() { _ = rsp.Body.Close() }()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
response := &PostApiConstellationForceReformResponse{
|
||||
Body: bodyBytes,
|
||||
HTTPResponse: rsp,
|
||||
}
|
||||
|
||||
switch {
|
||||
case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200:
|
||||
var dest UtilsAPIResponse
|
||||
if err := json.Unmarshal(bodyBytes, &dest); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
response.JSON200 = &dest
|
||||
|
||||
case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 400:
|
||||
var dest UtilsHTTPErrorResult
|
||||
if err := json.Unmarshal(bodyBytes, &dest); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
response.JSON400 = &dest
|
||||
|
||||
case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 403:
|
||||
var dest UtilsHTTPErrorResult
|
||||
if err := json.Unmarshal(bodyBytes, &dest); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
response.JSON403 = &dest
|
||||
|
||||
case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 500:
|
||||
var dest UtilsHTTPErrorResult
|
||||
if err := json.Unmarshal(bodyBytes, &dest); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
response.JSON500 = &dest
|
||||
|
||||
}
|
||||
|
||||
return response, nil
|
||||
}
|
||||
|
||||
// ParseGetApiConstellationGetNextIpResponse parses an HTTP response from a GetApiConstellationGetNextIpWithResponse call
|
||||
func ParseGetApiConstellationGetNextIpResponse(rsp *http.Response) (*GetApiConstellationGetNextIpResponse, error) {
|
||||
bodyBytes, err := io.ReadAll(rsp.Body)
|
||||
@@ -36256,318 +36440,321 @@ func ParsePostRcloneVfsStatsResponse(rsp *http.Response) (*PostRcloneVfsStatsRes
|
||||
// const string: with thousands of chunks the chained `+` fold is several
|
||||
// times slower for the Go compiler than parsing a slice literal.
|
||||
var swaggerSpec = []string{
|
||||
"7P3rbhs51igM3wqhbwPtALKcTnc/2E+A/cPxodszPkFyMth7HGRTVUsS21VkDcmSow4CfBfxXuF7JS+4",
|
||||
"yCpVSaRUJduJna4fMx2reOY6cR2/9CKRZoID16r39ktPwn9yUPqdiBngD+9EvDD/jQTXwLX5J82yhEVU",
|
||||
"M8EP/lSCm99UNIOUmn/pRQa9tz0x/hMi3fv69Wu/F4OKJMtMh97b3hV+IfdMz8hEkQmDJCYqg4hNFoxP",
|
||||
"iZ4BkZAKDb0+rodJiHtvtczha78XScEH/xDjoV3oP0ZXl62W9z8kTHpve/+/g+W+D+xXdeAb27OBf4gx",
|
||||
"YTFwzSYMpG+RuWaJGhwmIPWjLa46pmdR+IFEgk/YlOxdX41uiODJ4lWvvpw332o977OYaogJra/r/dqy",
|
||||
"joZXl0f4+ZGXVhl4w/pMq2bLe/NN11dfl+c6R5xmw8Oz4yc5u5XBN5xf0bLFWt9888VuXOTXvptNuXVN",
|
||||
"2JRmbHAkgWo4vD67EXfAHU0wTTIpMpDaEcjaTCX5U1oyPjUnAJ8zJhfHdKEqnxnXMAVpvrPsXzOmIWF2",
|
||||
"cKYhVd6B3A9USrowf3OagrdhBjJlSjHBVW3E7Sd5Xfb0TSiBxlc8WVQmHQuRAOX2q1lBpG/EkeBKQ5LQ",
|
||||
"lTMpG3+tksx/24187K9yjn7lLo4vR/Z2g9cQ5UqL9PhydMK1dL+12HltzW6Uhe8QYq4O45iZZjR5l4jo",
|
||||
"ztycand1MVfY9V1CbX//kcZcndIkGdPozjtszNW1kNrz7evm06RqNhZUxkORa1g/yyPBNWUc5FkUAOqy",
|
||||
"xTDn3Py2igjLFoTxiSB7EyHJ6GT44fD6mkgzrXrV63v2fLwFnf5gMZxKkZZ78B/dH0Jpb//gli5E7Een",
|
||||
"yxCeXVM9u5YwYZ+9n2+onIJ/Ee8VrCywsvT3CrwjV9Fn0+VCAg3IVoB6tEdNywgeTCZ3JoNPR+3a0LOt",
|
||||
"B4SoFr4NuC9xscHCpfi8wPYFy+v3zGg0eLqIcZeNrnw5UODel2cxOIY5iwBJ2Yo8Xt/e2LQI0Dgcolha",
|
||||
"Sj+fA5/qWe/tL2/6vZTx8s/++q44i+74Dj1XdlxZQmXMpru3YsLG7V/n44RFBuODPDsSKhXqsk6DKlLC",
|
||||
"Q46J8TlTbJyA/wZY5kdIdfKZ6ZUVVbupczad6ZnIFayTf0Puk/I7ilpecs/UuaDxO5pQHoEMzTSEhAbE",
|
||||
"jioMrC/BHpsq5l85t3XJKchNg/DCssaQchIzvRFO2h75ExynptNWskyA9FnWP1i+TeobpXKqTlREM4jX",
|
||||
"7+1G5kDYhEQiTSmPCVOEJkb2XBCwfcheCpSjrsAgH6GK0LJ5wji8Inv/YjwW98opFlj0auCFP6o1jWYj",
|
||||
"HYOU60s5xK+okVCa8pjKmICUQm4ei/FmQzGe5bpPUnoHimRCIZKSXKHQpEHSCDtvnEvkutlkItdZrr2D",
|
||||
"RWnsE+DseWpBZM7J/Qy4GUzqQkdTXrPBguair0gp436MPV5+8yAnGJE8E4x7NnxSfnvs9QKfr093zpQm",
|
||||
"YkKAz5kUPAWuyZxKRs39aUEUaML4Q2b9nAkFsZHuLcIkydWk9/bfm6UDTvXAdBmB7n392A8t2o5NMhz8",
|
||||
"a783A5roWTQDy6ObzbVE8j+weyGJrE37x3J0Yr+MQZGZuDcHZX+unZPBd7siPIlZhWuuDFx88bG8lE49",
|
||||
"XQzNNkdgJsQmhnQwTe6pIhlV5lTGC/xqpSAhyR4MpgMSiTyJyRiIWqRjkbDolW/WhI4hsfdVPhGva4Rv",
|
||||
"/QnnvSI7EIKRFkTPmKrB0RrFTWl0GMcS1Lqo3bugEXEfi60v7+6W3/JjyCREVEP81k5lNbJMkbj8QhTj",
|
||||
"EZDD6zMy/3nw668D8l4BOeEx4twItMEyNbgo10GY4X80HvjOiYO+F/LumCmDMB7ac6aIa0PiopGPcAn+",
|
||||
"LmeJZ4Cry3fvz86PSQqaxlRTomdUk3uQQGKYMA4xEbwCBsciugM5YQm0QlORAQ/Q+6sM6Y/55lu5mkGS",
|
||||
"rPcamZ+JkVywwf5EyNTc2vD9ZZ8cXRz3ycnlzfB/X1+dXd60Wimu5IpHHqQ4mxAtc+iTKBEK7JoJnWiQ",
|
||||
"eEA/E8trICZRwgyli5mKBOcQaeVnqkqLbMSmnPp2iL8jkdQiQ769BtjLbZg2NywFL5dzH8ge40RBJHis",
|
||||
"Xm0euCJQa70I8s2SZyojX6TKDEqJ1os+YTxK8tiwFHtObGLoB1OEC20PMPafieHq6xO+V3jKBjZZkiDj",
|
||||
"sgiKvHdPvTJ4xGKoo22f0EQJovLM0HAUGN5Opcgz3xHORZKn0IQqLemJnyq5kcheKnKu1SszcYzAusro",
|
||||
"1gY0qMz49Jh5zuAol9KClYRIC7kge9f/On61ZKBWDsEDGgNJaM4NNPb6DZVePja1LnobeWtuAbYZA9Qs",
|
||||
"hcFx7t7K63zv/4AUxMipCD6Mz0AyPSBFD0WoBMOLDbWEGBmRhU7CKRcOngdWE1FqNOtTDO0HA37mpHie",
|
||||
"jkGamzKvD4hyzeZAJpQluQRDUSEGlOfMZxaDrCKJWUDOHeMd3PLK8ou1e/EIRayzRz+9G0Oa3aiIflpD",
|
||||
"mmlSkVUUoZrEuSzkO1wJyUAyEZcru7Z/Puq6qhOtg7+9baYZTdhfQMYwERKcUdNeF/ZX9iJyA/jinuNF",
|
||||
"a6cbWpkSlC6u2LQwHTOQyBoqIhTVQTlqcMsPOTHntyAqYRGswSV2jWFC80QPbvkNSkAlmL695V++krdl",
|
||||
"48ol3PIvt73Lq8uT255p4bj1Wouji+PbXp+Yd99gMDAt4TNE5u/cCM/KIX+yKFrvj/44OT83fRz+mz6G",
|
||||
"PC7JgZ4RtVAa0p9UsXTLNVsxRr3kLo8EIY4rFTfGUnwS3FOmC2Ao8K98mdgrFGRG50BmOZ8OAtTMb/z2",
|
||||
"a3bbKYnM9uI8sUzqAco0J5Evh/NpR2IUtwaljaCmQyt/9Rh5aPaJxnE7BbHpFEuRtexl4SygrHMr/BTW",
|
||||
"52X5JzWjEgJWvyjLq18s6S4+gP608r36YMiAx+qTVfSGOPomAG52+Md2nitufun5gNHp1lpbvlp3+KSA",
|
||||
"ymjWsl9Nz7BFmeD5XD7s201rH9dt+2hJP5mHbsuzmbLYD10rL/qHQ0PlEb/6KA+/vT0qZZGEbRQ7vJ7X",
|
||||
"YDJh/K7lIaY0MjSleDyvtU8h/ZSwlGmf/KVELiNL0LWkjGvlE8HNEBKUEWaCu3fv3U9pyAzpGqgnQfsR",
|
||||
"SNRp2zm8R5sVuqgWpjmh9CfGlab2rduip2RzlsAU4rCzAZU6bOtqbQv0mNTWLII5N9zcO6kRtyXTi08i",
|
||||
"a0kx8B35SWTA/Vs1r9lPU0kj+JSVwqxPDhfZJ1U+un0vaUmnUKxvdxRTCxXph+Kpe317nskhkla8nze9",
|
||||
"bxvdNz5eBxfm/33X4V6qn2Imt1uCLKnbXcKpM9l1cUdwe8B+OA/iwNedF/RHnXGE38jrF1R5pQZeiRvh",
|
||||
"t3j77CS5rw7X+gAKwrduqUKNEA0aT2PJ5gHABG66fWLZ/L8ChjhznnV8rZllaeoFCae/aATsTTZ9dn14",
|
||||
"ESZ6wQ162e8jsHC/st/8StJcmTcUyTn7Tw7NFEAtT2DtwKdUw33NWlqhqfmYw6Mg4Ar/XQfDhFHVVtZm",
|
||||
"2fzXjaKNAc0NDVrv4gOSYo/vTxhJvjPE9HtWgvNPFHLganIwp0J60PfD4zKr4Ctyp7cZ8PkHKtsCWVja",
|
||||
"L0zYc8/t3AOava3BiRLGNVGisNKQiHLCplxIIK+9WsfHAJsU0vNCrg9J20GnQCMNv2M8Zny69RILs+wF",
|
||||
"zSp8+1okLPLTlYCsswHsrP3KUZEaTvp8zZ7wIbHpBaHs8tTTKi68Uzcwguw8s6N669NuuC/bZ8tFbSCb",
|
||||
"D/TedKuIcH53Zdd0kQgah0SgI3RQay0EhZ3UqQRuDQcTGiDAm/hrm32mbCqphhvxh1C6QOn6Jmda+93g",
|
||||
"zAe1fQnYv2jtW8qU6Vk+/hSJ9BP9K7+jVH6yPn/7Bi1AflIy+lT6ontFkcLHdmcwdiE7bpg1mW+52KFI",
|
||||
"4IGzmCE2zEDjlPGaY204sgDblt7BZ9ctORSdAtcXQac6mmthXXT938c0usuzZpt+h22X27Z9r9Dz6pj5",
|
||||
"MWQMmtr5A9iFzrMQHxkhYC2yoYEeu3LGS8hqGRGx3BKarOSijFw4UzW/bY9zmdxdVb0eKOVDcqrpmKqG",
|
||||
"vtPHrvVyQGezKmjDv6gsIho8lM62vcqAnx0fo81qY8P3CuQF5XQKhSrZ0xSJVJurOa72MIJbSlnSZoCT",
|
||||
"SgdU56aQ0Sm0GeKPeh9HKaM2I9zcXC97Mx7Jdw3QJWER8AC/SMR0yvj0HObgV4KlVN6BbrPNi2oPM4Lg",
|
||||
"U3H8zj+64EwL8xdGRT6UTNvISp/gWs5T9aBaB6w0i/kn/0qzOP8UkHvvz5YKW4/HFUL+EToCtdWxXlX6",
|
||||
"+iMuUqFhZBWVDblMtcvylhxnvjg9DOhUkeVaihp42mML5JOb1KkNAxRX16dnkLbCtptKh1XRo4r9NSz0",
|
||||
"iSH4XBtcOLe4dRkjoUrf0OkN8z2pz5cfC/O24VsE3YTNj+b1NmPO89h61t1TRcygRNPpFHz+gF4Qp4wP",
|
||||
"LktY9Bu6aZKIe0NFzkVEk7PrwyiqKzLqoXRHM5okwKdwLcWcxQGBNeYqKhoub2j3V+ZGUxmKi0dmuAmL",
|
||||
"qIbgcxPJTjze9C3YtxohsfZRpAwdaxYBIV2peyH9VnClEuQjAfwAvzytE9xw6Ns/Ifgk/hdL4ojKuHJe",
|
||||
"TYOxEJxGoPPMD0nHl6Ojx7z0NpAZJUAr3H/Vi9Qu4C3BZkX8svMjUWZHfo9+v9QXGrwm6xHn3OnTltkX",
|
||||
"46ERyEvaGFwxtiUovaPnsMYe/YdgJwQBboOf+M3N9ci3mcbYV5E5Qpu9loK4ZuQOFuT//f//P8TwiKWj",
|
||||
"cSUayt0i5fEtn9OE2Uj6OaOkNEYaeDl34znPLuAxERN76V636hZEYiXuopCgt8TYrTjLmov1ddmdanwf",
|
||||
"4oA61neMx/Y6lcfqgqCMathVn5FqKJrgQ4hyqZzGc72FGZVOSyeA7Zrf60qHSgT+qZAR1OZadUzwtSOS",
|
||||
"MgWKUG6DmAiz4QA4FYko5wKV5SmNgciiV7IgZtp9wZOF37W5WNXlyv79a6q2cqFPy0Vwwfe9M/fJONdE",
|
||||
"aZYkJAE6r/Uqe9zyvZwnoBSpzcKWoRQYW4Yk7s+cY3iVNzAsDCdHSa40yAqohBoKrpjSwK2SF3ieGmkt",
|
||||
"Kn429DWizpM5hgSmhgrgv9GXsSK+OUjv9z7vm2H251QalFRmvMo0pzn6PlZ+OSrGr/x2XJmq9rObtdzA",
|
||||
"canVaxQ5bt4G5ZnsyjrD525tIOuRxXW03Y5RVTxH7rt6nduHWAEBx24rd91giEqHlcwadaxxfs9qCeNL",
|
||||
"nDD4W0QN+uMwStvWSgyG9VpyoZEOlXglNsouklj3BManhodZVDOT9NFqc8tTujDkgrrAADeCJJQYZkwy",
|
||||
"qmd98zfMoTDnxINb7mZ3MRMuiKEIJEiziSJ7hdmOFl7Lc5rk4I250uEsCzhYq5u9qfYoYbFRT9OyNDS0",
|
||||
"mvRDrcsGFLiuc4+CoshMsjnFbFnLf0l0PjU4Xv5DqoTO0TcX/9uMvFTmHF6Xo1d+9f44HBWTVn70/TYc",
|
||||
"uTVV29nlldu+WbnGOgEo/OWaOW8LNThliZX01l23zc8FClhIzDMjDxvu7ziwYn/Bu4X2RWOM2F8oklt0",
|
||||
"Mg1rY/UN3xmbrhh1dzNjqoxpiQSfAyKBFgboXQ4GPnUe7mUUs0WEWx5XMdNMYTBuQE4Ntn2maZZA33xJ",
|
||||
"GM8/9zHC0TPTLTfCKCU/3f3UJz+lPxlc/Wn6E1ELrunnAXk3OjaoLvLpDFH1nsWGJy8x9p7p2S23+ss+",
|
||||
"yVG2IOgIOZ1p3KtdL274GmQEXNMp2NCXGvL7YkvCiHDj0LLAAMMEegXu9Rzi9/o9nrHM/O2IfEOQN6O/",
|
||||
"syOaf34oRjV/3LiRzb8vaQrxtZ3B/H1UzFKu88MqMfDZ95qpgWos+bH8JsSRyAI2F5WPDf1uaIUuLNzO",
|
||||
"Ir6+1T+E0meZP4b37LpQJiHbOLsuwlW9bzahdJHzaH0o86U2GMbHObeIfpudXNBs0+E20nqunorPhy00",
|
||||
"/wg2uGliuOZ63ypxe/ulNxEypbr3tpczrn954/WjyKRwec5+lyLP2mYLeqrsO03tu2b5x5AlYlGYV1a9",
|
||||
"5dLCHb+BeX+DE0XFkl2JjfmvX7eG1UjA3HaWVTHOUkOxfvYH0pUa5RW+Yj8QzDNGhkfnV5cnLkEmilzK",
|
||||
"OtHE5TkQFyNCBB+QoxlEdxDfci5i2MdYUsYx+irXcJhlyaLQJhVODrhxdwpE5lyhMoMSvC0+veVu7kyK",
|
||||
"OI9AkZGmOlenlCWoCZ4k4t6sSSLjQM3FZ6aQm00oS/YxqPbulv8np5JyzTigqDggl0ITSrKERmgtIxOW",
|
||||
"aJBviy07rQlTxNHzfbXgkdnanFFS03j1iRJG7JQLAgmbYloNs38yo447G1HV7UOBHtzy//HFXcDg8vDi",
|
||||
"5Kt9JCL02Hh4I4ErkeDDU1TenmbphtVSc5SDljHZkmqYLnxXbr8QBQlEWjmt+nVxNOV33EsRamU5sA1I",
|
||||
"NPL5EiIG5ATFaEMa8Xox3vS2J0XO430pxozf9rDnmEZ3+2bjVDv2/s8PBKzhmdxLpjXwZUxjmSwAb9iy",
|
||||
"8VIsXY7d6/cSoErvj3O1WOfClbQzK6F0dKocFCzvcQkf5kbVgBxWQR/DAk2///Xv2940y297H285Cj/4",
|
||||
"YkoMgApuu5L7mbneeuo/dJwb4MwuzMscFI40uOUXeaJZlgAxC0Yx5vDy+Kf4LSlm69/2+DyF295H4iiY",
|
||||
"ImNhoNvegA285OKWu30hbvGFBU+myn22AaUVcllQvX4RjFcSoRANtf4Hz40FrDuQOQw9Rc52zNSdX50f",
|
||||
"M+VPmFiwxHamjtX0T2b0cqxKz48blnwBchr29BpLyqNZa9eOmbjnAW1vzvV1MLpNZFoFr5LygKfCyjGU",
|
||||
"a65Nt/EMTLNwAs9dd8MBQSGgcfULse22WlmAG9G3zXpgcFUAY1z/16/+LBSLDNTgd0mzmZXtj71W2eLX",
|
||||
"XdPKiPv9BOaQEHcVZV6UvuHicxZbhnAHiwN8sJGMMll5pzJu98IEx7wwuNT94lHaR7a/FDpcND7OtG/b",
|
||||
"3nLGDXuyTdWsSKdTZpCozEATkuXSUDCbtw2XMXTX8RZVuL4UE5deI0U140+xebukQXjQTQ8Ee2NnKZ3C",
|
||||
"GTcn4AHlM19Km+PiYeIyH+87x3uMlz87Nouk3BrNKwdfphnHNGjevjGbgtIkokmUJ8jbJ1KkZqpbjsP9",
|
||||
"pJzs5GCT7FlxwuYyceo/O4qy2YcWIJW9m/HilpfG/Fe4sEsjNLl8AwYY7PQxm0xMr2Ju8n+HkIljO+r/",
|
||||
"JWNIxH3fyjG3fCaMQFWZ0voKpJSzCf6Ew0uYgEQLWLkCr93rA5M6p8mI/eUBgMrHMiRfaJrUFDR2+mVu",
|
||||
"F5okxSlgdpdb7nhqTMRkS9IkkRohKSbO6Inpkqz14g4yXQpZ91TGxApabMwSphc2p5Jd6FrypArNoDKa",
|
||||
"MQ2RzqXPKlf5Wj6FqYzvjbBydP2eVLsv00bYE0BpX3DvKdNcz4QvRx7+XuYgqeCb7eGc+mmZig9im5wt",
|
||||
"Eqk5KpcEwcEqKoxrbYuHxMXh2eXN4dnlyZDsLRNTlWlilhmcvKs3k4Evc9yR/YDoxV3SC5qQ1GDX1B1Q",
|
||||
"RDnm/gK9tnJhSFuaiWWauTCcNnOOW8ueWE04sDHPsr0BS0pWUpIUV4A4rUVhm1+u18H0UqmICdpqXjy2",
|
||||
"T9yvwrszAaCSvj1e/NYMLwJn6Vzxl+4aLfPWBTPWHdXHXgUNuyM3TiW5YIXA2gswVGPtBtAXygFQiQmo",
|
||||
"YK7BD2pMJcyZyBWZg0Tx2oxr4ZyMc5bE9skXVw/apr+SYBsQNDD2bznm0SzzVDFObFwOQQUwX6RCwtNd",
|
||||
"X+k34ssy6Z6jbdzJSkC0IoNd1S0fnh6RX3755b8xf9K+S6BE9lAsG54emU+XlItXQbEGX4h0TlmCfJVN",
|
||||
"SCZBIXGo5Ipzp1mch9AzkPdM+VHe6lQ+2PvzJbysfC7OwN125bLX0Nde7uNir2/506Vg2iJBjlemXUez",
|
||||
"SgtiBYLqfdCxyLVHbCvPwIJ5Hc1+Ugg8hdRjuIE1oAxswNfS67HZVla8JT0WoyK/4IogYT1+REQTi4JB",
|
||||
"mLNtnEYpppAKbmVkg6cZlXplZK0gmQysttfjjzQqgOiqtCCNrAWpMoZC+CkzpQb4vVBBsL1SYZBtPvkt",
|
||||
"t7OTPUBuT5NkgXTEpe59NfA7F0kvH7/G39eZYOZ+fyw8sV5bBbUwIgE+X1K0ay2zTOImD9xg9paTxYDc",
|
||||
"lIPiGHZENllZQ5YnSSHGF28CImHKlJZ+RJVLWdvneVN+tK+IxCX0C78m7DvArnpJEp2Mtiqn1263FNrt",
|
||||
"+gekVJkte0WU33JsWEA+6mOrbx9Qy6UYyTVXOa5lnUbXjw6YocfuBG+5PcLy7FC69By3mjnTa6WleyMp",
|
||||
"ZNH2EVTswPw6BW6A3Dyckbeq9XdYO22wucIbrxp06L7UL8+uHw0AB6iNrFEd+xVpD16SO2/3mjJ0qDzs",
|
||||
"8oJsHxzLSLyBC7LkCcfApVjkuOUlLHNhx/A839AGbg82ogpqkHPLb3s5t17it73VlsxK4dYBbAxVMBsv",
|
||||
"8PjPjluetxD6dNSWrQ1tr3VOYD8sdcZ+XlZwJtu6mk4UGRm+OcnZsc37qLwP2oe9ZEn1IbvUYm7SUWFi",
|
||||
"ax/J/WA/FItZe1y6jmTPiVHJghwOL9B16dWgjbZlWF7VarCC2VjLrCHOd6DB5LkqzWEbE+6FnZJ3LVBh",
|
||||
"oDOBQGaTmmV0U7UKXP/WogPh1W9Yg3+uMz5nW87KwNuN/woerZ7H9kM5F1PGNydR3PnmqoaESs+f3/zP",
|
||||
"Wtf/2XgXWywMuKFr12IIChpe93cptWJn7ze4oiEyYpDf95b+2ytr2ZU5b/RK799+rk/88y53XJ8geD6j",
|
||||
"PBYbz+ax4HA78L1XIN+cHqKnQdC8owOhdCuz2WbbpqqC+WaoeAilKKLlh6AywZUvsY19TdoEV9rmYnH+",
|
||||
"d+aZ9M9mTM4blh+IQYgP9XaDT6gKIahA78evQSjueSCE5xnU6+pbMPuDKr9tEL+O8om/stuGK4xjVBGe",
|
||||
"omlfhW4yeGg6xBjRKNduKUXt2ZWQyUjvcPA42GGkAydfS1PXcLSjsk+ZKS0QxYwRqZJNpyAhbpljZDXf",
|
||||
"XDW+dw6SBcIe9UwKrYMr0pJGd4xPL8AA3APSk6yf7jrZ3ODe3kKS9R382lxFzZPQWUbBFAar8LlZVFxP",
|
||||
"lbHuEIFfHxhBP2J8msBqTo71/BE2iL5puJivavGa1+H21NH+r9J8Hm9IJxgAyQfmAwpn/Fjb3CmTcE8T",
|
||||
"W/oUYl8mgK1c4/LwZjSsOEZ60mNXl7K5cN63q/tKH6Pu67hZ3dd4Y2aHmKvJlsKwgVJ2/V4sLoUeLXh0",
|
||||
"KeJQypmNsMYySXkg55uYg5QshksY5wktqtltzvKkZ0wd18obrjfJOXfu7o+a0/drM1wo4WING+4CEam7",
|
||||
"cPIqpt6FRH9fDptAktBlNsZ1GBhv+LgxY8HG2N4szzLYkGrJiPDBkedLa0Njjna8kjpnteQshjSa07oO",
|
||||
"uYGpO5Zdy5wDevQE0KFsVMnP2ZxVnNSz83hyWJxhtRYJN+ejHfBxIkUavEi/wCQ0myxQCRLymtt0yyHK",
|
||||
"kqvwDjZcffjkKhmCnj5lQxRBpg+TpLgMD9epZg1rkeEB246whsEJnzIeTkQ2c9F0oUB3bILFZEMtIhqF",
|
||||
"Augjmm0e3fGcSigePunDGVWiSs6Gosu/KNOBErb1Ds1SsAyt071UIX/eCDC/QyXufwjcSCj+VRdmmwsb",
|
||||
"zHCY69lKxoFK4625WwpUqMaEbNEAubQTUYO0E9hwpykKNlxkRbCFNQM+v4ZLtslFdF3pgER/nDA1uzi+",
|
||||
"HIWSLSWTm/NRMLGD+x6CyuVng5HKhbS3S7Fvh/hAExa/5zqUskYlsCkzRbQhM0VQDsBPOy9cJ2q+edFa",
|
||||
"5kpDbO6kdWrAXMGpkPdUxhCfChkk3M1za9QyYi7Lc5aoUoXpjxvJ/omUQg5B5Yk3rCuUJsl67QUyEhUa",
|
||||
"uub8Zy0v3vordYpxLiGdG+VxiG3fs3gK+sHFlu1KDRfCeGpPPrQWkRxSaBGJJFxWY3fBW7kaoQ8sPvFI",
|
||||
"2bo9yf3WjsnO1fbRYce0OQ7a3GCtX/Mry2XSasu1HHzrFR5Ccr0ROUKVWOLVNJi1qiiySQJfFm8gBxW1",
|
||||
"7zLm++f+m/7Pr/s//9x/87r/5uf+L6/7v/zc//V1/+fX5n8/f1yzZJex3s4V2FbmPxlefDo8vji77L3t",
|
||||
"OWclkam3xCXp7hM1y3Us7nmf5Bgk1SfarIfTpNevdP80PDk8RndzuCeJmKri69HV5enZ7++HhzdnVzgJ",
|
||||
"6IOM6mjmXEb7xDBOX+PakFGR38+2G54cn1zenB2ej+qtJGDAAE0U2QM+J3MqVZ8U4rzqE3sj6lUx0vnV",
|
||||
"77j3Q74wy55CvG8jNWSfXJwekvKWqu0//evk8J+BTlxgP1tCxHUanoyu3g+PTkZYjpijZ0qhBzNrsuSq",
|
||||
"X1Tr7xOn/usTTB67OsrqwTQdqE9S1NiWd/N+dDIclU6oplsC5r8QM1tVVfXJGzwEBbrWqbYELNieMKUP",
|
||||
"YtCUJZg6xAFbxS6DyQWq8NEe3gLT9Rvv4CFHttPN1YE3BPztgbcBvG6Fzo/+jCcrOF3Fcg8M1EDJD6lr",
|
||||
"WBDGdi/RCCF9DYXX8fPjkoLWRfw6zX+icllhFrTWdz3ReSkf3CBTbynXnr8LPqnO3400i+4WG9KBox6g",
|
||||
"VAKYt+XNqmRRy01eM/m1Nax5DYa+HOZxyjZlSM/17GSTjgi1zofXZ4fjXMGGJkciTQV/J3QoWaWQSkhW",
|
||||
"Vxw1t0IXqbaBxiD/MA+Pram2z2FKo4W5B9tLbWwd2D5WWKz0f0A6VxbDqRTpMVWzsaAyDmsO/Jb2KHA0",
|
||||
"Kf38jvL4ntUUlRX1SbgwYNAGPwd5L5lGVZbdfUCnq2fXEvzW7kL6s4JjUENfbTR0IuH74ZkKVi5rbKpX",
|
||||
"dyx7Pzw/wr/9LVIq9WjGIIkbGgaXHVwVFjNKJsRks9LPrDq79p1W1Tq8wWLr7MrXIC8YzzUESqEtC5u1",
|
||||
"KERcWEk2GFAg/iMEl7mCP94cBVUAK/3qH7edyH1ZuoGPzWO5ZQUHf4Fhd8zhZ0Ojp/CyRG8Lqr3psd2C",
|
||||
"D1YqZDzHZAtBG/q6DSPXYqRFVsbNqXAliJB12307FUHk2c0kJSETimkRSPMuQRuRU/AN1ZiCOgc/YFZm",
|
||||
"DAPnOgVad9TcxM9T+vn3RIxpMmJpnmjKQeQBU3oG8r0C+W6hYbXc1Xor51O3vWHjeW9YCu/yuH6ry0Jl",
|
||||
"Ge5+hOyA1y0otTRVRTlYzFGi2rmajDjNhodnx0FzqXkNXPHTEP2K22ZfWFvHLr4bGOrkPJRaqLojmY+P",
|
||||
"NiCaWvAo/H3DIW4hpO6M21KllbtpQZlu6jUUVpxfJUupXByJJODOZMNDNzQJT/xeeUvTb3TPDBsWEqr0",
|
||||
"qiW2/rXw7T6aUT4NzpAwftc+/f+KL/Om7yefs41NgitrG8rgenz0JQcuyiyPDBDZg38HVII8zK3cbM6x",
|
||||
"N7OyboFaLkUB+8tKSksQy5jZ99evWH5mIpzrhKaozVzLojI8Gd1g7PNESGLLoZGjROQxscVCSLos9GOE",
|
||||
"OI1OwK7hyDY5vD7rVVwdeq8Hb94MfvmtqKxCM9Z72/tl8Hrw2mWawS0efEKN0dsvPUdGV+PDdC65S2xm",
|
||||
"50nEFKNfCVU2pRrjRMNnTWwxN7dGl2ZU8LO497b3OxhAVMjCrMMzTvnm9eviYAr/P/isD3BMxHzE51q+",
|
||||
"mzHjVC48zs9rZ3r1T7P1X1//3HCKZlWFqsYjz6TvOXUAAbGd/pdvOf2pkGMWx4DE/7fGp/soc5+5ksMF",
|
||||
"PGLzGmZhHFwVp/798evHfk/lqSGoGLN+zxNB4wqoYaZODF38d8/GW/c+mjEPaMYOPqWQOvFrI/TaZiTH",
|
||||
"tBtOs+lSA7k1YzSjjfAe55htxgfDhxm7sDNuBWSaoQ+i6X3wpxKtD7waG9ACuB867aOA+DdYRA3Qf339",
|
||||
"27dfwQXomYgxseRhkoh7cxhtYP130EvwcwDq7MgbIJ6W1b8cwPtg1JUIszwQlH4n4mDu+GUTBmvlwR4E",
|
||||
"4y3yy4bh+/W3v9h3NCaV3KzPAbz/+9uvwEi/CVsRjbYC9TkGkMsiB9CymhyhBUwWoH3hjD8f8aWmPMB8",
|
||||
"LVQHzR00vzBorlPqgy/mnfDViiYJWLVsHcyP8fcS0C+LmE1JU7B2n3+v5UUzLYl70eCzBDNHlo+SUl1U",
|
||||
"PIEsnC4PdVV4/vgw7HrTodfzQa9fv/0KjBB0ir5qbUWgpSuCkM7DgNA1TCPjBSl18+vsY6so1OFUh1Md",
|
||||
"TrXCqSz3SWR5h1MdTnU4tRNOlYJhxvYxC4CqS4WrqqtUzG1BxbK+aWX4kAiZsRs79BqiPMpJW9c6mrGB",
|
||||
"m9IllihBa/3M8bvN3auFO7o1nP/6XPRpzwE1O53eC6MPx44erJQidhSggu8VYdWvqKZJUqb7hXg5nCJ7",
|
||||
"90zPilRvNNI5pvY0qDWjagbouOoVgGsUoVNZPwvwbq2EMHCxBIYwcBWKtJVsuC4wURFKONxXGAqWtEHb",
|
||||
"XpkTveIFhOYQWbEASnrvOu6pmbjnNmWm4BGsg1+hvftmHMn6qTfmSEXlRbL0dO94UseTnrHq0mfR/QYL",
|
||||
"eAS77pEr11+nPiEyti4oV7So7l26Aioom9vq40U9uHKanxSptO5XCVyfnF2T0oUUZftadhhSeBNb3w7v",
|
||||
"Y7igb03ew0tZ+Mnew49MVO3JNiCqp7aQnRbuodSR046c/kAivsWDRiJ+Qb4qecb8YpmlikYos02LKpSY",
|
||||
"9JszzWjC/nIllJaev5gGGyCGeECGRRnATELK8pQkLMKs2DbLP99fFglJRYxlJLxC2ju31KehJuGMaT5M",
|
||||
"qZxEEX/w2KSkU+I9mYfZC5RHxn6QK3C7wI01xN5f1lza/KYnKc2wBFuSeCerFDZzfm9FBOsdLGwSetfN",
|
||||
"yAeDwEPfrfOoiId9Hsz2Bb6zvVfUAiAqwuoWuFBIEr0TBkCi0eU3kUTfLQHqMUXRDuRenGVjCxyuWzTC",
|
||||
"8F8PiNpglXdDDJftnwNHf7m0qjx4rKTtdHpMkkREd+tOpU0u0BGxA8VppmZiq8vp2o0aIjQqOzenRmSv",
|
||||
"KMLlaqJXRN9Xz4RU/QDQ9v3J1MuVHkvsK5EDqx6uIeOC7FVw1GDfq12xUNMdMRA7dtjXYd8Pg31GZKmg",
|
||||
"mEENpjSLFNlT7C/o29i0SORc9wnoaNAI6Q4gtlHJG13RXfcT0/b760qsTioOPWA7nckz1Jm0gXQDZjWT",
|
||||
"xq6aijZO6G6Ql/mG7FjD35o1WJ+5VX168ZItammqyqukKeocSMCawE05hEGfoevyfbBoN9a0FUncpoib",
|
||||
"wD42ixM9i/vE5s6xhTzLsua2ZiQQG3XesabOJ/eHozwWLbAquCuuXFChAjvakJuWapddlS0d2+6Q55no",
|
||||
"U5a6lImQhBYOiZHDojbIk/NE2OJFTVn1e9ujQ5wOcV4S4liw9WsepZWFlaYJ1m2/a6Z8LHDoS4GPXw8m",
|
||||
"Iilym7bjRKeu4zdHq/5aXXO3InJ27J+iwqMfMM0xpiY1x29GR9nYFc6vjI+z/ycHNHq56d1iOkLREYqn",
|
||||
"47AOj2u2il1k0xppkKWLQ3O90pI+YO9nRR6IFmRSrOuxCEWHvR32PgB7LZ6QPYtjr6rCcQGOK09OWXWs",
|
||||
"2AWxVT625GLfKb72FfsL2osAo2Ic90AemVH+NvJAuftOHugoyvOzoVqVlUFstKKWL+8Sah3A0mZiQkT5",
|
||||
"vgIe75cJaTe6P97PQM9AVn0cZ1QR7EzMOIxPSUQzOmYJM/BZCYwNeEEeUT4CHmNR1GfjAfuMUhMu/Z9n",
|
||||
"EN0RNlk5baZIkU16QwrChm7P5l4neZLUHVgLY8yeAq6YZnMgExs5lFJ1BzHCIEYulBV7bK2hUHzz4/g7",
|
||||
"0yS5miDatLz3/lphYJfQe9M4U6Zn+fhTJNJP9K/8jlL5KcJUvvv2rD4pGX0qa0Vb6/8aXf743EOsX2hO",
|
||||
"Th+4VtDBQf9HW9skmq0jwDXIiZCpIlj9iJRAq8gejeO+Ec4SGlUKaqViDp/yzP0jFvf8FRGcYPFU4koo",
|
||||
"rQXumLnr4P/0YYBYWGlDDCB+d/F/y313kYBdJOAP5DKPmOdwuwmRyL08EkmAZZKGy0kIsEnHFSvc0LFL",
|
||||
"KoFkErBXbJ9/pZ+QHeNVIGL5SUnGLqxtHTbyShoUexCO6z1bUtKh8XPi+S/YpGOZZ1MhpC6TH8Q2p1hI",
|
||||
"MDE/gyLHl6N9CQn6jhalk52LjqEemZC6TyY0ScY0uusTLN6XMKWV9eqJcqUNteFasq2SyfG3SEV2ObKT",
|
||||
"bRBNji9HqwF2XcKCTkz5G9QZcBRlDQG2kJNlEpSDcSN3ilq5xXfOn+KJEH850eAYqwPjfA5g/zG6uvSS",
|
||||
"AGxpydlBzvG/JKOLRND4+SJ/F+K/GyNFiCBCkuKmKalBqCsrXceCSr3QEDKsab8CSqllly4i/4cAqN/B",
|
||||
"5l6McimBa3IJ4zyhm2hqM2jiEOl2xPXIdWpKXj/vL2iaeB29w9Wz3Pb+9+HF+RNHFnWQ/b2zoVh4InrG",
|
||||
"FOEixkS91bijOuH8cH1JOOh7Ie/awzsmXmkJ7rbPEwY11PdXJoZ0sgHZs6zikqbQJ0yds+lMz0SuoE9m",
|
||||
"rqZzn7BsSPkU82B2z4guYdFTJix6PHSMIUvEIjXb2p6uJknIsj2JYYI5ymrpi2qj71dGJ//8QPaupSAT",
|
||||
"oDqXsMGatxzguLK6Ts/3spMXR0muNEgS1+60gNbqr+E0xst8eQYLvLBInH/NAwDRx4JWIfHxH7WZFIPl",
|
||||
"LP7nq2e/Hbd59kqrF1PtrMZj1vE1iK5bWcvBDGhiSyhv5DAxKAPIZK4GRYZ7CXhYBM1lCPtZbVEkS3J1",
|
||||
"y80JxXkCkvwnp5Jyzbj1aYI6sg/IEGi8L3iyeEsUnaCkG9EkueXWgZIvUADuEy40+TNX9sm3HD7B6s+D",
|
||||
"ltzrD7v/joeVaPHb9+BhRvxiEZD3nM4pS+g4gZYY4pDCwrN1l6tJReoBSOLL0RAoCVNFgMeQvUon/RAA",
|
||||
"N8kGUeFPP2JWwWfBS15eSZQ2jKTfMF+mX/J7yPOig+8OvndJnLkO3J58mWuPm6YVDHaB89LF5znC+fd4",
|
||||
"NhVJsmr0p3szPe8308urC/C4D6YykcFYijsIVxDYLBGWz5afVO1dBJooYVrccg6fNZEQCR5h1rxFlACh",
|
||||
"WkOaafSNwYcXYZrQKWUrxGfQVndiUy3YPXXctnPQb+/McpQAlTVwr8F2a5Qz70HVxph/7Lp0Ns+XbVBB",
|
||||
"xbTPIcRGwVUN/bkCSfbcW59NCI1Txl9tsLX0Wxg3q/D0bVylrJqvma/Uqim0s/3/jQyLFh/Q5XfqCkhi",
|
||||
"/sQIpGYTszVQO1gcEd4PvrD460FmuHJ74nsWX5uOWyUIXL6REQ5CYd4sfpniQ2fK76K/d8f5a6xX7sd2",
|
||||
"LUiEwcESaDSzsdeL9ljO2wlVvBOoHmJbtwEZ9fs8vhxVQjQeRVThT1sgrj7Z5eiEa7kIxXKYrS0QWqve",
|
||||
"YZ1O55mS6pdpBPdj1E7k8ODLHSyaZWBfRbp/wmKrrFNixB029og69kNXzuuHjk0urW4ByCXjAkLCDKEF",
|
||||
"0+4gs4PMXQrNPQA6nfFsu9Hru0LndxeRCqtXlWd1AlL3ln3UStiB+JDd5SSImd53cXmtQkROYqaPi3i+",
|
||||
"b6VINZM2U6OafXUq1B9VhYplmqoGA69ipZIlrC1aTEHvc/is91nWRqvyO+hL+KzPsk638oOEnqK1vvRd",
|
||||
"JWfXhMaxBKWKsIc65B2dHQ+JpHzaPso5EdNWGrxz074Ds5cPZi7s98P1JeagMZQrsXfbDoA41WrfVR4O",
|
||||
"+f0fQ5RQieVGM5GI6YLsSZFAn/xxSFIRw6s+SdgcyOXhzchlxDmIEgZcH/wD9EhLoOnBPz9YR/8+cbHU",
|
||||
"GMIpcq4Vmq7uuLjnJKWcTkGqRp77l1SrUVE0uQPptioITVkCMd7a8p5cFWo0rscwzqdTa8JrCVUihl28",
|
||||
"ozCouYlflIEYCQq0IkyrWz7B9KhmB3fOZ4p4XaaYIhMJsPSYuhfyjghOmL7lO7tOXZrttnCaMu2LaOVX",
|
||||
"nd9U5ze15jdlEOEhHlNtLfbOUN8R0ZbXVeRnNgQH2Z/le1UexxShhj22JqJZPk5YtL+D89s19nxuLnDP",
|
||||
"IpztBXvBWXggjE+ETF2p1LHIrW2ZC76P+ZMg9rvLtQY/ZK9toG6IHToi0paI4LlZCrIU6h+WrkiC0lS2",
|
||||
"vT3s0t3fDvdnTm71BotnmRFVzeTuZdT6LnXOOSSt6P+N69Ld5c6+OjTCmgd1UqrLc916hdLuatONyTIt",
|
||||
"REUL7tv8sgmD0sI0vLq0Ceve9Lq6zJ3/TFv/GQR0IZ1bFjHgRP4U4zrfqUI6gut2R7QOqDugfnFAXaXa",
|
||||
"3vwOIQ8wKXiTUN1yIU8aqNseyzok6xwLdnIR6rtc7H2CauLCmc2Lb55I+5KXbBGPOtzqcKvDrR1xa5MH",
|
||||
"XodbHW51uLUrbhXiYkzVbCyojLdnzCtaunpUVR3uPdMzM7OmjIOs2l9HJ8MPh9fXodJylk8el4t4dqUV",
|
||||
"mYZUtSgPU+wEy9P1lpUUqZR08SJLK3aFUHYs7rhEGASmDSVQYqbutilHj7FNV4r5hRvAMIeju8oCHkZa",
|
||||
"SDqFVYA4sPQ17HNyit9tYT9XpR1iHN0SZPP7lM2BkwlLwNbWJXbQAbJgJkGRjCp1L2RsOYej6ANiXWkU",
|
||||
"yaSYotufyHWW66AzCcKnXdET+UUre0wDO4mZL+QKbVvYk3Ar2cEVWsNnfZAllPGW5Szs0UFMzAju3Igr",
|
||||
"e4wX8++r65Ph4c3Z1SUZvT86Ojk5Pjn+6BfIQov4Ngm5nnT27azmSaffQlOedO5HoCYOxqmFciFJRqVm",
|
||||
"K5ZWD2XhYR/JspgwxbiKRIi7PCtT1FhK8n547hzXlpW3JSiRzCEmZ9fzXws34ZCoh7rLjS82M4UWxajF",
|
||||
"k+0/OWCUh3uz5TL5pl5ljy9Zrq6wlWTY1b4MoE/Vom3AByEZkycb0N1Uaj7mah/TUQTx4wNIw2IN0FON",
|
||||
"AzNFIiElRDpZlC8vV1i+hjNhbED3q4YoYbNluB09E8To3C27mrJr3oSeCqiNUFBEdyD3ndfJ1rC8Y2zu",
|
||||
"Mq0/kdBplzSoTVXLarYp+fsjpDR7uBBaLsIKos9BzOzkvB2ShFgQLF2y9nBdCvYTdgev7LvClWZSfTIX",
|
||||
"SZ5CUUS5UMzVSgbgcBXkg/lKaSYfwzqZu8oDG9nVCH3JMHSXcTI8Pfrll1/+2708yd6b16//a//1z/uv",
|
||||
"39z8/Nvb17++ff3b/3kV4GYTKdLeJvbVX537hMePMrMW7ea9YJyleUoSMSUJzCEhexh70kdFaZ+oPIpA",
|
||||
"qT65p5Izbn5PMyE15bpPwMBFaCWJmJ6bAWvriWFCDdC97Znhe/3tCzzNk2QfX6QKqIxmRINMA1PaFu0O",
|
||||
"4N3o6pLgOIQqYp7mxDX2z1H82WKKazpl3BXOy6USkuxdobB8dkxm8NnNFzrHjGJw4hO+ETrt2nfWrlka",
|
||||
"ZonhhCUaJKaf4zHJStCp0MAL0JJF1QiUCZ2zqOb2uHJUYMvaI1ml+E8j57tu63J/3z2UzTKYNohBOWGp",
|
||||
"hUQfgT11C9j+INgHHokYYqKYBuJeCBOzvupyQsjX6o2whoiFgEOzDKMnySSh0yAtkXOaZY+IeSLSoPcx",
|
||||
"Ui6tA2OhMe2NGac2KcQ2OanJ8yM84Xd4hjzxYnZ5jjzxkh4p3LiKpFufIhMhI9i37u771uAaVsTfSDad",
|
||||
"glSEkpTynCbupY4zlSH0djBnvVUhPfqpmdhu1GY/6WKfOmvkA62RCFMVmFyDxCAaTEHvj2l0l2db/QMM",
|
||||
"37NNSSRUKtS+e6FYWXDCkhDP+x30OzvHA0H9ich/B9yd/qsNo3FY4CA+aPWfSpFnzWohu6y7UiRAbDeb",
|
||||
"voKcHZPb/PXrX+B/kV9Chch+t/N0bORlVzWeFtdYAJT7oWkN4zUYcjojmtr4vgxkypTCItuNqhZX4Opp",
|
||||
"im3Z9eM0G5St+H2p5YwxJ4jqchA++8JbXVDQbpmiEXl9dKDOWLDsxaYSrzYEyJCHNdIwIKcGiQibYEUa",
|
||||
"RagEojRLEkKVYlOOiZQI04Om1V4tsTiLtyk2LDY/rIYG4xqmILsiGh3SP8NKHi+I7JQpvoMkp0WFUezz",
|
||||
"k7ICh5Bt5I1cf18K8jTijT2kbeLNKYMkxvKI+VId1Ak2HY3raNyj0LiyqOpWsQrNNpXQ6i0FxDMJcyZy",
|
||||
"lSxIniWCxhBby08lKMf3XD8zbZpEtmFD1DJYmrrHeJTk6GUMnzVwQ1pffZekfE9ioPmOBohWZKYzFv0I",
|
||||
"xiLKV7B2k5YcG2xznTmzjbal1UTpyCbjcx4/lmiY5xZXGUQ6YGdlFbrxPNw/zVxqgPs+c0vvHB8eHVrd",
|
||||
"0S49xFaZTNDjy4LtQZYnSSPYvTYNHwC/me3/LYH34e6TZtE118nOebFF9UdzeCuAiamRKZEwZapeuGMT",
|
||||
"gO6zyT4+F7fmhF3C6tnkwvXogLYD2gcBreDJgrAJYZowRbjQhCYSaLwwYr4CrkkiIpokm8GZz9kmN5Lf",
|
||||
"XeXjwmLjUMTaNe5ggWYakVkXw2RBFPDYOpThwDYIzAVzYuZTAillSch+c2aX80Q1pBTIgZ1hS92aSxbd",
|
||||
"lUYo8z4gCOfPVs3RqRieiYqh81V4IKEbGsoVF9QjYfyuQr3Q7lMhXn+K8bYXzj9Mk86HeifrflEnIiZ/",
|
||||
"2lMM5GsxXw+WFr2N8VHmPqwKf5d0lpEUfPAPMa6R7+52Xy66l9acP8V4WaAEEweFMnX7gM8RgK2Q9zvo",
|
||||
"Dux+KJb7kn3ynFsQCpmJmCrz3KNF1pTIoMQWsJc5bwT2w5x3YN9R294FBgIkC6JtcICju1oQmfPtsMa3",
|
||||
"KzoctPFHKHvzd5S6XAXNZEHccTeRvJQWWSMyMDINOzrQ0QEDCIRWYSwMYglTej9mcgviG/A9ZnJ7FDYm",
|
||||
"/3HGYRcw/NYqqkIxsi7HVruw3GMmIdJCLkhG9awy1UE4FFfPulDcTuJ70mBgLJNsV+7EvbgA1I0pshAL",
|
||||
"UxcavB0TiyDiDmJ/gNSMy2hNCwDkDhb20aBnwCRJ6LhePGc9hDwRU8bDSn6zAODanAkq+nMF0kViVBXh",
|
||||
"hSK/T7BA6D/+dUMiIe4YEMGLXBIhxf45ruAJ9fo4wRa1PrYhkYTY7JY+54iMLvPUi8jo9t5gSuJgu8A/",
|
||||
"c4x15BO5DnrnnePLP6+X0EcMHC9IlAC1+SKWKGoLPBu8CyU0PLczPhdYfkbRyWsXZ88pcHMplXegm4VD",
|
||||
"2ra2+LESuYygSqMrW1Z9kiW5y5A7E/cRVc4BoQhRTkQehwp1X7gldZz9Zev+zO1jppIK2CQ2KqHGyvFz",
|
||||
"FSShUdB7JgU64VYT4jtnkqWOoUJTIEaSEwI6eH5J8LePY1D8RWa379juIyBYjZMW+cgCZB7kdLvl8gJb",
|
||||
"PW0acZxjQ+ANfn9A+vCuekvHmDbZBxC80jzRLEugkqFfkVwZKRQbyNPRRl1FMzXFUkWxuVSSSFO6r8A0",
|
||||
"MlwqwVKEk9pTGBNza8kgmJk7LSfr1HuPZK/AgITpVMIU78Wd8GpZkXVVRDqhmwLAh5CKuctg9+b0EF0M",
|
||||
"bXKsZTUJJOl7NE4ZJyVobYjzvpjQp9Q8GDnjzekhlhvfQLtLn0Inh+EubE550LhXmxqvU0g8XxfDl+xV",
|
||||
"V8KYU/Cty0N9/7ti1QX45urmusTLNaXFaiGGTIo5U0ygpSmcdb7A0Q7UX7jkbWEFAcXRbx+g+fXQH2jC",
|
||||
"XJC+hTIt7oD74UwLYvaK7mMosexPaKQNdNc0ZSF99LfhCZh5fgNPWG6yI/w/IDZgcY4F4kFxyaEnKJsa",
|
||||
"tNmfObzY/BK1jf8QSj9thQG3qhthproQsRdOzFEi1Tf/GJFMSL3mu9llh+jeqo/9VrWwSY5QaV4p9KkF",
|
||||
"MVhUFB7AXNCbIqBSkfMGOIetnlj7Y+bYpP0x3zvtT4dRT4RRCF60ovbZrOYxzbdqeWyjTrXyA/ih4IVD",
|
||||
"XFULbgQQR4APvrh/nB2vpP4LqGsubfPLotfWSGlH6EPptXhloBdYg6yD2weGFrmg6UIeGC8spAQFgqJi",
|
||||
"0RbSdlk06wDlByFwdUDxFqbqb5YSazDxZM8yzCsMbq5rV8bNp3F2EP8IJd+6B1mHRm3yAddRaTO1vS/C",
|
||||
"C7ag1n3L+ILHDCe4pqh5jGtRBbsGEqyXQRNJbE6LppjexeJ3qPoctv22dYa7wIW/c+BCDbFL8Nsk99+f",
|
||||
"caWpzeDl1/T/QXmcgHKae6UhKwrAMNvVMqxl9siYajqmCogCnWd9p2ys6RltYUlrFkYTQcH3QjaAy+VC",
|
||||
"n4Zfp5TxwXKWkFv6WXXLeBbOiv5MuXRXEub5O6ZfSxGBUoizbBW+NpZX4kKzSeGrvO0NVGu7hTc3q5SJ",
|
||||
"1j7HD6uj71yNteN+L9yTswYGpTHY67xcAHUdLgOwfSCBxm0AfAg03tVfrToz8YC9CuXzi9XLk/W6Z1wn",
|
||||
"bz4kS4W8W0F6qoi0yLcNw0UGnMXNCpg54RFicpUBPzsmR4JziDSJEmZeezXpMhRrdWUnfHbxEQz1842u",
|
||||
"1e7+CDfdW8ZLUCnp4kUGULzAXGsOAi3kVeUzB9BNK6o1gOTQc6gCyU/gFeWBNA8UVU/hib1IvkdI0sr+",
|
||||
"W2FWV8qkq9G2a402XqcvPvJSZ6BbC7UVfvrLoTeRnNLoFjD5WtKzvY5SnT48rJ7S87P0dtj1DOTmXUzM",
|
||||
"DdCrv600j2J8msBO/LsqiL5QHHppzLhD1ReGqq5+Th34V11BqrJ27sVWDNSv1xDcSeLOnwfGfi853xY4",
|
||||
"i0kn73fyfkfmHpHMFZUDWwj8RVavfQxEDZtsR7ayh7ecB9FiGdPKJu5Hpio6tpDW4doNhyGKTxmSVZuo",
|
||||
"Tf0Pu5nJWiWTLmSry2LyIButA8I1nAqHiWX1DMwrKoIZRHdYH52S98Nzg34SaDTD1IHVsFxKuOD7v33+",
|
||||
"TApwDTwprhvUqXo/PN8HHokYYqKYBpxaC5LZvj6b1n9aCSr9dUIk5yxy6ZNEDGSS0NBUCuScZlmve/93",
|
||||
"aPutSnVhuj4X6h70spBRIjgYnqup1Fus0ENsPHRtO3+GlqkPzKmhkWV4ZM6RSEiFBuIcQUlaxCiFXdps",
|
||||
"1TOQYenoyMWiKysD1cqk2dw5dL122l6lzNGrkIA0LOZ+QtmomGOLWDSsbqCoXrH00StOyQX9d8JRR2Uf",
|
||||
"Ihw5YLJ2TX/GkiV+KmiaHxvl/y5B9o+S1AYjE13yKXytzTG3+tY8VOucdyU1hy2RorAKP41gkieFi7Is",
|
||||
"GXEAvh6DTXdWpR/SZXgX0QUzBSPoVSDaalbWAXrftWwA165HkZAMsyakNJoxDmQP68mWmefvZ8DLeiGM",
|
||||
"KxYDocukC68248KoWHyHEV0qs4dT/RIpqhC78bElcg2qrVNgJsXnBXF9A/BdfHy5zn/XZpe4jyNLUToH",
|
||||
"wO/iALgCbQUsux+auv9VRmnm9FeB4KcyBq5DmOdl6V3xyzcAevbeGQE7p79v4vS3ShB8VKXOIQ++cJpC",
|
||||
"M7+/MKkh40URfx1w97NE59K22WhdsHShGs29ktvFfumc/TrT+rdy9tuGUk09/TbiD9PKj0NVufNFItBL",
|
||||
"5MIdrr5Eb78ahi15kk+ybuzs157pWVe/742w31O2L5z9ZCfjdzJ+R+Ee19GvqYDvHGG2GcdGRbMtVOqC",
|
||||
"fmZpnhKep2OQREyWuuCiHksuecApJ2Ep0z6XHMY1TLFoV5d98QdLqrcEj03JwAooPfhStj+Lvx40BNqj",
|
||||
"ZZ/taRqKdNFnx0TIDfw2qg3a5Q/920DwGVcZRHr5YFuF5O2ZRAPgTHMt9nMk4AdflKY6V1/bg/hhroXl",
|
||||
"AiMc4vuDfN9TV7rYKLH7JHtmEDP9hCYKXvkXoIoNdej2t0G3E46GZiFJzJR1Wa5Ajy1btIqBO+BeNIPo",
|
||||
"zo3aHufQxdoiXcdhOpB/aGJFA03kfgZ6hn5uJSD8pAhL6RTIjDrDHkgyB6mY4EunjB2gHz5nYqu3sQfu",
|
||||
"T2y/DuQ7kH8glUdAqsL6ihoNAV65CI8YJowzpyhqC+uJmKr2kH4ups9Rklp/bydiShLG4VGe2+sZiVmi",
|
||||
"bYF8ZcRcBVRGM5eqLhhqY9q0S3x8w1JQmqaZ2cQEdDSzU+ZcsyS0Har0ECJgcyhuq8WM1gxDBE8WROkY",
|
||||
"pN1kYC4wEH7Fk4VvkrEQCVDekaUfwanMlg+20PdowmZKOZ3CwRcamaV+DTqgHeJ39ZYoLbI+QQe3fuGe",
|
||||
"2Sd3LEn6GLUyhz7JaK6gT3Lu/iHB5gvvk7wQTBvSugtcnp38WdI8s7wUuCa0WKNnvvJbeCrgeYqOgVpg",
|
||||
"KODSP9v9y5ww/mCOGI8ix6BId8j4qczK7o75Y//5iiddAs4XHD8IciJkSihJVxGAGAHpMQjTWmmmuFlp",
|
||||
"Jg8RWanWFD9DQtK0QNQLfYt0yP6Si1RpGs3Wi1RNpEirr6QdahF1uNrpDX4YPDnUfjzRYhuWNOOEO2gK",
|
||||
"KrW+Oq1YB92PYiwvoBEjWfUMKj9UNGY2rxKHSENMtNgB6nFlDzE+jnCAZ2t4LLiI3Wf8tzE+dqLgyyUC",
|
||||
"I9AGNHOuYFm8nCnh6vY81tNvafpsKz0+K7vnkxXXLNd/KmQaCMpxG3S26a6qZkcVnowqlH6eq6j/kyIK",
|
||||
"tHalgzZQAZ1n4bRFTtmkUNgArpkEgiY3mthKe4TxpQdURJPkbVmKzxXh61dK7vWJyGw+FHIkuNJQVB1z",
|
||||
"8oot0Ve2sR0Pr8+IFnfAy4J9A3LFk8Vq+oFlGgQjATFOlpX1MA/cIBTjOsIzeMISfzhBKGkSfnxit/eO",
|
||||
"VPzApKKkBVcc9tVM6AILMinmTDHBrVE2mP5ApVTq/discrOcb9odw6TLitQ62Gh0cTi8IVRryca5rnou",
|
||||
"WHPi4c0hkr7LDxdAYsnmsDnhnOI0k5RtK083KpqtETffgSybMCiOxYwwPDw7tqErb3pfu/pxHRV6sC5D",
|
||||
"uOK7QAr4Wi+mtg782xTaHbB3wP7DAPsqpffmXgiZYV2fJvGkK2t62sjS3RCxw8MODx/llSwkiYssDX5c",
|
||||
"9MSBt+Y/HdZ1WNdh3aNhXYAR+rwmNz2BDFY2c2V8Qtzs+z070UU7fwT/xQWP0BNZ5uNevzdhn3v9HnAX",
|
||||
"DOLipp7CMbFD9b9hioFhzqsY7VwQ9wwQ9gnCYJ9M2Oc+sRDYLwL3Xm1Gc2te3ZaLU8+ARLmUwEt1lzPi",
|
||||
"LvPN12gM+kuLXEZAGJ8Iq2m2qjDsGfSLXpp7u7Rg8bPLlVzTstUAYaPGM49F2OgxhH2zY+DabArUsnTV",
|
||||
"PdMz8xer1HrSgkwl5ZpAAnNMZLNnhn9FMpApU8pbG74QGc06nrBugxl/S82GouwVqiHNwt0+nrUZosOC",
|
||||
"H6ESzolFGINBCHipiCFcwyHnWIhlq2fCe9fuadDKFYUZXJg5StHBBy64CuLW8PiJrDqJ62/+uHIARsmE",
|
||||
"JeA43CbBKs8SQatqRD/ze4/NMKudjTE3wxf1Gx2D3VN5lgmpFfkzm/bJnxlM+yTj0z6ZskmfjNOsT9R8",
|
||||
"2if3MM76RLPJpE/onE2CRYzsrE00J++osunvkGEhZ8a+EONKv0VSvDRPNMuo1AcTIdP9IhPccrR6pjg8",
|
||||
"xvWDPqudrt2EebQJmVLde9sbM04x1nTtEVnZx7/d6B+9WN85EXQJ7V5E1ZOXrOMyeFtSy02PDiORNygy",
|
||||
"QRI0Gk3Q4Rv72GdH6RVVxOv7Xqvvlc2g1jI5n53mGeTl+66lLszhdeUtOrryrEI+clXPiWj/blrvA7UW",
|
||||
"NIpQVEQy4ikzaQQpW2lyf0wVxETwsaAytm5jfnnNrerJNBd2G1t0F2YVpVNoUXCy01p0tKEZbfjONTte",
|
||||
"cGKyagWRlcqbBX2qST0HX7irmL+xeIh1YDG0q0a3xgtSdA/XDEGKdLlstrkouRm+Mqbvybj82lUP6UjN",
|
||||
"31QMKYuaBLB8SzETROOiEHShr1EZRGzCIA5jdeUx89JR+ntUHLAPme7h0lGMb08xbGkVRPw63/Y8YKiO",
|
||||
"Zj4NdEzLQvkF8VB5NCNUEUgpS9B2L0XiqaZihnxWpOOJ3kcnMdNbXkenDJJYWeWyi8ztHkYd7fmRDcox",
|
||||
"02FZpXiRzEWSp3Dwxf73srFX/Qds/6HstY2smDZF7Ww7l8FFN42XysyrY3f5Xf5GWb6clO2CuB20rHvF",
|
||||
"roVv25bbshN9cK06OPmxiubMy3ttm/ytChFPlirDTlLT5PoOxjZb6nG7fBmd28y3UV3W0ChEZhMxFY3c",
|
||||
"kY+ESoUqfGRMN5ut/vry99I4vEadz83oWwEbux9kfFo/1O2+IgEY/q3pDN8sd4F5NroDTOyReO3oMkoE",
|
||||
"hwPr2X3gUi5v9AgcYg8b0nRUpGjeleht9awbQirKIpbW4GZYeJ+YntbnvCIzdk6BHXV7Ouo2PDKQj3nh",
|
||||
"a7Uf7e9ehKoaZja9hqpIdVw8aJ4Mqa7wyxKbzKPK7qr2oOpw6Rni0i7PIL4T6OZptukZVANZ07ZLYtPq",
|
||||
"bvI0QylnkidJcT+rSasaXFPClLbXqpre1nmlS3dprd+LNVxSZA8pKOaPXWSAhW76hAvz/IqBa0YT9arZ",
|
||||
"VSo6byF7jegcuutrl/GTzgFxbnd0q2TzzH2XlNfu6H3VTvCU8rHLjLnk6J1g3AnG3yJNJifwmSnN+LSl",
|
||||
"jCEBI5MbMS2b+LpjVi/djD1NxJiWwo6WlKuJCy9WWwCmhA11QMci10345FXZ5xC77JDHBVt2xLIjlmuw",
|
||||
"b1bz8/dJLXGWZglWqYK4PQ66eFuSK/P/jE+EK7/nFAB7Wmia9EmuIO6TiQTYJr3OJ2pJyrfh5IeJWtLy",
|
||||
"Dhn/FmoIA3YfTkeWzjtow1hbiDdJCziFQTnrC5DLpPe2dxChTrv39ePX/y8AAP//",
|
||||
"7P3rbhs51igM3wqhbwPtALKcTvc82E+A/cOx7G7PxAdITgZ7j4NsqmpJYruKrCFZcjSNAN9FvFf4XskL",
|
||||
"LrJKVRIpVcl2Yqfrx0zHKp65TlzHP3uRSDPBgWvVe/tnT8K/c1D6nYgZ4A/vRLw0/40E18C1+SfNsoRF",
|
||||
"VDPBj/5QgpvfVDSHlJp/6WUGvbc9MfkDIt37+vVrvxeDiiTLTIfe294VfiH3TM/JVJEpgyQmKoOITZeM",
|
||||
"z4ieA5GQCg29Pq6HSYh7b7XM4Wu/F0nBB38Xk5Fd6N/HV5etlvc/JEx7b3v/v6PVvo/sV3XkG9uzgb+L",
|
||||
"CWExcM2mDKRvkblmiRocJyD1oy2uOqZnUfiBRIJP2YwcXF+Nb4jgyfJVr76cN99qPR+ymGqICa2v68PG",
|
||||
"sk5GV5cn+PmRl1YZeMv6TKtmy3vzTddXX5fnOsecZqPj8+GTnN3a4FvOr2jZYq1vvvlity7ya9/Npty6",
|
||||
"pmxGMzY4kUA1HF+f34g74I4mmCaZFBlI7QhkbaaS/CktGZ+ZE4AvGZPLIV2qymfGNcxAmu8s++ecaUiY",
|
||||
"HZxpSJV3IPcDlZIuzd+cpuBtmIFMmVJMcFUbcfdJXpc9fRNKoPEVT5aVSSdCJEC5/WpWEOkbcSK40pAk",
|
||||
"dO1MysZfqyTzX3Yjn/rrnKNfuYvh5djebvAaolxpkQ4vx6dcS/dbi53X1uxGWfoOIebqOI6ZaUaTd4mI",
|
||||
"7szNqXZXF3OFXd8l1Pb3H2nM1RlNkgmN7rzDxlxdC6k9375uP02q5hNBZTwSuYbNszwRXFPGQZ5HAaAu",
|
||||
"W4xyzs1v64iwakEYnwpyMBWSjE9HH4+vr4k006pXvb5nz8Md6PQ7i+FMirTcg//ofhdKe/sHt3QhYj86",
|
||||
"XYbw7Jrq+bWEKfvi/XxD5Qz8i/igYG2BlaV/UOAduYo+2y4XEmhAtgLUoz1qWkbwYDK5Nxl8OmrXhp7t",
|
||||
"PCBEtfBtwH2Jiw0WLsWXJbYvWF6/Z0ajwdNFjLtsdOWrgQL3vjqLwRAWLAIkZWvyeH17E9MiQONwiGJp",
|
||||
"Kf3yHvhMz3tvf3nT76WMl3/2N3fFWXTH9+i5tuPKEipjNt29FRO2bv86nyQsMhgf5NmRUKlQl3UaVJES",
|
||||
"HnJMjC+YYpME/DfAMj9CqtMvTK+tqNpNvWezuZ6LXMEm+TfkPim/o6jlJfdMvRc0fkcTyiOQoZlGkNCA",
|
||||
"2FGFgc0l2GNTxfxr57YpOQW5aRBeWNYYUk5jprfCSdsjf4Lj1HTWSpYJkD7L+gert0l9o1TO1KmKaAbx",
|
||||
"5r3dyBwIm5JIpCnlMWGK0MTInksCtg85SIFy1BUY5CNUEVo2TxiHV+Tgn4zH4l45xQKLXg288Ee1ptF8",
|
||||
"rGOQcnMpx/gVNRJKUx5TGROQUsjtYzHebCjGs1z3SUrvQJFMKERSkisUmjRIGmHnrXOJXDebTOQ6y7V3",
|
||||
"sCiNfQKcPU8tiMw5uZ8DN4NJXehoyms2WNBc9BUpZdyPscPVNw9yghHJM8G4Z8On5bfHXi/wxeZ075nS",
|
||||
"REwJ8AWTgqfANVlQyai5Py2IAk0Yf8isXzKhIDbSvUWYJLma9t7+a7t0wKkemC5j0L2vn/qhRduxSYaD",
|
||||
"f+335kATPY/mYHl0s7lWSP47di8kkY1pf1+NTuyXCSgyF/fmoOzPtXMy+G5XhCcxr3DNtYGLLz6Wl9KZ",
|
||||
"p4uh2eYIzITYxJAOpsk9VSSjypzKZIlfrRQkJDmAwWxAIpEnMZkAUct0IhIWvfLNmtAJJPa+yifidY3w",
|
||||
"bT7hvFdkB0Iw0oLoOVM1ONqguCmNjuNYgtoUtXsXNCLuY7H11d3d8ls+hExCRDXEb+1UViPLFInLL0Qx",
|
||||
"HgE5vj4ni58Hv/46IB8UkFMeI86NQRssU4OLch2EGf5H44HvnDjoeyHvhkwZhPHQnnNFXBsSF418hEvw",
|
||||
"dzlLPANcXb77cP5+SFLQNKaaEj2nmtyDBBLDlHGIieAVMBiK6A7klCXQCk1FBjxA768ypD/mm2/lag5J",
|
||||
"stlrbH4mRnLBBodTIVNza6MPl31ycjHsk9PLm9H/vr46v7xptVJcyRWPPEhxPiVa5tAnUSIU2DUTOtUg",
|
||||
"8YB+JpbXQEyihBlKFzMVCc4h0srPVJUW2ZjNOPXtEH9HIqlFhnx7A7BX2zBtblgKXi7nPpADxomCSPBY",
|
||||
"vdo+cEWg1noZ5Jslz1RGvkiVGZQSrZd9wniU5LFhKfac2NTQD6YIF9oeYOw/E8PVNyf8oPCUDWyyJEHG",
|
||||
"ZREUee+BemXwiMVQR9s+oYkSROWZoeEoMLydSZFnviNciCRPoQlVWtETP1VyI5GDVORcq1dm4hiBdZ3R",
|
||||
"bQxoUJnx2ZB5zuAkl9KClYRIC7kkB9f/HL5aMVArh+ABTYAkNOcGGnv9hkovH5vaFL2NvLWwANuMAWqW",
|
||||
"wmCYu7fyJt/7PyAFMXIqgg/jc5BMD0jRQxEqwfBiQy0hRkZkoZNwyoWD54HVRJQazfoUI/vBgJ85KZ6n",
|
||||
"E5DmpszrA6JcswWQKWVJLsFQVIgB5TnzmcUgq0hiFpBzx3gHt7yy/GLtXjxCEev80U/vxpBmNyqin9aQ",
|
||||
"ZppUZBVFqCZxLgv5DldCMpBMxOXKru2fj7qu6kSb4G9vm2lGE/YfIBOYCgnOqGmvC/srexG5AXxxz/Gi",
|
||||
"tdMNrU0JShdXbFqYjhlIZA0VEYrqoBw1uOXHnJjzWxKVsAg24BK7xjCleaIHt/wGJaASTN/e8j+/krdl",
|
||||
"48ol3PI/b3uXV5entz3TwnHrjRYnF8PbXp+Yd99gMDAt4QtE5u/cCM/KIX+yLFofjn8/ff/e9HH4b/oY",
|
||||
"8rgiB3pO1FJpSH9SxdIt12zFGPWKuzwShDiuVNwYS/FJcE+ZLoChwL/yZWKvUJA5XQCZ53w2CFAzv/Hb",
|
||||
"r9ltpyQy24vzxDKpByjTnES+Gs6nHYlR3BqUNoKaDq381WPkodlnGsftFMSmUyxF1rKXhbOAss6t8HNY",
|
||||
"n5fln9WcSghY/aIsr36xpLv4APrz2vfqgyEDHqvPVtEb4ujbALjZ4Q/tPFfc/NLzAaPTrbW2fLXu8FkB",
|
||||
"ldG8Zb+anmGHMsHzuXzYt5vWPq7b9tGSfjYP3ZZnM2OxH7rWXvQPh4bKI379UR5+e3tUyiIJ2yj2eD1v",
|
||||
"wGTC+F3LQ0xpZGhK8XjeaJ9C+jlhKdM++UuJXEaWoGtJGdfKJ4KbISQoI8wEd+/eu5/TkBnSNVBPgvZj",
|
||||
"kKjTtnN4jzYrdFEtTHNC6c+MK03tW7dFT8kWLIEZxGFnAyp12NbV2hboMaltWARzbri5d1Ijbkuml59F",
|
||||
"1pJi4Dvys8iA+7dqXrOfZ5JG8DkrhVmfHC6yz6p8dPte0pLOoFjf/iimlirSD8VT9/r2PJNDJK14P297",
|
||||
"3za6b3y8Di7M//uuw71UP8dM7rYEWVK3v4RTZ7Kb4o7g9oD9cB7Ega97L+j3OuMIv5E3L6jySg28ErfC",
|
||||
"b/H22UtyXx+u9QEUhG/TUoUaIRo0nsaSLQKACdx0+8yyxX8FDHHmPOv4WjPL0tQLEk5/0QjYm2z6/Pr4",
|
||||
"Ikz0ghv0st9HYOF+Zb/5laS5Mm8oknP27xyaKYBansDGgc+ohvuatbRCU/MJh0dBwDX+uwmGCaOqrazN",
|
||||
"ssWvW0UbA5pbGrTexUckxR7fnzCSfGeI6fesBOefKOTA1eRgzoT0oO/Hx2VWwVfkXm8z4IuPVLYFsrC0",
|
||||
"X5iwF57buQc0e1uDEyWMa6JEYaUhEeWEzbiQQF57tY6PATYppO8LuT4kbQedAo00/I7xmPHZzksszLIX",
|
||||
"NKvw7WuRsMhPVwKyzhaws/YrR0VqOOnzNXvCh8S2F4Syy1NPq7jwTt3ACLL3zI7qbU675b5snx0XtYVs",
|
||||
"PtB7060iwvndlV3TZSJoHBKBTtBBrbUQFHZSpxK4NRxMaYAAb+OvbfaZspmkGm7E70LpAqXrm5xr7XeD",
|
||||
"Mx/U7iVg/6K1bykzpuf55HMk0s/0P/kdpfKz9fk7NGgB8rOS0efSF90rihQ+tnuDsQvZccNsyHyrxY5E",
|
||||
"Ag+cxQyxZQYap4zXHGvDkQXYtvQOPr9uyaHoDLi+CDrV0VwL66Lr/z6h0V2eNdv0O2y72rbte4WeV0Pm",
|
||||
"x5AJaGrnD2AXOs9CfGKEgI3IhgZ67MoZryCrZUTEaktospLLMnLhXNX8tj3OZXJ/VfVmoJQPyammE6oa",
|
||||
"+k4PXevVgM5mVdCGf1JZRDR4KJ1te5UBPx8O0Wa1teEHBfKCcjqDQpXsaYpEqs3VDKs9jOCWUpa0GeC0",
|
||||
"0gHVuSlkdAZthvi93sdRyqjNCDc316vejEfyXQN0SVgEPMAvEjGbMT57DwvwK8FSKu9At9nmRbWHGUHw",
|
||||
"mRi+848uONPC/IVRkQ8l0zay0ie4lvNUPag2ASvNYv7Zv9Iszj8H5N7785XC1uNxhZB/go5AbXWsV5W+",
|
||||
"/oiLVGgYW0VlQy5T7bK6JceZL86OAzpVZLmWogae9tgC+eQ2dWrDAMX19ek5pK2w7abSYV30qGJ/DQt9",
|
||||
"Ygg+1wYXzi1uU8ZIqNI3dHbDfE/q96uPhXnb8C2CbsLmR/N6mzPneWw96+6pImZQoulsBj5/QC+IU8YH",
|
||||
"lyUs+g3dNEnEvaEi70VEk/Pr4yiqKzLqoXQnc5okwGdwLcWCxQGBNeYqKhqubmj/V+ZWUxmKiydmuCmL",
|
||||
"qIbgcxPJTjzZ9i3YtxohsfFRpAwda5YBIV2peyH9VnClEuQjAfwAvzytE9xw6Ns/IPgk/idL4ojKuHJe",
|
||||
"TYOxEJzGoPPMD0nDy/HJY156G8iMEqAV7r/uRWoX8JZgsyJ+2fmRKLMjv0e/X+oLDV6T9Yhz7vRpy+yL",
|
||||
"8dgI5CVtDK4Y2xKU3tFzWGOP/kOwE4IAt8VP/ObmeuzbTGPsq8gcoc1eS0FcM3IHS/L//v//H2J4xMrR",
|
||||
"uBIN5W6R8viWL2jCbCT9glFSGiMNvLx34znPLuAxEVN76V636hZEYi3uopCgd8TYrTnLmov1ddmfanwf",
|
||||
"4oA61neMx/Y6lcfqgqCMath1n5FqKJrgI4hyqZzGc7OFGZXOSieA3Zrf60qHSgT+mZAR1OZad0zwtSOS",
|
||||
"MgWKUG6DmAiz4QA4FYko5wKV5SmNgciiV7IkZtpDwZOl37W5WNXl2v79a6q2cqFPq0VwwQ+9M/fJJNdE",
|
||||
"aZYkJAG6qPUqe9zyg5wnoBSpzcJWoRQYW4Yk7o+cY3iVNzAsDCcnSa40yAqohBoKrpjSwK2SF3ieGmkt",
|
||||
"Kn429DWizpM5hgRmhgrgv9GXsSK+OUjv974cmmEOF1QalFRmvMo0Zzn6PlZ+OSnGr/w2rExV+9nNWm5g",
|
||||
"WGr1GkWOm7dBeSb7ss7wuVsbyGZkcR1td2NUFc+R+65f5+4h1kDAsdvKXTcYotJhLbNGHWuc37NawfgK",
|
||||
"Jwz+FlGD/jiM0ra1FoNhvZZcaKRDJV6JjbKLJNY9gfGZ4WEW1cwkfbTa3PKULg25oC4wwI0gCSWGGZOM",
|
||||
"6nnf/A0LKMw58eCWu9ldzIQLYigCCdJsqshBYbajhdfygiY5eGOudDjLAg7W6mZvqj1KWGzU07QsDQ2t",
|
||||
"Jv1Y67IFBa7r3KOgKDKTbEExW9bqXxKdTw2Ol/+QKqEL9M3F/zYjL5U5R9fl6JVfvT+OxsWklR99v43G",
|
||||
"bk3VdnZ55bZv1q6xTgAKf7lmzttCDc5YYiW9Tddt83OBAhYS88zIw4b7Ow6s2H/g3VL7ojHG7D8oklt0",
|
||||
"Mg1rY/UN35mYrhh1dzNnqoxpiQRfACKBFgboXQ4GPnMe7mUUs0WEWx5XMdNMYTBuQM4Mtn2haZZA33xJ",
|
||||
"GM+/9DHC0TPTLTfCKCU/3f3UJz+lPxlc/Wn2E1FLrumXAXk3HhpUF/lsjqh6z2LDk1cYe8/0/JZb/WWf",
|
||||
"5ChbEHSEnM017tWuFzd8DTICrukMbOhLDfl9sSVhRLhxaFlggGECvQL3eg7xe/0ez1hm/nZEviHIm9Hf",
|
||||
"2RHNPz8Wo5o/btzI5t+XNIX42s5g/j4pZinX+XGdGPjse83UQDWW/Fh+E+JEZAGbi8onhn43tEIXFm5n",
|
||||
"Ed/c6u9C6fPMH8N7fl0ok5BtnF8X4areN5tQush5tDmU+VIbDOPjnFtEv81OLmi27XAbaT3XT8Xnwxaa",
|
||||
"fwxb3DQxXHOzb5W4vf2zNxUypbr3tpczrn954/WjyKRwec5+kyLP2mYLeqrsO03tu2b5Q8gSsSzMK+ve",
|
||||
"cmnhjt/AvL/FiaJiya7ExvzXrzvDaiRgbjvLqhhnqaFYP/sD6UqN8hpfsR8I5hkjo5P3V5enLkEmilzK",
|
||||
"OtHE5TkQFyNCBB+QkzlEdxDfci5iOMRYUsYx+irXcJxlybLQJhVODrhxdwpE5lyhMoMSvC0+u+Vu7kyK",
|
||||
"OI9AkbGmOldnlCWoCZ4m4t6sSSLjQM3FF6aQm00pSw4xqPbulv87p5JyzTigqDggl0ITSrKERmgtI1OW",
|
||||
"aJBviy07rQlTxNHzQ7XkkdnaglFS03j1iRJG7JRLAgmbYVoNs38yp447G1HV7UOBHtzy//Gnu4DB5fHF",
|
||||
"6Vf7SETosfHwRgJXIsGHp6i8Pc3SDaul5igHLWOyJdUwW/qu3H4hChKItHJa9eviaMrvuJci1MpyYBuQ",
|
||||
"aOTzFUQMyCmK0YY04vVivOltT4qcx4dSTBi/7WHPCY3uDs3GqXbs/R8fCVjDM7mXTGvgq5jGMlkA3rBl",
|
||||
"46VYuhq71+8lQJU+nORqucmFK2ln1kLp6Ew5KFjd4wo+zI2qATmugj6GBZp+/+tft71Zlt/2Pt1yFH7w",
|
||||
"xZQYABXcdiX3c3O99dR/6Dg3wJldmJc5KBxpcMsv8kSzLAFiFoxizPHl8Kf4LSlm69/2+CKF294n4iiY",
|
||||
"IhNhoNvegA285OKWu30hbvGlBU+myn22AaU1cllQvX4RjFcSoRANtf4Hz40FbDqQOQw9Q842ZOrOr86P",
|
||||
"mfInTCxYYjtTx3r6JzN6OVal56ctS74AOQt7ek0k5dG8tWvHXNzzgLY35/o6GN0mMq2CV0l5wFNh7RjK",
|
||||
"Ndem23oGplk4gee+u+GAoBDQuPqF2HZbrSzAjejbZj0wuCqAMa7/61d/FoplBmrwm6TZ3Mr2Q69Vtvh1",
|
||||
"37Qy4v4wgQUkxF1FmRelb7j4gsWWIdzB8ggfbCSjTFbeqYzbvTDBMS8MLvWweJT2ke2vhA4XjY8zHdq2",
|
||||
"t5xxw55sUzUv0umUGSQqM9CEZLk0FMzmbcNljNx1vEUVri/FxKXXSFHN+FNs3i5pEB502wPB3th5Smdw",
|
||||
"zs0JeED53JfSZlg8TFzm40PneI/x8udDs0jKrdG8cvBlmnFMg+btG7MZKE0imkR5grx9KkVqprrlONxP",
|
||||
"yslODjbJgRUnbC4Tp/6zoyibfWgJUtm7mSxveWnMf4ULuzRCk8s3YIDBTh+z6dT0KuYm/3cEmRjaUf8v",
|
||||
"mUAi7vtWjrnlc2EEqsqU1lcgpZxN8SccXsIUJFrAyhV47V4fmdQ5TcbsPx4AqHwsQ/KFpklNQWOnX+V2",
|
||||
"oUlSnAJmd7nljqfGREx3JE0SqRGSYuKMnpguyVov7iDTpZB1T2VMrKDFJixhemlzKtmFbiRPqtAMKqM5",
|
||||
"0xDpXPqscpWv5VOYyvjeCCsn1x9ItfsqbYQ9AZT2BfeeMs31XPhy5OHvZQ6SCr7ZHs6pn5ap+CC2ydki",
|
||||
"kZqjckkQHKyiwrjWtnhIXByfX94cn1+ejsjBKjFVmSZmlcHJu3ozGfgyx53YD4he3CW9oAlJDXbN3AFF",
|
||||
"lGPuL9AbKxeGtKWZWKWZC8NpM+e4jeyJ1YQDW/Ms2xuwpGQtJUlxBYjTWhS2+dV6HUyvlIqYoK3mxWP7",
|
||||
"xP0qvDsTACrp2+PF35rhReAsnSv+yl2jZd66YMa6k/rY66Bhd+TGqSQXrBBYewGGamzcAPpCOQAqMQEV",
|
||||
"zDX4QY2phAUTuSILkChem3EtnJNJzpLYPvni6kHb9FcSbAOCBsb+Lcc8mmWeKsaJjcshqADmy1RIeLrr",
|
||||
"K/1GfFkm3XO0jTtZCYhWZLCruuWjsxPyyy+//DfmTzp0CZTIAYplo7MT8+mScvEqKNbgC5EuKEuQr7Ip",
|
||||
"ySQoJA6VXHHuNIvzEHoO8p4pP8pbncpHe3++hJeVz8UZuNuuXPYG+trLfVzs9S1/thJMWyTI8cq0m2hW",
|
||||
"aUGsQFC9DzoRufaIbeUZWDCvo9lPCoGnkHoMN7AGlIEN+Fp5PTbbypq3pMdiVOQXXBMkrMePiGhiUTAI",
|
||||
"c7aN0yjFFFLBrYxs8DSjUq+NrBUk04HV9nr8kcYFEF2VFqSxtSBVxlAIP2Wm1AC/FyoItlcqDLLNJ7/l",
|
||||
"dnZyAMjtaZIskY641L2vBn7nIunl49f4+yYTzNzvj4Un1muroBZGJMDnS4p2rVWWSdzkkRvM3nKyHJCb",
|
||||
"clAcw47IpmtryPIkKcT44k1AJMyY0tKPqHIla/s8b8qP9hWRuIR+4deEfQfYVa9IopPR1uX02u2WQrtd",
|
||||
"/4CUKrNVr4jyW44NC8hHfWz17QNqtRQjueYqx7Vs0uj60QEz9Nid4C23R1ieHUqXnuNWc2d6rbR0bySF",
|
||||
"LNo+goodmF9nwA2Qm4cz8la1+Q5rpw02V3jjVYOO3Jf65dn1owHgCLWRNapjvyLtwUty5+1eU4YOlYdd",
|
||||
"XpDtg2MZiTdwQZY84Ri4FIsct7yEZS7sGJ7nG9rA7cFGVEENcm75bS/n1kv8trfeklkp3DqATaAKZpMl",
|
||||
"Hv/5sOV5C6HPxm3Z2sj22uQE9sNKZ+znZQVnsq2r6USRkeGbk5wPbd5H5X3QPuwlS6oP2ZUWc5uOChNb",
|
||||
"+0juR/uhWMzG49J1JAdOjEqW5Hh0ga5LrwZttC2j8qrWgxXMxlpmDXG+Aw0mz1VpDtuacC/slLxvgQoD",
|
||||
"nQkEMpvULKPbqlXg+ncWHQivfssa/HOd8wXbcVYG3m78V/Bo9Tx2H8p7MWN8exLFvW+uakio9Pz5zf+s",
|
||||
"df2fjXexw8KAG7p2LUagoOF1f5dSK3b2foMrGiEjBvl9b+m/vbKWXZnzRq/0/tvP9Yl/3ueO6xMEz2ec",
|
||||
"x2Lr2TwWHO4Gvg8K5JuzY/Q0CJp3dCCUbm0222zXVFUw3w4VD6EURbT8CFQmuPIltrGvSZvgSttcLM7/",
|
||||
"zjyT/tGMyXnD8gMxCPGx3m3wCVUhBBXo/fg1CMU9D4TwPIN6XX0LZr9T5bcN4tdxPvVXdttyhXGMKsIz",
|
||||
"NO2r0E0GD02HGCMa5dotpag9uxYyGek9Dh4HO4504ORraeoajnZS9ikzpQWimDEiVbLZDCTELXOMrOeb",
|
||||
"q8b3LkCyQNijnkuhdXBFWtLojvHZBRiAe0B6ks3T3SSbW9zbW0iyvoPfmKuoeRI6yyiYwmAdPreLipup",
|
||||
"MjYdIvDrAyPox4zPEljPybGZP8IG0TcNF/NVLd7wOtydOtr/VZrPky3pBAMg+cB8QOGMHxubO2MS7mli",
|
||||
"S59C7MsEsJNrXB7fjEcVx0hPeuzqUrYXzvt2dV/pY9R9nTSr+xpvzewQczXdURg2UMqu34vFpdDjJY8u",
|
||||
"RRxKObMV1lgmKQ/kfBMLkJLFcAmTPKFFNbvtWZ70nKlhrbzhZpOcc+fu/qg5fb82w4USLjaw4S4QkboP",
|
||||
"J69i6l1I9PflsAkkCV1lY9yEgcmWj1szFmyN7c3yLIMtqZaMCB8cebGyNjTmaMO11DnrJWcxpNGc1nXI",
|
||||
"DUzdsexa5hzQoyeADmWjSn7O5qzitJ6dx5PD4hyrtUi4eT/eAx+nUqTBi/QLTEKz6RKVICGvuW23HKIs",
|
||||
"uQrvYMvVh0+ukiHo6VM2RBFk+jhJisvwcJ1q1rAWGR6w7RhrGJzyGePhRGRzF00XCnTHJlhMNtQiolEo",
|
||||
"gD6i2fbRHc+phOLhkz6cUSWq5GwouvyTMh0oYVvv0CwFy8g63UsV8ueNAPM7VOL+R8CNhOJfdWG2ubDB",
|
||||
"DMe5nq9lHKg03pm7pUCFakzIDg2QSzsRNUg7gQ33mqJgw0VWBFtYM+Dza7hkm1xE15UOSPQnCVPzi+Hl",
|
||||
"OJRsKZnevB8HEzu47yGoXH02GKlcSHu7FPt2iI80YfEHrkMpa1QC2zJTRFsyUwTlAPy098J1ohbbF61l",
|
||||
"rjTE5k5apwbMFZwJeU9lDPGZkEHC3Ty3Ri0j5qo8Z4kqVZj+tJXsn0op5AhUnnjDukJpkqzXXiAjUaGh",
|
||||
"a85/NvLibb5SZxjnEtK5UR6H2PY9i2egH1xs2a7UcCGMp/bkQ2sRySGFFpFIwmU19he8lasR+sDiE4+U",
|
||||
"rduT3G/jmOxcbR8ddkyb46DNDdb6Nb+yXCattlzLwbdZ4SEk1xuRI1SJJV5Pg1mriiKbJPBl8RZyUFH7",
|
||||
"rmK+f+6/6f/8uv/zz/03r/tvfu7/8rr/y8/9X1/3f35t/vfzpw1Ldhnr7VyBbWX+09HF5+Phxfll723P",
|
||||
"OSuJTL0lLkl3n6h5rmNxz/skxyCpPtFmPZwmvX6l++fR6fEQ3c3hniRipoqvJ1eXZ+e/fRgd35xf4SSg",
|
||||
"jzKqo7lzGe0Twzh9jWtDRkV+P9tudDo8vbw5P34/rreSgAEDNFHkAPiCLKhUfVKI86pP7I2oV8VI769+",
|
||||
"w70f86VZ9gziQxupIfvk4uyYlLdUbf/5n6fH/wh04gL72RIirtPodHz1YXRyOsZyxBw9Uwo9mFmTJVf9",
|
||||
"olp/nzj1X59g8tj1UdYPpulAfZKixra8mw/j09G4dEI13RIw/4WY2aqqqk/e4CEo0LVOtSVgwfaEKX0U",
|
||||
"g6YswdQhDtgqdhlMLlCFj/bwFpiu33gHDzmyvW6uDrwh4G8PvA3gdSd0fvJnPFnD6SqWe2CgBkp+SN3A",
|
||||
"gjC2e4lGCOlrKLyJn59WFLQu4tdp/hOVywqzoI2+m4nOS/ngBpl6S7n2/bvgk+r9u7Fm0d1ySzpw1AOU",
|
||||
"SgDztrxZlyxquclrJr+2hjWvwdCXwzxO2bYM6bmen27TEaHW+fj6/HiSK9jS5ESkqeDvhA4lqxRSCcnq",
|
||||
"iqPmVugi1TbQGOTv5uGxM9X2e5jRaGnuwfZSW1sHto8VFiv9H5DOlcVwJkU6pGo+EVTGYc2B39IeBY4m",
|
||||
"pV/eUR7fs5qisqI+CRcGDNrgFyDvJdOoyrK7D+h09fxagt/aXUh/VnAMauirjUZOJPwwOlfBymWNTfXq",
|
||||
"jmUfRu9P8G9/i5RKPZ4zSOKGhsFVB1eFxYySCTHdrvQzq86ufadVtQ5vsdg6u/I1yAvGcw2BUmirwmYt",
|
||||
"ChEXVpItBhSIfw/BZa7g9zcnQRXAWr/6x10ncl+WbuAT81huWcHBX2DYHXP42dDoKbwq0duCam97bLfg",
|
||||
"g5UKGc8x2ULQhr5pw8i1GGuRlXFzKlwJImTddt/ORBB59jNJSciEYloE0rxL0EbkFHxLNaagzsEPmJUZ",
|
||||
"w8C5SYE2HTW38fOUfvktEROajFmaJ5pyEHnAlJ6B/KBAvltqWC93tdnK+dTtbth43huWwrs8rt/qqlBZ",
|
||||
"hrsfIzvgdQtKLU1VUQ4Wc5Sodq4mY06z0fH5MGguNa+BK34Wol9x2+wLG+vYx3cDQ52ch1ILVXck88nJ",
|
||||
"FkRTSx6Fv285xB2E1J1xW6q0djctKNNNvYbCmvOrZCmVyxORBNyZbHjolibhiT8ob2n6re6ZYcNCQpVe",
|
||||
"t8TWvxa+3SdzymfBGRLG79qn/1/zZd72/fRLtrVJcGVtQxlcj0++5MBFmeWxASJ78O+ASpDHuZWbzTn2",
|
||||
"5lbWLVDLpShg/7GS0grEMmb2/fUrlp+ZCuc6oSlqMzeyqIxOxzcY+zwVkthyaOQkEXlMbLEQkq4K/Rgh",
|
||||
"TqMTsGs4tk2Or897FVeH3uvBm18Gr18f5lxpQyJev/65qLJCM9Z72/tl8Hrw2mWdwe0efUbt0ds/e46k",
|
||||
"rseK6Vxyl+TMzpmIGUbCEqpsejXGiYYvmtjCbm69LuWo4Odx723vNzBAqZCdWednnPLN69fFIRW+gPBF",
|
||||
"H+GYSAUQt2u5byaMU7n0OEJvnO/VP8zWf339c8MpmlUYqhqSPJN+4NQBB8R2+l++5fRnQk5YHAMygr81",
|
||||
"Pt1HmfvclR8uYBOb17AMY+Kq+PWvT18/9XsqTw1xxfj1e54IGldADbN2Yhjjv3o29rr3yYx5RDN29DmF",
|
||||
"1IliW6HXNiM5puBwWk6XJsitGSMbbbT3JMfMMz4YPs7YhZ1xJyDTDP0RTe+jP5RofeDVOIEWwP3QaR8F",
|
||||
"xL/BImqA/uvrv337FVyAnosYk0weJ4m4N4fRBtZ/A70CPwegzqa8BeJpWQnMAbwPRl25MMsPQel3Ig7m",
|
||||
"kV81YbBRKuxBMN4i12wYvl9/+4t9R2NSydP6HMD7v7/9CowknLA1MWknUL/HYHJZ5ANaVZYjtIDJArQv",
|
||||
"nCHoE77alAeYr4XqoLmD5hcGzXVKffSneTN8taJJAlZFWwfzIf5eAvplEb8paQrWBvSvjRxppiVxrxt8",
|
||||
"omAWyfKBUqqOiueQhdPVoa4Lz58ehl1vOvR6Puj167dfgRGCztBvra0ItHJLENJ5GxC6gWlksiSlnn6T",
|
||||
"fewUhTqc6nCqw6lWOJXlPoks73Cqw6kOp/bCqVIwzNghZgRQdalwXXWVioUtrljWOq0MHxIhM3Zjh95A",
|
||||
"lEc5aetmRzM2cFO6JBMlaG2eOX63eXy1cEe3gfNfn4s+7TmgZqfTe2H0YejowVpZYkcBKvheEVb9imqa",
|
||||
"JGXqX4hXwylycM/0vEj7RiOdY5pPg1pzquaATqxeAbhGETqV9bMA79ZKCAMXK2AIA1ehSFvLjOuCFBWh",
|
||||
"hMN9haFgeRu07ZX50SseQWgOkRULoKT3ruOBmot7btNnCh7BJvgV2rtvxpGsz3pjjlRUYSQrr/eOJ3U8",
|
||||
"6RmrLn0W3W+wgEew65640v116hMiY5uCckWL6t6la6CCsrmtRF7Uhiun+UmRSut+lcD1yfk1Kd1JUbav",
|
||||
"ZYohhWex9fPwPoYL+tbkPbyShZ/sPfzIRNWebAOiemaL2mnhHkodOe3I6Q8k4ls8aCTiF+SrknPML5ZZ",
|
||||
"qmiEMtu0qEiJCcA504wm7D+unNLKCxhTYgPEEA/IqCgJmElIWZ6ShEWYIdtm/OeHq4IhqYixpIRXSHvn",
|
||||
"lvo01CScPc2HKZWTKGIRHpuUdEq8J/Mwe4HyyMQPcgVuF7ixgdiHq/pL29/0JKUZlmNLEu9klSJnzu+t",
|
||||
"iGa9g6VNSO+6GflgEHjou3WeFLGxz4PZvsB3tveKWgBERVjdARcKSaJ3wgBINLr8JpLouxVAPaYo2oHc",
|
||||
"i7Ns7IDDTYtGGP7rwVFbrPJuiNGq/XPg6C+XVpUHj1W1nU6PSZKI6G7TqbTJBToidqQ4zdRc7HQ53bhR",
|
||||
"Q4TGZefm1IgcFAW5XH30iuj76pmQqh8A2r4/mXq50mOJfSVyYAXEDWRckoMKjhrse7UvFmq6JwZixw77",
|
||||
"Ouz7YbDPiCwVFDOowZRmkSIHiv0H+jY2LRI5130COho0QrojiG2E8lZXdNf91LT9/roSq5OKQw/YTmfy",
|
||||
"DHUmbSDdgFnNpLGvpqKNE7ob5GW+ITvW8JdmDdZnbl2fXrxki7qaqvIqaYo6RxKwPnBTDmHQZ+S6fB8s",
|
||||
"2o817UQStyniJrCPzeJEz+M+sXl0bFHPssS5rR8JxEadd6yp88n94SiPRQusEO4KLRdUqMCONuSmpdpl",
|
||||
"X2VLx7Y75Hkm+pSVLmUqJKGFQ2LksKgN8uQ8EbaQUVNW/cH26BCnQ5yXhDgWbP2aR2llYaVpgjXc75op",
|
||||
"Hwsc+rPAx69HU5EUeU7bcaIz1/Gbo1V/o8a5WxE5H/qnqPDoB0wzxDSl5vjN6CgbuyL6lfFx9n/ngEYv",
|
||||
"N71bTEcoOkLxdBzW4XHNVrGPbFojDbJ0cWiuV1rRB+z9rMgD0YJMi3U9FqHosLfD3gdgr8UTcmBx7FVV",
|
||||
"OC7Ace3JKauOFfsgtsonllwcOsXXoWL/gfYiwLgYxz2Qx2aUv4w8UO6+kwc6ivL8bKhWZWUQG62o5cu7",
|
||||
"hFoHsLSZmBBRfqiAx4dlctqt7o/3c9BzkFUfxzlVBDsTMw7jMxLRjE5Ywgx8VgJjA16QJ5SPgcdYIPXZ",
|
||||
"eMA+o9SEK//nOUR3hE3XTpspUmSW3pKCsKHbs7nXaZ4kdQfWwhhzoIArptkCyNRGDqVU3UGMMIiRC2X1",
|
||||
"Hlt3KBTf/Dj+zjRJrqaINi3vvb9RJNgl9942zozpeT75HIn0M/1Pfkep/BxhWt9De1aflYw+l3WjrfV/",
|
||||
"gy5/eu4h1i80J6cPXCvo4KD/k61zEs03EeAa5FTIVBGshERKoFXkgMZx3whnCY0qxbVSsYDPeeb+EYt7",
|
||||
"/ooITrCQKnHllDYCd8zcdfB/+jBALLK0JQYQv7v4v9W+u0jALhLwB3KZR8xzuN2ESOReHokkwDJJw+Uk",
|
||||
"BNik44oVbujYJZVAMgnYK7bPv9JPyI7xKhCx/KQkYx/WtgkbeSUNij0Ix/WeLSnp0Pg58fwXbNKxzLOp",
|
||||
"EFKXyY9im1MsJJiYn0GR4eX4UEKCvqNFGWXnomOoRyak7pMpTZIJje76BAv5JUxpZb16olxpQ224lmyn",
|
||||
"ZDL8FqnILsd2si2iyfByvB5g1yUs6MSUv0CdAUdRNhBgBzlZJUE5mjRyp6iVXnzn/CmeCPFXEw2GWCkY",
|
||||
"53MA+/fx1aWXBGBLS86Oco7/JRldJoLGzxf5uxD//RgpQgQRkhQ3TUkNQl2J6ToWVGqHhpBhQ/sVUEqt",
|
||||
"unQR+T8EQP0GNvdilEsJXJNLmOQJ3UZTm0ETh0i3I64nrlNT8vrlcEnTxOvoHa6e5bb3v48v3j9xZFEH",
|
||||
"2d87G4qFJ6LnTBEuYkzUW407qhPOj9eXhIO+F/KuPbxj4pWW4G77PGFQQ31/ZWJIJxuQA8sqLmkKfcLU",
|
||||
"ezab67nIFfTJ3NV37hOWjSifYR7M7hnRJSx6yoRFj4eOMWSJWKZmW7vT1SQJWbUnMUwxR1ktfVFt9MPK",
|
||||
"6OQfH8nBtRRkClTnErZY81YDDCur6/R8Lzt5cZTkSoMkce1OC2it/hpOY7zKl2ewwAuLxPnXPAAQfSxo",
|
||||
"HRIf/1GbSTFYzeJ/vnr223GbZ6+0ejHVzmo8ZhNfg+i6k7UczYEmtpzyVg4TgzKATBZqUGS4l4CHRdBc",
|
||||
"hrCf1RZFsiRXt9ycUJwnIMm/cyop14xbnyaoI/uAjIDGh4Iny7dE0SlKuhFNkltuHSj5EgXgPuFCkz9y",
|
||||
"ZZ98q+ETrAQ9aMm9frf773hYiRZ/+x48zIhfLALygdMFZQmdJNASQxxSWHi27nI1qUg9AEl8ORoCJWGq",
|
||||
"CPAYslfppB8C4CbZICr86UfMKvgseMnLK4nShpH0G+bL9Et+D3ledPDdwfc+iTM3gduTL3PjcdO0gsE+",
|
||||
"cF66+DxHOP8ez6YiSVaN/nRvpuf9Znp5dQEe98FUJjKYSHEH4QoC2yXC8tnyk6q9i0ATJUyLW87hiyYS",
|
||||
"IsEjzJq3jBIgVGtIM42+MfjwIkwTOqNsjfgM2upObKoFu6eO23YO+u2dWU4SoLIG7jXYbo1y5j2o2hjz",
|
||||
"h65LZ/N82QYVVEz7HEJsFFzV0J8rkOTAvfXZlNA4ZfzVFltLv4VxswpP38ZVyqr5mvlKrZtCO9v/X8iw",
|
||||
"aPEBXX5nroAk5k+MQGo2NVsDtYfFEeH96E8Wfz3KDFduT3zP42vTcacEgcs3MsJRKMybxS9TfOhM+V30",
|
||||
"9/44f431yv3YrgWJMDhYAo3mNvZ62R7LeTuhincC1UNs6zYgo36fw8txJUTjUUQV/rQF4uqTXY5PuZbL",
|
||||
"UCyH2doSobXqHdbpdJ4pqX6ZRnA/Ru1FDo/+vINlswzs60j3D1julHVKjLjDxh5Rx37oynn90LHJpdUt",
|
||||
"ALlkUkBImCG0YNodZHaQuU+huQdApzOe7TZ6fVfo/O4iUmH1qvKsTkDq3rKPWgk7EB+yv5wEMdOHLi6v",
|
||||
"VYjIacz0sIjn+1aKVDNpMzWq2VenQv1RVahYpqlqMPAqVipZwtqixVTICA4lTIVMw7bg34+J4MlyQN7l",
|
||||
"aWYzy4jsMBEzApmI5n0SS5EpkoiIJuTvoMdaAk2tl6xL8uB6YVq+lHI6A6lQ/ytBaSq1IpfHN+M+SYAu",
|
||||
"DN6vAsbuJdN0kgBhnJhl2q2n5pMSRcYr640YUU4mQIBLkSQQD8hQgEJ/Wy3yCCuXkpNjrH2t+kVJfCGL",
|
||||
"c4ypps0Mz2fm2Eb21Dpm18VgPUIa3gg2XW1tscxDA2aESsNPLV44BGqN7TPQhxy+6EOWtdGh/gb6Er7o",
|
||||
"86zTpP4ggebom1N6qpPza0LjWIJSRZBTnc+cnA9HRFI+a5/TIBGzVvr696Z9B2YvH8xckP/H60vMOGX4",
|
||||
"a2Lvth0AcarVoaszHoryGUKUUInFhTORiNmSHEiRQJ/8foyCwqs+SdgCUMRw+a+OooQB10eltHL0j4+F",
|
||||
"wOIyJ2DAtsiNYGEElTsu7nkpuzSK07mkWo2LEukdSLdVOGrKEojx1o7qUmVuXWlimOSzmTXYt4QqEcM+",
|
||||
"vpAokTbxgnSiLWhFmFa3fIrJkM0O7pyHJPE6SDJFphJg5R95L+QdEZwwfcv3dpS8NNtt4SJp2he5CV51",
|
||||
"XpKdl+SGl6RBhIf4R7b1z3FuOR0RbXldRTZ2Q3CQ/Vm+V+VxTBFq2GNrIprlk4RFh3u4ul5jz+fm8Pos",
|
||||
"gldfsM+rhQfC+EpNQycit54kXPBD1ARB7HeObQ1+yF7bQN0IO3REpC0RwXOzFGQl1D8sOZnT+rW8PezS",
|
||||
"3d8e92dObv0Gi2eZEVXN5O5l1Poudc45JK3o/43r0t3l3p55NMIKJ3VSqstz3XmF0u5q243JMglMxebl",
|
||||
"2/yqCYPSnjy6urTpKd/0uirsnbdcW285BHQhnRMmMeBE/hCTOt+pQjqC62630w6oO6B+cUBdpdrebC4h",
|
||||
"f08peJPA/HIhTxqW3x7LOiTr3Ij2cgjsu8oLfWveL1xXvfjmyatR8pId4lGHWx1udbi1J25t87ftcKvD",
|
||||
"rQ639sWtQlyMqZpPBJXx7vyYRUtXfa6qw71nem5m1pRxkFX76/h09PH4+jpUSNLyyWG5iGdXSJVpSFWL",
|
||||
"YlDFTrAYZW9VN5VKSZcvspBqV/Zoz1KuK4RBYNpS8Chm6m6XcnSIbbrC6y/cAIYZW91VFvAw1kLSGawD",
|
||||
"xJGlr2GfkzP8bt2mVQYRmzKIcXRLkM3vM7YATqYsAVtJ2/lHD5AFMwmKZFSpeyFjyzkcRR8Q60qjSCbF",
|
||||
"DN3+RK6zXAedSRA+7YqeKApC2WMa2EnMfKHAB9vCnoRbyR6BDxq+6KMsoYy3LF5jjw5iYkZw50ZckXO8",
|
||||
"mH9dXZ+Ojm/Ory7J+MPJyenp8HT4yS+QhRbxbdLvPensu1nNk06/g6Y86dyP4w9uYJxaKBeSZFRqtmZp",
|
||||
"9VAWHvaRLEuHU4yiSoS4y7MyIZWlJB9G753j2qrOvgQlkgXE5Px68WvhJhwS9VB3ufXFZqZAr3YctXiy",
|
||||
"/TsHjOlyb7ZcJt/Uq+zxJcv1FbaSDLtKtwH0qVq0DfggJGOqdAO6FdSw7LCOGYeYfCaIHx9BGhZrgJ5q",
|
||||
"HJgpEgkpIdLJsnx5QbyJM2FsQPerhihhc+O4HT0TxOjcLbsK0hvehJ56x41QUER3IA+d18nOINwhNnd1",
|
||||
"FZ5I6LRLGtSmquUw3Fbq4RESGD5cCC0XYQXR5yBmdnLeHimBLAiWLlkHuC4Fhwm7g1f2XeEKsak+WYgk",
|
||||
"T6EomV4o5moFQnC4CvLBYq0Qm49hnS5cnZGt7GqMvmQYqM84GZ2d/PLLL//tXp7k4M3r1/91+Prnw9dv",
|
||||
"bn7+29vXv759/bf/8yrAzaZSpL1t7Ku/Pvcpjx9lZi3azXvBOEvzlCRiRhJYQEIOMPakj4rSPlF5FIFS",
|
||||
"fXJPJWfc/J5mQmrKdZ+AgYvQShIxe28GrK0nhik1QPe2Z4bv9Xcv8CxPkkN8kSqgMpoTDRia7JvStmh3",
|
||||
"AO/GV5cExyFUEfM0J66xf47izxZTXNMZ465MZi6VkOTgCoXl8yGZwxc3X+gcM4rBiU/4Rui0a99Zu2Zp",
|
||||
"mCWGU5ZokJhsksckK0GnQgMvQEsWVSNQpnTBoprb49pRgY7mYKP9Ior/NHK+67Yp9/fdQ9ksg2mDGJQT",
|
||||
"llpI9BHYM7eA3Q+CQ+CRiCEmimkg7oUwNeurLieEfK3eCBuIWAg4NMtsmoVpQmdBWiIXNMseEfNEpEEf",
|
||||
"YqRcWgfGQmPamzBObQqYXXJSk+dHeMLv8Ax54sXs8xx54iU9UrhxFUl3PkVs1hPr7n5oDa5hRfyNZDOb",
|
||||
"rYSklOc0cS91nKkMobeDOeutCunRMc+D3ajNddTFPnXWyAdaI23ukBVMbkBiEA1moA8nNLrLs53+AYbv",
|
||||
"2aYkEioV6tC9UKwsOGVJiOf9BvqdneOBoP5E5L8D7k7/1YbROCxwEB+0+s+kyLNmlc9djm0pEiC2m01f",
|
||||
"Qc6H5DZ//foX+F/kl1DZwd/sPB0bedk1zGfFNRYA5X5oWrF8A4aczoimNr4vA5kypbCkfqMa5RW4eprS",
|
||||
"enb9OM0WZSt+X2k5Y8wJorqMo8++zF4XFLRfXnhEXh8dqDMWLHKzraCzDQEy5GGDNAzImUEiwqZYf0oR",
|
||||
"KoEozZKEUKXYjGMiJcL0oGltZ0sszuNdig2LzQ+rmMO4hhnIrmROh/TPsG7PCyI7ZUL/IMlpUU8Y+/yk",
|
||||
"rMAhZBt5I9ffl4I8jXhjD2mXeHPGIImxGGq+Ugd1gk1H4zoa9yg0riyhvFOsQrNNJbR6+6udZBIWTOQq",
|
||||
"WZI8SwSNIbaWn0pQju+5fm7aNIlsw4aoZbA09YDxKMnRyxi+aOCGtL76Lkn5nsRA8x0NEK3ITGcs+hGM",
|
||||
"RZSvYe02LTk22OU6c24b7UqridKRTcbnPH4s0TDPLa4yiHTAzsoqdON5uH+audQA933ult45Pjw6tLqj",
|
||||
"XXmIrTOZoMeXBdujLE+SRrB7bRo+AH4z2/9bAu/D3SfNomuuk53zYotar+bw1gATUyNTImHGVL1MzzYA",
|
||||
"PWTTQ3wu7swJu4LV8+mF69EBbQe0DwJawZMlYVPCNGG2XAxNJNB4acR8BVzbyjbJdnDmC7bNjeQ3V+e8",
|
||||
"sNg4FLF2jTtYoplGZNbFMFkSBTy2DmU4sA0Cc8GcmPmUQEpZErLfnNvlPFHFOAVyYGfYUaXqkkV3pREK",
|
||||
"65ggnD9bNUenYngmKobOV+GBhG5kKFdcUI+E8bsK9UK7T4V4/SEmu144fzdNOh/qvaz7RZ2ImPxhTzGQ",
|
||||
"r8V8PVpZ9LbGR5n7sCr8fdJZRlLwwd/FpEa+u9t9ueheWnP+EJNVgRJMHBTK1O0DPkcAdkLeb6A7sPuh",
|
||||
"WO5L9slzbkEoZCZipsxzjxZZUyKDEjvAXua8EdiPct6BfUdtexcYCJAsibbBAY7uakFkznfDGt+t6HDQ",
|
||||
"xh+h7M1fUepy9XKTJXHH3UTyUlpkjcjA2DTs6EBHBwwgEFqFsTCIJUzpw5jJHYhvwHfI5O4obEz+44zD",
|
||||
"LmD4rVVUhWJkXY6tdmG5QyYh0kIuSUb1vDLVUTgUV8+7UNxO4nvSYGBXK1ljSDCKe3EBqFtTZCEWpi40",
|
||||
"eDcmFkHEHcT+AKkZV9GaFgCwDjw+GvQcmCQJndSL52yGkCdixnhYyW8WAFybM0FFf65AukiMqiK8UOT3",
|
||||
"CRYI/fs/b0gkxB0DIniRSyKk2H+PK3hCvT5OsEOtj21IJCE2u6XPOSKjyzz1IjK6fTCYkjjYLvDPHGMd",
|
||||
"+USug9557/HlnzveYB8AFgMnSxIlQG2+iBWK2gLPBu9CCQ3f2xmfCyw/o+jkjYuz5xS4uZTKO9DNwiFt",
|
||||
"W1v8WIlcRlCl0ZUtqz7JktxlyJ2L+4gq54BQhCgnIo9Dhbov3JI6zv6ydX/m9jFTSQVsEhuVUGPl+LkK",
|
||||
"ktAo6D2TAp1wqwnxnTPJSsdQoSkQI8kJAR08vyT4u8cxKP4is9t3bPcREKzGSYt8ZAEyD3K223J5ga2e",
|
||||
"No04zrEl8Aa/PyB9eFe9pWNM2+wDCF5pnmiWJVDJ0K9IrowUig3k2XirrqKZmmKlotheKkmkKT1UYBoZ",
|
||||
"LpVgKcJp7SmMibm1ZBDMzJ2Wk3XqvUeyV2BAwmwmYYb34k54vazIpioindJtAeAjSMXCZbB7c3aMLoY2",
|
||||
"OdaqmgSS9AMap4yTErS2xHlfTOlTah6MnPHm7BjLjW+h3aVPoZPDcBc2pzxo3KtNjdcpJJ6vi+FL9qor",
|
||||
"Ycwp+Dblob7/XbHuAnxzdXNd4uWG0mK9EEMmxYIpJtDSFM46X+BoB+ovXPK2sIKA4ui3D9D8euiPNGEu",
|
||||
"SN9CmRZ3wP1wpgUxe0X3MZRYDqc00ga6a5qykD762/AEzDy/hSesNtkR/h8QG7A4xxLxoLjk0BOUzQza",
|
||||
"HM4dXmx/idrGvwuln7bCgFvVjTBTXYjYCyfmKJHqm3+MSSak3vDd7LJDdG/Vx36rWtgkJ6g0rxT61IIY",
|
||||
"LCoKD2Au6G0RUKnIeQOcw1ZPrP0xc2zT/pjvnfanw6gnwigEL1pR+2xX85jmO7U8tlGnWvkB/FDwwiGu",
|
||||
"qgW3AogjwEd/un+cD9dS/wXUNZe2+WXRa2ektCP0ofRavDLQC6xB1sHtA0OLXNB0IQ9MlhZSggJBUbFo",
|
||||
"B2m7LJp1gPKDELg6oHgLU/W3S4k1mHiyZxnmFQY317Ur4+bTODuIf4SSb92DrEOjNvmA66i0ndreF+EF",
|
||||
"O1DrvmV8wWOGE1xT1DzGtaiCfQMJNsugiSQ2p0VTTO9i8TtUfQ7bfts6w13gwl85cKGG2CX4bZP778+5",
|
||||
"0tRm8PJr+n+nPE5AOc290pAVBWCY7WoZ1ip7ZEw1nVAFRIHOs75TNtb0jLawpDULo4mg4HshG8DlaqFP",
|
||||
"w69TyvhgNUvILf28umU8C2dFf6ZcuisJ8/wd06+liEApxFm2Dl9byytxodm08FXe9Qaqtd3Bm5tVykRr",
|
||||
"n+OH1dH3rsbacb8X7slZA4PSGOx1Xi6Aug6XAdg+kkDjNgA+Ahrv669WnZl4wF6F8vnF6uXJet0zrpM3",
|
||||
"H5KlQt6tIT1VRFrk24XhIgPO4mYFzJzwCDG5yoCfD8mJ4BwiTaKEmddeTboMxVpd2QmfXXwEQ/18o2u1",
|
||||
"uz/BTfdW8RJUSrp8kQEULzDXmoNAC3lV+cwBdNOKag0gOfQcqkDyE3hFeSDNA0XVU3hiL5LvEZK0tv9W",
|
||||
"mNWVMulqtO1bo43X6YuPvNQZ6M5CbYWf/mrobSSnNLoFTL6W9Oyuo1SnDw+rp/T8LL0ddj0DuXkfE3MD",
|
||||
"9OrvKs2jGJ8lsBf/rgqiLxSHXhoz7lD1haGqq59TB/51V5CqrJ17sRUD9es1BPeSuPPngbHfS863Bc5i",
|
||||
"0sn7nbzfkblHJHNF5cAWAn+R1esQA1HDJtuxrezhLedBtFjFtLKp+5Gpio4tpHW4dsNhiOJThmTVJmpT",
|
||||
"/8NuZrpRyaQL2eqymDzIRuuAcAOnwmFiWT0D85qKYA7RHdZHp+TD6L1BPwk0mmPqwGpYLiVc8MO/fflC",
|
||||
"CnANPCmuG9Sp+jB6fwg8EjHERDENOLUWJLN9fTatf7cSVPqbhEguWOTSJ4kYyDShoakUyAXNsl73/u/Q",
|
||||
"9luV6sJ0fS7UPehlIaNEcDA8V1Opd1ihR9h45Np2/gwtUx+YU0Mjy+jEnCORkAoNxDmCkrSIUQq7tNmq",
|
||||
"ZyDD0tGJi0VXVgaqlUmzuXPoZu20g0qZo1chAWlUzP2EslExxw6xaFTdQFG9YuWjV5ySC/rvhKOOyj5E",
|
||||
"OHLAZO2a/owlK/xU0DQ/Nsr/XYLsHyWpDUYmuuRT+FpbYG71nXmoNjnvWmoOWyJFYRV+GsE0TwoXZVky",
|
||||
"4gB8PQab7qxKP6TL8D6iC2YKRtCrQLTVrGwC9KFr2QCuXY8iIRlmTUhpNGccyAHWky0zz9/PgZf1QhhX",
|
||||
"LAZCV0kXXm3HhXGx+A4julRmD6f6JVJUIXbrY0vkGlRbp8BMii9L4voG4Lv4+HKd/67NLnEfJ5aidA6A",
|
||||
"38UBcA3aClh2PzR1/6uM0szprwLBT2UM3IQwz8vSu+KXbwD07L0zAnZOf9/E6W+dIPioSp1DHv3JaQrN",
|
||||
"/P7CpIZMlkX8dcDdzxKdS9tmq3XB0oVqNPdabhf7pXP260zr38rZbxdKNfX024o/TCs/DlXlzheJQC+R",
|
||||
"C3e4+hK9/WoYtuJJPsm6sbNfe6ZnXf2+N8J+T9m+cPaTnYzfyfgdhXtcR7+mAr5zhNllHBsXzXZQqQv6",
|
||||
"haV5SnieTkASMV3pgot6LLnkAaechKVM+1xyGNcww6JdXfbFHyyp3go8tiUDK6D06M+y/Xn89agh0J6s",
|
||||
"+uxO01Ckiz4fEiG38NuoNmiXP/QvA8HnXGUQ6dWDbR2Sd2cSDYAzzbU4zJGAH/2pNNW5+toexI9zLSwX",
|
||||
"GOMQ3x/k+5660sVGid0nOTCDmOmnNFHwyr8AVWyoQ7e/DLqdcjQ0C0lipqzLcgV6bNmidQzcA/eiOUR3",
|
||||
"btT2OIcu1hbpOg7TgfxDEysaaCL3c9Bz9HMrAeEnRVhKZ0Dm1Bn2QJIFSMUEXzll7AH98CUTO72NPXB/",
|
||||
"avt1IN+B/AOpPAJSFdbX1GgI8MpFeMQwZZw5RVFbWE/ETLWH9Pdi9hwlqc33diJmJGEcHuW5vZmRmCXa",
|
||||
"FshXRsxVQGU0d6nqgqE2pk27xMc3LAWlaZqZTUxBR3M7Zc41S0LboUqPIAK2gOK2WsxozTBE8GRJlI5B",
|
||||
"2k0G5gID4Vc8WfommQiRAOUdWfoRnMps+WALfY8mbKaU0xkc/Ukjs9SvQQe0Y/yu3hKlRdYn6ODWL9wz",
|
||||
"++SOJUkfo1YW0CcZzRX0Sc7dPyTYfOF9kheCaUNad4HLs5M/S5pnlpcC14QWa/TMV34LTwU8T9ExUAsM",
|
||||
"BVz5Z7t/mRPGH8wR41HkGBTpDhk/lVnZ3TF/6j9f8aRLwPmC4wdBToVMCSXpOgIQIyA9BmHaKM0UNyvN",
|
||||
"5CEia9Wa4mdISJoWiHqhb5EO2V9ykSpNo/lmkaqpFGn1lbRHLaIOVzu9wQ+DJ8fajyda7MKSZpxwD01B",
|
||||
"pdZXpxXroPtRjOUFNGIkq55D5YeKxszmVeIQaYiJFntAPa7sIcbHMQ7wbA2PBRex+4z/MsbHThR8uURg",
|
||||
"DNqAZs4VrIqXMyVc3Z7HevqtTJ9tpcdnZfd8suKa5frPhEwDQTlug8423VXV7KjCk1GF0s9zHfV/UkSB",
|
||||
"1q500BYqoPMsnLbIKZsUChvANZNA0ORGE1tpjzC+8oCKaJK8LUvxuSJ8/UrJvT4Rmc2HQk4EVxqKqmNO",
|
||||
"XrEl+so2tuPx9TnR4g54WbBvQK54slxPP7BKg2AkIMbJqrIe5oEbhGJcx3gGT1jiDycIJU3Cj0/s9t6R",
|
||||
"ih+YVJS04IrDoZoLXWBBJsWCKSa4NcoG0x+olEp9GJtVbpfzTbshTLusSK2DjcYXx6MbQrWWbJLrqueC",
|
||||
"NSce3xwj6bv8eAEklmwB2xPOKU4zSdmu8nTjotkGcfMdyKoJg+JYzAij4/OhDV150/va1Y/rqNCDdRnC",
|
||||
"Fd8FUsDXZjG1TeDfpdDugL0D9h8G2NcpvTf3QsgM6/o0iSddW9PTRpbuh4gdHnZ4+CivZCFJXGRp8OOi",
|
||||
"Jw68Nf/psK7Dug7rHg3rAozQ5zW57QlksLKZK+MT4mbf79mJLtr5I/gvLnmEnsgyn/T6vSn70uv3gLtg",
|
||||
"EBc39RSOiR2q/wVTDIxyXsVo54J4YICwTxAG+2TKvvSJhcB+Ebj3ajuaW/Pqrlyceg4kyqUEXqq7nBF3",
|
||||
"lW++RmPQX1rkMgLC+FRYTbNVhWHPoF/0ytzbpQWLn12u5JqWrQYIWzWeeSzCRo8RHJodA9dmU6BWpavu",
|
||||
"mZ6bv1il1pMWZCYp1wQSWGAimwMz/CuSgUyZUt7a8IXIaNbxhHUbzPg7ajYUZa9QDWkW7vbxrM0QHRb8",
|
||||
"CJVwTi3CGAxCwEtFDOEaDjnHQiw7PRM+uHZPg1auKMzgwsxRig4+cMFVELeGx09k1Ulcf/HHlQMwSqYs",
|
||||
"AcfhtglWeZYIWlUj+pnfB2yGWe1sjLkZvqjf6BjsgcqzTEityB/ZrE/+yGDWJxmf9cmMTftkkmZ9ohaz",
|
||||
"PrmHSdYnmk2nfUIXbBosYmRnbaI5eUeVTX+HDAs5M/aFGFf6LZLipXmiWUalPpoKmR4WmeBWo9UzxeEx",
|
||||
"bh70ee107SbMo03IlOre296EcYqxphuPyMo+/uVG/+TF+s6JoEto9yKqnrxkHZfB25Jabnt0GIm8QZEJ",
|
||||
"kqDRaIoO39jHPjtKr6giXt/3Wv2gbAa1lsn57DTPIC/fdy11YQ6vK2/R0ZVnFfKRq3pORPt303ofqLWg",
|
||||
"UYSiIpIRT5lJI0jZSpOHE6ogJoJPBJWxdRvzy2tuVU+mubDb2KG7MKsonUKLgpOd1qKjDc1ow3eu2fGC",
|
||||
"E5NVK4isVd4s6FNN6jn6k7uK+VuLh1gHFkO7anRrsiRF93DNEKRIl6tm24uSm+ErY/qejKuvXfWQjtT8",
|
||||
"RcWQsqhJAMt3FDNBNC4KQRf6GpVBxKYM4jBWVx4zLx2lv0fFAfuQ6R4uHcX49hTDllZBxK/zbc8Dhupo",
|
||||
"7tNAx7QslF8QD5VHc0IVgZSyBG33UiSeaipmyGdFOp7ofXQaM73jdXTGIImVVS67yNzuYdTRnh/ZoBwz",
|
||||
"HZZVihfJQiR5Ckd/2v9eNvaq/4jtP5a9dpEV06aonW3nMrjopvFSmUV17C6/y18oy5eTsl0Qt4OWTa/Y",
|
||||
"jfBt23JXdqKPrlUHJz9W0ZxFea9tk79VIeLJUmXYSWqaXN/B2GYrPW6XL6Nzm/k2qssaGoXIbCJmopE7",
|
||||
"8olQqVCFj4zpZrPVX1/+VhqHN6jzezP6TsDG7kcZn9UPdbevSACG/9Z0hm+Wu8A8G90BJvZIvHZ0GSWC",
|
||||
"w5H17D5yKZe3egSOsIcNaTopUjTvS/R2etaNIBVlEUtrcDMsvE9MT+tzXpEZO6fAjro9HXUbnRjIx7zw",
|
||||
"tdqP9ncvQlUNM9teQ1WkGhYPmidDqiv8ssIm86iyu6o9qDpceoa4tM8ziO8FunmabXsG1UDWtO2S2LS6",
|
||||
"mzzNUMqZ5klS3M960qoG15Qwpe21qqa39b7Spbu01u/FGi4pcoAUFPPHLjPAQjd9woV5fsXANaOJetXs",
|
||||
"KhVdtJC9xnQB3fW1y/hJF4A4tz+6VbJ55r5Lymt39KFqJ3hK+dhlxlxx9E4w7gTjb5EmkxP4wpRmfNZS",
|
||||
"xpCAkcmNmJZNfN0xq5duxp4lYkJLYUdLytXUhRerHQBTwoY6ohOR6yZ88qrsc4xd9sjjgi07YtkRyw3Y",
|
||||
"N6v5+fukljhPswSrVEHcHgddvC3Jlfl/xqfCld9zCoADLTRN+iRXEPfJVALskl4XU7Ui5btw8uNUrWh5",
|
||||
"h4x/CTWEAbuPZ2NL5x20YawtxNukBZzCoJz1Bchl0nvbO4pQp937+unr/xcAAP//",
|
||||
}
|
||||
|
||||
// decodeSpec returns the embedded OpenAPI spec as raw JSON bytes,
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
package cosmossdk
|
||||
|
||||
// Version is the SDK version, synced with Cosmos Server at build time.
|
||||
const Version = "0.22.35"
|
||||
const Version = "0.23.00-unstable001"
|
||||
|
||||
@@ -45,6 +45,7 @@ require (
|
||||
golang.org/x/sys v0.47.0
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1
|
||||
gopkg.in/yaml.v2 v2.4.0
|
||||
modernc.org/sqlite v1.56.0
|
||||
)
|
||||
|
||||
require (
|
||||
@@ -262,7 +263,7 @@ require (
|
||||
github.com/magiconair/properties v1.8.10 // indirect
|
||||
github.com/mailru/easyjson v0.9.1 // indirect
|
||||
github.com/mattn/go-colorable v0.1.14 // indirect
|
||||
github.com/mattn/go-isatty v0.0.22 // indirect
|
||||
github.com/mattn/go-isatty v0.0.24 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.19 // indirect
|
||||
github.com/mattn/goveralls v0.0.6 // indirect
|
||||
github.com/mholt/archives v0.1.5 // indirect
|
||||
@@ -284,6 +285,7 @@ require (
|
||||
github.com/nats-io/jwt/v2 v2.5.5 // indirect
|
||||
github.com/nats-io/nkeys v0.4.7 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
github.com/ncruces/go-strftime v1.0.0 // indirect
|
||||
github.com/ncw/swift/v2 v2.0.5 // indirect
|
||||
github.com/nrdcg/auroradns v1.2.0 // indirect
|
||||
github.com/nrdcg/bunny-go v0.1.0 // indirect
|
||||
@@ -337,6 +339,7 @@ require (
|
||||
github.com/rclone/gofakes3 v0.0.4 // indirect
|
||||
github.com/regfish/regfish-dnsapi-go v0.1.1 // indirect
|
||||
github.com/relvacode/iso8601 v1.7.0 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
github.com/rfjakob/eme v1.1.2 // indirect
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
github.com/robfig/cron/v3 v3.0.1 // indirect
|
||||
@@ -444,6 +447,9 @@ require (
|
||||
gotest.tools/v3 v3.4.0 // indirect
|
||||
k8s.io/klog/v2 v2.80.1 // indirect
|
||||
k8s.io/utils v0.0.0-20240102154912-e7106e64919e // indirect
|
||||
modernc.org/libc v1.74.4 // indirect
|
||||
modernc.org/mathutil v1.7.1 // indirect
|
||||
modernc.org/memory v1.11.0 // indirect
|
||||
moul.io/http2curl/v2 v2.3.0 // indirect
|
||||
sigs.k8s.io/yaml v1.6.0 // indirect
|
||||
software.sslmate.com/src/go-pkcs12 v0.7.3 // indirect
|
||||
|
||||
@@ -1700,8 +1700,8 @@ github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLe
|
||||
github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
|
||||
github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
|
||||
github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
|
||||
github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad h1:a6HEuzUHeKH6hwfN/ZoQgRgVIWFJljSWa/zetS2WTvg=
|
||||
github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144=
|
||||
github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo=
|
||||
github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk=
|
||||
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
|
||||
github.com/google/s2a-go v0.1.3/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A=
|
||||
github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0=
|
||||
@@ -2115,8 +2115,8 @@ github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOA
|
||||
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
|
||||
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
|
||||
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw4=
|
||||
github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4=
|
||||
github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI=
|
||||
github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A=
|
||||
github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
|
||||
github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
|
||||
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
|
||||
@@ -2235,6 +2235,8 @@ github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI=
|
||||
github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc=
|
||||
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
||||
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
|
||||
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
|
||||
github.com/ncw/swift/v2 v2.0.5 h1:9o5Gsd7bInAFEqsGPcaUdsboMbqf8lnNtxqWKFT9iz8=
|
||||
github.com/ncw/swift/v2 v2.0.5/go.mod h1:cbAO76/ZwcFrFlHdXPjaqWZ9R7Hdar7HpjRXBfbjigk=
|
||||
github.com/nicksnyder/go-i18n v1.10.0/go.mod h1:HrK7VCrbOvQoUAQ7Vpy7i87N7JZZZ7R2xBGjv0j365Q=
|
||||
@@ -2486,6 +2488,8 @@ github.com/relvacode/iso8601 v1.7.0 h1:BXy+V60stMP6cpswc+a93Mq3e65PfXCgDFfhvNNGr
|
||||
github.com/relvacode/iso8601 v1.7.0/go.mod h1:FlNp+jz+TXpyRqgmM7tnzHHzBnz776kmAH2h3sZCn0I=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/rfjakob/eme v1.1.2 h1:SxziR8msSOElPayZNFfQw4Tjx/Sbaeeh3eRvrHVMUs4=
|
||||
github.com/rfjakob/eme v1.1.2/go.mod h1:cVvpasglm/G3ngEfcfT/Wt0GwhkuO32pf/poW6Nyk1k=
|
||||
github.com/rhnvrm/simples3 v0.5.0/go.mod h1:Y+3vYm2V7Y4VijFoJHHTrja6OgPrJ2cBti8dPGkC3sA=
|
||||
@@ -3975,17 +3979,30 @@ k8s.io/utils v0.0.0-20240102154912-e7106e64919e h1:eQ/4ljkx21sObifjzXwlPKpdGLrCf
|
||||
k8s.io/utils v0.0.0-20240102154912-e7106e64919e/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
|
||||
lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=
|
||||
lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=
|
||||
modernc.org/cc v1.0.0 h1:nPibNuDEx6tvYrUAtvDTTw98rx5juGsa5zuDnKwEEQQ=
|
||||
modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw=
|
||||
modernc.org/cc/v3 v3.36.0/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI=
|
||||
modernc.org/cc/v3 v3.36.2/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI=
|
||||
modernc.org/cc/v3 v3.36.3/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI=
|
||||
modernc.org/cc/v4 v4.29.1 h1:MKgdCV3WykTSPqpVrnxdEDS0HEd2FHpKZDzxzU5LyeI=
|
||||
modernc.org/cc/v4 v4.29.1/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
|
||||
modernc.org/ccgo/v3 v3.0.0-20220428102840-41399a37e894/go.mod h1:eI31LL8EwEBKPpNpA4bU1/i+sKOwOrQy8D87zWUcRZc=
|
||||
modernc.org/ccgo/v3 v3.0.0-20220430103911-bc99d88307be/go.mod h1:bwdAnOoaIt8Ax9YdWGjxWsdkPcZyRPHqrOvJxaKAKGw=
|
||||
modernc.org/ccgo/v3 v3.16.4/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ=
|
||||
modernc.org/ccgo/v3 v3.16.6/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ=
|
||||
modernc.org/ccgo/v3 v3.16.8/go.mod h1:zNjwkizS+fIFDrDjIAgBSCLkWbJuHF+ar3QRn+Z9aws=
|
||||
modernc.org/ccgo/v3 v3.16.9/go.mod h1:zNMzC9A9xeNUepy6KuZBbugn3c0Mc9TeiJO4lgvkJDo=
|
||||
modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU=
|
||||
modernc.org/ccgo/v4 v4.34.6/go.mod h1:SZ8YcN9NG7XVsQYdm6jYBvi8PQP1qi+kqB6OhjqI3Fk=
|
||||
modernc.org/ccorpus v1.11.6/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ=
|
||||
modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM=
|
||||
modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU=
|
||||
modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
|
||||
modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
|
||||
modernc.org/gc/v3 v3.1.4 h1:2g65LGVSmFQrXeITAw97x7hCRvZFcyE1uDP+7Vng7JI=
|
||||
modernc.org/gc/v3 v3.1.4/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
|
||||
modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks=
|
||||
modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI=
|
||||
modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk=
|
||||
modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM=
|
||||
modernc.org/libc v0.0.0-20220428101251-2d5f3daf273b/go.mod h1:p7Mg4+koNjc8jkqwcoFBJx7tXkpj00G77X7A72jXPXA=
|
||||
@@ -3995,21 +4012,37 @@ modernc.org/libc v1.16.17/go.mod h1:hYIV5VZczAmGZAnG15Vdngn5HSF5cSkbvfz2B7GRuVU=
|
||||
modernc.org/libc v1.16.19/go.mod h1:p7Mg4+koNjc8jkqwcoFBJx7tXkpj00G77X7A72jXPXA=
|
||||
modernc.org/libc v1.17.0/go.mod h1:XsgLldpP4aWlPlsjqKRdHPqCxCjISdHfM/yeWC5GyW0=
|
||||
modernc.org/libc v1.17.1/go.mod h1:FZ23b+8LjxZs7XtFMbSzL/EhPxNbfZbErxEHc7cbD9s=
|
||||
modernc.org/libc v1.74.4 h1:fX1Omw4o2/1C2iRkkIsrQTasJQldLhRmuPreXLoWs9k=
|
||||
modernc.org/libc v1.74.4/go.mod h1:eeQAS9W3sZeKYMFubydxJpII9ybHWshk+7or7bLG9co=
|
||||
modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k=
|
||||
modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
|
||||
modernc.org/mathutil v1.4.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
|
||||
modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
|
||||
modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
|
||||
modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
|
||||
modernc.org/memory v1.1.1/go.mod h1:/0wo5ibyrQiaoUoH7f9D8dnglAmILJ5/cxZlRECf+Nw=
|
||||
modernc.org/memory v1.2.0/go.mod h1:/0wo5ibyrQiaoUoH7f9D8dnglAmILJ5/cxZlRECf+Nw=
|
||||
modernc.org/memory v1.2.1/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU=
|
||||
modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI=
|
||||
modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
|
||||
modernc.org/opt v0.1.1/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
|
||||
modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
|
||||
modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg=
|
||||
modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
|
||||
modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
|
||||
modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
|
||||
modernc.org/sqlite v1.18.1/go.mod h1:6ho+Gow7oX5V+OiOQ6Tr4xeqbx13UZ6t+Fw9IRUG4d4=
|
||||
modernc.org/sqlite v1.56.0 h1:/D8e2RfFqoy/Zc6PuC76U28zFwmI/sYx1Kjm4yEn9e0=
|
||||
modernc.org/sqlite v1.56.0/go.mod h1:yCJ2cmAaIkHQ25oXWrF8H4O1lIfPYPR26yCEDj2P3pQ=
|
||||
modernc.org/strutil v1.1.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs=
|
||||
modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw=
|
||||
modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw=
|
||||
modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
|
||||
modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
|
||||
modernc.org/tcl v1.13.1/go.mod h1:XOLfOwzhkljL4itZkK6T72ckMgvj0BDsnKNdZVUOecw=
|
||||
modernc.org/token v1.0.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
|
||||
modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
|
||||
modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
|
||||
modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I=
|
||||
modernc.org/z v1.5.1/go.mod h1:eWFB510QWW5Th9YGZT81s+LwvaAs3Q2yr4sP0rmLkv8=
|
||||
moul.io/http2curl/v2 v2.3.0 h1:9r3JfDzWPcbIklMOs2TnIFzDYvfAZvjeavG6EzP7jYs=
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "cosmos-server",
|
||||
"version": "0.22.35",
|
||||
"version": "0.23.00-unstable001",
|
||||
"description": "",
|
||||
"main": "test-server.js",
|
||||
"bugs": {
|
||||
|
||||
@@ -15,8 +15,8 @@ import (
|
||||
"encoding/json"
|
||||
"net/url"
|
||||
"net/http/httptest"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
|
||||
"errors"
|
||||
|
||||
// "fmt"
|
||||
|
||||
"github.com/ory/fosite/compose"
|
||||
@@ -311,28 +311,22 @@ func detectCallbackEndpoint(w http.ResponseWriter, req *http.Request) {
|
||||
// get user from claims
|
||||
user := claims["sub"].(string)
|
||||
|
||||
c, closeDb, errCo := utils.GetEmbeddedCollection(utils.GetRootAppId(), "users")
|
||||
defer closeDb()
|
||||
if errCo != nil {
|
||||
utils.Error("Database Connect", errCo)
|
||||
utils.HTTPError(w, "Database Error", http.StatusInternalServerError, "DB001")
|
||||
return
|
||||
}
|
||||
|
||||
nickname := utils.Sanitize(user)
|
||||
|
||||
userInBase := utils.User{}
|
||||
|
||||
utils.Debug("UserLogin: Logging user " + nickname)
|
||||
|
||||
err3 := c.FindOne(nil, map[string]interface{}{
|
||||
"Nickname": nickname,
|
||||
}).Decode(&userInBase)
|
||||
userInBase, err3 := utils.GetUser(nickname)
|
||||
|
||||
if err3 == mongo.ErrNoDocuments {
|
||||
if errors.Is(err3, utils.ErrNotFound) {
|
||||
utils.Error("UserLogin: User not found", err3)
|
||||
utils.HTTPError(w, "User Logging Error", http.StatusInternalServerError, "UL001")
|
||||
return
|
||||
} else if err3 != nil {
|
||||
// pre-existing gap fix: without this branch a store error would fall through
|
||||
// towards token issuance guarded only by the zero-value Password check
|
||||
utils.Error("UserLogin: Error while finding user", err3)
|
||||
utils.HTTPError(w, "User Logging Error", http.StatusInternalServerError, "UL001")
|
||||
return
|
||||
} else if userInBase.Password == "" {
|
||||
utils.Error("UserLogin: User not registered", nil)
|
||||
utils.HTTPError(w, "User not registered", http.StatusInternalServerError, "UL002")
|
||||
|
||||
@@ -16,16 +16,7 @@ import (
|
||||
// source of truth shared by the userinfo endpoint and the id_token, so both
|
||||
// always return the same set of claims.
|
||||
func getUserClaims(nickname string, scopes fosite.Arguments) (map[string]interface{}, error) {
|
||||
c, closeDb, errCo := utils.GetEmbeddedCollection(utils.GetRootAppId(), "users")
|
||||
defer closeDb()
|
||||
if errCo != nil {
|
||||
return nil, errCo
|
||||
}
|
||||
|
||||
user := utils.User{}
|
||||
err := c.FindOne(nil, map[string]interface{}{
|
||||
"Nickname": nickname,
|
||||
}).Decode(&user)
|
||||
user, err := utils.GetUser(nickname)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
+23
-22
@@ -188,7 +188,6 @@ func createAPIToken(w http.ResponseWriter, req *http.Request) {
|
||||
}
|
||||
|
||||
utils.ConfigLock.Lock()
|
||||
defer utils.ConfigLock.Unlock()
|
||||
|
||||
config := utils.ReadConfigFromFile()
|
||||
|
||||
@@ -197,15 +196,21 @@ func createAPIToken(w http.ResponseWriter, req *http.Request) {
|
||||
}
|
||||
|
||||
if _, exists := config.APITokens[request.Name]; exists {
|
||||
utils.ConfigLock.Unlock()
|
||||
utils.HTTPError(w, "Token with this name already exists", http.StatusConflict, "AT013")
|
||||
return
|
||||
}
|
||||
|
||||
config.APITokens[request.Name] = tokenConfig
|
||||
utils.SetBaseMainConfig(config)
|
||||
tokens := config.APITokens
|
||||
// the lock must be released before publishing: the apply loop takes it to
|
||||
// install the op, and this handler waits for that apply
|
||||
utils.ConfigLock.Unlock()
|
||||
|
||||
utils.TouchDatabase()
|
||||
go constellation.SendNewDBSyncMessage()
|
||||
if err := constellation.PublishDomainOp(constellation.DomainAPITokens, tokens); err != nil {
|
||||
utils.HTTPStoreError(w, err, "AT015")
|
||||
return
|
||||
}
|
||||
|
||||
utils.TriggerEvent(
|
||||
"cosmos.api.token.created",
|
||||
@@ -266,25 +271,23 @@ func deleteAPIToken(w http.ResponseWriter, req *http.Request) {
|
||||
}
|
||||
|
||||
utils.ConfigLock.Lock()
|
||||
defer utils.ConfigLock.Unlock()
|
||||
|
||||
config := utils.ReadConfigFromFile()
|
||||
|
||||
if config.APITokens == nil {
|
||||
utils.HTTPError(w, "Token not found", http.StatusNotFound, "AT022")
|
||||
return
|
||||
}
|
||||
|
||||
if _, exists := config.APITokens[request.Name]; !exists {
|
||||
utils.ConfigLock.Unlock()
|
||||
utils.HTTPError(w, "Token not found", http.StatusNotFound, "AT022")
|
||||
return
|
||||
}
|
||||
|
||||
delete(config.APITokens, request.Name)
|
||||
utils.SetBaseMainConfig(config)
|
||||
tokens := config.APITokens
|
||||
utils.ConfigLock.Unlock()
|
||||
|
||||
utils.TouchDatabase()
|
||||
go constellation.SendNewDBSyncMessage()
|
||||
if err := constellation.PublishDomainOp(constellation.DomainAPITokens, tokens); err != nil {
|
||||
utils.HTTPStoreError(w, err, "AT023")
|
||||
return
|
||||
}
|
||||
|
||||
utils.TriggerEvent(
|
||||
"cosmos.api.token.deleted",
|
||||
@@ -344,17 +347,12 @@ func updateAPIToken(w http.ResponseWriter, req *http.Request) {
|
||||
}
|
||||
|
||||
utils.ConfigLock.Lock()
|
||||
defer utils.ConfigLock.Unlock()
|
||||
|
||||
config := utils.ReadConfigFromFile()
|
||||
|
||||
if config.APITokens == nil {
|
||||
utils.HTTPError(w, "Token not found", http.StatusNotFound, "AT032")
|
||||
return
|
||||
}
|
||||
|
||||
token, exists := config.APITokens[name]
|
||||
if !exists {
|
||||
utils.ConfigLock.Unlock()
|
||||
utils.HTTPError(w, "Token not found", http.StatusNotFound, "AT032")
|
||||
return
|
||||
}
|
||||
@@ -374,10 +372,13 @@ func updateAPIToken(w http.ResponseWriter, req *http.Request) {
|
||||
}
|
||||
|
||||
config.APITokens[name] = token
|
||||
utils.SetBaseMainConfig(config)
|
||||
tokens := config.APITokens
|
||||
utils.ConfigLock.Unlock()
|
||||
|
||||
utils.TouchDatabase()
|
||||
go constellation.SendNewDBSyncMessage()
|
||||
if err := constellation.PublishDomainOp(constellation.DomainAPITokens, tokens); err != nil {
|
||||
utils.HTTPStoreError(w, err, "AT033")
|
||||
return
|
||||
}
|
||||
|
||||
utils.TriggerEvent(
|
||||
"cosmos.api.token.updated",
|
||||
|
||||
+14
-10
@@ -43,9 +43,6 @@ func ConfigApiDNS(w http.ResponseWriter, req *http.Request) {
|
||||
|
||||
utils.Log("DNSConfigUpdate: Updating DNS configuration")
|
||||
|
||||
utils.ConfigLock.Lock()
|
||||
defer utils.ConfigLock.Unlock()
|
||||
|
||||
var updateReq DNSConfigRequest
|
||||
err := json.NewDecoder(req.Body).Decode(&updateReq)
|
||||
if err != nil {
|
||||
@@ -54,6 +51,8 @@ func ConfigApiDNS(w http.ResponseWriter, req *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
utils.ConfigLock.Lock()
|
||||
|
||||
config := utils.ReadConfigFromFile()
|
||||
|
||||
if updateReq.DNSPort != "" {
|
||||
@@ -72,11 +71,20 @@ func ConfigApiDNS(w http.ResponseWriter, req *http.Request) {
|
||||
config.ConstellationConfig.CustomDNSEntries = updateReq.CustomDNSEntries
|
||||
}
|
||||
|
||||
utils.SetBaseMainConfig(config)
|
||||
c := config.ConstellationConfig
|
||||
// released before publishing: the apply loop takes ConfigLock to install this
|
||||
utils.ConfigLock.Unlock()
|
||||
|
||||
err = utils.TouchDatabase()
|
||||
err = constellation.PublishDomainOp(constellation.DomainDNS, constellation.DNSPayload{
|
||||
DNSPort: c.DNSPort,
|
||||
DNSFallback: c.DNSFallback,
|
||||
DNSBlockBlacklist: c.DNSBlockBlacklist,
|
||||
DNSAdditionalBlocklists: c.DNSAdditionalBlocklists,
|
||||
CustomDNSEntries: c.CustomDNSEntries,
|
||||
})
|
||||
if err != nil {
|
||||
utils.Error("DNSConfigUpdate: Failed to touch database", err)
|
||||
utils.HTTPStoreError(w, err, "DNS003")
|
||||
return
|
||||
}
|
||||
|
||||
utils.TriggerEvent(
|
||||
@@ -87,10 +95,6 @@ func ConfigApiDNS(w http.ResponseWriter, req *http.Request) {
|
||||
map[string]interface{}{},
|
||||
)
|
||||
|
||||
go func() {
|
||||
constellation.SendNewDBSyncMessage()
|
||||
}()
|
||||
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"status": "OK",
|
||||
})
|
||||
|
||||
+26
-10
@@ -4,6 +4,7 @@ import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
|
||||
"github.com/azukaar/cosmos-server/src/constellation"
|
||||
"github.com/azukaar/cosmos-server/src/utils"
|
||||
"github.com/gorilla/mux"
|
||||
"golang.org/x/crypto/bcrypt"
|
||||
@@ -130,13 +131,13 @@ func createOpenIDClient(w http.ResponseWriter, req *http.Request) {
|
||||
}
|
||||
|
||||
utils.ConfigLock.Lock()
|
||||
defer utils.ConfigLock.Unlock()
|
||||
|
||||
config := utils.ReadConfigFromFile()
|
||||
clients := config.OpenIDClients
|
||||
|
||||
for _, client := range clients {
|
||||
if client.ID == newClient.ID {
|
||||
utils.ConfigLock.Unlock()
|
||||
utils.HTTPError(w, "OpenID client with this ID already exists", http.StatusConflict, "OID004")
|
||||
return
|
||||
}
|
||||
@@ -146,14 +147,21 @@ func createOpenIDClient(w http.ResponseWriter, req *http.Request) {
|
||||
plainSecret := utils.GenerateRandomString(32)
|
||||
hashedSecret, err := bcrypt.GenerateFromPassword([]byte(plainSecret), bcrypt.DefaultCost)
|
||||
if err != nil {
|
||||
utils.ConfigLock.Unlock()
|
||||
utils.Error("CreateOpenIDClient: Failed to hash secret", err)
|
||||
utils.HTTPError(w, "Failed to generate secret", http.StatusInternalServerError, "OID008")
|
||||
return
|
||||
}
|
||||
newClient.Secret = string(hashedSecret)
|
||||
|
||||
config.OpenIDClients = append(config.OpenIDClients, newClient)
|
||||
utils.SetBaseMainConfig(config)
|
||||
clients = append(clients, newClient)
|
||||
// released before publishing: the apply loop takes ConfigLock to install this
|
||||
utils.ConfigLock.Unlock()
|
||||
|
||||
if err := constellation.PublishDomainOp(constellation.DomainOpenIDClients, clients); err != nil {
|
||||
utils.HTTPStoreError(w, err, "OID009")
|
||||
return
|
||||
}
|
||||
|
||||
utils.Log("OpenID client created: " + newClient.ID)
|
||||
|
||||
@@ -207,7 +215,6 @@ func updateOpenIDClient(w http.ResponseWriter, req *http.Request) {
|
||||
}
|
||||
|
||||
utils.ConfigLock.Lock()
|
||||
defer utils.ConfigLock.Unlock()
|
||||
|
||||
config := utils.ReadConfigFromFile()
|
||||
clients := config.OpenIDClients
|
||||
@@ -221,13 +228,18 @@ func updateOpenIDClient(w http.ResponseWriter, req *http.Request) {
|
||||
}
|
||||
|
||||
if clientIndex == -1 {
|
||||
utils.ConfigLock.Unlock()
|
||||
utils.HTTPError(w, "OpenID client not found", http.StatusNotFound, "OID006")
|
||||
return
|
||||
}
|
||||
|
||||
clients[clientIndex] = updatedClient
|
||||
config.OpenIDClients = clients
|
||||
utils.SetBaseMainConfig(config)
|
||||
utils.ConfigLock.Unlock()
|
||||
|
||||
if err := constellation.PublishDomainOp(constellation.DomainOpenIDClients, clients); err != nil {
|
||||
utils.HTTPStoreError(w, err, "OID009")
|
||||
return
|
||||
}
|
||||
|
||||
utils.Log("OpenID client updated: " + id)
|
||||
|
||||
@@ -265,7 +277,6 @@ func deleteOpenIDClient(w http.ResponseWriter, req *http.Request) {
|
||||
id := mux.Vars(req)["id"]
|
||||
|
||||
utils.ConfigLock.Lock()
|
||||
defer utils.ConfigLock.Unlock()
|
||||
|
||||
config := utils.ReadConfigFromFile()
|
||||
clients := config.OpenIDClients
|
||||
@@ -279,14 +290,19 @@ func deleteOpenIDClient(w http.ResponseWriter, req *http.Request) {
|
||||
}
|
||||
|
||||
if clientIndex == -1 {
|
||||
utils.ConfigLock.Unlock()
|
||||
utils.HTTPError(w, "OpenID client not found", http.StatusNotFound, "OID007")
|
||||
return
|
||||
}
|
||||
|
||||
clients = append(clients[:clientIndex], clients[clientIndex+1:]...)
|
||||
config.OpenIDClients = clients
|
||||
utils.SetBaseMainConfig(config)
|
||||
|
||||
utils.ConfigLock.Unlock()
|
||||
|
||||
if err := constellation.PublishDomainOp(constellation.DomainOpenIDClients, clients); err != nil {
|
||||
utils.HTTPStoreError(w, err, "OID009")
|
||||
return
|
||||
}
|
||||
|
||||
utils.Log("OpenID client deleted: " + id)
|
||||
|
||||
utils.TriggerEvent(
|
||||
|
||||
+56
-1
@@ -3,9 +3,47 @@ package configapi
|
||||
import (
|
||||
"net/http"
|
||||
"encoding/json"
|
||||
"github.com/azukaar/cosmos-server/src/constellation"
|
||||
"github.com/azukaar/cosmos-server/src/utils"
|
||||
)
|
||||
|
||||
// publishReplicatedDomains publishes the three op-log domains this whole-config
|
||||
// PUT can touch. Full-state ops are idempotent, so publishing all three
|
||||
// unconditionally is cheaper than diffing and cannot miss a change. APITokens and
|
||||
// the auth keypair are restored from disk above, so they are not editable here.
|
||||
func publishReplicatedDomains(request utils.Config) error {
|
||||
c := request.ConstellationConfig
|
||||
err := constellation.PublishDomainOp(constellation.DomainDNS, constellation.DNSPayload{
|
||||
DNSPort: c.DNSPort,
|
||||
DNSFallback: c.DNSFallback,
|
||||
DNSBlockBlacklist: c.DNSBlockBlacklist,
|
||||
DNSAdditionalBlocklists: c.DNSAdditionalBlocklists,
|
||||
CustomDNSEntries: c.CustomDNSEntries,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := constellation.PublishDomainOp(constellation.DomainRoles, request.Roles); err != nil {
|
||||
return err
|
||||
}
|
||||
return constellation.PublishDomainOp(constellation.DomainOpenIDClients, request.OpenIDClients)
|
||||
}
|
||||
|
||||
// restoreReplicatedDomains takes the replicated fields from the freshly applied
|
||||
// config rather than from the request, so the local write can never roll back
|
||||
// what the apply loop just installed. Same treatment the auth keypair and API
|
||||
// tokens already get. Every OTHER ConstellationConfig field (Enabled, IPRange,
|
||||
// ThisDeviceName, DNSDisabled...) stays node-local from the request by design.
|
||||
func restoreReplicatedDomains(request *utils.Config, config utils.Config) {
|
||||
request.ConstellationConfig.DNSPort = config.ConstellationConfig.DNSPort
|
||||
request.ConstellationConfig.DNSFallback = config.ConstellationConfig.DNSFallback
|
||||
request.ConstellationConfig.DNSBlockBlacklist = config.ConstellationConfig.DNSBlockBlacklist
|
||||
request.ConstellationConfig.DNSAdditionalBlocklists = config.ConstellationConfig.DNSAdditionalBlocklists
|
||||
request.ConstellationConfig.CustomDNSEntries = config.ConstellationConfig.CustomDNSEntries
|
||||
request.Roles = config.Roles
|
||||
request.OpenIDClients = config.OpenIDClients
|
||||
}
|
||||
|
||||
// ConfigApiSet godoc
|
||||
// @Summary Update server configuration
|
||||
// @Description Replaces the entire server configuration (masked credential fields are preserved from existing config)
|
||||
@@ -100,8 +138,25 @@ func ConfigApiSet(w http.ResponseWriter, req *http.Request) {
|
||||
}
|
||||
}
|
||||
|
||||
// Replicated domains go through the op-log, never straight to disk. Published
|
||||
// BEFORE the local write so a read-only node fails here with nothing written,
|
||||
// rather than leaving a node-local edit the log will never hear about and the
|
||||
// next snapshot will silently discard. ConfigLock must NOT be held across
|
||||
// these — the apply loop takes it to install the op we are waiting on.
|
||||
if err := publishReplicatedDomains(request); err != nil {
|
||||
utils.HTTPStoreError(w, err, "UC004")
|
||||
return
|
||||
}
|
||||
|
||||
// The local write is the node-local half. Under the lock so it cannot
|
||||
// interleave with an apply-loop domain install, and re-reading here picks up
|
||||
// the state that loop just applied.
|
||||
utils.ConfigLock.Lock()
|
||||
config = utils.ReadConfigFromFile()
|
||||
restoreReplicatedDomains(&request, config)
|
||||
utils.SetBaseMainConfig(request)
|
||||
|
||||
utils.ConfigLock.Unlock()
|
||||
|
||||
utils.TriggerEvent(
|
||||
"cosmos.settings",
|
||||
"Settings updated",
|
||||
|
||||
+56
-40
@@ -31,6 +31,36 @@ func matchesDomain(qName string, hostname string) bool {
|
||||
return qName == hostname + "." || strings.HasSuffix(qName, "." + hostname + ".")
|
||||
}
|
||||
|
||||
// isAddrQuery reports whether q asks for an address record. AAAA is claimed and
|
||||
// answered empty rather than forwarded, so a client can't bypass us over IPv6.
|
||||
func isAddrQuery(q dns.Question) bool {
|
||||
return q.Qtype == dns.TypeA || q.Qtype == dns.TypeAAAA
|
||||
}
|
||||
|
||||
// answerA appends an A record, skipping malformed IPs instead of packing a nil
|
||||
// RR, and skipping duplicates so overlapping hostnames only answer once
|
||||
func answerA(m *dns.Msg, name string, ip string) {
|
||||
ip = cleanIp(ip)
|
||||
if ip == "" {
|
||||
utils.Error("DNS: no IP to answer " + name + " with", nil)
|
||||
return
|
||||
}
|
||||
|
||||
for _, existing := range m.Answer {
|
||||
if a, ok := existing.(*dns.A); ok && a.Hdr.Name == name && a.A.String() == ip {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
rr, err := dns.NewRR(name + " A " + ip)
|
||||
if err != nil {
|
||||
utils.Error("DNS: bad A record for "+name+" -> "+ip, err)
|
||||
return
|
||||
}
|
||||
|
||||
m.Answer = append(m.Answer, rr)
|
||||
}
|
||||
|
||||
// isBlacklisted reports whether domain or any of its parent domains is in DNSBlacklist,
|
||||
// so a blacklisted "example.com" also blocks "ads.example.com"
|
||||
func isBlacklisted(domain string) bool {
|
||||
@@ -81,17 +111,20 @@ func handleDNSRequest(w dns.ResponseWriter, r *dns.Msg) {
|
||||
hostname := entry.Key
|
||||
ip := entry.Value
|
||||
|
||||
if matchesDomain(q.Name, hostname) && q.Qtype == dns.TypeA {
|
||||
utils.Debug("DNS Overwrite " + hostname + " with " + ip)
|
||||
rr, _ := dns.NewRR(q.Name + " A " + ip)
|
||||
m.Answer = append(m.Answer, rr)
|
||||
if matchesDomain(q.Name, hostname) && isAddrQuery(q) {
|
||||
if q.Qtype == dns.TypeA {
|
||||
utils.Debug("DNS Overwrite " + hostname + " with " + ip)
|
||||
answerA(m, q.Name, ip)
|
||||
}
|
||||
customHandled = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Overwrite local hostnames with their Constellation IP. Prioritize local URLs over tunnels
|
||||
// Overwrite local hostnames with their Constellation IP. hostnames already
|
||||
// includes the tunnel routes this node load-balances, so a server that was
|
||||
// selected by DNS always answers for itself and keeps the request local.
|
||||
if !customHandled {
|
||||
thisIp, err := GetCurrentDeviceIP()
|
||||
if err != nil {
|
||||
@@ -100,40 +133,16 @@ func handleDNSRequest(w dns.ResponseWriter, r *dns.Msg) {
|
||||
for _, q := range r.Question {
|
||||
utils.Debug("DNS Question " + q.Name)
|
||||
for _, hostname := range hostnames {
|
||||
if matchesDomain(q.Name, hostname) && q.Qtype == dns.TypeA {
|
||||
utils.Debug("DNS Overwrite " + hostname + " with " + thisIp)
|
||||
rr, _ := dns.NewRR(q.Name + " A " + thisIp)
|
||||
m.Answer = append(m.Answer, rr)
|
||||
if matchesDomain(q.Name, hostname) && isAddrQuery(q) {
|
||||
if q.Qtype == dns.TypeA {
|
||||
utils.Debug("DNS Overwrite " + hostname + " with " + thisIp)
|
||||
answerA(m, q.Name, thisIp)
|
||||
}
|
||||
customHandled = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
remoteTunnels := GetLocalTunnelCache()
|
||||
currentName, err := GetCurrentDeviceName()
|
||||
if err != nil {
|
||||
utils.Error("[constellation] Failed to get current device name for DNS handling", err)
|
||||
} else {
|
||||
for _, q := range r.Question {
|
||||
for _, tunnel := range remoteTunnels {
|
||||
for _, target := range tunnel.Targets {
|
||||
if target.DeviceName == currentName {
|
||||
continue
|
||||
}
|
||||
destination := GetDeviceIp(target.DeviceName)
|
||||
if destination != "" {
|
||||
if matchesDomain(q.Name, tunnel.Route.Host) && q.Qtype == dns.TypeA {
|
||||
utils.Debug("DNS Overwrite " + tunnel.Route.Host + " with " + destination)
|
||||
rr, _ := dns.NewRR(q.Name + " A " + destination)
|
||||
m.Answer = append(m.Answer, rr)
|
||||
customHandled = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !customHandled {
|
||||
@@ -144,10 +153,11 @@ func handleDNSRequest(w dns.ResponseWriter, r *dns.Msg) {
|
||||
for deviceName, ip := range cachedNames {
|
||||
procDeviceName := strings.ReplaceAll(deviceName, " ", "-")
|
||||
|
||||
if matchesDomain(q.Name, procDeviceName) && q.Qtype == dns.TypeA {
|
||||
utils.Debug("DNS Overwrite " + procDeviceName + " with its IP")
|
||||
rr, _ := dns.NewRR(q.Name + " A " + ip)
|
||||
m.Answer = append(m.Answer, rr)
|
||||
if matchesDomain(q.Name, procDeviceName) && isAddrQuery(q) {
|
||||
if q.Qtype == dns.TypeA {
|
||||
utils.Debug("DNS Overwrite " + procDeviceName + " with its IP")
|
||||
answerA(m, q.Name, ip)
|
||||
}
|
||||
customHandled = true
|
||||
}
|
||||
}
|
||||
@@ -161,8 +171,7 @@ func handleDNSRequest(w dns.ResponseWriter, r *dns.Msg) {
|
||||
if isBlacklisted(noDot) {
|
||||
if q.Qtype == dns.TypeA {
|
||||
utils.Debug("DNS Block " + noDot)
|
||||
rr, _ := dns.NewRR(q.Name + " A 0.0.0.0")
|
||||
m.Answer = append(m.Answer, rr)
|
||||
answerA(m, q.Name, "0.0.0.0")
|
||||
}
|
||||
|
||||
customHandled = true
|
||||
@@ -247,6 +256,13 @@ func StopDNS() {
|
||||
}
|
||||
}
|
||||
|
||||
// ReloadDNS rebinds the DNS server against the current config. Reaction of the
|
||||
// dns op-log domain, replacing the old restart-everything sync.
|
||||
func ReloadDNS() {
|
||||
StopDNS()
|
||||
InitDNS()
|
||||
}
|
||||
|
||||
func InitDNS() {
|
||||
dnsMux.Lock()
|
||||
if dnsStarting || DNSStarted || dnsServer != nil {
|
||||
|
||||
+235
-77
@@ -9,6 +9,7 @@ import (
|
||||
"io/ioutil"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -58,7 +59,8 @@ type NodeHeartbeat struct {
|
||||
Tags []string `json:"tags,omitempty"`
|
||||
}
|
||||
|
||||
var ns *server.Server
|
||||
// atomic: written under natsStartMutex, but read lock-free by status/ping paths
|
||||
var ns atomic.Pointer[server.Server]
|
||||
|
||||
func truncateLog(s string) string {
|
||||
if len(s) > 100 {
|
||||
@@ -88,6 +90,35 @@ func sanitizeNATSUsername(username string) string {
|
||||
return username
|
||||
}
|
||||
|
||||
// the mobile app's only subject (see MasterNATSClientRouter)
|
||||
const publicDevicesSubject = "cosmos._global_.constellation.public-devices"
|
||||
|
||||
// natsUserPermissions returns a device's NATS subject rules.
|
||||
//
|
||||
// Cosmos servers get none: their traffic already crosses a leafnode link whose
|
||||
// users carry no permissions, so the allow-list only ever handicapped
|
||||
// manager-local callers relative to agents — which is how the missing $JS.FC.>
|
||||
// publish hid, stalling ordered consumers and KV watches under backlog.
|
||||
//
|
||||
// CosmosNode 0 is the mobile app: one request, one inbox. No publish on
|
||||
// _INBOX.> — the server checks the published subject, never the reply-to — and
|
||||
// no cosmos._global_.>, which would let a phone write the op-log stream.
|
||||
func natsUserPermissions(device utils.ConstellationDevice) *server.Permissions {
|
||||
if device.CosmosNode > 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
return &server.Permissions{
|
||||
Publish: &server.SubjectPermission{
|
||||
Allow: []string{publicDevicesSubject},
|
||||
},
|
||||
Subscribe: &server.SubjectPermission{
|
||||
// the app subscribes to _INBOX.<nuid>.>, not a literal inbox
|
||||
Allow: []string{"_INBOX.>"},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func GetClusterIPs() ([]*url.URL, error) {
|
||||
ipsMap := make(map[string]bool)
|
||||
|
||||
@@ -126,6 +157,20 @@ func GetClusterIPs() ([]*url.URL, error) {
|
||||
// natsLeafPort is the port managers listen on for agent leafnode connections.
|
||||
const natsLeafPort = 7422
|
||||
|
||||
// jetstreamDir is THE JetStream store directory — one expression, used by the
|
||||
// server option and by every path that deletes it.
|
||||
//
|
||||
// CONFIGFOLDER is copied verbatim out of COSMOS_CONFIG_FOLDER with no
|
||||
// trailing-slash normalization (index.go), so string concatenation gives two
|
||||
// different answers depending on the spelling: with COSMOS_CONFIG_FOLDER=/config,
|
||||
// `CONFIGFOLDER + "/jetstream"` is /config/jetstream while `CONFIGFOLDER +
|
||||
// "jetstream"` is /configjetstream. The second silently RemoveAll's a path that
|
||||
// does not exist and reports success, which would leave the abandoned cluster's
|
||||
// state in place through a force-reform — the one thing the reform must not do.
|
||||
func jetstreamDir() string {
|
||||
return filepath.Join(utils.CONFIGFOLDER, "jetstream")
|
||||
}
|
||||
|
||||
// readCstlnConfigField returns a cstln_* field from this node's nebula.yml;
|
||||
// the creator's own nebula.yml lacks these (its values live in the main config).
|
||||
func readCstlnConfigField(key string) interface{} {
|
||||
@@ -167,9 +212,51 @@ func getSeedManagerIPs() []string {
|
||||
return seeds
|
||||
}
|
||||
|
||||
// blockedDeviceIPs returns the nebula IPs that belong ONLY to blocked devices —
|
||||
// read from the store rather than the cache, since refreshDeviceCache drops
|
||||
// blocked devices and so can say who is live but never who was removed.
|
||||
//
|
||||
// Subtracting the live set is not a refinement, it is the whole correctness of
|
||||
// this function. An IP is not retired with its device: GetNextAvailableIP
|
||||
// allocates out of the unblocked devices only, so the replacement manager
|
||||
// enrolled after a force-reform normally takes the dead manager's address back.
|
||||
// Keyed on IP alone, this would then remove the LIVE replacement from the
|
||||
// cluster route list and JetStream would never elect — turning the reform into
|
||||
// the outage it was meant to repair. The schema allows the overlap on purpose
|
||||
// (UNIQUE(ip) WHERE blocked=0).
|
||||
func blockedDeviceIPs() map[string]bool {
|
||||
blocked := map[string]bool{}
|
||||
devices, err := utils.FindDevices(map[string]interface{}{"Blocked": true})
|
||||
if err != nil {
|
||||
utils.Debug("[NATS] Could not read blocked devices: " + err.Error())
|
||||
return blocked
|
||||
}
|
||||
for _, d := range devices {
|
||||
if d.IP != "" {
|
||||
blocked[cleanIp(d.IP)] = true
|
||||
}
|
||||
}
|
||||
if len(blocked) == 0 {
|
||||
return blocked
|
||||
}
|
||||
|
||||
live, err := utils.ListDevices(false)
|
||||
if err != nil {
|
||||
// Fail open: subtracting nothing leaves a stale route, which the epoch fence
|
||||
// still makes harmless. Subtracting wrongly drops a live manager out of the
|
||||
// cluster, which nothing else protects against.
|
||||
utils.Debug("[NATS] Could not read live devices, keeping every route: " + err.Error())
|
||||
return map[string]bool{}
|
||||
}
|
||||
for _, d := range live {
|
||||
delete(blocked, cleanIp(d.IP))
|
||||
}
|
||||
return blocked
|
||||
}
|
||||
|
||||
// getManagerIPs returns the nebula IPs of every known Manager node
|
||||
// (CosmosNode == 2), excluding excludeIP: device cache ∪ enrollment seeds,
|
||||
// falling back to the nebula lighthouses when both are empty.
|
||||
// (CosmosNode == 2), excluding excludeIP: device cache union enrollment seeds, minus
|
||||
// anything blocked, falling back to the nebula lighthouses when both are empty.
|
||||
func getManagerIPs(excludeIP string) []string {
|
||||
ipsMap := map[string]bool{}
|
||||
cachedDevices, _ := deviceCacheSnapshot()
|
||||
@@ -189,6 +276,17 @@ func getManagerIPs(excludeIP string) []string {
|
||||
}
|
||||
delete(ipsMap, cleanIp(excludeIP))
|
||||
|
||||
// The seeds above are a bootstrap hint frozen into nebula.yml at enrollment
|
||||
// and never rewritten, so on their own they can only ever ADD a manager and
|
||||
// never retire one. Subtracting the blocked set is what lets this list shrink
|
||||
// — which force-reform depends on, since these IPs become the cluster route
|
||||
// list and "routes must never point at non-voters" (see StartNATS): a reform
|
||||
// that wiped the store dir and then rebuilt the meta group around the two
|
||||
// managers it just fenced would have isolated nothing.
|
||||
for ip := range blockedDeviceIPs() {
|
||||
delete(ipsMap, ip)
|
||||
}
|
||||
|
||||
ips := make([]string, 0, len(ipsMap))
|
||||
for ip := range ipsMap {
|
||||
ips = append(ips, ip)
|
||||
@@ -265,11 +363,11 @@ func StartNATS() {
|
||||
natsStartMutex.Lock()
|
||||
defer natsStartMutex.Unlock()
|
||||
|
||||
if ns != nil {
|
||||
if ns.Load() != nil {
|
||||
return
|
||||
}
|
||||
|
||||
natsStartTime = time.Now()
|
||||
setNATSStartTime(time.Now())
|
||||
|
||||
ip, err := GetCurrentDeviceIP()
|
||||
if err != nil {
|
||||
@@ -329,45 +427,10 @@ func StartNATS() {
|
||||
seenUsers[username] = true
|
||||
utils.Debug("[NATS] Adding NATS user for device: " + devices.DeviceName + " With API Key: " + redactSecret(devices.APIKey))
|
||||
|
||||
// TODO: Agent / users with less permissions
|
||||
|
||||
users = append(users, &server.User{
|
||||
Username: username,
|
||||
Password: devices.APIKey,
|
||||
Permissions: &server.Permissions{
|
||||
Publish: &server.SubjectPermission{
|
||||
Allow: []string{
|
||||
"cosmos." + username + ".>", "_INBOX.>",
|
||||
"cosmos._global_.>",
|
||||
"_INBOX.>",
|
||||
// Scheduler: leader publishes per-target deployment commands
|
||||
// to cosmos.<target>.deployments.command. Scoped so non-leaders
|
||||
// can't fabricate arbitrary cross-node traffic.
|
||||
"cosmos.*.deployments.>",
|
||||
"$KV.constellation-nodes.>",
|
||||
"$KV.constellation-deployments.>",
|
||||
// Scheduler leader lease lives in its own bucket (see
|
||||
// src/pro scheduler election)
|
||||
"$KV.constellation-leader.>",
|
||||
"$JS.API.STREAM.INFO.>",
|
||||
"$JS.API.>",
|
||||
},
|
||||
},
|
||||
Subscribe: &server.SubjectPermission{
|
||||
Allow: []string{
|
||||
"cosmos." + username + ".>", "_INBOX.>",
|
||||
"cosmos._global_.>",
|
||||
"_INBOX.>",
|
||||
"$KV.constellation-nodes.>",
|
||||
"$KV.constellation-deployments.>",
|
||||
// Scheduler leader lease lives in its own bucket (see
|
||||
// src/pro scheduler election)
|
||||
"$KV.constellation-leader.>",
|
||||
"$JS.API.STREAM.INFO.>",
|
||||
"$JS.API.>",
|
||||
},
|
||||
},
|
||||
},
|
||||
Username: username,
|
||||
Password: devices.APIKey,
|
||||
Permissions: natsUserPermissions(devices),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -406,7 +469,7 @@ func StartNATS() {
|
||||
ServerName: natsName,
|
||||
|
||||
JetStream: !isAgent,
|
||||
StoreDir: utils.CONFIGFOLDER + "/jetstream",
|
||||
StoreDir: jetstreamDir(),
|
||||
|
||||
// Lets agents reach the manager's JetStream over leafnodes: nats-server
|
||||
// otherwise denies $JS.API.> across account leaf links. Enforced
|
||||
@@ -509,7 +572,7 @@ func StartNATS() {
|
||||
} else {
|
||||
// Clear clustered meta state ($SYS raft groups) left by older
|
||||
// versions so it cannot confuse a non-clustered start.
|
||||
os.RemoveAll(utils.CONFIGFOLDER + "/jetstream/$SYS")
|
||||
os.RemoveAll(filepath.Join(jetstreamDir(), "$SYS"))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -519,33 +582,35 @@ func StartNATS() {
|
||||
|
||||
for err != nil && retries < 5 {
|
||||
// drop any instance left over from a failed attempt
|
||||
if ns != nil {
|
||||
ns.Shutdown()
|
||||
ns.WaitForShutdown()
|
||||
ns = nil
|
||||
if old := ns.Load(); old != nil {
|
||||
old.Shutdown()
|
||||
old.WaitForShutdown()
|
||||
ns.Store(nil)
|
||||
}
|
||||
|
||||
ns, err = server.NewServer(opts)
|
||||
var srv *server.Server
|
||||
srv, err = server.NewServer(opts)
|
||||
if err != nil {
|
||||
retries++
|
||||
continue
|
||||
}
|
||||
ns.Store(srv)
|
||||
|
||||
if utils.LoggingLevelLabels[utils.GetMainConfig().LoggingLevel] == utils.DEBUG {
|
||||
ns.SetLogger(natsLogAdapter{}, true, true)
|
||||
srv.SetLogger(natsLogAdapter{}, true, true)
|
||||
}
|
||||
|
||||
if !NebulaStarted.Load() {
|
||||
utils.Error("[NATS] Nebula not started, aborting NATS server setup", nil)
|
||||
ns.Shutdown()
|
||||
ns = nil
|
||||
srv.Shutdown()
|
||||
ns.Store(nil)
|
||||
return
|
||||
}
|
||||
|
||||
go ns.Start()
|
||||
go srv.Start()
|
||||
|
||||
// Wait for the server to be ready
|
||||
if !ns.ReadyForConnections(time.Duration(2*(retries+1)) * time.Second) {
|
||||
if !srv.ReadyForConnections(time.Duration(2*(retries+1)) * time.Second) {
|
||||
retries++
|
||||
utils.Debug("[NATS] NATS server not ready...")
|
||||
err = errors.New("NATS server not ready")
|
||||
@@ -556,10 +621,10 @@ func StartNATS() {
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
if ns != nil {
|
||||
ns.Shutdown()
|
||||
ns.WaitForShutdown()
|
||||
ns = nil
|
||||
if old := ns.Load(); old != nil {
|
||||
old.Shutdown()
|
||||
old.WaitForShutdown()
|
||||
ns.Store(nil)
|
||||
}
|
||||
utils.MajorError("[NATS] Error starting NATS server", err)
|
||||
// hand off to the watchdog so a failed bring-up gets retried (standalone nodes don't need a server)
|
||||
@@ -632,23 +697,44 @@ func natsServerWatchdog() {
|
||||
}
|
||||
}
|
||||
|
||||
// natsStartTime marks the last constellation NATS bring-up; within
|
||||
// natsStartGracePeriod the UI reports failing steps as still starting.
|
||||
var natsStartTime time.Time
|
||||
// natsStartTimeNano marks the last constellation NATS bring-up (UnixNano, 0 =
|
||||
// never); within natsStartGracePeriod the UI reports failing steps as still
|
||||
// starting.
|
||||
//
|
||||
// Atomic rather than a time.Time: StartNATS writes it while GetNATSStatus reads
|
||||
// it from HTTP handlers, and a time.Time is multi-word (wall, ext, loc), so that
|
||||
// pairing is an unconditional data race, not a narrow interleaving. Polling
|
||||
// status during a nebula bounce makes the overlap structural.
|
||||
var natsStartTimeNano atomic.Int64
|
||||
|
||||
func setNATSStartTime(t time.Time) {
|
||||
natsStartTimeNano.Store(t.UnixNano())
|
||||
}
|
||||
|
||||
// natsStarting reports whether we are inside the post-start grace period.
|
||||
func natsStarting() bool {
|
||||
nano := natsStartTimeNano.Load()
|
||||
if nano == 0 {
|
||||
return false
|
||||
}
|
||||
return time.Since(time.Unix(0, nano)) < natsStartGracePeriod
|
||||
}
|
||||
|
||||
const natsStartGracePeriod = 2 * time.Minute
|
||||
|
||||
func StopNATS() {
|
||||
utils.Log("[NATS] Stopping NATS server...")
|
||||
|
||||
StopOplogApply()
|
||||
|
||||
// serialized with StartNATS so we never shut down a server mid-bring-up
|
||||
natsStartMutex.Lock()
|
||||
defer natsStartMutex.Unlock()
|
||||
|
||||
if ns != nil {
|
||||
ns.Shutdown()
|
||||
ns.WaitForShutdown()
|
||||
ns = nil
|
||||
if old := ns.Load(); old != nil {
|
||||
old.Shutdown()
|
||||
old.WaitForShutdown()
|
||||
ns.Store(nil)
|
||||
}
|
||||
NATSStarted.Store(false)
|
||||
}
|
||||
@@ -675,9 +761,43 @@ func connectNATSClient(url string, user string, pwd string) (*nats.Conn, error)
|
||||
nats.MaxReconnects(-1),
|
||||
|
||||
nats.NoEcho(),
|
||||
|
||||
nats.ErrorHandler(natsAsyncError),
|
||||
)
|
||||
}
|
||||
|
||||
// natsAsyncError surfaces the errors nats.go reports out of band instead of
|
||||
// returning them from a call.
|
||||
//
|
||||
// Nothing registered a handler before this, so every one of those was discarded
|
||||
// — which is why a survivor whose ordered consumer died during a quorum outage
|
||||
// produced no log line at all, only silence and 503s. The library knew
|
||||
// (ErrConsumerNotActive, and the consumer-recreation failures behind it); we
|
||||
// simply were not listening.
|
||||
//
|
||||
// This is diagnostics plus a hint, NOT the recovery mechanism. Recovery stays
|
||||
// with the progress probe (oplog_apply.go), which owes nothing to nats.go's
|
||||
// internals and so cannot be broken by a client upgrade.
|
||||
func natsAsyncError(_ *nats.Conn, sub *nats.Subscription, err error) {
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
|
||||
where := ""
|
||||
if sub != nil {
|
||||
where = " on " + sub.Subject
|
||||
}
|
||||
utils.Warn("[NATS] async error" + where + ": " + err.Error())
|
||||
|
||||
// The library spots an inactive consumer well before three probes can prove
|
||||
// it. Arming the detector rather than detaching here keeps the evidence
|
||||
// requirement intact: the next probe still has to see the log ahead of us AND
|
||||
// our position frozen, so a spurious signal costs nothing.
|
||||
if sub != nil && oplogOwnsSubscription(sub) {
|
||||
oplogArmStallDetector()
|
||||
}
|
||||
}
|
||||
|
||||
func InitNATSClient() error {
|
||||
if !NATSStarted.Load() {
|
||||
utils.Warn("[NATS] NATS server not started, cannot initialize client")
|
||||
@@ -772,13 +892,16 @@ func InitNATSClient() error {
|
||||
utils.Error("[NATS] Failed to get JetStream context", err)
|
||||
}
|
||||
|
||||
// standalone has no peers to heartbeat with or sync from
|
||||
// standalone has no peers to heartbeat with and writes straight to SQLite
|
||||
if IsConstellationStandalone() {
|
||||
utils.Debug("[NATS] Standalone constellation, skipping heartbeat and sync request")
|
||||
utils.Debug("[NATS] Standalone constellation, skipping heartbeat and op-log")
|
||||
} else {
|
||||
go ClientHeartbeatInit()
|
||||
|
||||
go SendRequestSyncMessage()
|
||||
// the apply loop materializes this node's config from the op-log; it
|
||||
// waits for JetStream itself, so starting it here is safe even if JS
|
||||
// isn't elected yet
|
||||
StartOplogApply()
|
||||
}
|
||||
|
||||
// POST CLIENT CONNECTION HOOK
|
||||
@@ -900,17 +1023,33 @@ type NATSStatus struct {
|
||||
// ManagerLinkUp: can this node reach a manager's NATS — matters on
|
||||
// agents, whose local client is always connected to its own server.
|
||||
ManagerLinkUp bool `json:"managerLinkUp"`
|
||||
|
||||
// op-log health. OplogAttached is also "config is writable here": writes
|
||||
// are published, not applied locally, so a detached node is read-only.
|
||||
OplogAttached bool `json:"oplogAttached"`
|
||||
OplogHalted bool `json:"oplogHalted"`
|
||||
OplogHaltReason string `json:"oplogHaltReason,omitempty"`
|
||||
OplogEpoch uint64 `json:"oplogEpoch"`
|
||||
OplogSeq uint64 `json:"oplogSeq"`
|
||||
ConfigWritable bool `json:"configWritable"`
|
||||
|
||||
// OplogReplicas is the log's actual replication factor, 0 when the stream is
|
||||
// not reachable. It is the only externally visible evidence that the R1→R3
|
||||
// scale-up happened: writes succeed perfectly well against an R1 stream, so a
|
||||
// cluster that silently never scaled looks identical to a healthy one.
|
||||
OplogReplicas int `json:"oplogReplicas"`
|
||||
}
|
||||
|
||||
func GetNATSStatus() NATSStatus {
|
||||
isAgent := utils.FBL.AgentMode
|
||||
|
||||
srv := ns.Load()
|
||||
status := NATSStatus{
|
||||
Role: "manager",
|
||||
HAMode: IsNATSHA(),
|
||||
NebulaStarted: NebulaStarted.Load(),
|
||||
ServerRunning: NATSStarted.Load() && ns != nil,
|
||||
Starting: !natsStartTime.IsZero() && time.Since(natsStartTime) < natsStartGracePeriod,
|
||||
ServerRunning: NATSStarted.Load() && srv != nil,
|
||||
Starting: natsStarting(),
|
||||
}
|
||||
if isAgent {
|
||||
status.Role = "agent"
|
||||
@@ -927,10 +1066,10 @@ func GetNATSStatus() NATSStatus {
|
||||
}
|
||||
status.KnownManagers = getManagerIPs(myIP)
|
||||
|
||||
if ns != nil {
|
||||
status.ConnectedClients = ns.NumClients()
|
||||
status.ConnectedLeafs = ns.NumLeafNodes()
|
||||
status.ClusterRoutes = ns.NumRoutes()
|
||||
if srv != nil {
|
||||
status.ConnectedClients = srv.NumClients()
|
||||
status.ConnectedLeafs = srv.NumLeafNodes()
|
||||
status.ClusterRoutes = srv.NumRoutes()
|
||||
}
|
||||
|
||||
if isAgent {
|
||||
@@ -942,6 +1081,15 @@ func GetNATSStatus() NATSStatus {
|
||||
|
||||
status.ClientConnected = IsClientConnected()
|
||||
|
||||
status.OplogAttached = oplogAttached.Load()
|
||||
status.OplogHalted = oplogHalted.Load()
|
||||
if reason, ok := oplogHaltReason.Load().(string); ok && status.OplogHalted {
|
||||
status.OplogHaltReason = reason
|
||||
}
|
||||
status.OplogEpoch = utils.GetOplogEpoch()
|
||||
status.OplogSeq = utils.GetLastAppliedSeq()
|
||||
status.ConfigWritable = oplogWriteMode() != oplogReadOnly
|
||||
|
||||
clientConfigLock.RLock()
|
||||
if js != nil {
|
||||
if _, err := js.AccountInfo(); err == nil {
|
||||
@@ -950,6 +1098,9 @@ func GetNATSStatus() NATSStatus {
|
||||
if _, err := js.KeyValue("constellation-nodes"); err == nil {
|
||||
status.KVNodesReady = true
|
||||
}
|
||||
if si, err := js.StreamInfo(oplogStreamName(status.OplogEpoch)); err == nil {
|
||||
status.OplogReplicas = si.Config.Replicas
|
||||
}
|
||||
}
|
||||
clientConfigLock.RUnlock()
|
||||
|
||||
@@ -959,6 +1110,9 @@ func GetNATSStatus() NATSStatus {
|
||||
func CloseNATSClient() {
|
||||
utils.Log("Closing NATS client...")
|
||||
|
||||
// before taking clientConfigLock: the loop's own goroutine reads it
|
||||
StopOplogApply()
|
||||
|
||||
StopHeartbeat()
|
||||
|
||||
utils.Debug("[NATS] Closing NATS client connection")
|
||||
@@ -1081,7 +1235,11 @@ func MasterNATSClientRouter() {
|
||||
m.Respond([]byte("Pong"))
|
||||
})
|
||||
|
||||
SyncNATSClientRouter(nc)
|
||||
// public device list survives the legacy sync retirement — it is a read-only
|
||||
// query, not a replication path
|
||||
nc.Subscribe("cosmos._global_.constellation.public-devices", PublicDeviceListNATS)
|
||||
|
||||
oplogSnapshotRouter(nc)
|
||||
|
||||
// Scheduler: subscribe this node to its own per-target deployment command
|
||||
// subject so the leader can dispatch apply/remove here.
|
||||
@@ -1102,7 +1260,7 @@ func PingNATSClient() bool {
|
||||
|
||||
// An isolated agent's local client still looks healthy — without a
|
||||
// leafnode link to a manager it isn't connected to the constellation.
|
||||
if utils.FBL.AgentMode && (ns == nil || ns.NumLeafNodes() == 0) {
|
||||
if srv := ns.Load(); utils.FBL.AgentMode && (srv == nil || srv.NumLeafNodes() == 0) {
|
||||
utils.Warn("[NATS] Agent has no leafnode link to a manager")
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -3,7 +3,6 @@ package constellation
|
||||
import (
|
||||
"net/http"
|
||||
"encoding/json"
|
||||
"time"
|
||||
|
||||
"github.com/azukaar/cosmos-server/src/utils"
|
||||
)
|
||||
@@ -55,23 +54,9 @@ func DeviceBlock(w http.ResponseWriter, req *http.Request) {
|
||||
|
||||
utils.Log("ConstellationDeviceBlocking: Blocking Device " + deviceName)
|
||||
|
||||
c, closeDb, errCo := utils.GetEmbeddedCollection(utils.GetRootAppId(), "devices")
|
||||
defer closeDb()
|
||||
|
||||
if errCo != nil {
|
||||
utils.Error("Database Connect", errCo)
|
||||
utils.HTTPError(w, "Database", http.StatusInternalServerError, "DB001")
|
||||
return
|
||||
}
|
||||
|
||||
device := utils.Device{}
|
||||
|
||||
utils.Debug("ConstellationDeviceBlocking: Blocking Device " + deviceName)
|
||||
|
||||
err2 := c.FindOne(nil, map[string]interface{}{
|
||||
"DeviceName": deviceName,
|
||||
"Blocked": false,
|
||||
}).Decode(&device)
|
||||
device, err2 := utils.GetDeviceByName(deviceName, true)
|
||||
|
||||
if err2 == nil {
|
||||
utils.Debug("ConstellationDeviceBlocking: Found Device " + deviceName)
|
||||
@@ -81,18 +66,15 @@ func DeviceBlock(w http.ResponseWriter, req *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
_, err3 := c.UpdateMany(nil, map[string]interface{}{
|
||||
err3 := utils.UpdateDevices(map[string]interface{}{
|
||||
"DeviceName": deviceName,
|
||||
}, map[string]interface{}{
|
||||
"$set": map[string]interface{}{
|
||||
"Blocked": request.Block,
|
||||
},
|
||||
"Blocked": request.Block,
|
||||
})
|
||||
|
||||
if err3 != nil {
|
||||
utils.Error("DeviceBlocking: Error while updating device", err3)
|
||||
utils.HTTPError(w, "Device Creation Error: " + err3.Error(),
|
||||
http.StatusInternalServerError, "DB001")
|
||||
utils.HTTPStoreError(w, err3, "DB001")
|
||||
return
|
||||
}
|
||||
|
||||
@@ -101,12 +83,6 @@ func DeviceBlock(w http.ResponseWriter, req *http.Request) {
|
||||
} else {
|
||||
utils.Log("ConstellationDeviceBlocking: Device " + deviceName + " unblocked")
|
||||
}
|
||||
|
||||
go func() {
|
||||
go SendNewDBSyncMessage()
|
||||
time.Sleep(2 * time.Second)
|
||||
RestartNebula()
|
||||
}()
|
||||
} else {
|
||||
utils.Error("DeviceBlocking: Error while finding device", err2)
|
||||
utils.HTTPError(w, "Device Creation Error: " + err2.Error(),
|
||||
|
||||
@@ -4,9 +4,7 @@ import (
|
||||
"net/http"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/azukaar/cosmos-server/src/utils"
|
||||
)
|
||||
@@ -182,16 +180,9 @@ func DeviceCreate_API(w http.ResponseWriter, req *http.Request) {
|
||||
"Invisible": request.Invisible,
|
||||
},
|
||||
})
|
||||
|
||||
go func() {
|
||||
go SendNewDBSyncMessage()
|
||||
time.Sleep(2 * time.Second)
|
||||
RestartNebula()
|
||||
}()
|
||||
} else {
|
||||
utils.Error("DeviceCreation: Error creating device", err)
|
||||
utils.HTTPError(w, "Device Creation Error: " + err.Error(),
|
||||
http.StatusInternalServerError, "DC004")
|
||||
utils.HTTPStoreError(w, err, "DC004")
|
||||
return
|
||||
}
|
||||
} else {
|
||||
@@ -211,38 +202,24 @@ func DeviceCreate(request DeviceCreateRequestJSON) (string, string, string, Devi
|
||||
|
||||
utils.Log("ConstellationDeviceCreation: Creating Device " + deviceName)
|
||||
|
||||
c, closeDb, errCo := utils.GetEmbeddedCollection(utils.GetRootAppId(), "devices")
|
||||
defer closeDb()
|
||||
|
||||
if errCo != nil {
|
||||
return "", "", "", DeviceCreateRequestJSON{}, errCo
|
||||
}
|
||||
|
||||
device := utils.Device{}
|
||||
|
||||
utils.Debug("ConstellationDeviceCreation: Creating Device " + deviceName)
|
||||
|
||||
err2 := c.FindOne(nil, map[string]interface{}{
|
||||
"DeviceName": deviceName,
|
||||
"Blocked": false,
|
||||
}).Decode(&device)
|
||||
|
||||
// read-then-write pre-checks kept for their error messages; the partial
|
||||
// unique indexes are the authority
|
||||
_, err2 := utils.GetDeviceByName(deviceName, true)
|
||||
|
||||
if err2 == nil {
|
||||
return "", "", "", DeviceCreateRequestJSON{}, errors.New("DeviceCreation: Device with this name already exists")
|
||||
} else if err2 != mongo.ErrNoDocuments {
|
||||
} else if !errors.Is(err2, utils.ErrNotFound) {
|
||||
return "", "", "", DeviceCreateRequestJSON{}, err2
|
||||
}
|
||||
|
||||
// Check if IP is already in use
|
||||
ipDevice := utils.Device{}
|
||||
errIP := c.FindOne(nil, map[string]interface{}{
|
||||
"IP": request.IP,
|
||||
"Blocked": false,
|
||||
}).Decode(&ipDevice)
|
||||
_, errIP := utils.GetDeviceByIP(request.IP)
|
||||
|
||||
if errIP == nil {
|
||||
return "", "", "", DeviceCreateRequestJSON{}, errors.New("DeviceCreation: IP address is already in use")
|
||||
} else if errIP != mongo.ErrNoDocuments {
|
||||
} else if !errors.Is(errIP, utils.ErrNotFound) {
|
||||
return "", "", "", DeviceCreateRequestJSON{}, errIP
|
||||
}
|
||||
|
||||
@@ -256,7 +233,7 @@ func DeviceCreate(request DeviceCreateRequestJSON) (string, string, string, Devi
|
||||
|
||||
// Check cosmos node and devices limit (skipped on Pro — unlimited).
|
||||
if !utils.IsPro() && request.CosmosNode > 0 {
|
||||
countManagers, errCount := c.CountDocuments(nil, map[string]interface{}{
|
||||
countManagers, errCount := utils.CountDevices(map[string]interface{}{
|
||||
"CosmosNode": 2,
|
||||
"Blocked": false,
|
||||
})
|
||||
@@ -265,7 +242,7 @@ func DeviceCreate(request DeviceCreateRequestJSON) (string, string, string, Devi
|
||||
return "", "", "", DeviceCreateRequestJSON{}, errCount
|
||||
}
|
||||
|
||||
countAgent, errCount := c.CountDocuments(nil, map[string]interface{}{
|
||||
countAgent, errCount := utils.CountDevices(map[string]interface{}{
|
||||
"CosmosNode": 1,
|
||||
"Blocked": false,
|
||||
})
|
||||
@@ -293,7 +270,7 @@ func DeviceCreate(request DeviceCreateRequestJSON) (string, string, string, Devi
|
||||
if !utils.IsPro() {
|
||||
totalClientLimit := 10 * int64(utils.GetNumberUsers())
|
||||
|
||||
countDevices, errCountDevices := c.CountDocuments(nil, map[string]interface{}{
|
||||
countDevices, errCountDevices := utils.CountDevices(map[string]interface{}{
|
||||
"Blocked": false,
|
||||
})
|
||||
|
||||
@@ -321,21 +298,21 @@ func DeviceCreate(request DeviceCreateRequestJSON) (string, string, string, Devi
|
||||
request.IsLoadBalancer = false
|
||||
}
|
||||
|
||||
_, err3 := c.InsertOne(nil, map[string]interface{}{
|
||||
"Nickname": nickname,
|
||||
"DeviceName": deviceName,
|
||||
"IP": request.IP,
|
||||
"IsLighthouse": request.IsLighthouse,
|
||||
"CosmosNode": request.CosmosNode,
|
||||
"IsRelay": request.IsRelay,
|
||||
"IsExitNode": request.IsExitNode,
|
||||
"PublicHostname": request.PublicHostname,
|
||||
"IsLoadBalancer": request.IsLoadBalancer,
|
||||
"Port": request.Port,
|
||||
"Fingerprint": fingerprint,
|
||||
"APIKey": APIKey,
|
||||
"Blocked": false,
|
||||
"Invisible": request.Invisible,
|
||||
err3 := utils.CreateDevice(utils.ConstellationDevice{
|
||||
Nickname: nickname,
|
||||
DeviceName: deviceName,
|
||||
IP: request.IP,
|
||||
IsLighthouse: request.IsLighthouse,
|
||||
CosmosNode: request.CosmosNode,
|
||||
IsRelay: request.IsRelay,
|
||||
IsExitNode: request.IsExitNode,
|
||||
PublicHostname: request.PublicHostname,
|
||||
IsLoadBalancer: request.IsLoadBalancer,
|
||||
Port: request.Port,
|
||||
Fingerprint: fingerprint,
|
||||
APIKey: APIKey,
|
||||
Blocked: false,
|
||||
Invisible: request.Invisible,
|
||||
})
|
||||
|
||||
if err3 != nil {
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/azukaar/cosmos-server/src/utils"
|
||||
)
|
||||
@@ -54,15 +53,6 @@ func DeviceEdit_API(w http.ResponseWriter, req *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
c, closeDb, errCo := utils.GetEmbeddedCollection(utils.GetRootAppId(), "devices")
|
||||
defer closeDb()
|
||||
|
||||
if errCo != nil {
|
||||
utils.Error("Database Connect", errCo)
|
||||
utils.HTTPError(w, "Database", http.StatusInternalServerError, "DB001")
|
||||
return
|
||||
}
|
||||
|
||||
// Non-lighthouses cannot be relay, exit, or load balancer
|
||||
if !request.IsLighthouse {
|
||||
request.IsRelay = false
|
||||
@@ -101,36 +91,22 @@ func DeviceEdit_API(w http.ResponseWriter, req *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
// Only topology-affecting flags require a mesh restart; Tags reach the scheduler
|
||||
// via the 2s heartbeat. If the current record can't be read, assume topology changed.
|
||||
topologyChanged := true
|
||||
var currentRecord utils.ConstellationDevice
|
||||
if errFind := c.FindOne(nil, map[string]interface{}{
|
||||
"DeviceName": deviceName,
|
||||
"Blocked": false,
|
||||
}).Decode(¤tRecord); errFind == nil {
|
||||
topologyChanged = currentRecord.IsLighthouse != request.IsLighthouse ||
|
||||
currentRecord.IsRelay != request.IsRelay ||
|
||||
currentRecord.IsExitNode != request.IsExitNode ||
|
||||
currentRecord.IsLoadBalancer != request.IsLoadBalancer
|
||||
}
|
||||
|
||||
_, err = c.UpdateOne(nil, map[string]interface{}{
|
||||
// restart-or-refresh is no longer decided here: the apply loop diffs the op
|
||||
// against the row's pre-image and reacts identically on every node
|
||||
err = utils.UpdateDevices(map[string]interface{}{
|
||||
"DeviceName": deviceName,
|
||||
"Blocked": false,
|
||||
}, map[string]interface{}{
|
||||
"$set": map[string]interface{}{
|
||||
"IsLighthouse": request.IsLighthouse,
|
||||
"IsRelay": request.IsRelay,
|
||||
"IsExitNode": request.IsExitNode,
|
||||
"IsLoadBalancer": request.IsLoadBalancer,
|
||||
"Tags": cleanTags,
|
||||
},
|
||||
"IsLighthouse": request.IsLighthouse,
|
||||
"IsRelay": request.IsRelay,
|
||||
"IsExitNode": request.IsExitNode,
|
||||
"IsLoadBalancer": request.IsLoadBalancer,
|
||||
"Tags": cleanTags,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
utils.Error("DeviceEdit: Error updating device", err)
|
||||
utils.HTTPError(w, "Device Edit Error: "+err.Error(), http.StatusInternalServerError, "DE003")
|
||||
utils.HTTPStoreError(w, err, "DE003")
|
||||
return
|
||||
}
|
||||
|
||||
@@ -146,20 +122,4 @@ func DeviceEdit_API(w http.ResponseWriter, req *http.Request) {
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"status": "OK",
|
||||
})
|
||||
|
||||
if !topologyChanged {
|
||||
// tag-only/no-op edit: local cache refresh first so the next heartbeat
|
||||
// carries the new Tags, then sync peers without triggering their restart
|
||||
go func() {
|
||||
refreshDeviceCache()
|
||||
SendNewDBSyncMessageNoRestart()
|
||||
}()
|
||||
return
|
||||
}
|
||||
|
||||
go func() {
|
||||
go SendNewDBSyncMessage()
|
||||
time.Sleep(2 * time.Second)
|
||||
RestartNebula()
|
||||
}()
|
||||
}
|
||||
|
||||
@@ -28,50 +28,29 @@ func DeviceList(w http.ResponseWriter, req *http.Request) {
|
||||
}
|
||||
|
||||
isAdmin := utils.HasPermission(req, utils.PERM_RESOURCES_READ)
|
||||
|
||||
// Connect to the collection
|
||||
c, closeDb, errCo := utils.GetEmbeddedCollection(utils.GetRootAppId(), "devices")
|
||||
defer closeDb()
|
||||
if errCo != nil {
|
||||
utils.Error("Database Connect", errCo)
|
||||
utils.HTTPError(w, "Database", http.StatusInternalServerError, "DB001")
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
var devices []utils.ConstellationDevice
|
||||
|
||||
|
||||
// Check if user is an admin
|
||||
if isAdmin {
|
||||
// If admin, get all devices
|
||||
cursor, err := c.Find(nil, map[string]interface{}{})
|
||||
var err error
|
||||
devices, err = utils.ListDevices(true)
|
||||
if err != nil {
|
||||
utils.Error("DeviceList: Error fetching devices", err)
|
||||
utils.HTTPError(w, "Error fetching devices", http.StatusInternalServerError, "DL001")
|
||||
return
|
||||
}
|
||||
defer cursor.Close(nil)
|
||||
|
||||
if err = cursor.All(nil, &devices); err != nil {
|
||||
utils.Error("DeviceList: Error decoding devices", err)
|
||||
utils.HTTPError(w, "Error decoding devices", http.StatusInternalServerError, "DL002")
|
||||
return
|
||||
}
|
||||
} else {
|
||||
// If not admin, get user's devices based on their nickname
|
||||
nickname := utils.GetAuthContext(req).Nickname
|
||||
cursor, err := c.Find(nil, map[string]interface{}{"Nickname": nickname})
|
||||
var err error
|
||||
devices, err = utils.FindDevices(map[string]interface{}{"Nickname": nickname})
|
||||
if err != nil {
|
||||
utils.Error("DeviceList: Error fetching devices", err)
|
||||
utils.HTTPError(w, "Error fetching devices", http.StatusInternalServerError, "DL003")
|
||||
return
|
||||
}
|
||||
defer cursor.Close(nil)
|
||||
|
||||
if err = cursor.All(nil, &devices); err != nil {
|
||||
utils.Error("DeviceList: Error decoding devices", err)
|
||||
utils.HTTPError(w, "Error decoding devices", http.StatusInternalServerError, "DL004")
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
n, _ := GetCurrentDeviceName()
|
||||
|
||||
@@ -41,15 +41,6 @@ func DevicePing(w http.ResponseWriter, req *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
// Connect to the collection
|
||||
c, closeDb, errCo := utils.GetEmbeddedCollection(utils.GetRootAppId(), "devices")
|
||||
defer closeDb()
|
||||
if errCo != nil {
|
||||
utils.Error("Database Connect", errCo)
|
||||
utils.HTTPError(w, "Database", http.StatusInternalServerError, "DB001")
|
||||
return
|
||||
}
|
||||
|
||||
currentDeviceName, err := GetCurrentDeviceName()
|
||||
if deviceID == currentDeviceName {
|
||||
// Respond with the ping result
|
||||
@@ -63,12 +54,8 @@ func DevicePing(w http.ResponseWriter, req *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
var device utils.ConstellationDevice
|
||||
|
||||
// Find the device by DeviceName
|
||||
err = c.FindOne(nil, map[string]interface{}{
|
||||
"DeviceName": deviceID,
|
||||
}).Decode(&device)
|
||||
device, err := utils.GetDeviceByName(deviceID, false)
|
||||
|
||||
if err != nil {
|
||||
utils.Error("DevicePing: Device not found", err)
|
||||
|
||||
@@ -43,19 +43,10 @@ func DevicePublicList(w http.ResponseWriter, req *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
// Connect to the collection
|
||||
c, closeDb, errCo := utils.GetEmbeddedCollection(utils.GetRootAppId(), "devices")
|
||||
defer closeDb()
|
||||
if errCo != nil {
|
||||
utils.Error("Database Connect", errCo)
|
||||
utils.HTTPError(w, "Database", http.StatusInternalServerError, "DB001")
|
||||
return
|
||||
}
|
||||
|
||||
utils.Log("DevicePublicList: Fetching devices with API key")
|
||||
|
||||
// Find all non-blocked devices that match the API key
|
||||
cursor, err := c.Find(nil, map[string]interface{}{
|
||||
// Find all non-blocked devices
|
||||
devices, err := utils.FindDevices(map[string]interface{}{
|
||||
"Blocked": false,
|
||||
"Invisible": false,
|
||||
})
|
||||
@@ -65,14 +56,6 @@ func DevicePublicList(w http.ResponseWriter, req *http.Request) {
|
||||
utils.HTTPError(w, "Error fetching devices", http.StatusInternalServerError, "DPL001")
|
||||
return
|
||||
}
|
||||
defer cursor.Close(nil)
|
||||
|
||||
var devices []utils.ConstellationDevice
|
||||
if err = cursor.All(nil, &devices); err != nil {
|
||||
utils.Error("DevicePublicList: Error decoding devices", err)
|
||||
utils.HTTPError(w, "Error decoding devices", http.StatusInternalServerError, "DPL002")
|
||||
return
|
||||
}
|
||||
|
||||
// Convert to public device info with limited fields
|
||||
publicDevices := make([]PublicDeviceInfo, len(devices))
|
||||
@@ -99,19 +82,10 @@ func DevicePublicList(w http.ResponseWriter, req *http.Request) {
|
||||
}
|
||||
|
||||
func PublicDeviceListNATS(m *nats.Msg) {
|
||||
// Connect to the collection
|
||||
c, closeDb, errCo := utils.GetEmbeddedCollection(utils.GetRootAppId(), "devices")
|
||||
defer closeDb()
|
||||
if errCo != nil {
|
||||
utils.Error("PublicDeviceListNATS: Database Connect", errCo)
|
||||
m.Respond([]byte(`{"status":"error","message":"Database error"}`))
|
||||
return
|
||||
}
|
||||
|
||||
utils.Debug("PublicDeviceListNATS: Fetching devices")
|
||||
|
||||
// Find all non-blocked, non-invisible devices
|
||||
cursor, err := c.Find(nil, map[string]interface{}{
|
||||
devices, err := utils.FindDevices(map[string]interface{}{
|
||||
"Blocked": false,
|
||||
"Invisible": false,
|
||||
})
|
||||
@@ -121,14 +95,6 @@ func PublicDeviceListNATS(m *nats.Msg) {
|
||||
m.Respond([]byte(`{"status":"error","message":"Error fetching devices"}`))
|
||||
return
|
||||
}
|
||||
defer cursor.Close(nil)
|
||||
|
||||
var devices []utils.ConstellationDevice
|
||||
if err = cursor.All(nil, &devices); err != nil {
|
||||
utils.Error("PublicDeviceListNATS: Error decoding devices", err)
|
||||
m.Respond([]byte(`{"status":"error","message":"Error decoding devices"}`))
|
||||
return
|
||||
}
|
||||
|
||||
// Convert to public device info with limited fields
|
||||
publicDevices := make([]PublicDeviceInfo, len(devices))
|
||||
|
||||
@@ -96,6 +96,26 @@ func getDNSEntry(w http.ResponseWriter, req *http.Request) {
|
||||
// @Failure 403 {object} utils.HTTPErrorResult
|
||||
// @Failure 409 {object} utils.HTTPErrorResult
|
||||
// @Router /api/constellation/dns [post]
|
||||
// dnsPayloadWithEntries builds the FULL dns domain state from config, swapping in
|
||||
// the given custom entries. The domain is full-state, so a handler that changes
|
||||
// one entry still has to send every DNS field — sending a partial payload would
|
||||
// blank the rest on every node. The copy keeps the published slice from aliasing
|
||||
// the config's backing array.
|
||||
//
|
||||
// These handlers publish and do NOT write locally: CustomDNSEntries is entirely
|
||||
// replicated, so the apply loop is the only writer. A local write here is what
|
||||
// made these entries silently vanish on the next unrelated DNS change.
|
||||
func dnsPayloadWithEntries(config utils.Config, entries []utils.ConstellationDNSEntry) DNSPayload {
|
||||
c := config.ConstellationConfig
|
||||
return DNSPayload{
|
||||
DNSPort: c.DNSPort,
|
||||
DNSFallback: c.DNSFallback,
|
||||
DNSBlockBlacklist: c.DNSBlockBlacklist,
|
||||
DNSAdditionalBlocklists: c.DNSAdditionalBlocklists,
|
||||
CustomDNSEntries: append([]utils.ConstellationDNSEntry{}, entries...),
|
||||
}
|
||||
}
|
||||
|
||||
func createDNSEntry(w http.ResponseWriter, req *http.Request) {
|
||||
if utils.CheckPermissions(w, req, utils.PERM_CONFIGURATION) != nil {
|
||||
return
|
||||
@@ -115,20 +135,27 @@ func createDNSEntry(w http.ResponseWriter, req *http.Request) {
|
||||
}
|
||||
|
||||
utils.ConfigLock.Lock()
|
||||
defer utils.ConfigLock.Unlock()
|
||||
|
||||
config := utils.ReadConfigFromFile()
|
||||
entries := config.ConstellationConfig.CustomDNSEntries
|
||||
|
||||
for _, entry := range entries {
|
||||
if entry.Key == newEntry.Key {
|
||||
utils.ConfigLock.Unlock()
|
||||
utils.HTTPError(w, "DNS entry with this key already exists", http.StatusConflict, "DNS004")
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
config.ConstellationConfig.CustomDNSEntries = append(config.ConstellationConfig.CustomDNSEntries, newEntry)
|
||||
utils.SetBaseMainConfig(config)
|
||||
payload := dnsPayloadWithEntries(config, append(entries, newEntry))
|
||||
// Must be released before publishing: the apply loop takes ConfigLock to
|
||||
// install the very op this call waits on, so holding it here deadlocks.
|
||||
utils.ConfigLock.Unlock()
|
||||
|
||||
if err := PublishDomainOp(DomainDNS, payload); err != nil {
|
||||
utils.HTTPStoreError(w, err, "DNS008")
|
||||
return
|
||||
}
|
||||
|
||||
utils.TriggerEvent(
|
||||
"cosmos.settings",
|
||||
@@ -173,7 +200,6 @@ func updateDNSEntry(w http.ResponseWriter, req *http.Request) {
|
||||
}
|
||||
|
||||
utils.ConfigLock.Lock()
|
||||
defer utils.ConfigLock.Unlock()
|
||||
|
||||
config := utils.ReadConfigFromFile()
|
||||
entries := config.ConstellationConfig.CustomDNSEntries
|
||||
@@ -187,13 +213,19 @@ func updateDNSEntry(w http.ResponseWriter, req *http.Request) {
|
||||
}
|
||||
|
||||
if entryIndex == -1 {
|
||||
utils.ConfigLock.Unlock()
|
||||
utils.HTTPError(w, "DNS entry not found", http.StatusNotFound, "DNS006")
|
||||
return
|
||||
}
|
||||
|
||||
entries[entryIndex] = updatedEntry
|
||||
config.ConstellationConfig.CustomDNSEntries = entries
|
||||
utils.SetBaseMainConfig(config)
|
||||
payload := dnsPayloadWithEntries(config, entries)
|
||||
utils.ConfigLock.Unlock()
|
||||
|
||||
if err := PublishDomainOp(DomainDNS, payload); err != nil {
|
||||
utils.HTTPStoreError(w, err, "DNS008")
|
||||
return
|
||||
}
|
||||
|
||||
utils.TriggerEvent(
|
||||
"cosmos.settings",
|
||||
@@ -227,7 +259,6 @@ func deleteDNSEntry(w http.ResponseWriter, req *http.Request) {
|
||||
key := mux.Vars(req)["key"]
|
||||
|
||||
utils.ConfigLock.Lock()
|
||||
defer utils.ConfigLock.Unlock()
|
||||
|
||||
config := utils.ReadConfigFromFile()
|
||||
entries := config.ConstellationConfig.CustomDNSEntries
|
||||
@@ -241,13 +272,18 @@ func deleteDNSEntry(w http.ResponseWriter, req *http.Request) {
|
||||
}
|
||||
|
||||
if entryIndex == -1 {
|
||||
utils.ConfigLock.Unlock()
|
||||
utils.HTTPError(w, "DNS entry not found", http.StatusNotFound, "DNS007")
|
||||
return
|
||||
}
|
||||
|
||||
entries = append(entries[:entryIndex], entries[entryIndex+1:]...)
|
||||
config.ConstellationConfig.CustomDNSEntries = entries
|
||||
utils.SetBaseMainConfig(config)
|
||||
payload := dnsPayloadWithEntries(config, append(entries[:entryIndex], entries[entryIndex+1:]...))
|
||||
utils.ConfigLock.Unlock()
|
||||
|
||||
if err := PublishDomainOp(DomainDNS, payload); err != nil {
|
||||
utils.HTTPStoreError(w, err, "DNS008")
|
||||
return
|
||||
}
|
||||
|
||||
utils.TriggerEvent(
|
||||
"cosmos.settings",
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
package constellation
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
|
||||
"github.com/azukaar/cosmos-server/src/utils"
|
||||
)
|
||||
|
||||
// Force-reform is the manual way out of a permanently lost quorum: an HA
|
||||
// constellation that has lost two of its three managers can no longer publish
|
||||
// anything, including the device edits that would replace them. It re-enters
|
||||
// formation on the survivor at a new epoch, so the survivor writes directly
|
||||
// again and replacement managers can be enrolled normally.
|
||||
//
|
||||
// Destructive and deliberately so — the old managers are fenced out, not
|
||||
// recovered. Everything that isn't cluster state (CA, keys, config, devices)
|
||||
// is left exactly as it is.
|
||||
|
||||
// blockDeadManagers blocks every manager but this one.
|
||||
//
|
||||
// Scope is CosmosNode == 2 on purpose. That is the definition of a cluster
|
||||
// voter, and voters are the only thing a reform has to fence: a CosmosNode 0 row
|
||||
// is a plain nebula client that runs no NATS server and holds no route, and
|
||||
// agents (CosmosNode == 1) are deliberately spared so they can follow the
|
||||
// survivor into the new epoch by snapshot.
|
||||
//
|
||||
// LOCAL writes, never published: the whole point of reform is that this node has
|
||||
// no quorum to publish through, and a blocked-manager record is node-local truth
|
||||
// until the replacements enroll and re-materialize from the new log. This is the
|
||||
// leave/teardown class of write (see utils.CommitMutationLocal), and formation is
|
||||
// exactly where it is legal.
|
||||
func blockDeadManagers(selfName string) ([]string, error) {
|
||||
managers, err := utils.FindDevices(map[string]interface{}{"CosmosNode": 2, "Blocked": false})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
blocked := []string{}
|
||||
for _, d := range managers {
|
||||
if d.DeviceName == "" || d.DeviceName == selfName {
|
||||
continue
|
||||
}
|
||||
err := utils.CommitMutationLocal(utils.Mutation{
|
||||
Table: "devices",
|
||||
Op: "updateMany",
|
||||
Filter: map[string]interface{}{"DeviceName": d.DeviceName, "Blocked": false},
|
||||
Doc: map[string]interface{}{"Blocked": true},
|
||||
})
|
||||
if err != nil {
|
||||
return blocked, err
|
||||
}
|
||||
blocked = append(blocked, d.DeviceName)
|
||||
}
|
||||
return blocked, nil
|
||||
}
|
||||
|
||||
// API_ForceReform godoc
|
||||
// @Summary Force the constellation to re-form around this manager
|
||||
// @Description HA only. Bumps the op-log epoch, drops local JetStream state, blocks the other managers and restarts NATS, leaving this node writable in formation mode so replacements can be enrolled. Does not touch the CA, keys, config or device data.
|
||||
// @Tags constellation
|
||||
// @Produce json
|
||||
// @Security BearerAuth
|
||||
// @Success 200 {object} utils.APIResponse
|
||||
// @Failure 400 {object} utils.HTTPErrorResult
|
||||
// @Failure 403 {object} utils.HTTPErrorResult
|
||||
// @Failure 500 {object} utils.HTTPErrorResult
|
||||
// @Router /api/constellation/force-reform [post]
|
||||
func API_ForceReform(w http.ResponseWriter, req *http.Request) {
|
||||
if utils.CheckPermissions(w, req, utils.PERM_ADMIN) != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if req.Method != "POST" {
|
||||
utils.Error("API_ForceReform: Method not allowed "+req.Method, nil)
|
||||
utils.HTTPError(w, "Method not allowed", http.StatusMethodNotAllowed, "HTTP001")
|
||||
return
|
||||
}
|
||||
|
||||
// Non-HA has nothing to reform: its single manager's JetStream is always up,
|
||||
// so it is never quorum-blocked in the first place.
|
||||
if !IsNATSHA() {
|
||||
utils.Error("API_ForceReform: not an HA constellation", nil)
|
||||
utils.HTTPError(w, "Force-reform only applies to an HA constellation",
|
||||
http.StatusBadRequest, "AFR001")
|
||||
return
|
||||
}
|
||||
|
||||
selfName, err := GetCurrentDeviceName()
|
||||
if err != nil || selfName == "" {
|
||||
utils.Error("API_ForceReform: cannot identify this device", err)
|
||||
utils.HTTPError(w, "Cannot identify this device", http.StatusInternalServerError, "AFR002")
|
||||
return
|
||||
}
|
||||
|
||||
epoch, blocked, err := performForceReform(selfName)
|
||||
if err != nil {
|
||||
utils.Error("API_ForceReform: reform failed", err)
|
||||
utils.HTTPError(w, "Force-reform failed: "+err.Error(),
|
||||
http.StatusInternalServerError, "AFR003")
|
||||
return
|
||||
}
|
||||
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"status": "OK",
|
||||
"data": map[string]interface{}{
|
||||
"epoch": epoch,
|
||||
"blockedDevices": blocked,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// performForceReform is the reform itself, callable without an HTTP request so
|
||||
// the E2E harness exercises this code rather than a copy of it. Callers own the
|
||||
// guards (permissions, method, HA-only); this owns the sequence.
|
||||
//
|
||||
// It schedules its own restart — callers must NOT trigger one.
|
||||
func performForceReform(selfName string) (uint64, []string, error) {
|
||||
utils.Log("[REFORM] Re-forming the constellation around " + selfName)
|
||||
|
||||
// One tx, and it comes first: the epoch bump is the fence. Everything after it
|
||||
// is cleanup a crash can safely repeat, but a node that tore down its cluster
|
||||
// state while still claiming the old epoch could rejoin the log it just left.
|
||||
epoch, err := utils.ReformOplogEpoch()
|
||||
if err != nil {
|
||||
return 0, nil, errors.New("could not start a new op-log epoch: " + err.Error())
|
||||
}
|
||||
// the new epoch has no log yet, which is what re-opens the direct write path
|
||||
oplogStreamSeen.Store(false)
|
||||
|
||||
// Unconditional, and deferred from here rather than run at the end: every step
|
||||
// below stops something (the apply loop, NATS, the client), so returning early
|
||||
// on an error without this would leave the node with its epoch bumped, NATS
|
||||
// down, and nothing scheduled to bring any of it back — recoverable only by
|
||||
// restarting the process.
|
||||
defer func() { go RestartNebula() }()
|
||||
|
||||
StopOplogApply()
|
||||
StopNATS()
|
||||
CloseNATSClient()
|
||||
|
||||
// ONLY the JetStream directory: the streams and meta-raft group belong to the
|
||||
// cluster we are abandoning. The CA, the node keys, nebula.yml, config.json and
|
||||
// the device rows are all still valid and are what the new epoch is seeded from.
|
||||
if err := os.RemoveAll(jetstreamDir()); err != nil {
|
||||
return epoch, nil, errors.New("could not remove local JetStream state: " + err.Error())
|
||||
}
|
||||
|
||||
blocked, err := blockDeadManagers(selfName)
|
||||
if err != nil {
|
||||
return epoch, blocked, errors.New("could not block the departed managers: " + err.Error())
|
||||
}
|
||||
|
||||
// so the restart builds cluster routes and NATS users from the survivors only:
|
||||
// routes come from getManagerIPs, which subtracts the blocked set, and the NATS
|
||||
// user list is rebuilt from this cache — which is how the fenced managers'
|
||||
// credentials die
|
||||
refreshDeviceCache()
|
||||
|
||||
utils.Log("[REFORM] Epoch is now " + strconv.FormatUint(epoch, 10) +
|
||||
", managers blocked: " + strconv.Itoa(len(blocked)))
|
||||
return epoch, blocked, nil
|
||||
}
|
||||
@@ -54,30 +54,13 @@ func incrementIP(ip net.IP) {
|
||||
|
||||
// GetNextAvailableIP fetches all used IPs from the database and returns the next available IP in the given CIDR range
|
||||
func GetNextAvailableIP(cidr string) string {
|
||||
c, closeDb, errCo := utils.GetEmbeddedCollection(utils.GetRootAppId(), "devices")
|
||||
defer closeDb()
|
||||
|
||||
if errCo != nil {
|
||||
utils.Error("GetNextAvailableIP: Database Connect", errCo)
|
||||
return ""
|
||||
}
|
||||
|
||||
cursor, err := c.Find(nil, map[string]interface{}{
|
||||
"Blocked": false,
|
||||
})
|
||||
devices, err := utils.ListDevices(false)
|
||||
if err != nil {
|
||||
utils.Error("GetNextAvailableIP: Error fetching devices", err)
|
||||
return ""
|
||||
}
|
||||
defer cursor.Close(nil)
|
||||
|
||||
usedIPs := make(map[string]bool)
|
||||
var devices []utils.ConstellationDevice
|
||||
if err = cursor.All(nil, &devices); err != nil {
|
||||
utils.Error("GetNextAvailableIP: Error decoding devices", err)
|
||||
return ""
|
||||
}
|
||||
|
||||
for _, device := range devices {
|
||||
usedIPs[device.IP] = true
|
||||
}
|
||||
|
||||
@@ -27,16 +27,10 @@ func CheckConstellationToken(req *http.Request) error {
|
||||
|
||||
// remove "Bearer " from auth header
|
||||
auth = strings.Replace(auth, "Bearer ", "", 1)
|
||||
|
||||
c, closeDb, errCo := utils.GetEmbeddedCollection(utils.GetRootAppId(), "devices")
|
||||
defer closeDb()
|
||||
if errCo != nil {
|
||||
return errCo
|
||||
}
|
||||
|
||||
utils.Log("DeviceConfigSync: Fetching devices for IP " + ip)
|
||||
|
||||
cursor, err := c.Find(nil, map[string]interface{}{
|
||||
devices, err := utils.FindDevices(map[string]interface{}{
|
||||
"IP": ip,
|
||||
"APIKey": auth,
|
||||
"Blocked": false,
|
||||
@@ -44,10 +38,9 @@ func CheckConstellationToken(req *http.Request) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer cursor.Close(nil)
|
||||
|
||||
// if any device is found, return config without keys
|
||||
if cursor.Next(nil) {
|
||||
if len(devices) > 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"gopkg.in/yaml.v2"
|
||||
|
||||
"github.com/azukaar/cosmos-server/src/utils"
|
||||
@@ -135,6 +136,14 @@ func API_NewConstellation(w http.ResponseWriter, req *http.Request) {
|
||||
}
|
||||
utils.SetBaseMainConfig(config)
|
||||
|
||||
// The creator seeds the op-log. In HA there is no JetStream at all until a
|
||||
// second manager enrolls, so it writes straight to SQLite until the log
|
||||
// exists; it is the only node licensed to, and the licence ends by itself
|
||||
// once the log is created and this node is applying it.
|
||||
if err := utils.SetFormationWriter(utils.GetOplogEpoch()); err != nil {
|
||||
utils.Error("API_NewConstellation: failed to enter formation mode", err)
|
||||
}
|
||||
|
||||
utils.TriggerEvent(
|
||||
"cosmos.settings",
|
||||
"Settings updated",
|
||||
@@ -229,10 +238,50 @@ func ConnectToExisting(yamlBody []byte, config utils.Config) (utils.Config, erro
|
||||
config.ConstellationConfig.IPRange = ipRange
|
||||
}
|
||||
|
||||
// Start at the constellation's real op-log epoch rather than the default 1.
|
||||
//
|
||||
// Forward only, and deliberately so. Adopting a LOWER epoch would walk this
|
||||
// node backwards into a log the cluster has already abandoned; the guard is
|
||||
// what keeps a stale enrollment bundle from being a downgrade attack on a
|
||||
// healthy node. At an equal epoch there is nothing to do, and clearing the
|
||||
// bootstrapped marker there would force a pointless re-snapshot.
|
||||
//
|
||||
// The seq resets to 0 and SetOplogState clears the bootstrapped marker, so the
|
||||
// joiner must re-materialize at this epoch before it may write — which is what
|
||||
// we want: it has the epoch's NAME, not its contents.
|
||||
if epoch, ok := yamlUint(configMap["cstln_oplog_epoch"]); ok && epoch > utils.GetOplogEpoch() {
|
||||
if err := utils.SetOplogState(epoch, 0); err != nil {
|
||||
return config, err
|
||||
}
|
||||
utils.Log("ConnectToExisting: joining at op-log epoch " + strconv.FormatUint(epoch, 10))
|
||||
}
|
||||
|
||||
utils.Log("ConnectToExisting: connected to an external Constellation")
|
||||
return config, nil
|
||||
}
|
||||
|
||||
// yamlUint reads a non-negative integer out of a decoded YAML value, whose Go
|
||||
// type depends on how the number was written.
|
||||
func yamlUint(v interface{}) (uint64, bool) {
|
||||
switch n := v.(type) {
|
||||
case int:
|
||||
if n >= 0 {
|
||||
return uint64(n), true
|
||||
}
|
||||
case int64:
|
||||
if n >= 0 {
|
||||
return uint64(n), true
|
||||
}
|
||||
case uint64:
|
||||
return n, true
|
||||
case float64:
|
||||
if n >= 0 {
|
||||
return uint64(n), true
|
||||
}
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
// API_ConnectToExisting godoc
|
||||
// @Summary Connect this node to an existing Constellation VPN network
|
||||
// @Tags constellation
|
||||
|
||||
@@ -101,21 +101,10 @@ func UpdateFirewallBlockedClients() error {
|
||||
}
|
||||
|
||||
// Get all devices from the database to map names to IPs
|
||||
c, closeDb, errCo := utils.GetEmbeddedCollection(utils.GetRootAppId(), "devices")
|
||||
if errCo != nil {
|
||||
return errCo
|
||||
}
|
||||
defer closeDb()
|
||||
|
||||
var devices []utils.ConstellationDevice
|
||||
cursor, err := c.Find(nil, map[string]interface{}{})
|
||||
devices, err := utils.ListDevices(true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer cursor.Close(nil)
|
||||
if err := cursor.All(nil, &devices); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Create a map of device names to IPs
|
||||
deviceIPs := make(map[string]string)
|
||||
|
||||
@@ -12,11 +12,17 @@ import (
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/azukaar/cosmos-server/src/utils"
|
||||
)
|
||||
|
||||
var e2eHTTP = &http.Client{Timeout: 20 * time.Second}
|
||||
@@ -127,6 +133,71 @@ func (c *e2eCluster) dbHash(name string) string {
|
||||
return h
|
||||
}
|
||||
|
||||
// oplogConverged reports whether every named node holds the same logical dump
|
||||
// AND sits at the same (epoch, seq) — matching dumps alone can be a coincidence
|
||||
// mid-replay, matching positions cannot.
|
||||
func (c *e2eCluster) oplogConverged(names ...string) (bool, string) {
|
||||
wantHash := ""
|
||||
wantPos := ""
|
||||
detail := ""
|
||||
|
||||
for _, name := range names {
|
||||
st, err := c.node(name).get("/oplog")
|
||||
if err != nil {
|
||||
return false, name + ": " + err.Error()
|
||||
}
|
||||
epoch, _ := st["epoch"].(float64)
|
||||
seq, _ := st["seq"].(float64)
|
||||
pos := fmt.Sprintf("e%.0f/%.0f", epoch, seq)
|
||||
hash := c.dbHash(name)
|
||||
|
||||
detail += fmt.Sprintf("%s=%s:%.8s ", name, pos, hash)
|
||||
|
||||
if halted, _ := st["halted"].(bool); halted {
|
||||
return false, detail + "(" + name + " HALTED: " + fmt.Sprint(st["haltReason"]) + ")"
|
||||
}
|
||||
if hash == "" {
|
||||
return false, detail + "(" + name + " has no dump)"
|
||||
}
|
||||
if wantHash == "" {
|
||||
wantHash, wantPos = hash, pos
|
||||
continue
|
||||
}
|
||||
if hash != wantHash || pos != wantPos {
|
||||
return false, detail
|
||||
}
|
||||
}
|
||||
return true, detail
|
||||
}
|
||||
|
||||
// waitStreamReplicas blocks until the op-log stream is both CONFIGURED for the
|
||||
// wanted replication and actually placed on that many peers.
|
||||
//
|
||||
// Any scenario that takes managers away has to wait for this first. The stream
|
||||
// is created R1 and raised to R3 asynchronously once the third manager is up, so
|
||||
// a test that kills two of three managers before the scale-up lands is killing
|
||||
// two nodes that may hold no replica at all: the survivor still has the only
|
||||
// copy, its publishes still succeed, and a "writes must fail without quorum"
|
||||
// assertion fails while the code under test is behaving correctly. Worse, which
|
||||
// single server an R1 stream lands on is arbitrary, so the same test passes or
|
||||
// fails run to run for reasons unrelated to the change being tested.
|
||||
func (c *e2eCluster) waitStreamReplicas(d time.Duration, node string, want int) {
|
||||
c.t.Helper()
|
||||
c.waitForDetail(d, fmt.Sprintf("op-log stream replicated R%d on %s", want, node),
|
||||
func() (bool, string) {
|
||||
out, err := c.node(node).get("/stream-info")
|
||||
if err != nil {
|
||||
return false, err.Error()
|
||||
}
|
||||
exists, _ := out["exists"].(bool)
|
||||
replicas, _ := out["replicas"].(float64)
|
||||
peers, _ := out["peers"].(float64)
|
||||
detail := fmt.Sprintf("exists=%v replicas=%v peers=%v leader=%v err=%v",
|
||||
exists, out["replicas"], out["peers"], out["leader"], out["error"])
|
||||
return exists && int(replicas) == want && int(peers) >= want, detail
|
||||
})
|
||||
}
|
||||
|
||||
// heartbeatNames returns the keys present in the node's view of the
|
||||
// constellation-nodes KV bucket.
|
||||
func (c *e2eCluster) heartbeatNames(name string) []string {
|
||||
@@ -179,11 +250,81 @@ func (c *e2eCluster) scanLogs() {
|
||||
}
|
||||
}
|
||||
|
||||
func e2eFileHash(path string) string {
|
||||
data, err := os.ReadFile(path)
|
||||
// e2eDumpHash hashes the canonical sorted-JSON logical dump of users+devices.
|
||||
func e2eDumpHash() string {
|
||||
data, err := utils.BuildLogicalDump()
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
sum := sha256.Sum256(data)
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// requireFreeControlPorts aborts immediately if a previous run's node is still
|
||||
// holding one of the fixed loopback control ports.
|
||||
//
|
||||
// The nodes bind hard-coded addresses, so only one suite can exist on a machine
|
||||
// at a time. Without this check a leftover process does not announce itself: the
|
||||
// new node's control server dies at startup with "address already in use", one
|
||||
// line deep in a child log, and the scenario then spends its full budget timing
|
||||
// out against a node that never came up. That reads exactly like a product
|
||||
// failure — a node unreachable while its peers are fine — and cost real time to
|
||||
// diagnose as an environment problem. Failing here instead turns minutes of
|
||||
// misleading red into one accurate sentence.
|
||||
func requireFreeControlPorts(t *testing.T, specs []e2eNodeSpec) {
|
||||
t.Helper()
|
||||
for _, s := range specs {
|
||||
addr := fmt.Sprintf("127.0.1.%d:%d", s.Octet, e2eControlPort)
|
||||
conn, err := net.DialTimeout("tcp", addr, 2*time.Second)
|
||||
if err != nil {
|
||||
continue // refused: nothing listening, which is what we want
|
||||
}
|
||||
conn.Close()
|
||||
t.Fatalf("e2e: %s is already in use — a node from a previous run is still alive. "+
|
||||
"Only one E2E suite can run at a time on this machine. "+
|
||||
"Clear it with: pkill -f 'constellation.test'", addr)
|
||||
}
|
||||
}
|
||||
|
||||
// reapNebulaStubs kills the fake nebula processes this cluster's nodes spawned.
|
||||
//
|
||||
// The stub is a child of the node process, and scenarios kill nodes with SIGKILL
|
||||
// to simulate a crash — which by definition gives them no chance to reap their
|
||||
// own children. So every scenario leaks a handful, and they accumulate: 728 were
|
||||
// found alive on this machine after a day of gate runs, each still pinning its
|
||||
// scenario's temp directory. They bind no ports and look harmless, which is why
|
||||
// nobody noticed, but several hundred stray processes are exactly the kind of
|
||||
// environmental drift that makes a gate's third run behave differently from its
|
||||
// first — and a three-run gate whose runs are not equivalent is not a gate.
|
||||
//
|
||||
// Matched on this cluster's OWN temp root, never on the process name, so a
|
||||
// concurrent run's stubs (or anything else on the box called nebula) are left
|
||||
// alone. Runs before the TempDir cleanup, which is registered earlier and so
|
||||
// fires later.
|
||||
func (c *e2eCluster) reapNebulaStubs() {
|
||||
entries, err := os.ReadDir("/proc")
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
reaped := 0
|
||||
for _, e := range entries {
|
||||
pid, err := strconv.Atoi(e.Name())
|
||||
if err != nil {
|
||||
continue // not a process directory
|
||||
}
|
||||
raw, err := os.ReadFile("/proc/" + e.Name() + "/cmdline")
|
||||
if err != nil {
|
||||
continue // exited between the listing and the read
|
||||
}
|
||||
cmd := strings.ReplaceAll(string(raw), "\x00", " ")
|
||||
if !strings.Contains(cmd, c.root) || !strings.Contains(cmd, "nebula") {
|
||||
continue
|
||||
}
|
||||
if syscall.Kill(pid, syscall.SIGKILL) == nil {
|
||||
reaped++
|
||||
}
|
||||
}
|
||||
if reaped > 0 {
|
||||
c.t.Logf("e2e: reaped %d leftover nebula stub process(es)", reaped)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,337 @@
|
||||
//go:build e2e
|
||||
|
||||
package constellation
|
||||
|
||||
// Control-API surface added for the op-log and M3 reform scenarios, split out of
|
||||
// e2e_nodemain_test.go to keep that file within the repo's size limit. These
|
||||
// handlers run inside the node child process, so they can call package-internal
|
||||
// code (DeviceCreate, connectNATSClient, API_ForceReform) directly rather than
|
||||
// reimplementing it in the parent test.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
"github.com/azukaar/cosmos-server/src/utils"
|
||||
)
|
||||
|
||||
// e2eWriteJSON / e2eFail / e2eFailStore are shared with e2eControlServer, which
|
||||
// aliases them; they are package-level so handlers can live in either file.
|
||||
func e2eWriteJSON(w http.ResponseWriter, code int, v interface{}) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(code)
|
||||
json.NewEncoder(w).Encode(v)
|
||||
}
|
||||
|
||||
func e2eFail(w http.ResponseWriter, err error) {
|
||||
e2eWriteJSON(w, 500, map[string]interface{}{"error": err.Error()})
|
||||
}
|
||||
|
||||
// e2eFailStore surfaces the op-log write contract to the parent test. e2eFail
|
||||
// collapses everything to 500, which would make "publish failed" and "apply
|
||||
// timed out" indistinguishable — and a regression turning every 409 into a 503
|
||||
// would sail straight through a test that can only assert "the write failed".
|
||||
func e2eFailStore(w http.ResponseWriter, err error) {
|
||||
status, code := 500, "internal"
|
||||
var ec *utils.ErrConstraint
|
||||
switch {
|
||||
case errors.Is(err, utils.ErrReadOnly):
|
||||
status, code = 409, "read-only"
|
||||
case errors.Is(err, utils.ErrApplyTimeout):
|
||||
status, code = 503, "apply-timeout"
|
||||
case errors.As(err, &ec):
|
||||
status, code = 409, "duplicate"
|
||||
case errors.Is(err, utils.ErrNotFound):
|
||||
status, code = 404, "not-found"
|
||||
}
|
||||
e2eWriteJSON(w, status, map[string]interface{}{
|
||||
"status": status, "code": code, "error": err.Error(),
|
||||
})
|
||||
}
|
||||
|
||||
// e2eRegisterOplogControl adds the op-log and reform endpoints to the node's
|
||||
// control mux.
|
||||
func e2eRegisterOplogControl(mux *http.ServeMux) {
|
||||
writeJSON := e2eWriteJSON
|
||||
fail := e2eFail
|
||||
failStore := e2eFailStore
|
||||
|
||||
// break-oplog-consumer kills the live subscription while deliberately LEAVING
|
||||
// oplogAttached set and oplogSub non-nil.
|
||||
//
|
||||
// That combination is not a simulation, it is the exact state a real quorum
|
||||
// outage was observed to leave behind (see TestE2EOplogDeadConsumerRecovery):
|
||||
// the ordered consumer is gone, but the stream still exists at the same epoch
|
||||
// and StreamInfo answers normally, so neither of the liveness probe's original
|
||||
// detach conditions can see it. The node publishes into a log it will never
|
||||
// read back.
|
||||
//
|
||||
// It exists because the natural trigger is a race inside JetStream — it
|
||||
// reproduced once in seven full-suite runs — and a defect that severe cannot
|
||||
// have its only coverage be a coin flip. Reproducing the STATE is deterministic;
|
||||
// reproducing the outage that sometimes causes it is not.
|
||||
mux.HandleFunc("/break-oplog-consumer", func(w http.ResponseWriter, r *http.Request) {
|
||||
oplogMu.Lock()
|
||||
sub := oplogSub
|
||||
oplogMu.Unlock()
|
||||
|
||||
if sub == nil {
|
||||
writeJSON(w, 200, map[string]interface{}{
|
||||
"broken": false, "reason": "no active subscription",
|
||||
})
|
||||
return
|
||||
}
|
||||
// Unsubscribe only. Clearing the latch or nilling oplogSub would hand the
|
||||
// supervisor the answer the probe is supposed to work out for itself.
|
||||
if err := sub.Unsubscribe(); err != nil {
|
||||
fail(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, map[string]interface{}{
|
||||
"broken": true,
|
||||
"attached": oplogAttached.Load(),
|
||||
"seq": utils.GetLastAppliedSeq(),
|
||||
})
|
||||
})
|
||||
|
||||
// stream-info exposes the op-log stream's actual replication. Nothing else can
|
||||
// see it, and without it two things are untestable: that R1 is raised to R3
|
||||
// once the third manager is up, and — the reason this was added — that a
|
||||
// "kill two of three managers" scenario is actually testing quorum loss. While
|
||||
// the stream is still R1 it lives on a single server, so if that server is the
|
||||
// survivor its writes keep succeeding and the scenario silently tests nothing.
|
||||
mux.HandleFunc("/stream-info", func(w http.ResponseWriter, r *http.Request) {
|
||||
clientConfigLock.RLock()
|
||||
jsl := js
|
||||
clientConfigLock.RUnlock()
|
||||
if jsl == nil {
|
||||
fail(w, errNoJS)
|
||||
return
|
||||
}
|
||||
name := oplogStreamName(utils.GetOplogEpoch())
|
||||
si, err := jsl.StreamInfo(name)
|
||||
if err != nil {
|
||||
writeJSON(w, 200, map[string]interface{}{
|
||||
"exists": false, "name": name, "error": err.Error(),
|
||||
})
|
||||
return
|
||||
}
|
||||
// configured replicas is what was asked for; the peer count is what JetStream
|
||||
// actually placed, and only the second one means the data is really replicated
|
||||
peers, leader := 0, ""
|
||||
if si.Cluster != nil {
|
||||
peers = len(si.Cluster.Replicas) + 1
|
||||
leader = si.Cluster.Leader
|
||||
}
|
||||
writeJSON(w, 200, map[string]interface{}{
|
||||
"exists": true, "name": name,
|
||||
"replicas": si.Config.Replicas, "peers": peers, "leader": leader,
|
||||
"msgs": si.State.Msgs, "firstSeq": si.State.FirstSeq, "lastSeq": si.State.LastSeq,
|
||||
})
|
||||
})
|
||||
|
||||
// enroll runs the real enrollment path in THIS node's process: DeviceCreate
|
||||
// plus the client YAML the new node boots from. A replacement manager joining
|
||||
// after a force-reform has to be enrolled by the survivor — enrolling it in
|
||||
// the parent test process would skip the survivor's post-reform device table
|
||||
// and its new epoch, which is the part under test.
|
||||
mux.HandleFunc("/enroll", func(w http.ResponseWriter, r *http.Request) {
|
||||
var body struct {
|
||||
DeviceName string
|
||||
IP string
|
||||
CosmosNode int
|
||||
IsLighthouse bool
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
fail(w, err)
|
||||
return
|
||||
}
|
||||
|
||||
cert, key, _, req, err := DeviceCreate(DeviceCreateRequestJSON{
|
||||
DeviceName: body.DeviceName,
|
||||
IP: body.IP,
|
||||
IsLighthouse: body.IsLighthouse,
|
||||
CosmosNode: body.CosmosNode,
|
||||
Port: "4242",
|
||||
})
|
||||
if err != nil {
|
||||
failStore(w, err)
|
||||
return
|
||||
}
|
||||
|
||||
capki, err := getCApki()
|
||||
if err != nil {
|
||||
fail(w, err)
|
||||
return
|
||||
}
|
||||
|
||||
yml, err := getYAMLClientConfig(body.DeviceName, utils.CONFIGFOLDER+"nebula.yml",
|
||||
capki, cert, key, req.APIKey, utils.ConstellationDevice{
|
||||
DeviceName: body.DeviceName,
|
||||
IP: req.IP,
|
||||
IsLighthouse: body.IsLighthouse,
|
||||
CosmosNode: body.CosmosNode,
|
||||
Port: "4242",
|
||||
APIKey: req.APIKey,
|
||||
}, false, false)
|
||||
if err != nil {
|
||||
fail(w, err)
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, 200, map[string]interface{}{
|
||||
"nebulaYml": yml,
|
||||
"apiKey": req.APIKey,
|
||||
})
|
||||
})
|
||||
|
||||
// edit-device mutates the local device DB the way the edit API does: under
|
||||
// the op-log that publishes one entry, which every node applies in order
|
||||
mux.HandleFunc("/edit-device", func(w http.ResponseWriter, r *http.Request) {
|
||||
var body struct{ DeviceName, Nickname string }
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
fail(w, err)
|
||||
return
|
||||
}
|
||||
err := utils.UpdateDevices(
|
||||
map[string]interface{}{"DeviceName": body.DeviceName},
|
||||
map[string]interface{}{"Nickname": body.Nickname})
|
||||
if err != nil {
|
||||
failStore(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, map[string]interface{}{"ok": true, "status": 200})
|
||||
})
|
||||
|
||||
mux.HandleFunc("/block-device", func(w http.ResponseWriter, r *http.Request) {
|
||||
var body struct {
|
||||
DeviceName string
|
||||
Blocked bool
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
fail(w, err)
|
||||
return
|
||||
}
|
||||
err := utils.UpdateDevices(
|
||||
map[string]interface{}{"DeviceName": body.DeviceName},
|
||||
map[string]interface{}{"Blocked": body.Blocked})
|
||||
if err != nil {
|
||||
failStore(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, map[string]interface{}{"ok": true, "status": 200})
|
||||
})
|
||||
|
||||
// create-device: ConflictingWrites races two nodes on the same name/IP, and
|
||||
// SnapshotFastForward needs bulk writes to push the log past its 100-entry
|
||||
// retention. The existing endpoints can only mutate pre-enrolled devices.
|
||||
mux.HandleFunc("/create-device", func(w http.ResponseWriter, r *http.Request) {
|
||||
var body struct{ DeviceName, IP string }
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
fail(w, err)
|
||||
return
|
||||
}
|
||||
err := utils.CreateDevice(utils.ConstellationDevice{
|
||||
Nickname: body.DeviceName,
|
||||
DeviceName: body.DeviceName,
|
||||
IP: body.IP,
|
||||
APIKey: "e2e-" + body.DeviceName,
|
||||
})
|
||||
if err != nil {
|
||||
failStore(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, map[string]interface{}{"ok": true, "status": 200})
|
||||
})
|
||||
|
||||
// create-user: the PRIMARY KEY rejection path (users are keyed by nickname, so
|
||||
// a duplicate arrives as a different SQLite code than a duplicate device does).
|
||||
mux.HandleFunc("/create-user", func(w http.ResponseWriter, r *http.Request) {
|
||||
var body struct{ Nickname string }
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
fail(w, err)
|
||||
return
|
||||
}
|
||||
err := utils.CreateUser(utils.User{Nickname: body.Nickname, Password: "e2e-hash"})
|
||||
if err != nil {
|
||||
failStore(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, map[string]interface{}{"ok": true, "status": 200})
|
||||
})
|
||||
|
||||
// create-users bulk-writes N users in one request. SnapshotFastForward has to
|
||||
// push the log past its 100-entry retention while a node is down, and users are
|
||||
// the only affordable way: a device op bounces nebula on every node, so 120
|
||||
// device writes would be 120 mesh restarts. Server-side looping also keeps the
|
||||
// burst from being dominated by HTTP round trips.
|
||||
mux.HandleFunc("/create-users", func(w http.ResponseWriter, r *http.Request) {
|
||||
var body struct {
|
||||
Prefix string
|
||||
Count int
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
fail(w, err)
|
||||
return
|
||||
}
|
||||
created := 0
|
||||
for i := 0; i < body.Count; i++ {
|
||||
err := utils.CreateUser(utils.User{
|
||||
Nickname: fmt.Sprintf("%s-%04d", body.Prefix, i),
|
||||
Password: "e2e-hash",
|
||||
})
|
||||
if err != nil {
|
||||
// partial progress is the useful part of the failure: it says whether
|
||||
// the burst died on write 1 or write 119
|
||||
writeJSON(w, 500, map[string]interface{}{
|
||||
"created": created, "failedAt": i, "error": err.Error(),
|
||||
})
|
||||
return
|
||||
}
|
||||
created++
|
||||
}
|
||||
writeJSON(w, 200, map[string]interface{}{"created": created})
|
||||
})
|
||||
|
||||
// probe-nats answers "would these credentials still get in?" directly, instead
|
||||
// of inferring revocation from something downstream. Credential fencing is
|
||||
// asserted nowhere else: a node's own client always dials its own server
|
||||
// (InitNATSClient builds nats://<own IP>:4222), so a fenced node still reports
|
||||
// clientConnected — the only honest test is to dial a *survivor* with the
|
||||
// fenced identity and see it refused.
|
||||
mux.HandleFunc("/probe-nats", func(w http.ResponseWriter, r *http.Request) {
|
||||
var body struct{ URL, User, Password string }
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
fail(w, err)
|
||||
return
|
||||
}
|
||||
conn, err := connectNATSClient(body.URL, sanitizeNATSUsername(body.User), body.Password)
|
||||
if err != nil {
|
||||
writeJSON(w, 200, map[string]interface{}{"connected": false, "error": err.Error()})
|
||||
return
|
||||
}
|
||||
conn.Close()
|
||||
writeJSON(w, 200, map[string]interface{}{"connected": true})
|
||||
})
|
||||
|
||||
// force-reform drives the REAL handler, API_ForceReform, rather than a copy of
|
||||
// its steps: the request is re-issued carrying an admin AuthContext so
|
||||
// CheckPermissions passes. Reimplementing the reform sequence here would test
|
||||
// the harness's copy of it and would silently stop testing the product the
|
||||
// first time the two drifted — and reform is precisely the operation where a
|
||||
// missed step (the epoch bump, the jetstream wipe, blocking the dead managers)
|
||||
// is invisible until a stale node turns up months later.
|
||||
mux.HandleFunc("/force-reform", func(w http.ResponseWriter, r *http.Request) {
|
||||
admin := &utils.AuthContext{
|
||||
Nickname: "e2e-admin",
|
||||
Permissions: []utils.Permission{utils.PERM_ADMIN},
|
||||
IsSudoed: true,
|
||||
}
|
||||
req := r.Clone(context.WithValue(r.Context(), utils.AuthCtxKey, admin))
|
||||
req.Method = "POST"
|
||||
API_ForceReform(w, req)
|
||||
})
|
||||
}
|
||||
@@ -53,6 +53,16 @@ type e2eCluster struct {
|
||||
root string
|
||||
ha bool
|
||||
nodes map[string]*e2eNode
|
||||
|
||||
// kept from bootstrap so nodes provisioned later (enrollVia) get the same
|
||||
// self-signed cert as the rest of the cluster's NATS TLS listeners
|
||||
tlsCert string
|
||||
tlsKey string
|
||||
|
||||
// each node's NATS password, by device name. Fencing scenarios need to dial a
|
||||
// survivor with a fenced node's identity, which is the only direct way to ask
|
||||
// whether its credentials are actually dead.
|
||||
apiKeys map[string]string
|
||||
}
|
||||
|
||||
func (n *e2eNode) controlURL() string {
|
||||
@@ -71,21 +81,13 @@ func newE2ECluster(t *testing.T, ha bool, specs []e2eNodeSpec) *e2eCluster {
|
||||
t.Fatal("e2e: first spec must be the creator (manager + lighthouse)")
|
||||
}
|
||||
|
||||
c := &e2eCluster{t: t, root: t.TempDir(), ha: ha, nodes: map[string]*e2eNode{}}
|
||||
requireFreeControlPorts(t, specs)
|
||||
|
||||
c := &e2eCluster{t: t, root: t.TempDir(), ha: ha,
|
||||
nodes: map[string]*e2eNode{}, apiKeys: map[string]string{}}
|
||||
|
||||
for _, s := range specs {
|
||||
n := &e2eNode{
|
||||
spec: s,
|
||||
ip: fmt.Sprintf("127.0.1.%d", s.Octet),
|
||||
configDir: filepath.Join(c.root, s.Name, "config") + "/",
|
||||
workDir: filepath.Join(c.root, s.Name, "work"),
|
||||
logPath: filepath.Join(c.root, s.Name, "node.log"),
|
||||
}
|
||||
if err := os.MkdirAll(n.configDir, 0755); err != nil {
|
||||
t.Fatal("e2e: mkdir config:", err)
|
||||
}
|
||||
writeFakeNebulaWorkDir(t, n.workDir)
|
||||
c.nodes[s.Name] = n
|
||||
c.provision(s)
|
||||
}
|
||||
|
||||
c.bootstrap(specs)
|
||||
@@ -94,6 +96,8 @@ func newE2ECluster(t *testing.T, ha bool, specs []e2eNodeSpec) *e2eCluster {
|
||||
for _, n := range c.nodes {
|
||||
n.killSilent()
|
||||
}
|
||||
// after the nodes, because a live node would just respawn its stub
|
||||
c.reapNebulaStubs()
|
||||
c.scanLogs()
|
||||
if t.Failed() {
|
||||
c.preserveLogs()
|
||||
@@ -103,6 +107,92 @@ func newE2ECluster(t *testing.T, ha bool, specs []e2eNodeSpec) *e2eCluster {
|
||||
return c
|
||||
}
|
||||
|
||||
// provision lays down a node's directories and fake nebula work dir. It does
|
||||
// NOT enroll the node — bootstrap does that for the initial set, enrollVia for
|
||||
// nodes that join a running cluster later.
|
||||
func (c *e2eCluster) provision(s e2eNodeSpec) *e2eNode {
|
||||
c.t.Helper()
|
||||
n := &e2eNode{
|
||||
spec: s,
|
||||
ip: fmt.Sprintf("127.0.1.%d", s.Octet),
|
||||
configDir: filepath.Join(c.root, s.Name, "config") + "/",
|
||||
workDir: filepath.Join(c.root, s.Name, "work"),
|
||||
logPath: filepath.Join(c.root, s.Name, "node.log"),
|
||||
}
|
||||
if err := os.MkdirAll(n.configDir, 0755); err != nil {
|
||||
c.t.Fatal("e2e: mkdir config:", err)
|
||||
}
|
||||
writeFakeNebulaWorkDir(c.t, n.workDir)
|
||||
c.nodes[s.Name] = n
|
||||
return n
|
||||
}
|
||||
|
||||
// enrollVia provisions a brand-new node and enrolls it through a LIVE node's
|
||||
// control API, rather than in the parent process the way bootstrap does.
|
||||
//
|
||||
// This is the only faithful way to test a replacement manager joining after a
|
||||
// force-reform: enrollment has to run on the survivor, in formation mode,
|
||||
// against the survivor's own post-reform device table and its new epoch.
|
||||
// Enrolling in the parent would bypass exactly the code the scenario is about.
|
||||
func (c *e2eCluster) enrollVia(enroller string, s e2eNodeSpec) {
|
||||
c.t.Helper()
|
||||
|
||||
if _, exists := c.nodes[s.Name]; exists {
|
||||
c.t.Fatal("e2e: node already provisioned: " + s.Name)
|
||||
}
|
||||
if c.tlsCert == "" {
|
||||
c.t.Fatal("e2e: cluster has no stored TLS cert; bootstrap did not run")
|
||||
}
|
||||
|
||||
// Retried rather than one-shot: the enroller may be inside a mesh bounce from
|
||||
// its own previous write. Every attempt is logged, so an enrollment that only
|
||||
// works on the fourth try still shows up as a problem instead of being hidden.
|
||||
var out map[string]interface{}
|
||||
var err error
|
||||
deadline := time.Now().Add(90 * time.Second)
|
||||
for attempt := 1; ; attempt++ {
|
||||
out, err = c.node(enroller).post("/enroll", map[string]interface{}{
|
||||
"DeviceName": s.Name,
|
||||
"IP": fmt.Sprintf("127.0.1.%d", s.Octet),
|
||||
"CosmosNode": s.CosmosNode,
|
||||
"IsLighthouse": s.Lighthouse,
|
||||
})
|
||||
if err == nil {
|
||||
if attempt > 1 {
|
||||
c.t.Logf("e2e: enroll %s via %s succeeded on attempt %d", s.Name, enroller, attempt)
|
||||
}
|
||||
break
|
||||
}
|
||||
c.t.Logf("e2e: enroll %s via %s attempt %d failed: %v", s.Name, enroller, attempt, err)
|
||||
if time.Now().After(deadline) {
|
||||
c.t.Fatalf("e2e: enroll %s via %s: %v", s.Name, enroller, err)
|
||||
}
|
||||
time.Sleep(3 * time.Second)
|
||||
}
|
||||
yml, _ := out["nebulaYml"].(string)
|
||||
if yml == "" {
|
||||
c.t.Fatalf("e2e: enroll %s via %s returned no nebula config", s.Name, enroller)
|
||||
}
|
||||
if key, ok := out["apiKey"].(string); ok {
|
||||
c.apiKeys[s.Name] = key
|
||||
}
|
||||
|
||||
n := c.provision(s)
|
||||
if err := os.WriteFile(n.configDir+"nebula.yml", []byte(yml), 0600); err != nil {
|
||||
c.t.Fatal("e2e: write enrolled nebula.yml:", err)
|
||||
}
|
||||
|
||||
// the node's own main config, written the same way bootstrap writes the
|
||||
// non-creator ones
|
||||
prevFolder := utils.CONFIGFOLDER
|
||||
defer func() {
|
||||
utils.CONFIGFOLDER = prevFolder
|
||||
utils.CloseStore()
|
||||
}()
|
||||
utils.CONFIGFOLDER = n.configDir
|
||||
utils.SetBaseMainConfig(c.nodeConfig(s, c.tlsCert, c.tlsKey))
|
||||
}
|
||||
|
||||
// bootstrap runs the creator-side enrollment inside the parent test process:
|
||||
// CA generation, one DeviceCreate per node, and device config YAML generation
|
||||
// for every non-creator node. Package globals touched here are restored.
|
||||
@@ -113,9 +203,12 @@ func (c *e2eCluster) bootstrap(specs []e2eNodeSpec) {
|
||||
prevFolder := utils.CONFIGFOLDER
|
||||
prevFBL := utils.FBL
|
||||
defer func() {
|
||||
utils.CloseEmbeddedDB()
|
||||
utils.CONFIGFOLDER = prevFolder
|
||||
utils.FBL = prevFBL
|
||||
// close with no reopen (faithful port of the old CloseEmbeddedDB): the parent
|
||||
// has no use for a store after bootstrap, and prevFolder is the package default
|
||||
// /var/lib/cosmos/, which does not exist under test
|
||||
utils.CloseStore()
|
||||
resetConstellationGlobals()
|
||||
}()
|
||||
|
||||
@@ -136,11 +229,30 @@ func (c *e2eCluster) bootstrap(specs []e2eNodeSpec) {
|
||||
if err != nil {
|
||||
t.Fatal("e2e: generate TLS cert:", err)
|
||||
}
|
||||
c.tlsCert, c.tlsKey = tlsCert, tlsKey
|
||||
|
||||
creator := c.nodes[specs[0].Name]
|
||||
utils.CONFIGFOLDER = creator.configDir
|
||||
utils.SetBaseMainConfig(c.nodeConfig(specs[0], tlsCert, tlsKey))
|
||||
|
||||
// DeviceCreate below writes through the store, which must point at the creator dir
|
||||
if err := utils.InitStore(); err != nil {
|
||||
t.Fatal("e2e: InitStore creator:", err)
|
||||
}
|
||||
|
||||
// The creator takes the formation write licence, exactly as the real creation
|
||||
// handler does (API_NewConstellation, api_nebula_connect.go:142). This harness
|
||||
// bootstraps by calling DeviceCreate in the parent process instead of going
|
||||
// through that handler, so it has to grant the licence itself — and it is not
|
||||
// optional bookkeeping: ensureOplogStream refuses to create a stream for a node
|
||||
// that is neither the formation writer nor already materialized from the log
|
||||
// (oplog.go:330-336), so without this NO node is licensed to create the op-log
|
||||
// and every node waits for a formation writer that does not exist. Observed as
|
||||
// a whole HA cluster sitting at e1/0 with divergent dumps and no stream.
|
||||
if err := utils.SetFormationWriter(utils.GetOplogEpoch()); err != nil {
|
||||
t.Fatal("e2e: set formation writer on the creator:", err)
|
||||
}
|
||||
|
||||
InitConfig()
|
||||
if err := generateNebulaCACert("cosmos-e2e"); err != nil {
|
||||
t.Fatal("e2e: generate CA:", err)
|
||||
@@ -168,6 +280,7 @@ func (c *e2eCluster) bootstrap(specs []e2eNodeSpec) {
|
||||
t.Fatal("e2e: DeviceCreate "+s.Name+":", err)
|
||||
}
|
||||
created[s.Name] = enrolled{cert: cert, key: key, req: req}
|
||||
c.apiKeys[s.Name] = req.APIKey
|
||||
}
|
||||
|
||||
capki, err := getCApki()
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"net/http"
|
||||
"os"
|
||||
"runtime"
|
||||
"sort"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -42,6 +43,10 @@ func TestE2ENodeMain(t *testing.T) {
|
||||
|
||||
utils.LoadBaseMainConfig(utils.ReadConfigFromFile())
|
||||
|
||||
if err := utils.InitStore(); err != nil {
|
||||
t.Fatal("e2e node: InitStore:", err)
|
||||
}
|
||||
|
||||
// control API first, so the parent can observe startup states too
|
||||
quit := make(chan struct{})
|
||||
go e2eControlServer(os.Getenv("COSMOS_E2E_CONTROL_ADDR"), quit)
|
||||
@@ -54,24 +59,32 @@ func TestE2ENodeMain(t *testing.T) {
|
||||
func e2eControlServer(addr string, quit chan struct{}) {
|
||||
mux := http.NewServeMux()
|
||||
|
||||
writeJSON := func(w http.ResponseWriter, code int, v interface{}) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(code)
|
||||
json.NewEncoder(w).Encode(v)
|
||||
}
|
||||
fail := func(w http.ResponseWriter, err error) {
|
||||
writeJSON(w, 500, map[string]interface{}{"error": err.Error()})
|
||||
}
|
||||
// defined at package level in e2e_control_oplog_test.go so handlers can live
|
||||
// in either file
|
||||
writeJSON := e2eWriteJSON
|
||||
fail := e2eFail
|
||||
failStore := e2eFailStore
|
||||
|
||||
e2eRegisterOplogControl(mux)
|
||||
|
||||
// fast, no network calls — safe to poll aggressively
|
||||
mux.HandleFunc("/status", func(w http.ResponseWriter, r *http.Request) {
|
||||
name, _ := GetCurrentDeviceName()
|
||||
leafs := -1
|
||||
routes := -1
|
||||
if ns != nil {
|
||||
leafs = ns.NumLeafNodes()
|
||||
routes = ns.NumRoutes()
|
||||
if srv := ns.Load(); srv != nil {
|
||||
leafs = srv.NumLeafNodes()
|
||||
routes = srv.NumRoutes()
|
||||
}
|
||||
// nebulaPid changing across an op is how the parent tells a mesh restart
|
||||
// from a plain device-cache refresh
|
||||
nebulaPid := -1
|
||||
ProcessMux.Lock()
|
||||
if process != nil && process.Process != nil {
|
||||
nebulaPid = process.Process.Pid
|
||||
}
|
||||
ProcessMux.Unlock()
|
||||
|
||||
writeJSON(w, 200, map[string]interface{}{
|
||||
"name": name,
|
||||
"agent": utils.FBL.AgentMode,
|
||||
@@ -82,6 +95,7 @@ func e2eControlServer(addr string, quit chan struct{}) {
|
||||
"goroutines": runtime.NumGoroutine(),
|
||||
"leafs": leafs,
|
||||
"routes": routes,
|
||||
"nebulaPid": nebulaPid,
|
||||
})
|
||||
})
|
||||
|
||||
@@ -104,12 +118,13 @@ func e2eControlServer(addr string, quit chan struct{}) {
|
||||
|
||||
// raw route/leaf state from the embedded server's monitoring API
|
||||
mux.HandleFunc("/routez", func(w http.ResponseWriter, r *http.Request) {
|
||||
if ns == nil {
|
||||
srv := ns.Load()
|
||||
if srv == nil {
|
||||
fail(w, errNoJS)
|
||||
return
|
||||
}
|
||||
routez, errR := ns.Routez(nil)
|
||||
leafz, errL := ns.Leafz(nil)
|
||||
routez, errR := srv.Routez(nil)
|
||||
leafz, errL := srv.Leafz(nil)
|
||||
if errR != nil || errL != nil {
|
||||
writeJSON(w, 500, map[string]interface{}{"routezErr": fmt.Sprint(errR), "leafzErr": fmt.Sprint(errL)})
|
||||
return
|
||||
@@ -156,7 +171,7 @@ func e2eControlServer(addr string, quit chan struct{}) {
|
||||
})
|
||||
}
|
||||
writeJSON(w, 200, map[string]interface{}{
|
||||
"hash": e2eFileHash(utils.CONFIGFOLDER + "database"),
|
||||
"hash": e2eDumpHash(),
|
||||
"devices": list,
|
||||
})
|
||||
})
|
||||
@@ -188,63 +203,90 @@ func e2eControlServer(addr string, quit chan struct{}) {
|
||||
writeJSON(w, 200, map[string]interface{}{"ok": true})
|
||||
})
|
||||
|
||||
mux.HandleFunc("/sync-push", func(w http.ResponseWriter, r *http.Request) {
|
||||
SendNewDBSyncMessage()
|
||||
writeJSON(w, 200, map[string]interface{}{"ok": true})
|
||||
// op-log position: nodes have converged when their dumps AND their
|
||||
// sequences match, which is stricter than matching dumps alone
|
||||
mux.HandleFunc("/oplog", func(w http.ResponseWriter, r *http.Request) {
|
||||
st := GetNATSStatus()
|
||||
writeJSON(w, 200, map[string]interface{}{
|
||||
"attached": st.OplogAttached,
|
||||
"halted": st.OplogHalted,
|
||||
"haltReason": st.OplogHaltReason,
|
||||
"epoch": st.OplogEpoch,
|
||||
"seq": st.OplogSeq,
|
||||
"configWritable": st.ConfigWritable,
|
||||
// formation is the one state where a write may legally bypass the log,
|
||||
// so a scenario has to be able to tell "writable because it publishes"
|
||||
// from "writable because it is the formation writer"
|
||||
"formationWriter": utils.IsFormationWriter(),
|
||||
"streamSeen": oplogStreamSeen.Load(),
|
||||
})
|
||||
})
|
||||
|
||||
mux.HandleFunc("/sync-request", func(w http.ResponseWriter, r *http.Request) {
|
||||
SendRequestSyncMessage()
|
||||
writeJSON(w, 200, map[string]interface{}{"ok": true})
|
||||
})
|
||||
|
||||
// edit-device mutates the local device DB the way the edit API does,
|
||||
// bumping the database file mtime that drives sync conflict resolution
|
||||
mux.HandleFunc("/edit-device", func(w http.ResponseWriter, r *http.Request) {
|
||||
var body struct{ DeviceName, Nickname string }
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
fail(w, err)
|
||||
return
|
||||
}
|
||||
c, closeDb, err := utils.GetEmbeddedCollection(utils.GetRootAppId(), "devices")
|
||||
if err != nil {
|
||||
fail(w, err)
|
||||
return
|
||||
}
|
||||
_, err = c.UpdateOne(nil,
|
||||
map[string]interface{}{"DeviceName": body.DeviceName},
|
||||
map[string]interface{}{"$set": map[string]interface{}{"Nickname": body.Nickname}})
|
||||
closeDb()
|
||||
if err != nil {
|
||||
fail(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, map[string]interface{}{"ok": true})
|
||||
})
|
||||
|
||||
mux.HandleFunc("/block-device", func(w http.ResponseWriter, r *http.Request) {
|
||||
// set-device-fields drives the reaction diff: Tags/Nickname must NOT bounce
|
||||
// nebula, IP/Blocked must. Pair it with nebulaPid from /status — asserting only
|
||||
// "pid unchanged after a tag edit" would also pass if the pre-image were always
|
||||
// empty, which silently degrades EVERY topology change to a cache refresh.
|
||||
mux.HandleFunc("/set-device-fields", func(w http.ResponseWriter, r *http.Request) {
|
||||
var body struct {
|
||||
DeviceName string
|
||||
Blocked bool
|
||||
Tags []string
|
||||
IP string
|
||||
Nickname string
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
fail(w, err)
|
||||
return
|
||||
}
|
||||
c, closeDb, err := utils.GetEmbeddedCollection(utils.GetRootAppId(), "devices")
|
||||
fields := map[string]interface{}{}
|
||||
if body.Tags != nil {
|
||||
fields["Tags"] = body.Tags
|
||||
}
|
||||
if body.IP != "" {
|
||||
fields["IP"] = body.IP
|
||||
}
|
||||
if body.Nickname != "" {
|
||||
fields["Nickname"] = body.Nickname
|
||||
}
|
||||
if len(fields) == 0 {
|
||||
fail(w, errors.New("no fields given"))
|
||||
return
|
||||
}
|
||||
err := utils.UpdateDevices(map[string]interface{}{"DeviceName": body.DeviceName}, fields)
|
||||
if err != nil {
|
||||
failStore(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, map[string]interface{}{"ok": true, "status": 200})
|
||||
})
|
||||
|
||||
// domain-op exercises kind:"domain" — otherwise the entire domain-registry path
|
||||
// is untested end to end. api_tokens is the cheapest domain to set and read back.
|
||||
mux.HandleFunc("/domain-op", func(w http.ResponseWriter, r *http.Request) {
|
||||
var body struct{ Tokens []string }
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
fail(w, err)
|
||||
return
|
||||
}
|
||||
_, err = c.UpdateOne(nil,
|
||||
map[string]interface{}{"DeviceName": body.DeviceName},
|
||||
map[string]interface{}{"$set": map[string]interface{}{"Blocked": body.Blocked}})
|
||||
closeDb()
|
||||
if err != nil {
|
||||
fail(w, err)
|
||||
tokens := map[string]utils.APITokenConfig{}
|
||||
for _, name := range body.Tokens {
|
||||
tokens[name] = utils.APITokenConfig{Name: name, Description: "e2e"}
|
||||
}
|
||||
if err := PublishDomainOp(DomainAPITokens, tokens); err != nil {
|
||||
failStore(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, map[string]interface{}{"ok": true})
|
||||
writeJSON(w, 200, map[string]interface{}{"ok": true, "status": 200})
|
||||
})
|
||||
|
||||
// domain-state reads the applied domain back, so a domain op's propagation is
|
||||
// observable the way /db makes a table op's propagation observable.
|
||||
mux.HandleFunc("/domain-state", func(w http.ResponseWriter, r *http.Request) {
|
||||
names := []string{}
|
||||
for name := range utils.GetMainConfig().APITokens {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
writeJSON(w, 200, map[string]interface{}{"apiTokens": names})
|
||||
})
|
||||
|
||||
mux.HandleFunc("/restart", func(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
@@ -0,0 +1,485 @@
|
||||
//go:build e2e
|
||||
|
||||
package constellation
|
||||
|
||||
// Availability and catch-up scenarios for the op-log: what a node does when the
|
||||
// stream stops accepting writes, and what it does when it has been away longer
|
||||
// than the log remembers. Both are the paths where a wrong answer is silent —
|
||||
// a node that writes locally instead of refusing, or one that quietly resumes
|
||||
// mid-log after entries it never saw, looks healthy and is permanently forked.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// a burst of writes outruns the 20s default client timeout; these calls are
|
||||
// server-side loops, not the fast control endpoints the rest of the suite polls
|
||||
var e2eHTTPLong = &http.Client{Timeout: 5 * time.Minute}
|
||||
|
||||
// burstUsers writes count users on the node in a single request and returns how
|
||||
// many landed. User ops are the cheap op kind: no reaction, so no mesh restart.
|
||||
func (c *e2eCluster) burstUsers(node, prefix string, count int) (int, error) {
|
||||
body := fmt.Sprintf(`{"Prefix":%q,"Count":%d}`, prefix, count)
|
||||
resp, err := e2eHTTPLong.Post(c.node(node).controlURL()+"/create-users",
|
||||
"application/json", strings.NewReader(body))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
out := map[string]interface{}{}
|
||||
json.NewDecoder(resp.Body).Decode(&out)
|
||||
created, _ := out["created"].(float64)
|
||||
if resp.StatusCode != 200 {
|
||||
return int(created), fmt.Errorf("create-users on %s: HTTP %d after %v writes: %v",
|
||||
node, resp.StatusCode, out["created"], out["error"])
|
||||
}
|
||||
return int(created), nil
|
||||
}
|
||||
|
||||
// oplogPos returns the node's (epoch, seq); -1 seq means the node did not answer.
|
||||
func (c *e2eCluster) oplogPos(node string) (uint64, int64) {
|
||||
st, err := c.node(node).get("/oplog")
|
||||
if err != nil {
|
||||
return 0, -1
|
||||
}
|
||||
epoch, _ := st["epoch"].(float64)
|
||||
seq, _ := st["seq"].(float64)
|
||||
return uint64(epoch), int64(seq)
|
||||
}
|
||||
|
||||
func (c *e2eCluster) assertNotHalted(what string, names ...string) {
|
||||
c.t.Helper()
|
||||
for _, name := range names {
|
||||
st, err := c.node(name).get("/oplog")
|
||||
if err != nil {
|
||||
c.t.Errorf("%s: /oplog on %s: %v", what, name, err)
|
||||
continue
|
||||
}
|
||||
if halted, _ := st["halted"].(bool); halted {
|
||||
c.t.Errorf("%s: %s apply loop HALTED: %v", what, name, st["haltReason"])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// tryWrite attempts one user write and reports the classified outcome: "" on
|
||||
// success, otherwise the control API's error code ("read-only", "duplicate", …).
|
||||
func (c *e2eCluster) tryWrite(node, nickname string) (code string, err error) {
|
||||
out, err := c.node(node).post("/create-user", map[string]string{"Nickname": nickname})
|
||||
if err == nil {
|
||||
return "", nil
|
||||
}
|
||||
code, _ = out["code"].(string)
|
||||
if code == "" {
|
||||
code = "<no code: " + err.Error() + ">"
|
||||
}
|
||||
return code, err
|
||||
}
|
||||
|
||||
// streamLastSeq returns the op-log stream's last sequence as the node sees it,
|
||||
// or -1 if the stream isn't reachable from there.
|
||||
func (c *e2eCluster) streamLastSeq(node string) int64 {
|
||||
out, err := c.node(node).get("/stream-info")
|
||||
if err != nil {
|
||||
return -1
|
||||
}
|
||||
if exists, _ := out["exists"].(bool); !exists {
|
||||
return -1
|
||||
}
|
||||
last, _ := out["lastSeq"].(float64)
|
||||
return int64(last)
|
||||
}
|
||||
|
||||
// TestE2EOplogDeadConsumerRecovery: a node whose consumer dies while its stream
|
||||
// stays perfectly healthy must notice and rebuild it.
|
||||
//
|
||||
// This is the deterministic form of a failure that was first seen by luck. In a
|
||||
// full-suite gate run, an HA manager that never went down lost its ordered
|
||||
// consumer during a JetStream quorum outage and never got it back: it logged one
|
||||
// "Attached to cosmos-oplog-e1 from seq 1" and nothing afterwards, while both
|
||||
// revived peers reached seq 2. Its publishes were still ACKED, so every write
|
||||
// returned 503 apply-timeout rather than a 409, it fell permanently behind, and
|
||||
// it reported attached and writable throughout. That reproduced once in seven
|
||||
// runs, which is no basis for coverage of a defect that leaves the surviving
|
||||
// manager of an HA outage silently dead.
|
||||
//
|
||||
// So the test creates the state rather than the weather that sometimes causes
|
||||
// it: /break-oplog-consumer unsubscribes the live subscription and leaves the
|
||||
// attached latch set, which is what the outage left behind. The stream is never
|
||||
// disturbed — same epoch, StreamInfo answering normally — which is precisely why
|
||||
// the two original detach conditions (epoch moved, ErrStreamNotFound) are blind
|
||||
// to it, and why the probe has to key on progress instead.
|
||||
//
|
||||
// The wedge is asserted BEFORE the recovery is, so the test cannot pass without
|
||||
// having reached the defect: on code without the progress check the first half
|
||||
// passes and the second half fails.
|
||||
func TestE2EOplogDeadConsumerRecovery(t *testing.T) {
|
||||
c := newE2ECluster(t, false, []e2eNodeSpec{
|
||||
{Name: "prime", Octet: 1, CosmosNode: 2, Lighthouse: true},
|
||||
{Name: "agentone", Octet: 2, CosmosNode: 1, Agent: true},
|
||||
})
|
||||
|
||||
c.start("prime")
|
||||
c.waitConnected("prime", 90*time.Second)
|
||||
c.start("agentone")
|
||||
c.waitConnected("agentone", 120*time.Second)
|
||||
c.waitFormed(300*time.Second, "prime", "agentone")
|
||||
c.waitConverged(180*time.Second, "initial convergence", "prime", "agentone")
|
||||
c.waitWritable(120*time.Second, "prime", "agentone")
|
||||
|
||||
_, seqBefore := c.oplogPos("prime")
|
||||
|
||||
out, err := c.node("prime").post("/break-oplog-consumer", map[string]interface{}{})
|
||||
if err != nil {
|
||||
t.Fatal("e2e: break prime's consumer:", err)
|
||||
}
|
||||
if broken, _ := out["broken"].(bool); !broken {
|
||||
t.Fatalf("e2e: prime had no live subscription to break (%v) — it was not attached, so this run would prove nothing", out)
|
||||
}
|
||||
if attached, _ := out["attached"].(bool); !attached {
|
||||
t.Fatal("e2e: breaking the subscription also cleared the attached latch; the wedged state was not created")
|
||||
}
|
||||
|
||||
// --- half one: prove the wedge was real.
|
||||
//
|
||||
// Two independent pieces of evidence, and they are split deliberately.
|
||||
//
|
||||
// The STATE is established by the break response above: a live subscription
|
||||
// existed, it was torn down, and the attached latch survived. That is the
|
||||
// defect's shape and it is proven without reference to timing.
|
||||
//
|
||||
// The CONSEQUENCE is established here: the node's own write must not succeed.
|
||||
// A write that succeeds would mean the consumer was never really dead, which
|
||||
// is the one outcome that would make everything below vacuous — the same
|
||||
// vacuous-pass trap that let an earlier version of the snapshot scenario go
|
||||
// green while testing nothing.
|
||||
//
|
||||
// Both refusal codes are accepted, because both are correct and which one
|
||||
// appears is a race this test deliberately does not pin. While the node still
|
||||
// believes it is attached the publish is ACKED and never applied, giving 503
|
||||
// apply-timeout; once the stall detector has detached it, the node is
|
||||
// read-only and gives 409. Detection can be one probe (~6s, when the client's
|
||||
// async error handler arms the counter) or three (~18s, when it does not), and
|
||||
// this write is issued within milliseconds of the break, so ordinarily it is
|
||||
// the 503 — but requiring that would make the test flaky against the fast path
|
||||
// for no gain in rigour. A position-based guard ("attached and behind") was
|
||||
// tried and discarded: a node merely catching up looks identical.
|
||||
code, errW := c.tryWrite("prime", "wedged-write")
|
||||
if errW == nil {
|
||||
t.Fatal("e2e: prime's write SUCCEEDED immediately after its consumer was killed — the wedged state was never created, so this run proves nothing about recovery")
|
||||
}
|
||||
switch code {
|
||||
case "apply-timeout", "read-only":
|
||||
t.Logf("e2e: wedged write refused as %q (apply-timeout = still latched, read-only = stall already detected)", code)
|
||||
default:
|
||||
t.Fatalf("e2e: prime's write while wedged returned %q, want apply-timeout or read-only; anything else means the write failed for a reason unrelated to the dead consumer and the recovery assertion below would not be testing the reported defect", code)
|
||||
}
|
||||
|
||||
// drive the log further forward from the OTHER node, so the gap prime has to
|
||||
// close is real work rather than its own single failed op
|
||||
if created, errB := c.burstUsers("agentone", "dead-consumer", 12); errB != nil {
|
||||
t.Fatalf("e2e: agent burst: %v (created %d)", errB, created)
|
||||
}
|
||||
|
||||
// --- half two: it must dig itself out.
|
||||
// The probe runs every 6s and needs 3 consecutive stalled observations, so
|
||||
// recovery is expected in roughly 20s; the budget is generous because the
|
||||
// rebuild itself can lose a round to a busy server.
|
||||
c.waitForDetail(180*time.Second, "prime rebuilds its consumer and catches up",
|
||||
func() (bool, string) {
|
||||
st := c.oplogState("prime")
|
||||
last := c.streamLastSeq("agentone")
|
||||
return st.attached && !st.halted && last >= 0 && st.seq >= last,
|
||||
fmt.Sprintf("prime=%s streamLastSeq=%d", st, last)
|
||||
})
|
||||
|
||||
if _, seqAfter := c.oplogPos("prime"); seqAfter <= seqBefore {
|
||||
t.Errorf("prime's sequence did not advance past %d (now %d) — it reattached without catching up", seqBefore, seqAfter)
|
||||
}
|
||||
|
||||
// The symptom the user would actually report: writes answering 503 forever.
|
||||
// A node that has genuinely rebuilt its consumer applies its own op again.
|
||||
recovered := 0
|
||||
c.waitForDetail(120*time.Second, "prime's own writes work again", func() (bool, string) {
|
||||
recovered++
|
||||
code, errW := c.tryWrite("prime", fmt.Sprintf("post-wedge-%d", recovered))
|
||||
return errW == nil, fmt.Sprintf("attempt %d -> %q", recovered, code)
|
||||
})
|
||||
|
||||
c.waitConverged(180*time.Second, "converged after the consumer rebuild", "prime", "agentone")
|
||||
c.assertNotHalted("after the consumer rebuild", "prime", "agentone")
|
||||
}
|
||||
|
||||
// TestE2ENoQuorumReadOnly: an HA manager that loses quorum must refuse writes
|
||||
// with 409 read-only and keep serving reads, then recover when quorum returns.
|
||||
//
|
||||
// The failure this guards against is not "the write errors" — it is a write that
|
||||
// SUCCEEDS. oplogWriteMode's direct-write branches are all reachable on a live
|
||||
// cluster member (a device reaction calls RestartNebula, which clears
|
||||
// NebulaStarted and the device cache, so a real member briefly looks standalone),
|
||||
// and a manager that answers 200 by writing to its own SQLite while cut off from
|
||||
// the log has forked the cluster silently. So the assertion below is two-sided:
|
||||
// the write must fail, and it must fail with the read-only classification rather
|
||||
// than any error at all.
|
||||
func TestE2ENoQuorumReadOnly(t *testing.T) {
|
||||
c := newE2ECluster(t, true, []e2eNodeSpec{
|
||||
{Name: "prime", Octet: 1, CosmosNode: 2, Lighthouse: true},
|
||||
{Name: "mantwo", Octet: 2, CosmosNode: 2},
|
||||
{Name: "manthree", Octet: 3, CosmosNode: 2},
|
||||
})
|
||||
|
||||
for _, name := range []string{"prime", "mantwo", "manthree"} {
|
||||
c.start(name)
|
||||
c.waitConnected(name, 180*time.Second)
|
||||
}
|
||||
// Formation must be OVER before anything here means what it says: while the
|
||||
// formation writer still holds its licence its writes go straight to SQLite,
|
||||
// so they succeed without the log and the sequence never moves.
|
||||
c.waitFormed(420*time.Second, "prime", "mantwo", "manthree")
|
||||
c.waitConverged(300*time.Second, "initial convergence", "prime", "mantwo", "manthree")
|
||||
|
||||
// The stream is created R1 and raised to R3 asynchronously. Killing two
|
||||
// managers before that lands would leave prime holding the only replica, its
|
||||
// writes would keep succeeding, and this test would fail while the product was
|
||||
// correct — so wait for the replication that makes "lose two of three" mean
|
||||
// quorum loss at all.
|
||||
c.waitStreamReplicas(420*time.Second, "prime", 3)
|
||||
|
||||
// Baseline: with all three up this call must succeed, so a later failure is
|
||||
// attributable to quorum loss rather than to the endpoint never having worked.
|
||||
// Polled, not one-shot — an unrelated device reaction can have prime briefly
|
||||
// detached, and that 409 is correct behavior rather than the thing under test.
|
||||
base := 0
|
||||
c.waitForDetail(180*time.Second, "baseline write with full quorum", func() (bool, string) {
|
||||
base++
|
||||
code, errW := c.tryWrite("prime", fmt.Sprintf("quorum-baseline-%d", base))
|
||||
return errW == nil, fmt.Sprintf("attempt %d -> %q", base, code)
|
||||
})
|
||||
c.waitConverged(180*time.Second, "baseline write converged", "prime", "mantwo", "manthree")
|
||||
|
||||
devicesBefore := len(c.deviceView("prime"))
|
||||
if devicesBefore == 0 {
|
||||
t.Fatal("e2e: prime reports no devices before the kill — the read-availability assertion below would be vacuous")
|
||||
}
|
||||
|
||||
// two of three: the R3 stream has no quorum and cannot accept a publish
|
||||
c.kill("mantwo")
|
||||
c.kill("manthree")
|
||||
|
||||
// Writes may still succeed for a moment while the cluster notices the loss, so
|
||||
// poll — but the outcome we are polling for is specific. An "internal" or a
|
||||
// bare transport error would mean the read-only contract collapsed into a
|
||||
// generic 500, which the UI shows as an unexplained failure instead of the
|
||||
// "config read-only" toast.
|
||||
attempt := 0
|
||||
seen := []string{}
|
||||
c.waitForDetail(180*time.Second, "prime refuses writes without quorum", func() (bool, string) {
|
||||
attempt++
|
||||
code, _ := c.tryWrite("prime", fmt.Sprintf("quorum-lost-%d", attempt))
|
||||
if code != "" {
|
||||
seen = append(seen, code)
|
||||
}
|
||||
return code == "read-only", fmt.Sprintf("attempt %d -> %q", attempt, code)
|
||||
})
|
||||
for _, code := range seen {
|
||||
switch code {
|
||||
case "read-only", "apply-timeout":
|
||||
default:
|
||||
t.Errorf("write without quorum was rejected as %q, want read-only (or apply-timeout during the transition)", code)
|
||||
}
|
||||
}
|
||||
|
||||
// Baseline taken only once the node is settled into read-only. Sampling it
|
||||
// before the kill would be wrong: an attempt made while the cluster had not
|
||||
// yet noticed the loss can legitimately have succeeded and moved the log.
|
||||
_, seqReadOnly := c.oplogPos("prime")
|
||||
|
||||
// Reads keep serving from local SQLite throughout, and every further write
|
||||
// stays refused. Sampled repeatedly rather than once, because a read path that
|
||||
// only breaks when the client's reconnect backoff kicks in, or a write path
|
||||
// that starts answering 200 once the node decides it looks standalone, would
|
||||
// both survive a single check.
|
||||
for i := 0; i < 5; i++ {
|
||||
if got := len(c.deviceView("prime")); got != devicesBefore {
|
||||
t.Errorf("read while read-only returned %d devices, want %d (reads must never depend on quorum)", got, devicesBefore)
|
||||
}
|
||||
attempt++
|
||||
if code, err := c.tryWrite("prime", fmt.Sprintf("quorum-lost-%d", attempt)); err == nil {
|
||||
t.Fatalf("write SUCCEEDED on a manager with no quorum (sample %d) — the write was applied outside the log and this node has forked", i)
|
||||
} else if code != "read-only" {
|
||||
t.Errorf("sustained write refusal returned %q, want read-only", code)
|
||||
}
|
||||
time.Sleep(3 * time.Second)
|
||||
}
|
||||
|
||||
// nothing above is an apply-time fault, so the loop must still be alive
|
||||
c.assertNotHalted("during quorum loss", "prime")
|
||||
|
||||
// refused writes leave no trace: the log cannot have advanced
|
||||
if _, seqNow := c.oplogPos("prime"); seqNow != seqReadOnly {
|
||||
t.Errorf("op-log sequence moved from %d to %d while every write was being refused", seqReadOnly, seqNow)
|
||||
}
|
||||
|
||||
// recovery
|
||||
c.start("mantwo")
|
||||
c.start("manthree")
|
||||
c.waitConnected("mantwo", 180*time.Second)
|
||||
c.waitConnected("manthree", 180*time.Second)
|
||||
|
||||
// This is the natural, racy path to the dead-consumer defect that
|
||||
// TestE2EOplogDeadConsumerRecovery now covers deterministically — it reproduced
|
||||
// here once in seven full-suite runs. It is kept as a second net over the real
|
||||
// outage rather than a synthetic one, so the detail below reports the state
|
||||
// that distinguishes the two ways this can fail: "read-only" means quorum never
|
||||
// came back, whereas "apply-timeout" with the stream ahead of prime's sequence
|
||||
// means quorum returned but prime's consumer did not, which is the defect.
|
||||
recovered := 0
|
||||
c.waitForDetail(300*time.Second, "writes recover once quorum returns", func() (bool, string) {
|
||||
recovered++
|
||||
code, err := c.tryWrite("prime", fmt.Sprintf("quorum-back-%d", recovered))
|
||||
return err == nil, fmt.Sprintf("attempt %d -> %q; prime=%s streamLastSeq=%d",
|
||||
recovered, code, c.oplogState("prime"), c.streamLastSeq("mantwo"))
|
||||
})
|
||||
c.waitConverged(300*time.Second, "converged after quorum recovery", "prime", "mantwo", "manthree")
|
||||
c.assertNotHalted("after quorum recovery", "prime", "mantwo", "manthree")
|
||||
}
|
||||
|
||||
// TestE2ESnapshotFastForward: a node that was down while more than the log's
|
||||
// 100-entry window went by cannot replay its way back, and must not try. It has
|
||||
// to detect the gap, take a peer snapshot, and re-attach at the snapshot's
|
||||
// sequence — with the retry loop covering retention moving again mid-install.
|
||||
//
|
||||
// Burst size is what makes this test real: 120 writes against MaxMsgs 100
|
||||
// guarantees the returning node's next-needed sequence has been discarded, so
|
||||
// the replay path is not merely unlikely, it is impossible. A smaller burst
|
||||
// would let the node catch up by ordinary replay and the snapshot path would go
|
||||
// untested while the test still passed.
|
||||
func TestE2ESnapshotFastForward(t *testing.T) {
|
||||
const burst = 120
|
||||
|
||||
// Agent variant: the agent reaches JetStream over its leafnode, so its
|
||||
// snapshot request and its re-attach both traverse a link the managers'
|
||||
// do not.
|
||||
t.Run("agent", func(t *testing.T) {
|
||||
c := newE2ECluster(t, false, []e2eNodeSpec{
|
||||
{Name: "prime", Octet: 1, CosmosNode: 2, Lighthouse: true},
|
||||
{Name: "agentone", Octet: 2, CosmosNode: 1, Agent: true},
|
||||
})
|
||||
|
||||
c.start("prime")
|
||||
c.waitConnected("prime", 90*time.Second)
|
||||
c.start("agentone")
|
||||
c.waitConnected("agentone", 120*time.Second)
|
||||
// formation must be over, or prime's burst writes go direct to SQLite,
|
||||
// the log never advances, and the retention guard below correctly refuses
|
||||
// to call this a test of the snapshot path
|
||||
c.waitFormed(300*time.Second, "prime", "agentone")
|
||||
c.waitConverged(180*time.Second, "initial convergence", "prime", "agentone")
|
||||
c.waitWritable(120*time.Second, "prime")
|
||||
|
||||
_, seqAtKill := c.oplogPos("agentone")
|
||||
if seqAtKill < 0 {
|
||||
t.Fatal("e2e: could not read agentone's op-log position before killing it")
|
||||
}
|
||||
c.kill("agentone")
|
||||
|
||||
created, err := c.burstUsers("prime", "ff-agent", burst)
|
||||
if err != nil {
|
||||
t.Fatalf("e2e: burst on prime: %v (created %d)", err, created)
|
||||
}
|
||||
if created != burst {
|
||||
t.Fatalf("e2e: burst wrote %d of %d users; below the 100-entry retention the gap path is not forced", created, burst)
|
||||
}
|
||||
|
||||
_, seqAfterBurst := c.oplogPos("prime")
|
||||
if seqAfterBurst < seqAtKill+int64(oplogMaxMsgs) {
|
||||
t.Fatalf("e2e: log advanced from %d to %d, less than the %d-entry window — the entry agentone needs may still be in the stream, so this run would not test the snapshot path",
|
||||
seqAtKill, seqAfterBurst, oplogMaxMsgs)
|
||||
}
|
||||
|
||||
c.start("agentone")
|
||||
c.waitConnected("agentone", 180*time.Second)
|
||||
c.waitConverged(300*time.Second, "agent caught up past retention", "prime", "agentone")
|
||||
c.assertNotHalted("after agent fast-forward", "prime", "agentone")
|
||||
|
||||
// converged at the burst's own sequence, not at some earlier point both
|
||||
// nodes happen to share
|
||||
_, seqAgent := c.oplogPos("agentone")
|
||||
if seqAgent < seqAfterBurst {
|
||||
t.Errorf("agentone converged at seq %d, behind the burst's %d", seqAgent, seqAfterBurst)
|
||||
}
|
||||
})
|
||||
|
||||
// Manager variant, with writes continuing *through* the rejoin. This is the
|
||||
// case the mandatory gap re-check exists for: retention can move past a
|
||||
// snapshot between building it and installing it, and a node that skips the
|
||||
// re-check lands one entry short of the stream and stays there — attached,
|
||||
// unhalted, and permanently behind.
|
||||
t.Run("manager", func(t *testing.T) {
|
||||
c := newE2ECluster(t, true, []e2eNodeSpec{
|
||||
{Name: "prime", Octet: 1, CosmosNode: 2, Lighthouse: true},
|
||||
{Name: "mantwo", Octet: 2, CosmosNode: 2},
|
||||
{Name: "manthree", Octet: 3, CosmosNode: 2},
|
||||
})
|
||||
|
||||
for _, name := range []string{"prime", "mantwo", "manthree"} {
|
||||
c.start(name)
|
||||
c.waitConnected(name, 180*time.Second)
|
||||
}
|
||||
c.waitFormed(420*time.Second, "prime", "mantwo", "manthree")
|
||||
c.waitConverged(300*time.Second, "initial convergence", "prime", "mantwo", "manthree")
|
||||
c.waitWritable(180*time.Second, "prime")
|
||||
// mantwo must actually hold a replica for its absence to be a real gap
|
||||
c.waitStreamReplicas(420*time.Second, "prime", 3)
|
||||
|
||||
_, seqAtKill := c.oplogPos("mantwo")
|
||||
if seqAtKill < 0 {
|
||||
t.Fatal("e2e: could not read mantwo's op-log position before killing it")
|
||||
}
|
||||
// one of three: the surviving two keep quorum, so writes continue
|
||||
c.kill("mantwo")
|
||||
|
||||
created, err := c.burstUsers("prime", "ff-mgr", burst)
|
||||
if err != nil {
|
||||
t.Fatalf("e2e: burst on prime: %v (created %d)", err, created)
|
||||
}
|
||||
if created != burst {
|
||||
t.Fatalf("e2e: burst wrote %d of %d users; below the 100-entry retention the gap path is not forced", created, burst)
|
||||
}
|
||||
|
||||
_, seqAfterBurst := c.oplogPos("prime")
|
||||
if seqAfterBurst < seqAtKill+int64(oplogMaxMsgs) {
|
||||
t.Fatalf("e2e: log advanced from %d to %d, less than the %d-entry window — mantwo could still replay, so this run would not test the snapshot path",
|
||||
seqAtKill, seqAfterBurst, oplogMaxMsgs)
|
||||
}
|
||||
|
||||
// keep the log moving while mantwo comes back, so its snapshot has a real
|
||||
// chance of ageing out during install
|
||||
churn := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := c.burstUsers("prime", "ff-mgr-churn", burst)
|
||||
churn <- err
|
||||
}()
|
||||
|
||||
c.start("mantwo")
|
||||
c.waitConnected("mantwo", 240*time.Second)
|
||||
|
||||
if err := <-churn; err != nil {
|
||||
// a churn write may legitimately lose to a bounce window; it must not
|
||||
// die on anything else
|
||||
t.Logf("e2e: churn burst ended early: %v", err)
|
||||
}
|
||||
|
||||
c.waitConverged(420*time.Second, "manager caught up past retention while the log moved", "prime", "mantwo", "manthree")
|
||||
c.assertNotHalted("after manager fast-forward", "prime", "mantwo", "manthree")
|
||||
|
||||
_, seqMantwo := c.oplogPos("mantwo")
|
||||
if seqMantwo < seqAfterBurst {
|
||||
t.Errorf("mantwo converged at seq %d, behind the first burst's %d", seqMantwo, seqAfterBurst)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,349 @@
|
||||
//go:build e2e
|
||||
|
||||
package constellation
|
||||
|
||||
// M2 op-log scenarios. Written by the test gate rather than the author of the
|
||||
// apply loop: a test written from the same mental model as the implementation
|
||||
// shares its blind spots, and the pre-image and rejection paths below are
|
||||
// exactly where that would bite.
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// nebulaPid reads the node's nebula sub-process id; a restart replaces it.
|
||||
func (c *e2eCluster) nebulaPid(name string) int {
|
||||
st, err := c.node(name).get("/status")
|
||||
if err != nil {
|
||||
return -1
|
||||
}
|
||||
pid, ok := st["nebulaPid"].(float64)
|
||||
if !ok {
|
||||
return -1
|
||||
}
|
||||
return int(pid)
|
||||
}
|
||||
|
||||
// waitConverged fails the test with per-node epoch/seq/hash detail rather than
|
||||
// a bare boolean, so a convergence failure is diagnosable from the log alone.
|
||||
func (c *e2eCluster) waitConverged(d time.Duration, what string, names ...string) {
|
||||
c.t.Helper()
|
||||
c.waitForDetail(d, what, func() (bool, string) {
|
||||
return c.oplogConverged(names...)
|
||||
})
|
||||
}
|
||||
|
||||
// deviceNames returns the node's view of the device table, keyed by name.
|
||||
func (c *e2eCluster) deviceView(node string) map[string]map[string]interface{} {
|
||||
out := map[string]map[string]interface{}{}
|
||||
db, err := c.node(node).get("/db")
|
||||
if err != nil {
|
||||
return out
|
||||
}
|
||||
devices, _ := db["devices"].([]interface{})
|
||||
for _, d := range devices {
|
||||
dev, _ := d.(map[string]interface{})
|
||||
if n, ok := dev["deviceName"].(string); ok {
|
||||
out[n] = dev
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// settle waits for the node's nebula pid to hold steady, so a later pid
|
||||
// comparison can't be confounded by an enrollment restart still in flight.
|
||||
// (Learned the hard way: sampling before the mesh quiesces misattributes an
|
||||
// enrollment bounce to whatever write happened next.)
|
||||
func (c *e2eCluster) settleNebula(node string, stable time.Duration, limit time.Duration) int {
|
||||
c.t.Helper()
|
||||
pid := -1
|
||||
since := time.Time{}
|
||||
c.waitForDetail(limit, "nebula pid stable on "+node, func() (bool, string) {
|
||||
cur := c.nebulaPid(node)
|
||||
if cur <= 0 {
|
||||
pid, since = -1, time.Time{}
|
||||
return false, fmt.Sprintf("%s pid=%d", node, cur)
|
||||
}
|
||||
if cur != pid {
|
||||
pid, since = cur, time.Now()
|
||||
return false, fmt.Sprintf("%s pid moved to %d", node, cur)
|
||||
}
|
||||
return time.Since(since) >= stable, fmt.Sprintf("%s pid=%d steady %s", node, cur, time.Since(since).Truncate(time.Second))
|
||||
})
|
||||
return pid
|
||||
}
|
||||
|
||||
// TestE2EOplogPropagation: one op of every kind published from one node must
|
||||
// land on every other node, with all nodes agreeing on (epoch, seq) — not just
|
||||
// on content. Equal dumps with unequal seq would mean divergent logs that
|
||||
// happen to look the same right now.
|
||||
// waitWritable blocks until every named node reports a writable op-log. Writes
|
||||
// racing a designed detach window (post-reaction RestartNebula, post-install
|
||||
// bounce) get a correct 409 — a scenario that means to test replication rather
|
||||
// than the window must wait it out first.
|
||||
func (c *e2eCluster) waitWritable(d time.Duration, names ...string) {
|
||||
c.t.Helper()
|
||||
for _, n := range names {
|
||||
node := n
|
||||
c.waitForDetail(d, "op-log writable on "+node, func() (bool, string) {
|
||||
st, err := c.node(node).get("/oplog")
|
||||
if err != nil {
|
||||
return false, err.Error()
|
||||
}
|
||||
w, _ := st["configWritable"].(bool)
|
||||
halted, _ := st["halted"].(bool)
|
||||
return w && !halted, fmt.Sprint("writable=", w, " halted=", halted)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestE2EOplogPropagation(t *testing.T) {
|
||||
c := newE2ECluster(t, false, []e2eNodeSpec{
|
||||
{Name: "prime", Octet: 1, CosmosNode: 2, Lighthouse: true},
|
||||
{Name: "agentone", Octet: 2, CosmosNode: 1, Agent: true},
|
||||
})
|
||||
|
||||
c.start("prime")
|
||||
c.waitConnected("prime", 90*time.Second)
|
||||
c.start("agentone")
|
||||
c.waitConnected("agentone", 120*time.Second)
|
||||
c.waitConverged(150*time.Second, "initial convergence", "prime", "agentone")
|
||||
c.waitWritable(90*time.Second, "prime")
|
||||
|
||||
// kind=db, table=devices
|
||||
if _, err := c.node("prime").post("/create-device",
|
||||
map[string]string{"DeviceName": "oplog-dev", "IP": "192.168.201.90"}); err != nil {
|
||||
t.Fatal("e2e: create-device:", err)
|
||||
}
|
||||
// the device insert's reaction bounces nebula (and detaches the loop) on every
|
||||
// node; wait out prime's own window before the next write
|
||||
c.waitWritable(90*time.Second, "prime")
|
||||
// kind=db, table=users (PRIMARY KEY table — different code path to devices)
|
||||
if _, err := c.node("prime").post("/create-user",
|
||||
map[string]string{"Nickname": "oplog-user"}); err != nil {
|
||||
t.Fatal("e2e: create-user:", err)
|
||||
}
|
||||
// kind=domain
|
||||
if _, err := c.node("prime").post("/domain-op",
|
||||
map[string]interface{}{"Tokens": []string{"tok-a", "tok-b"}}); err != nil {
|
||||
t.Fatal("e2e: domain-op:", err)
|
||||
}
|
||||
|
||||
c.waitConverged(120*time.Second, "all op kinds converged", "prime", "agentone")
|
||||
|
||||
if _, ok := c.deviceView("agentone")["oplog-dev"]; !ok {
|
||||
t.Error("device op did not reach agentone")
|
||||
}
|
||||
|
||||
// the domain op needs its own read side: converged dumps only cover SQLite,
|
||||
// and domain state lives in config.json
|
||||
c.waitForDetail(60*time.Second, "domain op visible on agentone", func() (bool, string) {
|
||||
st, err := c.node("agentone").get("/domain-state")
|
||||
if err != nil {
|
||||
return false, err.Error()
|
||||
}
|
||||
toks, _ := st["apiTokens"].([]interface{})
|
||||
return len(toks) == 2, fmt.Sprint("apiTokens=", toks)
|
||||
})
|
||||
}
|
||||
|
||||
// TestE2EOplogAgentWrite: an agent's write goes out over its leafnode to the
|
||||
// manager's JetStream and comes back through the log like any other. Agents
|
||||
// being uniform with managers is a load-bearing M2 claim.
|
||||
func TestE2EOplogAgentWrite(t *testing.T) {
|
||||
c := newE2ECluster(t, false, []e2eNodeSpec{
|
||||
{Name: "prime", Octet: 1, CosmosNode: 2, Lighthouse: true},
|
||||
{Name: "agentone", Octet: 2, CosmosNode: 1, Agent: true},
|
||||
})
|
||||
|
||||
c.start("prime")
|
||||
c.waitConnected("prime", 90*time.Second)
|
||||
c.start("agentone")
|
||||
c.waitConnected("agentone", 120*time.Second)
|
||||
c.waitConverged(150*time.Second, "initial convergence", "prime", "agentone")
|
||||
// the agent may still be inside its own post-snapshot-install mesh bounce;
|
||||
// its ladder correctly answers read-only until it re-attaches
|
||||
c.waitWritable(90*time.Second, "agentone", "prime")
|
||||
|
||||
if _, err := c.node("agentone").post("/create-device",
|
||||
map[string]string{"DeviceName": "from-agent", "IP": "192.168.201.91"}); err != nil {
|
||||
t.Fatal("e2e: agent create-device:", err)
|
||||
}
|
||||
|
||||
c.waitConverged(120*time.Second, "agent write converged", "prime", "agentone")
|
||||
|
||||
if _, ok := c.deviceView("prime")["from-agent"]; !ok {
|
||||
t.Error("agent-originated write never reached the manager")
|
||||
}
|
||||
}
|
||||
|
||||
// TestE2EOplogConflictingWrites: two nodes race the same device IP and the same
|
||||
// user nickname. Exactly one of each must win everywhere, and the loser must be
|
||||
// rejected with 409 duplicate rather than halting anyone's apply loop.
|
||||
//
|
||||
// Both paths matter and they are NOT the same code: devices reject through
|
||||
// SQLITE_CONSTRAINT_UNIQUE (partial indexes) and users through
|
||||
// SQLITE_CONSTRAINT_PRIMARYKEY (nickname PK). A classification that handles
|
||||
// only UNIQUE turns a duplicate user into an unrecognized error and halts the
|
||||
// apply loop on every node — a cluster-wide stop from a routine name clash.
|
||||
func TestE2EOplogConflictingWrites(t *testing.T) {
|
||||
c := newE2ECluster(t, false, []e2eNodeSpec{
|
||||
{Name: "prime", Octet: 1, CosmosNode: 2, Lighthouse: true},
|
||||
{Name: "agentone", Octet: 2, CosmosNode: 1, Agent: true},
|
||||
})
|
||||
|
||||
c.start("prime")
|
||||
c.waitConnected("prime", 90*time.Second)
|
||||
c.start("agentone")
|
||||
c.waitConnected("agentone", 120*time.Second)
|
||||
c.waitConverged(150*time.Second, "initial convergence", "prime", "agentone")
|
||||
|
||||
race := func(path string, body interface{}) (okCount, dupCount int, loserCodes []string, statuses []string) {
|
||||
var mu sync.Mutex
|
||||
var wg sync.WaitGroup
|
||||
for _, node := range []string{"prime", "agentone"} {
|
||||
wg.Add(1)
|
||||
go func(n string) {
|
||||
defer wg.Done()
|
||||
out, err := c.node(n).post(path, body)
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
code, _ := out["code"].(string)
|
||||
status, _ := out["status"].(float64)
|
||||
statuses = append(statuses, fmt.Sprintf("%s=%v/%v(err=%v)", n, status, code, err))
|
||||
switch {
|
||||
case err == nil:
|
||||
okCount++
|
||||
case code == "duplicate":
|
||||
dupCount++
|
||||
loserCodes = append(loserCodes, code)
|
||||
default:
|
||||
loserCodes = append(loserCodes, code)
|
||||
}
|
||||
}(node)
|
||||
}
|
||||
wg.Wait()
|
||||
return
|
||||
}
|
||||
|
||||
// A loser may legitimately lose to the index OR to a bounce window, but not
|
||||
// to anything else: an internal/not-found would mean a duplicate stopped
|
||||
// being classified, which is the regression that halts apply loops.
|
||||
assertLoserCodes := func(what string, codes []string) {
|
||||
t.Helper()
|
||||
for _, code := range codes {
|
||||
switch code {
|
||||
case "duplicate", "read-only", "apply-timeout":
|
||||
default:
|
||||
t.Errorf("%s: loser returned code %q, want duplicate/read-only/apply-timeout", what, code)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// devices: UNIQUE(ip) WHERE blocked=0. The winner's apply bounces nebula on
|
||||
// BOTH nodes mid-race, so the loser may see read-only/timeout instead of the
|
||||
// duplicate rejection — all are correct outcomes here. The invariants that
|
||||
// must hold: exactly one winner, never two, and nobody halts (checked below).
|
||||
c.waitWritable(90*time.Second, "prime", "agentone")
|
||||
okD, dupD, codesD, stD := race("/create-device", map[string]string{"DeviceName": "racer", "IP": "192.168.201.92"})
|
||||
if okD != 1 {
|
||||
t.Errorf("device race: want exactly 1 winner, got ok=%d dup=%d (%v)", okD, dupD, stD)
|
||||
}
|
||||
assertLoserCodes("device race", codesD)
|
||||
if dupD == 0 {
|
||||
t.Logf("device race: loser lost to the reaction window rather than the index (%v)", stD)
|
||||
}
|
||||
|
||||
// users: PRIMARY KEY(nickname) — a different SQLite code reaching the same
|
||||
// rejection; this is the one that halts the loop if misclassified. User ops
|
||||
// have no reaction, so no window opens mid-race: strict 1-winner-1-duplicate.
|
||||
c.waitWritable(90*time.Second, "prime", "agentone")
|
||||
okU, dupU, codesU, stU := race("/create-user", map[string]string{"Nickname": "racer-user"})
|
||||
if okU != 1 || dupU != 1 {
|
||||
t.Errorf("user race: want exactly 1 winner and 1 duplicate, got ok=%d dup=%d (%v)", okU, dupU, stU)
|
||||
}
|
||||
assertLoserCodes("user race", codesU)
|
||||
|
||||
// the decisive part: a rejection must commit its seq and leave every loop
|
||||
// running, so the cluster still converges afterwards
|
||||
c.waitConverged(120*time.Second, "converged after conflicting writes", "prime", "agentone")
|
||||
|
||||
for _, n := range []string{"prime", "agentone"} {
|
||||
st, err := c.node(n).get("/oplog")
|
||||
if err != nil {
|
||||
t.Fatalf("e2e: /oplog on %s: %v", n, err)
|
||||
}
|
||||
if halted, _ := st["halted"].(bool); halted {
|
||||
t.Errorf("%s apply loop HALTED after a duplicate: %v", n, st["haltReason"])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestE2EOplogDeviceReactionWiring pins the apply loop's pre-image diff in BOTH
|
||||
// directions, which is the only way to catch it.
|
||||
//
|
||||
// A Tags-only edit must not bounce nebula; an IP change must. Asserting only
|
||||
// the first is worse than useless: if ApplyOpTx hands the reaction an empty
|
||||
// pre-image, DeviceFieldsChanged returns false for everything, every topology
|
||||
// change silently degrades to a cache refresh, the mesh never reconverges — and
|
||||
// a pid-stability-only test goes GREEN precisely when the system is broken.
|
||||
func TestE2EOplogDeviceReactionWiring(t *testing.T) {
|
||||
c := newE2ECluster(t, false, []e2eNodeSpec{
|
||||
{Name: "prime", Octet: 1, CosmosNode: 2, Lighthouse: true},
|
||||
{Name: "agentone", Octet: 2, CosmosNode: 1, Agent: true},
|
||||
})
|
||||
|
||||
c.start("prime")
|
||||
c.waitConnected("prime", 90*time.Second)
|
||||
c.start("agentone")
|
||||
c.waitConnected("agentone", 120*time.Second)
|
||||
c.waitConverged(150*time.Second, "initial convergence", "prime", "agentone")
|
||||
|
||||
if _, err := c.node("prime").post("/create-device",
|
||||
map[string]string{"DeviceName": "reactor", "IP": "192.168.201.93"}); err != nil {
|
||||
t.Fatal("e2e: create-device:", err)
|
||||
}
|
||||
c.waitConverged(120*time.Second, "reactor device converged", "prime", "agentone")
|
||||
|
||||
before := c.settleNebula("agentone", 10*time.Second, 180*time.Second)
|
||||
if before <= 0 {
|
||||
t.Fatalf("e2e: agentone has no nebula process (pid %d); both assertions below would be vacuous", before)
|
||||
}
|
||||
|
||||
// --- direction 1: Tags-only edit must NOT restart nebula
|
||||
if _, err := c.node("prime").post("/set-device-fields",
|
||||
map[string]interface{}{"DeviceName": "reactor", "Tags": []string{"prod", "eu"}}); err != nil {
|
||||
t.Fatal("e2e: set tags:", err)
|
||||
}
|
||||
c.waitConverged(120*time.Second, "tags edit converged", "prime", "agentone")
|
||||
time.Sleep(8 * time.Second) // a bounce provoked by this op would land by now
|
||||
|
||||
afterTags := c.nebulaPid("agentone")
|
||||
if afterTags != before {
|
||||
t.Errorf("Tags-only edit bounced nebula on agentone: pid %d -> %d (pre-image diff classified Tags as topology)", before, afterTags)
|
||||
}
|
||||
|
||||
// --- direction 2: IP change MUST restart nebula.
|
||||
// This is the load-bearing half: it fails if the pre-image is empty, which
|
||||
// is the failure mode direction 1 cannot see.
|
||||
if _, err := c.node("prime").post("/set-device-fields",
|
||||
map[string]interface{}{"DeviceName": "reactor", "IP": "192.168.201.94"}); err != nil {
|
||||
t.Fatal("e2e: set ip:", err)
|
||||
}
|
||||
c.waitConverged(120*time.Second, "ip edit converged", "prime", "agentone")
|
||||
|
||||
restarted := false
|
||||
c.waitForDetail(90*time.Second, "IP change restarts nebula on agentone", func() (bool, string) {
|
||||
cur := c.nebulaPid("agentone")
|
||||
if cur > 0 && cur != afterTags {
|
||||
restarted = true
|
||||
}
|
||||
return restarted, fmt.Sprintf("pid %d -> %d", afterTags, cur)
|
||||
})
|
||||
if !restarted {
|
||||
t.Errorf("IP change did NOT restart nebula on agentone (pid stayed %d) — topology change silently degraded to a cache refresh; the mesh would never reconverge", afterTags)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,349 @@
|
||||
//go:build e2e
|
||||
|
||||
package constellation
|
||||
|
||||
// M3: force-reform and the fencing that has to hold around it.
|
||||
//
|
||||
// Reform is the one operation that deliberately splits a constellation's
|
||||
// history in two, and every guarantee it makes is about what the ABANDONED half
|
||||
// can no longer do. So these scenarios spend most of their assertions on the
|
||||
// losing side: a survivor that can write again is easy, a stale node that can
|
||||
// still be heard is the failure that matters and the one that looks like
|
||||
// nothing is wrong.
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// oplogState is the node's full op-log position and formation state in one read,
|
||||
// so a failure message can say which of the three it was that disagreed.
|
||||
type oplogState struct {
|
||||
epoch uint64
|
||||
seq int64
|
||||
attached bool
|
||||
halted bool
|
||||
formationWriter bool
|
||||
streamSeen bool
|
||||
configWritable bool
|
||||
err error
|
||||
}
|
||||
|
||||
func (c *e2eCluster) oplogState(node string) oplogState {
|
||||
st, err := c.node(node).get("/oplog")
|
||||
if err != nil {
|
||||
return oplogState{err: err, seq: -1}
|
||||
}
|
||||
epoch, _ := st["epoch"].(float64)
|
||||
seq, _ := st["seq"].(float64)
|
||||
attached, _ := st["attached"].(bool)
|
||||
halted, _ := st["halted"].(bool)
|
||||
fw, _ := st["formationWriter"].(bool)
|
||||
seen, _ := st["streamSeen"].(bool)
|
||||
writable, _ := st["configWritable"].(bool)
|
||||
return oplogState{
|
||||
epoch: uint64(epoch), seq: int64(seq), attached: attached, halted: halted,
|
||||
formationWriter: fw, streamSeen: seen, configWritable: writable,
|
||||
}
|
||||
}
|
||||
|
||||
func (s oplogState) String() string {
|
||||
if s.err != nil {
|
||||
return "unreachable(" + s.err.Error() + ")"
|
||||
}
|
||||
return fmt.Sprintf("e%d/%d attached=%v formationWriter=%v streamSeen=%v writable=%v halted=%v",
|
||||
s.epoch, s.seq, s.attached, s.formationWriter, s.streamSeen, s.configWritable, s.halted)
|
||||
}
|
||||
|
||||
// forceReform triggers the product's own force-reform handler on the node and
|
||||
// returns the epoch it reports. The handler answers in the standard API envelope
|
||||
// ({status, data:{...}}), so the epoch is unwrapped from data rather than read
|
||||
// off the top level.
|
||||
func (c *e2eCluster) forceReform(node string) uint64 {
|
||||
c.t.Helper()
|
||||
out, err := c.node(node).post("/force-reform", map[string]interface{}{})
|
||||
if err != nil {
|
||||
c.t.Fatalf("e2e: force-reform on %s: %v", node, err)
|
||||
}
|
||||
data, ok := out["data"].(map[string]interface{})
|
||||
if !ok {
|
||||
c.t.Fatalf("e2e: force-reform on %s returned no data envelope: %v", node, out)
|
||||
}
|
||||
epoch, ok := data["epoch"].(float64)
|
||||
if !ok || epoch == 0 {
|
||||
c.t.Fatalf("e2e: force-reform on %s reported no epoch: %v", node, data)
|
||||
}
|
||||
c.t.Logf("e2e: force-reform on %s -> epoch %d, blocked %v", node, uint64(epoch), data["blockedDevices"])
|
||||
return uint64(epoch)
|
||||
}
|
||||
|
||||
// waitFormed blocks until every named node has finished formation: a log exists
|
||||
// for its epoch, it is attached to it, and it no longer holds the direct-write
|
||||
// licence.
|
||||
//
|
||||
// This is a stricter precondition than convergence or writability, and both of
|
||||
// those are too weak to start a reform scenario from. An HA cluster mid-formation
|
||||
// converges perfectly well at seq 0 — the formation writer serves its store to
|
||||
// the others by snapshot, so every node reports the same dump at the same
|
||||
// position — and it reports itself writable, because the formation licence IS a
|
||||
// form of writability. Sampling there and calling it "a healthy cluster" starts
|
||||
// the scenario in the state the scenario is supposed to create.
|
||||
func (c *e2eCluster) waitFormed(d time.Duration, names ...string) {
|
||||
c.t.Helper()
|
||||
for _, n := range names {
|
||||
node := n
|
||||
c.waitForDetail(d, "formation complete on "+node, func() (bool, string) {
|
||||
st := c.oplogState(node)
|
||||
return st.attached && !st.formationWriter && !st.halted, node + ": " + st.String()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// waitReadOnly blocks until the node refuses writes, and returns the number of
|
||||
// attempts it took. Used to establish the pre-reform state: reform is only
|
||||
// meaningful from a manager that has genuinely lost its quorum.
|
||||
func (c *e2eCluster) waitReadOnly(d time.Duration, node, prefix string) {
|
||||
c.t.Helper()
|
||||
n := 0
|
||||
c.waitForDetail(d, node+" refuses writes", func() (bool, string) {
|
||||
n++
|
||||
code, _ := c.tryWrite(node, fmt.Sprintf("%s-%d", prefix, n))
|
||||
return code == "read-only", fmt.Sprintf("attempt %d -> %q", n, code)
|
||||
})
|
||||
}
|
||||
|
||||
// TestE2EForceReform: three managers and an agent, two managers lost for good.
|
||||
// The survivor must be able to reform, write again while no log exists, take on
|
||||
// a replacement manager, and bring the whole surviving cluster to the new epoch.
|
||||
//
|
||||
// The assertion that carries the most weight is not "reform returned 200" — it
|
||||
// is that the survivor becomes writable BECAUSE it holds the formation licence
|
||||
// and no stream exists yet (formationWriter && !streamSeen), and then stops
|
||||
// being a direct writer the moment the log appears. A reform that left the
|
||||
// licence in place after stream creation would keep answering 200 to writes
|
||||
// that never reach the log and get erased by the next snapshot, which no
|
||||
// convergence check performed straight afterwards would notice.
|
||||
func TestE2EForceReform(t *testing.T) {
|
||||
c := newE2ECluster(t, true, []e2eNodeSpec{
|
||||
{Name: "prime", Octet: 1, CosmosNode: 2, Lighthouse: true},
|
||||
{Name: "mantwo", Octet: 2, CosmosNode: 2},
|
||||
{Name: "manthree", Octet: 3, CosmosNode: 2},
|
||||
{Name: "agentone", Octet: 4, CosmosNode: 1, Agent: true},
|
||||
})
|
||||
|
||||
for _, name := range []string{"prime", "mantwo", "manthree", "agentone"} {
|
||||
c.start(name)
|
||||
c.waitConnected(name, 240*time.Second)
|
||||
}
|
||||
c.waitFormed(420*time.Second, "prime", "mantwo", "manthree", "agentone")
|
||||
c.waitConverged(420*time.Second, "initial convergence", "prime", "mantwo", "manthree", "agentone")
|
||||
c.waitWritable(180*time.Second, "prime")
|
||||
|
||||
// The R1->R3 scale-up is asserted here and nowhere else in the suite, and it is
|
||||
// also this scenario's precondition: until the log is genuinely on three peers,
|
||||
// killing two managers does not necessarily cost the survivor its quorum.
|
||||
c.waitStreamReplicas(420*time.Second, "prime", 3)
|
||||
|
||||
before := c.oplogState("prime")
|
||||
if before.err != nil {
|
||||
t.Fatal("e2e: reading prime's op-log state:", before.err)
|
||||
}
|
||||
|
||||
// the two-of-three loss that force-reform exists to escape
|
||||
c.kill("mantwo")
|
||||
c.kill("manthree")
|
||||
c.waitReadOnly(240*time.Second, "prime", "pre-reform")
|
||||
|
||||
newEpoch := c.forceReform("prime")
|
||||
if newEpoch != before.epoch+1 {
|
||||
t.Fatalf("force-reform moved the epoch to %d, want %d (epoch is the publish fence; a reform that does not advance it fences nothing)",
|
||||
newEpoch, before.epoch+1)
|
||||
}
|
||||
|
||||
// the survivor re-enters formation: new epoch, licence held, no log yet
|
||||
c.waitForDetail(180*time.Second, "prime re-enters formation at the new epoch", func() (bool, string) {
|
||||
st := c.oplogState("prime")
|
||||
return st.epoch == newEpoch && st.formationWriter && !st.streamSeen, st.String()
|
||||
})
|
||||
|
||||
// and the licence is real: a write must actually land while no stream exists
|
||||
wrote := 0
|
||||
c.waitForDetail(240*time.Second, "prime writes directly in formation", func() (bool, string) {
|
||||
wrote++
|
||||
code, errW := c.tryWrite("prime", fmt.Sprintf("in-formation-%d", wrote))
|
||||
return errW == nil, fmt.Sprintf("attempt %d -> %q (%s)", wrote, code, c.oplogState("prime"))
|
||||
})
|
||||
|
||||
// a replacement manager, enrolled BY the survivor, in formation, at the new
|
||||
// epoch — the normal enrollment flow is what the plan claims still works here
|
||||
c.enrollVia("prime", e2eNodeSpec{Name: "manfour", Octet: 5, CosmosNode: 2})
|
||||
c.start("manfour")
|
||||
c.waitConnected("manfour", 240*time.Second)
|
||||
|
||||
// JetStream elects with two managers, the formation writer creates the log for
|
||||
// the new epoch, and formation ends by itself
|
||||
c.waitForDetail(420*time.Second, "log created at the new epoch and formation ends", func() (bool, string) {
|
||||
st := c.oplogState("prime")
|
||||
return st.epoch == newEpoch && st.attached && !st.formationWriter, st.String()
|
||||
})
|
||||
|
||||
c.waitFormed(420*time.Second, "manfour")
|
||||
c.waitConverged(420*time.Second, "survivor and replacement converged at the new epoch", "prime", "manfour")
|
||||
|
||||
// The agent was never blocked and never left. It has to end up on the new
|
||||
// epoch by snapshot; this is checked on its own rather than folded into the
|
||||
// convergence wait so that an agent stranded at the old epoch reports as
|
||||
// exactly that instead of as a generic timeout.
|
||||
c.waitForDetail(420*time.Second, "agent adopts the new epoch", func() (bool, string) {
|
||||
st := c.oplogState("agentone")
|
||||
return st.epoch == newEpoch && st.attached, "agentone: " + st.String()
|
||||
})
|
||||
c.waitConverged(420*time.Second, "whole surviving cluster converged", "prime", "manfour", "agentone")
|
||||
c.assertNotHalted("after reform", "prime", "manfour", "agentone")
|
||||
}
|
||||
|
||||
// TestE2EStaleManagerFenced: revive a manager that was blocked and left behind
|
||||
// by a reform. It must never be heard from again in the new epoch.
|
||||
//
|
||||
// "Never heard from" is asserted three ways, because each can hold while another
|
||||
// fails: its writes must not reach the survivors, it must not sit attached to
|
||||
// the new log with state nobody else agreed to, and its credentials must be
|
||||
// refused by a survivor. The credential probe is the only one that tests
|
||||
// revocation directly — a stale node dials its OWN server for its own client
|
||||
// connection, so it will report itself connected no matter how thoroughly it has
|
||||
// been fenced, and any assertion built on its self-reported health is vacuous.
|
||||
func TestE2EStaleManagerFenced(t *testing.T) {
|
||||
c := newE2ECluster(t, true, []e2eNodeSpec{
|
||||
{Name: "prime", Octet: 1, CosmosNode: 2, Lighthouse: true},
|
||||
{Name: "mantwo", Octet: 2, CosmosNode: 2},
|
||||
{Name: "manthree", Octet: 3, CosmosNode: 2},
|
||||
})
|
||||
|
||||
for _, name := range []string{"prime", "mantwo", "manthree"} {
|
||||
c.start(name)
|
||||
c.waitConnected(name, 240*time.Second)
|
||||
}
|
||||
c.waitFormed(420*time.Second, "prime", "mantwo", "manthree")
|
||||
c.waitConverged(420*time.Second, "initial convergence", "prime", "mantwo", "manthree")
|
||||
c.waitWritable(180*time.Second, "prime")
|
||||
c.waitStreamReplicas(420*time.Second, "prime", 3)
|
||||
|
||||
staleKey := c.apiKeys["mantwo"]
|
||||
if staleKey == "" {
|
||||
t.Fatal("e2e: no recorded API key for mantwo; the credential probe below would be vacuous")
|
||||
}
|
||||
|
||||
// Positive control, taken while the cluster is healthy: these credentials DO
|
||||
// open a connection to prime right now. Without this, a later refusal proves
|
||||
// nothing — it could just as easily mean the probe never worked.
|
||||
probe := func(from, targetIP, user, pwd string) (bool, string) {
|
||||
out, err := c.node(from).post("/probe-nats", map[string]string{
|
||||
"URL": "nats://" + targetIP + ":4222",
|
||||
"User": user,
|
||||
"Password": pwd,
|
||||
})
|
||||
if err != nil {
|
||||
return false, "probe failed: " + err.Error()
|
||||
}
|
||||
connected, _ := out["connected"].(bool)
|
||||
detail, _ := out["error"].(string)
|
||||
return connected, detail
|
||||
}
|
||||
if ok, detail := probe("manthree", "127.0.1.1", "mantwo", staleKey); !ok {
|
||||
t.Fatalf("e2e: mantwo's credentials were already refused by prime BEFORE any fencing (%s) — the revocation assertion below would pass for the wrong reason", detail)
|
||||
}
|
||||
|
||||
before := c.oplogState("prime")
|
||||
if before.err != nil {
|
||||
t.Fatal("e2e: reading prime's op-log state:", before.err)
|
||||
}
|
||||
|
||||
// lose quorum, reform around prime; mantwo is the manager we will revive
|
||||
c.kill("mantwo")
|
||||
c.kill("manthree")
|
||||
c.waitReadOnly(240*time.Second, "prime", "pre-reform")
|
||||
|
||||
newEpoch := c.forceReform("prime")
|
||||
if newEpoch != before.epoch+1 {
|
||||
t.Fatalf("force-reform moved the epoch to %d, want %d", newEpoch, before.epoch+1)
|
||||
}
|
||||
|
||||
// bring the new epoch to a working cluster before reviving the stale node, so
|
||||
// anything it disturbs is disturbing something that demonstrably worked
|
||||
c.enrollVia("prime", e2eNodeSpec{Name: "manfour", Octet: 5, CosmosNode: 2})
|
||||
c.start("manfour")
|
||||
c.waitConnected("manfour", 240*time.Second)
|
||||
c.waitForDetail(420*time.Second, "new epoch log is live", func() (bool, string) {
|
||||
st := c.oplogState("prime")
|
||||
return st.epoch == newEpoch && st.attached && !st.formationWriter, st.String()
|
||||
})
|
||||
c.waitFormed(420*time.Second, "manfour")
|
||||
c.waitConverged(420*time.Second, "new epoch converged before the revival", "prime", "manfour")
|
||||
|
||||
healthyHash := c.dbHash("prime")
|
||||
if healthyHash == "" {
|
||||
t.Fatal("e2e: prime has no dump after reform")
|
||||
}
|
||||
|
||||
// --- the revival
|
||||
c.start("mantwo")
|
||||
c.waitFor(120*time.Second, "revived stale manager answers its control API", func() bool {
|
||||
_, errS := c.node("mantwo").get("/status")
|
||||
return errS == nil
|
||||
})
|
||||
|
||||
// 1. credentials. The device record was blocked by the reform and NATS was
|
||||
// restarted, so a survivor must refuse the stale identity.
|
||||
c.waitForDetail(240*time.Second, "prime refuses the stale manager's credentials", func() (bool, string) {
|
||||
ok, detail := probe("manfour", "127.0.1.1", "mantwo", staleKey)
|
||||
return !ok, fmt.Sprintf("connected=%v %s", ok, detail)
|
||||
})
|
||||
|
||||
// 2. it must never land an op in the new epoch. Its own writes must fail, and
|
||||
// nothing it attempted may appear on the survivors.
|
||||
for i := 0; i < 6; i++ {
|
||||
name := fmt.Sprintf("stale-write-%d", i)
|
||||
if _, errW := c.node("mantwo").post("/create-device",
|
||||
map[string]string{"DeviceName": name, "IP": fmt.Sprintf("192.168.209.%d", 10+i)}); errW == nil {
|
||||
t.Errorf("stale manager's write SUCCEEDED (%s) — a fenced node accepted a config change", name)
|
||||
}
|
||||
time.Sleep(5 * time.Second)
|
||||
}
|
||||
for name := range c.deviceView("prime") {
|
||||
if len(name) >= 12 && name[:12] == "stale-write-" {
|
||||
t.Errorf("a stale manager's write reached prime as device %q — the epoch fence did not hold", name)
|
||||
}
|
||||
}
|
||||
|
||||
// 3. it either adopted the new epoch or stayed isolated — the one state it
|
||||
// must never reach is attached to the new epoch while holding state the
|
||||
// cluster never agreed to.
|
||||
st := c.oplogState("mantwo")
|
||||
switch {
|
||||
case st.err != nil:
|
||||
t.Errorf("revived stale manager is unreachable: %v", st.err)
|
||||
case st.epoch == newEpoch:
|
||||
// it resynced: then it must actually agree with the cluster
|
||||
c.waitForDetail(300*time.Second, "resynced stale manager converges", func() (bool, string) {
|
||||
return c.dbHash("mantwo") == c.dbHash("prime"),
|
||||
fmt.Sprintf("mantwo=%.8s prime=%.8s (%s)", c.dbHash("mantwo"), c.dbHash("prime"), c.oplogState("mantwo"))
|
||||
})
|
||||
case st.epoch == before.epoch:
|
||||
if st.attached {
|
||||
t.Errorf("revived stale manager is ATTACHED at the abandoned epoch %d (%s) — it recreated the log it was fenced out of, which is a second live history",
|
||||
st.epoch, st)
|
||||
}
|
||||
if st.configWritable {
|
||||
t.Errorf("revived stale manager reports config writable at the abandoned epoch %d (%s)", st.epoch, st)
|
||||
}
|
||||
t.Logf("revived stale manager stayed isolated at epoch %d (%s)", st.epoch, st)
|
||||
default:
|
||||
t.Errorf("revived stale manager is at unexpected epoch %d, want %d (adopted) or %d (isolated): %s",
|
||||
st.epoch, newEpoch, before.epoch, st)
|
||||
}
|
||||
|
||||
// 4. whatever the stale node did, the live cluster is undisturbed
|
||||
c.assertNotHalted("after the stale manager was revived", "prime", "manfour")
|
||||
c.waitConverged(300*time.Second, "live cluster still converged after the revival", "prime", "manfour")
|
||||
}
|
||||
@@ -121,8 +121,9 @@ func TestE2EHAFormation(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
// TestE2ESyncPropagation: a DB edit on the manager propagates to the agent via
|
||||
// the NATS sync protocol and converges byte-for-byte.
|
||||
// TestE2ESyncPropagation: a DB edit on the manager propagates to the agent
|
||||
// through the JetStream op-log and converges byte-for-byte, with both nodes
|
||||
// landing on the same log sequence.
|
||||
func TestE2ESyncPropagation(t *testing.T) {
|
||||
c := newE2ECluster(t, false, []e2eNodeSpec{
|
||||
{Name: "prime", Octet: 1, CosmosNode: 2, Lighthouse: true},
|
||||
@@ -144,9 +145,8 @@ func TestE2ESyncPropagation(t *testing.T) {
|
||||
map[string]string{"DeviceName": "agentone", "Nickname": "renamed-by-e2e"}); err != nil {
|
||||
t.Fatal("e2e: edit-device:", err)
|
||||
}
|
||||
if _, err := c.node("prime").post("/sync-push", nil); err != nil {
|
||||
t.Fatal("e2e: sync-push:", err)
|
||||
}
|
||||
// no push step: the edit published one op-log entry, and every node applies
|
||||
// it from the stream on its own
|
||||
|
||||
c.waitFor(120*time.Second, "edited nickname visible on agent", func() bool {
|
||||
db, err := c.node("agentone").get("/db")
|
||||
@@ -162,6 +162,11 @@ func TestE2ESyncPropagation(t *testing.T) {
|
||||
}
|
||||
return false
|
||||
})
|
||||
|
||||
// op-log semantics: converged means same dump AND same position in the log
|
||||
c.waitForDetail(60*time.Second, "manager and agent at the same op-log sequence", func() (bool, string) {
|
||||
return c.oplogConverged("prime", "agentone")
|
||||
})
|
||||
}
|
||||
|
||||
// TestE2EManagerFailoverHA: with 3 managers and 1 agent, killing one manager
|
||||
|
||||
@@ -219,6 +219,64 @@ cstln_cosmos_node: 1
|
||||
}
|
||||
}
|
||||
|
||||
// A joining node must start at the constellation's real epoch, not the default 1.
|
||||
// Asking at 1 is outranked by every stale responder — including managers revived
|
||||
// from an epoch a force-reform abandoned — and the snapshot responder can only
|
||||
// silence peers BEHIND the asker, so a replacement could be captured at a dead
|
||||
// epoch. Adoption is forward-only: a lower epoch in an enrollment bundle must
|
||||
// never walk a healthy node backwards into a log its cluster has left.
|
||||
func TestUnitConnectToExistingAdoptsEpochForwardOnly(t *testing.T) {
|
||||
setupTestEnv(t, nil)
|
||||
|
||||
enroll := func(epoch string) {
|
||||
t.Helper()
|
||||
if _, err := ConnectToExisting([]byte("cstln_device_name: d\ncstln_oplog_epoch: "+epoch+"\n"),
|
||||
utils.GetMainConfig()); err != nil {
|
||||
t.Fatal("ConnectToExisting:", err)
|
||||
}
|
||||
}
|
||||
|
||||
enroll("4")
|
||||
if got := utils.GetOplogEpoch(); got != 4 {
|
||||
t.Fatalf("epoch = %d, want 4 — a joiner asking at the default 1 is outranked by stale peers", got)
|
||||
}
|
||||
// it has the epoch's name, not its contents: it must re-materialize before writing
|
||||
if utils.IsOplogBootstrapped() {
|
||||
t.Fatal("joiner claims materialized state for an epoch it has never seen")
|
||||
}
|
||||
|
||||
if err := utils.MarkOplogBootstrapped(4); err != nil {
|
||||
t.Fatal("MarkOplogBootstrapped:", err)
|
||||
}
|
||||
|
||||
// a stale bundle must not move it back, nor discard the state it holds
|
||||
enroll("2")
|
||||
if got := utils.GetOplogEpoch(); got != 4 {
|
||||
t.Fatalf("epoch = %d after a stale enrollment bundle, want 4 — adopting backwards "+
|
||||
"walks this node into a log the cluster has abandoned", got)
|
||||
}
|
||||
if !utils.IsOplogBootstrapped() {
|
||||
t.Fatal("a stale enrollment bundle cleared the bootstrapped marker")
|
||||
}
|
||||
|
||||
// an equal epoch is a no-op rather than a pointless re-snapshot
|
||||
enroll("4")
|
||||
if !utils.IsOplogBootstrapped() {
|
||||
t.Fatal("re-enrolling at the same epoch forced a needless re-materialization")
|
||||
}
|
||||
|
||||
// and a bundle without the key at all still works (older manager, mixed versions)
|
||||
enroll4Missing := func() {
|
||||
if _, err := ConnectToExisting([]byte("cstln_device_name: d\n"), utils.GetMainConfig()); err != nil {
|
||||
t.Fatal("ConnectToExisting without an epoch key:", err)
|
||||
}
|
||||
}
|
||||
enroll4Missing()
|
||||
if got := utils.GetOplogEpoch(); got != 4 {
|
||||
t.Fatalf("epoch = %d after an enrollment bundle with no epoch key, want 4", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnitConnectToExistingWrongTypes(t *testing.T) {
|
||||
// regression: wrong-typed fields must return an error, not panic
|
||||
tests := []struct {
|
||||
|
||||
@@ -13,7 +13,6 @@ var NebulaHasStarted = false
|
||||
var NATSStarted atomic.Bool
|
||||
var CachedDeviceNames = map[string]string{}
|
||||
var CachedDevices = map[string]utils.ConstellationDevice{}
|
||||
var needToSyncCA = false
|
||||
|
||||
// deviceCacheMux guards cachedCurrentDevice, CachedDevices and CachedDeviceNames,
|
||||
// rebuilt from goroutines while readers run concurrently. Writers must REPLACE
|
||||
@@ -27,35 +26,17 @@ func deviceCacheSnapshot() (map[string]utils.ConstellationDevice, map[string]str
|
||||
return CachedDevices, CachedDeviceNames
|
||||
}
|
||||
|
||||
func resyncConstellationNodes() {
|
||||
SendNewDBSyncMessage()
|
||||
}
|
||||
|
||||
// refreshDeviceCache rebuilds CachedDevices/CachedDeviceNames from the devices collection
|
||||
// and invalidates cachedCurrentDevice. Called at Init and whenever the on-disk database
|
||||
// changed without a full restart (e.g. a no-restart sync from a tag-only device edit).
|
||||
func refreshDeviceCache() {
|
||||
utils.Log("Constellation: populating device names cache...")
|
||||
c, closeDb, errCo := utils.GetEmbeddedCollection(utils.GetRootAppId(), "devices")
|
||||
defer closeDb()
|
||||
|
||||
if errCo != nil {
|
||||
utils.Error("Database Connect", errCo)
|
||||
return
|
||||
}
|
||||
|
||||
cursor, err := c.Find(nil, map[string]interface{}{})
|
||||
devices, err := utils.ListDevices(true)
|
||||
if err != nil {
|
||||
utils.Error("DeviceList: Error fetching devices", err)
|
||||
return
|
||||
}
|
||||
defer cursor.Close(nil)
|
||||
|
||||
var devices []utils.ConstellationDevice
|
||||
if err = cursor.All(nil, &devices); err != nil {
|
||||
utils.Error("DeviceList: Error decoding devices", err)
|
||||
return
|
||||
}
|
||||
|
||||
// build into locals and swap once under the write lock,
|
||||
// so concurrent readers never see a half-built cache
|
||||
@@ -190,7 +171,8 @@ func Init() {
|
||||
|
||||
utils.IsConstellationIP = IsConstellationIP
|
||||
|
||||
utils.ResyncConstellationNodes = resyncConstellationNodes
|
||||
// every user/device write in the product goes through here from now on
|
||||
utils.SetPublishOpHook(publishOp)
|
||||
utils.GetConstellationTunnelRoutes = getConstellationTunnelRoutes
|
||||
|
||||
NebulaStarted.Store(false)
|
||||
|
||||
+34
-53
@@ -319,18 +319,21 @@ func ResetNebula() error {
|
||||
os.RemoveAll(utils.CONFIGFOLDER + "ca.key")
|
||||
os.RemoveAll(utils.CONFIGFOLDER + "cosmos.crt")
|
||||
os.RemoveAll(utils.CONFIGFOLDER + "cosmos.key")
|
||||
os.RemoveAll(utils.CONFIGFOLDER + "jetstream")
|
||||
os.RemoveAll(jetstreamDir())
|
||||
|
||||
// remove everything in db
|
||||
|
||||
c, closeDb, err := utils.GetEmbeddedCollection(utils.GetRootAppId(), "devices")
|
||||
defer closeDb()
|
||||
// Wipe this node's devices LOCALLY. Deliberately not DeleteDevices: stop()
|
||||
// above leaves the apply loop attached, so a published delete-all — empty
|
||||
// filter, therefore an unqualified DELETE — would destroy the device registry
|
||||
// on every node in the constellation. Leaving is a local act by definition.
|
||||
err := utils.DeleteDevicesLocal(map[string]interface{}{})
|
||||
if err != nil {
|
||||
return err
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = c.DeleteMany(nil, map[string]interface{}{})
|
||||
if err != nil {
|
||||
// Drop any formation licence with it: this node keeps its kv epoch across a
|
||||
// reset, so a creator that resets and later joins someone else's constellation
|
||||
// at the same epoch number would otherwise still look licensed to write direct.
|
||||
if err := utils.ClearFormationWriter(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -341,6 +344,7 @@ func ResetNebula() error {
|
||||
config.ConstellationConfig.ThisDeviceName = ""
|
||||
config.ConstellationConfig.IPRange = ""
|
||||
config.ConstellationConfig.ConstellationHostname = strings.Join(GetDefaultHostnames(), ",")
|
||||
config.AgentMode = false
|
||||
|
||||
if config.Licence == "" {
|
||||
config.ServerToken = ""
|
||||
@@ -362,72 +366,31 @@ func ResetNebula() error {
|
||||
}
|
||||
|
||||
func GetAllDevicesEvenBlocked() ([]utils.ConstellationDevice, error) {
|
||||
c, closeDb, err := utils.GetEmbeddedCollection(utils.GetRootAppId(), "devices")
|
||||
defer closeDb()
|
||||
devices, err := utils.ListDevices(true)
|
||||
if err != nil {
|
||||
return []utils.ConstellationDevice{}, err
|
||||
}
|
||||
|
||||
var devices []utils.ConstellationDevice
|
||||
|
||||
cursor, err := c.Find(nil, map[string]interface{}{})
|
||||
if err != nil {
|
||||
return []utils.ConstellationDevice{}, err
|
||||
}
|
||||
defer cursor.Close(nil)
|
||||
|
||||
if err := cursor.All(nil, &devices); err != nil {
|
||||
return []utils.ConstellationDevice{}, err
|
||||
}
|
||||
|
||||
return devices, nil
|
||||
}
|
||||
|
||||
func GetAllDevices() ([]utils.ConstellationDevice, error) {
|
||||
c, closeDb, err := utils.GetEmbeddedCollection(utils.GetRootAppId(), "devices")
|
||||
defer closeDb()
|
||||
devices, err := utils.ListDevices(false)
|
||||
if err != nil {
|
||||
return []utils.ConstellationDevice{}, err
|
||||
}
|
||||
|
||||
var devices []utils.ConstellationDevice
|
||||
|
||||
cursor, err := c.Find(nil, map[string]interface{}{
|
||||
"Blocked": false,
|
||||
})
|
||||
if err != nil {
|
||||
return []utils.ConstellationDevice{}, err
|
||||
}
|
||||
defer cursor.Close(nil)
|
||||
|
||||
if err := cursor.All(nil, &devices); err != nil {
|
||||
return []utils.ConstellationDevice{}, err
|
||||
}
|
||||
|
||||
return devices, nil
|
||||
}
|
||||
|
||||
func GetAllLightHouses() ([]utils.ConstellationDevice, error) {
|
||||
c, closeDb, err := utils.GetEmbeddedCollection(utils.GetRootAppId(), "devices")
|
||||
defer closeDb()
|
||||
if err != nil {
|
||||
return []utils.ConstellationDevice{}, err
|
||||
}
|
||||
|
||||
var devices []utils.ConstellationDevice
|
||||
|
||||
cursor, err := c.Find(nil, map[string]interface{}{
|
||||
devices, err := utils.FindDevices(map[string]interface{}{
|
||||
"IsLighthouse": true,
|
||||
"Blocked": false,
|
||||
})
|
||||
if err != nil {
|
||||
return []utils.ConstellationDevice{}, err
|
||||
}
|
||||
defer cursor.Close(nil)
|
||||
|
||||
if err := cursor.All(nil, &devices); err != nil {
|
||||
return []utils.ConstellationDevice{}, err
|
||||
}
|
||||
|
||||
return devices, nil
|
||||
}
|
||||
@@ -587,6 +550,18 @@ func getYAMLClientConfig(name, configPath, capki, cert, key, APIKey string, devi
|
||||
}
|
||||
configMap["cstln_nats_managers"] = seedManagers
|
||||
|
||||
// The live op-log epoch, so a joining node starts where the constellation
|
||||
// actually is instead of at the default 1.
|
||||
//
|
||||
// This is a fence, not a convenience. The snapshot responder can only silence
|
||||
// peers BEHIND the asker, so a node asking at epoch 1 is outranked by every
|
||||
// stale responder — including managers revived from an epoch a force-reform
|
||||
// abandoned. Those answer, and whoever replies first wins, so a replacement
|
||||
// enrolled after a reform could be captured at the dead epoch and sit
|
||||
// read-only re-asking until it happened to reach a current peer. Seeded with
|
||||
// the real epoch it outranks them, and they decline.
|
||||
configMap["cstln_oplog_epoch"] = utils.GetOplogEpoch()
|
||||
|
||||
if getLicence && device.CosmosNode == 0 {
|
||||
// get client licence
|
||||
utils.Log("Creating client license for " + name)
|
||||
@@ -1199,12 +1174,18 @@ func GetCurrentDevice() (utils.ConstellationDevice, error) {
|
||||
}
|
||||
}
|
||||
|
||||
// Unreadable role means server, never 0: a node given the plain-device NATS
|
||||
// permissions can't run the constellation, and this path only runs before
|
||||
// the first sync. Agent, not manager, to keep the guess least-privilege.
|
||||
device.CosmosNode = 1
|
||||
if v, exists := configMap["cstln_cosmos_node"]; exists {
|
||||
if cosmosNode, ok := v.(int); ok {
|
||||
device.CosmosNode = cosmosNode
|
||||
} else {
|
||||
utils.Warn("GetCurrentDevice: cstln_cosmos_node is not an int, skipping")
|
||||
utils.Warn("GetCurrentDevice: cstln_cosmos_node is not an int, assuming agent")
|
||||
}
|
||||
} else {
|
||||
utils.Warn("GetCurrentDevice: cstln_cosmos_node missing, assuming agent")
|
||||
}
|
||||
|
||||
if v, exists := configMap["cstln_is_exit_node"]; exists {
|
||||
|
||||
@@ -0,0 +1,561 @@
|
||||
package constellation
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/azukaar/cosmos-server/src/utils"
|
||||
)
|
||||
|
||||
// The config op-log: one ordered JetStream stream is the single source of truth
|
||||
// for users, devices and the config domains below. Every node — manager and
|
||||
// agent alike — materializes its local state by applying that stream in order,
|
||||
// so there is no conflict resolution anywhere: ops simply apply in stream order
|
||||
// on every replica.
|
||||
|
||||
const oplogEnvVersion = 1
|
||||
|
||||
const (
|
||||
// publish must ack within this or the node reports itself read-only
|
||||
oplogPublishTimeout = 5 * time.Second
|
||||
// how long a writer waits to see its own op come back through the apply loop
|
||||
oplogApplyWait = 10 * time.Second
|
||||
// retention: the log is a catch-up buffer, not history — fall behind and you snapshot
|
||||
oplogMaxMsgs = 100
|
||||
)
|
||||
|
||||
// OpEnvelope is one entry in the log. Domain payloads carry the FULL domain
|
||||
// state with op "set", which makes them idempotent and last-writer-wins in
|
||||
// stream order; db payloads carry a filter+doc mutation.
|
||||
type OpEnvelope struct {
|
||||
V int `json:"v"`
|
||||
Epoch uint64 `json:"epoch"`
|
||||
Kind string `json:"kind"` // "db" | "domain"
|
||||
Table string `json:"table,omitempty"`
|
||||
Domain string `json:"domain,omitempty"`
|
||||
Op string `json:"op"`
|
||||
PK string `json:"pk,omitempty"` // informational, for log reading
|
||||
Filter map[string]interface{} `json:"filter,omitempty"`
|
||||
Doc json.RawMessage `json:"doc,omitempty"`
|
||||
Origin string `json:"origin"`
|
||||
ReqID string `json:"reqId"`
|
||||
}
|
||||
|
||||
func oplogStreamName(epoch uint64) string {
|
||||
return "cosmos-oplog-e" + strconv.FormatUint(epoch, 10)
|
||||
}
|
||||
|
||||
func oplogSubjectPrefix(epoch uint64) string {
|
||||
return "cosmos._global_.oplog.e" + strconv.FormatUint(epoch, 10) + "."
|
||||
}
|
||||
|
||||
func oplogSubjectWildcard(epoch uint64) string {
|
||||
return oplogSubjectPrefix(epoch) + ">"
|
||||
}
|
||||
|
||||
// subject tokens can't carry '.' or the domain wildcards; the envelope stays authoritative
|
||||
func sanitizeSubjectToken(s string) string {
|
||||
r := strings.NewReplacer(".", "_", " ", "_", "*", "_", ">", "_", ":", "_")
|
||||
if s == "" {
|
||||
return "_"
|
||||
}
|
||||
return r.Replace(s)
|
||||
}
|
||||
|
||||
func oplogSubject(env OpEnvelope) string {
|
||||
target := env.Table
|
||||
if env.Kind == "domain" {
|
||||
target = env.Domain
|
||||
}
|
||||
return oplogSubjectPrefix(env.Epoch) + env.Kind + "." + sanitizeSubjectToken(target)
|
||||
}
|
||||
|
||||
func oplogOrigin() string {
|
||||
name, err := GetCurrentDeviceName()
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
// oplogMode is where a write goes on this node right now.
|
||||
type oplogMode int
|
||||
|
||||
const (
|
||||
// single-writer by construction — safe to write straight to SQLite
|
||||
oplogDirect oplogMode = iota
|
||||
oplogPublish
|
||||
oplogReadOnly
|
||||
)
|
||||
|
||||
// oplogWriteMode decides the write path. Availability falls out of the topology
|
||||
// rather than the node's role: whoever cannot reach a writable stream is
|
||||
// read-only, whoever is provably the only writer goes direct.
|
||||
func oplogWriteMode() oplogMode {
|
||||
// Self-fencing, and it has to come before everything else including the
|
||||
// attached fast path — a fenced node is at its most dangerous precisely when
|
||||
// it looks healthy.
|
||||
//
|
||||
// A force-reform blocks the managers it removes, but blocking only revokes
|
||||
// their CLIENT credentials, and a manager's client dials its OWN server. The
|
||||
// cluster route listener has no user authentication at all, so a revived
|
||||
// manager re-establishes routes, rejoins the meta group, adopts the current
|
||||
// epoch by snapshot and writes — observed end to end in TestE2EStaleManagerFenced.
|
||||
// Consistency survives all of that (it applies the real log in order); what
|
||||
// does not survive is the operator's decision to remove it.
|
||||
//
|
||||
// So a node that can see it has been removed stops writing. This works because
|
||||
// the snapshot it adopts CONTAINS its own device record marked blocked.
|
||||
// Cooperative rather than enforced: a node that never reaches the cluster never
|
||||
// learns, and only transport-level auth can actually exclude it.
|
||||
if isSelfBlocked() {
|
||||
return oplogReadOnly
|
||||
}
|
||||
|
||||
// Checked first among the rest on purpose: once we hold a live log, writes
|
||||
// publish no matter what else looks true.
|
||||
if oplogAttached.Load() {
|
||||
return oplogPublish
|
||||
}
|
||||
|
||||
// Constellation off: no log exists for this node at all, so direct is the only
|
||||
// path. Also the zero-overhead fast path for single-node installs, and the exit
|
||||
// ResetNebula takes — it disables constellation but leaves a stale sequence in
|
||||
// kv, so this check must come before the one below or such a node would be
|
||||
// read-only forever with no way back.
|
||||
if !utils.GetMainConfig().ConstellationConfig.Enabled {
|
||||
return oplogDirect
|
||||
}
|
||||
|
||||
// Fail closed: a node that has materialized from this log must never write
|
||||
// outside it again. Both direct branches below are reachable during an ordinary
|
||||
// RestartNebula — stop() clears NebulaStarted and the device cache together
|
||||
// (nebula.go), which makes a real cluster member look both stopped and
|
||||
// standalone — and the apply loop's reaction to a device write is
|
||||
// `go RestartNebula()`. Without this guard the seconds after every device write
|
||||
// are a window where a second write lands locally, is never published, and is
|
||||
// silently discarded by the next snapshot or domain op.
|
||||
//
|
||||
// Keyed on the bootstrapped marker, NOT on last_applied_seq > 0. A node that
|
||||
// joined by installing a founder's snapshot legitimately sits at seq 0 with a
|
||||
// full store, so the sequence test let exactly those nodes through to a direct
|
||||
// write — an agent's create-device returned 200 having only ever written
|
||||
// locally, and a two-node race saw both writers "win".
|
||||
if utils.IsOplogBootstrapped() {
|
||||
return oplogReadOnly
|
||||
}
|
||||
|
||||
// Genuinely pre-log from here down: fresh single-node install...
|
||||
if IsConstellationStandalone() {
|
||||
return oplogDirect
|
||||
}
|
||||
// ...or pre-NATS boot, where migrations and new-install setup run before any
|
||||
// peer is reachable.
|
||||
if !NebulaStarted.Load() {
|
||||
return oplogDirect
|
||||
}
|
||||
|
||||
// Formation. An HA constellation has no JetStream until its second manager
|
||||
// enrolls, so the creator (and the survivor of a force-reform) writes straight
|
||||
// to SQLite until a log exists to publish into. Single-writer by construction:
|
||||
// the licence is granted to exactly one node, enrollment happens there, and
|
||||
// every other node's publish finds no stream and comes back read-only.
|
||||
//
|
||||
// Both halves matter. The licence alone is not enough — it ends the instant a
|
||||
// log exists, whether or not this node has attached to it yet, because a direct
|
||||
// write after stream creation would never reach the log and would be erased by
|
||||
// the first snapshot.
|
||||
if utils.IsFormationWriter() && !oplogStreamSeen.Load() {
|
||||
return oplogDirect
|
||||
}
|
||||
return oplogReadOnly
|
||||
}
|
||||
|
||||
// isSelfBlocked reports whether this node's own device record marks it blocked,
|
||||
// i.e. whether the constellation has removed it.
|
||||
//
|
||||
// Read from the store rather than the device cache: refreshDeviceCache drops
|
||||
// blocked devices entirely, and GetCurrentDevice falls back to nebula.yml, so the
|
||||
// cache can never say "you are blocked" — only "you are absent", which is also
|
||||
// what a not-yet-enrolled node looks like.
|
||||
//
|
||||
// Fails OPEN on every uncertainty (no name, no record, read error): a false
|
||||
// positive takes a healthy node read-only for a transient DB problem, while a
|
||||
// false negative is only the status quo this fixes. An absent record is NOT
|
||||
// blocked — that is a node mid-enrollment.
|
||||
func isSelfBlocked() bool {
|
||||
name, err := GetCurrentDeviceName()
|
||||
if err != nil || name == "" {
|
||||
return false
|
||||
}
|
||||
device, err := utils.GetDeviceByName(name, false)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return device.Blocked
|
||||
}
|
||||
|
||||
// publishOp is the utils.CommitMutation hook: every user/device write in the
|
||||
// product funnels through here.
|
||||
func publishOp(m utils.Mutation) error {
|
||||
switch oplogWriteMode() {
|
||||
case oplogDirect:
|
||||
// reactions live in the apply loop, so the direct path has to fire them itself
|
||||
var pre []utils.ConstellationDevice
|
||||
if err := utils.CommitMutationDirect(m, &pre); err != nil {
|
||||
return err
|
||||
}
|
||||
oplogReactToTableOp(m, pre)
|
||||
return nil
|
||||
case oplogReadOnly:
|
||||
// best-effort ops are dropped rather than written locally: a local-only
|
||||
// write would permanently diverge this node's materialization
|
||||
if m.BestEffort {
|
||||
utils.Debug("[OPLOG] dropping best-effort " + m.Table + " op, log not writable")
|
||||
return nil
|
||||
}
|
||||
return utils.ErrReadOnly
|
||||
}
|
||||
|
||||
doc, err := utils.EncodeOpDoc(m)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
filter, err := utils.NormalizeOpFilter(m.Table, m.Filter)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return publishEnvelope(OpEnvelope{
|
||||
V: oplogEnvVersion,
|
||||
Epoch: utils.GetOplogEpoch(),
|
||||
Kind: "db",
|
||||
Table: m.Table,
|
||||
Op: m.Op,
|
||||
Filter: filter,
|
||||
Doc: doc,
|
||||
Origin: oplogOrigin(),
|
||||
ReqID: utils.GenerateRandomString(24),
|
||||
}, m.BestEffort)
|
||||
}
|
||||
|
||||
// PublishDomainOp publishes the full state of a config domain and waits for the
|
||||
// apply loop to install it locally. Callers must NOT hold utils.ConfigLock —
|
||||
// the apply path takes it.
|
||||
func PublishDomainOp(domain string, state interface{}) error {
|
||||
raw, err := json.Marshal(state)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
switch oplogWriteMode() {
|
||||
case oplogDirect:
|
||||
return applyDomainLocal(domain, raw)
|
||||
case oplogReadOnly:
|
||||
return utils.ErrReadOnly
|
||||
}
|
||||
|
||||
return publishEnvelope(OpEnvelope{
|
||||
V: oplogEnvVersion,
|
||||
Epoch: utils.GetOplogEpoch(),
|
||||
Kind: "domain",
|
||||
Domain: domain,
|
||||
Op: "set",
|
||||
Doc: raw,
|
||||
Origin: oplogOrigin(),
|
||||
ReqID: utils.GenerateRandomString(24),
|
||||
}, false)
|
||||
}
|
||||
|
||||
func publishEnvelope(env OpEnvelope, bestEffort bool) error {
|
||||
data, err := json.Marshal(env)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// register before publishing: the apply loop can deliver our own op back
|
||||
// before js.Publish has even returned
|
||||
var wait chan error
|
||||
if !bestEffort {
|
||||
wait = registerOpWaiter(env.ReqID)
|
||||
defer releaseOpWaiter(env.ReqID)
|
||||
}
|
||||
|
||||
clientConfigLock.RLock()
|
||||
jsCtx := js
|
||||
clientConfigLock.RUnlock()
|
||||
|
||||
if jsCtx == nil {
|
||||
return utils.ErrReadOnly
|
||||
}
|
||||
|
||||
// epoch-in-subject is the publish fence: a stale-epoch node's subject matches
|
||||
// no stream, so it gets no ack and can never write
|
||||
_, err = jsCtx.Publish(oplogSubject(env), data, nats.MsgId(env.ReqID), nats.AckWait(oplogPublishTimeout))
|
||||
if err != nil {
|
||||
utils.Warn("[OPLOG] publish rejected (" + env.Kind + "/" + env.Table + env.Domain + "): " + err.Error())
|
||||
if bestEffort {
|
||||
return nil
|
||||
}
|
||||
return utils.ErrReadOnly
|
||||
}
|
||||
|
||||
if bestEffort {
|
||||
return nil
|
||||
}
|
||||
|
||||
select {
|
||||
case applyErr := <-wait:
|
||||
return applyErr
|
||||
case <-time.After(oplogApplyWait):
|
||||
return utils.ErrApplyTimeout
|
||||
}
|
||||
}
|
||||
|
||||
// waiter registry: an op's originator blocks until its own entry comes back
|
||||
// through the apply loop, so an HTTP handler returns only once the write is real.
|
||||
var opWaitersMu sync.Mutex
|
||||
var opWaiters = map[string]chan error{}
|
||||
|
||||
func registerOpWaiter(reqID string) chan error {
|
||||
ch := make(chan error, 1)
|
||||
opWaitersMu.Lock()
|
||||
opWaiters[reqID] = ch
|
||||
opWaitersMu.Unlock()
|
||||
return ch
|
||||
}
|
||||
|
||||
func releaseOpWaiter(reqID string) {
|
||||
opWaitersMu.Lock()
|
||||
delete(opWaiters, reqID)
|
||||
opWaitersMu.Unlock()
|
||||
}
|
||||
|
||||
func notifyOpWaiter(reqID string, err error) {
|
||||
opWaitersMu.Lock()
|
||||
ch, ok := opWaiters[reqID]
|
||||
opWaitersMu.Unlock()
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case ch <- err:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// createOplogStream creates the log for this epoch, and reports whether THIS
|
||||
// call is the one that made it. Agents never create — they wait for a manager.
|
||||
//
|
||||
// Callers must have established that no stream exists for the epoch AND that no
|
||||
// peer offered a snapshot first (see oplogAdoptFromPeer): by the time we are
|
||||
// here, "nobody has this log" is the conclusion, not the assumption.
|
||||
//
|
||||
// created is not bookkeeping — it is the only local evidence of which node
|
||||
// seeded the log, and it is what licenses oplogAttach to mark this node
|
||||
// bootstrapped from its own SQLite.
|
||||
func createOplogStream(jsCtx nats.JetStreamContext, epoch uint64, waitingSince time.Time) (*nats.StreamInfo, bool, error) {
|
||||
name := oplogStreamName(epoch)
|
||||
|
||||
if err := oplogMayCreateStream(name, waitingSince); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
|
||||
utils.Log("[OPLOG] Creating stream " + name)
|
||||
|
||||
// R1 at creation even in HA: a three-way replica group cannot be placed while
|
||||
// the cluster is still forming. It is raised to R3 by oplogMaintainReplicas
|
||||
// once the third manager is up.
|
||||
si, err := jsCtx.AddStream(&nats.StreamConfig{
|
||||
Name: name,
|
||||
Subjects: []string{oplogSubjectWildcard(epoch)},
|
||||
Storage: nats.FileStorage,
|
||||
MaxMsgs: oplogMaxMsgs,
|
||||
Discard: nats.DiscardOld,
|
||||
Replicas: 1,
|
||||
})
|
||||
if err != nil {
|
||||
// lost the race with another manager — its stream is just as good
|
||||
if si2, err2 := jsCtx.StreamInfo(name); err2 == nil {
|
||||
return si2, false, nil
|
||||
}
|
||||
return nil, false, err
|
||||
}
|
||||
return si, true, nil
|
||||
}
|
||||
|
||||
// oplogMayCreateStream decides whether this node is allowed to create the log
|
||||
// right now, and is the whole of createOplogStream's policy.
|
||||
//
|
||||
// Every "no" here is a DEFERRAL that times out, never a veto. That distinction
|
||||
// is the difference between deferring to a better-qualified creator and bricking
|
||||
// a constellation: an install upgrading from a release that predates the op-log
|
||||
// holds no formation licence and no bootstrapped marker, and nothing will ever
|
||||
// grant it either, so a permanent refusal would leave every node read-only with
|
||||
// no operator action that recovers it.
|
||||
func oplogMayCreateStream(name string, waitingSince time.Time) error {
|
||||
if utils.FBL.AgentMode {
|
||||
return errors.New("oplog: stream " + name + " does not exist yet")
|
||||
}
|
||||
|
||||
// Formation exit: the formation writer holds the seed state, so it creates the
|
||||
// log the moment JetStream elects — ahead of the election below, which could
|
||||
// otherwise hand creation to a peer that has nothing to seed with and leave the
|
||||
// writer's rows out of the log entirely.
|
||||
if utils.IsFormationWriter() {
|
||||
return nil
|
||||
}
|
||||
|
||||
// A node that HAS materialized from a log may never create one. This is a
|
||||
// refusal, not a deferral, and it is the single most important line here.
|
||||
//
|
||||
// Its own epoch having no stream means the cluster moved past it — a
|
||||
// force-reform renames the log — so the only correct move is to adopt what a
|
||||
// peer reports. Creating instead RESURRECTS the abandoned epoch's stream on the
|
||||
// live cluster, and the publish fence is built entirely on that stream not
|
||||
// existing: the old cluster's nodes would start getting acks, applying, and
|
||||
// reporting success, leaving two live streams and two disjoint materializations
|
||||
// that both look healthy. Being bootstrapped used to SKIP the checks below,
|
||||
// which is how a node that adopted a stale epoch from a revived peer could go
|
||||
// on to recreate that epoch's log here.
|
||||
//
|
||||
// It also makes losing a snapshot reply race harmless: if a stale peer answers
|
||||
// first, we decline to create, stay read-only, and the supervisor asks again on
|
||||
// the next tick until a current peer answers. Delay instead of divergence.
|
||||
if utils.IsOplogBootstrapped() {
|
||||
// Standalone cannot fork by definition — there is no peer holding another
|
||||
// materialization — so a single node whose store dir was lost can rebuild.
|
||||
if IsConstellationStandalone() {
|
||||
return nil
|
||||
}
|
||||
return errors.New("oplog: " + name + " is missing and this node has already materialized " +
|
||||
"from a log; waiting to adopt a peer's epoch rather than recreating it")
|
||||
}
|
||||
|
||||
// Genuinely pre-op-log from here down: a fresh founder, or an install upgrading
|
||||
// from a release that predates the op-log. Deferral, never veto — such an
|
||||
// install holds no licence, nothing will ever grant it one, and a permanent
|
||||
// refusal would leave every node read-only with no operator recovery.
|
||||
if IsConstellationStandalone() {
|
||||
return nil
|
||||
}
|
||||
if time.Since(waitingSince) < kvCreatorFallbackAfter {
|
||||
// long enough for any peer holding real state to answer a snapshot request
|
||||
// and for the formation writer of a normal creation to get there first
|
||||
return errors.New("oplog: waiting for a node holding materialized state to create " + name)
|
||||
}
|
||||
// After the window anyone may create; AddStream has exactly one winner and the
|
||||
// losers see created == false and snapshot from it. Deliberately not gated on
|
||||
// designatedKVCreator: on an upgraded install that election's winner may be a
|
||||
// node that is never coming back, and nothing else would then create the log.
|
||||
return nil
|
||||
}
|
||||
|
||||
// oplogTargetReplicas: R3 once the constellation is HA and has the three
|
||||
// managers to place them on, R1 while it is smaller than that.
|
||||
func oplogTargetReplicas() int {
|
||||
if IsNATSHA() && countKnownManagers() >= 3 {
|
||||
return 3
|
||||
}
|
||||
return 1
|
||||
}
|
||||
|
||||
// countKnownManagers counts distinct live managers in the device cache. The
|
||||
// cache indexes each device under its name AND every public hostname, so the
|
||||
// count has to be over names rather than entries.
|
||||
func countKnownManagers() int {
|
||||
devices, _ := deviceCacheSnapshot()
|
||||
names := map[string]bool{}
|
||||
for _, d := range devices {
|
||||
if d.CosmosNode == 2 && d.DeviceName != "" && !d.Blocked {
|
||||
names[d.DeviceName] = true
|
||||
}
|
||||
}
|
||||
return len(names)
|
||||
}
|
||||
|
||||
// oplogReplicaSettledFor paces the probe once the log is already the right size.
|
||||
// Only the settled path is paced: while a scale-up is actually outstanding the
|
||||
// supervisor's own 2s tick drives it, because placement only becomes possible
|
||||
// some seconds after the third manager's cluster route comes up.
|
||||
const oplogReplicaSettledFor = 20 * time.Second
|
||||
|
||||
var oplogReplicaMu sync.Mutex
|
||||
var oplogReplicaSettledAt time.Time
|
||||
var oplogReplicaWantedSince time.Time
|
||||
|
||||
func oplogReplicaSettle() {
|
||||
oplogReplicaMu.Lock()
|
||||
oplogReplicaSettledAt = time.Now()
|
||||
oplogReplicaWantedSince = time.Time{}
|
||||
oplogReplicaMu.Unlock()
|
||||
}
|
||||
|
||||
func oplogReplicaIsSettled() bool {
|
||||
oplogReplicaMu.Lock()
|
||||
defer oplogReplicaMu.Unlock()
|
||||
return time.Since(oplogReplicaSettledAt) < oplogReplicaSettledFor
|
||||
}
|
||||
|
||||
// oplogReplicaWaitedForCreator reports whether we have wanted R3 for long enough
|
||||
// that the designated creator has to be presumed gone. Without this the election
|
||||
// is a single point of failure exactly where HA was promised: countKnownManagers
|
||||
// counts cache entries rather than liveness, so if the lowest-named manager is
|
||||
// the one that died, every remaining manager computes a target of R3, defers to
|
||||
// a node that is not coming back, and the log silently stays R1 forever.
|
||||
func oplogReplicaWaitedForCreator() bool {
|
||||
oplogReplicaMu.Lock()
|
||||
defer oplogReplicaMu.Unlock()
|
||||
if oplogReplicaWantedSince.IsZero() {
|
||||
oplogReplicaWantedSince = time.Now()
|
||||
}
|
||||
return time.Since(oplogReplicaWantedSince) >= kvCreatorFallbackAfter
|
||||
}
|
||||
|
||||
// oplogMaintainReplicas scales the log up to R3 once the third manager enrolls.
|
||||
// UpdateStream, never drop-and-recreate: this is durable state, and recreating
|
||||
// it would restart the sequence out from under every node's last_applied_seq.
|
||||
func oplogMaintainReplicas() {
|
||||
if utils.FBL.AgentMode || !IsNATSHA() || oplogTargetReplicas() < 3 {
|
||||
return
|
||||
}
|
||||
if oplogReplicaIsSettled() {
|
||||
return
|
||||
}
|
||||
if _, isSelf := designatedKVCreator(); !isSelf && !oplogReplicaWaitedForCreator() {
|
||||
return
|
||||
}
|
||||
|
||||
clientConfigLock.RLock()
|
||||
jsCtx := js
|
||||
clientConfigLock.RUnlock()
|
||||
if jsCtx == nil {
|
||||
return
|
||||
}
|
||||
|
||||
name := oplogStreamName(utils.GetOplogEpoch())
|
||||
si, err := jsCtx.StreamInfo(name)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if si.Config.Replicas >= 3 {
|
||||
oplogReplicaSettle()
|
||||
return
|
||||
}
|
||||
|
||||
cfg := si.Config
|
||||
cfg.Replicas = 3
|
||||
if _, err := jsCtx.UpdateStream(&cfg); err != nil {
|
||||
// expected while the cluster is still placing the new manager; retried next tick
|
||||
utils.Debug("[OPLOG] " + name + " not ready to scale to 3 replicas yet: " + err.Error())
|
||||
return
|
||||
}
|
||||
oplogReplicaSettle()
|
||||
utils.Log("[OPLOG] Scaled " + name + " to 3 replicas")
|
||||
}
|
||||
@@ -0,0 +1,671 @@
|
||||
package constellation
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strconv"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/azukaar/cosmos-server/src/utils"
|
||||
)
|
||||
|
||||
// The apply loop is the only thing that mutates replicated state on this node.
|
||||
// It consumes the stream strictly in order and commits each entry's sequence in
|
||||
// the same SQLite transaction as its data, which is what makes apply
|
||||
// exactly-once regardless of JetStream redelivery.
|
||||
|
||||
var oplogAttached atomic.Bool
|
||||
var oplogHalted atomic.Bool
|
||||
var oplogHaltReason atomic.Value
|
||||
|
||||
// oplogStreamSeen: a log for the current epoch has been observed on this node.
|
||||
// It ends formation's direct-write licence (see oplogWriteMode) at stream
|
||||
// creation rather than at attach, so no write can slip past a log that already
|
||||
// exists. Deliberately NOT cleared by StopOplogApply — a RestartNebula bounce
|
||||
// must not reopen that window. Only a reform, which builds a new epoch, does.
|
||||
var oplogStreamSeen atomic.Bool
|
||||
|
||||
// oplogAttachedEpoch: the epoch of the stream the current subscription belongs
|
||||
// to, so the liveness probe can tell "my log is gone" from "my epoch moved".
|
||||
var oplogAttachedEpoch atomic.Uint64
|
||||
|
||||
var oplogMu sync.Mutex
|
||||
var oplogSub *nats.Subscription
|
||||
var oplogStop chan struct{}
|
||||
var oplogRunning bool
|
||||
|
||||
// serializes application so a redelivery can't race the entry behind it
|
||||
var oplogApplyMu sync.Mutex
|
||||
|
||||
const oplogSupervisorTick = 2 * time.Second
|
||||
|
||||
// StartOplogApply brings up the apply loop; safe to call repeatedly.
|
||||
func StartOplogApply() {
|
||||
oplogMu.Lock()
|
||||
if oplogRunning {
|
||||
oplogMu.Unlock()
|
||||
return
|
||||
}
|
||||
oplogRunning = true
|
||||
stop := make(chan struct{})
|
||||
oplogStop = stop
|
||||
oplogMu.Unlock()
|
||||
|
||||
oplogHalted.Store(false)
|
||||
|
||||
go oplogSupervisor(stop)
|
||||
}
|
||||
|
||||
// StopOplogApply tears the loop down. Never blocks: it runs from
|
||||
// CloseNATSClient/StopNATS, which RestartNebula calls from an apply reaction.
|
||||
func StopOplogApply() {
|
||||
oplogMu.Lock()
|
||||
if !oplogRunning {
|
||||
oplogMu.Unlock()
|
||||
return
|
||||
}
|
||||
oplogRunning = false
|
||||
stop := oplogStop
|
||||
oplogStop = nil
|
||||
sub := oplogSub
|
||||
oplogSub = nil
|
||||
oplogMu.Unlock()
|
||||
|
||||
oplogAttached.Store(false)
|
||||
if stop != nil {
|
||||
close(stop)
|
||||
}
|
||||
if sub != nil {
|
||||
sub.Unsubscribe()
|
||||
}
|
||||
}
|
||||
|
||||
// oplogSupervisor keeps the subscription attached: a node with no reachable
|
||||
// stream idles here and catches up (by snapshot if it fell off retention) as
|
||||
// soon as one appears.
|
||||
func oplogSupervisor(stop chan struct{}) {
|
||||
waitingSince := time.Now()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-stop:
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
if !oplogAttached.Load() && !oplogHalted.Load() {
|
||||
if err := oplogAttach(waitingSince); err != nil {
|
||||
utils.Debug("[OPLOG] not attached yet: " + err.Error())
|
||||
}
|
||||
} else if oplogAttached.Load() {
|
||||
// being attached is not evidence of being attached to anything
|
||||
oplogCheckAttachedStream()
|
||||
// doubles as the scale-up retry loop: placing a 3-way replica group
|
||||
// fails until the joining manager's cluster route is really up
|
||||
oplogMaintainReplicas()
|
||||
}
|
||||
|
||||
select {
|
||||
case <-stop:
|
||||
return
|
||||
case <-time.After(oplogSupervisorTick):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// oplogStreamCheckEvery paces the liveness probe below. The supervisor ticks
|
||||
// every 2s, which is more often than a StreamInfo round-trip needs to run.
|
||||
const oplogStreamCheckEvery = 6 * time.Second
|
||||
|
||||
// oplogStallProbes: how many consecutive probes may see the log ahead of us with
|
||||
// our own position frozen before we conclude the consumer is dead. Three at the
|
||||
// pacing above is ~18s of provable non-progress.
|
||||
const oplogStallProbes = 3
|
||||
|
||||
var oplogStreamCheckMu sync.Mutex
|
||||
var oplogStreamCheckedAt time.Time
|
||||
var oplogStallCount int
|
||||
var oplogStallLastApplied uint64
|
||||
|
||||
// oplogCheckAttachedStream detaches when the log this node is attached to is
|
||||
// gone, or when its epoch has moved on underneath it.
|
||||
//
|
||||
// oplogAttached is a latch: it is set once at subscribe time and cleared only by
|
||||
// an explicit detach. A JetStream subscription whose stream is DELETED does not
|
||||
// error — it simply goes quiet — so without this probe the latch stays set
|
||||
// forever, the supervisor keeps taking the attached branch, and oplogAttach is
|
||||
// never called again. Every recovery path lives inside oplogAttach, including
|
||||
// oplogAdoptFromPeer, so a node that was healthy at the moment of a force-reform
|
||||
// was the one node that could never learn about the new epoch. Worse, it kept
|
||||
// reporting itself writable (oplogWriteMode returns publish whenever the latch is
|
||||
// set) while every publish would have failed against a stream that no longer
|
||||
// exists: wedged, and looking perfectly healthy while wedged.
|
||||
func oplogCheckAttachedStream() {
|
||||
oplogStreamCheckMu.Lock()
|
||||
if time.Since(oplogStreamCheckedAt) < oplogStreamCheckEvery {
|
||||
oplogStreamCheckMu.Unlock()
|
||||
return
|
||||
}
|
||||
oplogStreamCheckedAt = time.Now()
|
||||
oplogStreamCheckMu.Unlock()
|
||||
|
||||
// a reform on THIS node moves the epoch without touching the subscription
|
||||
epoch := utils.GetOplogEpoch()
|
||||
if oplogAttachedEpoch.Load() != epoch {
|
||||
utils.Log("[OPLOG] Local epoch moved to " + strconv.FormatUint(epoch, 10) + ", re-attaching")
|
||||
oplogDetach()
|
||||
return
|
||||
}
|
||||
|
||||
clientConfigLock.RLock()
|
||||
jsCtx := js
|
||||
clientConfigLock.RUnlock()
|
||||
if jsCtx == nil {
|
||||
return
|
||||
}
|
||||
|
||||
si, err := jsCtx.StreamInfo(oplogStreamName(epoch))
|
||||
|
||||
// A definitive "it is not there" detaches. Any other error is UNOBSERVABLE
|
||||
// rather than benign: during a quorum outage StreamInfo simply doesn't answer,
|
||||
// so we learn nothing and must change nothing — including the stall counter
|
||||
// below, which has to survive the outage to be useful in the recovery window
|
||||
// right after it.
|
||||
if errors.Is(err, nats.ErrStreamNotFound) {
|
||||
utils.Warn("[OPLOG] " + oplogStreamName(epoch) + " no longer exists; detaching to find the current epoch")
|
||||
oplogDetach()
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if oplogNoteApplyProgress(si.State.LastSeq) {
|
||||
utils.Warn("[OPLOG] attached to " + oplogStreamName(epoch) + " but stuck at seq " +
|
||||
strconv.FormatUint(utils.GetLastAppliedSeq(), 10) + " while the log is at " +
|
||||
strconv.FormatUint(si.State.LastSeq, 10) + "; rebuilding the consumer")
|
||||
oplogDetach()
|
||||
}
|
||||
}
|
||||
|
||||
// oplogNoteApplyProgress reports whether the subscription has provably stopped
|
||||
// consuming: the log is ahead of us and our own position has not moved across
|
||||
// oplogStallProbes probes.
|
||||
//
|
||||
// This keys on PROGRESS rather than on error classification, and that is the
|
||||
// whole point. A consumer can die without producing an error anywhere — an
|
||||
// ordered consumer lost during a quorum outage leaves the stream healthy, the
|
||||
// epoch unchanged, and StreamInfo answering normally, so both of the checks above
|
||||
// pass while nothing is ever delivered again. The node then publishes into a log
|
||||
// it will never read back: the ack succeeds, no waiter is ever notified, and every
|
||||
// write returns 503 apply-timeout forever while the node reports itself attached
|
||||
// and writable, falling further behind for as long as it runs.
|
||||
//
|
||||
// Detaching restores the truthful answer too. A detached node is read-only, so
|
||||
// the same outage returns 409 "config read-only" — which is what the plan
|
||||
// specifies for lost quorum — instead of a 503 that claims the write might yet
|
||||
// land. It also stops NATSStatus reporting ConfigWritable on a node that cannot
|
||||
// apply anything, which was the UI telling an operator a wedged node was healthy.
|
||||
//
|
||||
// TIMING, because "3 probes × 6s" invites the wrong reading: the ~18s is measured
|
||||
// from when JetStream starts ANSWERING again, not from the fault. The counter can
|
||||
// only accumulate on a successful StreamInfo, so during the outage itself nothing
|
||||
// is observed and nothing is counted — which is deliberate (see the caller) and is
|
||||
// what makes the evidence land in the recovery window where the damage shows.
|
||||
//
|
||||
// Safe against the blips the caller deliberately ignores: any real delivery moves
|
||||
// last_applied_seq and resets the count, so a healthy consumer working through a
|
||||
// backlog, or one merely lagging a busy log between probes, never trips it. That
|
||||
// includes a run of REJECTED entries — a constraint violation still commits its
|
||||
// sequence (see oplogApplyEnvelope), which is what we want here: the test is
|
||||
// whether the consumer is alive, not whether its entries were accepted.
|
||||
//
|
||||
// One known false positive, left deliberately: GetLastAppliedSeq returns 0 when
|
||||
// the store cannot be read, which looks frozen and would cycle detach/re-attach.
|
||||
// A node whose SQLite reads are failing is already broken in a much larger way,
|
||||
// and everything else in this file fails closed on the same condition — so this
|
||||
// is documented rather than special-cased, to save the next reader diagnosing a
|
||||
// reattach loop that is really a disk problem.
|
||||
func oplogNoteApplyProgress(streamLast uint64) bool {
|
||||
applied := utils.GetLastAppliedSeq()
|
||||
|
||||
oplogStreamCheckMu.Lock()
|
||||
defer oplogStreamCheckMu.Unlock()
|
||||
|
||||
// caught up, or we moved since the last probe: the consumer is alive
|
||||
if streamLast <= applied || applied != oplogStallLastApplied {
|
||||
oplogStallLastApplied = applied
|
||||
oplogStallCount = 0
|
||||
return false
|
||||
}
|
||||
|
||||
oplogStallCount++
|
||||
if oplogStallCount < oplogStallProbes {
|
||||
return false
|
||||
}
|
||||
oplogStallCount = 0
|
||||
return true
|
||||
}
|
||||
|
||||
// oplogOwnsSubscription reports whether sub is the apply loop's current
|
||||
// subscription, so an async error about someone else's doesn't arm anything.
|
||||
//
|
||||
// The pointer comparison relies on nats.go MUTATING the Subscription during its
|
||||
// own recovery (resetOrderedConsumer calls applyNewSID on the same object)
|
||||
// rather than allocating a replacement. That happens to hold in the exact
|
||||
// failure this exists for, but it is a property of the client, not a contract.
|
||||
// If a future version allocates instead, this stops matching and we lose the
|
||||
// acceleration only — the progress probe still detaches, just at ~18s instead of
|
||||
// ~6s. That degradation is silent by design: nothing fails and no test goes red.
|
||||
// So do NOT "harden" this into something load-bearing; its whole value is that
|
||||
// losing it costs speed and never correctness.
|
||||
func oplogOwnsSubscription(sub *nats.Subscription) bool {
|
||||
oplogMu.Lock()
|
||||
defer oplogMu.Unlock()
|
||||
return oplogSub != nil && oplogSub == sub
|
||||
}
|
||||
|
||||
// oplogArmStallDetector brings the counter to one short of the threshold, so the
|
||||
// NEXT probe can act rather than waiting out the full window.
|
||||
//
|
||||
// Deliberately not a detach. nats.go knows a consumer is inactive long before
|
||||
// three 6s probes can prove it, and using that shortens recovery from ~18s to
|
||||
// ~6s — but the signal is a hint, not evidence. The probe still has to observe
|
||||
// the log ahead of us with our position frozen, so a spurious or transient
|
||||
// report costs nothing at all.
|
||||
func oplogArmStallDetector() {
|
||||
oplogStreamCheckMu.Lock()
|
||||
if oplogStallCount < oplogStallProbes-1 {
|
||||
oplogStallCount = oplogStallProbes - 1
|
||||
}
|
||||
oplogStreamCheckMu.Unlock()
|
||||
}
|
||||
|
||||
// oplogResetStallTracking clears the progress window, so a fresh subscription is
|
||||
// never judged on the dead one's history.
|
||||
func oplogResetStallTracking() {
|
||||
oplogStreamCheckMu.Lock()
|
||||
oplogStallCount = 0
|
||||
oplogStallLastApplied = utils.GetLastAppliedSeq()
|
||||
oplogStreamCheckMu.Unlock()
|
||||
}
|
||||
|
||||
func oplogAttach(waitingSince time.Time) error {
|
||||
if err := ClientConnectToJS(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
clientConfigLock.RLock()
|
||||
jsCtx := js
|
||||
clientConfigLock.RUnlock()
|
||||
|
||||
if jsCtx == nil {
|
||||
return errors.New("oplog: no JetStream context")
|
||||
}
|
||||
|
||||
epoch := utils.GetOplogEpoch()
|
||||
|
||||
si, errInfo := jsCtx.StreamInfo(oplogStreamName(epoch))
|
||||
created := false
|
||||
if errInfo != nil {
|
||||
// No log at OUR epoch. That is not only the founder's situation — it is
|
||||
// equally what a node left behind by a force-reform sees (the reform renames
|
||||
// the stream, so its own epoch's name resolves to nothing) and what a
|
||||
// replacement manager enrolling at the default epoch 1 sees on a cluster that
|
||||
// has moved past it.
|
||||
//
|
||||
// So ask the cluster before creating anything. A snapshot reply is the only
|
||||
// message that carries the authoritative epoch, and creating here instead
|
||||
// would be actively harmful in both of those cases: it resurrects a stream
|
||||
// for an abandoned epoch, and the publish fence's entire premise is that
|
||||
// `...oplog.e<abandoned>.>` matches no stream. Previously this path returned
|
||||
// early, above every snapshot call site, so nothing ever triggered adoption
|
||||
// and a stale node either looped forever (agent) or seeded the dead epoch
|
||||
// (manager).
|
||||
if oplogAdoptFromPeer() {
|
||||
return errors.New("oplog: adopted a peer's epoch, re-attaching")
|
||||
}
|
||||
|
||||
var errCreate error
|
||||
si, created, errCreate = createOplogStream(jsCtx, epoch, waitingSince)
|
||||
if errCreate != nil {
|
||||
return errCreate
|
||||
}
|
||||
}
|
||||
|
||||
// formation is over for this node the moment a log exists, attached or not
|
||||
oplogStreamSeen.Store(true)
|
||||
|
||||
// The gap path detaches asynchronously, so a message can still be in flight
|
||||
// while we catch up. Holding the apply lock makes it wait and then re-gate
|
||||
// against the post-snapshot position instead of writing over it.
|
||||
oplogApplyMu.Lock()
|
||||
defer oplogApplyMu.Unlock()
|
||||
|
||||
last := utils.GetLastAppliedSeq()
|
||||
|
||||
switch {
|
||||
// We just created this log, so it is empty and no peer can be serving it —
|
||||
// oplogSnapshotRouter only answers once attached, and nobody can be attached to
|
||||
// a stream that did not exist a moment ago. Our own store is the seed.
|
||||
//
|
||||
// Recording it HERE, before the subscribe below, is what closes the restart
|
||||
// window: until this is persisted, a node that died after creating the stream
|
||||
// would come back with its formation licence still set, oplogStreamSeen false
|
||||
// again (it is in-memory), and therefore a direct-write path into a log that
|
||||
// already exists. It also skips a fast-forward that is provably useless, which
|
||||
// used to stall every formation exit for the full 15s snapshot timeout.
|
||||
case created:
|
||||
if err := utils.MarkOplogBootstrapped(epoch); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// the entry we need has aged out of the 100-message window
|
||||
case si.State.Msgs > 0 && si.State.FirstSeq > last+1:
|
||||
utils.Warn("[OPLOG] behind retention (need " + strconv.FormatUint(last+1, 10) +
|
||||
", stream starts at " + strconv.FormatUint(si.State.FirstSeq, 10) + "), fast-forwarding")
|
||||
if err := oplogFastForward(jsCtx, epoch); err != nil {
|
||||
return err
|
||||
}
|
||||
last = utils.GetLastAppliedSeq()
|
||||
|
||||
// A node that has never materialized from this log cannot rebuild state by
|
||||
// replaying a log of deltas, however short it looks — the entries that created
|
||||
// the existing users, devices and CA are simply not in it. It has to snapshot.
|
||||
// If no peer answers, this node IS the seed and its own SQLite is the truth.
|
||||
//
|
||||
// Gated on the bootstrapped marker rather than on seq == 0: a founder seeds at
|
||||
// seq 0 and serves snapshots stamped seq 0, so a joiner that installs one is
|
||||
// still at seq 0 afterwards. Keyed on the sequence, this branch re-fired on
|
||||
// every attempt and the joiner never reached the Subscribe below.
|
||||
case !utils.IsOplogBootstrapped():
|
||||
if err := oplogFastForward(jsCtx, epoch); err != nil {
|
||||
// Only a node that can prove it holds the seed truth may answer an
|
||||
// unreachable peer by declaring its own store authoritative. Creating the
|
||||
// log is handled above, so what remains here is the formation writer
|
||||
// re-attaching to a stream it created in an earlier process. Anyone else
|
||||
// is a joiner whose peer is merely slow, and self-seeding there forks the
|
||||
// constellation into two histories that both look valid.
|
||||
if !utils.IsFormationWriter() {
|
||||
return err
|
||||
}
|
||||
utils.Warn("[OPLOG] no peer snapshot available, seeding the log from local state: " + err.Error())
|
||||
// founder: our own store is the seed truth for this epoch
|
||||
if errMark := utils.MarkOplogBootstrapped(epoch); errMark != nil {
|
||||
return errMark
|
||||
}
|
||||
}
|
||||
last = utils.GetLastAppliedSeq()
|
||||
}
|
||||
|
||||
// fast-forward may have adopted a newer epoch, which retargets the stream
|
||||
if utils.GetOplogEpoch() != epoch {
|
||||
return errors.New("oplog: epoch changed while catching up, re-attaching")
|
||||
}
|
||||
|
||||
sub, err := jsCtx.Subscribe(oplogSubjectWildcard(epoch), oplogHandleMsg,
|
||||
nats.OrderedConsumer(), nats.StartSequence(last+1))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
oplogMu.Lock()
|
||||
// a Stop raced us in: drop the subscription instead of running detached
|
||||
if !oplogRunning {
|
||||
oplogMu.Unlock()
|
||||
sub.Unsubscribe()
|
||||
return errors.New("oplog: stopped while attaching")
|
||||
}
|
||||
oplogSub = sub
|
||||
oplogMu.Unlock()
|
||||
|
||||
oplogResetStallTracking()
|
||||
oplogAttachedEpoch.Store(epoch)
|
||||
oplogAttached.Store(true)
|
||||
utils.Log("[OPLOG] Attached to " + oplogStreamName(epoch) + " from seq " + strconv.FormatUint(last+1, 10))
|
||||
|
||||
// Formation ends here: the log exists and this node is applying it, so every
|
||||
// write from now on publishes like any other node's.
|
||||
//
|
||||
// Clearing LATE — at attach rather than at creation — is load-bearing, and the
|
||||
// self-seed gate above depends on it. If the writer creates the stream and the
|
||||
// process dies before attaching, the next start sees created == false (the
|
||||
// stream now exists) but still holds the licence, so it is still allowed to seed
|
||||
// itself. Move this to creation and that restart wedges forever: the one node
|
||||
// that IS the seed would sit waiting for a peer snapshot that nobody can serve.
|
||||
if utils.IsFormationWriter() {
|
||||
if err := utils.ClearFormationWriter(); err != nil {
|
||||
utils.Error("[OPLOG] Failed to end formation mode", err)
|
||||
} else {
|
||||
utils.Log("[OPLOG] Formation complete, this node now publishes to " + oplogStreamName(epoch))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func oplogDetach() {
|
||||
oplogMu.Lock()
|
||||
sub := oplogSub
|
||||
oplogSub = nil
|
||||
oplogMu.Unlock()
|
||||
|
||||
oplogAttached.Store(false)
|
||||
if sub != nil {
|
||||
sub.Unsubscribe()
|
||||
}
|
||||
}
|
||||
|
||||
// oplogHalt stops applying rather than skipping an entry we don't understand:
|
||||
// skipping would silently fork this node's state from the rest of the cluster.
|
||||
func oplogHalt(err error, seq uint64) {
|
||||
oplogHalted.Store(true)
|
||||
oplogHaltReason.Store("seq " + strconv.FormatUint(seq, 10) + ": " + err.Error())
|
||||
utils.MajorError("[OPLOG] Apply loop halted at seq "+strconv.FormatUint(seq, 10)+
|
||||
", config replication has stopped on this node", err)
|
||||
go oplogDetach()
|
||||
}
|
||||
|
||||
type seqAction int
|
||||
|
||||
const (
|
||||
seqApply seqAction = iota
|
||||
seqSkip
|
||||
seqGap
|
||||
)
|
||||
|
||||
// oplogSeqAction gates delivery on strict contiguity: anything already applied
|
||||
// is dropped, and anything past the next entry means the ones in between aged
|
||||
// out of the log, which only a snapshot can repair.
|
||||
func oplogSeqAction(seq uint64, last uint64) seqAction {
|
||||
switch {
|
||||
case seq <= last:
|
||||
return seqSkip
|
||||
case seq > last+1:
|
||||
return seqGap
|
||||
}
|
||||
return seqApply
|
||||
}
|
||||
|
||||
func oplogHandleMsg(m *nats.Msg) {
|
||||
meta, err := m.Metadata()
|
||||
if err != nil {
|
||||
utils.Error("[OPLOG] message carries no JetStream metadata", err)
|
||||
return
|
||||
}
|
||||
seq := meta.Sequence.Stream
|
||||
|
||||
oplogApplyMu.Lock()
|
||||
defer oplogApplyMu.Unlock()
|
||||
|
||||
if oplogHalted.Load() {
|
||||
return
|
||||
}
|
||||
|
||||
last := utils.GetLastAppliedSeq()
|
||||
|
||||
switch oplogSeqAction(seq, last) {
|
||||
case seqSkip:
|
||||
// redelivery of something already committed — idempotent no-op
|
||||
utils.Debug("[OPLOG] skipping already-applied seq " + strconv.FormatUint(seq, 10))
|
||||
return
|
||||
case seqGap:
|
||||
utils.Warn("[OPLOG] sequence gap at " + strconv.FormatUint(seq, 10) +
|
||||
" (last applied " + strconv.FormatUint(last, 10) + "), fast-forwarding")
|
||||
oplogAttached.Store(false)
|
||||
go oplogDetach()
|
||||
return
|
||||
}
|
||||
|
||||
var env OpEnvelope
|
||||
if err := json.Unmarshal(m.Data, &env); err != nil {
|
||||
oplogHalt(err, seq)
|
||||
return
|
||||
}
|
||||
|
||||
if env.Epoch != utils.GetOplogEpoch() {
|
||||
// can't happen while the epoch is in the subject; consume it so the loop
|
||||
// stays contiguous rather than wedging on a forged entry
|
||||
utils.Warn("[OPLOG] epoch mismatch at seq " + strconv.FormatUint(seq, 10) + ", skipping entry")
|
||||
if err := utils.CommitOplogSeq(seq); err != nil {
|
||||
oplogHalt(err, seq)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if err := oplogApplyEnvelope(env, seq); err != nil {
|
||||
oplogHalt(err, seq)
|
||||
}
|
||||
}
|
||||
|
||||
func oplogApplyEnvelope(env OpEnvelope, seq uint64) error {
|
||||
if env.V != oplogEnvVersion {
|
||||
return errors.New("oplog: unsupported envelope version " + strconv.Itoa(env.V))
|
||||
}
|
||||
|
||||
// the waiter registry is what identifies our own ops: ReqID is unique, so
|
||||
// notifying an id we never registered is a no-op. Origin stays informational,
|
||||
// which keeps waiters working even when this node can't read its own name.
|
||||
if env.Kind == "domain" {
|
||||
return oplogApplyDomainEnvelope(env, seq)
|
||||
}
|
||||
if env.Kind != "db" {
|
||||
return errors.New("oplog: unknown op kind " + env.Kind)
|
||||
}
|
||||
|
||||
doc, err := utils.DecodeOpDoc(env.Table, env.Op, env.Doc)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
m := utils.Mutation{Table: env.Table, Op: env.Op, Filter: env.Filter, Doc: doc}
|
||||
|
||||
var pre []utils.ConstellationDevice
|
||||
err = utils.ApplyOpTx(m, seq, &pre)
|
||||
|
||||
var ec *utils.ErrConstraint
|
||||
if errors.As(err, &ec) {
|
||||
// A uniqueness rejection is a first-class outcome, not a fault: every replica
|
||||
// rejects it identically, so the sequence is still consumed and only the
|
||||
// originator learns it lost. (This is why mapSQLError must recognise BOTH
|
||||
// UNIQUE and PRIMARY KEY — users are keyed by nickname, so a duplicate user
|
||||
// arrives as a PK violation, and an unrecognised error here halts every node.)
|
||||
//
|
||||
// Deliberately two transactions rather than a SAVEPOINT: applyOpTx has already
|
||||
// rolled the data back, and we commit the sequence separately below. The crash
|
||||
// window in between is benign — on restart the node re-attaches at this same
|
||||
// sequence, hits the identical violation (the conflicting row is still there),
|
||||
// and commits. A replay re-rejects to the same end state; the only thing lost
|
||||
// is a waiter notification whose HTTP request died with the process.
|
||||
utils.Warn("[OPLOG] entry at seq " + strconv.FormatUint(seq, 10) + " rejected: " + err.Error())
|
||||
if errSeq := utils.CommitOplogSeq(seq); errSeq != nil {
|
||||
return errSeq
|
||||
}
|
||||
notifyOpWaiter(env.ReqID, err)
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
notifyOpWaiter(env.ReqID, nil)
|
||||
oplogReactToTableOp(m, pre)
|
||||
return nil
|
||||
}
|
||||
|
||||
func oplogApplyDomainEnvelope(env OpEnvelope, seq uint64) error {
|
||||
d, ok := oplogDomains[env.Domain]
|
||||
if !ok {
|
||||
return errors.New("oplog: unknown domain " + env.Domain)
|
||||
}
|
||||
|
||||
old, _ := d.Snapshot()
|
||||
|
||||
utils.ConfigLock.Lock()
|
||||
err := d.Apply(env.Doc)
|
||||
utils.ConfigLock.Unlock()
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Apply MUST precede the sequence commit, and this ordering is load-bearing:
|
||||
// no transaction spans config.json and SQLite, so a crash between them has to
|
||||
// land on the safe side. Applied-but-not-committed replays the same full state,
|
||||
// which is harmless because domain payloads are idempotent "set" ops. Committed-
|
||||
// but-not-applied would drop the change permanently with nothing to detect it.
|
||||
// Do not hoist this above d.Apply.
|
||||
if err := utils.CommitOplogSeq(seq); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
notifyOpWaiter(env.ReqID, nil)
|
||||
if d.React != nil {
|
||||
d.React(old, env.Doc)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// deviceTopologyFields are the columns that reshape the nebula mesh or the NATS
|
||||
// credential set; anything else is cosmetic to the network.
|
||||
var deviceTopologyFields = map[string]bool{
|
||||
"DeviceName": true,
|
||||
"IP": true,
|
||||
"Blocked": true,
|
||||
"CosmosNode": true,
|
||||
"IsLighthouse": true,
|
||||
"IsRelay": true,
|
||||
"IsExitNode": true,
|
||||
"IsLoadBalancer": true,
|
||||
"PublicHostname": true,
|
||||
"Port": true,
|
||||
"PublicKey": true,
|
||||
"APIKey": true,
|
||||
"Fingerprint": true,
|
||||
}
|
||||
|
||||
// oplogReactToTableOp bounces only what the change actually invalidates: user
|
||||
// rows need nothing, a tag rename needs the device cache, and only a real
|
||||
// topology move is worth a nebula restart.
|
||||
func oplogReactToTableOp(m utils.Mutation, pre []utils.ConstellationDevice) {
|
||||
if m.Table != "devices" {
|
||||
return
|
||||
}
|
||||
|
||||
switch m.Op {
|
||||
case "insert", "insertMany", "delete", "deleteMany":
|
||||
go RestartNebula()
|
||||
return
|
||||
}
|
||||
|
||||
fields, _ := m.Doc.(map[string]interface{})
|
||||
if utils.DeviceFieldsChanged(pre, fields, deviceTopologyFields) {
|
||||
go RestartNebula()
|
||||
return
|
||||
}
|
||||
|
||||
go refreshDeviceCache()
|
||||
}
|
||||
@@ -0,0 +1,280 @@
|
||||
package constellation
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
|
||||
"github.com/azukaar/cosmos-server/src/utils"
|
||||
)
|
||||
|
||||
// Everything the op-log replicates that isn't a SQLite row lives here, one entry
|
||||
// per domain. Adding a future domain is one registry entry — that is the point
|
||||
// of the indirection.
|
||||
//
|
||||
// Apply runs with utils.ConfigLock already held by the caller; React runs after
|
||||
// the sequence is committed, and must not block the apply loop.
|
||||
type Domain struct {
|
||||
Name string
|
||||
Apply func(state json.RawMessage) error
|
||||
React func(old json.RawMessage, new json.RawMessage)
|
||||
Snapshot func() (json.RawMessage, error)
|
||||
}
|
||||
|
||||
const (
|
||||
DomainAuthKeys = "auth_keys"
|
||||
DomainDNS = "dns"
|
||||
DomainAPITokens = "api_tokens"
|
||||
DomainRoles = "roles"
|
||||
DomainOpenIDClients = "openid_clients"
|
||||
DomainFileCACrt = "file:ca.crt"
|
||||
DomainFileCAKey = "file:ca.key"
|
||||
DomainFileRclone = "file:rclone.conf"
|
||||
)
|
||||
|
||||
type AuthKeysPayload struct {
|
||||
AuthPrivateKey string `json:"authPrivateKey"`
|
||||
AuthPublicKey string `json:"authPublicKey"`
|
||||
}
|
||||
|
||||
type DNSPayload struct {
|
||||
DNSPort string `json:"dnsPort"`
|
||||
DNSFallback string `json:"dnsFallback"`
|
||||
DNSBlockBlacklist bool `json:"dnsBlockBlacklist"`
|
||||
DNSAdditionalBlocklists []string `json:"dnsAdditionalBlocklists"`
|
||||
CustomDNSEntries []utils.ConstellationDNSEntry `json:"customDNSEntries"`
|
||||
}
|
||||
|
||||
// FilePayload carries a loose config file as base64. Empty Data means "no such
|
||||
// file here" and is never applied, so a node that lacks a CA can't wipe everyone's.
|
||||
type FilePayload struct {
|
||||
Data string `json:"data"`
|
||||
}
|
||||
|
||||
var oplogDomains = map[string]Domain{}
|
||||
|
||||
func init() {
|
||||
register := func(d Domain) { oplogDomains[d.Name] = d }
|
||||
|
||||
register(Domain{
|
||||
Name: DomainAuthKeys,
|
||||
Apply: func(state json.RawMessage) error {
|
||||
var p AuthKeysPayload
|
||||
if err := json.Unmarshal(state, &p); err != nil {
|
||||
return err
|
||||
}
|
||||
config := utils.ReadConfigFromFile()
|
||||
config.HTTPConfig.AuthPrivateKey = p.AuthPrivateKey
|
||||
config.HTTPConfig.AuthPublicKey = p.AuthPublicKey
|
||||
utils.SetBaseMainConfig(config)
|
||||
return nil
|
||||
},
|
||||
// rotating the JWT keypair invalidates live sessions, so only bounce
|
||||
// the server when the keys actually moved
|
||||
React: func(old json.RawMessage, new json.RawMessage) {
|
||||
if !bytes.Equal(old, new) {
|
||||
go utils.RestartHTTPServer()
|
||||
}
|
||||
},
|
||||
Snapshot: func() (json.RawMessage, error) {
|
||||
config := utils.GetMainConfig()
|
||||
return json.Marshal(AuthKeysPayload{
|
||||
AuthPrivateKey: config.HTTPConfig.AuthPrivateKey,
|
||||
AuthPublicKey: config.HTTPConfig.AuthPublicKey,
|
||||
})
|
||||
},
|
||||
})
|
||||
|
||||
register(Domain{
|
||||
Name: DomainDNS,
|
||||
// Assigns the five replicated fields individually, and must keep doing so:
|
||||
// ConstellationConfig also holds per-node state (Enabled, IPRange,
|
||||
// ThisDeviceName, ConstellationHostname, NATSReplicas, DNSDisabled,
|
||||
// FirewallBlockedClients) that api_nebula_connect.go writes locally by design.
|
||||
// Replacing the struct wholesale here would push one node's identity to every
|
||||
// other node.
|
||||
Apply: func(state json.RawMessage) error {
|
||||
var p DNSPayload
|
||||
if err := json.Unmarshal(state, &p); err != nil {
|
||||
return err
|
||||
}
|
||||
config := utils.ReadConfigFromFile()
|
||||
config.ConstellationConfig.DNSPort = p.DNSPort
|
||||
config.ConstellationConfig.DNSFallback = p.DNSFallback
|
||||
config.ConstellationConfig.DNSBlockBlacklist = p.DNSBlockBlacklist
|
||||
config.ConstellationConfig.DNSAdditionalBlocklists = p.DNSAdditionalBlocklists
|
||||
config.ConstellationConfig.CustomDNSEntries = p.CustomDNSEntries
|
||||
utils.SetBaseMainConfig(config)
|
||||
return nil
|
||||
},
|
||||
React: func(old json.RawMessage, new json.RawMessage) {
|
||||
if !bytes.Equal(old, new) {
|
||||
go ReloadDNS()
|
||||
}
|
||||
},
|
||||
Snapshot: func() (json.RawMessage, error) {
|
||||
c := utils.GetMainConfig().ConstellationConfig
|
||||
return json.Marshal(DNSPayload{
|
||||
DNSPort: c.DNSPort,
|
||||
DNSFallback: c.DNSFallback,
|
||||
DNSBlockBlacklist: c.DNSBlockBlacklist,
|
||||
DNSAdditionalBlocklists: c.DNSAdditionalBlocklists,
|
||||
CustomDNSEntries: c.CustomDNSEntries,
|
||||
})
|
||||
},
|
||||
})
|
||||
|
||||
// api_tokens / roles / openid_clients need no reaction: every one of them is
|
||||
// read per request straight off the config, never cached in a running server
|
||||
register(Domain{
|
||||
Name: DomainAPITokens,
|
||||
Apply: func(state json.RawMessage) error {
|
||||
var tokens map[string]utils.APITokenConfig
|
||||
if err := json.Unmarshal(state, &tokens); err != nil {
|
||||
return err
|
||||
}
|
||||
config := utils.ReadConfigFromFile()
|
||||
config.APITokens = tokens
|
||||
utils.SetBaseMainConfig(config)
|
||||
return nil
|
||||
},
|
||||
Snapshot: func() (json.RawMessage, error) {
|
||||
return json.Marshal(utils.GetMainConfig().APITokens)
|
||||
},
|
||||
})
|
||||
|
||||
register(Domain{
|
||||
Name: DomainRoles,
|
||||
Apply: func(state json.RawMessage) error {
|
||||
var roles map[utils.Role]utils.RoleConfig
|
||||
if err := json.Unmarshal(state, &roles); err != nil {
|
||||
return err
|
||||
}
|
||||
config := utils.ReadConfigFromFile()
|
||||
config.Roles = roles
|
||||
utils.SetBaseMainConfig(config)
|
||||
return nil
|
||||
},
|
||||
Snapshot: func() (json.RawMessage, error) {
|
||||
return json.Marshal(utils.GetMainConfig().Roles)
|
||||
},
|
||||
})
|
||||
|
||||
register(Domain{
|
||||
Name: DomainOpenIDClients,
|
||||
Apply: func(state json.RawMessage) error {
|
||||
var clients []utils.OpenIDClient
|
||||
if err := json.Unmarshal(state, &clients); err != nil {
|
||||
return err
|
||||
}
|
||||
config := utils.ReadConfigFromFile()
|
||||
config.OpenIDClients = clients
|
||||
utils.SetBaseMainConfig(config)
|
||||
return nil
|
||||
},
|
||||
Snapshot: func() (json.RawMessage, error) {
|
||||
return json.Marshal(utils.GetMainConfig().OpenIDClients)
|
||||
},
|
||||
})
|
||||
|
||||
register(fileDomain(DomainFileCACrt, "ca.crt", nil))
|
||||
register(fileDomain(DomainFileCAKey, "ca.key", nil))
|
||||
register(fileDomain(DomainFileRclone, "rclone.conf", func(old json.RawMessage, new json.RawMessage) {
|
||||
if bytes.Equal(old, new) || utils.InitRemoteStorage == nil {
|
||||
return
|
||||
}
|
||||
go utils.InitRemoteStorage()
|
||||
}))
|
||||
}
|
||||
|
||||
func fileDomain(name string, filename string, react func(old json.RawMessage, new json.RawMessage)) Domain {
|
||||
path := func() string { return utils.CONFIGFOLDER + filename }
|
||||
return Domain{
|
||||
Name: name,
|
||||
React: react,
|
||||
Apply: func(state json.RawMessage) error {
|
||||
var p FilePayload
|
||||
if err := json.Unmarshal(state, &p); err != nil {
|
||||
return err
|
||||
}
|
||||
if p.Data == "" {
|
||||
return nil
|
||||
}
|
||||
data, err := base64.StdEncoding.DecodeString(p.Data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return writeFileAtomic(path(), data, 0600)
|
||||
},
|
||||
Snapshot: func() (json.RawMessage, error) {
|
||||
data, err := ioutil.ReadFile(path())
|
||||
if err != nil {
|
||||
return json.Marshal(FilePayload{})
|
||||
}
|
||||
return json.Marshal(FilePayload{Data: base64.StdEncoding.EncodeToString(data)})
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// PublishFileDomain publishes a loose config file's current contents as the
|
||||
// full state of its domain.
|
||||
func PublishFileDomain(domain string, filename string) error {
|
||||
data, err := ioutil.ReadFile(utils.CONFIGFOLDER + filename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return PublishDomainOp(domain, FilePayload{Data: base64.StdEncoding.EncodeToString(data)})
|
||||
}
|
||||
|
||||
// writeFileAtomic keeps a crash from leaving a half-written cert on disk.
|
||||
func writeFileAtomic(path string, data []byte, perm os.FileMode) error {
|
||||
tmp := path + ".tmp"
|
||||
if err := ioutil.WriteFile(tmp, data, perm); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(tmp, path); err != nil {
|
||||
os.Remove(tmp)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// applyDomainLocal installs a domain's state under ConfigLock and fires its
|
||||
// reaction. Shared by the apply loop and the direct write path.
|
||||
func applyDomainLocal(name string, state json.RawMessage) error {
|
||||
d, ok := oplogDomains[name]
|
||||
if !ok {
|
||||
return errors.New("oplog: unknown domain " + name)
|
||||
}
|
||||
|
||||
old, _ := d.Snapshot()
|
||||
|
||||
utils.ConfigLock.Lock()
|
||||
err := d.Apply(state)
|
||||
utils.ConfigLock.Unlock()
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.React != nil {
|
||||
d.React(old, state)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// snapshotDomains captures every domain's current state for a fast-forward payload.
|
||||
func snapshotDomains() map[string]json.RawMessage {
|
||||
out := map[string]json.RawMessage{}
|
||||
for name, d := range oplogDomains {
|
||||
state, err := d.Snapshot()
|
||||
if err != nil {
|
||||
utils.Warn("[OPLOG] snapshot of domain " + name + " failed: " + err.Error())
|
||||
continue
|
||||
}
|
||||
out[name] = state
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,366 @@
|
||||
package constellation
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/azukaar/cosmos-server/src/utils"
|
||||
)
|
||||
|
||||
// A node that has been down longer than the log's 100-entry window can't replay
|
||||
// its way back. It asks any healthy peer for a snapshot instead: the same
|
||||
// mechanism for managers and agents, and the only way a brand-new node gets its
|
||||
// initial state.
|
||||
|
||||
const oplogSnapshotSubject = "cosmos._global_.oplog.snapshot-request"
|
||||
const oplogSnapshotTimeout = 15 * time.Second
|
||||
const oplogFastForwardAttempts = 5
|
||||
|
||||
type OplogSnapshot struct {
|
||||
Epoch uint64 `json:"epoch"`
|
||||
Seq uint64 `json:"seq"`
|
||||
Dump json.RawMessage `json:"dump"`
|
||||
Domains map[string]json.RawMessage `json:"domains"`
|
||||
}
|
||||
|
||||
// oplogSnapshotRequest carries the asker's epoch so the exchange is fenced on
|
||||
// BOTH sides, not just at the requester.
|
||||
type oplogSnapshotRequest struct {
|
||||
Epoch uint64 `json:"epoch"`
|
||||
}
|
||||
|
||||
// oplogMayServeSnapshot reports whether this node is entitled to hand its state
|
||||
// to a peer at all. Extracted from the router so the policy is testable on its
|
||||
// own; the per-request epoch check stays with the request.
|
||||
func oplogMayServeSnapshot() bool {
|
||||
if oplogHalted.Load() {
|
||||
return false
|
||||
}
|
||||
|
||||
// A node the constellation has REMOVED must not be a source of truth for
|
||||
// anyone, and this needs its own check rather than falling out of the attached
|
||||
// gate below: self-fencing deliberately leaves a blocked node attached so it
|
||||
// can go on materializing as a read-only member, which means it would
|
||||
// otherwise sail straight past that gate and answer every joiner.
|
||||
//
|
||||
// This matters more than the write fence it completes. A snapshot is not just
|
||||
// device rows — snapshotDomains() walks every registered domain, so the payload
|
||||
// carries file:ca.key (the constellation CA private key), the auth_keys signing
|
||||
// keypair, api_tokens, openid_clients with their secrets, and file:rclone.conf
|
||||
// credentials. Answering here would let a removed machine hand the cluster's
|
||||
// entire secret material to every node that enrolls or falls off retention
|
||||
// afterwards.
|
||||
if isSelfBlocked() {
|
||||
return false
|
||||
}
|
||||
|
||||
return oplogAttached.Load() || utils.IsFormationWriter()
|
||||
}
|
||||
|
||||
// oplogSnapshotRouter answers snapshot requests from peers.
|
||||
func oplogSnapshotRouter(conn *nats.Conn) {
|
||||
conn.Subscribe(oplogSnapshotSubject, func(m *nats.Msg) {
|
||||
// Only a node whose own loop is healthy and current can hand out state —
|
||||
// this is also what stops a mid-bootstrap node (attached to nothing yet)
|
||||
// from answering with the empty store it is about to fill.
|
||||
//
|
||||
// The formation writer is the one exception, and it has to be: during HA
|
||||
// formation there is no log to attach to, yet its store IS the seed truth
|
||||
// every enrolling peer needs.
|
||||
//
|
||||
// INVARIANT, and it lives in three files — do not weaken any of them
|
||||
// separately. Serving while unattached is safe ONLY because no peer can
|
||||
// reach this path until the stream exists: a non-licensed node is refused by
|
||||
// oplogMayCreateStream (oplog.go) while no stream exists, and the moment one
|
||||
// does exist oplogStreamSeen is set (oplog_apply.go), which is the same
|
||||
// moment the writer's direct-write licence stops applying. So by the time
|
||||
// anyone can ask, the writer's store is frozen and the snapshot is
|
||||
// consistent. If a peer could snapshot mid-formation it would install state
|
||||
// at (epoch, 0) while the writer kept making direct writes that appear in no
|
||||
// log and in no snapshot, and nothing would ever reconcile the two.
|
||||
if !oplogMayServeSnapshot() {
|
||||
return
|
||||
}
|
||||
|
||||
// Epoch fence, responder side. A node behind the asker must stay SILENT
|
||||
// rather than answer with state from an epoch the asker has already left.
|
||||
// The asker would reject it — but core request/reply takes the first reply
|
||||
// and gives up, so one fast stale responder is enough to starve the current
|
||||
// peers and leave the asker believing nobody can help. Its next move is to
|
||||
// create a stream for its own epoch, which is exactly how an abandoned
|
||||
// epoch's log gets resurrected on a live cluster.
|
||||
//
|
||||
// A body we can't read, or a legacy one, decodes to epoch 0 and is served
|
||||
// as before.
|
||||
var req oplogSnapshotRequest
|
||||
json.Unmarshal(m.Data, &req)
|
||||
if req.Epoch > utils.GetOplogEpoch() {
|
||||
utils.Debug("[OPLOG] declining a snapshot request from epoch " +
|
||||
strconv.FormatUint(req.Epoch, 10) + ": this node is behind it")
|
||||
return
|
||||
}
|
||||
|
||||
snap, err := buildOplogSnapshot()
|
||||
if err != nil {
|
||||
utils.Error("[OPLOG] Failed to build snapshot for a peer", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Refuse to serve an empty store. Install is replace-all, so answering
|
||||
// here would wipe the requester's good state. Guarded on both sides: the
|
||||
// requester cannot tell an empty peer from a legitimately empty cluster,
|
||||
// and this node can.
|
||||
if dumpIsEmpty(snap.Dump) {
|
||||
utils.Warn("[OPLOG] declining to serve a snapshot: this node's store is empty")
|
||||
return
|
||||
}
|
||||
|
||||
payload, err := json.Marshal(snap)
|
||||
if err != nil {
|
||||
utils.Error("[OPLOG] Failed to marshal snapshot for a peer", err)
|
||||
return
|
||||
}
|
||||
m.Respond(payload)
|
||||
})
|
||||
}
|
||||
|
||||
func buildOplogSnapshot() (OplogSnapshot, error) {
|
||||
// rows and the log position they belong to come out of one read tx
|
||||
dump, epoch, seq, err := utils.BuildSnapshot()
|
||||
if err != nil {
|
||||
return OplogSnapshot{}, err
|
||||
}
|
||||
|
||||
// domain state lives in config.json and loose files rather than SQLite, so
|
||||
// the config lock is the closest thing to that tx we have
|
||||
utils.ConfigLock.Lock()
|
||||
domains := snapshotDomains()
|
||||
utils.ConfigLock.Unlock()
|
||||
|
||||
return OplogSnapshot{Epoch: epoch, Seq: seq, Dump: dump, Domains: domains}, nil
|
||||
}
|
||||
|
||||
// dumpIsEmpty reports whether a snapshot body carries no rows at all.
|
||||
//
|
||||
// A length check does NOT answer this and is actively misleading: an empty store
|
||||
// marshals to `{"devices":[],"users":[]}` — 25 bytes, not zero — so `len(dump) == 0`
|
||||
// only catches a missing field and would wave through a semantically empty
|
||||
// snapshot straight into a replace-all install.
|
||||
func dumpIsEmpty(dump []byte) bool {
|
||||
if len(dump) == 0 {
|
||||
return true
|
||||
}
|
||||
var d struct {
|
||||
Users []json.RawMessage `json:"users"`
|
||||
Devices []json.RawMessage `json:"devices"`
|
||||
}
|
||||
if err := json.Unmarshal(dump, &d); err != nil {
|
||||
return true
|
||||
}
|
||||
return len(d.Users) == 0 && len(d.Devices) == 0
|
||||
}
|
||||
|
||||
// oplogAdoptFromPeer asks the cluster for state when this node's own epoch has
|
||||
// no log behind it, and installs whatever epoch comes back. Reports whether it
|
||||
// adopted anything — if it did, the caller must re-attach, because the epoch and
|
||||
// therefore the stream name have changed under it.
|
||||
//
|
||||
// This is the reform recovery path. It deliberately runs BEFORE stream creation:
|
||||
// a node whose epoch has been left behind cannot tell "I am the founder" from "I
|
||||
// have been reformed out" by looking at anything local, and only a peer can
|
||||
// settle it.
|
||||
func oplogAdoptFromPeer() bool {
|
||||
snap, err := requestOplogSnapshot()
|
||||
if err != nil {
|
||||
utils.Debug("[OPLOG] no peer answered a snapshot request: " + err.Error())
|
||||
return false
|
||||
}
|
||||
// see dumpIsEmpty: install is replace-all, so an empty reply would wipe us
|
||||
if dumpIsEmpty(snap.Dump) {
|
||||
return false
|
||||
}
|
||||
|
||||
local := utils.GetOplogEpoch()
|
||||
if snap.Epoch < local {
|
||||
utils.Warn("[OPLOG] ignoring a snapshot from stale epoch " + strconv.FormatUint(snap.Epoch, 10))
|
||||
return false
|
||||
}
|
||||
// Same epoch and we already hold its state: there is nothing to learn, and
|
||||
// re-installing on every supervisor tick while we wait for the founder to
|
||||
// create the stream would be a busy loop over the whole store.
|
||||
if snap.Epoch == local && utils.IsOplogBootstrapped() {
|
||||
return false
|
||||
}
|
||||
|
||||
before, _ := utils.BuildLogicalDump()
|
||||
if err := installOplogSnapshot(snap); err != nil {
|
||||
utils.Error("[OPLOG] Failed to install a peer's snapshot", err)
|
||||
return false
|
||||
}
|
||||
utils.Log("[OPLOG] Installed a peer snapshot at epoch " + strconv.FormatUint(snap.Epoch, 10) +
|
||||
" seq " + strconv.FormatUint(snap.Seq, 10))
|
||||
|
||||
// moving epochs means the JetStream state we hold belongs to a cluster that
|
||||
// no longer exists; oplogDropStaleJetStream takes the restart over when so
|
||||
if snap.Epoch != local && oplogDropStaleJetStream() {
|
||||
return true
|
||||
}
|
||||
afterInstallReaction(before)
|
||||
return true
|
||||
}
|
||||
|
||||
// oplogFastForward replaces local state wholesale from a peer's snapshot and
|
||||
// loops until the stream's retention window actually covers where it landed.
|
||||
func oplogFastForward(jsCtx nats.JetStreamContext, epoch uint64) error {
|
||||
for attempt := 1; attempt <= oplogFastForwardAttempts; attempt++ {
|
||||
snap, err := requestOplogSnapshot()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// see dumpIsEmpty: a byte-length check here would accept `{"devices":[],"users":[]}`
|
||||
if dumpIsEmpty(snap.Dump) {
|
||||
return errors.New("oplog: peer returned an empty snapshot, refusing to replace local state")
|
||||
}
|
||||
|
||||
// a snapshot from an older epoch is a stale peer talking; try again
|
||||
if snap.Epoch < utils.GetOplogEpoch() {
|
||||
return errors.New("oplog: rejected snapshot from stale epoch " + strconv.FormatUint(snap.Epoch, 10))
|
||||
}
|
||||
|
||||
before, _ := utils.BuildLogicalDump()
|
||||
|
||||
if err := installOplogSnapshot(snap); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if snap.Epoch != epoch {
|
||||
// adopted a newer epoch: the stream we were about to attach to is gone
|
||||
utils.Log("[OPLOG] Adopted epoch " + strconv.FormatUint(snap.Epoch, 10) + " from snapshot")
|
||||
if !oplogDropStaleJetStream() {
|
||||
afterInstallReaction(before)
|
||||
}
|
||||
return errors.New("oplog: epoch changed during fast-forward, re-attaching")
|
||||
}
|
||||
|
||||
si, errInfo := jsCtx.StreamInfo(oplogStreamName(snap.Epoch))
|
||||
if errInfo != nil {
|
||||
return errInfo
|
||||
}
|
||||
|
||||
// mandatory gap check: retention can move past our snapshot while we
|
||||
// install it, which would leave us permanently one entry short
|
||||
if si.State.Msgs == 0 || si.State.FirstSeq <= snap.Seq+1 {
|
||||
utils.Log("[OPLOG] Fast-forwarded to seq " + strconv.FormatUint(snap.Seq, 10))
|
||||
afterInstallReaction(before)
|
||||
return nil
|
||||
}
|
||||
|
||||
utils.Warn("[OPLOG] snapshot aged out while installing (attempt " + strconv.Itoa(attempt) + "), refetching")
|
||||
}
|
||||
return errors.New("oplog: could not catch up, the log outruns snapshot installs")
|
||||
}
|
||||
|
||||
func requestOplogSnapshot() (OplogSnapshot, error) {
|
||||
var snap OplogSnapshot
|
||||
|
||||
clientConfigLock.RLock()
|
||||
conn := nc
|
||||
clientConfigLock.RUnlock()
|
||||
|
||||
if conn == nil {
|
||||
return snap, errors.New("oplog: NATS client not connected")
|
||||
}
|
||||
|
||||
// our epoch travels with the request so a peer behind us declines rather than
|
||||
// answering with state we would only have to throw away
|
||||
body, err := json.Marshal(oplogSnapshotRequest{Epoch: utils.GetOplogEpoch()})
|
||||
if err != nil {
|
||||
return snap, err
|
||||
}
|
||||
|
||||
msg, err := conn.Request(oplogSnapshotSubject, body, oplogSnapshotTimeout)
|
||||
if err != nil {
|
||||
return snap, err
|
||||
}
|
||||
if err := json.Unmarshal(msg.Data, &snap); err != nil {
|
||||
return snap, err
|
||||
}
|
||||
return snap, nil
|
||||
}
|
||||
|
||||
func installOplogSnapshot(snap OplogSnapshot) error {
|
||||
utils.ConfigLock.Lock()
|
||||
defer utils.ConfigLock.Unlock()
|
||||
|
||||
if err := utils.ApplyLogicalDump(snap.Dump, snap.Epoch, snap.Seq); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for name, state := range snap.Domains {
|
||||
d, ok := oplogDomains[name]
|
||||
if !ok {
|
||||
utils.Warn("[OPLOG] snapshot carries unknown domain " + name + ", ignoring")
|
||||
continue
|
||||
}
|
||||
if err := d.Apply(state); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// oplogDropStaleJetStream throws away the JetStream state a manager built under
|
||||
// the epoch it has just been overtaken by, and restarts into the new one.
|
||||
//
|
||||
// A force-reform abandons a cluster: its meta-raft group and its stream stay on
|
||||
// every manager's disk. A manager that was down for the reform and comes back
|
||||
// would otherwise carry that group into the constellation it is rejoining, where
|
||||
// it is a claim on a quorum that no longer exists. Agents run no JetStream, so
|
||||
// they have nothing to drop and restart through the ordinary reaction instead.
|
||||
//
|
||||
// Reports whether it took the restart over. Runs async: the caller is inside the
|
||||
// apply path, and the teardown stops the very loop it is running under.
|
||||
//
|
||||
// KNOWN RESIDUAL, so nobody rediscovers it as a bug: this only reaches a manager
|
||||
// that ADOPTS a newer epoch, which means one that could not attach. Managers
|
||||
// revived together in numbers that still hold quorum of the ABANDONED cluster's
|
||||
// meta group elect among themselves, attach to their own old stream, and never
|
||||
// re-enter oplogAttach — so they never ask for a snapshot, never see the new
|
||||
// epoch, and never get here. They stay a self-consistent island at the dead
|
||||
// epoch. They cannot contaminate the live cluster (no current-epoch node will
|
||||
// create or adopt into their epoch, and survivors no longer dial them), and an
|
||||
// operator restart kills their credentials since reform blocked their device
|
||||
// records. Fencing them at the transport needs authenticated cluster routes,
|
||||
// which the routes here deliberately do not have.
|
||||
func oplogDropStaleJetStream() bool {
|
||||
if utils.FBL.AgentMode {
|
||||
return false
|
||||
}
|
||||
go func() {
|
||||
StopNATS()
|
||||
CloseNATSClient()
|
||||
os.RemoveAll(jetstreamDir())
|
||||
RestartNebula()
|
||||
}()
|
||||
return true
|
||||
}
|
||||
|
||||
// afterInstallReaction bounces the mesh only when the snapshot actually moved
|
||||
// the device set; a no-op catch-up costs nothing.
|
||||
func afterInstallReaction(before []byte) {
|
||||
after, err := utils.BuildLogicalDump()
|
||||
if err != nil {
|
||||
utils.Error("[OPLOG] Failed to read back state after snapshot install", err)
|
||||
return
|
||||
}
|
||||
if bytes.Equal(before, after) {
|
||||
return
|
||||
}
|
||||
go RestartNebula()
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,9 +1,7 @@
|
||||
package constellation
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -260,23 +258,3 @@ func TestUnitRefreshDeviceCacheInvalidatesStaleCurrentDevice(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnitSyncPayloadNoRestartWireCompat(t *testing.T) {
|
||||
// payloads from old builds lack the field and must decode to restart (false)
|
||||
var legacy SyncPayload
|
||||
if err := json.Unmarshal([]byte(`{"database":"","lastEdited":1}`), &legacy); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if legacy.NoRestart {
|
||||
t.Error("legacy payload decoded NoRestart = true, want false (restart)")
|
||||
}
|
||||
|
||||
// the flag round-trips when set, and omitempty keeps default payloads unchanged
|
||||
on, _ := json.Marshal(SyncPayload{NoRestart: true})
|
||||
if !strings.Contains(string(on), `"noRestart":true`) {
|
||||
t.Errorf("NoRestart=true not serialized: %s", on)
|
||||
}
|
||||
off, _ := json.Marshal(SyncPayload{})
|
||||
if strings.Contains(string(off), "noRestart") {
|
||||
t.Errorf("default payload should omit noRestart: %s", off)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,382 +0,0 @@
|
||||
package constellation
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"strconv"
|
||||
"io/ioutil"
|
||||
"time"
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/azukaar/cosmos-server/src/utils"
|
||||
)
|
||||
|
||||
type SyncPayload struct {
|
||||
Database string `json:"database"`
|
||||
AuthPrivateKey string `json:"authPrivateKey"`
|
||||
AuthPublicKey string `json:"authPublicKey"`
|
||||
CART string `json:"caCrt"`
|
||||
CAKey string `json:"caKey"`
|
||||
DNS SyncDNSPayload `json:"dns"`
|
||||
RcloneConfig string `json:"rcloneConfig"`
|
||||
ConfigData SyncConfigPayload `json:"configData"`
|
||||
LastEdited int64 `json:"lastEdited"`
|
||||
// NoRestart: receivers apply the payload but skip the nebula+NATS restart.
|
||||
// Inverse-named so the zero value keeps wire compat with old builds (= restart).
|
||||
NoRestart bool `json:"noRestart,omitempty"`
|
||||
}
|
||||
|
||||
type SyncDNSPayload struct {
|
||||
DNSPort string `json:"dnsPort"`
|
||||
DNSFallback string `json:"dnsFallback"`
|
||||
DNSBlockBlacklist bool `json:"dnsBlockBlacklist"`
|
||||
DNSAdditionalBlocklists []string `json:"dnsAdditionalBlocklists"`
|
||||
CustomDNSEntries []utils.ConstellationDNSEntry `json:"customDNSEntries"`
|
||||
}
|
||||
|
||||
type SyncConfigPayload struct {
|
||||
APITokens map[string]utils.APITokenConfig `json:"apiTokens,omitempty"`
|
||||
Roles map[utils.Role]utils.RoleConfig `json:"roles,omitempty"`
|
||||
}
|
||||
|
||||
type SyncRequestPayload struct {
|
||||
EditedAt int64 `json:"editedAt"`
|
||||
}
|
||||
|
||||
func MakeSyncPayload(rawPayload string) string {
|
||||
// restart-required is the historical default for every existing call site
|
||||
return makeSyncPayload(rawPayload, false)
|
||||
}
|
||||
|
||||
func makeSyncPayload(rawPayload string, noRestart bool) string {
|
||||
utils.Log("Constellation: MakeSyncPayload: Making sync payload")
|
||||
|
||||
// Read database file
|
||||
dbPath := utils.CONFIGFOLDER + "database"
|
||||
dbData, err := ioutil.ReadFile(dbPath)
|
||||
if err != nil {
|
||||
utils.Error("Constellation: MakeSyncPayload: Failed to read database file", err)
|
||||
return ""
|
||||
}
|
||||
|
||||
if (rawPayload != "") {
|
||||
var payload SyncPayload
|
||||
err := json.Unmarshal([]byte(rawPayload), &payload)
|
||||
if err != nil {
|
||||
utils.Error("Constellation: ReceiveSyncPayload: Failed to unmarshal payload", err)
|
||||
return ""
|
||||
}
|
||||
|
||||
editedAt := payload.LastEdited
|
||||
|
||||
// If our local is older or same, skip sending
|
||||
if utils.GetFileLastModifiedTime(dbPath).Unix() <= editedAt {
|
||||
utils.Warn("Constellation: MakeSyncPayload: Local database is older or same as requested one, skipping sending")
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
// Encode database content to base64
|
||||
dbBase64 := base64.StdEncoding.EncodeToString(dbData)
|
||||
|
||||
// Read auth keys
|
||||
AuthPrivateKey := utils.GetMainConfig().HTTPConfig.AuthPrivateKey
|
||||
AuthPublicKey := utils.GetMainConfig().HTTPConfig.AuthPublicKey
|
||||
|
||||
// Read CA
|
||||
CART := ""
|
||||
CAKey := ""
|
||||
|
||||
caCrtData, err := ioutil.ReadFile(utils.CONFIGFOLDER + "ca.crt")
|
||||
if err == nil {
|
||||
CART = base64.StdEncoding.EncodeToString(caCrtData)
|
||||
}
|
||||
|
||||
caKeyData, err := ioutil.ReadFile(utils.CONFIGFOLDER + "ca.key")
|
||||
if err == nil {
|
||||
CAKey = base64.StdEncoding.EncodeToString(caKeyData)
|
||||
}
|
||||
|
||||
// Read rclone config
|
||||
rcloneConfig := ""
|
||||
rcloneConfigData, err := ioutil.ReadFile(utils.CONFIGFOLDER + "rclone.conf")
|
||||
if err == nil {
|
||||
rcloneConfig = base64.StdEncoding.EncodeToString(rcloneConfigData)
|
||||
}
|
||||
|
||||
// Create payload
|
||||
|
||||
constellationConfig := utils.GetMainConfig().ConstellationConfig
|
||||
|
||||
sendPayload := SyncPayload{
|
||||
Database: dbBase64,
|
||||
AuthPrivateKey: AuthPrivateKey,
|
||||
AuthPublicKey: AuthPublicKey,
|
||||
CART: CART,
|
||||
CAKey: CAKey,
|
||||
DNS: SyncDNSPayload{
|
||||
DNSPort: constellationConfig.DNSPort,
|
||||
DNSFallback: constellationConfig.DNSFallback,
|
||||
DNSBlockBlacklist: constellationConfig.DNSBlockBlacklist,
|
||||
DNSAdditionalBlocklists: constellationConfig.DNSAdditionalBlocklists,
|
||||
CustomDNSEntries: constellationConfig.CustomDNSEntries,
|
||||
},
|
||||
RcloneConfig: rcloneConfig,
|
||||
ConfigData: SyncConfigPayload{
|
||||
APITokens: utils.GetMainConfig().APITokens,
|
||||
Roles: utils.GetMainConfig().Roles,
|
||||
},
|
||||
LastEdited: utils.GetFileLastModifiedTime(dbPath).Unix(),
|
||||
NoRestart: noRestart,
|
||||
}
|
||||
|
||||
// JSON encode the payload
|
||||
payloadBytes, err := json.Marshal(sendPayload)
|
||||
if err != nil {
|
||||
utils.Error("Constellation: MakeSyncPayload: Failed to marshal payload", err)
|
||||
return ""
|
||||
}
|
||||
|
||||
return string(payloadBytes)
|
||||
}
|
||||
|
||||
var lastDBSyncTime time.Time
|
||||
|
||||
func ReceiveSyncPayload(rawPayload string) bool {
|
||||
utils.Log("Constellation: ReceiveSyncPayload: Received sync payload")
|
||||
|
||||
var payload SyncPayload
|
||||
err := json.Unmarshal([]byte(rawPayload), &payload)
|
||||
if err != nil {
|
||||
utils.Error("Constellation: ReceiveSyncPayload: Failed to unmarshal payload " + rawPayload, err)
|
||||
return false
|
||||
}
|
||||
|
||||
// Decode base64 database content
|
||||
dbData, err := base64.StdEncoding.DecodeString(payload.Database)
|
||||
if err != nil {
|
||||
utils.Error("Constellation: ReceiveSyncPayload: Failed to decode database content", err)
|
||||
return false
|
||||
}
|
||||
|
||||
// Write database file
|
||||
dbPath := utils.CONFIGFOLDER + "database"
|
||||
|
||||
// Check if local database has 0 constellation devices - if so, always accept sync
|
||||
skipDateCheck := false
|
||||
c, closeDb, errCo := utils.GetEmbeddedCollection(utils.GetRootAppId(), "devices")
|
||||
if errCo == nil {
|
||||
count, errCount := c.CountDocuments(nil, map[string]interface{}{
|
||||
"Blocked": false,
|
||||
})
|
||||
closeDb()
|
||||
if errCount == nil && count == 0 {
|
||||
utils.Log("Constellation: ReceiveSyncPayload: Local database has 0 devices, skipping date check")
|
||||
skipDateCheck = true
|
||||
}
|
||||
}
|
||||
|
||||
// if local database is newer or same, skip update (unless we have 0 devices)
|
||||
if !skipDateCheck && utils.FileExists(dbPath) && utils.GetFileLastModifiedTime(dbPath).Unix() >= payload.LastEdited {
|
||||
utils.Warn("Constellation: ReceiveSyncPayload: Local database is newer or same as received one, skipping update - local time:" + strconv.FormatInt(utils.GetFileLastModifiedTime(dbPath).Unix(), 10) + " received time:" + strconv.FormatInt(payload.LastEdited, 10))
|
||||
return false
|
||||
}
|
||||
|
||||
err = ioutil.WriteFile(dbPath, dbData, 0600)
|
||||
if err != nil {
|
||||
utils.Error("Constellation: ReceiveSyncPayload: Failed to write database file", err)
|
||||
return false
|
||||
}
|
||||
|
||||
// set the last modified time to the one received
|
||||
err = utils.SetFileLastModifiedTime(dbPath, payload.LastEdited)
|
||||
if err != nil {
|
||||
utils.Error("Constellation: ReceiveSyncPayload: Failed to set database file last modified time", err)
|
||||
return false
|
||||
}
|
||||
|
||||
utils.Warn("Constellation: ReceiveSyncPayload: Database file updated - local time:" + strconv.FormatInt(utils.GetFileLastModifiedTime(dbPath).Unix(), 10) + " received time:" + strconv.FormatInt(payload.LastEdited, 10))
|
||||
|
||||
// set the CA filses if present
|
||||
if payload.CART != "" && payload.CAKey != "" {
|
||||
caCrtData, err := base64.StdEncoding.DecodeString(payload.CART)
|
||||
if err != nil {
|
||||
utils.Error("Constellation: ReceiveSyncPayload: Failed to decode CA crt", err)
|
||||
} else {
|
||||
err = ioutil.WriteFile(utils.CONFIGFOLDER + "ca.crt", caCrtData, 0600)
|
||||
if err != nil {
|
||||
utils.Error("Constellation: ReceiveSyncPayload: Failed to write CA crt file", err)
|
||||
} else {
|
||||
caKeyData, err := base64.StdEncoding.DecodeString(payload.CAKey)
|
||||
if err != nil {
|
||||
utils.Error("Constellation: ReceiveSyncPayload: Failed to decode CA key", err)
|
||||
} else {
|
||||
err = ioutil.WriteFile(utils.CONFIGFOLDER + "ca.key", caKeyData, 0600)
|
||||
if err != nil {
|
||||
utils.Error("Constellation: ReceiveSyncPayload: Failed to write CA key file", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Write rclone config if present
|
||||
needRcloneRemount := false
|
||||
if payload.RcloneConfig != "" {
|
||||
rcloneConfigData, err := base64.StdEncoding.DecodeString(payload.RcloneConfig)
|
||||
if err != nil {
|
||||
utils.Error("Constellation: ReceiveSyncPayload: Failed to decode rclone config", err)
|
||||
} else {
|
||||
err = ioutil.WriteFile(utils.CONFIGFOLDER+"rclone.conf", rcloneConfigData, 0600)
|
||||
if err != nil {
|
||||
utils.Error("Constellation: ReceiveSyncPayload: Failed to write rclone config file", err)
|
||||
} else {
|
||||
needRcloneRemount = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Update auth keys and DNS config
|
||||
config := utils.ReadConfigFromFile()
|
||||
config.HTTPConfig.AuthPrivateKey = payload.AuthPrivateKey
|
||||
config.HTTPConfig.AuthPublicKey = payload.AuthPublicKey
|
||||
config.ConstellationConfig.DNSPort = payload.DNS.DNSPort
|
||||
config.ConstellationConfig.DNSFallback = payload.DNS.DNSFallback
|
||||
config.ConstellationConfig.DNSBlockBlacklist = payload.DNS.DNSBlockBlacklist
|
||||
config.ConstellationConfig.DNSAdditionalBlocklists = payload.DNS.DNSAdditionalBlocklists
|
||||
config.ConstellationConfig.CustomDNSEntries = payload.DNS.CustomDNSEntries
|
||||
|
||||
if payload.ConfigData.APITokens != nil {
|
||||
config.APITokens = payload.ConfigData.APITokens
|
||||
}
|
||||
if payload.ConfigData.Roles != nil {
|
||||
config.Roles = payload.ConfigData.Roles
|
||||
}
|
||||
|
||||
utils.SetBaseMainConfig(config)
|
||||
|
||||
utils.CloseEmbeddedDB()
|
||||
|
||||
if needRcloneRemount && utils.InitRemoteStorage != nil {
|
||||
go utils.InitRemoteStorage()
|
||||
}
|
||||
|
||||
lastDBSyncTime = time.Now()
|
||||
|
||||
utils.TriggerEvent(
|
||||
"cosmos.settings",
|
||||
"Settings updated",
|
||||
"success",
|
||||
"",
|
||||
map[string]interface{}{},
|
||||
)
|
||||
|
||||
if payload.NoRestart {
|
||||
// non-topology changes only (e.g. Tags): rebuild the device cache in place
|
||||
// (async — this runs on the NATS handler goroutine) instead of restarting
|
||||
go refreshDeviceCache()
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func SendRequestSyncMessage() {
|
||||
if !NebulaStarted.Load() {
|
||||
utils.Warn("Constellation: SendRequestSyncMessage: Nebula not started, skipping sync request")
|
||||
return
|
||||
}
|
||||
|
||||
if !lastDBSyncTime.IsZero() && time.Since(lastDBSyncTime) < 10*time.Second {
|
||||
utils.Warn("Constellation: SendRequestSyncMessage: Last DB sync was less than 10s ago, skipping")
|
||||
return
|
||||
}
|
||||
|
||||
utils.Log("Constellation: SendRequestSyncMessage: Requesting sync payload")
|
||||
|
||||
payload := SyncRequestPayload{
|
||||
EditedAt: utils.GetFileLastModifiedTime(utils.CONFIGFOLDER + "database").Unix(),
|
||||
}
|
||||
|
||||
payloadBytes, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
utils.Error("Constellation: SendRequestSyncMessage: Failed to marshal request payload", err)
|
||||
return
|
||||
}
|
||||
|
||||
payloadStr := string(payloadBytes)
|
||||
|
||||
needRestart := false
|
||||
|
||||
SendNATSMessageAllReply("cosmos._global_.constellation.data.sync-request", payloadStr, 2*time.Second, func(response string) {
|
||||
utils.Log("Constellation: SendRequestSyncMessage: Received sync response")
|
||||
needRestart = ReceiveSyncPayload(response) || needRestart
|
||||
})
|
||||
|
||||
if needRestart {
|
||||
go func() {
|
||||
utils.RestartHTTPServer()
|
||||
RestartNebula()
|
||||
}()
|
||||
}
|
||||
}
|
||||
|
||||
func SendNewDBSyncMessage() {
|
||||
sendNewDBSyncMessage(false)
|
||||
}
|
||||
|
||||
// SendNewDBSyncMessageNoRestart broadcasts the same whole-database sync but tells
|
||||
// receivers not to bounce nebula+NATS — for edits that don't touch mesh topology.
|
||||
func SendNewDBSyncMessageNoRestart() {
|
||||
sendNewDBSyncMessage(true)
|
||||
}
|
||||
|
||||
func sendNewDBSyncMessage(noRestart bool) {
|
||||
if !NebulaStarted.Load() {
|
||||
return
|
||||
}
|
||||
|
||||
if IsConstellationStandalone() {
|
||||
return
|
||||
}
|
||||
|
||||
utils.Log("Constellation: SendNewDBSyncMessage: sending sync payload")
|
||||
|
||||
payload := makeSyncPayload("", noRestart)
|
||||
if payload == "" {
|
||||
utils.Warn("Constellation: SendNewDBSyncMessage: No sync payload to send")
|
||||
return
|
||||
}
|
||||
|
||||
err := PublishNATSMessage("cosmos._global_.constellation.data.sync-receive", payload)
|
||||
|
||||
if err != nil {
|
||||
utils.Error("Constellation: SendNewDBSyncMessage: Failed to send request", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func SyncNATSClientRouter(nc *nats.Conn) {
|
||||
nc.Subscribe("cosmos._global_.constellation.data.sync-request", func(m *nats.Msg) {
|
||||
utils.Log("[NATS] Constellation data sync request received")
|
||||
|
||||
payload := m.Data
|
||||
response := MakeSyncPayload((string)(payload))
|
||||
if response != "" {
|
||||
m.Respond([]byte(response))
|
||||
}
|
||||
})
|
||||
|
||||
nc.Subscribe("cosmos._global_.constellation.public-devices", PublicDeviceListNATS)
|
||||
|
||||
nc.Subscribe("cosmos._global_.constellation.data.sync-receive", func(m *nats.Msg) {
|
||||
utils.Log("[NATS] Constellation data sync received")
|
||||
|
||||
payload := m.Data
|
||||
if ReceiveSyncPayload((string)(payload)) {
|
||||
go RestartNebula()
|
||||
}
|
||||
|
||||
m.Respond([]byte("ack"))
|
||||
})
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gopkg.in/yaml.v2"
|
||||
|
||||
@@ -39,11 +40,13 @@ func setupTestEnv(t *testing.T, mutate func(*utils.Config)) string {
|
||||
}
|
||||
utils.LoadBaseMainConfig(cfg)
|
||||
|
||||
utils.CloseEmbeddedDB()
|
||||
if err := utils.InitStore(); err != nil {
|
||||
t.Fatal("testenv: InitStore:", err)
|
||||
}
|
||||
resetConstellationGlobals()
|
||||
|
||||
t.Cleanup(func() {
|
||||
utils.CloseEmbeddedDB()
|
||||
utils.CloseStore()
|
||||
resetConstellationGlobals()
|
||||
utils.CONFIGFOLDER = prevFolder
|
||||
})
|
||||
@@ -61,6 +64,19 @@ func resetConstellationGlobals() {
|
||||
NebulaHasStarted = false
|
||||
NATSStarted.Store(false)
|
||||
|
||||
oplogAttached.Store(false)
|
||||
oplogHalted.Store(false)
|
||||
oplogStreamSeen.Store(false)
|
||||
oplogAttachedEpoch.Store(0)
|
||||
oplogStreamCheckMu.Lock()
|
||||
oplogStreamCheckedAt = time.Time{}
|
||||
oplogStallCount = 0
|
||||
oplogStallLastApplied = 0
|
||||
oplogStreamCheckMu.Unlock()
|
||||
opWaitersMu.Lock()
|
||||
opWaiters = map[string]chan error{}
|
||||
opWaitersMu.Unlock()
|
||||
|
||||
dnsMux.Lock()
|
||||
DNSStarted = false
|
||||
dnsServer = nil
|
||||
@@ -117,3 +133,8 @@ func chdirWithCertBinary(t *testing.T) string {
|
||||
|
||||
return work
|
||||
}
|
||||
|
||||
// readConfigFile reads a loose file out of the test CONFIGFOLDER.
|
||||
func readConfigFile(name string) ([]byte, error) {
|
||||
return os.ReadFile(utils.CONFIGFOLDER + name)
|
||||
}
|
||||
|
||||
+13
-6
@@ -13,6 +13,7 @@ import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"strconv"
|
||||
"runtime"
|
||||
"github.com/azukaar/cosmos-server/src/utils"
|
||||
@@ -47,17 +48,23 @@ func getIdFromName(name string) (string, error) {
|
||||
return "", errors.New("Container not found")
|
||||
}
|
||||
|
||||
var DockerIsConnected = false
|
||||
// race fix: atomic flag + a mutex serializing Connect (concurrent heartbeat goroutines
|
||||
// raced on this flag and on the DockerClient pointer writes below)
|
||||
var DockerIsConnected atomic.Bool
|
||||
var dockerConnectLock sync.Mutex
|
||||
|
||||
func Connect() error {
|
||||
dockerConnectLock.Lock()
|
||||
defer dockerConnectLock.Unlock()
|
||||
|
||||
if DockerClient != nil {
|
||||
// check if connection is still alive
|
||||
ping, err := DockerClient.Ping(DockerContext)
|
||||
if ping.APIVersion != "" && err == nil {
|
||||
DockerIsConnected = true
|
||||
DockerIsConnected.Store(true)
|
||||
return nil
|
||||
} else {
|
||||
DockerIsConnected = false
|
||||
DockerIsConnected.Store(false)
|
||||
DockerClient = nil
|
||||
utils.Error("Docker Connection died, will try to connect again", err)
|
||||
}
|
||||
@@ -66,7 +73,7 @@ func Connect() error {
|
||||
ctx := context.Background()
|
||||
client, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
|
||||
if err != nil {
|
||||
DockerIsConnected = false
|
||||
DockerIsConnected.Store(false)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -75,10 +82,10 @@ func Connect() error {
|
||||
|
||||
ping, err := DockerClient.Ping(DockerContext)
|
||||
if ping.APIVersion != "" && err == nil {
|
||||
DockerIsConnected = true
|
||||
DockerIsConnected.Store(true)
|
||||
utils.Log("Docker Connected")
|
||||
} else {
|
||||
DockerIsConnected = false
|
||||
DockerIsConnected.Store(false)
|
||||
utils.Error("Docker Connection - Cannot ping Daemon. Is it running?", nil)
|
||||
return errors.New("Docker Connection - Cannot ping Daemon. Is it running?")
|
||||
}
|
||||
|
||||
+27
-7
@@ -424,6 +424,26 @@ func CertificateIsExpired(validUntil time.Time) bool {
|
||||
return isValid
|
||||
}
|
||||
|
||||
var appliedDNSChallengeEnv []string
|
||||
|
||||
// env vars are process-wide and survive soft restarts, so drop the ones removed from the config
|
||||
func applyDNSChallengeEnv(newEnv map[string]string) {
|
||||
for _, key := range appliedDNSChallengeEnv {
|
||||
if _, ok := newEnv[key]; !ok {
|
||||
os.Unsetenv(key)
|
||||
}
|
||||
}
|
||||
|
||||
applied := make([]string, 0, len(newEnv))
|
||||
for key, value := range newEnv {
|
||||
key = strings.TrimSpace(key)
|
||||
os.Setenv(key, strings.TrimSpace(value))
|
||||
applied = append(applied, key)
|
||||
}
|
||||
|
||||
appliedDNSChallengeEnv = applied
|
||||
}
|
||||
|
||||
func InitServer() *mux.Router {
|
||||
utils.RestartHTTPServer = RestartHTTPServer
|
||||
baseMainConfig := utils.GetBaseMainConfig()
|
||||
@@ -491,14 +511,13 @@ func InitServer() *mux.Router {
|
||||
config.HTTPConfig.TLSKeyHostsCached[0] == config.HTTPConfig.Hostname &&
|
||||
CertificateIsExpired(baseMainConfig.HTTPConfig.TLSValidUntil)
|
||||
|
||||
if letsEncryptNeedsRefresh && config.HTTPConfig.HTTPSCertificateMode == utils.HTTPSCertModeList["LETSENCRYPT"] {
|
||||
if config.HTTPConfig.DNSChallengeProvider != "" {
|
||||
newEnv := config.HTTPConfig.DNSChallengeConfig
|
||||
for key, value := range newEnv {
|
||||
os.Setenv(key, value)
|
||||
}
|
||||
}
|
||||
dnsChallengeEnv := map[string]string{}
|
||||
if config.HTTPConfig.DNSChallengeProvider != "" {
|
||||
dnsChallengeEnv = config.HTTPConfig.DNSChallengeConfig
|
||||
}
|
||||
applyDNSChallengeEnv(dnsChallengeEnv)
|
||||
|
||||
if letsEncryptNeedsRefresh && config.HTTPConfig.HTTPSCertificateMode == utils.HTTPSCertModeList["LETSENCRYPT"] {
|
||||
// Get Certificates
|
||||
pub, priv := utils.DoLetsEncrypt()
|
||||
|
||||
@@ -691,6 +710,7 @@ func InitServer() *mux.Router {
|
||||
srapiAdmin.HandleFunc("/api/constellation/devices/{id}/ping", constellation.DevicePing)
|
||||
srapiAdmin.HandleFunc("/api/constellation/restart", constellation.API_Restart)
|
||||
srapiAdmin.HandleFunc("/api/constellation/reset", constellation.API_Reset)
|
||||
srapiAdmin.HandleFunc("/api/constellation/force-reform", constellation.API_ForceReform)
|
||||
srapiAdmin.HandleFunc("/api/constellation/connect", constellation.API_ConnectToExisting)
|
||||
srapiAdmin.HandleFunc("/api/constellation/create", constellation.API_NewConstellation)
|
||||
srapiAdmin.HandleFunc("/api/constellation/config", constellation.API_GetConfig)
|
||||
|
||||
+6
-1
@@ -206,7 +206,7 @@ func main() {
|
||||
}
|
||||
|
||||
// @title Cosmos Server API
|
||||
// @version 0.22.35
|
||||
// @version 0.23.00-unstable001
|
||||
// @description REST API for Cosmos Cloud server management
|
||||
// @BasePath /cosmos
|
||||
// @securityDefinitions.apikey BearerAuth
|
||||
@@ -319,10 +319,15 @@ func cosmos() {
|
||||
proxy.InitSocketShield()
|
||||
proxy.InitUDPShield()
|
||||
|
||||
if err := utils.InitStore(); err != nil {
|
||||
utils.Fatal("Cannot open auth.db", err)
|
||||
}
|
||||
|
||||
if !config.NewInstall {
|
||||
MigratePre013()
|
||||
MigratePre014()
|
||||
MigratePre02231()
|
||||
MigratePre02236()
|
||||
|
||||
utils.CheckInternet()
|
||||
|
||||
|
||||
+124
-55
@@ -2,6 +2,7 @@ package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"context"
|
||||
// "fmt"
|
||||
@@ -11,6 +12,8 @@ import (
|
||||
"go.mongodb.org/mongo-driver/mongo/writeconcern"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
|
||||
lungo "github.com/256dpi/lungo"
|
||||
|
||||
"github.com/azukaar/cosmos-server/src/utils"
|
||||
"github.com/azukaar/cosmos-server/src/docker"
|
||||
)
|
||||
@@ -26,60 +29,70 @@ func MigratePre014Coll(collection string, from *mongo.Client) {
|
||||
applicationId := utils.GetRootAppId()
|
||||
|
||||
cf := from.Database(name).Collection(applicationId + "_" + collection)
|
||||
ct, closeDb, err := utils.GetEmbeddedCollection(utils.GetRootAppId(), collection)
|
||||
if err != nil {
|
||||
utils.Error("Error getting collection " + applicationId + "_" + collection + " from database " + name, err)
|
||||
return
|
||||
}
|
||||
defer closeDb()
|
||||
|
||||
// get all documents
|
||||
opts := options.Find()
|
||||
cur, err := cf.Find(context.Background(), bson.D{}, opts)
|
||||
cur, err := cf.Find(context.Background(), bson.D{}, options.Find())
|
||||
if err != nil {
|
||||
utils.Error("Error getting documents from " + collection + " collection", err)
|
||||
return
|
||||
}
|
||||
defer cur.Close(context.Background())
|
||||
|
||||
var batch []interface{}
|
||||
batchSize := 100 // Define a suitable batch size
|
||||
|
||||
for cur.Next(context.Background()) {
|
||||
var elem bson.D
|
||||
if err := cur.Decode(&elem); err != nil {
|
||||
utils.Error("Error decoding document from " + collection + " collection", err)
|
||||
continue
|
||||
}
|
||||
|
||||
batch = append(batch, elem)
|
||||
|
||||
if len(batch) >= batchSize {
|
||||
if _, err := ct.InsertMany(context.Background(), batch); err != nil {
|
||||
utils.Error("Error inserting batch into " + collection + " collection", err)
|
||||
}
|
||||
batch = batch[:0] // Clear the batch
|
||||
}
|
||||
// typed decode + store-layer write
|
||||
switch collection {
|
||||
case "users":
|
||||
users := []utils.User{}
|
||||
if err := cur.All(context.Background(), &users); err != nil {
|
||||
utils.Error("Error decoding documents from " + collection + " collection", err)
|
||||
return
|
||||
}
|
||||
if len(users) == 0 {
|
||||
return
|
||||
}
|
||||
if err := utils.CommitMutation(utils.Mutation{Table: "users", Op: "insertMany", Doc: users}); err != nil {
|
||||
utils.Error("Error inserting documents into " + collection + " collection", err)
|
||||
}
|
||||
case "devices":
|
||||
devices := []utils.ConstellationDevice{}
|
||||
if err := cur.All(context.Background(), &devices); err != nil {
|
||||
utils.Error("Error decoding documents from " + collection + " collection", err)
|
||||
return
|
||||
}
|
||||
if len(devices) == 0 {
|
||||
return
|
||||
}
|
||||
if err := utils.CommitMutation(utils.Mutation{Table: "devices", Op: "insertMany", Doc: devices}); err != nil {
|
||||
utils.Error("Error inserting documents into " + collection + " collection", err)
|
||||
}
|
||||
default:
|
||||
utils.Error("MigratePre014Coll: unknown collection " + collection, nil)
|
||||
}
|
||||
}
|
||||
|
||||
// Insert any remaining documents
|
||||
if len(batch) > 0 {
|
||||
if _, err := ct.InsertMany(context.Background(), batch); err != nil {
|
||||
utils.Error("Error inserting remaining documents into " + collection + " collection", err)
|
||||
}
|
||||
// migratePre014Needed: only if no legacy lungo file and the store is still empty
|
||||
// (auth.db is created by InitStore before migrations, so "auth.db does not exist"
|
||||
// is expressed as "store has no data") and MongoDB is configured.
|
||||
func migratePre014Needed(config utils.Config) bool {
|
||||
if config.MongoDB == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
if err := cur.Err(); err != nil {
|
||||
utils.Error("Error iterating over documents from " + collection + " collection", err)
|
||||
if _, err := os.Stat(utils.CONFIGFOLDER + "database"); err == nil {
|
||||
return false
|
||||
}
|
||||
// a read failure must skip the migration, never trigger a re-import
|
||||
users, errU := utils.CountUsers()
|
||||
devices, errD := utils.CountDevices(map[string]interface{}{})
|
||||
if errU != nil || errD != nil {
|
||||
utils.Error("MigratePre014: cannot inspect auth.db, skipping migration gate", errU)
|
||||
return false
|
||||
}
|
||||
return users == 0 && devices == 0
|
||||
}
|
||||
|
||||
|
||||
func MigratePre014() {
|
||||
config := utils.GetMainConfig()
|
||||
|
||||
// check if COSMOS.db does NOT exist
|
||||
if _, err := os.Stat(utils.CONFIGFOLDER + "database"); err != nil && config.MongoDB != "" {
|
||||
if migratePre014Needed(config) {
|
||||
utils.Log("MigratePre014: Migration of database...")
|
||||
|
||||
// connect to MongoDB
|
||||
@@ -124,25 +137,7 @@ func MigratePre014() {
|
||||
}
|
||||
|
||||
utils.Log("Successfully connected to the database.")
|
||||
|
||||
ct, closeDb, err := utils.GetEmbeddedCollection(utils.GetRootAppId(), "events")
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer closeDb()
|
||||
|
||||
// Create a date index
|
||||
model := mongo.IndexModel{
|
||||
Keys: bson.M{"Date": -1},
|
||||
}
|
||||
|
||||
// Creating the index
|
||||
_, err = ct.Indexes().CreateOne(context.Background(), model)
|
||||
if err != nil {
|
||||
utils.Error("Metrics - Create Index", err)
|
||||
return // Handle error appropriately
|
||||
}
|
||||
|
||||
MigratePre014Coll("users", client)
|
||||
MigratePre014Coll("devices", client)
|
||||
|
||||
@@ -279,3 +274,77 @@ func MigratePre02231() {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MigratePre02236 one-shot imports the legacy lungo embedded database into auth.db.
|
||||
// The rename to database.backup makes it naturally one-shot. Requires InitStore().
|
||||
func MigratePre02236() {
|
||||
dbPath := utils.CONFIGFOLDER + "database"
|
||||
if _, err := os.Stat(dbPath); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
utils.Log("MigratePre02236: importing legacy embedded database into auth.db...")
|
||||
|
||||
opts := lungo.Options{
|
||||
Store: lungo.NewFileStore(dbPath, 0700),
|
||||
}
|
||||
|
||||
client, engine, err := lungo.Open(nil, opts)
|
||||
if err != nil {
|
||||
utils.MajorError("MigratePre02236: cannot open legacy embedded database", err)
|
||||
return
|
||||
}
|
||||
|
||||
name := os.Getenv("MONGODB_NAME")
|
||||
if name == "" {
|
||||
name = "COSMOS"
|
||||
}
|
||||
applicationId := utils.GetRootAppId()
|
||||
|
||||
users := []utils.User{}
|
||||
devices := []utils.ConstellationDevice{}
|
||||
|
||||
curU, err := client.Database(name).Collection(applicationId + "_users").Find(nil, bson.D{})
|
||||
if err == nil {
|
||||
err = curU.All(nil, &users)
|
||||
}
|
||||
if err != nil {
|
||||
utils.MajorError("MigratePre02236: cannot read legacy users", err)
|
||||
engine.Close()
|
||||
return
|
||||
}
|
||||
|
||||
curD, err := client.Database(name).Collection(applicationId + "_devices").Find(nil, bson.D{})
|
||||
if err == nil {
|
||||
err = curD.All(nil, &devices)
|
||||
}
|
||||
if err != nil {
|
||||
utils.MajorError("MigratePre02236: cannot read legacy devices", err)
|
||||
engine.Close()
|
||||
return
|
||||
}
|
||||
|
||||
engine.Close()
|
||||
|
||||
// one tx via the store layer
|
||||
ms := []utils.Mutation{}
|
||||
if len(users) > 0 {
|
||||
ms = append(ms, utils.Mutation{Table: "users", Op: "insertMany", Doc: users})
|
||||
}
|
||||
if len(devices) > 0 {
|
||||
ms = append(ms, utils.Mutation{Table: "devices", Op: "insertMany", Doc: devices})
|
||||
}
|
||||
if len(ms) > 0 {
|
||||
if err := utils.CommitMutations(ms); err != nil {
|
||||
utils.MajorError("MigratePre02236: cannot import legacy data into auth.db", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if err := os.Rename(dbPath, dbPath+".backup"); err != nil {
|
||||
utils.MajorError("MigratePre02236: cannot rename legacy database file", err)
|
||||
return
|
||||
}
|
||||
|
||||
utils.Log("MigratePre02236: imported " + strconv.Itoa(len(users)) + " users and " + strconv.Itoa(len(devices)) + " devices; legacy file renamed to database.backup")
|
||||
}
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
lungo "github.com/256dpi/lungo"
|
||||
|
||||
"github.com/azukaar/cosmos-server/src/utils"
|
||||
)
|
||||
|
||||
func setupMigrateEnv(t *testing.T) string {
|
||||
t.Helper()
|
||||
tmp := t.TempDir() + "/"
|
||||
prev := utils.CONFIGFOLDER
|
||||
utils.CONFIGFOLDER = tmp
|
||||
t.Cleanup(func() {
|
||||
utils.CloseStore()
|
||||
utils.CONFIGFOLDER = prev
|
||||
})
|
||||
return tmp
|
||||
}
|
||||
|
||||
func writeLungoFixture(t *testing.T, path string, users []utils.User, devices []utils.ConstellationDevice) {
|
||||
t.Helper()
|
||||
opts := lungo.Options{Store: lungo.NewFileStore(path, 0700)}
|
||||
client, engine, err := lungo.Open(nil, opts)
|
||||
if err != nil {
|
||||
t.Fatal("lungo open:", err)
|
||||
}
|
||||
appId := utils.GetRootAppId()
|
||||
for _, u := range users {
|
||||
if _, err := client.Database("COSMOS").Collection(appId+"_users").InsertOne(nil, u); err != nil {
|
||||
t.Fatal("lungo insert user:", err)
|
||||
}
|
||||
}
|
||||
for _, d := range devices {
|
||||
if _, err := client.Database("COSMOS").Collection(appId+"_devices").InsertOne(nil, d); err != nil {
|
||||
t.Fatal("lungo insert device:", err)
|
||||
}
|
||||
}
|
||||
engine.Close()
|
||||
}
|
||||
|
||||
func TestMigratePre02236LungoImport(t *testing.T) {
|
||||
tmp := setupMigrateEnv(t)
|
||||
|
||||
created := time.Date(2023, 2, 3, 4, 5, 6, 0, time.UTC)
|
||||
writeLungoFixture(t, tmp+"database",
|
||||
[]utils.User{
|
||||
{Nickname: "alice", Password: "hash1", Role: utils.ADMIN, CreatedAt: created},
|
||||
// zero times stay zero through the import
|
||||
{Nickname: "bob", Password: "", RegisterKey: "rk"},
|
||||
},
|
||||
[]utils.ConstellationDevice{
|
||||
{DeviceName: "node1", IP: "192.168.201.2", APIKey: "apikey1", Fingerprint: "fp1", PublicKey: "pk1", Tags: []string{"gpu"}},
|
||||
// blocked device sharing name+ip with the live one (partial-index case)
|
||||
{DeviceName: "node1", IP: "192.168.201.2", APIKey: "apikey-old", Blocked: true},
|
||||
})
|
||||
|
||||
if err := utils.InitStore(); err != nil {
|
||||
t.Fatal("InitStore:", err)
|
||||
}
|
||||
|
||||
MigratePre02236()
|
||||
|
||||
// legacy file renamed, not deleted
|
||||
if _, err := os.Stat(tmp + "database"); !os.IsNotExist(err) {
|
||||
t.Fatal("legacy database file should be gone")
|
||||
}
|
||||
if _, err := os.Stat(tmp + "database.backup"); err != nil {
|
||||
t.Fatal("database.backup missing:", err)
|
||||
}
|
||||
|
||||
u, err := utils.GetUser("alice")
|
||||
if err != nil || u.Password != "hash1" || u.Role != utils.ADMIN || !u.CreatedAt.Equal(created) {
|
||||
t.Fatalf("alice not imported correctly: %+v, %v", u, err)
|
||||
}
|
||||
b, err := utils.GetUser("bob")
|
||||
if err != nil || b.RegisterKey != "rk" || !b.CreatedAt.IsZero() || !b.LastLogin.IsZero() {
|
||||
t.Fatalf("bob not imported correctly: %+v, %v", b, err)
|
||||
}
|
||||
|
||||
devices, err := utils.ListDevices(true)
|
||||
if err != nil || len(devices) != 2 {
|
||||
t.Fatalf("devices not imported: %v, %v", devices, err)
|
||||
}
|
||||
active, err := utils.GetDeviceByName("node1", true)
|
||||
if err != nil || active.APIKey != "apikey1" || active.Fingerprint != "fp1" || active.PublicKey != "pk1" {
|
||||
t.Fatalf("secrets did not carry over: %+v, %v", active, err)
|
||||
}
|
||||
if len(active.Tags) != 1 || active.Tags[0] != "gpu" {
|
||||
t.Fatalf("tags did not carry over: %+v", active)
|
||||
}
|
||||
|
||||
// second run is a no-op (file renamed away)
|
||||
MigratePre02236()
|
||||
count, _ := utils.CountUsers()
|
||||
if count != 2 {
|
||||
t.Fatalf("second run must be a no-op, users = %d", count)
|
||||
}
|
||||
devices, _ = utils.ListDevices(true)
|
||||
if len(devices) != 2 {
|
||||
t.Fatalf("second run must be a no-op, devices = %d", len(devices))
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigratePre02236NoFile(t *testing.T) {
|
||||
setupMigrateEnv(t)
|
||||
if err := utils.InitStore(); err != nil {
|
||||
t.Fatal("InitStore:", err)
|
||||
}
|
||||
// no legacy file: must be a silent no-op
|
||||
MigratePre02236()
|
||||
if _, err := utils.GetUser("anyone"); !errors.Is(err, utils.ErrNotFound) {
|
||||
t.Fatal("store should stay empty")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigratePre014Gate(t *testing.T) {
|
||||
tmp := setupMigrateEnv(t)
|
||||
if err := utils.InitStore(); err != nil {
|
||||
t.Fatal("InitStore:", err)
|
||||
}
|
||||
|
||||
cfg := utils.Config{}
|
||||
|
||||
// no MongoDB configured -> never
|
||||
cfg.MongoDB = ""
|
||||
if migratePre014Needed(cfg) {
|
||||
t.Fatal("gate must be closed without MongoDB")
|
||||
}
|
||||
|
||||
// MongoDB configured, empty store, no legacy file -> run
|
||||
cfg.MongoDB = "mongodb://example:27017"
|
||||
if !migratePre014Needed(cfg) {
|
||||
t.Fatal("gate must open for Mongo-only installs")
|
||||
}
|
||||
|
||||
// legacy lungo file present -> skip (MigratePre02236 owns it)
|
||||
if err := os.WriteFile(tmp+"database", []byte("x"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if migratePre014Needed(cfg) {
|
||||
t.Fatal("gate must be closed when the lungo file exists")
|
||||
}
|
||||
os.Remove(tmp + "database")
|
||||
|
||||
// store already has data -> skip
|
||||
if err := utils.CreateUser(utils.User{Nickname: "existing"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if migratePre014Needed(cfg) {
|
||||
t.Fatal("gate must be closed once auth.db has data")
|
||||
}
|
||||
}
|
||||
+21
-35
@@ -170,14 +170,6 @@ func NewInstallRoute(w http.ResponseWriter, req *http.Request) {
|
||||
}
|
||||
|
||||
// Admin User
|
||||
c, closeDb, errCo := utils.GetEmbeddedCollection(utils.GetRootAppId(), "users")
|
||||
defer closeDb()
|
||||
if errCo != nil {
|
||||
utils.Error("Database Connect", errCo)
|
||||
utils.HTTPError(w, "Database", http.StatusInternalServerError, "DB001")
|
||||
return
|
||||
}
|
||||
|
||||
nickname := utils.Sanitize(request.Nickname)
|
||||
hashedPassword, err2 := bcrypt.GenerateFromPassword([]byte(request.Password), 14)
|
||||
|
||||
@@ -188,8 +180,9 @@ func NewInstallRoute(w http.ResponseWriter, req *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
// pre-remove every users
|
||||
_, err4 := c.DeleteMany(nil, map[string]interface{}{})
|
||||
// Local wipe, not a cluster op: setup means "this box starts empty".
|
||||
// Published, the empty filter becomes an unqualified DELETE everywhere.
|
||||
err4 := utils.DeleteAllUsersLocal()
|
||||
if err4 != nil {
|
||||
utils.Error("NewInstall: Error deleting users", err4)
|
||||
utils.HTTPError(w, "New Install: Error deleting users " + err4.Error(),
|
||||
@@ -197,14 +190,14 @@ func NewInstallRoute(w http.ResponseWriter, req *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
_, err3 := c.InsertOne(nil, map[string]interface{}{
|
||||
"Nickname": nickname,
|
||||
"Email": request.Email,
|
||||
"Password": hashedPassword,
|
||||
"Role": utils.ADMIN,
|
||||
"PasswordCycle": 0,
|
||||
"CreatedAt": time.Now(),
|
||||
"RegisteredAt": time.Now(),
|
||||
err3 := utils.CreateUser(utils.User{
|
||||
Nickname: nickname,
|
||||
Email: request.Email,
|
||||
Password: string(hashedPassword),
|
||||
Role: utils.ADMIN,
|
||||
PasswordCycle: 0,
|
||||
CreatedAt: time.Now(),
|
||||
RegisteredAt: time.Now(),
|
||||
})
|
||||
|
||||
if err3 != nil {
|
||||
@@ -395,14 +388,6 @@ func SetupRoute(w http.ResponseWriter, req *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
c, closeDb, errCo := utils.GetEmbeddedCollection(utils.GetRootAppId(), "users")
|
||||
defer closeDb()
|
||||
if errCo != nil {
|
||||
utils.Error("Setup: Database Connect", errCo)
|
||||
utils.HTTPError(w, "Database error", http.StatusInternalServerError, "SU006")
|
||||
return
|
||||
}
|
||||
|
||||
nickname := utils.Sanitize(request.Nickname)
|
||||
hashedPassword, err := bcrypt.GenerateFromPassword([]byte(request.Password), 14)
|
||||
if err != nil {
|
||||
@@ -411,15 +396,16 @@ func SetupRoute(w http.ResponseWriter, req *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
c.DeleteMany(nil, map[string]interface{}{})
|
||||
_, err = c.InsertOne(nil, map[string]interface{}{
|
||||
"Nickname": nickname,
|
||||
"Email": request.Email,
|
||||
"Password": hashedPassword,
|
||||
"Role": utils.ADMIN,
|
||||
"PasswordCycle": 0,
|
||||
"CreatedAt": time.Now(),
|
||||
"RegisteredAt": time.Now(),
|
||||
// local wipe — see DeleteAllUsersLocal; setup must never empty the cluster
|
||||
utils.DeleteAllUsersLocal()
|
||||
err = utils.CreateUser(utils.User{
|
||||
Nickname: nickname,
|
||||
Email: request.Email,
|
||||
Password: string(hashedPassword),
|
||||
Role: utils.ADMIN,
|
||||
PasswordCycle: 0,
|
||||
CreatedAt: time.Now(),
|
||||
RegisteredAt: time.Now(),
|
||||
})
|
||||
if err != nil {
|
||||
utils.Error("Setup: Error creating admin user", err)
|
||||
|
||||
@@ -127,9 +127,13 @@ func API_RClone_ConfigCreate(w http.ResponseWriter, req *http.Request) {
|
||||
}
|
||||
|
||||
config.SaveConfig()
|
||||
utils.TouchDatabase()
|
||||
go constellation.SendNewDBSyncMessage()
|
||||
go InitRemoteStorage()
|
||||
|
||||
// rclone.conf is replicated as a file domain; the apply loop writes it here
|
||||
// too and fires the remount, so this handler must not do either itself
|
||||
if err := constellation.PublishFileDomain(constellation.DomainFileRclone, "rclone.conf"); err != nil {
|
||||
utils.HTTPStoreError(w, err, "RCL010")
|
||||
return
|
||||
}
|
||||
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{"status": "OK"})
|
||||
}
|
||||
@@ -177,9 +181,13 @@ func API_RClone_ConfigUpdate(w http.ResponseWriter, req *http.Request) {
|
||||
}
|
||||
|
||||
config.SaveConfig()
|
||||
utils.TouchDatabase()
|
||||
go constellation.SendNewDBSyncMessage()
|
||||
go InitRemoteStorage()
|
||||
|
||||
// rclone.conf is replicated as a file domain; the apply loop writes it here
|
||||
// too and fires the remount, so this handler must not do either itself
|
||||
if err := constellation.PublishFileDomain(constellation.DomainFileRclone, "rclone.conf"); err != nil {
|
||||
utils.HTTPStoreError(w, err, "RCL010")
|
||||
return
|
||||
}
|
||||
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{"status": "OK"})
|
||||
}
|
||||
@@ -213,9 +221,13 @@ func API_RClone_ConfigDelete(w http.ResponseWriter, req *http.Request) {
|
||||
|
||||
config.DeleteRemote(payload.Name)
|
||||
config.SaveConfig()
|
||||
utils.TouchDatabase()
|
||||
go constellation.SendNewDBSyncMessage()
|
||||
go InitRemoteStorage()
|
||||
|
||||
// rclone.conf is replicated as a file domain; the apply loop writes it here
|
||||
// too and fires the remount, so this handler must not do either itself
|
||||
if err := constellation.PublishFileDomain(constellation.DomainFileRclone, "rclone.conf"); err != nil {
|
||||
utils.HTTPStoreError(w, err, "RCL010")
|
||||
return
|
||||
}
|
||||
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{"status": "OK"})
|
||||
}
|
||||
|
||||
+2
-2
@@ -52,7 +52,7 @@ func StatusRoute(w http.ResponseWriter, req *http.Request) {
|
||||
}
|
||||
}
|
||||
|
||||
if(!docker.DockerIsConnected) {
|
||||
if(!docker.DockerIsConnected.Load()) {
|
||||
ed := docker.Connect()
|
||||
if ed != nil {
|
||||
utils.Error("Status: Docker error", ed)
|
||||
@@ -89,7 +89,7 @@ func StatusRoute(w http.ResponseWriter, req *http.Request) {
|
||||
"containerized": utils.IsInsideContainer,
|
||||
"hostmode": utils.IsHostNetwork || !utils.IsInsideContainer || utils.GetMainConfig().DisableHostModeWarning,
|
||||
"database": utils.DBStatus,
|
||||
"docker": docker.DockerIsConnected,
|
||||
"docker": docker.DockerIsConnected.Load(),
|
||||
"backup_status": docker.ExportError,
|
||||
"letsencrypt": utils.GetMainConfig().HTTPConfig.HTTPSCertificateMode == "LETSENCRYPT" && utils.GetMainConfig().HTTPConfig.SSLEmail == "",
|
||||
"domain": utils.GetMainConfig().HTTPConfig.Hostname == "localhost" || utils.GetMainConfig().HTTPConfig.Hostname == "0.0.0.0",
|
||||
|
||||
+3
-20
@@ -39,20 +39,7 @@ func Check2FA(w http.ResponseWriter, req *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
c, closeDb, errCo := utils.GetEmbeddedCollection(utils.GetRootAppId(), "users")
|
||||
defer closeDb()
|
||||
|
||||
if errCo != nil {
|
||||
utils.Error("Database Connect", errCo)
|
||||
utils.HTTPError(w, "Database", http.StatusInternalServerError, "DB001")
|
||||
return
|
||||
}
|
||||
|
||||
userInBase := utils.User{}
|
||||
|
||||
err := c.FindOne(nil, map[string]interface{}{
|
||||
"Nickname": nickname,
|
||||
}).Decode(&userInBase)
|
||||
userInBase, err := utils.GetUser(nickname)
|
||||
|
||||
if err != nil {
|
||||
utils.Error("UserGet: Error while getting user", err)
|
||||
@@ -76,12 +63,8 @@ func Check2FA(w http.ResponseWriter, req *http.Request) {
|
||||
"Was2FAVerified": true,
|
||||
}
|
||||
|
||||
_, err = c.UpdateOne(nil, map[string]interface{}{
|
||||
"Nickname": nickname,
|
||||
}, map[string]interface{}{
|
||||
"$set": toSet,
|
||||
})
|
||||
|
||||
err = utils.UpdateUser(nickname, toSet)
|
||||
|
||||
if err != nil {
|
||||
utils.Error("2FA: Cannot update user", err)
|
||||
utils.HTTPError(w, "2FA Error", http.StatusInternalServerError, "2FA004")
|
||||
|
||||
+2
-18
@@ -46,19 +46,7 @@ func New2FA(w http.ResponseWriter, req *http.Request) {
|
||||
"Was2FAVerified": false,
|
||||
}
|
||||
|
||||
c, closeDb, errCo := utils.GetEmbeddedCollection(utils.GetRootAppId(), "users")
|
||||
defer closeDb()
|
||||
if errCo != nil {
|
||||
utils.Error("Database Connect", errCo)
|
||||
utils.HTTPError(w, "Database", http.StatusInternalServerError, "DB001")
|
||||
return
|
||||
}
|
||||
|
||||
userInBase := utils.User{}
|
||||
|
||||
err = c.FindOne(nil, map[string]interface{}{
|
||||
"Nickname": nickname,
|
||||
}).Decode(&userInBase)
|
||||
userInBase, err := utils.GetUser(nickname)
|
||||
|
||||
if err != nil {
|
||||
utils.Error("UserGet: Error while getting user", err)
|
||||
@@ -72,11 +60,7 @@ func New2FA(w http.ResponseWriter, req *http.Request) {
|
||||
}
|
||||
}
|
||||
|
||||
_, err = c.UpdateOne(nil, map[string]interface{}{
|
||||
"Nickname": nickname,
|
||||
}, map[string]interface{}{
|
||||
"$set": toSet,
|
||||
})
|
||||
err = utils.UpdateUser(nickname, toSet)
|
||||
|
||||
if err != nil {
|
||||
utils.Error("2FA: Cannot update user", err)
|
||||
|
||||
+3
-19
@@ -39,37 +39,21 @@ func Delete2FA(w http.ResponseWriter, req *http.Request) {
|
||||
|
||||
nickname := request.Nickname
|
||||
|
||||
c, closeDb, errCo := utils.GetEmbeddedCollection(utils.GetRootAppId(), "users")
|
||||
defer closeDb()
|
||||
if errCo != nil {
|
||||
utils.Error("Database Connect", errCo)
|
||||
utils.HTTPError(w, "Database", http.StatusInternalServerError, "DB001")
|
||||
return
|
||||
}
|
||||
|
||||
userInBase := utils.User{}
|
||||
|
||||
err := c.FindOne(nil, map[string]interface{}{
|
||||
"Nickname": nickname,
|
||||
}).Decode(&userInBase)
|
||||
userInBase, err := utils.GetUser(nickname)
|
||||
|
||||
if err != nil {
|
||||
utils.Error("UserGet: Error while getting user", err)
|
||||
utils.HTTPError(w, "User Get Error", http.StatusInternalServerError, "2FA002")
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
toSet := map[string]interface{}{
|
||||
"Was2FAVerified": false,
|
||||
"MFAKey": "",
|
||||
"PasswordCycle": userInBase.PasswordCycle + 1,
|
||||
}
|
||||
|
||||
_, err = c.UpdateOne(nil, map[string]interface{}{
|
||||
"Nickname": nickname,
|
||||
}, map[string]interface{}{
|
||||
"$set": toSet,
|
||||
})
|
||||
err = utils.UpdateUser(nickname, toSet)
|
||||
|
||||
if err != nil {
|
||||
utils.Error("UserGet: Error while getting user", err)
|
||||
|
||||
+14
-28
@@ -3,10 +3,10 @@ package user
|
||||
import (
|
||||
"net/http"
|
||||
"encoding/json"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/azukaar/cosmos-server/src/utils"
|
||||
"github.com/azukaar/cosmos-server/src/utils"
|
||||
)
|
||||
|
||||
type CreateRequestJSON struct {
|
||||
@@ -68,17 +68,6 @@ func UserCreate(w http.ResponseWriter, req *http.Request) {
|
||||
role = utils.USER
|
||||
}
|
||||
|
||||
c, closeDb, errCo := utils.GetEmbeddedCollection(utils.GetRootAppId(), "users")
|
||||
|
||||
defer closeDb()
|
||||
if errCo != nil {
|
||||
utils.Error("Database Connect", errCo)
|
||||
utils.HTTPError(w, "Database", http.StatusInternalServerError, "DB001")
|
||||
return
|
||||
}
|
||||
|
||||
user := utils.User{}
|
||||
|
||||
utils.Debug("UserCreation: Creating user " + nickname)
|
||||
|
||||
// count users
|
||||
@@ -95,23 +84,21 @@ func UserCreate(w http.ResponseWriter, req *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
err2 := c.FindOne(nil, map[string]interface{}{
|
||||
"Nickname": nickname,
|
||||
}).Decode(&user)
|
||||
_, err2 := utils.GetUser(nickname)
|
||||
|
||||
if err2 == mongo.ErrNoDocuments {
|
||||
if errors.Is(err2, utils.ErrNotFound) {
|
||||
RegisterKey := utils.GenerateRandomString(24)
|
||||
RegisterKeyExp := time.Now().Add(time.Hour * 24 * 7)
|
||||
|
||||
_, err3 := c.InsertOne(nil, map[string]interface{}{
|
||||
"Nickname": nickname,
|
||||
"Email": email,
|
||||
"Password": "",
|
||||
"RegisterKey": RegisterKey,
|
||||
"RegisterKeyExp": RegisterKeyExp,
|
||||
"Role": role,
|
||||
"PasswordCycle": 0,
|
||||
"CreatedAt": time.Now(),
|
||||
err3 := utils.CreateUser(utils.User{
|
||||
Nickname: nickname,
|
||||
Email: email,
|
||||
Password: "",
|
||||
RegisterKey: RegisterKey,
|
||||
RegisterKeyExp: RegisterKeyExp,
|
||||
Role: role,
|
||||
PasswordCycle: 0,
|
||||
CreatedAt: time.Now(),
|
||||
})
|
||||
|
||||
if err3 != nil {
|
||||
@@ -137,8 +124,7 @@ func UserCreate(w http.ResponseWriter, req *http.Request) {
|
||||
"registerKeyExp": RegisterKeyExp,
|
||||
},
|
||||
})
|
||||
|
||||
go utils.ResyncConstellationNodes()
|
||||
|
||||
} else if err2 == nil {
|
||||
utils.Error("UserCreation: User already exists", nil)
|
||||
utils.HTTPError(w, "User already exists", http.StatusConflict, "UC002")
|
||||
|
||||
+2
-14
@@ -38,20 +38,9 @@ func UserDelete(w http.ResponseWriter, req *http.Request) {
|
||||
|
||||
if(req.Method == "DELETE") {
|
||||
|
||||
c, closeDb, errCo := utils.GetEmbeddedCollection(utils.GetRootAppId(), "users")
|
||||
defer closeDb()
|
||||
|
||||
if errCo != nil {
|
||||
utils.Error("Database Connect", errCo)
|
||||
utils.HTTPError(w, "Database", http.StatusInternalServerError, "DB001")
|
||||
return
|
||||
}
|
||||
|
||||
utils.Debug("UserDeletion: Deleting user " + nickname)
|
||||
|
||||
_, err := c.DeleteOne(nil, map[string]interface{}{
|
||||
"Nickname": nickname,
|
||||
})
|
||||
err := utils.DeleteUser(nickname)
|
||||
|
||||
if err != nil {
|
||||
utils.Error("UserDeletion: Error while deleting user", err)
|
||||
@@ -62,8 +51,7 @@ func UserDelete(w http.ResponseWriter, req *http.Request) {
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"status": "OK",
|
||||
})
|
||||
|
||||
go utils.ResyncConstellationNodes()
|
||||
|
||||
} else {
|
||||
utils.Error("UserDeletion: Method not allowed" + req.Method, nil)
|
||||
utils.HTTPError(w, "Method not allowed", http.StatusMethodNotAllowed, "HTTP001")
|
||||
|
||||
+2
-15
@@ -59,14 +59,6 @@ func UserEdit(w http.ResponseWriter, req *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
c, closeDb, errCo := utils.GetEmbeddedCollection(utils.GetRootAppId(), "users")
|
||||
defer closeDb()
|
||||
if errCo != nil {
|
||||
utils.Error("Database Connect", errCo)
|
||||
utils.HTTPError(w, "Database", http.StatusInternalServerError, "DB001")
|
||||
return
|
||||
}
|
||||
|
||||
utils.Debug("UserEdit: Edit user " + nickname)
|
||||
|
||||
toSet := map[string]interface{}{}
|
||||
@@ -86,11 +78,7 @@ func UserEdit(w http.ResponseWriter, req *http.Request) {
|
||||
toSet["Role"] = *request.Role
|
||||
}
|
||||
|
||||
_, err := c.UpdateOne(nil, map[string]interface{}{
|
||||
"Nickname": nickname,
|
||||
}, map[string]interface{}{
|
||||
"$set": toSet,
|
||||
})
|
||||
err := utils.UpdateUser(nickname, toSet)
|
||||
|
||||
if err != nil {
|
||||
utils.Error("UserEdit: Error while getting user", err)
|
||||
@@ -101,8 +89,7 @@ func UserEdit(w http.ResponseWriter, req *http.Request) {
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"status": "OK",
|
||||
})
|
||||
|
||||
go utils.ResyncConstellationNodes()
|
||||
|
||||
} else {
|
||||
utils.Error("UserEdit: Method not allowed" + req.Method, nil)
|
||||
utils.HTTPError(w, "Method not allowed", http.StatusMethodNotAllowed, "HTTP001")
|
||||
|
||||
+1
-13
@@ -33,21 +33,9 @@ func UserGet(w http.ResponseWriter, req *http.Request) {
|
||||
}
|
||||
|
||||
if(req.Method == "GET") {
|
||||
c, closeDb, errCo := utils.GetEmbeddedCollection(utils.GetRootAppId(), "users")
|
||||
defer closeDb()
|
||||
if errCo != nil {
|
||||
utils.Error("Database Connect", errCo)
|
||||
utils.HTTPError(w, "Database", http.StatusInternalServerError, "DB001")
|
||||
return
|
||||
}
|
||||
|
||||
utils.Debug("UserGet: Get user " + nickname)
|
||||
|
||||
user := utils.User{}
|
||||
|
||||
err := c.FindOne(nil, map[string]interface{}{
|
||||
"Nickname": nickname,
|
||||
}).Decode(&user)
|
||||
user, err := utils.GetUser(nickname)
|
||||
|
||||
if err != nil {
|
||||
utils.Error("UserGet: Error while getting user", err)
|
||||
|
||||
+6
-36
@@ -3,8 +3,7 @@ package user
|
||||
import (
|
||||
"net/http"
|
||||
"encoding/json"
|
||||
"github.com/azukaar/cosmos-server/src/utils"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
"github.com/azukaar/cosmos-server/src/utils"
|
||||
"strconv"
|
||||
"math"
|
||||
)
|
||||
@@ -37,36 +36,13 @@ func UserList(w http.ResponseWriter, req *http.Request) {
|
||||
}
|
||||
|
||||
if(req.Method == "GET") {
|
||||
c, closeDb, errCo := utils.GetEmbeddedCollection(utils.GetRootAppId(), "users")
|
||||
defer closeDb()
|
||||
if errCo != nil {
|
||||
utils.Error("Database Connect", errCo)
|
||||
utils.HTTPError(w, "Database", http.StatusInternalServerError, "DB001")
|
||||
return
|
||||
}
|
||||
|
||||
utils.Debug("UserList: List user ")
|
||||
|
||||
userList := []utils.User{}
|
||||
|
||||
l := int64(math.Max((float64)(maxLimit), (float64)(limit)))
|
||||
|
||||
fOpt := options.FindOptions{
|
||||
Limit: &l,
|
||||
}
|
||||
l := int(math.Max((float64)(maxLimit), (float64)(limit)))
|
||||
|
||||
// TODO: Implement pagination
|
||||
|
||||
cursor, errDB := c.Find(
|
||||
nil,
|
||||
map[string]interface{}{
|
||||
// "_id": map[string]interface{}{
|
||||
// "$gt": from,
|
||||
// },
|
||||
},
|
||||
&fOpt,
|
||||
)
|
||||
defer cursor.Close(nil)
|
||||
users, errDB := utils.ListUsersPage(l)
|
||||
|
||||
if errDB != nil {
|
||||
utils.Error("UserList: Error while getting user", errDB)
|
||||
@@ -74,19 +50,13 @@ func UserList(w http.ResponseWriter, req *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
for cursor.Next(nil) {
|
||||
user := utils.User{}
|
||||
errDec := cursor.Decode(&user)
|
||||
if errDec != nil {
|
||||
utils.Error("UserList: Error while decoding user", errDec)
|
||||
utils.HTTPError(w, "User Get Error", http.StatusInternalServerError, "UL001")
|
||||
return
|
||||
}
|
||||
userList := []utils.User{}
|
||||
for _, user := range users {
|
||||
user.Link = "/api/user/" + user.Nickname
|
||||
userList = append(userList, user)
|
||||
}
|
||||
|
||||
|
||||
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"status": "OK",
|
||||
"data": userList,
|
||||
|
||||
+9
-19
@@ -2,11 +2,11 @@ package user
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"golang.org/x/crypto/bcrypt"
|
||||
|
||||
"github.com/azukaar/cosmos-server/src/utils"
|
||||
@@ -47,26 +47,14 @@ func UserLogin(w http.ResponseWriter, req *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
c, closeDb, errCo := utils.GetEmbeddedCollection(utils.GetRootAppId(), "users")
|
||||
defer closeDb()
|
||||
if errCo != nil {
|
||||
utils.Error("Database Connect", errCo)
|
||||
utils.HTTPError(w, "Database Error", http.StatusInternalServerError, "DB001")
|
||||
return
|
||||
}
|
||||
|
||||
nickname := utils.Sanitize(request.Nickname)
|
||||
password := request.Password
|
||||
|
||||
user := utils.User{}
|
||||
|
||||
utils.Debug("UserLogin: Logging user " + nickname)
|
||||
|
||||
err3 := c.FindOne(nil, map[string]interface{}{
|
||||
"Nickname": nickname,
|
||||
}).Decode(&user)
|
||||
user, err3 := utils.GetUser(nickname)
|
||||
|
||||
if err3 == mongo.ErrNoDocuments {
|
||||
if errors.Is(err3, utils.ErrNotFound) {
|
||||
bcrypt.CompareHashAndPassword([]byte("$2a$14$4nzsVwEnR3.jEbMTME7kqeCo4gMgR/Tuk7ivNExvXjr73nKvLgHka"), []byte("dummyPassword"))
|
||||
utils.Error("UserLogin: User not found", err3)
|
||||
utils.HTTPError(w, "User Logging Error", http.StatusInternalServerError, "UL001")
|
||||
@@ -102,12 +90,14 @@ func UserLogin(w http.ResponseWriter, req *http.Request) {
|
||||
"status": "OK",
|
||||
})
|
||||
|
||||
_, errE := c.UpdateOne(nil, map[string]interface{}{
|
||||
"Nickname": nickname,
|
||||
}, map[string]interface{}{
|
||||
"$set": map[string]interface{}{
|
||||
errE := utils.CommitMutation(utils.Mutation{
|
||||
Table: "users",
|
||||
Op: "update",
|
||||
Filter: map[string]interface{}{"Nickname": nickname},
|
||||
Doc: map[string]interface{}{
|
||||
"LastLogin": time.Now(),
|
||||
},
|
||||
BestEffort: true,
|
||||
})
|
||||
|
||||
if errE != nil {
|
||||
|
||||
+1
-13
@@ -27,19 +27,7 @@ func Me(w http.ResponseWriter, req *http.Request) {
|
||||
|
||||
nickname := utils.GetAuthContext(req).Nickname
|
||||
|
||||
c, closeDb, errCo := utils.GetEmbeddedCollection(utils.GetRootAppId(), "users")
|
||||
defer closeDb()
|
||||
if errCo != nil {
|
||||
utils.Error("Database Connect", errCo)
|
||||
utils.HTTPError(w, "Database", http.StatusInternalServerError, "DB001")
|
||||
return
|
||||
}
|
||||
|
||||
user := utils.User{}
|
||||
|
||||
err := c.FindOne(nil, map[string]interface{}{
|
||||
"Nickname": nickname,
|
||||
}).Decode(&user)
|
||||
user, err := utils.GetUser(nickname)
|
||||
|
||||
if err != nil {
|
||||
utils.Error("UserGet: Error while getting user", err)
|
||||
|
||||
@@ -53,21 +53,13 @@ func ResetPassword(w http.ResponseWriter, req *http.Request) {
|
||||
|
||||
utils.Debug("Sending password reset to: " + nickname)
|
||||
|
||||
c, closeDb, errCo := utils.GetEmbeddedCollection(utils.GetRootAppId(), "users")
|
||||
defer closeDb()
|
||||
if errCo != nil {
|
||||
utils.Error("Database Connect", errCo)
|
||||
utils.HTTPError(w, "Database", http.StatusInternalServerError, "DB001")
|
||||
return
|
||||
user, err := utils.GetUser(nickname)
|
||||
// email mismatch behaves like not-found, matching the old compound filter
|
||||
if err == nil && user.Email != request.Email {
|
||||
user = utils.User{}
|
||||
err = utils.ErrNotFound
|
||||
}
|
||||
|
||||
user := utils.User{}
|
||||
|
||||
err := c.FindOne(nil, map[string]interface{}{
|
||||
"Nickname": nickname,
|
||||
"Email": request.Email,
|
||||
}).Decode(&user)
|
||||
|
||||
if err != nil {
|
||||
utils.Error("PasswordReset: Error while finding user", err)
|
||||
utils.HTTPError(w, "User Send Invite Error", http.StatusInternalServerError, "PR001")
|
||||
@@ -79,13 +71,9 @@ func ResetPassword(w http.ResponseWriter, req *http.Request) {
|
||||
utils.Debug(RegisterKey)
|
||||
utils.Debug(RegisterKeyExp.String())
|
||||
|
||||
_, err := c.UpdateOne(nil, map[string]interface{}{
|
||||
"Nickname": nickname,
|
||||
}, map[string]interface{}{
|
||||
"$set": map[string]interface{}{
|
||||
"RegisterKeyExp": RegisterKeyExp,
|
||||
"RegisterKey": RegisterKey,
|
||||
},
|
||||
err := utils.UpdateUser(nickname, map[string]interface{}{
|
||||
"RegisterKeyExp": RegisterKeyExp,
|
||||
"RegisterKey": RegisterKey,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
@@ -117,8 +105,7 @@ func ResetPassword(w http.ResponseWriter, req *http.Request) {
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"status": "OK",
|
||||
})
|
||||
|
||||
go utils.ResyncConstellationNodes()
|
||||
|
||||
}
|
||||
} else {
|
||||
utils.Error("PasswordReset: Method not allowed" + req.Method, nil)
|
||||
|
||||
+130
-136
@@ -1,137 +1,131 @@
|
||||
package user
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"math/rand"
|
||||
"encoding/json"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"time"
|
||||
"golang.org/x/crypto/bcrypt"
|
||||
|
||||
"github.com/azukaar/cosmos-server/src/utils"
|
||||
)
|
||||
|
||||
type RegisterRequestJSON struct {
|
||||
Nickname string `validate:"required,min=3,max=32,alphanum"`
|
||||
Password string `validate:"required,min=9,max=128,containsany=~!@#$%^&*()_+=-{[}]:;"'<>.?/,containsany=ABCDEFGHIJKLMNOPQRSTUVWXYZ,containsany=abcdefghijklmnopqrstuvwxyz,containsany=0123456789"`
|
||||
RegisterKey string `validate:"required,min=1,max=512,alphanum"`
|
||||
}
|
||||
|
||||
// UserRegister godoc
|
||||
// @Summary Register a new user
|
||||
// @Description Completes user registration using a registration key (invite link)
|
||||
// @Tags auth
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param request body RegisterRequestJSON true "Registration details including register key"
|
||||
// @Success 200 {object} utils.APIResponse
|
||||
// @Failure 401 {object} utils.HTTPErrorResult
|
||||
// @Failure 405 {object} utils.HTTPErrorResult
|
||||
// @Failure 500 {object} utils.HTTPErrorResult
|
||||
// @Router /api/register [post]
|
||||
func UserRegister(w http.ResponseWriter, req *http.Request) {
|
||||
if(req.Method == "POST") {
|
||||
time.Sleep(time.Duration(rand.Float64()*2)*time.Second)
|
||||
|
||||
var request RegisterRequestJSON
|
||||
err1 := json.NewDecoder(req.Body).Decode(&request)
|
||||
if err1 != nil {
|
||||
utils.Error("UserRegister: Invalid User Request", err1)
|
||||
utils.HTTPError(w, "User Register Error", http.StatusInternalServerError, "UR001")
|
||||
return
|
||||
}
|
||||
|
||||
errV := utils.Validate.Struct(request)
|
||||
if errV != nil {
|
||||
utils.Error("UserRegister: Invalid User Request", errV)
|
||||
utils.HTTPError(w, "User Register Error: " + errV.Error(), http.StatusInternalServerError, "UR002")
|
||||
return
|
||||
}
|
||||
|
||||
nickname := utils.Sanitize(request.Nickname)
|
||||
password := request.Password
|
||||
registerKey := request.RegisterKey
|
||||
|
||||
utils.Debug("UserRegister: Registering user " + nickname)
|
||||
|
||||
hashedPassword, err2 := bcrypt.GenerateFromPassword([]byte(password), 14)
|
||||
|
||||
if err2 != nil {
|
||||
utils.Error("UserRegister: Encryption error", err2)
|
||||
utils.HTTPError(w, "User Register Error", http.StatusUnauthorized, "UR001")
|
||||
return
|
||||
}
|
||||
|
||||
c, closeDb, errCo := utils.GetEmbeddedCollection(utils.GetRootAppId(), "users")
|
||||
defer closeDb()
|
||||
if errCo != nil {
|
||||
utils.Error("Database Connect", errCo)
|
||||
utils.HTTPError(w, "Database", http.StatusInternalServerError, "DB001")
|
||||
return
|
||||
}
|
||||
|
||||
user := utils.User{}
|
||||
|
||||
err3 := c.FindOne(nil, map[string]interface{}{
|
||||
"Nickname": nickname,
|
||||
"RegisterKey": registerKey,
|
||||
}).Decode(&user)
|
||||
|
||||
if err3 == mongo.ErrNoDocuments {
|
||||
utils.Error("UserRegister: User not found", err3)
|
||||
utils.HTTPError(w, "User Register Error", http.StatusInternalServerError, "UR001")
|
||||
return
|
||||
} else if err3 != nil {
|
||||
utils.Error("UserRegister: Error while finding user", err3)
|
||||
utils.HTTPError(w, "User Register Error", http.StatusInternalServerError, "UR001")
|
||||
return
|
||||
} else if user.RegisterKeyExp.Before(time.Now()) {
|
||||
utils.Error("UserRegister: Link expired", nil)
|
||||
utils.HTTPError(w, "User Register Error", http.StatusInternalServerError, "UR001")
|
||||
return
|
||||
} else {
|
||||
RegisteredAt := user.RegisteredAt
|
||||
if RegisteredAt.IsZero() {
|
||||
RegisteredAt = time.Now()
|
||||
}
|
||||
_, err4 := c.UpdateOne(nil, map[string]interface{}{
|
||||
"Nickname": nickname,
|
||||
"RegisterKey": registerKey,
|
||||
}, map[string]interface{}{
|
||||
"$set": map[string]interface{}{
|
||||
"Password": hashedPassword,
|
||||
"RegisterKey": "",
|
||||
"RegisterKeyExp": time.Time{},
|
||||
"RegisteredAt": RegisteredAt,
|
||||
"LastPasswordChangedAt": time.Now(),
|
||||
"PasswordCycle": user.PasswordCycle + 1,
|
||||
},
|
||||
})
|
||||
|
||||
if err4 != nil {
|
||||
utils.Error("UserRegister: Error while updating user", err4)
|
||||
utils.HTTPError(w, "User Register Error", http.StatusInternalServerError, "UR001")
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
utils.TriggerEvent(
|
||||
"cosmos.user.register",
|
||||
"User registered",
|
||||
"success",
|
||||
"",
|
||||
map[string]interface{}{
|
||||
"nickname": nickname,
|
||||
})
|
||||
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"status": "OK",
|
||||
})
|
||||
|
||||
go utils.ResyncConstellationNodes()
|
||||
} else {
|
||||
utils.Error("UserRegister: Method not allowed" + req.Method, nil)
|
||||
utils.HTTPError(w, "Method not allowed", http.StatusMethodNotAllowed, "HTTP001")
|
||||
return
|
||||
}
|
||||
package user
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"math/rand"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"time"
|
||||
"golang.org/x/crypto/bcrypt"
|
||||
|
||||
"github.com/azukaar/cosmos-server/src/utils"
|
||||
)
|
||||
|
||||
type RegisterRequestJSON struct {
|
||||
Nickname string `validate:"required,min=3,max=32,alphanum"`
|
||||
Password string `validate:"required,min=9,max=128,containsany=~!@#$%^&*()_+=-{[}]:;"'<>.?/,containsany=ABCDEFGHIJKLMNOPQRSTUVWXYZ,containsany=abcdefghijklmnopqrstuvwxyz,containsany=0123456789"`
|
||||
RegisterKey string `validate:"required,min=1,max=512,alphanum"`
|
||||
}
|
||||
|
||||
// UserRegister godoc
|
||||
// @Summary Register a new user
|
||||
// @Description Completes user registration using a registration key (invite link)
|
||||
// @Tags auth
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param request body RegisterRequestJSON true "Registration details including register key"
|
||||
// @Success 200 {object} utils.APIResponse
|
||||
// @Failure 401 {object} utils.HTTPErrorResult
|
||||
// @Failure 405 {object} utils.HTTPErrorResult
|
||||
// @Failure 500 {object} utils.HTTPErrorResult
|
||||
// @Router /api/register [post]
|
||||
func UserRegister(w http.ResponseWriter, req *http.Request) {
|
||||
if(req.Method == "POST") {
|
||||
time.Sleep(time.Duration(rand.Float64()*2)*time.Second)
|
||||
|
||||
var request RegisterRequestJSON
|
||||
err1 := json.NewDecoder(req.Body).Decode(&request)
|
||||
if err1 != nil {
|
||||
utils.Error("UserRegister: Invalid User Request", err1)
|
||||
utils.HTTPError(w, "User Register Error", http.StatusInternalServerError, "UR001")
|
||||
return
|
||||
}
|
||||
|
||||
errV := utils.Validate.Struct(request)
|
||||
if errV != nil {
|
||||
utils.Error("UserRegister: Invalid User Request", errV)
|
||||
utils.HTTPError(w, "User Register Error: " + errV.Error(), http.StatusInternalServerError, "UR002")
|
||||
return
|
||||
}
|
||||
|
||||
nickname := utils.Sanitize(request.Nickname)
|
||||
password := request.Password
|
||||
registerKey := request.RegisterKey
|
||||
|
||||
utils.Debug("UserRegister: Registering user " + nickname)
|
||||
|
||||
hashedPassword, err2 := bcrypt.GenerateFromPassword([]byte(password), 14)
|
||||
|
||||
if err2 != nil {
|
||||
utils.Error("UserRegister: Encryption error", err2)
|
||||
utils.HTTPError(w, "User Register Error", http.StatusUnauthorized, "UR001")
|
||||
return
|
||||
}
|
||||
|
||||
user, err3 := utils.GetUser(nickname)
|
||||
// key mismatch behaves like not-found, matching the old compound filter
|
||||
if err3 == nil && user.RegisterKey != registerKey {
|
||||
user = utils.User{}
|
||||
err3 = utils.ErrNotFound
|
||||
}
|
||||
|
||||
if errors.Is(err3, utils.ErrNotFound) {
|
||||
utils.Error("UserRegister: User not found", err3)
|
||||
utils.HTTPError(w, "User Register Error", http.StatusInternalServerError, "UR001")
|
||||
return
|
||||
} else if err3 != nil {
|
||||
utils.Error("UserRegister: Error while finding user", err3)
|
||||
utils.HTTPError(w, "User Register Error", http.StatusInternalServerError, "UR001")
|
||||
return
|
||||
} else if user.RegisterKeyExp.Before(time.Now()) {
|
||||
utils.Error("UserRegister: Link expired", nil)
|
||||
utils.HTTPError(w, "User Register Error", http.StatusInternalServerError, "UR001")
|
||||
return
|
||||
} else {
|
||||
RegisteredAt := user.RegisteredAt
|
||||
if RegisteredAt.IsZero() {
|
||||
RegisteredAt = time.Now()
|
||||
}
|
||||
err4 := utils.CommitMutation(utils.Mutation{
|
||||
Table: "users",
|
||||
Op: "update",
|
||||
Filter: map[string]interface{}{
|
||||
"Nickname": nickname,
|
||||
"RegisterKey": registerKey,
|
||||
},
|
||||
Doc: map[string]interface{}{
|
||||
"Password": hashedPassword,
|
||||
"RegisterKey": "",
|
||||
"RegisterKeyExp": time.Time{},
|
||||
"RegisteredAt": RegisteredAt,
|
||||
"LastPasswordChangedAt": time.Now(),
|
||||
"PasswordCycle": user.PasswordCycle + 1,
|
||||
},
|
||||
})
|
||||
|
||||
if err4 != nil {
|
||||
utils.Error("UserRegister: Error while updating user", err4)
|
||||
utils.HTTPError(w, "User Register Error", http.StatusInternalServerError, "UR001")
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
utils.TriggerEvent(
|
||||
"cosmos.user.register",
|
||||
"User registered",
|
||||
"success",
|
||||
"",
|
||||
map[string]interface{}{
|
||||
"nickname": nickname,
|
||||
})
|
||||
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"status": "OK",
|
||||
})
|
||||
|
||||
} else {
|
||||
utils.Error("UserRegister: Method not allowed" + req.Method, nil)
|
||||
utils.HTTPError(w, "Method not allowed", http.StatusMethodNotAllowed, "HTTP001")
|
||||
return
|
||||
}
|
||||
}
|
||||
+7
-23
@@ -3,10 +3,10 @@ package user
|
||||
import (
|
||||
"net/http"
|
||||
"encoding/json"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/azukaar/cosmos-server/src/utils"
|
||||
"github.com/azukaar/cosmos-server/src/utils"
|
||||
)
|
||||
|
||||
type InviteRequestJSON struct {
|
||||
@@ -47,21 +47,9 @@ func UserResendInviteLink(w http.ResponseWriter, req *http.Request) {
|
||||
|
||||
utils.Debug("Re-Sending an invite to " + nickname)
|
||||
|
||||
c, closeDb, errCo := utils.GetEmbeddedCollection(utils.GetRootAppId(), "users")
|
||||
defer closeDb()
|
||||
if errCo != nil {
|
||||
utils.Error("Database Connect", errCo)
|
||||
utils.HTTPError(w, "Database", http.StatusInternalServerError, "DB001")
|
||||
return
|
||||
}
|
||||
user, err := utils.GetUser(nickname)
|
||||
|
||||
user := utils.User{}
|
||||
|
||||
err := c.FindOne(nil, map[string]interface{}{
|
||||
"Nickname": nickname,
|
||||
}).Decode(&user)
|
||||
|
||||
if err == mongo.ErrNoDocuments {
|
||||
if errors.Is(err, utils.ErrNotFound) {
|
||||
utils.Error("UserInvite: User not found", err)
|
||||
utils.HTTPError(w, "User Send Invite Error", http.StatusNotFound, "US001")
|
||||
return
|
||||
@@ -76,13 +64,9 @@ func UserResendInviteLink(w http.ResponseWriter, req *http.Request) {
|
||||
utils.Debug(RegisterKey)
|
||||
utils.Debug(RegisterKeyExp.String())
|
||||
|
||||
_, err := c.UpdateOne(nil, map[string]interface{}{
|
||||
"Nickname": nickname,
|
||||
}, map[string]interface{}{
|
||||
"$set": map[string]interface{}{
|
||||
"RegisterKeyExp": RegisterKeyExp,
|
||||
"RegisterKey": RegisterKey,
|
||||
},
|
||||
err := utils.UpdateUser(nickname, map[string]interface{}{
|
||||
"RegisterKeyExp": RegisterKeyExp,
|
||||
"RegisterKey": RegisterKey,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
|
||||
+3
-15
@@ -2,10 +2,10 @@ package user
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"time"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"golang.org/x/crypto/bcrypt"
|
||||
|
||||
"github.com/azukaar/cosmos-server/src/utils"
|
||||
@@ -53,26 +53,14 @@ func UserSudo(w http.ResponseWriter, req *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
c, closeDb, errCo := utils.GetEmbeddedCollection(utils.GetRootAppId(), "users")
|
||||
defer closeDb()
|
||||
if errCo != nil {
|
||||
utils.Error("Database Connect", errCo)
|
||||
utils.HTTPError(w, "Database Error", http.StatusInternalServerError, "DB001")
|
||||
return
|
||||
}
|
||||
|
||||
nickname := utils.Sanitize(userNickname)
|
||||
password := request.Password
|
||||
|
||||
user := utils.User{}
|
||||
|
||||
utils.Debug("UserSudo: Logging user " + nickname)
|
||||
|
||||
err3 := c.FindOne(nil, map[string]interface{}{
|
||||
"Nickname": nickname,
|
||||
}).Decode(&user)
|
||||
user, err3 := utils.GetUser(nickname)
|
||||
|
||||
if err3 == mongo.ErrNoDocuments {
|
||||
if errors.Is(err3, utils.ErrNotFound) {
|
||||
bcrypt.CompareHashAndPassword([]byte("$2a$14$4nzsVwEnR3.jEbMTME7kqeCo4gMgR/Tuk7ivNExvXjr73nKvLgHka"), []byte("dummyPassword"))
|
||||
utils.Error("UserSudo: User not found", err3)
|
||||
utils.HTTPError(w, "User Logging Error", http.StatusInternalServerError, "UL001")
|
||||
|
||||
+1
-14
@@ -140,20 +140,7 @@ func RefreshUserToken(w http.ResponseWriter, req *http.Request) ([]utils.Permiss
|
||||
return nil, false, utils.User{}, errors.New("JWT Token not valid for this domain")
|
||||
}
|
||||
|
||||
userInBase := utils.User{}
|
||||
|
||||
c, closeDb, errCo := utils.GetEmbeddedCollection(utils.GetRootAppId(), "users")
|
||||
defer closeDb()
|
||||
|
||||
if errCo != nil {
|
||||
utils.Error("Database Connect", errCo)
|
||||
utils.HTTPError(w, "Database", http.StatusInternalServerError, "DB001")
|
||||
return nil, false, utils.User{}, errCo
|
||||
}
|
||||
|
||||
errDB := c.FindOne(nil, map[string]interface{}{
|
||||
"Nickname": nickname,
|
||||
}).Decode(&userInBase)
|
||||
userInBase, errDB := utils.GetUser(nickname)
|
||||
|
||||
if errDB != nil {
|
||||
utils.Error("UserToken: User not found", errDB)
|
||||
|
||||
+1
-101
@@ -12,9 +12,7 @@ import (
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
"go.mongodb.org/mongo-driver/mongo/readpref"
|
||||
"go.mongodb.org/mongo-driver/mongo/writeconcern"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
|
||||
lungo "github.com/256dpi/lungo"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
)
|
||||
|
||||
|
||||
@@ -132,65 +130,6 @@ func DisconnectDB() {
|
||||
client = nil
|
||||
}
|
||||
|
||||
var embeddedClient lungo.IClient
|
||||
var embeddedClientClose func()
|
||||
|
||||
func CloseEmbeddedDB() {
|
||||
if embeddedClientClose != nil {
|
||||
embeddedClientClose()
|
||||
embeddedClientClose = nil
|
||||
}
|
||||
embeddedClient = nil
|
||||
}
|
||||
|
||||
func GetEmbeddedCollection(applicationId string, collection string) (lungo.ICollection, func(), error) {
|
||||
opts := lungo.Options{
|
||||
Store: lungo.NewFileStore(CONFIGFOLDER + "database", 0700),
|
||||
}
|
||||
|
||||
name := os.Getenv("MONGODB_NAME"); if name == "" {
|
||||
name = "COSMOS"
|
||||
}
|
||||
|
||||
if embeddedClient != nil {
|
||||
c := embeddedClient.Database(name).Collection(applicationId + "_" + collection)
|
||||
return c, func() {
|
||||
//engine.Close()
|
||||
}, nil
|
||||
}
|
||||
|
||||
// open database
|
||||
client, engine, err := lungo.Open(nil, opts)
|
||||
if err != nil {
|
||||
return nil, func() {
|
||||
//engine.Close()
|
||||
}, err
|
||||
}
|
||||
|
||||
// ensure engine is closed
|
||||
// defer engine.Close()
|
||||
|
||||
c := client.Database(name).Collection(applicationId + "_" + collection)
|
||||
|
||||
embeddedClient = client
|
||||
embeddedClientClose = engine.Close
|
||||
|
||||
return c, func() {
|
||||
//engine.Close()
|
||||
}, nil
|
||||
}
|
||||
|
||||
// CountUsers returns the number of users registered on this server.
|
||||
func CountUsers() (int64, error) {
|
||||
c, closeDb, errCo := GetEmbeddedCollection(GetRootAppId(), "users")
|
||||
if errCo != nil {
|
||||
return 0, errCo
|
||||
}
|
||||
defer closeDb()
|
||||
|
||||
return c.CountDocuments(nil, map[string]interface{}{})
|
||||
}
|
||||
|
||||
func GetCollection(applicationId string, collection string) (*mongo.Collection, error) {
|
||||
if client == nil {
|
||||
errCo := DB()
|
||||
@@ -316,45 +255,6 @@ func WriteToDatabase(collection *mongo.Collection, objects []map[string]interfac
|
||||
return nil
|
||||
}
|
||||
|
||||
func ListAllUsers(role string) []User {
|
||||
// list all users
|
||||
c, closeDb, errCo := GetEmbeddedCollection(GetRootAppId(), "users")
|
||||
defer closeDb()
|
||||
if errCo != nil {
|
||||
Error("Database Connect", errCo)
|
||||
return []User{}
|
||||
}
|
||||
|
||||
users := []User{}
|
||||
|
||||
condition := map[string]interface{}{}
|
||||
|
||||
if role == "admin" {
|
||||
condition = map[string]interface{}{
|
||||
"Role": 2,
|
||||
}
|
||||
} else if role == "user" {
|
||||
condition = map[string]interface{}{
|
||||
"Role": 1,
|
||||
}
|
||||
}
|
||||
|
||||
cursor, err := c.Find(nil, condition)
|
||||
defer cursor.Close(nil)
|
||||
|
||||
if err != nil {
|
||||
Error("Database: Error while getting users", err)
|
||||
return []User{}
|
||||
}
|
||||
|
||||
if err = cursor.All(nil, &users); err != nil {
|
||||
Error("Database: Error while decoding users", err)
|
||||
return []User{}
|
||||
}
|
||||
|
||||
return users
|
||||
}
|
||||
|
||||
func initDB() {
|
||||
c, errCo := GetCollection(GetRootAppId(), "events")
|
||||
if errCo != nil {
|
||||
|
||||
@@ -193,7 +193,7 @@ func WriteNotification(notification Notification) {
|
||||
// list all users
|
||||
users := ListAllUsers(notification.Recipient)
|
||||
|
||||
Debug("Notifications: Sending notification to " + string(len(users)) + " users")
|
||||
Debug("Notifications: Sending notification to " + fmt.Sprint(len(users)) + " users")
|
||||
|
||||
for _, user := range users {
|
||||
BufferedDBWrite("notifications", map[string]interface{}{
|
||||
|
||||
@@ -0,0 +1,233 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"sort"
|
||||
"strconv"
|
||||
"sync"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
// ErrNotFound is returned by typed reads when no row matches.
|
||||
var ErrNotFound = errors.New("store: not found")
|
||||
|
||||
// ErrConstraint is returned when a unique index rejects a write.
|
||||
type ErrConstraint struct {
|
||||
Table string
|
||||
Index string
|
||||
}
|
||||
|
||||
func (e *ErrConstraint) Error() string {
|
||||
return "store: unique constraint violation on " + e.Table + " (" + e.Index + ")"
|
||||
}
|
||||
|
||||
var (
|
||||
storeMu sync.RWMutex
|
||||
writeDB *sql.DB
|
||||
readDB *sql.DB
|
||||
)
|
||||
|
||||
// InitStore opens auth.db (two handles: single writer, pooled readers) and runs schema migrations.
|
||||
// Deliberately close-then-open: calling it again re-points the store at the current CONFIGFOLDER
|
||||
// (tests and the E2E harness rely on this).
|
||||
func InitStore() error {
|
||||
storeMu.Lock()
|
||||
defer storeMu.Unlock()
|
||||
|
||||
closeStoreLocked()
|
||||
|
||||
dsn := "file:" + CONFIGFOLDER + "auth.db" +
|
||||
"?_pragma=busy_timeout(5000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL)&_pragma=foreign_keys(ON)"
|
||||
|
||||
w, err := sql.Open("sqlite", dsn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
w.SetMaxOpenConns(1)
|
||||
|
||||
if err := initSchema(w); err != nil {
|
||||
w.Close()
|
||||
return err
|
||||
}
|
||||
|
||||
r, err := sql.Open("sqlite", dsn)
|
||||
if err != nil {
|
||||
w.Close()
|
||||
return err
|
||||
}
|
||||
r.SetMaxOpenConns(4)
|
||||
|
||||
writeDB = w
|
||||
readDB = r
|
||||
return nil
|
||||
}
|
||||
|
||||
// CloseStore closes both handles; only for shutdown and tests.
|
||||
func CloseStore() {
|
||||
storeMu.Lock()
|
||||
defer storeMu.Unlock()
|
||||
closeStoreLocked()
|
||||
}
|
||||
|
||||
func closeStoreLocked() {
|
||||
if writeDB != nil {
|
||||
writeDB.Close()
|
||||
writeDB = nil
|
||||
}
|
||||
if readDB != nil {
|
||||
readDB.Close()
|
||||
readDB = nil
|
||||
}
|
||||
}
|
||||
|
||||
func getWriteDB() (*sql.DB, error) {
|
||||
storeMu.RLock()
|
||||
defer storeMu.RUnlock()
|
||||
if writeDB == nil {
|
||||
return nil, errors.New("store: not initialized")
|
||||
}
|
||||
return writeDB, nil
|
||||
}
|
||||
|
||||
func getReadDB() (*sql.DB, error) {
|
||||
storeMu.RLock()
|
||||
defer storeMu.RUnlock()
|
||||
if readDB == nil {
|
||||
return nil, errors.New("store: not initialized")
|
||||
}
|
||||
return readDB, nil
|
||||
}
|
||||
|
||||
func kvGet(q interface {
|
||||
QueryRow(string, ...interface{}) *sql.Row
|
||||
}, key string) (string, error) {
|
||||
var value string
|
||||
err := q.QueryRow("SELECT value FROM kv WHERE key = ?", key).Scan(&value)
|
||||
if err == sql.ErrNoRows {
|
||||
return "", ErrNotFound
|
||||
}
|
||||
return value, err
|
||||
}
|
||||
|
||||
func kvSetTx(tx *sql.Tx, key string, value string) error {
|
||||
_, err := tx.Exec("INSERT INTO kv(key, value) VALUES(?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value", key, value)
|
||||
return err
|
||||
}
|
||||
|
||||
// BuildLogicalDump serializes users+devices as canonical sorted JSON in one read tx.
|
||||
// Body of the op-log snapshot, and the canonical form E2E convergence asserts on.
|
||||
func BuildLogicalDump() ([]byte, error) {
|
||||
db, err := getReadDB()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tx, err := db.Begin()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
return buildLogicalDumpTx(tx)
|
||||
}
|
||||
|
||||
func buildLogicalDumpTx(tx *sql.Tx) ([]byte, error) {
|
||||
users, err := scanUsers(tx, "SELECT "+userCols+" FROM users")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
devices, err := scanDevices(tx, "SELECT "+deviceCols+" FROM devices")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
sort.Slice(users, func(i, j int) bool { return users[i].Nickname < users[j].Nickname })
|
||||
sort.Slice(devices, func(i, j int) bool {
|
||||
if devices[i].DeviceName != devices[j].DeviceName {
|
||||
return devices[i].DeviceName < devices[j].DeviceName
|
||||
}
|
||||
if devices[i].IP != devices[j].IP {
|
||||
return devices[i].IP < devices[j].IP
|
||||
}
|
||||
return !devices[i].Blocked && devices[j].Blocked
|
||||
})
|
||||
|
||||
userMaps := make([]map[string]interface{}, len(users))
|
||||
for i, u := range users {
|
||||
userMaps[i] = dumpUser(u)
|
||||
}
|
||||
deviceMaps := make([]map[string]interface{}, len(devices))
|
||||
for i, d := range devices {
|
||||
deviceMaps[i] = dumpDevice(d)
|
||||
}
|
||||
|
||||
return json.Marshal(map[string]interface{}{
|
||||
"users": userMaps,
|
||||
"devices": deviceMaps,
|
||||
})
|
||||
}
|
||||
|
||||
// ApplyLogicalDump replaces users+devices with the snapshot content and adopts
|
||||
// its (epoch, seq) in the same tx, so a crash mid-install can never leave the
|
||||
// node claiming a log position its data doesn't match.
|
||||
func ApplyLogicalDump(data []byte, epoch uint64, seq uint64) error {
|
||||
var dump struct {
|
||||
Users []map[string]interface{} `json:"users"`
|
||||
Devices []map[string]interface{} `json:"devices"`
|
||||
}
|
||||
if err := json.Unmarshal(data, &dump); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
users := make([]User, len(dump.Users))
|
||||
for i, m := range dump.Users {
|
||||
users[i] = parseDumpUser(m)
|
||||
}
|
||||
devices := make([]ConstellationDevice, len(dump.Devices))
|
||||
for i, m := range dump.Devices {
|
||||
devices[i] = parseDumpDevice(m)
|
||||
}
|
||||
|
||||
db, err := getWriteDB()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tx, err := db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec("DELETE FROM users"); err != nil {
|
||||
tx.Rollback()
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec("DELETE FROM devices"); err != nil {
|
||||
tx.Rollback()
|
||||
return err
|
||||
}
|
||||
for _, u := range users {
|
||||
if err := insertUserTx(tx, u); err != nil {
|
||||
tx.Rollback()
|
||||
return mapSQLError(err)
|
||||
}
|
||||
}
|
||||
for _, d := range devices {
|
||||
if err := insertDeviceTx(tx, d); err != nil {
|
||||
tx.Rollback()
|
||||
return mapSQLError(err)
|
||||
}
|
||||
}
|
||||
if err := setOplogStateTx(tx, epoch, seq); err != nil {
|
||||
tx.Rollback()
|
||||
return err
|
||||
}
|
||||
// Bootstrapped in the SAME tx as the rows: a node that holds snapshot data must
|
||||
// never be able to look un-bootstrapped afterwards, or it re-snapshots forever
|
||||
// (a snapshot from a founder is legitimately stamped seq 0, so the sequence
|
||||
// cannot carry this fact).
|
||||
if err := kvSetTx(tx, kvOplogBootstrapped, strconv.FormatUint(epoch, 10)); err != nil {
|
||||
tx.Rollback()
|
||||
return err
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
@@ -0,0 +1,202 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
)
|
||||
|
||||
const deviceCols = "nickname, device_name, public_key, ip, is_lighthouse, cosmos_node, is_relay, is_load_balancer, is_exit_node, public_hostname, port, blocked, fingerprint, api_key, invisible, tags"
|
||||
|
||||
func insertDeviceTx(tx *sql.Tx, d ConstellationDevice) error {
|
||||
_, err := tx.Exec("INSERT INTO devices ("+deviceCols+") VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
||||
d.Nickname, d.DeviceName, d.PublicKey, d.IP, boolToDB(d.IsLighthouse), d.CosmosNode,
|
||||
boolToDB(d.IsRelay), boolToDB(d.IsLoadBalancer), boolToDB(d.IsExitNode), d.PublicHostname,
|
||||
d.Port, boolToDB(d.Blocked), d.Fingerprint, d.APIKey, boolToDB(d.Invisible), tagsToDB(d.Tags))
|
||||
return err
|
||||
}
|
||||
|
||||
func scanDevices(q rowQuerier, query string, args ...interface{}) ([]ConstellationDevice, error) {
|
||||
rows, err := q.Query(query, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
devices := []ConstellationDevice{}
|
||||
for rows.Next() {
|
||||
var d ConstellationDevice
|
||||
var isLighthouse, isRelay, isLoadBalancer, isExitNode, blocked, invisible int
|
||||
var tags string
|
||||
if err := rows.Scan(&d.Nickname, &d.DeviceName, &d.PublicKey, &d.IP, &isLighthouse, &d.CosmosNode,
|
||||
&isRelay, &isLoadBalancer, &isExitNode, &d.PublicHostname, &d.Port, &blocked,
|
||||
&d.Fingerprint, &d.APIKey, &invisible, &tags); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
d.IsLighthouse = isLighthouse != 0
|
||||
d.IsRelay = isRelay != 0
|
||||
d.IsLoadBalancer = isLoadBalancer != 0
|
||||
d.IsExitNode = isExitNode != 0
|
||||
d.Blocked = blocked != 0
|
||||
d.Invisible = invisible != 0
|
||||
d.Tags = tagsFromDB(tags)
|
||||
devices = append(devices, d)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return devices, nil
|
||||
}
|
||||
|
||||
// GetDeviceByName returns one device by name (active only when mustBeActive);
|
||||
// on any error it returns a zero-value struct, never partial data.
|
||||
func GetDeviceByName(name string, mustBeActive bool) (ConstellationDevice, error) {
|
||||
db, err := getReadDB()
|
||||
if err != nil {
|
||||
return ConstellationDevice{}, err
|
||||
}
|
||||
query := "SELECT " + deviceCols + " FROM devices WHERE device_name = ?"
|
||||
if mustBeActive {
|
||||
query += " AND blocked = 0"
|
||||
}
|
||||
devices, err := scanDevices(db, query+" LIMIT 1", name)
|
||||
if err != nil {
|
||||
return ConstellationDevice{}, err
|
||||
}
|
||||
if len(devices) == 0 {
|
||||
return ConstellationDevice{}, ErrNotFound
|
||||
}
|
||||
return devices[0], nil
|
||||
}
|
||||
|
||||
// GetDeviceByIP returns the active (non-blocked) device holding this IP.
|
||||
func GetDeviceByIP(ip string) (ConstellationDevice, error) {
|
||||
db, err := getReadDB()
|
||||
if err != nil {
|
||||
return ConstellationDevice{}, err
|
||||
}
|
||||
devices, err := scanDevices(db, "SELECT "+deviceCols+" FROM devices WHERE ip = ? AND blocked = 0 LIMIT 1", ip)
|
||||
if err != nil {
|
||||
return ConstellationDevice{}, err
|
||||
}
|
||||
if len(devices) == 0 {
|
||||
return ConstellationDevice{}, ErrNotFound
|
||||
}
|
||||
return devices[0], nil
|
||||
}
|
||||
|
||||
func ListDevices(includeBlocked bool) ([]ConstellationDevice, error) {
|
||||
db, err := getReadDB()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if includeBlocked {
|
||||
return scanDevices(db, "SELECT "+deviceCols+" FROM devices")
|
||||
}
|
||||
return scanDevices(db, "SELECT "+deviceCols+" FROM devices WHERE blocked = 0")
|
||||
}
|
||||
|
||||
// FindDevices returns devices matching a flat equality filter with bson field names.
|
||||
func FindDevices(filter map[string]interface{}) ([]ConstellationDevice, error) {
|
||||
db, err := getReadDB()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
where, args, err := buildWhere("devices", filter)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return scanDevices(db, "SELECT "+deviceCols+" FROM devices"+where, args...)
|
||||
}
|
||||
|
||||
func CountDevices(filter map[string]interface{}) (int64, error) {
|
||||
db, err := getReadDB()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
where, args, err := buildWhere("devices", filter)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var count int64
|
||||
err = db.QueryRow("SELECT COUNT(*) FROM devices"+where, args...).Scan(&count)
|
||||
return count, err
|
||||
}
|
||||
|
||||
func CreateDevice(d ConstellationDevice) error {
|
||||
return CommitMutation(Mutation{Table: "devices", Op: "insert", Doc: d})
|
||||
}
|
||||
|
||||
// UpdateDevices sets the given bson-named fields on all devices matching the filter.
|
||||
func UpdateDevices(filter map[string]interface{}, fields map[string]interface{}) error {
|
||||
return CommitMutation(Mutation{Table: "devices", Op: "updateMany", Filter: filter, Doc: fields})
|
||||
}
|
||||
|
||||
func DeleteDevices(filter map[string]interface{}) error {
|
||||
return CommitMutation(Mutation{Table: "devices", Op: "deleteMany", Filter: filter})
|
||||
}
|
||||
|
||||
// DeleteDevicesLocal removes devices on this node only, never publishing.
|
||||
// Reserved for leave/reset semantics — see CommitMutationLocal for why a
|
||||
// published delete-all is a cluster-wide data loss rather than a local teardown.
|
||||
func DeleteDevicesLocal(filter map[string]interface{}) error {
|
||||
return CommitMutationLocal(Mutation{Table: "devices", Op: "deleteMany", Filter: filter})
|
||||
}
|
||||
|
||||
func dumpDevice(d ConstellationDevice) map[string]interface{} {
|
||||
tags := d.Tags
|
||||
if tags == nil {
|
||||
tags = []string{}
|
||||
}
|
||||
return map[string]interface{}{
|
||||
"Nickname": d.Nickname,
|
||||
"DeviceName": d.DeviceName,
|
||||
"PublicKey": d.PublicKey,
|
||||
"IP": d.IP,
|
||||
"IsLighthouse": d.IsLighthouse,
|
||||
"CosmosNode": d.CosmosNode,
|
||||
"IsRelay": d.IsRelay,
|
||||
"IsLoadBalancer": d.IsLoadBalancer,
|
||||
"IsExitNode": d.IsExitNode,
|
||||
"PublicHostname": d.PublicHostname,
|
||||
"Port": d.Port,
|
||||
"Blocked": d.Blocked,
|
||||
"Fingerprint": d.Fingerprint,
|
||||
"APIKey": d.APIKey,
|
||||
"Invisible": d.Invisible,
|
||||
"Tags": tags,
|
||||
}
|
||||
}
|
||||
|
||||
func dumpTags(m map[string]interface{}, key string) []string {
|
||||
list, ok := m[key].([]interface{})
|
||||
if !ok || len(list) == 0 {
|
||||
return nil
|
||||
}
|
||||
tags := make([]string, 0, len(list))
|
||||
for _, v := range list {
|
||||
if s, ok := v.(string); ok {
|
||||
tags = append(tags, s)
|
||||
}
|
||||
}
|
||||
return tags
|
||||
}
|
||||
|
||||
func parseDumpDevice(m map[string]interface{}) ConstellationDevice {
|
||||
return ConstellationDevice{
|
||||
Nickname: dumpStr(m, "Nickname"),
|
||||
DeviceName: dumpStr(m, "DeviceName"),
|
||||
PublicKey: dumpStr(m, "PublicKey"),
|
||||
IP: dumpStr(m, "IP"),
|
||||
IsLighthouse: dumpBool(m, "IsLighthouse"),
|
||||
CosmosNode: dumpInt(m, "CosmosNode"),
|
||||
IsRelay: dumpBool(m, "IsRelay"),
|
||||
IsLoadBalancer: dumpBool(m, "IsLoadBalancer"),
|
||||
IsExitNode: dumpBool(m, "IsExitNode"),
|
||||
PublicHostname: dumpStr(m, "PublicHostname"),
|
||||
Port: dumpStr(m, "Port"),
|
||||
Blocked: dumpBool(m, "Blocked"),
|
||||
Fingerprint: dumpStr(m, "Fingerprint"),
|
||||
APIKey: dumpStr(m, "APIKey"),
|
||||
Invisible: dumpBool(m, "Invisible"),
|
||||
Tags: dumpTags(m, "Tags"),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,339 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
sqlite "modernc.org/sqlite"
|
||||
sqlite3 "modernc.org/sqlite/lib"
|
||||
)
|
||||
|
||||
// Mutation is the single write descriptor; Filter/Doc keys use bson field names.
|
||||
type Mutation struct {
|
||||
Table string
|
||||
Op string // insert|insertMany|update|updateMany|delete|deleteMany
|
||||
Filter map[string]interface{}
|
||||
Doc interface{}
|
||||
BestEffort bool // LastLogin only
|
||||
}
|
||||
|
||||
// CommitMutation routes one mutation through the op-log when it is wired, and
|
||||
// writes it directly otherwise. The hook itself decides between publishing and
|
||||
// falling back to a direct tx (standalone install, pre-NATS boot).
|
||||
func CommitMutation(m Mutation) error {
|
||||
if hook := getPublishOpHook(); hook != nil {
|
||||
return hook(m)
|
||||
}
|
||||
return CommitMutations([]Mutation{m})
|
||||
}
|
||||
|
||||
// CommitMutations applies several mutations in one direct tx; never published
|
||||
// (migrations and the op-log's own direct fallback are its only callers).
|
||||
func CommitMutations(ms []Mutation) error {
|
||||
db, err := getWriteDB()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tx, err := db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, m := range ms {
|
||||
if err := applyMutation(tx, m); err != nil {
|
||||
tx.Rollback()
|
||||
return mapSQLError(err)
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// applyMutation is the one writer; reused verbatim by the M2 apply loop.
|
||||
func applyMutation(tx *sql.Tx, m Mutation) error {
|
||||
switch m.Op {
|
||||
case "insert":
|
||||
return insertDocTx(tx, m.Table, m.Doc)
|
||||
case "insertMany":
|
||||
return insertManyTx(tx, m.Table, m.Doc)
|
||||
case "update", "updateMany":
|
||||
return updateTx(tx, m)
|
||||
case "delete", "deleteMany":
|
||||
return deleteTx(tx, m)
|
||||
}
|
||||
return errors.New("store: unknown mutation op " + m.Op)
|
||||
}
|
||||
|
||||
func insertDocTx(tx *sql.Tx, table string, doc interface{}) error {
|
||||
switch table {
|
||||
case "users":
|
||||
switch v := doc.(type) {
|
||||
case User:
|
||||
return insertUserTx(tx, v)
|
||||
case *User:
|
||||
return insertUserTx(tx, *v)
|
||||
}
|
||||
case "devices":
|
||||
switch v := doc.(type) {
|
||||
case ConstellationDevice:
|
||||
return insertDeviceTx(tx, v)
|
||||
case *ConstellationDevice:
|
||||
return insertDeviceTx(tx, *v)
|
||||
}
|
||||
}
|
||||
return errors.New("store: unsupported insert doc for table " + table)
|
||||
}
|
||||
|
||||
func insertManyTx(tx *sql.Tx, table string, doc interface{}) error {
|
||||
switch v := doc.(type) {
|
||||
case []User:
|
||||
for _, u := range v {
|
||||
if err := insertUserTx(tx, u); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
case []ConstellationDevice:
|
||||
for _, d := range v {
|
||||
if err := insertDeviceTx(tx, d); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return errors.New("store: unsupported insertMany doc for table " + table)
|
||||
}
|
||||
|
||||
func updateTx(tx *sql.Tx, m Mutation) error {
|
||||
fields, ok := m.Doc.(map[string]interface{})
|
||||
if !ok {
|
||||
return errors.New("store: update doc must be a set-fields map")
|
||||
}
|
||||
if len(fields) == 0 {
|
||||
return nil
|
||||
}
|
||||
cols, err := columnsFor(m.Table)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
keys := sortedKeys(fields)
|
||||
set := make([]string, 0, len(keys))
|
||||
args := make([]interface{}, 0, len(keys))
|
||||
for _, k := range keys {
|
||||
col, ok := cols[k]
|
||||
if !ok {
|
||||
return errors.New("store: unknown field " + k + " for table " + m.Table)
|
||||
}
|
||||
v, err := toDBValue(fields[k])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
set = append(set, col+" = ?")
|
||||
args = append(args, v)
|
||||
}
|
||||
where, whereArgs, err := buildWhere(m.Table, m.Filter)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = tx.Exec("UPDATE "+m.Table+" SET "+strings.Join(set, ", ")+where, append(args, whereArgs...)...)
|
||||
return err
|
||||
}
|
||||
|
||||
func deleteTx(tx *sql.Tx, m Mutation) error {
|
||||
where, args, err := buildWhere(m.Table, m.Filter)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = tx.Exec("DELETE FROM "+m.Table+where, args...)
|
||||
return err
|
||||
}
|
||||
|
||||
func buildWhere(table string, filter map[string]interface{}) (string, []interface{}, error) {
|
||||
if len(filter) == 0 {
|
||||
return "", nil, nil
|
||||
}
|
||||
cols, err := columnsFor(table)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
keys := sortedKeys(filter)
|
||||
clauses := make([]string, 0, len(keys))
|
||||
args := make([]interface{}, 0, len(keys))
|
||||
for _, k := range keys {
|
||||
col, ok := cols[k]
|
||||
if !ok {
|
||||
return "", nil, errors.New("store: unknown filter field " + k + " for table " + table)
|
||||
}
|
||||
v, err := toDBValue(filter[k])
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
clauses = append(clauses, col+" = ?")
|
||||
args = append(args, v)
|
||||
}
|
||||
return " WHERE " + strings.Join(clauses, " AND "), args, nil
|
||||
}
|
||||
|
||||
// bson field name -> column, one map per entity
|
||||
var userColumns = map[string]string{
|
||||
"Nickname": "nickname",
|
||||
"Password": "password",
|
||||
"RegisterKey": "register_key",
|
||||
"RegisterKeyExp": "register_key_exp",
|
||||
"Role": "role",
|
||||
"PasswordCycle": "password_cycle",
|
||||
"Email": "email",
|
||||
"RegisteredAt": "registered_at",
|
||||
"LastPasswordChangedAt": "last_password_changed_at",
|
||||
"CreatedAt": "created_at",
|
||||
"LastLogin": "last_login",
|
||||
"MFAKey": "mfa_key",
|
||||
"Was2FAVerified": "was_2fa_verified",
|
||||
}
|
||||
|
||||
var deviceColumns = map[string]string{
|
||||
"Nickname": "nickname",
|
||||
"DeviceName": "device_name",
|
||||
"PublicKey": "public_key",
|
||||
"IP": "ip",
|
||||
"IsLighthouse": "is_lighthouse",
|
||||
"CosmosNode": "cosmos_node",
|
||||
"IsRelay": "is_relay",
|
||||
"IsLoadBalancer": "is_load_balancer",
|
||||
"IsExitNode": "is_exit_node",
|
||||
"PublicHostname": "public_hostname",
|
||||
"Port": "port",
|
||||
"Blocked": "blocked",
|
||||
"Fingerprint": "fingerprint",
|
||||
"APIKey": "api_key",
|
||||
"Invisible": "invisible",
|
||||
"Tags": "tags",
|
||||
}
|
||||
|
||||
func columnsFor(table string) (map[string]string, error) {
|
||||
switch table {
|
||||
case "users":
|
||||
return userColumns, nil
|
||||
case "devices":
|
||||
return deviceColumns, nil
|
||||
}
|
||||
return nil, errors.New("store: unknown table " + table)
|
||||
}
|
||||
|
||||
func sortedKeys(m map[string]interface{}) []string {
|
||||
keys := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
return keys
|
||||
}
|
||||
|
||||
func toDBValue(v interface{}) (interface{}, error) {
|
||||
switch t := v.(type) {
|
||||
case nil:
|
||||
// fail loudly: '' in an INTEGER column would corrupt the row silently
|
||||
return nil, errors.New("store: nil value in mutation")
|
||||
case string:
|
||||
return t, nil
|
||||
case []byte:
|
||||
return string(t), nil
|
||||
case bool:
|
||||
return boolToDB(t), nil
|
||||
case int:
|
||||
return t, nil
|
||||
case int64:
|
||||
return t, nil
|
||||
case float64:
|
||||
return int64(t), nil
|
||||
case Role:
|
||||
return int(t), nil
|
||||
case time.Time:
|
||||
return timeToDB(t), nil
|
||||
case []string:
|
||||
return tagsToDB(t), nil
|
||||
}
|
||||
return nil, errors.New("store: unsupported value type in mutation")
|
||||
}
|
||||
|
||||
func boolToDB(b bool) int {
|
||||
if b {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// time columns: RFC3339Nano UTC TEXT, '' = zero
|
||||
func timeToDB(t time.Time) string {
|
||||
if t.IsZero() {
|
||||
return ""
|
||||
}
|
||||
return t.UTC().Format(time.RFC3339Nano)
|
||||
}
|
||||
|
||||
func timeFromDB(s string) time.Time {
|
||||
if s == "" {
|
||||
return time.Time{}
|
||||
}
|
||||
t, err := time.Parse(time.RFC3339Nano, s)
|
||||
if err != nil {
|
||||
return time.Time{}
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
func tagsToDB(tags []string) string {
|
||||
if len(tags) == 0 {
|
||||
return ""
|
||||
}
|
||||
data, err := json.Marshal(tags)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return string(data)
|
||||
}
|
||||
|
||||
func tagsFromDB(s string) []string {
|
||||
if s == "" {
|
||||
return nil
|
||||
}
|
||||
var tags []string
|
||||
if err := json.Unmarshal([]byte(s), &tags); err != nil {
|
||||
return nil
|
||||
}
|
||||
return tags
|
||||
}
|
||||
|
||||
// mapSQLError converts a uniqueness rejection into *ErrConstraint; the message
|
||||
// parse only fills in Table/Index ("devices.device_name"). PRIMARY KEY counts:
|
||||
// users are keyed by nickname, so a duplicate user arrives as a PK violation and
|
||||
// must be a rejection like any other, not an error that halts the apply loop.
|
||||
func mapSQLError(err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
var serr *sqlite.Error
|
||||
if !errors.As(err, &serr) {
|
||||
return err
|
||||
}
|
||||
if serr.Code() != sqlite3.SQLITE_CONSTRAINT_UNIQUE && serr.Code() != sqlite3.SQLITE_CONSTRAINT_PRIMARYKEY {
|
||||
return err
|
||||
}
|
||||
ec := &ErrConstraint{}
|
||||
msg := err.Error()
|
||||
marker := "UNIQUE constraint failed: "
|
||||
if idx := strings.Index(msg, marker); idx >= 0 {
|
||||
target := msg[idx+len(marker):]
|
||||
if end := strings.IndexAny(target, " ,("); end > 0 {
|
||||
target = target[:end]
|
||||
}
|
||||
ec.Table = target
|
||||
if parts := strings.SplitN(target, ".", 2); len(parts) == 2 {
|
||||
ec.Table = parts[0]
|
||||
ec.Index = parts[1]
|
||||
}
|
||||
}
|
||||
return ec
|
||||
}
|
||||
@@ -0,0 +1,358 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// ErrReadOnly: the op-log publish failed, so this node cannot write config.
|
||||
var ErrReadOnly = errors.New("store: config is read-only, no writable op-log")
|
||||
|
||||
// ErrApplyTimeout: the op was published but did not apply locally in time.
|
||||
var ErrApplyTimeout = errors.New("store: timed out waiting for the op to apply")
|
||||
|
||||
const (
|
||||
kvOplogEpoch = "oplog_epoch"
|
||||
kvLastAppliedSeq = "last_applied_seq"
|
||||
kvOplogBootstrapped = "oplog_bootstrapped"
|
||||
kvFormationWriter = "formation_writer"
|
||||
)
|
||||
|
||||
// publishOpHook is the op-log write path; when nil, CommitMutation writes directly.
|
||||
// Guarded because it is set from constellation Init while HTTP handlers read it.
|
||||
var publishOpMu sync.RWMutex
|
||||
var publishOpHook func(Mutation) error
|
||||
|
||||
// SetPublishOpHook wires the op-log publisher; nil restores direct writes.
|
||||
func SetPublishOpHook(h func(Mutation) error) {
|
||||
publishOpMu.Lock()
|
||||
defer publishOpMu.Unlock()
|
||||
publishOpHook = h
|
||||
}
|
||||
|
||||
func getPublishOpHook() func(Mutation) error {
|
||||
publishOpMu.RLock()
|
||||
defer publishOpMu.RUnlock()
|
||||
return publishOpHook
|
||||
}
|
||||
|
||||
type rowScanner interface {
|
||||
QueryRow(string, ...interface{}) *sql.Row
|
||||
}
|
||||
|
||||
func kvUint(q rowScanner, key string, def uint64) uint64 {
|
||||
value, err := kvGet(q, key)
|
||||
if err != nil {
|
||||
return def
|
||||
}
|
||||
n, err := strconv.ParseUint(value, 10, 64)
|
||||
if err != nil {
|
||||
return def
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// GetOplogEpoch returns the current op-log epoch (1 until a reform bumps it).
|
||||
func GetOplogEpoch() uint64 {
|
||||
db, err := getReadDB()
|
||||
if err != nil {
|
||||
return 1
|
||||
}
|
||||
return kvUint(db, kvOplogEpoch, 1)
|
||||
}
|
||||
|
||||
// GetLastAppliedSeq returns the last op-log sequence committed to this node.
|
||||
func GetLastAppliedSeq() uint64 {
|
||||
db, err := getReadDB()
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return kvUint(db, kvLastAppliedSeq, 0)
|
||||
}
|
||||
|
||||
// IsOplogBootstrapped reports whether this node holds state it materialized from
|
||||
// the log for the CURRENT epoch — either by installing a snapshot or by seeding
|
||||
// as founder.
|
||||
//
|
||||
// This exists because `last_applied_seq > 0` cannot answer the question. A
|
||||
// founder that seeded its own store, and every peer that snapshots from it,
|
||||
// legitimately sit at seq 0 with real rows; seq 0 is therefore not evidence of a
|
||||
// fresh node. Using it as such both re-triggered the always-snapshot rule forever
|
||||
// (the joiner never attached) and let an enrolled node fall through to a direct
|
||||
// write during a bounce (the write forked, never published).
|
||||
//
|
||||
// The marker stores the epoch it was set for, so a reform that bumps the epoch
|
||||
// invalidates it automatically — there is no separate clear path to forget.
|
||||
func IsOplogBootstrapped() bool {
|
||||
db, err := getReadDB()
|
||||
if err != nil {
|
||||
// fail closed: unknown state must not read as "fresh node, safe to write"
|
||||
return true
|
||||
}
|
||||
return kvUint(db, kvOplogBootstrapped, 0) == kvUint(db, kvOplogEpoch, 1)
|
||||
}
|
||||
|
||||
// MarkOplogBootstrapped records that this node holds valid state for the epoch.
|
||||
// Snapshot installs set it inside their own tx; this is for the founder-seed path.
|
||||
func MarkOplogBootstrapped(epoch uint64) error {
|
||||
return kvSetOne(kvOplogBootstrapped, strconv.FormatUint(epoch, 10))
|
||||
}
|
||||
|
||||
// IsFormationWriter reports whether this node is the constellation's single
|
||||
// writer during formation — the creator, or the survivor of a force-reform.
|
||||
//
|
||||
// Epoch-tied for the same reason as the bootstrapped marker: the flag stores the
|
||||
// epoch it was granted for, so a reform that bumps the epoch invalidates any
|
||||
// older grant automatically and no node can carry a formation licence forward
|
||||
// into a log it did not seed.
|
||||
func IsFormationWriter() bool {
|
||||
db, err := getReadDB()
|
||||
if err != nil {
|
||||
// fail closed: an unreadable store must not read as "licensed to write"
|
||||
return false
|
||||
}
|
||||
return kvUint(db, kvFormationWriter, 0) == kvUint(db, kvOplogEpoch, 1)
|
||||
}
|
||||
|
||||
// SetFormationWriter grants the formation write licence for an epoch.
|
||||
func SetFormationWriter(epoch uint64) error {
|
||||
return kvSetOne(kvFormationWriter, strconv.FormatUint(epoch, 10))
|
||||
}
|
||||
|
||||
// ClearFormationWriter ends formation: from here this node publishes like any other.
|
||||
func ClearFormationWriter() error {
|
||||
return kvSetOne(kvFormationWriter, "0")
|
||||
}
|
||||
|
||||
// ReformOplogEpoch re-enters formation on this node in ONE tx: the epoch moves
|
||||
// past anything the old cluster can publish (its subjects match no stream), the
|
||||
// sequence restarts, the bootstrapped marker is dropped so peers must
|
||||
// re-materialize, and this node takes the formation write licence.
|
||||
func ReformOplogEpoch() (uint64, error) {
|
||||
db, err := getWriteDB()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
tx, err := db.Begin()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
epoch := kvUint(tx, kvOplogEpoch, 1) + 1
|
||||
if err := setOplogStateTx(tx, epoch, 0); err != nil {
|
||||
tx.Rollback()
|
||||
return 0, err
|
||||
}
|
||||
if err := kvSetTx(tx, kvOplogBootstrapped, "0"); err != nil {
|
||||
tx.Rollback()
|
||||
return 0, err
|
||||
}
|
||||
if err := kvSetTx(tx, kvFormationWriter, strconv.FormatUint(epoch, 10)); err != nil {
|
||||
tx.Rollback()
|
||||
return 0, err
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return epoch, nil
|
||||
}
|
||||
|
||||
func kvSetOne(key string, value string) error {
|
||||
db, err := getWriteDB()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tx, err := db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := kvSetTx(tx, key, value); err != nil {
|
||||
tx.Rollback()
|
||||
return err
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// SetOplogState overwrites (epoch, seq) and CLEARS the bootstrapped marker; used
|
||||
// by reform, which deliberately makes the node re-materialize at the new epoch.
|
||||
func SetOplogState(epoch uint64, seq uint64) error {
|
||||
db, err := getWriteDB()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tx, err := db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := setOplogStateTx(tx, epoch, seq); err != nil {
|
||||
tx.Rollback()
|
||||
return err
|
||||
}
|
||||
// reform: the node must re-materialize at the new epoch before it may write
|
||||
if err := kvSetTx(tx, kvOplogBootstrapped, "0"); err != nil {
|
||||
tx.Rollback()
|
||||
return err
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
func setOplogStateTx(tx *sql.Tx, epoch uint64, seq uint64) error {
|
||||
if err := kvSetTx(tx, kvOplogEpoch, strconv.FormatUint(epoch, 10)); err != nil {
|
||||
return err
|
||||
}
|
||||
return kvSetTx(tx, kvLastAppliedSeq, strconv.FormatUint(seq, 10))
|
||||
}
|
||||
|
||||
// ApplyOpTx applies one op-log mutation and commits its sequence in the SAME tx —
|
||||
// that atomicity, not JetStream ack state, is what makes apply exactly-once.
|
||||
// preDevices, when non-nil, receives the matching device rows read inside the tx
|
||||
// (pre-image the apply loop diffs to pick a reaction).
|
||||
func ApplyOpTx(m Mutation, seq uint64, preDevices *[]ConstellationDevice) error {
|
||||
return applyOpTx(m, seq, true, preDevices)
|
||||
}
|
||||
|
||||
// CommitMutationDirect writes one mutation straight to SQLite, still capturing
|
||||
// the pre-image so the direct write path fires the same reactions as the loop.
|
||||
func CommitMutationDirect(m Mutation, preDevices *[]ConstellationDevice) error {
|
||||
return applyOpTx(m, 0, false, preDevices)
|
||||
}
|
||||
|
||||
// CommitMutationLocal writes to this node ONLY, bypassing the op-log entirely and
|
||||
// unconditionally — it never consults the write-mode ladder.
|
||||
//
|
||||
// RESERVED for leave/teardown semantics, where the intent is inherently local:
|
||||
// "delete my devices because I am leaving the constellation" must never become a
|
||||
// cluster op. Routed through CommitMutation instead, a reset's delete-all carries
|
||||
// an empty filter, which compiles to an unqualified DELETE and wipes the device
|
||||
// registry on every node that applies it.
|
||||
//
|
||||
// Do NOT use for ordinary writes — they must go through CommitMutation so the
|
||||
// cluster stays a single materialization of one log.
|
||||
func CommitMutationLocal(m Mutation) error {
|
||||
return applyOpTx(m, 0, false, nil)
|
||||
}
|
||||
|
||||
func applyOpTx(m Mutation, seq uint64, commitSeq bool, preDevices *[]ConstellationDevice) error {
|
||||
db, err := getWriteDB()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tx, err := db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if preDevices != nil && m.Table == "devices" {
|
||||
where, args, errW := buildWhere("devices", m.Filter)
|
||||
if errW != nil {
|
||||
tx.Rollback()
|
||||
return errW
|
||||
}
|
||||
rows, errQ := scanDevices(tx, "SELECT "+deviceCols+" FROM devices"+where, args...)
|
||||
if errQ != nil {
|
||||
tx.Rollback()
|
||||
return errQ
|
||||
}
|
||||
*preDevices = rows
|
||||
}
|
||||
|
||||
if err := applyMutation(tx, m); err != nil {
|
||||
tx.Rollback()
|
||||
return mapSQLError(err)
|
||||
}
|
||||
if commitSeq {
|
||||
if err := kvSetTx(tx, kvLastAppliedSeq, strconv.FormatUint(seq, 10)); err != nil {
|
||||
tx.Rollback()
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// CommitOplogSeq advances last_applied_seq alone — a rejected op still consumes
|
||||
// its sequence so every replica stays at the same position.
|
||||
func CommitOplogSeq(seq uint64) error {
|
||||
db, err := getWriteDB()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tx, err := db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := kvSetTx(tx, kvLastAppliedSeq, strconv.FormatUint(seq, 10)); err != nil {
|
||||
tx.Rollback()
|
||||
return err
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// BuildSnapshot returns the users+devices dump plus the (epoch, seq) it is
|
||||
// consistent with, all read in one tx.
|
||||
func BuildSnapshot() ([]byte, uint64, uint64, error) {
|
||||
db, err := getReadDB()
|
||||
if err != nil {
|
||||
return nil, 0, 0, err
|
||||
}
|
||||
tx, err := db.Begin()
|
||||
if err != nil {
|
||||
return nil, 0, 0, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
dump, err := buildLogicalDumpTx(tx)
|
||||
if err != nil {
|
||||
return nil, 0, 0, err
|
||||
}
|
||||
return dump, kvUint(tx, kvOplogEpoch, 1), kvUint(tx, kvLastAppliedSeq, 0), nil
|
||||
}
|
||||
|
||||
// dbValueKey renders a value in its DB form as a string, so a wire value
|
||||
// (JSON numbers, DB-normalized strings) compares equal to the same value read
|
||||
// back out of a row.
|
||||
func dbValueKey(v interface{}) string {
|
||||
dv, err := toDBValue(v)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprint(dv)
|
||||
}
|
||||
|
||||
// DeviceFieldsChanged reports whether an update actually moves any of the named
|
||||
// fields away from what the matched rows already hold. Lets the apply loop tell
|
||||
// a real topology move from an edit that merely resubmits the same values.
|
||||
func DeviceFieldsChanged(pre []ConstellationDevice, fields map[string]interface{}, names map[string]bool) bool {
|
||||
for k, v := range fields {
|
||||
if !names[k] {
|
||||
continue
|
||||
}
|
||||
want := dbValueKey(v)
|
||||
for _, d := range pre {
|
||||
if dbValueKey(dumpDevice(d)[k]) != want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// HTTPStoreError maps a store/op-log failure onto the status the client expects:
|
||||
// read-only and duplicate-value both surface as 409, a lost apply as 503.
|
||||
func HTTPStoreError(w http.ResponseWriter, err error, userCode string) {
|
||||
var ec *ErrConstraint
|
||||
switch {
|
||||
case errors.Is(err, ErrReadOnly):
|
||||
HTTPError(w, "Configuration is read-only: no writable op-log (no quorum)", http.StatusConflict, userCode)
|
||||
case errors.Is(err, ErrApplyTimeout):
|
||||
HTTPError(w, "Timed out waiting for the change to be applied", http.StatusServiceUnavailable, userCode)
|
||||
case errors.As(err, &ec):
|
||||
HTTPError(w, "Duplicate value rejected by "+ec.Table+"."+ec.Index+": already in use", http.StatusConflict, userCode)
|
||||
case errors.Is(err, ErrNotFound):
|
||||
HTTPError(w, "Not found", http.StatusNotFound, userCode)
|
||||
default:
|
||||
HTTPError(w, err.Error(), http.StatusInternalServerError, userCode)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
)
|
||||
|
||||
// The op-log wire form deliberately does NOT reuse the structs' json tags:
|
||||
// User.Password, ConstellationDevice.APIKey and friends are `json:"-"` and would
|
||||
// be dropped. Docs travel as the bson-named dump maps instead, and set-field maps
|
||||
// travel pre-converted to their DB representation so every replica writes bytes
|
||||
// identical to the originating node's.
|
||||
|
||||
// EncodeOpDoc renders a mutation's Doc into its canonical wire form.
|
||||
func EncodeOpDoc(m Mutation) (json.RawMessage, error) {
|
||||
switch m.Op {
|
||||
case "insert":
|
||||
doc, err := dumpDoc(m.Table, m.Doc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return json.Marshal(doc)
|
||||
case "insertMany":
|
||||
docs, err := dumpDocs(m.Table, m.Doc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return json.Marshal(docs)
|
||||
case "update", "updateMany":
|
||||
fields, ok := m.Doc.(map[string]interface{})
|
||||
if !ok {
|
||||
return nil, errors.New("store: update doc must be a set-fields map")
|
||||
}
|
||||
norm, err := NormalizeOpFields(m.Table, fields)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return json.Marshal(norm)
|
||||
case "delete", "deleteMany":
|
||||
return nil, nil
|
||||
}
|
||||
return nil, errors.New("store: unknown mutation op " + m.Op)
|
||||
}
|
||||
|
||||
// DecodeOpDoc rebuilds a Doc from the wire form for this table and op.
|
||||
func DecodeOpDoc(table string, op string, raw json.RawMessage) (interface{}, error) {
|
||||
switch op {
|
||||
case "insert":
|
||||
var m map[string]interface{}
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch table {
|
||||
case "users":
|
||||
return parseDumpUser(m), nil
|
||||
case "devices":
|
||||
return parseDumpDevice(m), nil
|
||||
}
|
||||
case "insertMany":
|
||||
var list []map[string]interface{}
|
||||
if err := json.Unmarshal(raw, &list); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch table {
|
||||
case "users":
|
||||
users := make([]User, len(list))
|
||||
for i, m := range list {
|
||||
users[i] = parseDumpUser(m)
|
||||
}
|
||||
return users, nil
|
||||
case "devices":
|
||||
devices := make([]ConstellationDevice, len(list))
|
||||
for i, m := range list {
|
||||
devices[i] = parseDumpDevice(m)
|
||||
}
|
||||
return devices, nil
|
||||
}
|
||||
case "update", "updateMany":
|
||||
var m map[string]interface{}
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return m, nil
|
||||
case "delete", "deleteMany":
|
||||
return nil, nil
|
||||
default:
|
||||
return nil, errors.New("store: unknown mutation op " + op)
|
||||
}
|
||||
return nil, errors.New("store: unsupported doc for table " + table)
|
||||
}
|
||||
|
||||
// NormalizeOpFields converts a set-fields map to its DB representation, rejecting
|
||||
// unknown field names before they ever reach the log.
|
||||
func NormalizeOpFields(table string, fields map[string]interface{}) (map[string]interface{}, error) {
|
||||
cols, err := columnsFor(table)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make(map[string]interface{}, len(fields))
|
||||
for k, v := range fields {
|
||||
if _, ok := cols[k]; !ok {
|
||||
return nil, errors.New("store: unknown field " + k + " for table " + table)
|
||||
}
|
||||
dbv, err := toDBValue(v)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[k] = dbv
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// NormalizeOpFilter does the same for a filter's equality values.
|
||||
func NormalizeOpFilter(table string, filter map[string]interface{}) (map[string]interface{}, error) {
|
||||
if filter == nil {
|
||||
return nil, nil
|
||||
}
|
||||
return NormalizeOpFields(table, filter)
|
||||
}
|
||||
|
||||
func dumpDoc(table string, doc interface{}) (map[string]interface{}, error) {
|
||||
switch table {
|
||||
case "users":
|
||||
switch v := doc.(type) {
|
||||
case User:
|
||||
return dumpUser(v), nil
|
||||
case *User:
|
||||
return dumpUser(*v), nil
|
||||
}
|
||||
case "devices":
|
||||
switch v := doc.(type) {
|
||||
case ConstellationDevice:
|
||||
return dumpDevice(v), nil
|
||||
case *ConstellationDevice:
|
||||
return dumpDevice(*v), nil
|
||||
}
|
||||
}
|
||||
return nil, errors.New("store: unsupported doc for table " + table)
|
||||
}
|
||||
|
||||
func dumpDocs(table string, doc interface{}) ([]map[string]interface{}, error) {
|
||||
switch v := doc.(type) {
|
||||
case []User:
|
||||
out := make([]map[string]interface{}, len(v))
|
||||
for i, u := range v {
|
||||
out[i] = dumpUser(u)
|
||||
}
|
||||
return out, nil
|
||||
case []ConstellationDevice:
|
||||
out := make([]map[string]interface{}, len(v))
|
||||
for i, d := range v {
|
||||
out[i] = dumpDevice(d)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
return nil, errors.New("store: unsupported insertMany doc for table " + table)
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
const storeSchemaVersion = 1
|
||||
|
||||
// schemaV1: users keyed by nickname, devices with partial unique indexes on active rows only
|
||||
// (blocked devices may share name/IP with a live replacement), kv for stamps and versions.
|
||||
const schemaV1 = `
|
||||
CREATE TABLE IF NOT EXISTS kv (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
nickname TEXT PRIMARY KEY,
|
||||
password TEXT NOT NULL DEFAULT '',
|
||||
register_key TEXT NOT NULL DEFAULT '',
|
||||
register_key_exp TEXT NOT NULL DEFAULT '',
|
||||
role INTEGER NOT NULL DEFAULT 0,
|
||||
password_cycle INTEGER NOT NULL DEFAULT 0,
|
||||
email TEXT NOT NULL DEFAULT '',
|
||||
registered_at TEXT NOT NULL DEFAULT '',
|
||||
last_password_changed_at TEXT NOT NULL DEFAULT '',
|
||||
created_at TEXT NOT NULL DEFAULT '',
|
||||
last_login TEXT NOT NULL DEFAULT '',
|
||||
mfa_key TEXT NOT NULL DEFAULT '',
|
||||
was_2fa_verified INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS devices (
|
||||
nickname TEXT NOT NULL DEFAULT '',
|
||||
device_name TEXT NOT NULL DEFAULT '',
|
||||
public_key TEXT NOT NULL DEFAULT '',
|
||||
ip TEXT NOT NULL DEFAULT '',
|
||||
is_lighthouse INTEGER NOT NULL DEFAULT 0,
|
||||
cosmos_node INTEGER NOT NULL DEFAULT 0,
|
||||
is_relay INTEGER NOT NULL DEFAULT 0,
|
||||
is_load_balancer INTEGER NOT NULL DEFAULT 0,
|
||||
is_exit_node INTEGER NOT NULL DEFAULT 0,
|
||||
public_hostname TEXT NOT NULL DEFAULT '',
|
||||
port TEXT NOT NULL DEFAULT '',
|
||||
blocked INTEGER NOT NULL DEFAULT 0,
|
||||
fingerprint TEXT NOT NULL DEFAULT '',
|
||||
api_key TEXT NOT NULL DEFAULT '',
|
||||
invisible INTEGER NOT NULL DEFAULT 0,
|
||||
tags TEXT NOT NULL DEFAULT ''
|
||||
);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_devices_active_name ON devices(device_name) WHERE blocked = 0;
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_devices_active_ip ON devices(ip) WHERE blocked = 0;
|
||||
`
|
||||
|
||||
func initSchema(db *sql.DB) error {
|
||||
tx, err := db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := tx.Exec(schemaV1); err != nil {
|
||||
tx.Rollback()
|
||||
return err
|
||||
}
|
||||
|
||||
current := 0
|
||||
value, err := kvGet(tx, "schema_version")
|
||||
if err == nil {
|
||||
current, _ = strconv.Atoi(value)
|
||||
} else if err != ErrNotFound {
|
||||
tx.Rollback()
|
||||
return err
|
||||
}
|
||||
|
||||
// future schema migrations go here, gated on current < N
|
||||
_ = current
|
||||
|
||||
if err := kvSetTx(tx, "schema_version", strconv.Itoa(storeSchemaVersion)); err != nil {
|
||||
tx.Rollback()
|
||||
return err
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
@@ -0,0 +1,327 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func setupStore(t *testing.T) {
|
||||
t.Helper()
|
||||
tmp := t.TempDir() + "/"
|
||||
prev := CONFIGFOLDER
|
||||
CONFIGFOLDER = tmp
|
||||
if err := InitStore(); err != nil {
|
||||
t.Fatal("InitStore:", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
CloseStore()
|
||||
CONFIGFOLDER = prev
|
||||
})
|
||||
}
|
||||
|
||||
func TestStoreUserCRUD(t *testing.T) {
|
||||
setupStore(t)
|
||||
|
||||
created := time.Date(2024, 5, 1, 12, 30, 45, 123456789, time.UTC)
|
||||
err := CreateUser(User{
|
||||
Nickname: "alice",
|
||||
Password: "hashed-secret",
|
||||
Email: "[email protected]",
|
||||
Role: ADMIN,
|
||||
CreatedAt: created,
|
||||
// RegisterKeyExp / LastLogin left zero on purpose
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal("CreateUser:", err)
|
||||
}
|
||||
|
||||
u, err := GetUser("alice")
|
||||
if err != nil {
|
||||
t.Fatal("GetUser:", err)
|
||||
}
|
||||
if u.Password != "hashed-secret" || u.Email != "[email protected]" || u.Role != ADMIN {
|
||||
t.Fatalf("round-trip mismatch: %+v", u)
|
||||
}
|
||||
if !u.CreatedAt.Equal(created) {
|
||||
t.Fatalf("CreatedAt mismatch: %v != %v", u.CreatedAt, created)
|
||||
}
|
||||
if !u.RegisterKeyExp.IsZero() || !u.LastLogin.IsZero() {
|
||||
t.Fatalf("zero times must round-trip as zero: %+v", u)
|
||||
}
|
||||
|
||||
exp := time.Now().Add(time.Hour).UTC()
|
||||
if err := UpdateUser("alice", map[string]interface{}{
|
||||
"RegisterKey": "key123",
|
||||
"RegisterKeyExp": exp,
|
||||
"Was2FAVerified": true,
|
||||
"PasswordCycle": 3,
|
||||
}); err != nil {
|
||||
t.Fatal("UpdateUser:", err)
|
||||
}
|
||||
u, _ = GetUser("alice")
|
||||
if u.RegisterKey != "key123" || !u.RegisterKeyExp.Equal(exp) || !u.Was2FAVerified || u.PasswordCycle != 3 {
|
||||
t.Fatalf("update mismatch: %+v", u)
|
||||
}
|
||||
|
||||
// zero a time back out
|
||||
if err := UpdateUser("alice", map[string]interface{}{"RegisterKeyExp": time.Time{}}); err != nil {
|
||||
t.Fatal("UpdateUser zero time:", err)
|
||||
}
|
||||
u, _ = GetUser("alice")
|
||||
if !u.RegisterKeyExp.IsZero() {
|
||||
t.Fatal("zeroed time did not round-trip as zero")
|
||||
}
|
||||
|
||||
count, err := CountUsers()
|
||||
if err != nil || count != 1 {
|
||||
t.Fatalf("CountUsers = %d, %v", count, err)
|
||||
}
|
||||
|
||||
if err := DeleteUser("alice"); err != nil {
|
||||
t.Fatal("DeleteUser:", err)
|
||||
}
|
||||
if _, err := GetUser("alice"); !errors.Is(err, ErrNotFound) {
|
||||
t.Fatalf("expected ErrNotFound after delete, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStoreErrNotFound(t *testing.T) {
|
||||
setupStore(t)
|
||||
|
||||
if u, err := GetUser("ghost"); !errors.Is(err, ErrNotFound) || u.Nickname != "" || u.Password != "" {
|
||||
t.Fatalf("GetUser: want zero User + ErrNotFound, got %+v, %v", u, err)
|
||||
}
|
||||
if d, err := GetDeviceByName("ghost", false); !errors.Is(err, ErrNotFound) || d.DeviceName != "" || d.APIKey != "" {
|
||||
t.Fatalf("GetDeviceByName: want zero device + ErrNotFound, got %+v, %v", d, err)
|
||||
}
|
||||
if d, err := GetDeviceByIP("10.9.9.9"); !errors.Is(err, ErrNotFound) || d.DeviceName != "" {
|
||||
t.Fatalf("GetDeviceByIP: want zero device + ErrNotFound, got %+v, %v", d, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStoreDeviceCRUDAndTags(t *testing.T) {
|
||||
setupStore(t)
|
||||
|
||||
err := CreateDevice(ConstellationDevice{
|
||||
DeviceName: "node1",
|
||||
Nickname: "alice",
|
||||
IP: "192.168.201.2",
|
||||
APIKey: "secret-api-key",
|
||||
Tags: []string{"gpu", "eu-west"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal("CreateDevice:", err)
|
||||
}
|
||||
|
||||
d, err := GetDeviceByName("node1", true)
|
||||
if err != nil {
|
||||
t.Fatal("GetDeviceByName:", err)
|
||||
}
|
||||
if d.APIKey != "secret-api-key" || len(d.Tags) != 2 || d.Tags[0] != "gpu" || d.Tags[1] != "eu-west" {
|
||||
t.Fatalf("device round-trip mismatch: %+v", d)
|
||||
}
|
||||
|
||||
// no tags round-trips as nil
|
||||
if err := CreateDevice(ConstellationDevice{DeviceName: "node2", IP: "192.168.201.3"}); err != nil {
|
||||
t.Fatal("CreateDevice node2:", err)
|
||||
}
|
||||
d2, _ := GetDeviceByName("node2", true)
|
||||
if d2.Tags != nil {
|
||||
t.Fatalf("empty tags should round-trip as nil, got %v", d2.Tags)
|
||||
}
|
||||
|
||||
if err := UpdateDevices(
|
||||
map[string]interface{}{"DeviceName": "node1", "Blocked": false},
|
||||
map[string]interface{}{"Tags": []string{"cpu"}, "IsRelay": true},
|
||||
); err != nil {
|
||||
t.Fatal("UpdateDevices:", err)
|
||||
}
|
||||
d, _ = GetDeviceByName("node1", true)
|
||||
if !d.IsRelay || len(d.Tags) != 1 || d.Tags[0] != "cpu" {
|
||||
t.Fatalf("device update mismatch: %+v", d)
|
||||
}
|
||||
|
||||
devices, err := FindDevices(map[string]interface{}{"Nickname": "alice"})
|
||||
if err != nil || len(devices) != 1 {
|
||||
t.Fatalf("FindDevices = %v, %v", devices, err)
|
||||
}
|
||||
|
||||
if err := DeleteDevices(map[string]interface{}{"DeviceName": "node2"}); err != nil {
|
||||
t.Fatal("DeleteDevices:", err)
|
||||
}
|
||||
all, _ := ListDevices(true)
|
||||
if len(all) != 1 {
|
||||
t.Fatalf("expected 1 device after delete, got %d", len(all))
|
||||
}
|
||||
}
|
||||
|
||||
func TestStorePartialUniqueIndexes(t *testing.T) {
|
||||
setupStore(t)
|
||||
|
||||
if err := CreateDevice(ConstellationDevice{DeviceName: "dup", IP: "192.168.201.10"}); err != nil {
|
||||
t.Fatal("CreateDevice active:", err)
|
||||
}
|
||||
|
||||
// a blocked device may share name AND ip with a live replacement
|
||||
if err := CreateDevice(ConstellationDevice{DeviceName: "dup", IP: "192.168.201.10", Blocked: true}); err != nil {
|
||||
t.Fatal("blocked duplicate must be allowed:", err)
|
||||
}
|
||||
|
||||
// an active duplicate name is rejected by the partial index
|
||||
err := CreateDevice(ConstellationDevice{DeviceName: "dup", IP: "192.168.201.11"})
|
||||
var ec *ErrConstraint
|
||||
if !errors.As(err, &ec) {
|
||||
t.Fatalf("expected ErrConstraint for active dup name, got %v", err)
|
||||
}
|
||||
if ec.Table != "devices" {
|
||||
t.Fatalf("ErrConstraint.Table = %q", ec.Table)
|
||||
}
|
||||
|
||||
// an active duplicate ip is rejected too
|
||||
err = CreateDevice(ConstellationDevice{DeviceName: "other", IP: "192.168.201.10"})
|
||||
if !errors.As(err, &ec) {
|
||||
t.Fatalf("expected ErrConstraint for active dup ip, got %v", err)
|
||||
}
|
||||
|
||||
// blocking the live one frees the name for a replacement
|
||||
if err := UpdateDevices(map[string]interface{}{"DeviceName": "dup"}, map[string]interface{}{"Blocked": true}); err != nil {
|
||||
t.Fatal("block:", err)
|
||||
}
|
||||
if err := CreateDevice(ConstellationDevice{DeviceName: "dup", IP: "192.168.201.12"}); err != nil {
|
||||
t.Fatal("replacement after block:", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStoreConcurrentReadWrite(t *testing.T) {
|
||||
setupStore(t)
|
||||
|
||||
const writers = 4
|
||||
const perWriter = 20
|
||||
done := make(chan struct{})
|
||||
var wg sync.WaitGroup
|
||||
|
||||
// concurrent readers
|
||||
for r := 0; r < 3; r++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for {
|
||||
select {
|
||||
case <-done:
|
||||
return
|
||||
default:
|
||||
ListUsers("")
|
||||
GetUser("w0-5")
|
||||
CountDevices(map[string]interface{}{})
|
||||
GetLastAppliedSeq()
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
var werr sync.Map
|
||||
var wwg sync.WaitGroup
|
||||
for wi := 0; wi < writers; wi++ {
|
||||
wwg.Add(1)
|
||||
go func(wi int) {
|
||||
defer wwg.Done()
|
||||
for i := 0; i < perWriter; i++ {
|
||||
nick := fmt.Sprintf("w%d-%d", wi, i)
|
||||
if err := CreateUser(User{Nickname: nick, CreatedAt: time.Now()}); err != nil {
|
||||
werr.Store(nick, err)
|
||||
}
|
||||
}
|
||||
}(wi)
|
||||
}
|
||||
wwg.Wait()
|
||||
close(done)
|
||||
wg.Wait()
|
||||
|
||||
werr.Range(func(k, v interface{}) bool {
|
||||
t.Errorf("write %v failed: %v", k, v)
|
||||
return true
|
||||
})
|
||||
|
||||
count, err := CountUsers()
|
||||
if err != nil || count != writers*perWriter {
|
||||
t.Fatalf("CountUsers = %d, %v (want %d)", count, err, writers*perWriter)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStoreLogicalDumpRoundTrip(t *testing.T) {
|
||||
setupStore(t)
|
||||
|
||||
CreateUser(User{Nickname: "bob", Password: "hash", Role: USER})
|
||||
CreateDevice(ConstellationDevice{DeviceName: "d1", IP: "192.168.201.5", APIKey: "k", Tags: []string{"t"}})
|
||||
CreateDevice(ConstellationDevice{DeviceName: "d1", IP: "192.168.201.5", Blocked: true})
|
||||
|
||||
dump, err := BuildLogicalDump()
|
||||
if err != nil {
|
||||
t.Fatal("BuildLogicalDump:", err)
|
||||
}
|
||||
|
||||
if err := ApplyLogicalDump(dump, 3, 4242); err != nil {
|
||||
t.Fatal("ApplyLogicalDump:", err)
|
||||
}
|
||||
|
||||
if epoch, seq := GetOplogEpoch(), GetLastAppliedSeq(); epoch != 3 || seq != 4242 {
|
||||
t.Fatalf("op-log position not adopted: epoch %d seq %d", epoch, seq)
|
||||
}
|
||||
|
||||
u, err := GetUser("bob")
|
||||
if err != nil || u.Password != "hash" {
|
||||
t.Fatalf("user lost in dump round-trip: %+v, %v", u, err)
|
||||
}
|
||||
devices, _ := ListDevices(true)
|
||||
if len(devices) != 2 {
|
||||
t.Fatalf("devices lost in dump round-trip: %d", len(devices))
|
||||
}
|
||||
|
||||
// canonical: same content must serialize identically
|
||||
dump2, err := BuildLogicalDump()
|
||||
if err != nil {
|
||||
t.Fatal("BuildLogicalDump 2:", err)
|
||||
}
|
||||
if string(dump) != string(dump2) {
|
||||
t.Fatal("dump is not canonical across rebuilds")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStoreOplogPosition(t *testing.T) {
|
||||
setupStore(t)
|
||||
|
||||
if GetOplogEpoch() != 1 || GetLastAppliedSeq() != 0 {
|
||||
t.Fatal("fresh store should start at epoch 1, seq 0")
|
||||
}
|
||||
|
||||
// applying an op commits its sequence in the same tx as the data
|
||||
m := Mutation{Table: "users", Op: "insert", Doc: User{Nickname: "seqbob"}}
|
||||
if err := ApplyOpTx(m, 7, nil); err != nil {
|
||||
t.Fatal("ApplyOpTx:", err)
|
||||
}
|
||||
if seq := GetLastAppliedSeq(); seq != 7 {
|
||||
t.Fatalf("seq not committed with the data: %d", seq)
|
||||
}
|
||||
if _, err := GetUser("seqbob"); err != nil {
|
||||
t.Fatal("data not committed with the seq:", err)
|
||||
}
|
||||
|
||||
// a rejected op still consumes its sequence
|
||||
if err := CommitOplogSeq(8); err != nil {
|
||||
t.Fatal("CommitOplogSeq:", err)
|
||||
}
|
||||
if seq := GetLastAppliedSeq(); seq != 8 {
|
||||
t.Fatalf("rejected op did not consume its seq: %d", seq)
|
||||
}
|
||||
|
||||
// a direct write leaves the log position untouched
|
||||
if err := CommitMutationDirect(Mutation{Table: "users", Op: "insert", Doc: User{Nickname: "direct"}}, nil); err != nil {
|
||||
t.Fatal("CommitMutationDirect:", err)
|
||||
}
|
||||
if seq := GetLastAppliedSeq(); seq != 8 {
|
||||
t.Fatalf("direct write moved the log position: %d", seq)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,209 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
)
|
||||
|
||||
const userCols = "nickname, password, register_key, register_key_exp, role, password_cycle, email, registered_at, last_password_changed_at, created_at, last_login, mfa_key, was_2fa_verified"
|
||||
|
||||
type rowQuerier interface {
|
||||
Query(string, ...interface{}) (*sql.Rows, error)
|
||||
}
|
||||
|
||||
func insertUserTx(tx *sql.Tx, u User) error {
|
||||
_, err := tx.Exec("INSERT INTO users ("+userCols+") VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
||||
u.Nickname, u.Password, u.RegisterKey, timeToDB(u.RegisterKeyExp), int(u.Role), u.PasswordCycle,
|
||||
u.Email, timeToDB(u.RegisteredAt), timeToDB(u.LastPasswordChangedAt), timeToDB(u.CreatedAt),
|
||||
timeToDB(u.LastLogin), u.MFAKey, boolToDB(u.Was2FAVerified))
|
||||
return err
|
||||
}
|
||||
|
||||
func scanUsers(q rowQuerier, query string, args ...interface{}) ([]User, error) {
|
||||
rows, err := q.Query(query, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
// convention: on ANY error return nil, never a partial struct or slice —
|
||||
// call sites' fail-closed field checks depend on this
|
||||
users := []User{}
|
||||
for rows.Next() {
|
||||
var u User
|
||||
var role, was2fa int
|
||||
var regExp, regAt, lastPw, createdAt, lastLogin string
|
||||
if err := rows.Scan(&u.Nickname, &u.Password, &u.RegisterKey, ®Exp, &role, &u.PasswordCycle,
|
||||
&u.Email, ®At, &lastPw, &createdAt, &lastLogin, &u.MFAKey, &was2fa); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
u.RegisterKeyExp = timeFromDB(regExp)
|
||||
u.Role = Role(role)
|
||||
u.RegisteredAt = timeFromDB(regAt)
|
||||
u.LastPasswordChangedAt = timeFromDB(lastPw)
|
||||
u.CreatedAt = timeFromDB(createdAt)
|
||||
u.LastLogin = timeFromDB(lastLogin)
|
||||
u.Was2FAVerified = was2fa != 0
|
||||
users = append(users, u)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return users, nil
|
||||
}
|
||||
|
||||
// GetUser returns the user by nickname; on any error it returns a zero-value User
|
||||
// (never partial data) so field checks at call sites stay fail-closed.
|
||||
func GetUser(nickname string) (User, error) {
|
||||
db, err := getReadDB()
|
||||
if err != nil {
|
||||
return User{}, err
|
||||
}
|
||||
users, err := scanUsers(db, "SELECT "+userCols+" FROM users WHERE nickname = ?", nickname)
|
||||
if err != nil {
|
||||
return User{}, err
|
||||
}
|
||||
if len(users) == 0 {
|
||||
return User{}, ErrNotFound
|
||||
}
|
||||
return users[0], nil
|
||||
}
|
||||
|
||||
// ListUsers returns users filtered by role ("admin", "user", or "" for all).
|
||||
func ListUsers(role string) ([]User, error) {
|
||||
db, err := getReadDB()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch role {
|
||||
case "admin":
|
||||
return scanUsers(db, "SELECT "+userCols+" FROM users WHERE role = ?", int(ADMIN))
|
||||
case "user":
|
||||
return scanUsers(db, "SELECT "+userCols+" FROM users WHERE role = ?", int(USER))
|
||||
}
|
||||
return scanUsers(db, "SELECT "+userCols+" FROM users")
|
||||
}
|
||||
|
||||
// ListAllUsers keeps the legacy signature for existing callers.
|
||||
func ListAllUsers(role string) []User {
|
||||
users, err := ListUsers(role)
|
||||
if err != nil {
|
||||
Error("Database: Error while getting users", err)
|
||||
return []User{}
|
||||
}
|
||||
return users
|
||||
}
|
||||
|
||||
func ListUsersPage(limit int) ([]User, error) {
|
||||
db, err := getReadDB()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return scanUsers(db, "SELECT "+userCols+" FROM users ORDER BY nickname LIMIT ?", limit)
|
||||
}
|
||||
|
||||
// CountUsers returns the number of users registered on this server.
|
||||
func CountUsers() (int64, error) {
|
||||
db, err := getReadDB()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var count int64
|
||||
err = db.QueryRow("SELECT COUNT(*) FROM users").Scan(&count)
|
||||
return count, err
|
||||
}
|
||||
|
||||
func CreateUser(u User) error {
|
||||
return CommitMutation(Mutation{Table: "users", Op: "insert", Doc: u})
|
||||
}
|
||||
|
||||
// UpdateUser sets the given bson-named fields on the user row.
|
||||
func UpdateUser(nickname string, fields map[string]interface{}) error {
|
||||
return CommitMutation(Mutation{
|
||||
Table: "users",
|
||||
Op: "update",
|
||||
Filter: map[string]interface{}{"Nickname": nickname},
|
||||
Doc: fields,
|
||||
})
|
||||
}
|
||||
|
||||
func DeleteUser(nickname string) error {
|
||||
return CommitMutation(Mutation{
|
||||
Table: "users",
|
||||
Op: "delete",
|
||||
Filter: map[string]interface{}{"Nickname": nickname},
|
||||
})
|
||||
}
|
||||
|
||||
func DeleteAllUsers() error {
|
||||
return CommitMutation(Mutation{Table: "users", Op: "deleteMany", Filter: map[string]interface{}{}})
|
||||
}
|
||||
|
||||
// DeleteAllUsersLocal wipes users on this node only, never publishing. Reserved
|
||||
// for fresh-install setup, where the intent is "this box starts empty" rather
|
||||
// than "empty the cluster" — an empty filter compiles to an unqualified DELETE,
|
||||
// so published it would take every user on every node with it.
|
||||
func DeleteAllUsersLocal() error {
|
||||
return CommitMutationLocal(Mutation{Table: "users", Op: "deleteMany", Filter: map[string]interface{}{}})
|
||||
}
|
||||
|
||||
// dump encoding uses bson field names and RFC3339Nano strings for canonical JSON
|
||||
func dumpUser(u User) map[string]interface{} {
|
||||
return map[string]interface{}{
|
||||
"Nickname": u.Nickname,
|
||||
"Password": u.Password,
|
||||
"RegisterKey": u.RegisterKey,
|
||||
"RegisterKeyExp": timeToDB(u.RegisterKeyExp),
|
||||
"Role": int(u.Role),
|
||||
"PasswordCycle": u.PasswordCycle,
|
||||
"Email": u.Email,
|
||||
"RegisteredAt": timeToDB(u.RegisteredAt),
|
||||
"LastPasswordChangedAt": timeToDB(u.LastPasswordChangedAt),
|
||||
"CreatedAt": timeToDB(u.CreatedAt),
|
||||
"LastLogin": timeToDB(u.LastLogin),
|
||||
"MFAKey": u.MFAKey,
|
||||
"Was2FAVerified": u.Was2FAVerified,
|
||||
}
|
||||
}
|
||||
|
||||
func parseDumpUser(m map[string]interface{}) User {
|
||||
return User{
|
||||
Nickname: dumpStr(m, "Nickname"),
|
||||
Password: dumpStr(m, "Password"),
|
||||
RegisterKey: dumpStr(m, "RegisterKey"),
|
||||
RegisterKeyExp: timeFromDB(dumpStr(m, "RegisterKeyExp")),
|
||||
Role: Role(dumpInt(m, "Role")),
|
||||
PasswordCycle: dumpInt(m, "PasswordCycle"),
|
||||
Email: dumpStr(m, "Email"),
|
||||
RegisteredAt: timeFromDB(dumpStr(m, "RegisteredAt")),
|
||||
LastPasswordChangedAt: timeFromDB(dumpStr(m, "LastPasswordChangedAt")),
|
||||
CreatedAt: timeFromDB(dumpStr(m, "CreatedAt")),
|
||||
LastLogin: timeFromDB(dumpStr(m, "LastLogin")),
|
||||
MFAKey: dumpStr(m, "MFAKey"),
|
||||
Was2FAVerified: dumpBool(m, "Was2FAVerified"),
|
||||
}
|
||||
}
|
||||
|
||||
func dumpStr(m map[string]interface{}, key string) string {
|
||||
if v, ok := m[key].(string); ok {
|
||||
return v
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func dumpInt(m map[string]interface{}, key string) int {
|
||||
switch v := m[key].(type) {
|
||||
case float64:
|
||||
return int(v)
|
||||
case int:
|
||||
return v
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func dumpBool(m map[string]interface{}, key string) bool {
|
||||
if v, ok := m[key].(bool); ok {
|
||||
return v
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
+2
-20
@@ -32,7 +32,6 @@ import (
|
||||
"golang.org/x/net/publicsuffix"
|
||||
"github.com/Masterminds/semver"
|
||||
"golang.org/x/crypto/bcrypt"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
)
|
||||
|
||||
// JSONEquals reports whether two values serialize to identical JSON. Handy for
|
||||
@@ -86,7 +85,6 @@ var IsPro func() bool = func() bool { return false }
|
||||
|
||||
var InitPremiumFeatures func()
|
||||
|
||||
var ResyncConstellationNodes = func() {}
|
||||
var GetConstellationTunnelRoutes = func() []ProxyRouteConfig { return []ProxyRouteConfig{} }
|
||||
|
||||
var LetsEncryptErrors = []string{}
|
||||
@@ -1044,20 +1042,9 @@ func CompareSemver(v1, v2 string) (int, error) {
|
||||
func CheckPassword(nickname, password string) error {
|
||||
time.Sleep(time.Duration(rand.Float64()*1)*time.Second)
|
||||
|
||||
c, closeDb, errCo := GetEmbeddedCollection(GetRootAppId(), "users")
|
||||
defer closeDb()
|
||||
if errCo != nil {
|
||||
return errCo
|
||||
}
|
||||
user, err3 := GetUser(nickname)
|
||||
|
||||
user := User{}
|
||||
|
||||
err3 := c.FindOne(nil, map[string]interface{}{
|
||||
"Nickname": nickname,
|
||||
}).Decode(&user)
|
||||
|
||||
|
||||
if err3 == mongo.ErrNoDocuments {
|
||||
if errors.Is(err3, ErrNotFound) {
|
||||
bcrypt.CompareHashAndPassword([]byte("$2a$14$4nzsVwEnR3.jEbMTME7kqeCo4gMgR/Tuk7ivNExvXjr73nKvLgHka"), []byte("dummyPassword"))
|
||||
return err3
|
||||
} else if err3 != nil {
|
||||
@@ -1406,8 +1393,3 @@ func SetFileLastModifiedTime(path string, modTime int64) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func TouchDatabase() error {
|
||||
dbPath := CONFIGFOLDER + "database"
|
||||
now := time.Now()
|
||||
return os.Chtimes(dbPath, now, now)
|
||||
}
|
||||
Reference in New Issue
Block a user