Broker unterstützt jetzt mehrere WSGI Worker

This commit is contained in:
concierge committed 2026-11-03 16:47:20 +01:00
1 parent 1e2a58e563
commit f5b0c9509c
13 files changed
+1252 -327

No files matched your search

+535
View File
@@ -0,0 +1,535 @@
"""Prozessübergreifender Zustand des WebSocket-Brokers.
Der Broker hält weiterhin nur die WebSocket-Objekte im eigenen Prozess.
Alles, was ein Reconnect auf einem anderen WSGI-Worker wiederfinden muss,
liegt in SQLite:
- ``ws_clients``: bekannte Client-IDs, Account, letzte Sequenznummer und
Verbindungsstatus,
- ``ws_client_queue``: persistente Zustellaufträge pro Client,
- ``ws_group_presence``: gemeinsame Pong-Zeitpunkte der Gruppen.
Die Tabellen werden über ``ensure_schema()`` angelegt. Die Funktionen hier
erwarten einen aktiven Flask-App-Kontext, weil sie ``get_db()`` verwenden.
"""
import time
from cmbnet.db import get_db
# Kleine Hilfsfunktion, damit alle Zeitstempel dieselbe Basis haben.
def now():
return time.time()
# ----------------------------------------------------------------------
# Clients
# ----------------------------------------------------------------------
def get_client(tenant_id, client_id):
"""Liefert die Zeile eines bekannten Clients oder ``None``."""
return get_db().execute(
'SELECT * FROM ws_clients WHERE tenant_id = ? AND client_id = ?',
(tenant_id, client_id)
).fetchone()
def get_client_account(tenant_id, client_id):
"""Liefert die Account-ID, die zu einer Client-ID gehört."""
row = get_client(tenant_id, client_id)
return row['account_id'] if row else None
def client_exists(tenant_id, client_id):
return get_client(tenant_id, client_id) is not None
def client_is_connected(tenant_id, client_id):
row = get_client(tenant_id, client_id)
return bool(row and row['disconnected_at'] is None)
def ensure_client(tenant_id, client_id, account_id, worker_id=None):
"""Registriert eine Client-ID bzw. aktualisiert ihren Verbindungsstatus.
Gibt ``False`` zurück, wenn die Client-ID bereits einem anderen Account
gehört. Ein solcher Wechsel wäre ein Hinweis auf eine manipulierte
Zuordnung und wird vom Broker abgelehnt.
"""
db = get_db()
row = get_client(tenant_id, client_id)
current = now()
if row is not None:
if row['account_id'] != account_id:
return False
db.execute(
'UPDATE ws_clients SET worker_id = ?, connected_at = ?, '
'disconnected_at = NULL, authenticated = 0 '
'WHERE tenant_id = ? AND client_id = ?',
(worker_id, current, tenant_id, client_id)
)
db.commit()
return True
db.execute(
'INSERT INTO ws_clients '
'(tenant_id, client_id, account_id, worker_id, connected_at, '
' authenticated, last_seq) '
'VALUES (?, ?, ?, ?, ?, 0, 0)',
(tenant_id, client_id, account_id, worker_id, current)
)
db.commit()
return True
def mark_authenticated(tenant_id, client_id, authenticated=True, worker_id=None):
db = get_db()
if worker_id is None:
db.execute(
'UPDATE ws_clients SET authenticated = ? '
'WHERE tenant_id = ? AND client_id = ?',
(1 if authenticated else 0, tenant_id, client_id)
)
else:
# Nur der Worker, dem die Client-ID aktuell zugeordnet ist, darf den
# Authentifizierungsstatus setzen. So überschreibt ein verspäteter
# Handshake eines alten Workers keinen Reconnect auf einem anderen
# Worker.
db.execute(
'UPDATE ws_clients SET authenticated = ? '
'WHERE tenant_id = ? AND client_id = ? '
'AND (worker_id IS NULL OR worker_id = ?)',
(1 if authenticated else 0, tenant_id, client_id, worker_id)
)
db.commit()
def mark_disconnected(tenant_id, client_id, worker_id=None):
"""Markiert eine Client-ID als getrennt, ohne ihre Queue zu löschen.
Mit ``worker_id`` wird nur die Zuordnung dieses Workers getrennt. Hat
inzwischen ein anderer Worker dieselbe Client-ID übernommen, bleibt dessen
Verbindungsstatus unberührt.
"""
db = get_db()
if worker_id is None:
db.execute(
'UPDATE ws_clients SET worker_id = NULL, disconnected_at = ?, '
'authenticated = 0 WHERE tenant_id = ? AND client_id = ?',
(now(), tenant_id, client_id)
)
else:
db.execute(
'UPDATE ws_clients SET worker_id = NULL, disconnected_at = ?, '
'authenticated = 0 WHERE tenant_id = ? AND client_id = ? '
'AND (worker_id IS NULL OR worker_id = ?)',
(now(), tenant_id, client_id, worker_id)
)
db.commit()
def delete_client(tenant_id, client_id, worker_id=None):
"""Entfernt eine Client-ID samt Queue und Sequenzzustand.
Mit ``worker_id`` wird nur gelöscht, solange die Client-ID noch diesem
Worker gehört. Das verhindert, dass ein scheidender Worker einen
zwischenzeitlichen Reconnect auf einem anderen Worker entfernt.
"""
db = get_db()
if worker_id is None:
cursor = db.execute(
'DELETE FROM ws_clients WHERE tenant_id = ? AND client_id = ?',
(tenant_id, client_id)
)
else:
cursor = db.execute(
'DELETE FROM ws_clients WHERE tenant_id = ? AND client_id = ? '
'AND (worker_id IS NULL OR worker_id = ?)',
(tenant_id, client_id, worker_id)
)
if cursor.rowcount:
db.execute(
'DELETE FROM ws_client_queue WHERE tenant_id = ? AND client_id = ?',
(tenant_id, client_id)
)
db.commit()
def get_last_seq(tenant_id, client_id):
row = get_client(tenant_id, client_id)
return int(row['last_seq'] or 0) if row else 0
def set_last_seq(tenant_id, client_id, seq, account_id=None):
db = get_db()
cursor = db.execute(
'UPDATE ws_clients SET last_seq = ? WHERE tenant_id = ? AND client_id = ?',
(seq, tenant_id, client_id)
)
if cursor.rowcount:
db.commit()
return
# Die Client-Zeile kann durch einen Cleanup verschwunden sein. In diesem
# Fall wird sie mit der bekannten Account-ID neu angelegt, damit der
# Sequenzstand nicht verloren geht.
if account_id is not None:
db.execute(
'INSERT OR IGNORE INTO ws_clients '
'(tenant_id, client_id, account_id, last_seq) VALUES (?, ?, ?, ?)',
(tenant_id, client_id, account_id, seq)
)
db.commit()
def client_ids_for_accounts(tenant_id, account_ids):
"""Expandiert Empfänger-Accounts auf alle bekannten Client-IDs.
Anders als früher werden auch Clients berücksichtigt, die an einem
anderen Worker verbunden oder momentan getrennt sind. Nur so können
Nachrichten für einen Reconnect auf einem beliebigen Worker gepuffert
werden.
"""
account_ids = list(dict.fromkeys(account_ids))
if not account_ids:
return set()
db = get_db()
result = set()
chunk_size = 400
for start in range(0, len(account_ids), chunk_size):
chunk = account_ids[start:start + chunk_size]
placeholders = ','.join('?' for _ in chunk)
rows = db.execute(
f'SELECT client_id FROM ws_clients '
f'WHERE tenant_id = ? AND account_id IN ({placeholders})',
(tenant_id, *chunk)
).fetchall()
result.update(row['client_id'] for row in rows)
return result
def describe_connections(tenant_id):
"""Liefert alle aktuell verbundenen Clients inklusive Account und Gruppe."""
rows = get_db().execute(
'''SELECT c.client_id, c.account_id, c.authenticated,
a.name AS account_name, a.role,
g.id AS group_id, g.aliasname AS group_alias
FROM ws_clients c
JOIN accounts a
ON a.id = c.account_id AND a.tenant_id = c.tenant_id
LEFT JOIN group_accounts ga
ON ga.tenant_id = c.tenant_id AND ga.account_id = c.account_id
LEFT JOIN groups g
ON g.id = ga.group_id AND g.tenant_id = c.tenant_id
WHERE c.tenant_id = ? AND c.disconnected_at IS NULL
ORDER BY c.client_id''',
(tenant_id,)
).fetchall()
return [dict(row) for row in rows]
# ----------------------------------------------------------------------
# Client-Queues
# ----------------------------------------------------------------------
def get_queue_started_at(tenant_id, client_id):
row = get_client(tenant_id, client_id)
return row['queue_started_at'] if row else None
def _set_queue_started_at(db, tenant_id, client_id, started_at):
db.execute(
'UPDATE ws_clients SET queue_started_at = ? '
'WHERE tenant_id = ? AND client_id = ?',
(started_at, tenant_id, client_id)
)
def client_has_queue(tenant_id, client_id):
"""Prüft, ob für einen Client noch zustellbare Aufträge warten."""
row = get_db().execute(
'SELECT 1 FROM ws_client_queue '
'WHERE tenant_id = ? AND client_id = ? '
'AND (expires_at IS NULL OR expires_at > ?) LIMIT 1',
(tenant_id, client_id, now())
).fetchone()
return row is not None
def enqueue_message(tenant_id, client_id, code, payload, max_age_seconds,
timeout_payload=None, ephemeral_seconds=None):
"""Legt einen Zustellauftrag für einen Client an.
``max_age_seconds`` steuert die Lebensdauer einer normalen Queue.
``timeout_payload`` wird als Code-99-Tombstone eingesetzt, wenn die Queue
zu alt ist oder eine Code-99-Nachricht alle älteren Aufträge verdrängt.
``ephemeral_seconds`` gilt für Code 2: Der Auftrag verfällt schnell und
wird nicht für getrennte Clients vorgehalten.
"""
db = get_db()
current = now()
row = db.execute(
'SELECT queue_started_at FROM ws_clients '
'WHERE tenant_id = ? AND client_id = ?',
(tenant_id, client_id)
).fetchone()
if row is None:
# Unbekannte Client-IDs werden nicht gepuffert. Der Broker legt die
# Zeile beim Code-0-Handshake an.
return False
started_at = row['queue_started_at']
if started_at is None:
started_at = current
# Eine Code-99-Nachricht verdrängt alle älteren Aufträge und blockiert
# weitere Nachrichten, bis sie zugestellt wurde.
if code != 99:
tombstone = db.execute(
'SELECT 1 FROM ws_client_queue '
'WHERE tenant_id = ? AND client_id = ? AND code99_only = 1 LIMIT 1',
(tenant_id, client_id)
).fetchone()
if tombstone:
return False
# Abgelaufene Queue: Inhalt verwerfen und einen Code-99-Hinweis hinterlassen.
if current - started_at >= max_age_seconds:
db.execute(
'DELETE FROM ws_client_queue WHERE tenant_id = ? AND client_id = ?',
(tenant_id, client_id)
)
if timeout_payload is not None:
db.execute(
'INSERT INTO ws_client_queue '
'(tenant_id, client_id, code, payload, code99_only, created_at) '
'VALUES (?, ?, 99, ?, 1, ?)',
(tenant_id, client_id, timeout_payload, current)
)
_set_queue_started_at(db, tenant_id, client_id, current)
else:
_set_queue_started_at(db, tenant_id, client_id, None)
db.commit()
return True
if code == 99:
db.execute(
'DELETE FROM ws_client_queue WHERE tenant_id = ? AND client_id = ?',
(tenant_id, client_id)
)
db.execute(
'INSERT INTO ws_client_queue '
'(tenant_id, client_id, code, payload, code99_only, created_at) '
'VALUES (?, ?, 99, ?, 1, ?)',
(tenant_id, client_id, payload, current)
)
_set_queue_started_at(db, tenant_id, client_id, current)
db.commit()
return True
expires_at = None
if ephemeral_seconds is not None:
expires_at = current + ephemeral_seconds
db.execute(
'INSERT INTO ws_client_queue '
'(tenant_id, client_id, code, payload, code99_only, created_at, expires_at) '
'VALUES (?, ?, ?, ?, 0, ?, ?)',
(tenant_id, client_id, code, payload, current, expires_at)
)
_set_queue_started_at(db, tenant_id, client_id, started_at)
db.commit()
return True
def fetch_queue(tenant_id, client_id):
"""Liefert alle noch gültigen Aufträge eines Clients in Reihenfolge."""
return get_db().execute(
'SELECT * FROM ws_client_queue '
'WHERE tenant_id = ? AND client_id = ? '
'AND (expires_at IS NULL OR expires_at > ?) '
'ORDER BY id ASC',
(tenant_id, client_id, now())
).fetchall()
def clients_with_queue(pairs):
"""Liefert diejenigen ``(tenant_id, client_id)``, die Aufträge haben."""
if not pairs:
return set()
db = get_db()
by_tenant = {}
for tenant_id, client_id in pairs:
by_tenant.setdefault(tenant_id, set()).add(client_id)
result = set()
chunk_size = 400
for tenant_id, client_ids in by_tenant.items():
ids = list(client_ids)
for start in range(0, len(ids), chunk_size):
chunk = ids[start:start + chunk_size]
placeholders = ','.join('?' for _ in chunk)
rows = db.execute(
f'SELECT DISTINCT client_id FROM ws_client_queue '
f'WHERE tenant_id = ? AND client_id IN ({placeholders}) '
f'AND (expires_at IS NULL OR expires_at > ?)',
(tenant_id, *chunk, now())
).fetchall()
result.update((tenant_id, row['client_id']) for row in rows)
return result
def delete_queue_row(row_id):
"""Löscht einen zugestellten Auftrag und räumt den Queue-Start auf."""
db = get_db()
row = db.execute(
'SELECT tenant_id, client_id FROM ws_client_queue WHERE id = ?',
(row_id,)
).fetchone()
if row is None:
return
db.execute('DELETE FROM ws_client_queue WHERE id = ?', (row_id,))
remaining = db.execute(
'SELECT 1 FROM ws_client_queue '
'WHERE tenant_id = ? AND client_id = ? '
'AND (expires_at IS NULL OR expires_at > ?) LIMIT 1',
(row['tenant_id'], row['client_id'], now())
).fetchone()
if remaining is None:
_set_queue_started_at(db, row['tenant_id'], row['client_id'], None)
db.commit()
def replace_queue_with_timeout(tenant_id, client_id, payload):
"""Ersetzt eine abgelaufene Queue durch einen Code-99-Hinweis."""
db = get_db()
current = now()
db.execute(
'DELETE FROM ws_client_queue WHERE tenant_id = ? AND client_id = ?',
(tenant_id, client_id)
)
db.execute(
'INSERT INTO ws_client_queue '
'(tenant_id, client_id, code, payload, code99_only, created_at) '
'VALUES (?, ?, 99, ?, 1, ?)',
(tenant_id, client_id, payload, current)
)
_set_queue_started_at(db, tenant_id, client_id, current)
db.commit()
def expire_queues(max_age_seconds, timeout_payload):
"""Wandelt zu alte Queues in Code-99-Tombstones um.
Die Funktion ist idempotent und kann von jedem Worker regelmäßig
aufgerufen werden. Nach dem ersten Lauf ist ``queue_started_at``
aktualisiert, sodass weitere Worker die Queue nicht erneut umwandeln.
"""
db = get_db()
cutoff = now() - max_age_seconds
rows = db.execute(
'SELECT tenant_id, client_id FROM ws_clients '
'WHERE queue_started_at IS NOT NULL AND queue_started_at <= ?',
(cutoff,)
).fetchall()
current = now()
for row in rows:
tenant_id = row['tenant_id']
client_id = row['client_id']
db.execute(
'DELETE FROM ws_client_queue WHERE tenant_id = ? AND client_id = ?',
(tenant_id, client_id)
)
db.execute(
'INSERT INTO ws_client_queue '
'(tenant_id, client_id, code, payload, code99_only, created_at) '
'VALUES (?, ?, 99, ?, 1, ?)',
(tenant_id, client_id, timeout_payload, current)
)
_set_queue_started_at(db, tenant_id, client_id, current)
if rows:
db.commit()
def cleanup_clients(retention_seconds):
"""Entfernt dauerhaft getrennte Clients ohne zustellbare Restnachrichten."""
db = get_db()
cutoff = now() - retention_seconds
rows = db.execute(
'SELECT tenant_id, client_id FROM ws_clients '
'WHERE disconnected_at IS NOT NULL AND disconnected_at < ?',
(cutoff,)
).fetchall()
for row in rows:
tenant_id = row['tenant_id']
client_id = row['client_id']
pending = db.execute(
'SELECT 1 FROM ws_client_queue '
'WHERE tenant_id = ? AND client_id = ? AND code99_only = 0 '
'AND (expires_at IS NULL OR expires_at > ?) LIMIT 1',
(tenant_id, client_id, now())
).fetchone()
if pending is not None:
# Noch zustellbare Nachrichten halten den Client am Leben.
continue
db.execute(
'DELETE FROM ws_client_queue WHERE tenant_id = ? AND client_id = ?',
(tenant_id, client_id)
)
db.execute(
'DELETE FROM ws_clients WHERE tenant_id = ? AND client_id = ?',
(tenant_id, client_id)
)
if rows:
db.commit()
def purge_expired_queue_rows():
"""Löscht abgelaufene ephemere Aufträge (z. B. Code 2)."""
db = get_db()
db.execute(
'DELETE FROM ws_client_queue WHERE expires_at IS NOT NULL AND expires_at <= ?',
(now(),)
)
db.commit()
# ----------------------------------------------------------------------
# Gruppenpräsenz
# ----------------------------------------------------------------------
def get_group_presence(tenant_id, group_id):
return get_db().execute(
'SELECT * FROM ws_group_presence WHERE tenant_id = ? AND group_id = ?',
(tenant_id, group_id)
).fetchone()
def record_group_pong(tenant_id, group_id, timestamp=None):
db = get_db()
current = timestamp if timestamp is not None else now()
db.execute(
'INSERT OR IGNORE INTO ws_group_presence (tenant_id, group_id) VALUES (?, ?)',
(tenant_id, group_id)
)
db.execute(
'UPDATE ws_group_presence SET last_pong_at = ?, updated_at = CURRENT_TIMESTAMP '
'WHERE tenant_id = ? AND group_id = ?',
(current, tenant_id, group_id)
)
db.commit()
def set_group_online_since(tenant_id, group_id, timestamp=None):
db = get_db()
current = timestamp if timestamp is not None else now()
db.execute(
'INSERT OR IGNORE INTO ws_group_presence (tenant_id, group_id) VALUES (?, ?)',
(tenant_id, group_id)
)
db.execute(
'UPDATE ws_group_presence SET online_since = ?, updated_at = CURRENT_TIMESTAMP '
'WHERE tenant_id = ? AND group_id = ?',
(current, tenant_id, group_id)
)
db.commit()