536 lines
18 KiB
Python
536 lines
18 KiB
Python
"""Prozessübergreifender Zustand des WebSocket-Brokers.
|
|
|
|
Der Broker hält weiterhin nur die WebSocket-Objekte im eigenen Prozess.
|
|
Alles, was ein Reconnect auf einem anderen WSGI-Worker wiederfinden muss,
|
|
liegt in SQLite:
|
|
|
|
- ``ws_clients``: bekannte Client-IDs, Account, letzte Sequenznummer und
|
|
Verbindungsstatus,
|
|
- ``ws_client_queue``: persistente Zustellaufträge pro Client,
|
|
- ``ws_group_presence``: gemeinsame Pong-Zeitpunkte der Gruppen.
|
|
|
|
Die Tabellen werden über ``ensure_schema()`` angelegt. Die Funktionen hier
|
|
erwarten einen aktiven Flask-App-Kontext, weil sie ``get_db()`` verwenden.
|
|
"""
|
|
import time
|
|
|
|
from cmbnet.db import get_db
|
|
|
|
# Kleine Hilfsfunktion, damit alle Zeitstempel dieselbe Basis haben.
|
|
def now():
|
|
return time.time()
|
|
|
|
|
|
# ----------------------------------------------------------------------
|
|
# Clients
|
|
# ----------------------------------------------------------------------
|
|
def get_client(tenant_id, client_id):
|
|
"""Liefert die Zeile eines bekannten Clients oder ``None``."""
|
|
return get_db().execute(
|
|
'SELECT * FROM ws_clients WHERE tenant_id = ? AND client_id = ?',
|
|
(tenant_id, client_id)
|
|
).fetchone()
|
|
|
|
|
|
def get_client_account(tenant_id, client_id):
|
|
"""Liefert die Account-ID, die zu einer Client-ID gehört."""
|
|
row = get_client(tenant_id, client_id)
|
|
return row['account_id'] if row else None
|
|
|
|
|
|
def client_exists(tenant_id, client_id):
|
|
return get_client(tenant_id, client_id) is not None
|
|
|
|
|
|
def client_is_connected(tenant_id, client_id):
|
|
row = get_client(tenant_id, client_id)
|
|
return bool(row and row['disconnected_at'] is None)
|
|
|
|
|
|
def ensure_client(tenant_id, client_id, account_id, worker_id=None):
|
|
"""Registriert eine Client-ID bzw. aktualisiert ihren Verbindungsstatus.
|
|
|
|
Gibt ``False`` zurück, wenn die Client-ID bereits einem anderen Account
|
|
gehört. Ein solcher Wechsel wäre ein Hinweis auf eine manipulierte
|
|
Zuordnung und wird vom Broker abgelehnt.
|
|
"""
|
|
db = get_db()
|
|
row = get_client(tenant_id, client_id)
|
|
current = now()
|
|
|
|
if row is not None:
|
|
if row['account_id'] != account_id:
|
|
return False
|
|
db.execute(
|
|
'UPDATE ws_clients SET worker_id = ?, connected_at = ?, '
|
|
'disconnected_at = NULL, authenticated = 0 '
|
|
'WHERE tenant_id = ? AND client_id = ?',
|
|
(worker_id, current, tenant_id, client_id)
|
|
)
|
|
db.commit()
|
|
return True
|
|
|
|
db.execute(
|
|
'INSERT INTO ws_clients '
|
|
'(tenant_id, client_id, account_id, worker_id, connected_at, '
|
|
' authenticated, last_seq) '
|
|
'VALUES (?, ?, ?, ?, ?, 0, 0)',
|
|
(tenant_id, client_id, account_id, worker_id, current)
|
|
)
|
|
db.commit()
|
|
return True
|
|
|
|
|
|
def mark_authenticated(tenant_id, client_id, authenticated=True, worker_id=None):
|
|
db = get_db()
|
|
if worker_id is None:
|
|
db.execute(
|
|
'UPDATE ws_clients SET authenticated = ? '
|
|
'WHERE tenant_id = ? AND client_id = ?',
|
|
(1 if authenticated else 0, tenant_id, client_id)
|
|
)
|
|
else:
|
|
# Nur der Worker, dem die Client-ID aktuell zugeordnet ist, darf den
|
|
# Authentifizierungsstatus setzen. So überschreibt ein verspäteter
|
|
# Handshake eines alten Workers keinen Reconnect auf einem anderen
|
|
# Worker.
|
|
db.execute(
|
|
'UPDATE ws_clients SET authenticated = ? '
|
|
'WHERE tenant_id = ? AND client_id = ? '
|
|
'AND (worker_id IS NULL OR worker_id = ?)',
|
|
(1 if authenticated else 0, tenant_id, client_id, worker_id)
|
|
)
|
|
db.commit()
|
|
|
|
|
|
def mark_disconnected(tenant_id, client_id, worker_id=None):
|
|
"""Markiert eine Client-ID als getrennt, ohne ihre Queue zu löschen.
|
|
|
|
Mit ``worker_id`` wird nur die Zuordnung dieses Workers getrennt. Hat
|
|
inzwischen ein anderer Worker dieselbe Client-ID übernommen, bleibt dessen
|
|
Verbindungsstatus unberührt.
|
|
"""
|
|
db = get_db()
|
|
if worker_id is None:
|
|
db.execute(
|
|
'UPDATE ws_clients SET worker_id = NULL, disconnected_at = ?, '
|
|
'authenticated = 0 WHERE tenant_id = ? AND client_id = ?',
|
|
(now(), tenant_id, client_id)
|
|
)
|
|
else:
|
|
db.execute(
|
|
'UPDATE ws_clients SET worker_id = NULL, disconnected_at = ?, '
|
|
'authenticated = 0 WHERE tenant_id = ? AND client_id = ? '
|
|
'AND (worker_id IS NULL OR worker_id = ?)',
|
|
(now(), tenant_id, client_id, worker_id)
|
|
)
|
|
db.commit()
|
|
|
|
|
|
def delete_client(tenant_id, client_id, worker_id=None):
|
|
"""Entfernt eine Client-ID samt Queue und Sequenzzustand.
|
|
|
|
Mit ``worker_id`` wird nur gelöscht, solange die Client-ID noch diesem
|
|
Worker gehört. Das verhindert, dass ein scheidender Worker einen
|
|
zwischenzeitlichen Reconnect auf einem anderen Worker entfernt.
|
|
"""
|
|
db = get_db()
|
|
if worker_id is None:
|
|
cursor = db.execute(
|
|
'DELETE FROM ws_clients WHERE tenant_id = ? AND client_id = ?',
|
|
(tenant_id, client_id)
|
|
)
|
|
else:
|
|
cursor = db.execute(
|
|
'DELETE FROM ws_clients WHERE tenant_id = ? AND client_id = ? '
|
|
'AND (worker_id IS NULL OR worker_id = ?)',
|
|
(tenant_id, client_id, worker_id)
|
|
)
|
|
if cursor.rowcount:
|
|
db.execute(
|
|
'DELETE FROM ws_client_queue WHERE tenant_id = ? AND client_id = ?',
|
|
(tenant_id, client_id)
|
|
)
|
|
db.commit()
|
|
|
|
|
|
def get_last_seq(tenant_id, client_id):
|
|
row = get_client(tenant_id, client_id)
|
|
return int(row['last_seq'] or 0) if row else 0
|
|
|
|
|
|
def set_last_seq(tenant_id, client_id, seq, account_id=None):
|
|
db = get_db()
|
|
cursor = db.execute(
|
|
'UPDATE ws_clients SET last_seq = ? WHERE tenant_id = ? AND client_id = ?',
|
|
(seq, tenant_id, client_id)
|
|
)
|
|
if cursor.rowcount:
|
|
db.commit()
|
|
return
|
|
# Die Client-Zeile kann durch einen Cleanup verschwunden sein. In diesem
|
|
# Fall wird sie mit der bekannten Account-ID neu angelegt, damit der
|
|
# Sequenzstand nicht verloren geht.
|
|
if account_id is not None:
|
|
db.execute(
|
|
'INSERT OR IGNORE INTO ws_clients '
|
|
'(tenant_id, client_id, account_id, last_seq) VALUES (?, ?, ?, ?)',
|
|
(tenant_id, client_id, account_id, seq)
|
|
)
|
|
db.commit()
|
|
|
|
|
|
def client_ids_for_accounts(tenant_id, account_ids):
|
|
"""Expandiert Empfänger-Accounts auf alle bekannten Client-IDs.
|
|
|
|
Anders als früher werden auch Clients berücksichtigt, die an einem
|
|
anderen Worker verbunden oder momentan getrennt sind. Nur so können
|
|
Nachrichten für einen Reconnect auf einem beliebigen Worker gepuffert
|
|
werden.
|
|
"""
|
|
account_ids = list(dict.fromkeys(account_ids))
|
|
if not account_ids:
|
|
return set()
|
|
|
|
db = get_db()
|
|
result = set()
|
|
chunk_size = 400
|
|
for start in range(0, len(account_ids), chunk_size):
|
|
chunk = account_ids[start:start + chunk_size]
|
|
placeholders = ','.join('?' for _ in chunk)
|
|
rows = db.execute(
|
|
f'SELECT client_id FROM ws_clients '
|
|
f'WHERE tenant_id = ? AND account_id IN ({placeholders})',
|
|
(tenant_id, *chunk)
|
|
).fetchall()
|
|
result.update(row['client_id'] for row in rows)
|
|
return result
|
|
|
|
|
|
def describe_connections(tenant_id):
|
|
"""Liefert alle aktuell verbundenen Clients inklusive Account und Gruppe."""
|
|
rows = get_db().execute(
|
|
'''SELECT c.client_id, c.account_id, c.authenticated,
|
|
a.name AS account_name, a.role,
|
|
g.id AS group_id, g.aliasname AS group_alias
|
|
FROM ws_clients c
|
|
JOIN accounts a
|
|
ON a.id = c.account_id AND a.tenant_id = c.tenant_id
|
|
LEFT JOIN group_accounts ga
|
|
ON ga.tenant_id = c.tenant_id AND ga.account_id = c.account_id
|
|
LEFT JOIN groups g
|
|
ON g.id = ga.group_id AND g.tenant_id = c.tenant_id
|
|
WHERE c.tenant_id = ? AND c.disconnected_at IS NULL
|
|
ORDER BY c.client_id''',
|
|
(tenant_id,)
|
|
).fetchall()
|
|
return [dict(row) for row in rows]
|
|
|
|
|
|
# ----------------------------------------------------------------------
|
|
# Client-Queues
|
|
# ----------------------------------------------------------------------
|
|
def get_queue_started_at(tenant_id, client_id):
|
|
row = get_client(tenant_id, client_id)
|
|
return row['queue_started_at'] if row else None
|
|
|
|
|
|
def _set_queue_started_at(db, tenant_id, client_id, started_at):
|
|
db.execute(
|
|
'UPDATE ws_clients SET queue_started_at = ? '
|
|
'WHERE tenant_id = ? AND client_id = ?',
|
|
(started_at, tenant_id, client_id)
|
|
)
|
|
|
|
|
|
def client_has_queue(tenant_id, client_id):
|
|
"""Prüft, ob für einen Client noch zustellbare Aufträge warten."""
|
|
row = get_db().execute(
|
|
'SELECT 1 FROM ws_client_queue '
|
|
'WHERE tenant_id = ? AND client_id = ? '
|
|
'AND (expires_at IS NULL OR expires_at > ?) LIMIT 1',
|
|
(tenant_id, client_id, now())
|
|
).fetchone()
|
|
return row is not None
|
|
|
|
|
|
def enqueue_message(tenant_id, client_id, code, payload, max_age_seconds,
|
|
timeout_payload=None, ephemeral_seconds=None):
|
|
"""Legt einen Zustellauftrag für einen Client an.
|
|
|
|
``max_age_seconds`` steuert die Lebensdauer einer normalen Queue.
|
|
``timeout_payload`` wird als Code-99-Tombstone eingesetzt, wenn die Queue
|
|
zu alt ist oder eine Code-99-Nachricht alle älteren Aufträge verdrängt.
|
|
``ephemeral_seconds`` gilt für Code 2: Der Auftrag verfällt schnell und
|
|
wird nicht für getrennte Clients vorgehalten.
|
|
"""
|
|
db = get_db()
|
|
current = now()
|
|
row = db.execute(
|
|
'SELECT queue_started_at FROM ws_clients '
|
|
'WHERE tenant_id = ? AND client_id = ?',
|
|
(tenant_id, client_id)
|
|
).fetchone()
|
|
if row is None:
|
|
# Unbekannte Client-IDs werden nicht gepuffert. Der Broker legt die
|
|
# Zeile beim Code-0-Handshake an.
|
|
return False
|
|
|
|
started_at = row['queue_started_at']
|
|
if started_at is None:
|
|
started_at = current
|
|
|
|
# Eine Code-99-Nachricht verdrängt alle älteren Aufträge und blockiert
|
|
# weitere Nachrichten, bis sie zugestellt wurde.
|
|
if code != 99:
|
|
tombstone = db.execute(
|
|
'SELECT 1 FROM ws_client_queue '
|
|
'WHERE tenant_id = ? AND client_id = ? AND code99_only = 1 LIMIT 1',
|
|
(tenant_id, client_id)
|
|
).fetchone()
|
|
if tombstone:
|
|
return False
|
|
|
|
# Abgelaufene Queue: Inhalt verwerfen und einen Code-99-Hinweis hinterlassen.
|
|
if current - started_at >= max_age_seconds:
|
|
db.execute(
|
|
'DELETE FROM ws_client_queue WHERE tenant_id = ? AND client_id = ?',
|
|
(tenant_id, client_id)
|
|
)
|
|
if timeout_payload is not None:
|
|
db.execute(
|
|
'INSERT INTO ws_client_queue '
|
|
'(tenant_id, client_id, code, payload, code99_only, created_at) '
|
|
'VALUES (?, ?, 99, ?, 1, ?)',
|
|
(tenant_id, client_id, timeout_payload, current)
|
|
)
|
|
_set_queue_started_at(db, tenant_id, client_id, current)
|
|
else:
|
|
_set_queue_started_at(db, tenant_id, client_id, None)
|
|
db.commit()
|
|
return True
|
|
|
|
if code == 99:
|
|
db.execute(
|
|
'DELETE FROM ws_client_queue WHERE tenant_id = ? AND client_id = ?',
|
|
(tenant_id, client_id)
|
|
)
|
|
db.execute(
|
|
'INSERT INTO ws_client_queue '
|
|
'(tenant_id, client_id, code, payload, code99_only, created_at) '
|
|
'VALUES (?, ?, 99, ?, 1, ?)',
|
|
(tenant_id, client_id, payload, current)
|
|
)
|
|
_set_queue_started_at(db, tenant_id, client_id, current)
|
|
db.commit()
|
|
return True
|
|
|
|
expires_at = None
|
|
if ephemeral_seconds is not None:
|
|
expires_at = current + ephemeral_seconds
|
|
|
|
db.execute(
|
|
'INSERT INTO ws_client_queue '
|
|
'(tenant_id, client_id, code, payload, code99_only, created_at, expires_at) '
|
|
'VALUES (?, ?, ?, ?, 0, ?, ?)',
|
|
(tenant_id, client_id, code, payload, current, expires_at)
|
|
)
|
|
_set_queue_started_at(db, tenant_id, client_id, started_at)
|
|
db.commit()
|
|
return True
|
|
|
|
|
|
def fetch_queue(tenant_id, client_id):
|
|
"""Liefert alle noch gültigen Aufträge eines Clients in Reihenfolge."""
|
|
return get_db().execute(
|
|
'SELECT * FROM ws_client_queue '
|
|
'WHERE tenant_id = ? AND client_id = ? '
|
|
'AND (expires_at IS NULL OR expires_at > ?) '
|
|
'ORDER BY id ASC',
|
|
(tenant_id, client_id, now())
|
|
).fetchall()
|
|
|
|
|
|
def clients_with_queue(pairs):
|
|
"""Liefert diejenigen ``(tenant_id, client_id)``, die Aufträge haben."""
|
|
if not pairs:
|
|
return set()
|
|
|
|
db = get_db()
|
|
by_tenant = {}
|
|
for tenant_id, client_id in pairs:
|
|
by_tenant.setdefault(tenant_id, set()).add(client_id)
|
|
|
|
result = set()
|
|
chunk_size = 400
|
|
for tenant_id, client_ids in by_tenant.items():
|
|
ids = list(client_ids)
|
|
for start in range(0, len(ids), chunk_size):
|
|
chunk = ids[start:start + chunk_size]
|
|
placeholders = ','.join('?' for _ in chunk)
|
|
rows = db.execute(
|
|
f'SELECT DISTINCT client_id FROM ws_client_queue '
|
|
f'WHERE tenant_id = ? AND client_id IN ({placeholders}) '
|
|
f'AND (expires_at IS NULL OR expires_at > ?)',
|
|
(tenant_id, *chunk, now())
|
|
).fetchall()
|
|
result.update((tenant_id, row['client_id']) for row in rows)
|
|
return result
|
|
|
|
|
|
def delete_queue_row(row_id):
|
|
"""Löscht einen zugestellten Auftrag und räumt den Queue-Start auf."""
|
|
db = get_db()
|
|
row = db.execute(
|
|
'SELECT tenant_id, client_id FROM ws_client_queue WHERE id = ?',
|
|
(row_id,)
|
|
).fetchone()
|
|
if row is None:
|
|
return
|
|
db.execute('DELETE FROM ws_client_queue WHERE id = ?', (row_id,))
|
|
remaining = db.execute(
|
|
'SELECT 1 FROM ws_client_queue '
|
|
'WHERE tenant_id = ? AND client_id = ? '
|
|
'AND (expires_at IS NULL OR expires_at > ?) LIMIT 1',
|
|
(row['tenant_id'], row['client_id'], now())
|
|
).fetchone()
|
|
if remaining is None:
|
|
_set_queue_started_at(db, row['tenant_id'], row['client_id'], None)
|
|
db.commit()
|
|
|
|
|
|
def replace_queue_with_timeout(tenant_id, client_id, payload):
|
|
"""Ersetzt eine abgelaufene Queue durch einen Code-99-Hinweis."""
|
|
db = get_db()
|
|
current = now()
|
|
db.execute(
|
|
'DELETE FROM ws_client_queue WHERE tenant_id = ? AND client_id = ?',
|
|
(tenant_id, client_id)
|
|
)
|
|
db.execute(
|
|
'INSERT INTO ws_client_queue '
|
|
'(tenant_id, client_id, code, payload, code99_only, created_at) '
|
|
'VALUES (?, ?, 99, ?, 1, ?)',
|
|
(tenant_id, client_id, payload, current)
|
|
)
|
|
_set_queue_started_at(db, tenant_id, client_id, current)
|
|
db.commit()
|
|
|
|
|
|
def expire_queues(max_age_seconds, timeout_payload):
|
|
"""Wandelt zu alte Queues in Code-99-Tombstones um.
|
|
|
|
Die Funktion ist idempotent und kann von jedem Worker regelmäßig
|
|
aufgerufen werden. Nach dem ersten Lauf ist ``queue_started_at``
|
|
aktualisiert, sodass weitere Worker die Queue nicht erneut umwandeln.
|
|
"""
|
|
db = get_db()
|
|
cutoff = now() - max_age_seconds
|
|
rows = db.execute(
|
|
'SELECT tenant_id, client_id FROM ws_clients '
|
|
'WHERE queue_started_at IS NOT NULL AND queue_started_at <= ?',
|
|
(cutoff,)
|
|
).fetchall()
|
|
|
|
current = now()
|
|
for row in rows:
|
|
tenant_id = row['tenant_id']
|
|
client_id = row['client_id']
|
|
db.execute(
|
|
'DELETE FROM ws_client_queue WHERE tenant_id = ? AND client_id = ?',
|
|
(tenant_id, client_id)
|
|
)
|
|
db.execute(
|
|
'INSERT INTO ws_client_queue '
|
|
'(tenant_id, client_id, code, payload, code99_only, created_at) '
|
|
'VALUES (?, ?, 99, ?, 1, ?)',
|
|
(tenant_id, client_id, timeout_payload, current)
|
|
)
|
|
_set_queue_started_at(db, tenant_id, client_id, current)
|
|
if rows:
|
|
db.commit()
|
|
|
|
|
|
def cleanup_clients(retention_seconds):
|
|
"""Entfernt dauerhaft getrennte Clients ohne zustellbare Restnachrichten."""
|
|
db = get_db()
|
|
cutoff = now() - retention_seconds
|
|
rows = db.execute(
|
|
'SELECT tenant_id, client_id FROM ws_clients '
|
|
'WHERE disconnected_at IS NOT NULL AND disconnected_at < ?',
|
|
(cutoff,)
|
|
).fetchall()
|
|
|
|
for row in rows:
|
|
tenant_id = row['tenant_id']
|
|
client_id = row['client_id']
|
|
pending = db.execute(
|
|
'SELECT 1 FROM ws_client_queue '
|
|
'WHERE tenant_id = ? AND client_id = ? AND code99_only = 0 '
|
|
'AND (expires_at IS NULL OR expires_at > ?) LIMIT 1',
|
|
(tenant_id, client_id, now())
|
|
).fetchone()
|
|
if pending is not None:
|
|
# Noch zustellbare Nachrichten halten den Client am Leben.
|
|
continue
|
|
db.execute(
|
|
'DELETE FROM ws_client_queue WHERE tenant_id = ? AND client_id = ?',
|
|
(tenant_id, client_id)
|
|
)
|
|
db.execute(
|
|
'DELETE FROM ws_clients WHERE tenant_id = ? AND client_id = ?',
|
|
(tenant_id, client_id)
|
|
)
|
|
if rows:
|
|
db.commit()
|
|
|
|
|
|
def purge_expired_queue_rows():
|
|
"""Löscht abgelaufene ephemere Aufträge (z. B. Code 2)."""
|
|
db = get_db()
|
|
db.execute(
|
|
'DELETE FROM ws_client_queue WHERE expires_at IS NOT NULL AND expires_at <= ?',
|
|
(now(),)
|
|
)
|
|
db.commit()
|
|
|
|
|
|
# ----------------------------------------------------------------------
|
|
# Gruppenpräsenz
|
|
# ----------------------------------------------------------------------
|
|
def get_group_presence(tenant_id, group_id):
|
|
return get_db().execute(
|
|
'SELECT * FROM ws_group_presence WHERE tenant_id = ? AND group_id = ?',
|
|
(tenant_id, group_id)
|
|
).fetchone()
|
|
|
|
|
|
def record_group_pong(tenant_id, group_id, timestamp=None):
|
|
db = get_db()
|
|
current = timestamp if timestamp is not None else now()
|
|
db.execute(
|
|
'INSERT OR IGNORE INTO ws_group_presence (tenant_id, group_id) VALUES (?, ?)',
|
|
(tenant_id, group_id)
|
|
)
|
|
db.execute(
|
|
'UPDATE ws_group_presence SET last_pong_at = ?, updated_at = CURRENT_TIMESTAMP '
|
|
'WHERE tenant_id = ? AND group_id = ?',
|
|
(current, tenant_id, group_id)
|
|
)
|
|
db.commit()
|
|
|
|
|
|
def set_group_online_since(tenant_id, group_id, timestamp=None):
|
|
db = get_db()
|
|
current = timestamp if timestamp is not None else now()
|
|
db.execute(
|
|
'INSERT OR IGNORE INTO ws_group_presence (tenant_id, group_id) VALUES (?, ?)',
|
|
(tenant_id, group_id)
|
|
)
|
|
db.execute(
|
|
'UPDATE ws_group_presence SET online_since = ?, updated_at = CURRENT_TIMESTAMP '
|
|
'WHERE tenant_id = ? AND group_id = ?',
|
|
(current, tenant_id, group_id)
|
|
)
|
|
db.commit()
|