Broker unterstützt jetzt mehrere WSGI Worker
This commit is contained in:
1 parent
1e2a58e563
commit
f5b0c9509c
13 files changed
+1252
-327
No files matched your search
@@ -156,6 +156,12 @@ def create_app(config_name=None):
|
||||
|
||||
# WebSocket Routes laden
|
||||
from cmbnet.ws import routes
|
||||
|
||||
# Optionale feste Worker-ID aus der Konfiguration übernehmen. Der Broker
|
||||
# liest sonst CMBNET_WS_WORKER_ID bzw. bildet <hostname>:<pid>.
|
||||
from cmbnet.ws import broker as ws_broker
|
||||
if app.config.get('WS_WORKER_ID'):
|
||||
ws_broker.worker_id = app.config['WS_WORKER_ID']
|
||||
|
||||
# Heartbeat starten (Code 1 jede Minute, Aufräumen alter Nachrichten)
|
||||
from cmbnet.ws.heartbeat import heartbeat_manager
|
||||
@@ -163,6 +169,16 @@ def create_app(config_name=None):
|
||||
app,
|
||||
interval=app.config.get('WS_HEARTBEAT_INTERVAL', 60)
|
||||
)
|
||||
|
||||
# Worker-lokale Schleife für die gemeinsame Zustell-Queue. Jeder
|
||||
# Gunicorn-/WSGI-Worker muss eine eigene Instanz starten. Bei
|
||||
# ``preload_app=True`` würde sie im Master starten und in den Workern
|
||||
# fehlen; deshalb ist der Betrieb ohne Preload erforderlich.
|
||||
from cmbnet.ws.delivery import delivery_worker
|
||||
delivery_worker.start(
|
||||
app,
|
||||
interval=app.config.get('WS_BUS_POLL_INTERVAL', 0.5)
|
||||
)
|
||||
|
||||
# Favicon explizit bedienen, damit /favicon.ico nicht als Tenant-Pfad
|
||||
# interpretiert wird.
|
||||
|
||||
+17
-31
@@ -356,38 +356,24 @@ def import_streets(**kwargs):
|
||||
@admin_api_bp.route('/broker/status')
|
||||
@role_required('Administrator')
|
||||
def broker_status(**kwargs):
|
||||
"""Status aktiver WebSocket-Verbindungen"""
|
||||
connections = broker.connections.get(g.tenant_id, {}) if hasattr(broker, 'connections') else {}
|
||||
"""Status aktiver WebSocket-Verbindungen aller Worker.
|
||||
|
||||
Die Verbindungsliste liegt in der gemeinsamen Tabelle ``ws_clients``.
|
||||
Dadurch zeigt das Dashboard auch Clients, die an einem anderen
|
||||
WSGI-Worker hängen.
|
||||
"""
|
||||
connections = broker.describe_connections(g.tenant_id)
|
||||
result = []
|
||||
for client_id, conn in connections.items():
|
||||
account_id = conn.get('account_id')
|
||||
account = get_db().execute(
|
||||
'SELECT id, name, role FROM accounts WHERE id = ? AND tenant_id = ?',
|
||||
(account_id, g.tenant_id)
|
||||
).fetchone()
|
||||
if not account:
|
||||
continue
|
||||
entry = {
|
||||
'client_id': client_id,
|
||||
'account_id': account['id'],
|
||||
'account_name': account['name'],
|
||||
'role': account['role'],
|
||||
'authenticated': broker.is_authenticated(g.tenant_id, client_id),
|
||||
'group_id': None,
|
||||
'group_alias': None
|
||||
}
|
||||
if account['role'] == 'Gruppe':
|
||||
group = get_db().execute(
|
||||
'''SELECT g.id, g.aliasname FROM groups g
|
||||
JOIN group_accounts ga
|
||||
ON ga.group_id = g.id AND ga.tenant_id = g.tenant_id
|
||||
WHERE ga.account_id = ? AND g.tenant_id = ?''',
|
||||
(account['id'], g.tenant_id)
|
||||
).fetchone()
|
||||
if group:
|
||||
entry['group_id'] = group['id']
|
||||
entry['group_alias'] = group['aliasname']
|
||||
result.append(entry)
|
||||
for conn in connections:
|
||||
result.append({
|
||||
'client_id': conn['client_id'],
|
||||
'account_id': conn['account_id'],
|
||||
'account_name': conn['account_name'],
|
||||
'role': conn['role'],
|
||||
'authenticated': bool(conn['authenticated']),
|
||||
'group_id': conn['group_id'],
|
||||
'group_alias': conn['group_alias']
|
||||
})
|
||||
return jsonify({'connections': result})
|
||||
|
||||
|
||||
|
||||
@@ -11,8 +11,16 @@ def get_db():
|
||||
db_path = current_app.config['DATABASE_PATH']
|
||||
# Stelle sicher, dass das instance Verzeichnis existiert
|
||||
Path(db_path).parent.mkdir(parents=True, exist_ok=True)
|
||||
g.db = sqlite3.connect(db_path)
|
||||
g.db = sqlite3.connect(db_path, timeout=30)
|
||||
g.db.row_factory = sqlite3.Row
|
||||
# Mehrere WSGI-Worker schreiben gleichzeitig in dieselbe SQLite-Datei.
|
||||
# busy_timeout zuerst setzen, damit auch das Umschalten auf WAL bei
|
||||
# parallelen Verbindungen warten kann statt sofort zu scheitern.
|
||||
# WAL erlaubt parallele Leser und reduziert die Blockierzeit bei
|
||||
# Schreibern.
|
||||
g.db.execute('PRAGMA busy_timeout = 30000')
|
||||
g.db.execute('PRAGMA journal_mode = WAL')
|
||||
g.db.execute('PRAGMA synchronous = NORMAL')
|
||||
return g.db
|
||||
|
||||
|
||||
@@ -23,6 +31,65 @@ def close_db(e=None):
|
||||
db.close()
|
||||
|
||||
|
||||
def ensure_ws_schema(db):
|
||||
"""Legt die Tabellen für den prozessübergreifenden Broker-Zustand an."""
|
||||
db.executescript('''
|
||||
CREATE TABLE IF NOT EXISTS ws_clients (
|
||||
tenant_id INTEGER NOT NULL,
|
||||
client_id TEXT NOT NULL,
|
||||
account_id INTEGER NOT NULL,
|
||||
last_seq INTEGER NOT NULL DEFAULT 0,
|
||||
worker_id TEXT,
|
||||
authenticated INTEGER NOT NULL DEFAULT 0,
|
||||
connected_at REAL,
|
||||
disconnected_at REAL,
|
||||
queue_started_at REAL,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (tenant_id, client_id),
|
||||
FOREIGN KEY (tenant_id) REFERENCES tenants(id),
|
||||
FOREIGN KEY (account_id) REFERENCES accounts(id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ws_client_queue (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
tenant_id INTEGER NOT NULL,
|
||||
client_id TEXT NOT NULL,
|
||||
code INTEGER NOT NULL,
|
||||
payload TEXT NOT NULL,
|
||||
code99_only INTEGER NOT NULL DEFAULT 0,
|
||||
created_at REAL NOT NULL,
|
||||
expires_at REAL,
|
||||
FOREIGN KEY (tenant_id) REFERENCES tenants(id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ws_group_presence (
|
||||
tenant_id INTEGER NOT NULL,
|
||||
group_id INTEGER NOT NULL,
|
||||
last_pong_at REAL,
|
||||
online_since REAL,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (tenant_id, group_id),
|
||||
FOREIGN KEY (tenant_id) REFERENCES tenants(id),
|
||||
FOREIGN KEY (group_id) REFERENCES groups(id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS cache_state (
|
||||
key TEXT PRIMARY KEY,
|
||||
revision INTEGER NOT NULL DEFAULT 0,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_ws_clients_account
|
||||
ON ws_clients(tenant_id, account_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_ws_client_queue_client
|
||||
ON ws_client_queue(tenant_id, client_id, id);
|
||||
CREATE INDEX IF NOT EXISTS idx_ws_client_queue_expires
|
||||
ON ws_client_queue(expires_at);
|
||||
''')
|
||||
db.commit()
|
||||
|
||||
|
||||
def ensure_schema():
|
||||
"""Ergänzt bestehende Datenbanken um das Gruppen-Account-Zuordnungsmodell.
|
||||
|
||||
@@ -32,6 +99,7 @@ def ensure_schema():
|
||||
und die alte Spalte durch einen Tabellen-Neubau entfernt.
|
||||
"""
|
||||
db = get_db()
|
||||
ensure_ws_schema(db)
|
||||
groups_table = db.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'groups'"
|
||||
).fetchone()
|
||||
|
||||
@@ -10,10 +10,61 @@ from cmbnet.db import get_db
|
||||
class GroupsService:
|
||||
# In-Memory Cache: groups_cache[tenant_id][group_id] = group_dict
|
||||
_cache = {}
|
||||
|
||||
# Gemeinsame Cache-Revision je Mandant. Sie liegt in der Tabelle
|
||||
# ``cache_state`` und wird bei jeder schreibenden Änderung erhöht. Dadurch
|
||||
# verwirft jeder Worker seinen lokalen Cache, sobald ein anderer Worker
|
||||
# Gruppen- oder Zuordnungsdaten geändert hat.
|
||||
_cache_revision = {}
|
||||
|
||||
@classmethod
|
||||
def _revision_key(cls, tenant_id):
|
||||
return f'groups:{tenant_id}'
|
||||
|
||||
@classmethod
|
||||
def _current_revision(cls, tenant_id):
|
||||
db = get_db()
|
||||
key = cls._revision_key(tenant_id)
|
||||
row = db.execute(
|
||||
'SELECT revision FROM cache_state WHERE key = ?', (key,)
|
||||
).fetchone()
|
||||
if row is None:
|
||||
db.execute(
|
||||
'INSERT OR IGNORE INTO cache_state (key, revision) VALUES (?, 0)',
|
||||
(key,)
|
||||
)
|
||||
db.commit()
|
||||
return 0
|
||||
return int(row['revision'] or 0)
|
||||
|
||||
@classmethod
|
||||
def _bump_revision(cls, tenant_id):
|
||||
"""Erhöht die gemeinsame Revision und gibt den neuen Wert zurück."""
|
||||
db = get_db()
|
||||
key = cls._revision_key(tenant_id)
|
||||
db.execute(
|
||||
'INSERT OR IGNORE INTO cache_state (key, revision) VALUES (?, 0)',
|
||||
(key,)
|
||||
)
|
||||
db.execute(
|
||||
'UPDATE cache_state SET revision = revision + 1, '
|
||||
'updated_at = CURRENT_TIMESTAMP WHERE key = ?',
|
||||
(key,)
|
||||
)
|
||||
row = db.execute(
|
||||
'SELECT revision FROM cache_state WHERE key = ?', (key,)
|
||||
).fetchone()
|
||||
db.commit()
|
||||
return int(row['revision'] or 0)
|
||||
|
||||
@classmethod
|
||||
def load_from_db(cls, tenant_id):
|
||||
"""Lädt alle Gruppen eines Mandanten aus der Datenbank in den Cache"""
|
||||
# Revision vor dem Lesen der Daten ermitteln. Schreibt ein anderer
|
||||
# Worker währenddessen, bleibt die gespeicherte Revision älter als die
|
||||
# gemeinsame Revision und der Cache wird beim nächsten Zugriff erneut
|
||||
# geladen. Die umgekehrte Reihenfolge könnte dagegen veraltete Daten
|
||||
# dauerhaft mit einer aktuellen Revision verknüpfen.
|
||||
revision = cls._current_revision(tenant_id)
|
||||
groups = GroupModel.get_all_by_tenant(tenant_id)
|
||||
memberships = GroupModel.get_account_ids_by_group(tenant_id)
|
||||
# Vollständig neu aufbauen, damit gelöschte Gruppen nicht im Cache
|
||||
@@ -33,32 +84,47 @@ class GroupsService:
|
||||
'created_at': group['created_at'],
|
||||
'updated_at': group['updated_at']
|
||||
}
|
||||
cls._cache_revision[tenant_id] = revision
|
||||
return cls._cache[tenant_id]
|
||||
|
||||
@classmethod
|
||||
def get_all(cls, tenant_id, force_reload=False):
|
||||
"""Gibt alle Gruppen für einen Mandanten zurück (aus Cache)"""
|
||||
if force_reload or tenant_id not in cls._cache or not cls._cache[tenant_id]:
|
||||
revision = cls._current_revision(tenant_id)
|
||||
if (
|
||||
force_reload
|
||||
or tenant_id not in cls._cache
|
||||
or not cls._cache[tenant_id]
|
||||
or cls._cache_revision.get(tenant_id) != revision
|
||||
):
|
||||
cls.load_from_db(tenant_id)
|
||||
return cls._cache.get(tenant_id, {})
|
||||
|
||||
@classmethod
|
||||
def get_by_id(cls, tenant_id, group_id):
|
||||
"""Gibt eine spezifische Gruppe aus dem Cache zurück"""
|
||||
if tenant_id not in cls._cache or not cls._cache[tenant_id]:
|
||||
revision = cls._current_revision(tenant_id)
|
||||
if (
|
||||
tenant_id not in cls._cache
|
||||
or not cls._cache[tenant_id]
|
||||
or cls._cache_revision.get(tenant_id) != revision
|
||||
):
|
||||
cls.load_from_db(tenant_id)
|
||||
return cls._cache.get(tenant_id, {}).get(group_id)
|
||||
|
||||
@classmethod
|
||||
def invalidate(cls, tenant_id):
|
||||
"""Invalidiert den Cache für einen Mandanten (erzwingt Neuladen)"""
|
||||
"""Invalidiert den Cache für einen Mandanten (erzwingt Neuladen)."""
|
||||
if tenant_id in cls._cache:
|
||||
cls._cache[tenant_id] = {}
|
||||
cls._cache_revision.pop(tenant_id, None)
|
||||
cls._bump_revision(tenant_id)
|
||||
|
||||
@classmethod
|
||||
def invalidate_all(cls):
|
||||
"""Invalidiert den gesamten Cache"""
|
||||
"""Invalidiert den gesamten Cache."""
|
||||
cls._cache = {}
|
||||
cls._cache_revision = {}
|
||||
|
||||
@classmethod
|
||||
def is_account_linked(cls, account_id, group_id, tenant_id):
|
||||
@@ -87,14 +153,20 @@ class GroupsService:
|
||||
|
||||
@classmethod
|
||||
def update_cache_entry(cls, tenant_id, group_id, data):
|
||||
"""Aktualisiert einen Eintrag im Cache.
|
||||
"""Aktualisiert einen Eintrag im Cache und invalidiert andere Worker.
|
||||
|
||||
Der Cache wird vor dem Aktualisieren vollständig geladen, damit ein
|
||||
gezieltes Update (z. B. nach Code 200/300) keinen Teil-Cache erzeugt,
|
||||
der anschließend beim Initialload unvollständige Gruppendaten liefert.
|
||||
Anschließend wird die gemeinsame Revision erhöht, damit andere Worker
|
||||
ihre lokalen Caches verwerfen. Der lokale Cache wird bewusst nicht auf
|
||||
die neue Revision gesetzt: Beim nächsten Zugriff wird er neu geladen
|
||||
und kann so keine Änderungen verpassen, die ein anderer Worker zwischen
|
||||
dem Laden und dem Erhöhen der Revision geschrieben hat.
|
||||
"""
|
||||
if tenant_id not in cls._cache or not cls._cache[tenant_id]:
|
||||
cls.load_from_db(tenant_id)
|
||||
cls.get_all(tenant_id)
|
||||
if group_id not in cls._cache[tenant_id]:
|
||||
cls._cache[tenant_id][group_id] = {}
|
||||
cls._cache[tenant_id][group_id].update(data)
|
||||
cls._bump_revision(tenant_id)
|
||||
cls._cache_revision.pop(tenant_id, None)
|
||||
+226
-209
@@ -23,14 +23,17 @@ Wichtige Regeln aus der Spezifikation:
|
||||
Zeitfenster wieder zurück auf "unterwegs".
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import socket
|
||||
import time
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from threading import Lock
|
||||
from threading import Event, Lock
|
||||
from flask import current_app
|
||||
from cmbnet.db import get_db
|
||||
from cmbnet.services.groups_service import GroupsService
|
||||
from cmbnet.ws import store
|
||||
from cmbnet.ws.protocol import WSMessage
|
||||
|
||||
|
||||
@@ -43,41 +46,35 @@ class WebSocketBroker:
|
||||
# 'group_id': <verknüpfte Gruppe oder None>,
|
||||
# }
|
||||
# Eine Client-ID gehört genau zu einer Webview. Mehrere Clients
|
||||
# desselben Accounts besitzen daher getrennte Einträge.
|
||||
# desselben Accounts besitzen daher getrennte Einträge. Dieses Dict
|
||||
# enthält bewusst nur die Verbindungen des aktuellen Worker-Prozesses.
|
||||
self.connections = {}
|
||||
# authenticated[tenant_id][client_id] = True
|
||||
# Erst nach erfolgreichem Code-0-Handshake darf ein Client Nachrichten
|
||||
# über die Verbindung senden.
|
||||
# über die Verbindung senden. Auch dieser Status ist prozesslokal.
|
||||
self.authenticated = {}
|
||||
|
||||
# Nicht zustellbare Nachrichten werden pro Client gepuffert.
|
||||
# Schlüssel: (tenant_id, client_id).
|
||||
self.client_queues = {}
|
||||
# client_accounts[(tenant_id, client_id)] = account_id
|
||||
# Bleibt auch nach einem Verbindungsabbruch erhalten, damit weitere
|
||||
# Nachrichten für denselben Client gepuffert werden können.
|
||||
self.client_accounts = {}
|
||||
# account_clients[(tenant_id, account_id)] = {client_id, ...}
|
||||
# Wird für die Empfängerermittlung benötigt: Ein Account kann mehrere
|
||||
# Clients gleichzeitig betreiben.
|
||||
self.account_clients = {}
|
||||
# disconnected_clients[(tenant_id, client_id)] = monotonic Zeitpunkt.
|
||||
# Dient dem Aufräumen dauerhaft verlassener Webviews.
|
||||
self.disconnected_clients = {}
|
||||
self.queue_lock = Lock()
|
||||
# Gemeinsamer Zustand liegt in SQLite (siehe cmbnet/ws/store.py):
|
||||
# ws_clients, ws_client_queue und ws_group_presence. Dadurch finden
|
||||
# Reconnects auf einem anderen Worker ihre Queue und Sequenznummer.
|
||||
self.worker_id = os.environ.get('CMBNET_WS_WORKER_ID') or ''
|
||||
if not self.worker_id:
|
||||
self.worker_id = f'{socket.gethostname()}:{os.getpid()}'
|
||||
|
||||
# Sequenznummern werden pro Client-ID vergeben und bleiben über einen
|
||||
# WebSocket-Reconnect derselben Webview hinweg erhalten. Dadurch kann
|
||||
# der Client nach einem Reconnect erkennen, ob ihm Frames fehlen.
|
||||
# client_sequences[(tenant_id, client_id)] = zuletzt zugestellte Sequenz
|
||||
# Sequenznummern werden pro Client-ID vergeben. Der lokale Cache wird
|
||||
# beim Registrieren aus der Datenbank geladen; nach jedem erfolgreichen
|
||||
# Versand wird die gemeinsame Zählung fortgeschrieben.
|
||||
self.client_sequences = {}
|
||||
# Ein Lock pro Client-ID serialisiert Sequenzvergabe und Sendevorgang
|
||||
# auch dann, wenn die Verbindung während eines Reconnects ersetzt wird.
|
||||
self.client_sequence_locks = {}
|
||||
|
||||
# Zeitpunkt des letzten Code-2-Pongs je Gruppe (monotonic).
|
||||
self.group_last_pong = {}
|
||||
self.group_online_since = {}
|
||||
# Serialisiert das Ausliefern der gemeinsamen Queue pro Client, damit
|
||||
# Authentifizierung und Polling-Schleife nicht parallel dieselben
|
||||
# Aufträge versenden.
|
||||
self.client_delivery_locks = {}
|
||||
# Weckt die lokale Polling-Schleife nach einer neu eingestellten
|
||||
# Nachricht. Workerübergreifend übernimmt das reguläre Polling.
|
||||
self.delivery_wakeup = Event()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Verbindungen und Authentifizierung
|
||||
@@ -102,10 +99,15 @@ class WebSocketBroker:
|
||||
|
||||
# Eine Client-ID ist an den Account gebunden, der sie im Handshake
|
||||
# verwendet hat. Ein Wechsel des Accounts unter derselben ID wäre ein
|
||||
# Hinweis auf eine falsche bzw. manipulierte Zuordnung.
|
||||
existing_account = self.client_accounts.get((tenant_id, client_id))
|
||||
# Hinweis auf eine falsche bzw. manipulierte Zuordnung. Die Zuordnung
|
||||
# liegt prozessübergreifend in ws_clients.
|
||||
existing_account = store.get_client_account(tenant_id, client_id)
|
||||
if existing_account is not None and existing_account != account_id:
|
||||
return False
|
||||
if not store.ensure_client(
|
||||
tenant_id, client_id, account_id, worker_id=self.worker_id
|
||||
):
|
||||
return False
|
||||
|
||||
if role is None:
|
||||
role = account['role']
|
||||
@@ -123,6 +125,7 @@ class WebSocketBroker:
|
||||
group_id = group['id'] if group else None
|
||||
|
||||
connections = self.connections.setdefault(tenant_id, {})
|
||||
key = (tenant_id, client_id)
|
||||
|
||||
# Eine neue Verbindung mit derselben Client-ID ersetzt eine eventuell
|
||||
# noch registrierte alte Verbindung (z. B. nach einem schnellen
|
||||
@@ -148,9 +151,9 @@ class WebSocketBroker:
|
||||
'connected_at': time.monotonic(),
|
||||
'last_pong_at': None,
|
||||
# Zuletzt an diese Client-ID zugestellte Sequenznummer. Die
|
||||
# maßgebliche Zählung liegt in self.client_sequences, damit sie
|
||||
# einen Reconnect überlebt.
|
||||
'seq': self.client_sequences.get((tenant_id, client_id), 0),
|
||||
# maßgebliche Zählung liegt in ws_clients und überlebt damit einen
|
||||
# Worker-Wechsel.
|
||||
'seq': store.get_last_seq(tenant_id, client_id),
|
||||
# Ob diese Client-ID bereits vor diesem Handshake bekannt war.
|
||||
# Nur dann bleibt ihre Queue auch bei einem fehlgeschlagenen
|
||||
# erneuten Handshake erhalten.
|
||||
@@ -163,9 +166,8 @@ class WebSocketBroker:
|
||||
'send_lock': Lock()
|
||||
}
|
||||
|
||||
self.client_accounts[(tenant_id, client_id)] = account_id
|
||||
self.account_clients.setdefault((tenant_id, account_id), set()).add(client_id)
|
||||
self.disconnected_clients.pop((tenant_id, client_id), None)
|
||||
self.client_sequences[key] = store.get_last_seq(tenant_id, client_id)
|
||||
self.client_delivery_locks.setdefault(key, Lock())
|
||||
return True
|
||||
|
||||
def send_to_connection(self, tenant_id, client_id, data):
|
||||
@@ -191,7 +193,7 @@ class WebSocketBroker:
|
||||
|
||||
last_seq = self.client_sequences.get(key)
|
||||
if last_seq is None:
|
||||
last_seq = int(conn.get('seq') or 0)
|
||||
last_seq = store.get_last_seq(tenant_id, client_id)
|
||||
next_seq = last_seq + 1
|
||||
payload['seq'] = next_seq
|
||||
framed = json.dumps(payload, ensure_ascii=False)
|
||||
@@ -205,6 +207,12 @@ class WebSocketBroker:
|
||||
|
||||
self.client_sequences[key] = next_seq
|
||||
conn['seq'] = next_seq
|
||||
# Die Sequenz wird erst nach erfolgreichem Senden persistiert,
|
||||
# damit ein fehlgeschlagener Versand keine Lücke erzeugt.
|
||||
store.set_last_seq(
|
||||
tenant_id, client_id, next_seq,
|
||||
account_id=conn.get('account_id')
|
||||
)
|
||||
return True
|
||||
except Exception:
|
||||
# Eine fehlerhafte Verbindung darf den Versand an andere Clients
|
||||
@@ -216,6 +224,10 @@ class WebSocketBroker:
|
||||
"""Prüft, ob ein Client den Code-0-Handshake erfolgreich absolviert hat."""
|
||||
return self.authenticated.get(tenant_id, {}).get(client_id, False)
|
||||
|
||||
def describe_connections(self, tenant_id):
|
||||
"""Liefert alle verbundenen Clients des Mandanten aus der gemeinsamen DB."""
|
||||
return store.describe_connections(tenant_id)
|
||||
|
||||
def authenticate(self, tenant_id, account_id, client_id, token):
|
||||
"""Authentifiziert eine Client-Verbindung anhand des Einmal-Tokens (Code 0).
|
||||
|
||||
@@ -257,13 +269,15 @@ class WebSocketBroker:
|
||||
if conn.get('role') == 'Gruppe' and conn.get('group_id'):
|
||||
# Ein frisch verbundener Gruppen-Client erhält eine
|
||||
# Zwei-Minuten-Kulanz, bis der erste Code-2-Pong eintrifft.
|
||||
key = (tenant_id, conn['group_id'])
|
||||
self.group_online_since[key] = time.monotonic()
|
||||
store.set_group_online_since(tenant_id, conn['group_id'])
|
||||
|
||||
self.authenticated.setdefault(tenant_id, {})[client_id] = True
|
||||
self.client_accounts[(tenant_id, client_id)] = account_id
|
||||
self.account_clients.setdefault((tenant_id, account_id), set()).add(client_id)
|
||||
self.disconnected_clients.pop((tenant_id, client_id), None)
|
||||
store.mark_authenticated(
|
||||
tenant_id, client_id, True, worker_id=self.worker_id
|
||||
)
|
||||
self.client_sequences[(tenant_id, client_id)] = store.get_last_seq(
|
||||
tenant_id, client_id
|
||||
)
|
||||
|
||||
# Zuerst die für diesen Client gepufferten Nachrichten ausliefern.
|
||||
# Wurde die Queue wegen Überschreitung der Maximalzeit verworfen, wird
|
||||
@@ -313,23 +327,26 @@ class WebSocketBroker:
|
||||
if not authenticated:
|
||||
self.authenticated.pop(tenant_id, None)
|
||||
|
||||
key = (tenant_id, client_id)
|
||||
self.client_sequences.pop(key, None)
|
||||
self.client_sequence_locks.pop(key, None)
|
||||
|
||||
if not was_authenticated and not was_known_client:
|
||||
# Ein nie erfolgreich authentifizierter neuer Client hinterlässt
|
||||
# keine Zuordnung, Queue oder Empfängerregistrierung.
|
||||
account_id = self.client_accounts.pop((tenant_id, client_id), None)
|
||||
self.client_queues.pop((tenant_id, client_id), None)
|
||||
self.disconnected_clients.pop((tenant_id, client_id), None)
|
||||
self.client_sequences.pop((tenant_id, client_id), None)
|
||||
self.client_sequence_locks.pop((tenant_id, client_id), None)
|
||||
if account_id is not None:
|
||||
clients = self.account_clients.get((tenant_id, account_id))
|
||||
if clients is not None:
|
||||
clients.discard(client_id)
|
||||
if not clients:
|
||||
self.account_clients.pop((tenant_id, account_id), None)
|
||||
store.delete_client(
|
||||
tenant_id, client_id, worker_id=self.worker_id
|
||||
)
|
||||
self.client_delivery_locks.pop(key, None)
|
||||
return
|
||||
|
||||
self.disconnected_clients[(tenant_id, client_id)] = time.monotonic()
|
||||
# Die Client-ID bleibt samt Queue bekannt. Ein Reconnect kann auf
|
||||
# einem beliebigen Worker landen und findet den Zustand dort wieder.
|
||||
# Das Delivery-Lock bleibt bestehen, damit eine laufende Zustellung
|
||||
# nicht mit einer neuen Verbindung derselben Client-ID kollidiert.
|
||||
store.mark_disconnected(
|
||||
tenant_id, client_id, worker_id=self.worker_id
|
||||
)
|
||||
|
||||
def _get_connection(self, tenant_id, client_id):
|
||||
return self.connections.get(tenant_id, {}).get(client_id)
|
||||
@@ -369,7 +386,8 @@ class WebSocketBroker:
|
||||
|
||||
recipient_rows = []
|
||||
for recipient_id in sorted(set(recipients or [])):
|
||||
account_id = self.client_accounts.get((tenant_id, recipient_id))
|
||||
client_row = store.get_client(tenant_id, recipient_id)
|
||||
account_id = client_row['account_id'] if client_row else None
|
||||
account = None
|
||||
if account_id is not None:
|
||||
account = db.execute(
|
||||
@@ -388,8 +406,12 @@ class WebSocketBroker:
|
||||
|
||||
info = self._get_sender_info(tenant_id, account)
|
||||
info['client_id'] = recipient_id
|
||||
info['connected'] = bool(self._get_connection(tenant_id, recipient_id))
|
||||
info['authenticated'] = self.is_authenticated(tenant_id, recipient_id)
|
||||
info['connected'] = bool(
|
||||
client_row and client_row['disconnected_at'] is None
|
||||
)
|
||||
info['authenticated'] = bool(
|
||||
client_row and client_row['authenticated']
|
||||
)
|
||||
recipient_rows.append(info)
|
||||
|
||||
payload = message.msg
|
||||
@@ -454,95 +476,62 @@ class WebSocketBroker:
|
||||
minutes = 60
|
||||
return max(self._client_queue_max_age_seconds(), minutes * 60)
|
||||
|
||||
def _ensure_client_queue(self, tenant_id, client_id, now=None):
|
||||
"""Liefert die Queue eines Clients und legt sie bei Bedarf an."""
|
||||
key = (tenant_id, client_id)
|
||||
queue = self.client_queues.get(key)
|
||||
if queue is None:
|
||||
queue = {
|
||||
'messages': [],
|
||||
'created_at': now if now is not None else time.monotonic(),
|
||||
'expired': False,
|
||||
'code99_only': False,
|
||||
'code99_message': None,
|
||||
'account_id': self.client_accounts.get(key),
|
||||
}
|
||||
self.client_queues[key] = queue
|
||||
return queue
|
||||
|
||||
def _expire_client_queue(self, queue, now=None):
|
||||
"""Verwirft zu alte Nachrichten und hinterlässt einen Code-99-Hinweis."""
|
||||
if not queue or queue.get('code99_only') or queue.get('expired'):
|
||||
return
|
||||
|
||||
now = now if now is not None else time.monotonic()
|
||||
if now - queue.get('created_at', now) < self._client_queue_max_age_seconds():
|
||||
return
|
||||
|
||||
queue['messages'].clear()
|
||||
queue['expired'] = True
|
||||
def _ephemeral_ttl_seconds(self):
|
||||
try:
|
||||
return max(1.0, float(current_app.config.get(
|
||||
'WS_EPHEMERAL_TTL_SECONDS', 30
|
||||
)))
|
||||
except (TypeError, ValueError):
|
||||
return 30.0
|
||||
|
||||
def _queue_undeliverable(self, tenant_id, client_id, message):
|
||||
"""Puffert eine nicht zustellbare Nachricht für einen Client."""
|
||||
if message.code < 100 and message.code != 99:
|
||||
return
|
||||
"""Puffert eine nicht zustellbare Nachricht für einen Client.
|
||||
|
||||
if (tenant_id, client_id) not in self.client_accounts:
|
||||
return
|
||||
Bei Code 2 handelt es sich um ein kurzlebiges Lebenszeichen. Es wird
|
||||
nur für aktuell verbundene Clients gepuffert, damit ein späterer
|
||||
Reconnect kein veraltetes Pong erhält.
|
||||
"""
|
||||
if message.code == 2:
|
||||
if not store.client_is_connected(tenant_id, client_id):
|
||||
return False
|
||||
elif message.code < 100 and message.code != 99:
|
||||
return False
|
||||
|
||||
with self.queue_lock:
|
||||
queue = self._ensure_client_queue(tenant_id, client_id)
|
||||
now = time.monotonic()
|
||||
if not store.client_exists(tenant_id, client_id):
|
||||
return False
|
||||
|
||||
if message.code == 99:
|
||||
# Eine nicht zustellbare Reload-Aufforderung verdrängt alle
|
||||
# älteren Nachrichten. Danach wird nichts weiter gepuffert.
|
||||
queue['messages'].clear()
|
||||
queue['code99_only'] = True
|
||||
queue['code99_message'] = message.to_json()
|
||||
queue['expired'] = False
|
||||
return
|
||||
|
||||
self._expire_client_queue(queue, now)
|
||||
if queue.get('expired') or queue.get('code99_only'):
|
||||
return
|
||||
|
||||
queue['messages'].append(message.to_json())
|
||||
queued = store.enqueue_message(
|
||||
tenant_id,
|
||||
client_id,
|
||||
message.code,
|
||||
message.to_json(),
|
||||
self._client_queue_max_age_seconds(),
|
||||
timeout_payload=self._client_queue_timeout_message(),
|
||||
ephemeral_seconds=(
|
||||
self._ephemeral_ttl_seconds() if message.code == 2 else None
|
||||
)
|
||||
)
|
||||
if queued:
|
||||
self.delivery_wakeup.set()
|
||||
return queued
|
||||
|
||||
def cleanup_client_queues(self):
|
||||
"""Räumt abgelaufene Client-Queues und verwaiste Webviews auf."""
|
||||
now = time.monotonic()
|
||||
retention = self._client_retention_seconds()
|
||||
with self.queue_lock:
|
||||
for queue in self.client_queues.values():
|
||||
self._expire_client_queue(queue, now)
|
||||
"""Räumt abgelaufene Queues und verwaiste Clients in der DB auf."""
|
||||
store.purge_expired_queue_rows()
|
||||
store.expire_queues(
|
||||
self._client_queue_max_age_seconds(),
|
||||
self._client_queue_timeout_message()
|
||||
)
|
||||
store.cleanup_clients(self._client_retention_seconds())
|
||||
|
||||
for key, disconnected_at in list(self.disconnected_clients.items()):
|
||||
tenant_id, client_id = key
|
||||
if self._get_connection(tenant_id, client_id):
|
||||
self.disconnected_clients.pop(key, None)
|
||||
continue
|
||||
if now - disconnected_at < retention:
|
||||
continue
|
||||
|
||||
queue = self.client_queues.get(key)
|
||||
# Solange noch zustellbare Nachrichten warten, bleibt der
|
||||
# Client registriert. Danach darf die verwaiste Webview
|
||||
# inklusive Code-99-Tombstone verworfen werden.
|
||||
if queue and not queue.get('expired') and not queue.get('code99_only'):
|
||||
continue
|
||||
|
||||
account_id = self.client_accounts.pop(key, None)
|
||||
self.client_queues.pop(key, None)
|
||||
self.disconnected_clients.pop(key, None)
|
||||
# Lokale Caches für inzwischen entfernte Clients bereinigen.
|
||||
for key in list(self.client_sequences):
|
||||
if not store.client_exists(*key):
|
||||
self.client_sequences.pop(key, None)
|
||||
self.client_sequence_locks.pop(key, None)
|
||||
if account_id is not None:
|
||||
clients = self.account_clients.get((tenant_id, account_id))
|
||||
if clients is not None:
|
||||
clients.discard(client_id)
|
||||
if not clients:
|
||||
self.account_clients.pop((tenant_id, account_id), None)
|
||||
for key in list(self.client_delivery_locks):
|
||||
if not store.client_exists(*key):
|
||||
self.client_delivery_locks.pop(key, None)
|
||||
|
||||
def _client_queue_timeout_message(self):
|
||||
minutes = current_app.config.get('WS_QUEUE_MAX_AGE_MINUTES', 15)
|
||||
@@ -556,8 +545,13 @@ class WebSocketBroker:
|
||||
)
|
||||
).to_json()
|
||||
|
||||
def _client_delivery_lock(self, tenant_id, client_id):
|
||||
return self.client_delivery_locks.setdefault(
|
||||
(tenant_id, client_id), Lock()
|
||||
)
|
||||
|
||||
def restore_client_queue(self, tenant_id, client_id):
|
||||
"""Liefert gepufferte Nachrichten eines wiedererkannten Clients aus.
|
||||
"""Liefert persistierte Nachrichten eines wiedererkannten Clients aus.
|
||||
|
||||
Rückgabe: (Menge der Roh-Nachrichten aus der Queue, Late-Entry-Sperre).
|
||||
Die Menge dient dazu, dieselben Nachrichten nicht zusätzlich als
|
||||
@@ -565,57 +559,66 @@ class WebSocketBroker:
|
||||
war oder nur noch eine Code-99-Nachricht enthielt. In diesem Fall darf
|
||||
keine zusätzliche Late-Entry-Historie mehr gesendet werden.
|
||||
"""
|
||||
key = (tenant_id, client_id)
|
||||
now = time.monotonic()
|
||||
|
||||
with self.queue_lock:
|
||||
queue = self.client_queues.get(key)
|
||||
if not queue:
|
||||
with self._client_delivery_lock(tenant_id, client_id):
|
||||
rows = store.fetch_queue(tenant_id, client_id)
|
||||
if not rows:
|
||||
return set(), False
|
||||
self._expire_client_queue(queue, now)
|
||||
|
||||
if queue.get('code99_only'):
|
||||
raw_message = queue.get('code99_message') or self._client_queue_timeout_message()
|
||||
self.client_queues.pop(key, None)
|
||||
send_only = True
|
||||
messages = []
|
||||
elif queue.get('expired'):
|
||||
raw_message = self._client_queue_timeout_message()
|
||||
self.client_queues.pop(key, None)
|
||||
send_only = True
|
||||
messages = []
|
||||
started_at = store.get_queue_started_at(tenant_id, client_id)
|
||||
max_age = self._client_queue_max_age_seconds()
|
||||
if started_at is not None and store.now() - started_at >= max_age:
|
||||
store.replace_queue_with_timeout(
|
||||
tenant_id, client_id, self._client_queue_timeout_message()
|
||||
)
|
||||
rows = store.fetch_queue(tenant_id, client_id)
|
||||
suppress_late_entries = True
|
||||
else:
|
||||
raw_message = None
|
||||
send_only = False
|
||||
messages = list(queue.get('messages', []))
|
||||
queue['messages'] = []
|
||||
suppress_late_entries = any(
|
||||
row['code99_only'] for row in rows
|
||||
)
|
||||
|
||||
if send_only:
|
||||
if raw_message and not self.send_to_connection(tenant_id, client_id, raw_message):
|
||||
# Verbindung ist bereits wieder weg: Queue für den nächsten
|
||||
# Versuch erhalten.
|
||||
with self.queue_lock:
|
||||
self.client_queues.setdefault(key, queue)
|
||||
return set(), True
|
||||
raw_messages = set()
|
||||
for row in rows:
|
||||
raw_message = row['payload']
|
||||
if not self.send_to_connection(tenant_id, client_id, raw_message):
|
||||
# Verbindung ist bereits wieder weg. Noch nicht
|
||||
# zugestellte Aufträge bleiben in der Queue.
|
||||
break
|
||||
if not row['code99_only']:
|
||||
raw_messages.add(raw_message)
|
||||
store.delete_queue_row(row['id'])
|
||||
|
||||
unsent = []
|
||||
for index, raw_message in enumerate(messages):
|
||||
if not self.send_to_connection(tenant_id, client_id, raw_message):
|
||||
unsent = messages[index:]
|
||||
break
|
||||
return raw_messages, suppress_late_entries
|
||||
|
||||
with self.queue_lock:
|
||||
current = self.client_queues.get(key)
|
||||
if current is not None:
|
||||
if unsent:
|
||||
current['messages'] = unsent + current.get('messages', [])
|
||||
if not current.get('messages'):
|
||||
self.client_queues.pop(key, None)
|
||||
def drain_client_queue(self, tenant_id, client_id):
|
||||
"""Stellt die gemeinsame Queue eines lokal verbundenen Clients zu."""
|
||||
self.restore_client_queue(tenant_id, client_id)
|
||||
|
||||
# Auch nicht zugestellte Queue-Nachrichten von den Late Entries
|
||||
# ausnehmen. Sie bleiben in der Queue und werden beim nächsten
|
||||
# Reconnect erneut versucht, statt doppelt gepuffert zu werden.
|
||||
return set(messages), False
|
||||
def drain_local_clients(self):
|
||||
"""Zustellrunde für alle lokal verbundenen, authentifizierten Clients."""
|
||||
local_clients = set()
|
||||
for tenant_id, connections in list(self.connections.items()):
|
||||
for client_id in list(connections):
|
||||
if self.is_authenticated(tenant_id, client_id):
|
||||
local_clients.add((tenant_id, client_id))
|
||||
|
||||
if not local_clients:
|
||||
return
|
||||
|
||||
# Nur Clients mit tatsächlich wartenden Aufträgen öffnen eine
|
||||
# Transaktion. Das hält die Polling-Schleife bei vielen verbundenen
|
||||
# Clients günstig.
|
||||
for tenant_id, client_id in store.clients_with_queue(local_clients):
|
||||
try:
|
||||
self.drain_client_queue(tenant_id, client_id)
|
||||
except Exception as exc:
|
||||
try:
|
||||
current_app.logger.warning(
|
||||
'WS-Queue-Zustellung fehlgeschlagen: tenant=%s client=%s error=%s',
|
||||
tenant_id, client_id, exc
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Late Entry
|
||||
@@ -881,7 +884,7 @@ class WebSocketBroker:
|
||||
"""Wertet ein Code-2-Lebenszeichen eines Clients aus."""
|
||||
conn = self._get_connection(tenant_id, client_id) if client_id else None
|
||||
if conn and conn.get('account_id') == account_id:
|
||||
conn['last_pong_at'] = time.monotonic()
|
||||
conn['last_pong_at'] = time.time()
|
||||
|
||||
if not sender or sender['role'] != 'Gruppe':
|
||||
return
|
||||
@@ -897,9 +900,8 @@ class WebSocketBroker:
|
||||
if not group:
|
||||
return
|
||||
|
||||
now = time.monotonic()
|
||||
group_id = group['id']
|
||||
self.group_last_pong[(tenant_id, group_id)] = now
|
||||
store.record_group_pong(tenant_id, group_id)
|
||||
|
||||
if group['state'] == 'unterwegs':
|
||||
self._set_group_state_internal(
|
||||
@@ -917,11 +919,16 @@ class WebSocketBroker:
|
||||
return False
|
||||
|
||||
old_state = group['state']
|
||||
db.execute(
|
||||
# Der Vergleich im UPDATE verhindert, dass zwei Worker denselben
|
||||
# automatischen Statuswechsel gleichzeitig in die Historie schreiben.
|
||||
cursor = db.execute(
|
||||
'UPDATE groups SET state = ?, updated_at = CURRENT_TIMESTAMP '
|
||||
'WHERE id = ? AND tenant_id = ?',
|
||||
(new_state, group_id, tenant_id)
|
||||
'WHERE id = ? AND tenant_id = ? AND state = ?',
|
||||
(new_state, group_id, tenant_id, old_state)
|
||||
)
|
||||
if cursor.rowcount == 0:
|
||||
db.rollback()
|
||||
return False
|
||||
db.execute(
|
||||
'''INSERT INTO group_state_history
|
||||
(tenant_id, group_id, from_state, to_state, last_lat, last_lon,
|
||||
@@ -937,7 +944,7 @@ class WebSocketBroker:
|
||||
)
|
||||
|
||||
if new_state == 'online':
|
||||
self.group_online_since[(tenant_id, group_id)] = time.monotonic()
|
||||
store.set_group_online_since(tenant_id, group_id)
|
||||
|
||||
message = WSMessage(
|
||||
sender_id=actor_account_id or 0,
|
||||
@@ -972,20 +979,24 @@ class WebSocketBroker:
|
||||
except (TypeError, ValueError):
|
||||
timeout = 120.0
|
||||
|
||||
now = time.monotonic()
|
||||
current = store.now()
|
||||
for row in rows:
|
||||
key = (row['tenant_id'], row['id'])
|
||||
reference = self.group_last_pong.get(key)
|
||||
online_since = self.group_online_since.get(key)
|
||||
presence = store.get_group_presence(row['tenant_id'], row['id'])
|
||||
last_pong = presence['last_pong_at'] if presence else None
|
||||
online_since = presence['online_since'] if presence else None
|
||||
reference = last_pong
|
||||
if online_since is not None:
|
||||
reference = max(reference, online_since) if reference else online_since
|
||||
reference = (
|
||||
max(reference, online_since)
|
||||
if reference is not None else online_since
|
||||
)
|
||||
if reference is None:
|
||||
# Zustand war bereits vor dem Start des Trackings "online".
|
||||
# Ab jetzt läuft die Kulanzfrist.
|
||||
self.group_online_since[key] = now
|
||||
store.set_group_online_since(row['tenant_id'], row['id'], current)
|
||||
continue
|
||||
|
||||
if now - reference > timeout:
|
||||
if current - reference > timeout:
|
||||
self._set_group_state_internal(
|
||||
row['tenant_id'], row['id'], 'unterwegs'
|
||||
)
|
||||
@@ -1158,7 +1169,7 @@ class WebSocketBroker:
|
||||
db.commit()
|
||||
GroupsService.update_cache_entry(tenant_id, group_id, {'state': new_state})
|
||||
if new_state == 'online':
|
||||
self.group_online_since[(tenant_id, group_id)] = time.monotonic()
|
||||
store.set_group_online_since(tenant_id, group_id)
|
||||
self._log_business(tenant_id, sender['id'], 'group.state_change', 'group',
|
||||
group_id, {'from': group['state'], 'to': new_state})
|
||||
|
||||
@@ -1402,34 +1413,40 @@ class WebSocketBroker:
|
||||
return message
|
||||
|
||||
def _deliver(self, tenant_id, message, sender):
|
||||
"""Stellt eine Nachricht an alle berechtigten Clients zu."""
|
||||
"""Stellt eine Nachricht an alle berechtigten Clients zu.
|
||||
|
||||
Lokal verbundene Clients erhalten den Frame direkt. Für alle anderen
|
||||
Clients wird ein persistenter Auftrag in der gemeinsamen Queue
|
||||
angelegt. Jeder Worker pollt diese Queue für seine eigenen Clients.
|
||||
"""
|
||||
recipients = sorted(self._get_recipients(tenant_id, message, sender))
|
||||
self.log_incoming(tenant_id, message, sender, recipients=recipients)
|
||||
for client_id in recipients:
|
||||
self._send(tenant_id, client_id, message)
|
||||
self.delivery_wakeup.set()
|
||||
|
||||
def _send(self, tenant_id, client_id, message):
|
||||
"""Sendet eine Nachricht an einen Client oder puffert sie für ihn."""
|
||||
# Erst nach erfolgreicher Token-Authentifizierung dürfen Nachrichten
|
||||
# an einen Client zugestellt werden.
|
||||
if not self.is_authenticated(tenant_id, client_id):
|
||||
self._queue_undeliverable(tenant_id, client_id, message)
|
||||
return False
|
||||
conn = self._get_connection(tenant_id, client_id)
|
||||
authenticated = self.is_authenticated(tenant_id, client_id)
|
||||
|
||||
if self.send_to_connection(tenant_id, client_id, message.to_json()):
|
||||
return True
|
||||
# Ein lokal verbundener Client ohne wartende Queue wird direkt
|
||||
# bedient. Existiert bereits eine Queue, wird auch diese Nachricht
|
||||
# eingereiht, damit die Reihenfolge erhalten bleibt.
|
||||
if conn and authenticated and not store.client_has_queue(tenant_id, client_id):
|
||||
if self.send_to_connection(tenant_id, client_id, message.to_json()):
|
||||
return True
|
||||
|
||||
self._queue_undeliverable(tenant_id, client_id, message)
|
||||
return False
|
||||
|
||||
def _client_ids_for_accounts(self, tenant_id, account_ids):
|
||||
"""Expandiert Empfänger-Accounts auf alle bekannten Client-IDs."""
|
||||
client_ids = set()
|
||||
for account_id in account_ids:
|
||||
client_ids.update(
|
||||
self.account_clients.get((tenant_id, account_id), ())
|
||||
)
|
||||
return client_ids
|
||||
"""Expandiert Empfänger-Accounts auf alle bekannten Client-IDs.
|
||||
|
||||
Die Zuordnung liegt in ws_clients und umfasst damit auch Clients, die
|
||||
an anderen Workern verbunden oder momentan getrennt sind.
|
||||
"""
|
||||
return store.client_ids_for_accounts(tenant_id, account_ids)
|
||||
|
||||
def _get_recipients(self, tenant_id, message, sender):
|
||||
"""Bestimmt den Empfängerkreis abhängig vom Nachrichtencode.
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
"""Worker-lokale Zustellschleife für die gemeinsame WebSocket-Queue.
|
||||
|
||||
Jeder WSGI-Worker besitzt eigene WebSocket-Objekte. Die persistente Queue in
|
||||
``ws_client_queue`` ist dagegen gemeinsam. Diese Schleife prüft regelmäßig, ob
|
||||
für die lokal verbundenen Clients Aufträge eingegangen sind, und stellt sie
|
||||
über den lokalen Broker zu.
|
||||
"""
|
||||
import threading
|
||||
|
||||
from cmbnet.ws import broker
|
||||
|
||||
|
||||
class DeliveryWorker:
|
||||
def __init__(self, interval=0.5):
|
||||
self.interval = interval
|
||||
self.app = None
|
||||
self.running = False
|
||||
self._thread = None
|
||||
self._stop = threading.Event()
|
||||
|
||||
def start(self, app=None, interval=None):
|
||||
if app is not None:
|
||||
self.app = app
|
||||
if interval is not None:
|
||||
self.interval = interval
|
||||
if self.running:
|
||||
return
|
||||
if self.app is None:
|
||||
return
|
||||
|
||||
self.running = True
|
||||
self._stop.clear()
|
||||
self._thread = threading.Thread(
|
||||
target=self._run,
|
||||
name='cmbnet-ws-delivery',
|
||||
daemon=True
|
||||
)
|
||||
self._thread.start()
|
||||
|
||||
def stop(self):
|
||||
self.running = False
|
||||
self._stop.set()
|
||||
broker.delivery_wakeup.set()
|
||||
# Der Thread ist ein Daemon. Ein Join mit Timeout verhindert, dass
|
||||
# ein laufender Datenbankzugriff den Shutdown blockiert.
|
||||
if self._thread is not None:
|
||||
self._thread.join(timeout=2.0)
|
||||
self._thread = None
|
||||
|
||||
def wake(self):
|
||||
broker.delivery_wakeup.set()
|
||||
|
||||
def _run(self):
|
||||
# Ein App-Kontext für die gesamte Lebensdauer der Schleife hält die
|
||||
# SQLite-Verbindung dieses Threads offen. Bei einem Polling-Intervall
|
||||
# von 0,5 s wäre ein Verbindungsaufbau pro Runde unnötig teuer.
|
||||
with self.app.app_context():
|
||||
while not self._stop.is_set() and self.running:
|
||||
try:
|
||||
broker.drain_local_clients()
|
||||
except Exception as exc:
|
||||
try:
|
||||
self.app.logger.warning(
|
||||
'WS-Zustellschleife fehlgeschlagen: %s', exc
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Neue lokale Nachrichten wecken die Schleife sofort. Für
|
||||
# Nachrichten aus anderen Workern greift der reguläre Timeout.
|
||||
broker.delivery_wakeup.wait(self.interval)
|
||||
broker.delivery_wakeup.clear()
|
||||
|
||||
|
||||
delivery_worker = DeliveryWorker()
|
||||
@@ -69,17 +69,24 @@ class HeartbeatManager:
|
||||
|
||||
msg = WSMessage(sender_id=0, code=1, msg='ping')
|
||||
|
||||
# Nur authentifizierte Verbindungen erhalten einen Heartbeat.
|
||||
for tenant_id, connections in list(broker.connections.items()):
|
||||
for client_id in list(connections):
|
||||
if not broker.is_authenticated(tenant_id, client_id):
|
||||
continue
|
||||
broker.send_to_connection(tenant_id, client_id, msg.to_json())
|
||||
|
||||
# Abgelaufene persistierte Nachrichten und Client-Queues löschen.
|
||||
if self.app is not None:
|
||||
try:
|
||||
# Auch der Versand braucht einen App-Kontext: Die
|
||||
# Sequenznummern liegen inzwischen in der gemeinsamen
|
||||
# SQLite-Datenbank und werden beim Senden fortgeschrieben.
|
||||
with self.app.app_context():
|
||||
# Nur authentifizierte Verbindungen erhalten einen
|
||||
# Heartbeat.
|
||||
for tenant_id, connections in list(broker.connections.items()):
|
||||
for client_id in list(connections):
|
||||
if not broker.is_authenticated(tenant_id, client_id):
|
||||
continue
|
||||
broker.send_to_connection(
|
||||
tenant_id, client_id, msg.to_json()
|
||||
)
|
||||
|
||||
# Abgelaufene persistierte Nachrichten und Client-Queues
|
||||
# löschen.
|
||||
broker.cleanup_client_queues()
|
||||
broker.cleanup_old_messages()
|
||||
except Exception as exc:
|
||||
|
||||
@@ -0,0 +1,535 @@
|
||||
"""Prozessübergreifender Zustand des WebSocket-Brokers.
|
||||
|
||||
Der Broker hält weiterhin nur die WebSocket-Objekte im eigenen Prozess.
|
||||
Alles, was ein Reconnect auf einem anderen WSGI-Worker wiederfinden muss,
|
||||
liegt in SQLite:
|
||||
|
||||
- ``ws_clients``: bekannte Client-IDs, Account, letzte Sequenznummer und
|
||||
Verbindungsstatus,
|
||||
- ``ws_client_queue``: persistente Zustellaufträge pro Client,
|
||||
- ``ws_group_presence``: gemeinsame Pong-Zeitpunkte der Gruppen.
|
||||
|
||||
Die Tabellen werden über ``ensure_schema()`` angelegt. Die Funktionen hier
|
||||
erwarten einen aktiven Flask-App-Kontext, weil sie ``get_db()`` verwenden.
|
||||
"""
|
||||
import time
|
||||
|
||||
from cmbnet.db import get_db
|
||||
|
||||
# Kleine Hilfsfunktion, damit alle Zeitstempel dieselbe Basis haben.
|
||||
def now():
|
||||
return time.time()
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Clients
|
||||
# ----------------------------------------------------------------------
|
||||
def get_client(tenant_id, client_id):
|
||||
"""Liefert die Zeile eines bekannten Clients oder ``None``."""
|
||||
return get_db().execute(
|
||||
'SELECT * FROM ws_clients WHERE tenant_id = ? AND client_id = ?',
|
||||
(tenant_id, client_id)
|
||||
).fetchone()
|
||||
|
||||
|
||||
def get_client_account(tenant_id, client_id):
|
||||
"""Liefert die Account-ID, die zu einer Client-ID gehört."""
|
||||
row = get_client(tenant_id, client_id)
|
||||
return row['account_id'] if row else None
|
||||
|
||||
|
||||
def client_exists(tenant_id, client_id):
|
||||
return get_client(tenant_id, client_id) is not None
|
||||
|
||||
|
||||
def client_is_connected(tenant_id, client_id):
|
||||
row = get_client(tenant_id, client_id)
|
||||
return bool(row and row['disconnected_at'] is None)
|
||||
|
||||
|
||||
def ensure_client(tenant_id, client_id, account_id, worker_id=None):
|
||||
"""Registriert eine Client-ID bzw. aktualisiert ihren Verbindungsstatus.
|
||||
|
||||
Gibt ``False`` zurück, wenn die Client-ID bereits einem anderen Account
|
||||
gehört. Ein solcher Wechsel wäre ein Hinweis auf eine manipulierte
|
||||
Zuordnung und wird vom Broker abgelehnt.
|
||||
"""
|
||||
db = get_db()
|
||||
row = get_client(tenant_id, client_id)
|
||||
current = now()
|
||||
|
||||
if row is not None:
|
||||
if row['account_id'] != account_id:
|
||||
return False
|
||||
db.execute(
|
||||
'UPDATE ws_clients SET worker_id = ?, connected_at = ?, '
|
||||
'disconnected_at = NULL, authenticated = 0 '
|
||||
'WHERE tenant_id = ? AND client_id = ?',
|
||||
(worker_id, current, tenant_id, client_id)
|
||||
)
|
||||
db.commit()
|
||||
return True
|
||||
|
||||
db.execute(
|
||||
'INSERT INTO ws_clients '
|
||||
'(tenant_id, client_id, account_id, worker_id, connected_at, '
|
||||
' authenticated, last_seq) '
|
||||
'VALUES (?, ?, ?, ?, ?, 0, 0)',
|
||||
(tenant_id, client_id, account_id, worker_id, current)
|
||||
)
|
||||
db.commit()
|
||||
return True
|
||||
|
||||
|
||||
def mark_authenticated(tenant_id, client_id, authenticated=True, worker_id=None):
|
||||
db = get_db()
|
||||
if worker_id is None:
|
||||
db.execute(
|
||||
'UPDATE ws_clients SET authenticated = ? '
|
||||
'WHERE tenant_id = ? AND client_id = ?',
|
||||
(1 if authenticated else 0, tenant_id, client_id)
|
||||
)
|
||||
else:
|
||||
# Nur der Worker, dem die Client-ID aktuell zugeordnet ist, darf den
|
||||
# Authentifizierungsstatus setzen. So überschreibt ein verspäteter
|
||||
# Handshake eines alten Workers keinen Reconnect auf einem anderen
|
||||
# Worker.
|
||||
db.execute(
|
||||
'UPDATE ws_clients SET authenticated = ? '
|
||||
'WHERE tenant_id = ? AND client_id = ? '
|
||||
'AND (worker_id IS NULL OR worker_id = ?)',
|
||||
(1 if authenticated else 0, tenant_id, client_id, worker_id)
|
||||
)
|
||||
db.commit()
|
||||
|
||||
|
||||
def mark_disconnected(tenant_id, client_id, worker_id=None):
|
||||
"""Markiert eine Client-ID als getrennt, ohne ihre Queue zu löschen.
|
||||
|
||||
Mit ``worker_id`` wird nur die Zuordnung dieses Workers getrennt. Hat
|
||||
inzwischen ein anderer Worker dieselbe Client-ID übernommen, bleibt dessen
|
||||
Verbindungsstatus unberührt.
|
||||
"""
|
||||
db = get_db()
|
||||
if worker_id is None:
|
||||
db.execute(
|
||||
'UPDATE ws_clients SET worker_id = NULL, disconnected_at = ?, '
|
||||
'authenticated = 0 WHERE tenant_id = ? AND client_id = ?',
|
||||
(now(), tenant_id, client_id)
|
||||
)
|
||||
else:
|
||||
db.execute(
|
||||
'UPDATE ws_clients SET worker_id = NULL, disconnected_at = ?, '
|
||||
'authenticated = 0 WHERE tenant_id = ? AND client_id = ? '
|
||||
'AND (worker_id IS NULL OR worker_id = ?)',
|
||||
(now(), tenant_id, client_id, worker_id)
|
||||
)
|
||||
db.commit()
|
||||
|
||||
|
||||
def delete_client(tenant_id, client_id, worker_id=None):
|
||||
"""Entfernt eine Client-ID samt Queue und Sequenzzustand.
|
||||
|
||||
Mit ``worker_id`` wird nur gelöscht, solange die Client-ID noch diesem
|
||||
Worker gehört. Das verhindert, dass ein scheidender Worker einen
|
||||
zwischenzeitlichen Reconnect auf einem anderen Worker entfernt.
|
||||
"""
|
||||
db = get_db()
|
||||
if worker_id is None:
|
||||
cursor = db.execute(
|
||||
'DELETE FROM ws_clients WHERE tenant_id = ? AND client_id = ?',
|
||||
(tenant_id, client_id)
|
||||
)
|
||||
else:
|
||||
cursor = db.execute(
|
||||
'DELETE FROM ws_clients WHERE tenant_id = ? AND client_id = ? '
|
||||
'AND (worker_id IS NULL OR worker_id = ?)',
|
||||
(tenant_id, client_id, worker_id)
|
||||
)
|
||||
if cursor.rowcount:
|
||||
db.execute(
|
||||
'DELETE FROM ws_client_queue WHERE tenant_id = ? AND client_id = ?',
|
||||
(tenant_id, client_id)
|
||||
)
|
||||
db.commit()
|
||||
|
||||
|
||||
def get_last_seq(tenant_id, client_id):
|
||||
row = get_client(tenant_id, client_id)
|
||||
return int(row['last_seq'] or 0) if row else 0
|
||||
|
||||
|
||||
def set_last_seq(tenant_id, client_id, seq, account_id=None):
|
||||
db = get_db()
|
||||
cursor = db.execute(
|
||||
'UPDATE ws_clients SET last_seq = ? WHERE tenant_id = ? AND client_id = ?',
|
||||
(seq, tenant_id, client_id)
|
||||
)
|
||||
if cursor.rowcount:
|
||||
db.commit()
|
||||
return
|
||||
# Die Client-Zeile kann durch einen Cleanup verschwunden sein. In diesem
|
||||
# Fall wird sie mit der bekannten Account-ID neu angelegt, damit der
|
||||
# Sequenzstand nicht verloren geht.
|
||||
if account_id is not None:
|
||||
db.execute(
|
||||
'INSERT OR IGNORE INTO ws_clients '
|
||||
'(tenant_id, client_id, account_id, last_seq) VALUES (?, ?, ?, ?)',
|
||||
(tenant_id, client_id, account_id, seq)
|
||||
)
|
||||
db.commit()
|
||||
|
||||
|
||||
def client_ids_for_accounts(tenant_id, account_ids):
|
||||
"""Expandiert Empfänger-Accounts auf alle bekannten Client-IDs.
|
||||
|
||||
Anders als früher werden auch Clients berücksichtigt, die an einem
|
||||
anderen Worker verbunden oder momentan getrennt sind. Nur so können
|
||||
Nachrichten für einen Reconnect auf einem beliebigen Worker gepuffert
|
||||
werden.
|
||||
"""
|
||||
account_ids = list(dict.fromkeys(account_ids))
|
||||
if not account_ids:
|
||||
return set()
|
||||
|
||||
db = get_db()
|
||||
result = set()
|
||||
chunk_size = 400
|
||||
for start in range(0, len(account_ids), chunk_size):
|
||||
chunk = account_ids[start:start + chunk_size]
|
||||
placeholders = ','.join('?' for _ in chunk)
|
||||
rows = db.execute(
|
||||
f'SELECT client_id FROM ws_clients '
|
||||
f'WHERE tenant_id = ? AND account_id IN ({placeholders})',
|
||||
(tenant_id, *chunk)
|
||||
).fetchall()
|
||||
result.update(row['client_id'] for row in rows)
|
||||
return result
|
||||
|
||||
|
||||
def describe_connections(tenant_id):
|
||||
"""Liefert alle aktuell verbundenen Clients inklusive Account und Gruppe."""
|
||||
rows = get_db().execute(
|
||||
'''SELECT c.client_id, c.account_id, c.authenticated,
|
||||
a.name AS account_name, a.role,
|
||||
g.id AS group_id, g.aliasname AS group_alias
|
||||
FROM ws_clients c
|
||||
JOIN accounts a
|
||||
ON a.id = c.account_id AND a.tenant_id = c.tenant_id
|
||||
LEFT JOIN group_accounts ga
|
||||
ON ga.tenant_id = c.tenant_id AND ga.account_id = c.account_id
|
||||
LEFT JOIN groups g
|
||||
ON g.id = ga.group_id AND g.tenant_id = c.tenant_id
|
||||
WHERE c.tenant_id = ? AND c.disconnected_at IS NULL
|
||||
ORDER BY c.client_id''',
|
||||
(tenant_id,)
|
||||
).fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Client-Queues
|
||||
# ----------------------------------------------------------------------
|
||||
def get_queue_started_at(tenant_id, client_id):
|
||||
row = get_client(tenant_id, client_id)
|
||||
return row['queue_started_at'] if row else None
|
||||
|
||||
|
||||
def _set_queue_started_at(db, tenant_id, client_id, started_at):
|
||||
db.execute(
|
||||
'UPDATE ws_clients SET queue_started_at = ? '
|
||||
'WHERE tenant_id = ? AND client_id = ?',
|
||||
(started_at, tenant_id, client_id)
|
||||
)
|
||||
|
||||
|
||||
def client_has_queue(tenant_id, client_id):
|
||||
"""Prüft, ob für einen Client noch zustellbare Aufträge warten."""
|
||||
row = get_db().execute(
|
||||
'SELECT 1 FROM ws_client_queue '
|
||||
'WHERE tenant_id = ? AND client_id = ? '
|
||||
'AND (expires_at IS NULL OR expires_at > ?) LIMIT 1',
|
||||
(tenant_id, client_id, now())
|
||||
).fetchone()
|
||||
return row is not None
|
||||
|
||||
|
||||
def enqueue_message(tenant_id, client_id, code, payload, max_age_seconds,
|
||||
timeout_payload=None, ephemeral_seconds=None):
|
||||
"""Legt einen Zustellauftrag für einen Client an.
|
||||
|
||||
``max_age_seconds`` steuert die Lebensdauer einer normalen Queue.
|
||||
``timeout_payload`` wird als Code-99-Tombstone eingesetzt, wenn die Queue
|
||||
zu alt ist oder eine Code-99-Nachricht alle älteren Aufträge verdrängt.
|
||||
``ephemeral_seconds`` gilt für Code 2: Der Auftrag verfällt schnell und
|
||||
wird nicht für getrennte Clients vorgehalten.
|
||||
"""
|
||||
db = get_db()
|
||||
current = now()
|
||||
row = db.execute(
|
||||
'SELECT queue_started_at FROM ws_clients '
|
||||
'WHERE tenant_id = ? AND client_id = ?',
|
||||
(tenant_id, client_id)
|
||||
).fetchone()
|
||||
if row is None:
|
||||
# Unbekannte Client-IDs werden nicht gepuffert. Der Broker legt die
|
||||
# Zeile beim Code-0-Handshake an.
|
||||
return False
|
||||
|
||||
started_at = row['queue_started_at']
|
||||
if started_at is None:
|
||||
started_at = current
|
||||
|
||||
# Eine Code-99-Nachricht verdrängt alle älteren Aufträge und blockiert
|
||||
# weitere Nachrichten, bis sie zugestellt wurde.
|
||||
if code != 99:
|
||||
tombstone = db.execute(
|
||||
'SELECT 1 FROM ws_client_queue '
|
||||
'WHERE tenant_id = ? AND client_id = ? AND code99_only = 1 LIMIT 1',
|
||||
(tenant_id, client_id)
|
||||
).fetchone()
|
||||
if tombstone:
|
||||
return False
|
||||
|
||||
# Abgelaufene Queue: Inhalt verwerfen und einen Code-99-Hinweis hinterlassen.
|
||||
if current - started_at >= max_age_seconds:
|
||||
db.execute(
|
||||
'DELETE FROM ws_client_queue WHERE tenant_id = ? AND client_id = ?',
|
||||
(tenant_id, client_id)
|
||||
)
|
||||
if timeout_payload is not None:
|
||||
db.execute(
|
||||
'INSERT INTO ws_client_queue '
|
||||
'(tenant_id, client_id, code, payload, code99_only, created_at) '
|
||||
'VALUES (?, ?, 99, ?, 1, ?)',
|
||||
(tenant_id, client_id, timeout_payload, current)
|
||||
)
|
||||
_set_queue_started_at(db, tenant_id, client_id, current)
|
||||
else:
|
||||
_set_queue_started_at(db, tenant_id, client_id, None)
|
||||
db.commit()
|
||||
return True
|
||||
|
||||
if code == 99:
|
||||
db.execute(
|
||||
'DELETE FROM ws_client_queue WHERE tenant_id = ? AND client_id = ?',
|
||||
(tenant_id, client_id)
|
||||
)
|
||||
db.execute(
|
||||
'INSERT INTO ws_client_queue '
|
||||
'(tenant_id, client_id, code, payload, code99_only, created_at) '
|
||||
'VALUES (?, ?, 99, ?, 1, ?)',
|
||||
(tenant_id, client_id, payload, current)
|
||||
)
|
||||
_set_queue_started_at(db, tenant_id, client_id, current)
|
||||
db.commit()
|
||||
return True
|
||||
|
||||
expires_at = None
|
||||
if ephemeral_seconds is not None:
|
||||
expires_at = current + ephemeral_seconds
|
||||
|
||||
db.execute(
|
||||
'INSERT INTO ws_client_queue '
|
||||
'(tenant_id, client_id, code, payload, code99_only, created_at, expires_at) '
|
||||
'VALUES (?, ?, ?, ?, 0, ?, ?)',
|
||||
(tenant_id, client_id, code, payload, current, expires_at)
|
||||
)
|
||||
_set_queue_started_at(db, tenant_id, client_id, started_at)
|
||||
db.commit()
|
||||
return True
|
||||
|
||||
|
||||
def fetch_queue(tenant_id, client_id):
|
||||
"""Liefert alle noch gültigen Aufträge eines Clients in Reihenfolge."""
|
||||
return get_db().execute(
|
||||
'SELECT * FROM ws_client_queue '
|
||||
'WHERE tenant_id = ? AND client_id = ? '
|
||||
'AND (expires_at IS NULL OR expires_at > ?) '
|
||||
'ORDER BY id ASC',
|
||||
(tenant_id, client_id, now())
|
||||
).fetchall()
|
||||
|
||||
|
||||
def clients_with_queue(pairs):
|
||||
"""Liefert diejenigen ``(tenant_id, client_id)``, die Aufträge haben."""
|
||||
if not pairs:
|
||||
return set()
|
||||
|
||||
db = get_db()
|
||||
by_tenant = {}
|
||||
for tenant_id, client_id in pairs:
|
||||
by_tenant.setdefault(tenant_id, set()).add(client_id)
|
||||
|
||||
result = set()
|
||||
chunk_size = 400
|
||||
for tenant_id, client_ids in by_tenant.items():
|
||||
ids = list(client_ids)
|
||||
for start in range(0, len(ids), chunk_size):
|
||||
chunk = ids[start:start + chunk_size]
|
||||
placeholders = ','.join('?' for _ in chunk)
|
||||
rows = db.execute(
|
||||
f'SELECT DISTINCT client_id FROM ws_client_queue '
|
||||
f'WHERE tenant_id = ? AND client_id IN ({placeholders}) '
|
||||
f'AND (expires_at IS NULL OR expires_at > ?)',
|
||||
(tenant_id, *chunk, now())
|
||||
).fetchall()
|
||||
result.update((tenant_id, row['client_id']) for row in rows)
|
||||
return result
|
||||
|
||||
|
||||
def delete_queue_row(row_id):
|
||||
"""Löscht einen zugestellten Auftrag und räumt den Queue-Start auf."""
|
||||
db = get_db()
|
||||
row = db.execute(
|
||||
'SELECT tenant_id, client_id FROM ws_client_queue WHERE id = ?',
|
||||
(row_id,)
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return
|
||||
db.execute('DELETE FROM ws_client_queue WHERE id = ?', (row_id,))
|
||||
remaining = db.execute(
|
||||
'SELECT 1 FROM ws_client_queue '
|
||||
'WHERE tenant_id = ? AND client_id = ? '
|
||||
'AND (expires_at IS NULL OR expires_at > ?) LIMIT 1',
|
||||
(row['tenant_id'], row['client_id'], now())
|
||||
).fetchone()
|
||||
if remaining is None:
|
||||
_set_queue_started_at(db, row['tenant_id'], row['client_id'], None)
|
||||
db.commit()
|
||||
|
||||
|
||||
def replace_queue_with_timeout(tenant_id, client_id, payload):
|
||||
"""Ersetzt eine abgelaufene Queue durch einen Code-99-Hinweis."""
|
||||
db = get_db()
|
||||
current = now()
|
||||
db.execute(
|
||||
'DELETE FROM ws_client_queue WHERE tenant_id = ? AND client_id = ?',
|
||||
(tenant_id, client_id)
|
||||
)
|
||||
db.execute(
|
||||
'INSERT INTO ws_client_queue '
|
||||
'(tenant_id, client_id, code, payload, code99_only, created_at) '
|
||||
'VALUES (?, ?, 99, ?, 1, ?)',
|
||||
(tenant_id, client_id, payload, current)
|
||||
)
|
||||
_set_queue_started_at(db, tenant_id, client_id, current)
|
||||
db.commit()
|
||||
|
||||
|
||||
def expire_queues(max_age_seconds, timeout_payload):
|
||||
"""Wandelt zu alte Queues in Code-99-Tombstones um.
|
||||
|
||||
Die Funktion ist idempotent und kann von jedem Worker regelmäßig
|
||||
aufgerufen werden. Nach dem ersten Lauf ist ``queue_started_at``
|
||||
aktualisiert, sodass weitere Worker die Queue nicht erneut umwandeln.
|
||||
"""
|
||||
db = get_db()
|
||||
cutoff = now() - max_age_seconds
|
||||
rows = db.execute(
|
||||
'SELECT tenant_id, client_id FROM ws_clients '
|
||||
'WHERE queue_started_at IS NOT NULL AND queue_started_at <= ?',
|
||||
(cutoff,)
|
||||
).fetchall()
|
||||
|
||||
current = now()
|
||||
for row in rows:
|
||||
tenant_id = row['tenant_id']
|
||||
client_id = row['client_id']
|
||||
db.execute(
|
||||
'DELETE FROM ws_client_queue WHERE tenant_id = ? AND client_id = ?',
|
||||
(tenant_id, client_id)
|
||||
)
|
||||
db.execute(
|
||||
'INSERT INTO ws_client_queue '
|
||||
'(tenant_id, client_id, code, payload, code99_only, created_at) '
|
||||
'VALUES (?, ?, 99, ?, 1, ?)',
|
||||
(tenant_id, client_id, timeout_payload, current)
|
||||
)
|
||||
_set_queue_started_at(db, tenant_id, client_id, current)
|
||||
if rows:
|
||||
db.commit()
|
||||
|
||||
|
||||
def cleanup_clients(retention_seconds):
|
||||
"""Entfernt dauerhaft getrennte Clients ohne zustellbare Restnachrichten."""
|
||||
db = get_db()
|
||||
cutoff = now() - retention_seconds
|
||||
rows = db.execute(
|
||||
'SELECT tenant_id, client_id FROM ws_clients '
|
||||
'WHERE disconnected_at IS NOT NULL AND disconnected_at < ?',
|
||||
(cutoff,)
|
||||
).fetchall()
|
||||
|
||||
for row in rows:
|
||||
tenant_id = row['tenant_id']
|
||||
client_id = row['client_id']
|
||||
pending = db.execute(
|
||||
'SELECT 1 FROM ws_client_queue '
|
||||
'WHERE tenant_id = ? AND client_id = ? AND code99_only = 0 '
|
||||
'AND (expires_at IS NULL OR expires_at > ?) LIMIT 1',
|
||||
(tenant_id, client_id, now())
|
||||
).fetchone()
|
||||
if pending is not None:
|
||||
# Noch zustellbare Nachrichten halten den Client am Leben.
|
||||
continue
|
||||
db.execute(
|
||||
'DELETE FROM ws_client_queue WHERE tenant_id = ? AND client_id = ?',
|
||||
(tenant_id, client_id)
|
||||
)
|
||||
db.execute(
|
||||
'DELETE FROM ws_clients WHERE tenant_id = ? AND client_id = ?',
|
||||
(tenant_id, client_id)
|
||||
)
|
||||
if rows:
|
||||
db.commit()
|
||||
|
||||
|
||||
def purge_expired_queue_rows():
|
||||
"""Löscht abgelaufene ephemere Aufträge (z. B. Code 2)."""
|
||||
db = get_db()
|
||||
db.execute(
|
||||
'DELETE FROM ws_client_queue WHERE expires_at IS NOT NULL AND expires_at <= ?',
|
||||
(now(),)
|
||||
)
|
||||
db.commit()
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Gruppenpräsenz
|
||||
# ----------------------------------------------------------------------
|
||||
def get_group_presence(tenant_id, group_id):
|
||||
return get_db().execute(
|
||||
'SELECT * FROM ws_group_presence WHERE tenant_id = ? AND group_id = ?',
|
||||
(tenant_id, group_id)
|
||||
).fetchone()
|
||||
|
||||
|
||||
def record_group_pong(tenant_id, group_id, timestamp=None):
|
||||
db = get_db()
|
||||
current = timestamp if timestamp is not None else now()
|
||||
db.execute(
|
||||
'INSERT OR IGNORE INTO ws_group_presence (tenant_id, group_id) VALUES (?, ?)',
|
||||
(tenant_id, group_id)
|
||||
)
|
||||
db.execute(
|
||||
'UPDATE ws_group_presence SET last_pong_at = ?, updated_at = CURRENT_TIMESTAMP '
|
||||
'WHERE tenant_id = ? AND group_id = ?',
|
||||
(current, tenant_id, group_id)
|
||||
)
|
||||
db.commit()
|
||||
|
||||
|
||||
def set_group_online_since(tenant_id, group_id, timestamp=None):
|
||||
db = get_db()
|
||||
current = timestamp if timestamp is not None else now()
|
||||
db.execute(
|
||||
'INSERT OR IGNORE INTO ws_group_presence (tenant_id, group_id) VALUES (?, ?)',
|
||||
(tenant_id, group_id)
|
||||
)
|
||||
db.execute(
|
||||
'UPDATE ws_group_presence SET online_since = ?, updated_at = CURRENT_TIMESTAMP '
|
||||
'WHERE tenant_id = ? AND group_id = ?',
|
||||
(current, tenant_id, group_id)
|
||||
)
|
||||
db.commit()
|
||||
Reference in new issue
Block a user