Add Loki and Sentinel utility classes for web API endpoints

- Implemented LokiUtils class with GET and POST endpoints for managing scripts, jobs, and payloads.
- Added SentinelUtils class with GET and POST endpoints for managing events, rules, devices, and notifications.
- Both classes include error handling and JSON response formatting.
This commit is contained in:
infinition
2026-03-14 22:33:10 +01:00
parent eb20b168a6
commit aac77a3e76
525 changed files with 29400 additions and 13136 deletions

116
db_utils/bifrost.py Normal file
View File

@@ -0,0 +1,116 @@
"""
Bifrost DB operations — networks, handshakes, epochs, activity, peers, plugin data.
"""
import logging
from logger import Logger
logger = Logger(name="db_utils.bifrost", level=logging.DEBUG)
class BifrostOps:
def __init__(self, base):
self.base = base
def create_tables(self):
"""Create all Bifrost tables."""
# WiFi networks discovered by Bifrost
self.base.execute("""
CREATE TABLE IF NOT EXISTS bifrost_networks (
bssid TEXT PRIMARY KEY,
essid TEXT DEFAULT '',
channel INTEGER DEFAULT 0,
encryption TEXT DEFAULT '',
rssi INTEGER DEFAULT 0,
vendor TEXT DEFAULT '',
num_clients INTEGER DEFAULT 0,
first_seen TEXT DEFAULT CURRENT_TIMESTAMP,
last_seen TEXT DEFAULT CURRENT_TIMESTAMP,
handshake INTEGER DEFAULT 0,
deauthed INTEGER DEFAULT 0,
associated INTEGER DEFAULT 0,
whitelisted INTEGER DEFAULT 0
)
""")
# Captured handshakes
self.base.execute("""
CREATE TABLE IF NOT EXISTS bifrost_handshakes (
id INTEGER PRIMARY KEY AUTOINCREMENT,
ap_mac TEXT NOT NULL,
sta_mac TEXT NOT NULL,
ap_essid TEXT DEFAULT '',
channel INTEGER DEFAULT 0,
rssi INTEGER DEFAULT 0,
filename TEXT DEFAULT '',
captured_at TEXT DEFAULT CURRENT_TIMESTAMP,
uploaded INTEGER DEFAULT 0,
cracked INTEGER DEFAULT 0,
UNIQUE(ap_mac, sta_mac)
)
""")
# Epoch history
self.base.execute("""
CREATE TABLE IF NOT EXISTS bifrost_epochs (
id INTEGER PRIMARY KEY AUTOINCREMENT,
epoch_num INTEGER NOT NULL,
started_at TEXT NOT NULL,
duration_secs REAL DEFAULT 0,
num_deauths INTEGER DEFAULT 0,
num_assocs INTEGER DEFAULT 0,
num_handshakes INTEGER DEFAULT 0,
num_hops INTEGER DEFAULT 0,
num_missed INTEGER DEFAULT 0,
num_peers INTEGER DEFAULT 0,
mood TEXT DEFAULT 'ready',
reward REAL DEFAULT 0,
cpu_load REAL DEFAULT 0,
mem_usage REAL DEFAULT 0,
temperature REAL DEFAULT 0,
meta_json TEXT DEFAULT '{}'
)
""")
# Activity log (event feed)
self.base.execute("""
CREATE TABLE IF NOT EXISTS bifrost_activity (
id INTEGER PRIMARY KEY AUTOINCREMENT,
timestamp TEXT DEFAULT CURRENT_TIMESTAMP,
event_type TEXT NOT NULL,
title TEXT NOT NULL,
details TEXT DEFAULT '',
meta_json TEXT DEFAULT '{}'
)
""")
self.base.execute(
"CREATE INDEX IF NOT EXISTS idx_bifrost_activity_ts "
"ON bifrost_activity(timestamp DESC)"
)
# Peers (mesh networking — Phase 2)
self.base.execute("""
CREATE TABLE IF NOT EXISTS bifrost_peers (
peer_id TEXT PRIMARY KEY,
name TEXT DEFAULT '',
version TEXT DEFAULT '',
face TEXT DEFAULT '',
encounters INTEGER DEFAULT 0,
last_channel INTEGER DEFAULT 0,
last_seen TEXT DEFAULT CURRENT_TIMESTAMP,
first_seen TEXT DEFAULT CURRENT_TIMESTAMP
)
""")
# Plugin persistent state
self.base.execute("""
CREATE TABLE IF NOT EXISTS bifrost_plugin_data (
plugin_name TEXT NOT NULL,
key TEXT NOT NULL,
value TEXT DEFAULT '',
PRIMARY KEY (plugin_name, key)
)
""")
logger.debug("Bifrost tables created/verified")

51
db_utils/loki.py Normal file
View File

@@ -0,0 +1,51 @@
"""
Loki DB operations — HID scripts and job tracking.
"""
import logging
from logger import Logger
logger = Logger(name="db_utils.loki", level=logging.DEBUG)
class LokiOps:
def __init__(self, base):
self.base = base
def create_tables(self):
"""Create all Loki tables."""
# User-saved HID scripts
self.base.execute("""
CREATE TABLE IF NOT EXISTS loki_scripts (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL UNIQUE,
description TEXT DEFAULT '',
content TEXT NOT NULL DEFAULT '',
category TEXT DEFAULT 'general',
target_os TEXT DEFAULT 'any',
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
updated_at TEXT DEFAULT CURRENT_TIMESTAMP
)
""")
# Job execution history
self.base.execute("""
CREATE TABLE IF NOT EXISTS loki_jobs (
id TEXT PRIMARY KEY,
script_id INTEGER,
script_name TEXT DEFAULT '',
status TEXT DEFAULT 'pending',
output TEXT DEFAULT '',
error TEXT DEFAULT '',
started_at TEXT,
finished_at TEXT,
created_at TEXT DEFAULT CURRENT_TIMESTAMP
)
""")
self.base.execute(
"CREATE INDEX IF NOT EXISTS idx_loki_jobs_status "
"ON loki_jobs(status)"
)
logger.debug("Loki tables created/verified")

View File

@@ -65,6 +65,20 @@ class QueueOps:
WHERE status='scheduled';
""")
# Circuit breaker table for ORCH-01
self.base.execute("""
CREATE TABLE IF NOT EXISTS action_circuit_breaker (
action_name TEXT NOT NULL,
mac_address TEXT NOT NULL DEFAULT '',
failure_streak INTEGER NOT NULL DEFAULT 0,
last_failure_at TEXT,
circuit_status TEXT NOT NULL DEFAULT 'closed',
opened_at TEXT,
cooldown_until TEXT,
PRIMARY KEY (action_name, mac_address)
);
""")
logger.debug("Action queue table created/verified")
# =========================================================================
@@ -398,6 +412,120 @@ class QueueOps:
# HELPER METHODS
# =========================================================================
# =========================================================================
# CIRCUIT BREAKER OPERATIONS (ORCH-01)
# =========================================================================
def record_circuit_breaker_failure(self, action_name: str, mac: str = '',
threshold: int = 3) -> None:
"""Increment failure streak; open circuit if streak >= threshold."""
now_str = self.base.query_one("SELECT datetime('now') AS ts")['ts']
# Upsert the row
self.base.execute("""
INSERT INTO action_circuit_breaker (action_name, mac_address, failure_streak,
last_failure_at, circuit_status)
VALUES (?, ?, 1, ?, 'closed')
ON CONFLICT(action_name, mac_address) DO UPDATE SET
failure_streak = failure_streak + 1,
last_failure_at = excluded.last_failure_at
""", (action_name, mac or '', now_str))
# Check if we need to open the circuit
row = self.base.query_one(
"SELECT failure_streak FROM action_circuit_breaker WHERE action_name=? AND mac_address=?",
(action_name, mac or '')
)
if row and row['failure_streak'] >= threshold:
streak = row['failure_streak']
cooldown_secs = min(2 ** streak * 60, 3600)
self.base.execute("""
UPDATE action_circuit_breaker
SET circuit_status = 'open',
opened_at = ?,
cooldown_until = datetime(?, '+' || ? || ' seconds')
WHERE action_name=? AND mac_address=?
""", (now_str, now_str, str(cooldown_secs), action_name, mac or ''))
def record_circuit_breaker_success(self, action_name: str, mac: str = '') -> None:
"""Reset failure streak and close circuit on success."""
self.base.execute("""
INSERT INTO action_circuit_breaker (action_name, mac_address, failure_streak,
circuit_status)
VALUES (?, ?, 0, 'closed')
ON CONFLICT(action_name, mac_address) DO UPDATE SET
failure_streak = 0,
circuit_status = 'closed',
opened_at = NULL,
cooldown_until = NULL
""", (action_name, mac or ''))
def is_circuit_open(self, action_name: str, mac: str = '') -> bool:
"""Return True if circuit is open AND cooldown hasn't expired.
If cooldown has expired, transition to half_open and return False."""
row = self.base.query_one(
"SELECT circuit_status, cooldown_until FROM action_circuit_breaker "
"WHERE action_name=? AND mac_address=?",
(action_name, mac or '')
)
if not row:
return False
status = row['circuit_status']
if status == 'closed':
return False
if status == 'open':
cooldown = row.get('cooldown_until')
if cooldown:
# Check if cooldown has expired
expired = self.base.query_one(
"SELECT datetime('now') >= datetime(?) AS expired",
(cooldown,)
)
if expired and expired['expired']:
# Transition to half_open
self.base.execute("""
UPDATE action_circuit_breaker SET circuit_status='half_open'
WHERE action_name=? AND mac_address=?
""", (action_name, mac or ''))
return False # Allow one attempt through
return True # Still in cooldown
# half_open: allow one attempt through
return False
def get_circuit_breaker_status(self, action_name: str, mac: str = '') -> Optional[Dict[str, Any]]:
"""Return full circuit breaker status dict."""
row = self.base.query_one(
"SELECT * FROM action_circuit_breaker WHERE action_name=? AND mac_address=?",
(action_name, mac or '')
)
return dict(row) if row else None
def reset_circuit_breaker(self, action_name: str, mac: str = '') -> None:
"""Manual reset of circuit breaker."""
self.base.execute("""
DELETE FROM action_circuit_breaker WHERE action_name=? AND mac_address=?
""", (action_name, mac or ''))
# =========================================================================
# CONCURRENCY OPERATIONS (ORCH-02)
# =========================================================================
def count_running_actions(self, action_name: Optional[str] = None) -> int:
"""Count currently running actions, optionally filtered by action_name."""
if action_name:
row = self.base.query_one(
"SELECT COUNT(*) AS cnt FROM action_queue WHERE status='running' AND action_name=?",
(action_name,)
)
else:
row = self.base.query_one(
"SELECT COUNT(*) AS cnt FROM action_queue WHERE status='running'"
)
return int(row['cnt']) if row else 0
# =========================================================================
# HELPER METHODS
# =========================================================================
def _format_ts_for_raw(self, ts_db: Optional[str]) -> str:
"""
Convert SQLite 'YYYY-MM-DD HH:MM:SS' to 'YYYYMMDD_HHMMSS'.

314
db_utils/sentinel.py Normal file
View File

@@ -0,0 +1,314 @@
"""
Sentinel DB operations — events, rules, known devices baseline.
"""
import json
import logging
from typing import Any, Dict, List, Optional
from logger import Logger
logger = Logger(name="db_utils.sentinel", level=logging.DEBUG)
class SentinelOps:
def __init__(self, base):
self.base = base
def create_tables(self):
"""Create all Sentinel tables."""
# Known device baselines — MAC → expected behavior
self.base.execute("""
CREATE TABLE IF NOT EXISTS sentinel_devices (
mac_address TEXT PRIMARY KEY,
alias TEXT,
trusted INTEGER DEFAULT 0,
watch INTEGER DEFAULT 1,
first_seen TEXT DEFAULT CURRENT_TIMESTAMP,
last_seen TEXT DEFAULT CURRENT_TIMESTAMP,
expected_ips TEXT DEFAULT '',
expected_ports TEXT DEFAULT '',
notes TEXT DEFAULT ''
)
""")
# Events / alerts log
self.base.execute("""
CREATE TABLE IF NOT EXISTS sentinel_events (
id INTEGER PRIMARY KEY AUTOINCREMENT,
timestamp TEXT DEFAULT CURRENT_TIMESTAMP,
event_type TEXT NOT NULL,
severity TEXT DEFAULT 'info',
title TEXT NOT NULL,
details TEXT DEFAULT '',
mac_address TEXT,
ip_address TEXT,
acknowledged INTEGER DEFAULT 0,
notified INTEGER DEFAULT 0,
meta_json TEXT DEFAULT '{}'
)
""")
self.base.execute(
"CREATE INDEX IF NOT EXISTS idx_sentinel_events_ts "
"ON sentinel_events(timestamp DESC)"
)
self.base.execute(
"CREATE INDEX IF NOT EXISTS idx_sentinel_events_type "
"ON sentinel_events(event_type)"
)
self.base.execute(
"CREATE INDEX IF NOT EXISTS idx_sentinel_events_ack "
"ON sentinel_events(acknowledged)"
)
# Configurable rules (AND/OR composable)
self.base.execute("""
CREATE TABLE IF NOT EXISTS sentinel_rules (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
enabled INTEGER DEFAULT 1,
trigger_type TEXT NOT NULL,
conditions TEXT DEFAULT '{}',
logic TEXT DEFAULT 'AND',
actions TEXT DEFAULT '["notify_web"]',
cooldown_s INTEGER DEFAULT 60,
last_fired TEXT,
created_at TEXT DEFAULT CURRENT_TIMESTAMP
)
""")
# ARP cache snapshots for spoof detection
self.base.execute("""
CREATE TABLE IF NOT EXISTS sentinel_arp_cache (
mac_address TEXT NOT NULL,
ip_address TEXT NOT NULL,
first_seen TEXT DEFAULT CURRENT_TIMESTAMP,
last_seen TEXT DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (mac_address, ip_address)
)
""")
# Insert default rules if empty
existing = self.base.query("SELECT COUNT(*) AS c FROM sentinel_rules")
if existing and existing[0].get('c', 0) == 0:
self._insert_default_rules()
def _insert_default_rules(self):
"""Seed default Sentinel rules."""
defaults = [
{
"name": "New Device Detected",
"trigger_type": "new_device",
"conditions": "{}",
"logic": "AND",
"actions": '["notify_web"]',
"cooldown_s": 0,
},
{
"name": "Device Joined Network",
"trigger_type": "device_join",
"conditions": "{}",
"logic": "AND",
"actions": '["notify_web"]',
"cooldown_s": 30,
},
{
"name": "Device Left Network",
"trigger_type": "device_leave",
"conditions": "{}",
"logic": "AND",
"actions": '["notify_web"]',
"cooldown_s": 30,
},
{
"name": "ARP Spoofing Detected",
"trigger_type": "arp_spoof",
"conditions": "{}",
"logic": "AND",
"actions": '["notify_web", "notify_discord"]',
"cooldown_s": 10,
},
{
"name": "Port Change on Host",
"trigger_type": "port_change",
"conditions": "{}",
"logic": "AND",
"actions": '["notify_web"]',
"cooldown_s": 120,
},
{
"name": "Rogue DHCP Server",
"trigger_type": "rogue_dhcp",
"conditions": "{}",
"logic": "AND",
"actions": '["notify_web", "notify_discord"]',
"cooldown_s": 60,
},
]
for rule in defaults:
self.base.execute(
"""INSERT INTO sentinel_rules
(name, trigger_type, conditions, logic, actions, cooldown_s)
VALUES (?, ?, ?, ?, ?, ?)""",
(rule["name"], rule["trigger_type"], rule["conditions"],
rule["logic"], rule["actions"], rule["cooldown_s"])
)
# ── Events ──────────────────────────────────────────────────────────
def insert_event(self, event_type: str, severity: str, title: str,
details: str = "", mac: str = "", ip: str = "",
meta: Optional[Dict] = None) -> int:
return self.base.execute(
"""INSERT INTO sentinel_events
(event_type, severity, title, details, mac_address, ip_address, meta_json)
VALUES (?, ?, ?, ?, ?, ?, ?)""",
(event_type, severity, title, details, mac, ip,
json.dumps(meta or {}))
)
def get_events(self, limit: int = 100, offset: int = 0,
event_type: str = "", unread_only: bool = False) -> List[Dict]:
sql = "SELECT * FROM sentinel_events WHERE 1=1"
params: list = []
if event_type:
sql += " AND event_type = ?"
params.append(event_type)
if unread_only:
sql += " AND acknowledged = 0"
sql += " ORDER BY timestamp DESC LIMIT ? OFFSET ?"
params.extend([limit, offset])
return self.base.query(sql, params)
def count_unread(self) -> int:
row = self.base.query_one(
"SELECT COUNT(*) AS c FROM sentinel_events WHERE acknowledged = 0"
)
return int(row.get("c", 0)) if row else 0
def acknowledge_event(self, event_id: int):
self.base.execute(
"UPDATE sentinel_events SET acknowledged = 1 WHERE id = ?",
(event_id,)
)
def acknowledge_all(self):
self.base.execute("UPDATE sentinel_events SET acknowledged = 1")
def clear_events(self):
self.base.execute("DELETE FROM sentinel_events")
# ── Rules ───────────────────────────────────────────────────────────
def get_rules(self) -> List[Dict]:
return self.base.query("SELECT * FROM sentinel_rules ORDER BY id")
def get_enabled_rules(self, trigger_type: str = "") -> List[Dict]:
if trigger_type:
return self.base.query(
"SELECT * FROM sentinel_rules WHERE enabled = 1 AND trigger_type = ?",
(trigger_type,)
)
return self.base.query(
"SELECT * FROM sentinel_rules WHERE enabled = 1"
)
def upsert_rule(self, data: Dict) -> Dict:
rule_id = data.get("id")
if rule_id:
self.base.execute(
"""UPDATE sentinel_rules SET
name=?, enabled=?, trigger_type=?, conditions=?,
logic=?, actions=?, cooldown_s=?
WHERE id=?""",
(data["name"], int(data.get("enabled", 1)),
data["trigger_type"], json.dumps(data.get("conditions", {})),
data.get("logic", "AND"),
json.dumps(data.get("actions", ["notify_web"])),
int(data.get("cooldown_s", 60)), rule_id)
)
else:
self.base.execute(
"""INSERT INTO sentinel_rules
(name, enabled, trigger_type, conditions, logic, actions, cooldown_s)
VALUES (?, ?, ?, ?, ?, ?, ?)""",
(data["name"], int(data.get("enabled", 1)),
data["trigger_type"], json.dumps(data.get("conditions", {})),
data.get("logic", "AND"),
json.dumps(data.get("actions", ["notify_web"])),
int(data.get("cooldown_s", 60)))
)
return {"status": "ok"}
def delete_rule(self, rule_id: int):
self.base.execute("DELETE FROM sentinel_rules WHERE id = ?", (rule_id,))
def update_rule_fired(self, rule_id: int):
self.base.execute(
"UPDATE sentinel_rules SET last_fired = CURRENT_TIMESTAMP WHERE id = ?",
(rule_id,)
)
# ── Devices baseline ────────────────────────────────────────────────
def get_known_device(self, mac: str) -> Optional[Dict]:
return self.base.query_one(
"SELECT * FROM sentinel_devices WHERE mac_address = ?", (mac,)
)
def upsert_device(self, mac: str, **kwargs):
existing = self.get_known_device(mac)
if existing:
sets = []
params = []
for k, v in kwargs.items():
if k in ("alias", "trusted", "watch", "expected_ips",
"expected_ports", "notes"):
sets.append(f"{k} = ?")
params.append(v)
sets.append("last_seen = CURRENT_TIMESTAMP")
if sets:
params.append(mac)
self.base.execute(
f"UPDATE sentinel_devices SET {', '.join(sets)} WHERE mac_address = ?",
params
)
else:
self.base.execute(
"""INSERT INTO sentinel_devices
(mac_address, alias, trusted, watch, expected_ips, expected_ports, notes)
VALUES (?, ?, ?, ?, ?, ?, ?)""",
(mac, kwargs.get("alias", ""),
int(kwargs.get("trusted", 0)),
int(kwargs.get("watch", 1)),
kwargs.get("expected_ips", ""),
kwargs.get("expected_ports", ""),
kwargs.get("notes", ""))
)
def get_all_known_devices(self) -> List[Dict]:
return self.base.query("SELECT * FROM sentinel_devices ORDER BY last_seen DESC")
# ── ARP cache ───────────────────────────────────────────────────────
def update_arp_entry(self, mac: str, ip: str):
self.base.execute(
"""INSERT INTO sentinel_arp_cache (mac_address, ip_address)
VALUES (?, ?)
ON CONFLICT(mac_address, ip_address)
DO UPDATE SET last_seen = CURRENT_TIMESTAMP""",
(mac, ip)
)
def get_arp_for_ip(self, ip: str) -> List[Dict]:
return self.base.query(
"SELECT * FROM sentinel_arp_cache WHERE ip_address = ?", (ip,)
)
def get_arp_for_mac(self, mac: str) -> List[Dict]:
return self.base.query(
"SELECT * FROM sentinel_arp_cache WHERE mac_address = ?", (mac,)
)
def get_full_arp_cache(self) -> List[Dict]:
return self.base.query("SELECT * FROM sentinel_arp_cache ORDER BY last_seen DESC")