DocRock3r/Bjorn
1
0
Fork 0
mirror of https://github.com/infinition/Bjorn.git synced 2026-03-11 07:01:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add RLUtils class for managing RL/AI dashboard endpoints

Browse Source 
- Implemented methods for fetching AI stats, training history, and recent experiences.
- Added functionality to set operation mode (MANUAL, AUTO, AI) with appropriate handling.
- Included helper methods for querying the database and sending JSON responses.
- Integrated model metadata extraction for visualization purposes.
This commit is contained in:
Fabien POLLY
2026-02-18 22:36:10 +01:00
parent b8a13cc698
commit eb20b168a6
684 changed files with 53278 additions and 27977 deletions
Download Patch File Download Diff File Expand all files Collapse all files

239
web_utils/action_utils.py
View File

@@ -17,7 +17,6 @@ This file merges previous modules:
from __future__ import annotations
import ast
import cgi
import io
import json
import os
@@ -38,6 +37,86 @@ from logger import Logger
# Single shared logger for the whole file
logger = Logger(name="action_utils.py", level=logging.DEBUG)
# --- Multipart form helpers (replaces cgi module removed in Python 3.13) ---
def _parse_header(line):
parts = line.split(';')
key = parts[0].strip()
pdict = {}
for p in parts[1:]:
if '=' in p:
k, v = p.strip().split('=', 1)
pdict[k.strip()] = v.strip().strip('"')
return key, pdict
class _FormField:
__slots__ = ('name', 'filename', 'file', 'value')
def __init__(self, name, filename=None, data=b''):
self.name = name
self.filename = filename
if filename:
self.file = BytesIO(data)
self.value = data
else:
self.value = data.decode('utf-8', errors='replace').strip()
self.file = None
class _MultipartForm:
"""Minimal replacement for _MultipartForm."""
def __init__(self, fp, headers, environ=None, keep_blank_values=False):
import re as _re
self._fields = {}
ct = headers.get('Content-Type', '') if hasattr(headers, 'get') else ''
_, params = _parse_header(ct)
boundary = params.get('boundary', '').encode()
if hasattr(fp, 'read'):
cl = headers.get('Content-Length') if hasattr(headers, 'get') else None
body = fp.read(int(cl)) if cl else fp.read()
else:
body = fp
for part in body.split(b'--' + boundary)[1:]:
part = part.strip(b'\r\n')
if part == b'--' or not part:
continue
sep = b'\r\n\r\n' if b'\r\n\r\n' in part else b'\n\n'
if sep not in part:
continue
hdr, data = part.split(sep, 1)
hdr_s = hdr.decode('utf-8', errors='replace')
nm = _re.search(r'name="([^"]*)"', hdr_s)
fn = _re.search(r'filename="([^"]*)"', hdr_s)
if not nm:
continue
name = nm.group(1)
filename = fn.group(1) if fn else None
field = _FormField(name, filename, data)
if name in self._fields:
existing = self._fields[name]
if isinstance(existing, list):
existing.append(field)
else:
self._fields[name] = [existing, field]
else:
self._fields[name] = field
def __contains__(self, key):
return key in self._fields
def __getitem__(self, key):
return self._fields[key]
def getvalue(self, key, default=None):
if key not in self._fields:
return default
f = self._fields[key]
if isinstance(f, list):
return [x.value for x in f]
return f.value
ALLOWED_IMAGE_EXTS = {'.bmp', '.png', '.jpg', '.jpeg', '.gif', '.ico', '.webp'}
@@ -169,7 +248,12 @@ class ActionUtils:
except Exception:
font = ImageFont.load_default()
tw, th = draw.textsize(text, font=font)
try:
bbox = draw.textbbox((0, 0), text, font=font)
tw = bbox[2] - bbox[0]
th = bbox[3] - bbox[1]
except AttributeError:
tw, th = draw.textsize(text, font=font)
draw.text(((size - tw) / 2, (size - th) / 2), text, fill=ring_color, font=font)
out = BytesIO()
@@ -197,10 +281,16 @@ class ActionUtils:
def serve_bjorn_character(self, handler):
try:
# Convertir l'image PIL en bytes
# Fallback robust: use current character sprite, or static default "bjorn1"
img = self.shared_data.bjorn_character or getattr(self.shared_data, 'bjorn1', None)
if img is None:
raise ValueError("No character image (bjorn_character or bjorn1) available")
img_byte_arr = io.BytesIO()
self.shared_data.bjorn_character.save(img_byte_arr, format='PNG')
img.save(img_byte_arr, format='PNG')
img_byte_arr = img_byte_arr.getvalue()
handler.send_response(200)
handler.send_header('Content-Type', 'image/png')
handler.send_header('Cache-Control', 'no-cache')
@@ -221,11 +311,16 @@ class ActionUtils:
handler.send_header("Content-Type", "application/json")
handler.end_headers()
handler.wfile.write(json.dumps(bjorn_says_data).encode('utf-8'))
except BrokenPipeError:
pass
except Exception as e:
handler.send_response(500)
handler.send_header("Content-Type", "application/json")
handler.end_headers()
handler.wfile.write(json.dumps({"status": "error", "message": str(e)}).encode('utf-8'))
try:
handler.send_response(500)
handler.send_header("Content-Type", "application/json")
handler.end_headers()
handler.wfile.write(json.dumps({"status": "error", "message": str(e)}).encode('utf-8'))
except BrokenPipeError:
pass
def create_action(self, handler):
"""
@@ -246,7 +341,7 @@ class ActionUtils:
content_length = int(handler.headers.get("Content-Length", 0))
body = handler.rfile.read(content_length)
form = cgi.FieldStorage(
form = _MultipartForm(
fp=BytesIO(body),
headers=handler.headers,
environ={"REQUEST_METHOD": "POST"},
@@ -299,12 +394,15 @@ class ActionUtils:
meta.setdefault("b_module", module_name)
self.shared_data.db.upsert_simple_action(**meta)
def delete_action(self, handler):
def delete_action(self, handler, data=None):
"""Delete action: python script + images + comment section."""
try:
content_length = int(handler.headers.get("Content-Length", 0))
body = handler.rfile.read(content_length) if content_length > 0 else b"{}"
data = json.loads(body)
if data is None:
content_length = int(handler.headers.get("Content-Length", 0))
body = handler.rfile.read(content_length) if content_length > 0 else b"{}"
data = json.loads(body)
elif not isinstance(data, dict):
raise ValueError("Invalid JSON payload")
action_name = (data.get("action_name") or "").strip()
if not action_name:
@@ -518,6 +616,8 @@ class ActionUtils:
handler.send_header("Content-Length", str(len(content)))
handler.end_headers()
handler.wfile.write(content)
except BrokenPipeError:
pass
except Exception as e:
self.logger.error(f"serve_status_image: {e}")
handler.send_error(500, "Internal Server Error")
@@ -545,13 +645,13 @@ class ActionUtils:
def upload_static_image(self, handler):
"""Upload a static image; store as BMP. Optional manual size via flags."""
try:
ctype, pdict = cgi.parse_header(handler.headers.get("Content-Type"))
ctype, pdict = _parse_header(handler.headers.get("Content-Type"))
if ctype != "multipart/form-data":
raise ValueError("Content-Type must be multipart/form-data")
pdict["boundary"] = bytes(pdict["boundary"], "utf-8")
pdict["CONTENT-LENGTH"] = int(handler.headers.get("Content-Length"))
form = cgi.FieldStorage(
form = _MultipartForm(
fp=BytesIO(handler.rfile.read(pdict["CONTENT-LENGTH"])),
headers=handler.headers,
environ={"REQUEST_METHOD": "POST"},
@@ -674,10 +774,13 @@ class ActionUtils:
self._send_error(handler, str(e))
def rename_image(self, handler):
def rename_image(self, handler, data=None):
"""Rename a static image, an action image, or an action folder."""
try:
data = json.loads(handler.rfile.read(int(handler.headers["Content-Length"])).decode("utf-8"))
if data is None:
data = json.loads(handler.rfile.read(int(handler.headers["Content-Length"])).decode("utf-8"))
elif not isinstance(data, dict):
raise ValueError("Invalid JSON payload")
entity_type = data.get("type") # 'action' | 'static' | 'image'
old_name = data.get("old_name")
new_name = data.get("new_name")
@@ -731,15 +834,15 @@ class ActionUtils:
self._send_error(handler, str(e))
def replace_image(self, h):
import cgi
try:
ctype, pdict = cgi.parse_header(h.headers.get('Content-Type'))
ctype, pdict = _parse_header(h.headers.get('Content-Type'))
if ctype != 'multipart/form-data':
raise ValueError('Content-Type must be multipart/form-data')
pdict['boundary'] = bytes(pdict['boundary'], 'utf-8')
pdict['CONTENT-LENGTH'] = int(h.headers.get('Content-Length'))
form = cgi.FieldStorage(
form = _MultipartForm(
fp=BytesIO(h.rfile.read(pdict['CONTENT-LENGTH'])),
headers=h.headers,
environ={'REQUEST_METHOD': 'POST'},
@@ -819,13 +922,13 @@ class ActionUtils:
Creates the action folder if it doesn't exist.
"""
try:
ctype, pdict = cgi.parse_header(handler.headers.get("Content-Type"))
ctype, pdict = _parse_header(handler.headers.get("Content-Type"))
if ctype != "multipart/form-data":
raise ValueError("Content-Type must be multipart/form-data")
pdict["boundary"] = bytes(pdict["boundary"], "utf-8")
pdict["CONTENT-LENGTH"] = int(handler.headers.get("Content-Length"))
form = cgi.FieldStorage(
form = _MultipartForm(
fp=BytesIO(handler.rfile.read(pdict["CONTENT-LENGTH"])),
headers=handler.headers,
environ={"REQUEST_METHOD": "POST"},
@@ -867,13 +970,13 @@ class ActionUtils:
Always resized to 78x78 BMP.
"""
try:
ctype, pdict = cgi.parse_header(handler.headers.get("Content-Type"))
ctype, pdict = _parse_header(handler.headers.get("Content-Type"))
if ctype != "multipart/form-data":
raise ValueError("Content-Type must be multipart/form-data")
pdict["boundary"] = bytes(pdict["boundary"], "utf-8")
pdict["CONTENT-LENGTH"] = int(handler.headers.get("Content-Length"))
form = cgi.FieldStorage(
form = _MultipartForm(
fp=BytesIO(handler.rfile.read(pdict["CONTENT-LENGTH"])),
headers=handler.headers,
environ={"REQUEST_METHOD": "POST"},
@@ -1117,6 +1220,8 @@ class ActionUtils:
handler.send_header("Content-Type", "image/bmp" if full.lower().endswith(".bmp") else "image/jpeg")
handler.end_headers()
handler.wfile.write(data)
except BrokenPipeError:
pass
except Exception as e:
self.logger.error(f"serve_static_image: {e}")
handler.send_response(404)
@@ -1175,10 +1280,13 @@ class ActionUtils:
handler.send_response(404)
handler.end_headers()
def create_character(self, handler):
def create_character(self, handler, data=None):
"""Create a new character by copying current character's images."""
try:
data = json.loads(handler.rfile.read(int(handler.headers["Content-Length"])).decode("utf-8"))
if data is None:
data = json.loads(handler.rfile.read(int(handler.headers["Content-Length"])).decode("utf-8"))
elif not isinstance(data, dict):
raise ValueError("Invalid JSON payload")
name = (data.get("character_name") or "").strip()
if not name:
raise ValueError("character_name is required")
@@ -1193,10 +1301,13 @@ class ActionUtils:
self.logger.error(f"create_character: {e}")
self._send_error(handler, str(e))
def switch_character(self, handler):
def switch_character(self, handler, data=None):
"""Switch character: persist current images, load selected images as active."""
try:
data = json.loads(handler.rfile.read(int(handler.headers["Content-Length"])).decode("utf-8"))
if data is None:
data = json.loads(handler.rfile.read(int(handler.headers["Content-Length"])).decode("utf-8"))
elif not isinstance(data, dict):
raise ValueError("Invalid JSON payload")
target = (data.get("character_name") or "").strip()
if not target:
raise ValueError("character_name is required")
@@ -1230,10 +1341,13 @@ class ActionUtils:
self.logger.error(f"switch_character: {e}")
self._send_error(handler, str(e))
def delete_character(self, handler):
def delete_character(self, handler, data=None):
"""Delete a character; if it's the current one, switch back to BJORN first."""
try:
data = json.loads(handler.rfile.read(int(handler.headers["Content-Length"])).decode("utf-8"))
if data is None:
data = json.loads(handler.rfile.read(int(handler.headers["Content-Length"])).decode("utf-8"))
elif not isinstance(data, dict):
raise ValueError("Invalid JSON payload")
name = (data.get("character_name") or "").strip()
if not name:
raise ValueError("character_name is required")
@@ -1519,12 +1633,15 @@ class ActionUtils:
self._send_error_response(handler, str(e))
def set_action_enabled(self, handler):
def set_action_enabled(self, handler, data=None):
"""Body: { action_name: str, enabled: 0|1 }"""
try:
length = int(handler.headers.get('Content-Length', 0))
body = handler.rfile.read(length) if length else b'{}'
data = json.loads(body or b'{}')
if data is None:
length = int(handler.headers.get('Content-Length', 0))
body = handler.rfile.read(length) if length else b'{}'
data = json.loads(body or b'{}')
elif not isinstance(data, dict):
raise ValueError("Invalid JSON payload")
action_name = (data.get('action_name') or '').strip()
enabled = 1 if int(data.get('enabled', 0)) else 0
@@ -1539,6 +1656,15 @@ class ActionUtils:
if not rowcount:
raise ValueError(f"Action '{action_name}' not found (b_class)")
# Best-effort sync to actions_studio when present.
try:
self.shared_data.db.execute(
"UPDATE actions_studio SET b_enabled = ?, updated_at = CURRENT_TIMESTAMP WHERE b_class = ?;",
(enabled, action_name)
)
except Exception as e:
self.logger.debug(f"set_action_enabled studio sync skipped for {action_name}: {e}")
out = {"status": "success", "action_name": action_name, "enabled": enabled}
handler.send_response(200)
handler.send_header('Content-Type', 'application/json')
@@ -1579,7 +1705,7 @@ class ActionUtils:
if 'multipart/form-data' not in ctype:
raise ValueError("Content-Type must be multipart/form-data.")
form = cgi.FieldStorage(fp=handler.rfile, headers=handler.headers, environ={'REQUEST_METHOD': 'POST'})
form = _MultipartForm(fp=handler.rfile, headers=handler.headers, environ={'REQUEST_METHOD': 'POST'})
if 'attack_file' not in form:
raise ValueError("No attack_file field in form.")
@@ -1614,11 +1740,14 @@ class ActionUtils:
self.logger.error(f"Error importing attack: {e}")
self._send_error_response(handler, str(e))
def remove_attack(self, handler):
def remove_attack(self, handler, data=None):
"""Remove an attack (file + DB row)."""
try:
body = handler.rfile.read(int(handler.headers.get('Content-Length', 0)) or 0)
data = json.loads(body or "{}")
if data is None:
body = handler.rfile.read(int(handler.headers.get('Content-Length', 0)) or 0)
data = json.loads(body or "{}")
elif not isinstance(data, dict):
raise ValueError("Invalid JSON payload")
attack_name = (data.get("name") or "").strip()
if not attack_name:
raise ValueError("Attack name not provided.")
@@ -1638,11 +1767,14 @@ class ActionUtils:
self.logger.error(f"Error removing attack: {e}")
self._send_error_response(handler, str(e))
def save_attack(self, handler):
def save_attack(self, handler, data=None):
"""Save/update attack source code and refresh DB metadata if b_class changed."""
try:
body = handler.rfile.read(int(handler.headers.get('Content-Length', 0)) or 0)
data = json.loads(body or "{}")
if data is None:
body = handler.rfile.read(int(handler.headers.get('Content-Length', 0)) or 0)
data = json.loads(body or "{}")
elif not isinstance(data, dict):
raise ValueError("Invalid JSON payload")
attack_name = (data.get('name') or '').strip()
content = data.get('content') or ""
if not attack_name or not content:
@@ -1675,11 +1807,14 @@ class ActionUtils:
self.logger.error(f"Error saving attack: {e}")
self._send_error_response(handler, str(e))
def restore_attack(self, handler):
def restore_attack(self, handler, data=None):
"""Restore an attack from default_actions_dir and re-upsert metadata."""
try:
body = handler.rfile.read(int(handler.headers.get('Content-Length', 0)) or 0)
data = json.loads(body or "{}")
if data is None:
body = handler.rfile.read(int(handler.headers.get('Content-Length', 0)) or 0)
data = json.loads(body or "{}")
elif not isinstance(data, dict):
raise ValueError("Invalid JSON payload")
attack_name = (data.get('name') or '').strip()
if not attack_name:
raise ValueError("Attack name not provided.")
@@ -1777,12 +1912,12 @@ class ActionUtils:
except Exception as e: self.logger.error(e); self._err(h, str(e))
def upload_web_image(self, h):
import cgi
try:
ctype, pdict = cgi.parse_header(h.headers.get('Content-Type'))
ctype, pdict = _parse_header(h.headers.get('Content-Type'))
if ctype != 'multipart/form-data': raise ValueError('Content-Type doit être multipart/form-data')
pdict['boundary']=bytes(pdict['boundary'],'utf-8'); pdict['CONTENT-LENGTH']=int(h.headers.get('Content-Length'))
form = cgi.FieldStorage(fp=BytesIO(h.rfile.read(pdict['CONTENT-LENGTH'])),
form = _MultipartForm(fp=BytesIO(h.rfile.read(pdict['CONTENT-LENGTH'])),
headers=h.headers, environ={'REQUEST_METHOD':'POST'}, keep_blank_values=True)
if 'web_image' not in form or not getattr(form['web_image'],'filename',''): raise ValueError('Aucun fichier web_image fourni')
file_item = form['web_image']; filename = self._safe(file_item.filename)
@@ -1823,12 +1958,12 @@ class ActionUtils:
except Exception as e: self.logger.error(e); self._err(h, str(e))
def upload_actions_icon(self, h):
import cgi
try:
ctype, pdict = cgi.parse_header(h.headers.get('Content-Type'))
ctype, pdict = _parse_header(h.headers.get('Content-Type'))
if ctype != 'multipart/form-data': raise ValueError('Content-Type doit être multipart/form-data')
pdict['boundary']=bytes(pdict['boundary'],'utf-8'); pdict['CONTENT-LENGTH']=int(h.headers.get('Content-Length'))
form = cgi.FieldStorage(fp=BytesIO(h.rfile.read(pdict['CONTENT-LENGTH'])),
form = _MultipartForm(fp=BytesIO(h.rfile.read(pdict['CONTENT-LENGTH'])),
headers=h.headers, environ={'REQUEST_METHOD':'POST'}, keep_blank_values=True)
if 'icon_image' not in form or not getattr(form['icon_image'],'filename',''): raise ValueError('Aucun fichier icon_image fourni')
file_item = form['icon_image']; filename = self._safe(file_item.filename)
@@ -1846,7 +1981,7 @@ class ActionUtils:
if fmt in ('JPEG','BMP'): im = im.convert('RGB')
im.save(out, fmt)
data = out.getvalue()
with open(os.path.join(self.web_images_dir, filename), 'wb') as f:
with open(os.path.join(self.actions_icons_dir, filename), 'wb') as f:
f.write(data)
self._send_json(h, {'status':'success','message':'Action icon uploaded','file':filename})
except Exception as e:
@@ -1863,4 +1998,4 @@ class ActionUtils:
h.send_response(200); h.send_header('Content-Type', self._mime(image_path))
h.send_header('Content-Length', str(len(content))); h.end_headers(); h.wfile.write(content)
except Exception as e:
self.logger.error(e); h.send_error(500,"Internal Server Error")
self.logger.error(e); h.send_error(500,"Internal Server Error")

82
web_utils/attack_utils.py
View File

@@ -7,8 +7,86 @@ from __future__ import annotations
import json
import os
import ast
import cgi
import shutil
from io import BytesIO
# --- Multipart form helpers (replaces cgi module removed in Python 3.13) ---
def _parse_header(line):
parts = line.split(';')
key = parts[0].strip()
pdict = {}
for p in parts[1:]:
if '=' in p:
k, v = p.strip().split('=', 1)
pdict[k.strip()] = v.strip().strip('"')
return key, pdict
class _FormField:
__slots__ = ('name', 'filename', 'file', 'value')
def __init__(self, name, filename=None, data=b''):
self.name = name
self.filename = filename
if filename:
self.file = BytesIO(data)
self.value = data
else:
self.value = data.decode('utf-8', errors='replace').strip()
self.file = None
class _MultipartForm:
"""Minimal replacement for _MultipartForm."""
def __init__(self, fp, headers, environ=None, keep_blank_values=False):
import re as _re
self._fields = {}
ct = headers.get('Content-Type', '') if hasattr(headers, 'get') else ''
_, params = _parse_header(ct)
boundary = params.get('boundary', '').encode()
if hasattr(fp, 'read'):
cl = headers.get('Content-Length') if hasattr(headers, 'get') else None
body = fp.read(int(cl)) if cl else fp.read()
else:
body = fp
for part in body.split(b'--' + boundary)[1:]:
part = part.strip(b'\r\n')
if part == b'--' or not part:
continue
sep = b'\r\n\r\n' if b'\r\n\r\n' in part else b'\n\n'
if sep not in part:
continue
hdr, data = part.split(sep, 1)
hdr_s = hdr.decode('utf-8', errors='replace')
nm = _re.search(r'name="([^"]*)"', hdr_s)
fn = _re.search(r'filename="([^"]*)"', hdr_s)
if not nm:
continue
name = nm.group(1)
filename = fn.group(1) if fn else None
field = _FormField(name, filename, data)
if name in self._fields:
existing = self._fields[name]
if isinstance(existing, list):
existing.append(field)
else:
self._fields[name] = [existing, field]
else:
self._fields[name] = field
def __contains__(self, key):
return key in self._fields
def __getitem__(self, key):
return self._fields[key]
def getvalue(self, key, default=None):
if key not in self._fields:
return default
f = self._fields[key]
if isinstance(f, list):
return [x.value for x in f]
return f.value
from typing import Any, Dict, Optional
from urllib.parse import urlparse, parse_qs
import logging
@@ -107,7 +185,7 @@ class AttackUtils:
if 'multipart/form-data' not in ctype:
raise ValueError("Content-Type must be multipart/form-data.")
form = cgi.FieldStorage(fp=handler.rfile, headers=handler.headers, environ={'REQUEST_METHOD': 'POST'})
form = _MultipartForm(fp=handler.rfile, headers=handler.headers, environ={'REQUEST_METHOD': 'POST'})
if 'attack_file' not in form:
raise ValueError("No attack_file field in form.")

8
web_utils/backup_utils.py
View File

@@ -436,14 +436,14 @@ class BackupUtils:
return
try:
with open(backup_path, 'rb') as f:
file_data = f.read()
file_size = os.path.getsize(backup_path)
handler.send_response(200)
handler.send_header('Content-Type', 'application/octet-stream')
handler.send_header('Content-Disposition', f'attachment; filename="{filename}"')
handler.send_header('Content-Length', str(len(file_data)))
handler.send_header('Content-Length', str(file_size))
handler.end_headers()
handler.wfile.write(file_data)
with open(backup_path, 'rb') as f:
shutil.copyfileobj(f, handler.wfile)
except Exception as e:
self.logger.error(f"Error downloading backup: {e}")
handler.send_response(500)

83
web_utils/character_utils.py
View File

@@ -15,9 +15,86 @@ from typing import Any, Dict, Optional
from urllib.parse import urlparse, parse_qs
import io
import cgi
from PIL import Image
# --- Multipart form helpers (replaces cgi module removed in Python 3.13) ---
def _parse_header(line):
parts = line.split(';')
key = parts[0].strip()
pdict = {}
for p in parts[1:]:
if '=' in p:
k, v = p.strip().split('=', 1)
pdict[k.strip()] = v.strip().strip('"')
return key, pdict
class _FormField:
__slots__ = ('name', 'filename', 'file', 'value')
def __init__(self, name, filename=None, data=b''):
self.name = name
self.filename = filename
if filename:
self.file = BytesIO(data)
self.value = data
else:
self.value = data.decode('utf-8', errors='replace').strip()
self.file = None
class _MultipartForm:
"""Minimal replacement for _MultipartForm."""
def __init__(self, fp, headers, environ=None, keep_blank_values=False):
import re as _re
self._fields = {}
ct = headers.get('Content-Type', '') if hasattr(headers, 'get') else ''
_, params = _parse_header(ct)
boundary = params.get('boundary', '').encode()
if hasattr(fp, 'read'):
cl = headers.get('Content-Length') if hasattr(headers, 'get') else None
body = fp.read(int(cl)) if cl else fp.read()
else:
body = fp
for part in body.split(b'--' + boundary)[1:]:
part = part.strip(b'\r\n')
if part == b'--' or not part:
continue
sep = b'\r\n\r\n' if b'\r\n\r\n' in part else b'\n\n'
if sep not in part:
continue
hdr, data = part.split(sep, 1)
hdr_s = hdr.decode('utf-8', errors='replace')
nm = _re.search(r'name="([^"]*)"', hdr_s)
fn = _re.search(r'filename="([^"]*)"', hdr_s)
if not nm:
continue
name = nm.group(1)
filename = fn.group(1) if fn else None
field = _FormField(name, filename, data)
if name in self._fields:
existing = self._fields[name]
if isinstance(existing, list):
existing.append(field)
else:
self._fields[name] = [existing, field]
else:
self._fields[name] = field
def __contains__(self, key):
return key in self._fields
def __getitem__(self, key):
return self._fields[key]
def getvalue(self, key, default=None):
if key not in self._fields:
return default
f = self._fields[key]
if isinstance(f, list):
return [x.value for x in f]
return f.value
from logger import Logger
logger = Logger(name="character_utils.py", level=logging.DEBUG)
@@ -323,14 +400,14 @@ class CharacterUtils:
def upload_character_images(self, handler):
"""Ajoute des images de characters pour une action existante (toujours BMP + numérotation)."""
try:
ctype, pdict = cgi.parse_header(handler.headers.get('Content-Type'))
ctype, pdict = _parse_header(handler.headers.get('Content-Type'))
if ctype != 'multipart/form-data':
raise ValueError('Content-Type doit être multipart/form-data')
pdict['boundary'] = bytes(pdict['boundary'], "utf-8")
pdict['CONTENT-LENGTH'] = int(handler.headers.get('Content-Length'))
form = cgi.FieldStorage(
form = _MultipartForm(
fp=io.BytesIO(handler.rfile.read(pdict['CONTENT-LENGTH'])),
headers=handler.headers,
environ={'REQUEST_METHOD': 'POST'},

13
web_utils/db_utils.py
View File

@@ -33,16 +33,11 @@ class DBUtils:
def _db_table_info(self, table: str):
"""Get table info (primary key and columns)."""
table = self._db_safe_ident(table)
cols = [r["name"] for r in self.shared_data.db.query(f"PRAGMA table_info({table});")]
if not cols:
rows = self.shared_data.db.query(f"PRAGMA table_info({table});")
if not rows:
raise ValueError("Table not found")
pk = None
for r in self.shared_data.db.query(f"PRAGMA table_info({table});"):
if int(r["pk"] or 0) == 1:
pk = r["name"]
break
cols = [r["name"] for r in rows]
pk = next((r["name"] for r in rows if int(r["pk"] or 0) == 1), None)
if not pk:
pk = "id" if "id" in cols else cols[0]
return pk, cols

536
web_utils/debug_utils.py Normal file
View File

@@ -0,0 +1,536 @@
"""
Debug / Profiling utilities for the Bjorn Debug page.
Exposes process-level and per-thread metrics via /proc (no external deps).
Designed for Pi Zero 2: lightweight reads, no subprocess spawning.
OPTIMIZED: minimal allocations, cached tracemalloc, /proc/self/smaps for C memory.
"""
import json
import os
import sys
import threading
import time
import tracemalloc
from logger import Logger
logger = Logger(name="debug_utils")
_SC_CLK_TCK = os.sysconf("SC_CLK_TCK") if hasattr(os, "sysconf") else 100
# ---------------------------------------------------------------------------
# /proc helpers
# ---------------------------------------------------------------------------
def _read_proc_status():
result = {}
try:
with open("/proc/self/status", "r", encoding="utf-8") as f:
for line in f:
if line.startswith("VmRSS:"):
result["vm_rss_kb"] = int(line.split()[1])
elif line.startswith("VmSize:"):
result["vm_size_kb"] = int(line.split()[1])
elif line.startswith("VmPeak:"):
result["vm_peak_kb"] = int(line.split()[1])
elif line.startswith("VmSwap:"):
result["vm_swap_kb"] = int(line.split()[1])
elif line.startswith("FDSize:"):
result["fd_slots"] = int(line.split()[1])
elif line.startswith("Threads:"):
result["kernel_threads"] = int(line.split()[1])
elif line.startswith("RssAnon:"):
result["rss_anon_kb"] = int(line.split()[1])
elif line.startswith("RssFile:"):
result["rss_file_kb"] = int(line.split()[1])
elif line.startswith("RssShmem:"):
result["rss_shmem_kb"] = int(line.split()[1])
except Exception:
pass
return result
def _fd_count():
try:
return len(os.listdir("/proc/self/fd"))
except Exception:
return -1
def _read_open_files():
"""Read open FDs — reuses a single dict to minimize allocations."""
fd_dir = "/proc/self/fd"
fd_map = {}
try:
fds = os.listdir(fd_dir)
except Exception:
return []
for fd in fds:
try:
target = os.readlink(fd_dir + "/" + fd)
except Exception:
target = "???"
if target.startswith("/"):
ftype = "device" if "/dev/" in target else "proc" if target.startswith("/proc/") else "temp" if (target.startswith("/tmp/") or target.startswith("/run/")) else "file"
elif target.startswith("socket:"):
ftype = "socket"
elif target.startswith("pipe:"):
ftype = "pipe"
elif target.startswith("anon_inode:"):
ftype = "anon"
else:
ftype = "other"
entry = fd_map.get(target)
if entry is None:
entry = {"target": target, "type": ftype, "count": 0, "fds": []}
fd_map[target] = entry
entry["count"] += 1
if len(entry["fds"]) < 5:
entry["fds"].append(int(fd))
result = sorted(fd_map.values(), key=lambda x: (-x["count"], x["target"]))
return result
def _read_thread_stats():
threads = []
task_dir = "/proc/self/task"
try:
tids = os.listdir(task_dir)
except Exception:
return threads
for tid in tids:
try:
with open(task_dir + "/" + tid + "/stat", "r", encoding="utf-8") as f:
raw = f.read()
i1 = raw.find("(")
i2 = raw.rfind(")")
if i1 < 0 or i2 < 0:
continue
name = raw[i1 + 1:i2]
fields = raw[i2 + 2:].split()
state = fields[0] if fields else "?"
utime = int(fields[11]) if len(fields) > 11 else 0
stime = int(fields[12]) if len(fields) > 12 else 0
threads.append({
"tid": int(tid),
"name": name,
"state": state,
"cpu_ticks": utime + stime,
})
except Exception:
continue
return threads
def _get_python_threads_rich():
"""Enumerate Python threads with target + current frame. Minimal allocations."""
frames = sys._current_frames()
result = []
for t in threading.enumerate():
ident = t.ident
nid = getattr(t, "native_id", None)
# Target function info
target = getattr(t, "_target", None)
if target is not None:
tf = getattr(target, "__qualname__", getattr(target, "__name__", "?"))
tm = getattr(target, "__module__", "")
# Source file — use __code__ directly (avoids importing inspect)
tfile = ""
code = getattr(target, "__code__", None)
if code:
tfile = getattr(code, "co_filename", "")
else:
tf = "(main)" if t.name == "MainThread" else "(no target)"
tm = ""
tfile = ""
# Current stack — top 5 frames, build compact strings directly
stack = []
frame = frames.get(ident)
depth = 0
while frame is not None and depth < 5:
co = frame.f_code
fn = co.co_filename
# Shorten: last 2 path components
sep = fn.rfind("/")
if sep > 0:
sep2 = fn.rfind("/", 0, sep)
short = fn[sep2 + 1:] if sep2 >= 0 else fn
else:
short = fn
stack.append({
"file": short,
"line": frame.f_lineno,
"func": co.co_name,
})
frame = frame.f_back
depth += 1
# Release frame reference immediately
del frame
result.append({
"name": t.name,
"daemon": t.daemon,
"alive": t.is_alive(),
"ident": ident,
"native_id": nid,
"target_func": tf,
"target_module": tm,
"target_file": tfile,
"stack_top": stack,
})
# Release all frame references
del frames
return result
def _system_cpu_mem():
result = {"cpu_count": 1, "mem_total_kb": 0, "mem_available_kb": 0}
try:
with open("/proc/meminfo", "r", encoding="utf-8") as f:
for line in f:
if line.startswith("MemTotal:"):
result["mem_total_kb"] = int(line.split()[1])
elif line.startswith("MemAvailable:"):
result["mem_available_kb"] = int(line.split()[1])
except Exception:
pass
try:
result["cpu_count"] = len(os.sched_getaffinity(0))
except Exception:
try:
result["cpu_count"] = os.cpu_count() or 1
except Exception:
pass
return result
def _read_smaps_rollup():
"""
Read /proc/self/smaps_rollup for a breakdown of what consumes RSS.
This shows: Shared_Clean, Shared_Dirty, Private_Clean, Private_Dirty,
which helps identify C extension memory vs Python heap vs mmap.
"""
result = {}
try:
with open("/proc/self/smaps_rollup", "r", encoding="utf-8") as f:
for line in f:
parts = line.split()
if len(parts) >= 2:
key = parts[0].rstrip(":")
if key in ("Rss", "Pss", "Shared_Clean", "Shared_Dirty",
"Private_Clean", "Private_Dirty", "Referenced",
"Anonymous", "Swap", "Locked"):
result[key.lower() + "_kb"] = int(parts[1])
except Exception:
pass
return result
# ---------------------------------------------------------------------------
# Cached tracemalloc — take snapshot at most every 5s to reduce overhead
# ---------------------------------------------------------------------------
_tm_cache_lock = threading.Lock()
_tm_cache = None # (current, peak, by_file, by_line)
_tm_cache_time = 0.0
_TM_CACHE_TTL = 5.0 # seconds
def _get_tracemalloc_cached():
"""Return cached tracemalloc data, refreshing at most every 5s."""
global _tm_cache, _tm_cache_time
if not tracemalloc.is_tracing():
return 0, 0, [], []
now = time.monotonic()
with _tm_cache_lock:
if _tm_cache is not None and (now - _tm_cache_time) < _TM_CACHE_TTL:
return _tm_cache
# Take snapshot outside the lock (it's slow)
current, peak = tracemalloc.get_traced_memory()
snap = tracemalloc.take_snapshot()
# Single statistics call — use lineno (more useful), derive file-level client-side
stats_line = snap.statistics("lineno")[:30]
top_by_line = []
file_agg = {}
for s in stats_line:
frame = s.traceback[0] if s.traceback else None
if frame is None:
continue
fn = frame.filename
sep = fn.rfind("/")
if sep > 0:
sep2 = fn.rfind("/", 0, sep)
short = fn[sep2 + 1:] if sep2 >= 0 else fn
else:
short = fn
top_by_line.append({
"file": short,
"full_path": fn,
"line": frame.lineno,
"size_kb": round(s.size / 1024, 1),
"count": s.count,
})
# Aggregate by file
if fn not in file_agg:
file_agg[fn] = {"file": short, "full_path": fn, "size_kb": 0, "count": 0}
file_agg[fn]["size_kb"] += round(s.size / 1024, 1)
file_agg[fn]["count"] += s.count
# Also get file-level stats for files that don't appear in line-level top
stats_file = snap.statistics("filename")[:20]
for s in stats_file:
fn = str(s.traceback) if hasattr(s.traceback, '__str__') else ""
# traceback for filename stats is just the filename
raw_fn = s.traceback[0].filename if s.traceback else fn
if raw_fn not in file_agg:
sep = raw_fn.rfind("/")
if sep > 0:
sep2 = raw_fn.rfind("/", 0, sep)
short = raw_fn[sep2 + 1:] if sep2 >= 0 else raw_fn
else:
short = raw_fn
file_agg[raw_fn] = {"file": short, "full_path": raw_fn, "size_kb": 0, "count": 0}
entry = file_agg[raw_fn]
# Use the larger of aggregated or direct stats
direct_kb = round(s.size / 1024, 1)
if direct_kb > entry["size_kb"]:
entry["size_kb"] = direct_kb
if s.count > entry["count"]:
entry["count"] = s.count
top_by_file = sorted(file_agg.values(), key=lambda x: -x["size_kb"])[:20]
# Release snapshot immediately
del snap
result = (current, peak, top_by_file, top_by_line)
with _tm_cache_lock:
_tm_cache = result
_tm_cache_time = now
return result
# ---------------------------------------------------------------------------
# Snapshot + history ring buffer
# ---------------------------------------------------------------------------
_MAX_HISTORY = 120
_history_lock = threading.Lock()
_history = []
_prev_thread_ticks = {}
_prev_proc_ticks = 0
_prev_wall = 0.0
def _take_snapshot():
global _prev_thread_ticks, _prev_proc_ticks, _prev_wall
now = time.time()
wall_delta = now - _prev_wall if _prev_wall > 0 else 1.0
tick_budget = wall_delta * _SC_CLK_TCK
# Process-level
status = _read_proc_status()
fd_open = _fd_count()
sys_info = _system_cpu_mem()
smaps = _read_smaps_rollup()
# Thread CPU from /proc
raw_threads = _read_thread_stats()
thread_details = []
new_ticks_map = {}
total_proc_ticks = 0
for t in raw_threads:
tid = t["tid"]
prev = _prev_thread_ticks.get(tid, t["cpu_ticks"])
delta = max(0, t["cpu_ticks"] - prev)
cpu_pct = (delta / tick_budget * 100.0) if tick_budget > 0 else 0.0
new_ticks_map[tid] = t["cpu_ticks"]
total_proc_ticks += t["cpu_ticks"]
thread_details.append({
"tid": tid,
"name": t["name"],
"state": t["state"],
"cpu_pct": round(cpu_pct, 2),
"cpu_ticks_total": t["cpu_ticks"],
})
thread_details.sort(key=lambda x: x["cpu_pct"], reverse=True)
proc_delta = total_proc_ticks - _prev_proc_ticks if _prev_proc_ticks else 0
proc_cpu_pct = (proc_delta / tick_budget * 100.0) if tick_budget > 0 else 0.0
_prev_thread_ticks = new_ticks_map
_prev_proc_ticks = total_proc_ticks
_prev_wall = now
# Python threads
py_threads = _get_python_threads_rich()
# Match kernel TIDs to Python threads
native_to_py = {}
for pt in py_threads:
nid = pt.get("native_id")
if nid is not None:
native_to_py[nid] = pt
for td in thread_details:
pt = native_to_py.get(td["tid"])
if pt:
td["py_name"] = pt["name"]
td["py_target"] = pt.get("target_func", "")
td["py_module"] = pt.get("target_module", "")
td["py_file"] = pt.get("target_file", "")
if pt.get("stack_top"):
top = pt["stack_top"][0]
td["py_current"] = f"{top['file']}:{top['line']} {top['func']}()"
# tracemalloc (cached, refreshes every 5s)
tm_current, tm_peak, tm_by_file, tm_by_line = _get_tracemalloc_cached()
# Open files
open_files = _read_open_files()
# Memory breakdown
rss_kb = status.get("vm_rss_kb", 0)
tm_current_kb = round(tm_current / 1024, 1)
# C/native memory = RSS - Python traced (approximation)
rss_anon_kb = status.get("rss_anon_kb", 0)
rss_file_kb = status.get("rss_file_kb", 0)
snapshot = {
"ts": round(now, 3),
"proc_cpu_pct": round(proc_cpu_pct, 2),
"rss_kb": rss_kb,
"vm_size_kb": status.get("vm_size_kb", 0),
"vm_peak_kb": status.get("vm_peak_kb", 0),
"vm_swap_kb": status.get("vm_swap_kb", 0),
"fd_open": fd_open,
"fd_slots": status.get("fd_slots", 0),
"kernel_threads": status.get("kernel_threads", 0),
"py_thread_count": len(py_threads),
"sys_cpu_count": sys_info["cpu_count"],
"sys_mem_total_kb": sys_info["mem_total_kb"],
"sys_mem_available_kb": sys_info["mem_available_kb"],
# Memory breakdown
"rss_anon_kb": rss_anon_kb,
"rss_file_kb": rss_file_kb,
"rss_shmem_kb": status.get("rss_shmem_kb", 0),
"private_dirty_kb": smaps.get("private_dirty_kb", 0),
"private_clean_kb": smaps.get("private_clean_kb", 0),
"shared_dirty_kb": smaps.get("shared_dirty_kb", 0),
"shared_clean_kb": smaps.get("shared_clean_kb", 0),
# Data
"threads": thread_details,
"py_threads": py_threads,
"tracemalloc_active": tracemalloc.is_tracing(),
"tracemalloc_current_kb": tm_current_kb,
"tracemalloc_peak_kb": round(tm_peak / 1024, 1),
"tracemalloc_by_file": tm_by_file,
"tracemalloc_by_line": tm_by_line,
"open_files": open_files,
}
with _history_lock:
_history.append({
"ts": snapshot["ts"],
"proc_cpu_pct": snapshot["proc_cpu_pct"],
"rss_kb": rss_kb,
"fd_open": fd_open,
"py_thread_count": snapshot["py_thread_count"],
"kernel_threads": snapshot["kernel_threads"],
"vm_swap_kb": snapshot["vm_swap_kb"],
"private_dirty_kb": snapshot["private_dirty_kb"],
})
if len(_history) > _MAX_HISTORY:
del _history[: len(_history) - _MAX_HISTORY]
return snapshot
# ---------------------------------------------------------------------------
# WebUtils class
# ---------------------------------------------------------------------------
class DebugUtils:
def __init__(self, shared_data):
self.shared_data = shared_data
def get_snapshot(self, handler):
try:
data = _take_snapshot()
self._send_json(handler, data)
except Exception as exc:
logger.error(f"debug snapshot error: {exc}")
self._send_json(handler, {"error": str(exc)}, status=500)
def get_history(self, handler):
try:
with _history_lock:
data = list(_history)
self._send_json(handler, {"history": data})
except Exception as exc:
logger.error(f"debug history error: {exc}")
self._send_json(handler, {"error": str(exc)}, status=500)
def toggle_tracemalloc(self, data):
global _tm_cache, _tm_cache_time
action = data.get("action", "status")
try:
if action == "start":
if not tracemalloc.is_tracing():
tracemalloc.start(int(data.get("nframes", 10)))
return {"status": "ok", "tracing": True}
elif action == "stop":
if tracemalloc.is_tracing():
tracemalloc.stop()
with _tm_cache_lock:
_tm_cache = None
_tm_cache_time = 0.0
return {"status": "ok", "tracing": False}
else:
return {"status": "ok", "tracing": tracemalloc.is_tracing()}
except Exception as exc:
return {"status": "error", "message": str(exc)}
def get_gc_stats(self, handler):
import gc
try:
counts = gc.get_count()
thresholds = gc.get_threshold()
self._send_json(handler, {
"gc_enabled": gc.isenabled(),
"counts": {"gen0": counts[0], "gen1": counts[1], "gen2": counts[2]},
"thresholds": {"gen0": thresholds[0], "gen1": thresholds[1], "gen2": thresholds[2]},
})
except Exception as exc:
self._send_json(handler, {"error": str(exc)}, status=500)
def force_gc(self, data):
import gc
try:
return {"status": "ok", "collected": gc.collect()}
except Exception as exc:
return {"status": "error", "message": str(exc)}
@staticmethod
def _send_json(handler, data, status=200):
handler.send_response(status)
handler.send_header("Content-Type", "application/json")
handler.end_headers()
handler.wfile.write(json.dumps(data, default=str).encode("utf-8"))

3
web_utils/file_utils.py
View File

@@ -7,7 +7,6 @@ from __future__ import annotations
import os
import json
import shutil
import cgi
from pathlib import Path
from io import BytesIO
from typing import Any, Dict, Optional
@@ -155,7 +154,7 @@ class FileUtils:
handler.send_header("Content-Disposition", f'attachment; filename="{os.path.basename(file_path)}"')
handler.end_headers()
with open(file_path, 'rb') as file:
handler.wfile.write(file.read())
shutil.copyfileobj(file, handler.wfile)
else:
handler.send_response(404)
handler.end_headers()

109
web_utils/image_utils.py
View File

@@ -10,6 +10,85 @@ from logger import Logger
logger = Logger(name="image_utils.py", level=logging.DEBUG)
# --- Multipart form helpers (replaces cgi module removed in Python 3.13) ---
def _parse_header(line):
parts = line.split(';')
key = parts[0].strip()
pdict = {}
for p in parts[1:]:
if '=' in p:
k, v = p.strip().split('=', 1)
pdict[k.strip()] = v.strip().strip('"')
return key, pdict
class _FormField:
__slots__ = ('name', 'filename', 'file', 'value')
def __init__(self, name, filename=None, data=b''):
self.name = name
self.filename = filename
if filename:
self.file = BytesIO(data)
self.value = data
else:
self.value = data.decode('utf-8', errors='replace').strip()
self.file = None
class _MultipartForm:
"""Minimal replacement for _MultipartForm."""
def __init__(self, fp, headers, environ=None, keep_blank_values=False):
import re as _re
self._fields = {}
ct = headers.get('Content-Type', '') if hasattr(headers, 'get') else ''
_, params = _parse_header(ct)
boundary = params.get('boundary', '').encode()
if hasattr(fp, 'read'):
cl = headers.get('Content-Length') if hasattr(headers, 'get') else None
body = fp.read(int(cl)) if cl else fp.read()
else:
body = fp
for part in body.split(b'--' + boundary)[1:]:
part = part.strip(b'\r\n')
if part == b'--' or not part:
continue
sep = b'\r\n\r\n' if b'\r\n\r\n' in part else b'\n\n'
if sep not in part:
continue
hdr, data = part.split(sep, 1)
hdr_s = hdr.decode('utf-8', errors='replace')
nm = _re.search(r'name="([^"]*)"', hdr_s)
fn = _re.search(r'filename="([^"]*)"', hdr_s)
if not nm:
continue
name = nm.group(1)
filename = fn.group(1) if fn else None
field = _FormField(name, filename, data)
if name in self._fields:
existing = self._fields[name]
if isinstance(existing, list):
existing.append(field)
else:
self._fields[name] = [existing, field]
else:
self._fields[name] = field
def __contains__(self, key):
return key in self._fields
def __getitem__(self, key):
return self._fields[key]
def getvalue(self, key, default=None):
if key not in self._fields:
return default
f = self._fields[key]
if isinstance(f, list):
return [x.value for x in f]
return f.value
ALLOWED_IMAGE_EXTS = {'.bmp', '.png', '.jpg', '.jpeg', '.gif', '.ico', '.webp'}
class ImageUtils:
@@ -149,12 +228,12 @@ class ImageUtils:
def upload_status_image(self, h):
"""Add/replace <action>/<action>.bmp (always 28x28 BMP)."""
import cgi
try:
ctype, pdict = cgi.parse_header(h.headers.get('Content-Type'))
ctype, pdict = _parse_header(h.headers.get('Content-Type'))
if ctype != 'multipart/form-data': raise ValueError('Content-Type doit être multipart/form-data')
pdict['boundary']=bytes(pdict['boundary'],'utf-8'); pdict['CONTENT-LENGTH']=int(h.headers.get('Content-Length'))
form = cgi.FieldStorage(fp=BytesIO(h.rfile.read(pdict['CONTENT-LENGTH'])),
form = _MultipartForm(fp=BytesIO(h.rfile.read(pdict['CONTENT-LENGTH'])),
headers=h.headers, environ={'REQUEST_METHOD':'POST'}, keep_blank_values=True)
for key in ('type','action_name','status_image'):
if key not in form: raise ValueError(f'Missing field: {key}')
@@ -179,12 +258,12 @@ class ImageUtils:
except Exception as e: self.logger.error(e); self._err(h, str(e))
def upload_static_image(self, h):
import cgi
try:
ctype, pdict = cgi.parse_header(h.headers.get('Content-Type'))
ctype, pdict = _parse_header(h.headers.get('Content-Type'))
if ctype != 'multipart/form-data': raise ValueError('Content-Type must be multipart/form-data')
pdict['boundary']=bytes(pdict['boundary'],'utf-8'); pdict['CONTENT-LENGTH']=int(h.headers.get('Content-Length'))
form = cgi.FieldStorage(fp=BytesIO(h.rfile.read(pdict['CONTENT-LENGTH'])),
form = _MultipartForm(fp=BytesIO(h.rfile.read(pdict['CONTENT-LENGTH'])),
headers=h.headers, environ={'REQUEST_METHOD':'POST'}, keep_blank_values=True)
if 'static_image' not in form or not getattr(form['static_image'],'filename',''): raise ValueError('No static_image provided')
filename = self._safe(form['static_image'].filename); base, _ = os.path.splitext(filename); filename = base + '.bmp'
@@ -216,12 +295,12 @@ class ImageUtils:
except Exception as e: self.logger.error(e); self._err(h, str(e))
def upload_web_image(self, h):
import cgi
try:
ctype, pdict = cgi.parse_header(h.headers.get('Content-Type'))
ctype, pdict = _parse_header(h.headers.get('Content-Type'))
if ctype != 'multipart/form-data': raise ValueError('Content-Type doit être multipart/form-data')
pdict['boundary']=bytes(pdict['boundary'],'utf-8'); pdict['CONTENT-LENGTH']=int(h.headers.get('Content-Length'))
form = cgi.FieldStorage(fp=BytesIO(h.rfile.read(pdict['CONTENT-LENGTH'])),
form = _MultipartForm(fp=BytesIO(h.rfile.read(pdict['CONTENT-LENGTH'])),
headers=h.headers, environ={'REQUEST_METHOD':'POST'}, keep_blank_values=True)
if 'web_image' not in form or not getattr(form['web_image'],'filename',''): raise ValueError('Aucun fichier web_image fourni')
file_item = form['web_image']; filename = self._safe(file_item.filename)
@@ -250,12 +329,12 @@ class ImageUtils:
except Exception as e: self.logger.error(e); self._err(h, str(e))
def upload_actions_icon(self, h):
import cgi
try:
ctype, pdict = cgi.parse_header(h.headers.get('Content-Type'))
ctype, pdict = _parse_header(h.headers.get('Content-Type'))
if ctype != 'multipart/form-data': raise ValueError('Content-Type doit être multipart/form-data')
pdict['boundary']=bytes(pdict['boundary'],'utf-8'); pdict['CONTENT-LENGTH']=int(h.headers.get('Content-Length'))
form = cgi.FieldStorage(fp=BytesIO(h.rfile.read(pdict['CONTENT-LENGTH'])),
form = _MultipartForm(fp=BytesIO(h.rfile.read(pdict['CONTENT-LENGTH'])),
headers=h.headers, environ={'REQUEST_METHOD':'POST'}, keep_blank_values=True)
if 'icon_image' not in form or not getattr(form['icon_image'],'filename',''): raise ValueError('Aucun fichier icon_image fourni')
file_item = form['icon_image']; filename = self._safe(file_item.filename)
@@ -315,12 +394,12 @@ class ImageUtils:
def replace_image(self, h):
"""Replace image. For type='action': status icon here; character images delegated to CharacterUtils."""
import cgi
try:
ctype, pdict = cgi.parse_header(h.headers.get('Content-Type'))
ctype, pdict = _parse_header(h.headers.get('Content-Type'))
if ctype != 'multipart/form-data': raise ValueError('Content-Type must be multipart/form-data')
pdict['boundary']=bytes(pdict['boundary'],'utf-8'); pdict['CONTENT-LENGTH']=int(h.headers.get('Content-Length'))
form = cgi.FieldStorage(fp=BytesIO(h.rfile.read(pdict['CONTENT-LENGTH'])),
form = _MultipartForm(fp=BytesIO(h.rfile.read(pdict['CONTENT-LENGTH'])),
headers=h.headers, environ={'REQUEST_METHOD':'POST'}, keep_blank_values=True)
tp = form.getvalue('type'); image_name = self._safe(form.getvalue('image_name') or '')
file_item = form['new_image'] if 'new_image' in form else None

110
web_utils/index_utils.py
View File

@@ -213,48 +213,32 @@ class IndexUtils:
return ts
except Exception:
pass
ts = int(time.time())
self._cfg_set("first_init_ts", ts)
return ts
# ---------------------- Monitoring ressources ----------------------
def _cpu_pct(self) -> int:
try:
return int(psutil.cpu_percent(interval=0.5))
except Exception:
return 0
def _mem_bytes(self) -> Tuple[int, int]:
try:
vm = psutil.virtual_memory()
return int(vm.total - vm.available), int(vm.total)
except Exception:
try:
info = self._read_text("/proc/meminfo") or ""
def kb(k):
line = next((l for l in info.splitlines() if l.startswith(k + ":")), None)
return int(line.split()[1]) * 1024 if line else 0
total = kb("MemTotal")
free = kb("MemFree") + kb("Buffers") + kb("Cached")
used = max(0, total - free)
return used, total
except Exception:
return 0, 0
def _disk_bytes(self) -> Tuple[int, int]:
try:
usage = psutil.disk_usage("/")
return int(usage.used), int(usage.total)
except Exception:
try:
st = os.statvfs("/")
total = st.f_frsize * st.f_blocks
free = st.f_frsize * st.f_bavail
return int(total - free), int(total)
except Exception:
return 0, 0
return 0
def _battery_probe(self) -> Dict[str, Any]:
try:
# Prefer runtime battery telemetry (PiSugar/shared_data) when available.
present = bool(getattr(self.shared_data, "battery_present", False))
last_update = float(getattr(self.shared_data, "battery_last_update", 0.0))
source = str(getattr(self.shared_data, "battery_source", "shared"))
if last_update > 0 and (present or source == "none"):
level = int(getattr(self.shared_data, "battery_percent", 0))
charging = bool(getattr(self.shared_data, "battery_is_charging", False))
state = "Charging" if charging else "Discharging"
if not present:
state = "No battery"
return {
"present": present,
"level_pct": max(0, min(100, level)),
"state": state,
"charging": charging,
"voltage": getattr(self.shared_data, "battery_voltage", None),
"source": source,
"updated_at": last_update,
}
except Exception:
pass
base = "/sys/class/power_supply"
try:
if not os.path.isdir(base):
@@ -414,11 +398,53 @@ class IndexUtils:
except Exception:
return 0
def _cpu_pct(self) -> int:
# OPTIMIZATION: Use shared_data from display loop to avoid blocking 0.5s
# Old method:
# try:
# return int(psutil.cpu_percent(interval=0.5))
# except Exception:
# return 0
return int(getattr(self.shared_data, "system_cpu", 0))
def _mem_bytes(self) -> Tuple[int, int]:
# OPTIMIZATION: Use shared_data from display loop
# Old method:
# try:
# vm = psutil.virtual_memory()
# return int(vm.total - vm.available), int(vm.total)
# except Exception:
# try:
# info = self._read_text("/proc/meminfo") or ""
# def kb(k):
# line = next((l for l in info.splitlines() if l.startswith(k + ":")), None)
# return int(line.split()[1]) * 1024 if line else 0
# total = kb("MemTotal")
# free = kb("MemFree") + kb("Buffers") + kb("Cached")
# used = max(0, total - free)
# return used, total
# except Exception:
# return 0, 0
return int(getattr(self.shared_data, "system_mem_used", 0)), int(getattr(self.shared_data, "system_mem_total", 0))
def _disk_bytes(self) -> Tuple[int, int]:
try:
usage = psutil.disk_usage("/")
return int(usage.used), int(usage.total)
except Exception:
try:
st = os.statvfs("/")
total = st.f_frsize * st.f_blocks
free = st.f_frsize * st.f_bavail
return int(total - free), int(total)
except Exception:
return 0, 0
def _alive_hosts_db(self) -> Tuple[int, int]:
try:
row = self.db.query_one(
"""
SELECT
SELECT
SUM(CASE WHEN alive=1 THEN 1 ELSE 0 END) AS alive,
COUNT(*) AS total
FROM hosts
@@ -462,7 +488,7 @@ class IndexUtils:
def _zombies_count_db(self) -> int:
try:
row = self.db.query_one("SELECT COUNT(*) AS c FROM stats WHERE id=1;")
row = self.db.query_one("SELECT COALESCE(zombie_count, 0) AS c FROM stats WHERE id=1;")
if row and row.get("c") is not None:
return int(row["c"])
except Exception:

23
web_utils/netkb_utils.py
View File

@@ -94,20 +94,21 @@ class NetKBUtils:
def serve_network_data(self, handler):
"""Serve network data as HTML table."""
try:
html = ['<table><tr><th>ESSID</th><th>IP</th><th>Hostname</th><th>MAC Address</th><th>Vendor</th><th>Ports</th></tr>']
import html as _html
rows = ['<table><tr><th>ESSID</th><th>IP</th><th>Hostname</th><th>MAC Address</th><th>Vendor</th><th>Ports</th></tr>']
for h in self.shared_data.db.get_all_hosts():
if int(h.get("alive") or 0) != 1:
continue
html.append(
f"<tr><td>{h.get('essid', '')}</td>"
f"<td>{h.get('ips', '')}</td>"
f"<td>{h.get('hostnames', '')}</td>"
f"<td>{h.get('mac_address', '')}</td>"
f"<td>{h.get('vendor', '')}</td>"
f"<td>{h.get('ports', '')}</td></tr>"
rows.append(
f"<tr><td>{_html.escape(str(h.get('essid') or ''))}</td>"
f"<td>{_html.escape(str(h.get('ips') or ''))}</td>"
f"<td>{_html.escape(str(h.get('hostnames') or ''))}</td>"
f"<td>{_html.escape(str(h.get('mac_address') or ''))}</td>"
f"<td>{_html.escape(str(h.get('vendor') or ''))}</td>"
f"<td>{_html.escape(str(h.get('ports') or ''))}</td></tr>"
)
html.append("</table>")
table_html = "\n".join(html)
rows.append("</table>")
table_html = "\n".join(rows)
handler.send_response(200)
handler.send_header("Content-type", "text/html")
handler.end_headers()
@@ -193,7 +194,7 @@ class NetKBUtils:
limit = int((qs.get("limit", ["200"])[0] or 200))
include_superseded = (qs.get("include_superseded", ["true"])[0] or "true").lower() in ("1", "true", "yes", "on")
if not action or mac is None:
if not action or not mac:
raise ValueError("missing required parameters: action, mac")
db = self.shared_data.db

9
web_utils/orchestrator_utils.py
View File

@@ -42,7 +42,7 @@ class OrchestratorUtils:
raise Exception(f"No data found for IP: {ip}")
action_key = action_instance.action_name
self.logger.info(f"Executing {action_key} on {ip}:{port}")
self.logger.info(f"Executing [MANUAL]: {action_key} on {ip}:{port}")
result = action_instance.execute(ip, port, row, action_key)
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
@@ -97,7 +97,10 @@ class OrchestratorUtils:
"""Start the orchestrator."""
try:
bjorn_instance = self.shared_data.bjorn_instance
self.shared_data.manual_mode = False
if getattr(self.shared_data, "ai_mode", False):
self.shared_data.operation_mode = "AI"
else:
self.shared_data.operation_mode = "AUTO"
self.shared_data.orchestrator_should_exit = False
bjorn_instance.start_orchestrator()
return {"status": "success", "message": "Orchestrator starting..."}
@@ -109,7 +112,7 @@ class OrchestratorUtils:
"""Stop the orchestrator."""
try:
bjorn_instance = self.shared_data.bjorn_instance
self.shared_data.manual_mode = False
self.shared_data.operation_mode = "MANUAL"
bjorn_instance.stop_orchestrator()
self.shared_data.orchestrator_should_exit = True
return {"status": "success", "message": "Orchestrator stopping..."}

194
web_utils/rl_utils.py Normal file
View File

@@ -0,0 +1,194 @@
import json
from typing import Any, Dict, List
from ai_engine import get_or_create_ai_engine
from logger import Logger
logger = Logger(name="rl_utils")
class RLUtils:
"""
Backend utilities for RL/AI dashboard endpoints.
"""
def __init__(self, shared_data):
self.shared_data = shared_data
# Use the process-level singleton to avoid reloading model weights
self.ai_engine = get_or_create_ai_engine(shared_data)
def get_stats(self, handler) -> None:
"""
API Endpoint: GET /api/rl/stats
"""
try:
ai_stats = self.ai_engine.get_stats() if self.ai_engine else {}
ai_stats = ai_stats if isinstance(ai_stats, dict) else {}
episodes = self._query_scalar("SELECT COUNT(*) AS c FROM ml_features", key="c", default=0)
recent_activity = self._query_rows(
"""
SELECT action_name AS action, reward, success, timestamp
FROM ml_features
ORDER BY timestamp DESC
LIMIT 5
"""
)
payload = {
"enabled": bool(self.ai_engine is not None),
"episodes": int(episodes),
"epsilon": float(getattr(self.shared_data, "ai_exploration_rate", 0.1)),
"q_table_size": int(ai_stats.get("q_table_size", 0) or 0),
"recent_activity": recent_activity,
"last_loss": 0.0,
"status": self.shared_data.get_status().get("status", "Idle"),
"ai_mode": bool(getattr(self.shared_data, "ai_mode", False)),
"mode": str(getattr(self.shared_data, "operation_mode", "AUTO")),
"manual_mode": bool(getattr(self.shared_data, "manual_mode", False)),
"model_loaded": bool(ai_stats.get("model_loaded", False)),
"model_version": ai_stats.get("model_version"),
"model_trained_at": ai_stats.get("model_trained_at"),
"model_accuracy": ai_stats.get("model_accuracy"),
"training_samples": ai_stats.get("training_samples"),
}
payload.update(self._extract_model_meta())
self._send_json(handler, payload)
except Exception as exc:
logger.error(f"Error fetching AI stats: {exc}")
self._send_json(handler, {"error": str(exc)}, 500)
def get_training_history(self, handler) -> None:
"""
API Endpoint: GET /api/rl/history
"""
try:
rows = self._query_rows(
"""
SELECT id, id AS batch_id, record_count, file_path AS filepath, created_at AS timestamp
FROM ml_export_batches
ORDER BY created_at DESC
LIMIT 50
"""
)
self._send_json(handler, {"history": rows})
except Exception as exc:
logger.error(f"Error fetching training history: {exc}")
self._send_json(handler, {"error": str(exc)}, 500)
def get_recent_experiences(self, handler) -> None:
"""
API Endpoint: GET /api/rl/experiences
"""
try:
rows = self._query_rows(
"""
SELECT action_name, reward, success, duration_seconds, timestamp, ip_address
FROM ml_features
ORDER BY timestamp DESC
LIMIT 20
"""
)
self._send_json(handler, {"experiences": rows})
except Exception as exc:
logger.error(f"Error fetching experiences: {exc}")
self._send_json(handler, {"error": str(exc)}, 500)
def set_mode(self, handler, data: Dict) -> Dict:
"""
API Endpoint: POST /api/rl/config
"""
try:
mode = str(data.get("mode", "")).upper()
if mode not in ["MANUAL", "AUTO", "AI"]:
return {"status": "error", "message": f"Invalid mode: {mode}"}
self.shared_data.operation_mode = mode
bjorn = getattr(self.shared_data, "bjorn_instance", None)
if bjorn:
if mode == "MANUAL":
bjorn.stop_orchestrator()
else:
bjorn.check_and_start_orchestrator()
else:
logger.warning("Bjorn instance not found in shared_data")
return {
"status": "ok",
"mode": mode,
"manual_mode": bool(getattr(self.shared_data, "manual_mode", False)),
"ai_mode": bool(getattr(self.shared_data, "ai_mode", False)),
}
except Exception as exc:
logger.error(f"Error setting mode: {exc}")
return {"status": "error", "message": str(exc)}
# ------------------------------------------------------------------ helpers
def _extract_model_meta(self) -> Dict[str, Any]:
"""
Returns model metadata useful for abstract visualization only.
"""
default = {
"model_param_count": 0,
"model_layer_count": 0,
"model_feature_count": 0,
}
if not self.ai_engine or not self.ai_engine.model_loaded:
return default
try:
param_count = 0
layer_count = 0
weights = self.ai_engine.model_weights or {}
for name, arr in weights.items():
shape = getattr(arr, "shape", None)
if shape is not None:
try:
size = int(arr.size)
except Exception:
size = 0
param_count += max(0, size)
if isinstance(name, str) and name.startswith("w"):
layer_count += 1
feature_count = 0
cfg = self.ai_engine.model_config or {}
arch = cfg.get("architecture", {}) if isinstance(cfg, dict) else {}
feats = arch.get("feature_names", []) if isinstance(arch, dict) else []
if isinstance(feats, list):
feature_count = len(feats)
return {
"model_param_count": int(param_count),
"model_layer_count": int(layer_count),
"model_feature_count": int(feature_count),
}
except Exception as exc:
logger.error(f"Failed extracting model meta: {exc}")
return default
def _query_rows(self, sql: str) -> List[Dict[str, Any]]:
try:
return self.shared_data.db.query(sql) or []
except Exception as exc:
logger.error(f"DB query failed: {exc}")
return []
def _query_scalar(self, sql: str, key: str, default: int = 0) -> int:
rows = self._query_rows(sql)
if not rows:
return default
try:
return int(rows[0].get(key, default) or default)
except Exception:
return default
def _send_json(self, handler, data, status: int = 200):
handler.send_response(status)
handler.send_header("Content-Type", "application/json")
handler.end_headers()
handler.wfile.write(json.dumps(data).encode("utf-8"))

83
web_utils/script_utils.py
View File

@@ -12,8 +12,85 @@ import threading
import importlib.util
import ast
import html
import cgi
from pathlib import Path
# --- Multipart form helpers (replaces cgi module removed in Python 3.13) ---
def _parse_header(line):
parts = line.split(';')
key = parts[0].strip()
pdict = {}
for p in parts[1:]:
if '=' in p:
k, v = p.strip().split('=', 1)
pdict[k.strip()] = v.strip().strip('"')
return key, pdict
class _FormField:
__slots__ = ('name', 'filename', 'file', 'value')
def __init__(self, name, filename=None, data=b''):
self.name = name
self.filename = filename
if filename:
self.file = BytesIO(data)
self.value = data
else:
self.value = data.decode('utf-8', errors='replace').strip()
self.file = None
class _MultipartForm:
"""Minimal replacement for _MultipartForm."""
def __init__(self, fp, headers, environ=None, keep_blank_values=False):
import re as _re
self._fields = {}
ct = headers.get('Content-Type', '') if hasattr(headers, 'get') else ''
_, params = _parse_header(ct)
boundary = params.get('boundary', '').encode()
if hasattr(fp, 'read'):
cl = headers.get('Content-Length') if hasattr(headers, 'get') else None
body = fp.read(int(cl)) if cl else fp.read()
else:
body = fp
for part in body.split(b'--' + boundary)[1:]:
part = part.strip(b'\r\n')
if part == b'--' or not part:
continue
sep = b'\r\n\r\n' if b'\r\n\r\n' in part else b'\n\n'
if sep not in part:
continue
hdr, data = part.split(sep, 1)
hdr_s = hdr.decode('utf-8', errors='replace')
nm = _re.search(r'name="([^"]*)"', hdr_s)
fn = _re.search(r'filename="([^"]*)"', hdr_s)
if not nm:
continue
name = nm.group(1)
filename = fn.group(1) if fn else None
field = _FormField(name, filename, data)
if name in self._fields:
existing = self._fields[name]
if isinstance(existing, list):
existing.append(field)
else:
self._fields[name] = [existing, field]
else:
self._fields[name] = field
def __contains__(self, key):
return key in self._fields
def __getitem__(self, key):
return self._fields[key]
def getvalue(self, key, default=None):
if key not in self._fields:
return default
f = self._fields[key]
if isinstance(f, list):
return [x.value for x in f]
return f.value
from typing import Any, Dict, Optional, List
from io import BytesIO
import logging
@@ -439,7 +516,7 @@ class ScriptUtils:
def upload_script(self, handler) -> None:
"""Upload a new script file."""
try:
form = cgi.FieldStorage(
form = _MultipartForm(
fp=handler.rfile,
headers=handler.headers,
environ={'REQUEST_METHOD': 'POST'}
@@ -519,7 +596,7 @@ class ScriptUtils:
def upload_project(self, handler) -> None:
"""Upload a project with multiple files."""
try:
form = cgi.FieldStorage(
form = _MultipartForm(
fp=handler.rfile,
headers=handler.headers,
environ={'REQUEST_METHOD': 'POST'}

51
web_utils/system_utils.py
View File

@@ -238,11 +238,11 @@ class SystemUtils:
return {"status": "error", "message": str(e)}
def serve_current_config(self, handler):
"""Serve current configuration as JSON."""
"""Serve current configuration as JSON (Optimized via SharedData cache)."""
handler.send_response(200)
handler.send_header("Content-type", "application/json")
handler.end_headers()
handler.wfile.write(json.dumps(self.shared_data.config).encode('utf-8'))
handler.wfile.write(self.shared_data.config_json.encode('utf-8'))
def restore_default_config(self, handler):
"""Restore default configuration."""
@@ -318,17 +318,46 @@ class SystemUtils:
finally:
self.logger.info("SSE stream closed")
def _parse_progress(self):
"""Parse bjorn_progress ('42%', '', 0, '100%') → int 0-100."""
raw = getattr(self.shared_data, "bjorn_progress", 0)
if isinstance(raw, (int, float)):
return max(0, min(int(raw), 100))
if isinstance(raw, str):
cleaned = raw.strip().rstrip('%').strip()
if not cleaned:
return 0
try:
return max(0, min(int(cleaned), 100))
except (ValueError, TypeError):
return 0
return 0
def serve_bjorn_status(self, handler):
"""Serve Bjorn status information."""
try:
status_data = {
"status": self.shared_data.bjorn_orch_status,
"status2": self.shared_data.bjorn_status_text2,
"image_path": "/bjorn_status_image?t=" + str(int(time.time()))
# 🟢 PROGRESS — parse "42%" / "" / 0 safely
"progress": self._parse_progress(),
"image_path": "/bjorn_status_image?t=" + str(int(time.time())),
"battery": {
"present": bool(getattr(self.shared_data, "battery_present", False)),
"level_pct": int(getattr(self.shared_data, "battery_percent", 0)),
"charging": bool(getattr(self.shared_data, "battery_is_charging", False)),
"voltage": getattr(self.shared_data, "battery_voltage", None),
"source": getattr(self.shared_data, "battery_source", "unknown"),
"updated_at": float(getattr(self.shared_data, "battery_last_update", 0.0)),
},
}
handler.send_response(200)
handler.send_header("Content-Type", "application/json")
handler.send_header("Cache-Control", "no-cache, no-store, must-revalidate")
handler.send_header("Pragma", "no-cache")
handler.send_header("Expires", "0")
handler.end_headers()
handler.wfile.write(json.dumps(status_data).encode('utf-8'))
except BrokenPipeError:
@@ -342,10 +371,12 @@ class SystemUtils:
handler.send_response(200)
handler.send_header("Content-type", "text/plain")
handler.end_headers()
handler.wfile.write(str(self.shared_data.manual_mode).encode('utf-8'))
handler.wfile.write(str(self.shared_data.operation_mode).encode('utf-8'))
except (ConnectionResetError, ConnectionAbortedError, BrokenPipeError):
# Client closed the socket before response flush: normal with polling/XHR aborts.
return
except Exception as e:
handler.send_response(500)
handler.end_headers()
self.logger.error(f"check_manual_mode failed: {e}")
def check_console_autostart(self, handler):
"""Check console autostart setting."""
@@ -354,6 +385,8 @@ class SystemUtils:
handler.send_header("Content-type", "text/plain")
handler.end_headers()
handler.wfile.write(str(self.shared_data.consoleonwebstart).encode('utf-8'))
except (ConnectionResetError, ConnectionAbortedError, BrokenPipeError):
# Client closed the socket before response flush: normal with polling/XHR aborts.
return
except Exception as e:
handler.send_response(500)
handler.end_headers()
self.logger.error(f"check_console_autostart failed: {e}")

386
web_utils/vuln_utils.py
View File

@@ -431,6 +431,390 @@ class VulnUtils:
logger.exception("serve_cve_bulk failed")
self._send_json(handler, 500, {"status": "error", "message": str(e)})
def serve_cve_bulk_exploits(self, handler, data: Dict[str, Any]) -> None:
"""Bulk exploit search for a list of CVE IDs.
Called by the frontend "Search All Exploits" button via
POST /api/cve/bulk_exploits { "cves": ["CVE-XXXX-YYYY", ...] }
For every CVE the method:
1. Checks the local DB cache first (avoids hammering external APIs on
low-power hardware like the Pi Zero).
2. If the cached exploit list is empty or the record is stale (>48 h),
attempts to fetch exploit hints from:
- GitHub Advisory / search (ghsa-style refs stored in NVD)
- Rapid7 AttackerKB (public, no key required)
3. Persists the updated exploit list back to cve_meta so subsequent
calls are served instantly from cache.
Returns a summary dict so the frontend can update counters.
"""
try:
cves: List[str] = data.get("cves") or []
if not cves:
self._send_json(handler, 200, {"status": "ok", "processed": 0, "with_exploits": 0})
return
# cap per-chunk to avoid timeouts on Pi Zero
cves = [c for c in cves if c and c.upper().startswith("CVE-")][:20]
db = self.shared_data.db
processed = 0
with_exploits = 0
results: Dict[str, Any] = {}
EXPLOIT_STALE_TTL = 48 * 3600 # re-check after 48 h
for cve_id in cves:
try:
# --- 1. DB cache lookup ---
row = None
try:
row = db.get_cve_meta(cve_id)
except Exception:
pass
exploits: List[Dict[str, Any]] = []
cache_fresh = False
if row:
cached_exploits = row.get("exploits_json") or []
if isinstance(cached_exploits, str):
try:
cached_exploits = json.loads(cached_exploits)
except Exception:
cached_exploits = []
age = 0
try:
age = time.time() - int(row.get("updated_at") or 0)
except Exception:
pass
if cached_exploits and age < EXPLOIT_STALE_TTL:
exploits = cached_exploits
cache_fresh = True
# --- 2. External fetch if cache is stale / empty ---
if not cache_fresh:
exploits = self._fetch_exploits_for_cve(cve_id)
# Persist back to DB (merge with any existing meta)
try:
existing = self.cve_enricher.get(cve_id, use_cache_only=True) if self.cve_enricher else {}
patch = {
"cve_id": cve_id,
"description": existing.get("description") or f"{cve_id} vulnerability",
"cvss": existing.get("cvss"),
"references": existing.get("references") or [],
"affected": existing.get("affected") or [],
"exploits": exploits,
"is_kev": existing.get("is_kev", False),
"epss": existing.get("epss"),
"epss_percentile": existing.get("epss_percentile"),
"updated_at": time.time(),
}
db.upsert_cve_meta(patch)
except Exception:
logger.debug("Failed to persist exploits for %s", cve_id, exc_info=True)
processed += 1
if exploits:
with_exploits += 1
results[cve_id] = {
"exploit_count": len(exploits),
"exploits": exploits,
"from_cache": cache_fresh,
}
except Exception:
logger.debug("Exploit search failed for %s", cve_id, exc_info=True)
results[cve_id] = {"exploit_count": 0, "exploits": [], "from_cache": False}
self._send_json(handler, 200, {
"status": "ok",
"processed": processed,
"with_exploits": with_exploits,
"results": results,
})
except Exception as e:
logger.exception("serve_cve_bulk_exploits failed")
self._send_json(handler, 500, {"status": "error", "message": str(e)})
def _fetch_exploits_for_cve(self, cve_id: str) -> List[Dict[str, Any]]:
"""Look up exploit data from the local exploit_feeds table.
No external API calls — populated by serve_feed_sync().
"""
try:
rows = self.shared_data.db.query(
"""
SELECT source, edb_id, title, url, published, platform, type, verified
FROM exploit_feeds
WHERE cve_id = ?
ORDER BY verified DESC, published DESC
LIMIT 10
""",
(cve_id,),
)
return [
{
"source": r.get("source", ""),
"edb_id": r.get("edb_id"),
"description": r.get("title", ""),
"url": r.get("url", ""),
"published": r.get("published", ""),
"platform": r.get("platform", ""),
"type": r.get("type", ""),
"verified": bool(r.get("verified")),
}
for r in (rows or [])
]
except Exception:
logger.debug("Local exploit lookup failed for %s", cve_id, exc_info=True)
return []
# ------------------------------------------------------------------
# Feed sync — called by POST /api/feeds/sync
# ------------------------------------------------------------------
# Schema created lazily on first sync
_FEED_SCHEMA = """
CREATE TABLE IF NOT EXISTS exploit_feeds (
id INTEGER PRIMARY KEY AUTOINCREMENT,
cve_id TEXT NOT NULL,
source TEXT NOT NULL,
edb_id TEXT,
title TEXT,
url TEXT,
published TEXT,
platform TEXT,
type TEXT,
verified INTEGER DEFAULT 0,
UNIQUE(cve_id, source, edb_id)
);
CREATE INDEX IF NOT EXISTS idx_ef_cve ON exploit_feeds(cve_id);
CREATE TABLE IF NOT EXISTS feed_sync_state (
feed TEXT PRIMARY KEY,
last_synced INTEGER DEFAULT 0,
record_count INTEGER DEFAULT 0,
status TEXT DEFAULT 'never'
);
"""
def _ensure_feed_schema(self) -> None:
for stmt in self._FEED_SCHEMA.strip().split(";"):
stmt = stmt.strip()
if stmt:
try:
self.shared_data.db.execute(stmt)
except Exception:
pass
def _set_sync_state(self, feed: str, count: int, status: str) -> None:
try:
self.shared_data.db.execute(
"""
INSERT INTO feed_sync_state (feed, last_synced, record_count, status)
VALUES (?, ?, ?, ?)
ON CONFLICT(feed) DO UPDATE SET
last_synced = excluded.last_synced,
record_count = excluded.record_count,
status = excluded.status
""",
(feed, int(time.time()), count, status),
)
except Exception:
logger.debug("Failed to update feed_sync_state for %s", feed, exc_info=True)
def serve_feed_sync(self, handler) -> None:
"""POST /api/feeds/sync — download CISA KEV + Exploit-DB + EPSS into local DB."""
self._ensure_feed_schema()
results: Dict[str, Any] = {}
# ── 1. CISA KEV ────────────────────────────────────────────────
try:
kev_count = self._sync_cisa_kev()
self._set_sync_state("cisa_kev", kev_count, "ok")
results["cisa_kev"] = {"status": "ok", "count": kev_count}
logger.info("CISA KEV synced — %d records", kev_count)
except Exception as e:
self._set_sync_state("cisa_kev", 0, "error")
results["cisa_kev"] = {"status": "error", "message": str(e)}
logger.exception("CISA KEV sync failed")
# ── 2. Exploit-DB CSV ───────────────────────────────────────────
try:
edb_count = self._sync_exploitdb()
self._set_sync_state("exploitdb", edb_count, "ok")
results["exploitdb"] = {"status": "ok", "count": edb_count}
logger.info("Exploit-DB synced — %d records", edb_count)
except Exception as e:
self._set_sync_state("exploitdb", 0, "error")
results["exploitdb"] = {"status": "error", "message": str(e)}
logger.exception("Exploit-DB sync failed")
# ── 3. EPSS scores ──────────────────────────────────────────────
try:
epss_count = self._sync_epss()
self._set_sync_state("epss", epss_count, "ok")
results["epss"] = {"status": "ok", "count": epss_count}
logger.info("EPSS synced — %d records", epss_count)
except Exception as e:
self._set_sync_state("epss", 0, "error")
results["epss"] = {"status": "error", "message": str(e)}
logger.exception("EPSS sync failed")
any_ok = any(v.get("status") == "ok" for v in results.values())
self._send_json(handler, 200, {
"status": "ok" if any_ok else "error",
"feeds": results,
"synced_at": int(time.time()),
})
def serve_feed_status(self, handler) -> None:
"""GET /api/feeds/status — return last sync timestamps and counts."""
try:
self._ensure_feed_schema()
rows = self.shared_data.db.query(
"SELECT feed, last_synced, record_count, status FROM feed_sync_state"
) or []
state = {r["feed"]: {
"last_synced": r["last_synced"],
"record_count": r["record_count"],
"status": r["status"],
} for r in rows}
# total exploits in local DB
try:
total_row = self.shared_data.db.query_one(
"SELECT COUNT(*) as n FROM exploit_feeds"
)
total = total_row["n"] if total_row else 0
except Exception:
total = 0
self._send_json(handler, 200, {"feeds": state, "total_exploits": total})
except Exception as e:
logger.exception("serve_feed_status failed")
self._send_json(handler, 500, {"status": "error", "message": str(e)})
# ── Feed downloaders ────────────────────────────────────────────────
def _sync_cisa_kev(self) -> int:
import urllib.request, json
url = "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json"
req = urllib.request.Request(url, headers={"User-Agent": "BjornVulnScanner/1.0"})
with urllib.request.urlopen(req, timeout=30) as r:
data = json.loads(r.read().decode("utf-8"))
vulns = data.get("vulnerabilities") or []
count = 0
for v in vulns:
cve_id = (v.get("cveID") or "").strip()
if not cve_id:
continue
try:
self.shared_data.db.execute(
"""
INSERT OR IGNORE INTO exploit_feeds
(cve_id, source, title, url, published, type, verified)
VALUES (?, 'CISA KEV', ?, ?, ?, 'known-exploited', 1)
""",
(
cve_id,
(v.get("vulnerabilityName") or cve_id)[:255],
f"https://www.cisa.gov/known-exploited-vulnerabilities-catalog",
v.get("dateAdded") or "",
),
)
# also flag cve_meta.is_kev
try:
self.shared_data.db.execute(
"UPDATE cve_meta SET is_kev = 1 WHERE cve_id = ?", (cve_id,)
)
except Exception:
pass
count += 1
except Exception:
pass
return count
def _sync_exploitdb(self) -> int:
import urllib.request, csv, io
url = "https://gitlab.com/exploit-database/exploitdb/-/raw/main/files_exploits.csv"
req = urllib.request.Request(url, headers={"User-Agent": "BjornVulnScanner/1.0"})
with urllib.request.urlopen(req, timeout=60) as r:
content = r.read().decode("utf-8", errors="replace")
reader = csv.DictReader(io.StringIO(content))
count = 0
for row in reader:
# exploit-db CSV columns: id, file, description, date_published,
# author, type, platform, port, date_added, verified, codes, tags, aliases, screenshot_url, application_url, source_url
codes = row.get("codes") or ""
# 'codes' field contains semicolon-separated CVE IDs
cve_ids = [c.strip() for c in codes.split(";") if c.strip().upper().startswith("CVE-")]
if not cve_ids:
continue
edb_id = (row.get("id") or "").strip()
title = (row.get("description") or "")[:255]
published = (row.get("date_published") or row.get("date_added") or "").strip()
platform = (row.get("platform") or "").strip()
etype = (row.get("type") or "").strip()
verified = 1 if str(row.get("verified") or "0").strip() == "1" else 0
url_path = (row.get("file") or "").strip()
edb_url = f"https://www.exploit-db.com/exploits/{edb_id}" if edb_id else ""
for cve_id in cve_ids:
try:
self.shared_data.db.execute(
"""
INSERT OR IGNORE INTO exploit_feeds
(cve_id, source, edb_id, title, url, published, platform, type, verified)
VALUES (?, 'Exploit-DB', ?, ?, ?, ?, ?, ?, ?)
""",
(cve_id, edb_id, title, edb_url, published, platform, etype, verified),
)
count += 1
except Exception:
pass
return count
def _sync_epss(self) -> int:
import urllib.request, gzip, csv, io
url = "https://epss.cyentia.com/epss_scores-current.csv.gz"
req = urllib.request.Request(url, headers={"User-Agent": "BjornVulnScanner/1.0"})
count = 0
with urllib.request.urlopen(req, timeout=60) as r:
with gzip.GzipFile(fileobj=r) as gz:
wrapper = io.TextIOWrapper(gz, encoding="utf-8", errors="replace")
# skip leading comment lines (#model_version:...)
reader = csv.DictReader(
(line for line in wrapper if not line.startswith("#"))
)
for row in reader:
cve_id = (row.get("cve") or "").strip()
if not cve_id:
continue
try:
epss = float(row.get("epss") or 0)
pct = float(row.get("percentile") or 0)
self.shared_data.db.execute(
"""
INSERT INTO cve_meta (cve_id, epss, epss_percentile, updated_at)
VALUES (?, ?, ?, ?)
ON CONFLICT(cve_id) DO UPDATE SET
epss = excluded.epss,
epss_percentile = excluded.epss_percentile,
updated_at = excluded.updated_at
""",
(cve_id, epss, pct, int(time.time())),
)
count += 1
except Exception:
pass
return count
def serve_exploitdb_by_cve(self, handler, cve_id: str) -> None:
"""Get Exploit-DB entries for a CVE."""
try:
@@ -580,4 +964,4 @@ class VulnUtils:
except Exception as e:
logger.exception("serve_vulns_stats failed")
self._send_json(handler, 500, {"error": str(e)})
self._send_json(handler, 500, {"error": str(e)})