feat: merge Survey into Scanning and harden live recon
- Delete the Survey stack (sampler, handlers, routes, view, recordings); Scanning gains AP compare (cap 6), whole-page selection filter, channel map (20 MHz lobes, the radio does not report width), and auto-follow that prefers the newest non-empty scan - Richer HTML scan reports: stat cards, band/encryption breakdowns, channel occupancy, color-coded signal cells, GPS line only on a fix - Harden live recon: bounded + retried scan-detail reads (503 on lock), serialize recon starts (409 + scan_remaining while running), GPS and archive discovery move to a 30s poll, GPS_GET skipped when gpsd is down - Read-only history for pineapd-rotated databases (error-*/diagnostic-*): archives list/detail/download endpoints, Previous Scans optgroup, delete disabled, traversal-guarded - Status flags: hopper-radio-offline and history-reset banners; empty scans greyed in the picker - Tests: recon suite grows to 99 cases; all 14 modules green Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
parent
251b1f6261
commit
d8a7074e1f
@@ -45,18 +45,12 @@ _recon_scans_cache = {'db': None, 'updated': 0, 'data': {'scans': []}}
|
||||
_recon_status_cache = {
|
||||
'db': None, 'updated': 0, 'last_scan': None, 'last_activity': None}
|
||||
|
||||
SURVEY_DIR = os.environ.get('PAGER_SURVEY_DIR', '/root/loot/recon-surveys')
|
||||
WIGLE_DIR = os.environ.get('PAGER_WIGLE_DIR', '/root/loot/wigle')
|
||||
GPSD_CONFIG = os.environ.get('PAGER_GPSD_CONFIG', '/etc/config/gpsd')
|
||||
GPSD_INIT = os.environ.get('PAGER_GPSD_INIT', '/etc/init.d/gpsd')
|
||||
SERIAL_DIR = os.environ.get('PAGER_SERIAL_DIR', '/dev/serial/by-path')
|
||||
SURVEY_MAX_SAMPLES = int(os.environ.get('PAGER_SURVEY_MAX_SAMPLES', '1800'))
|
||||
SURVEY_SAMPLE_INTERVAL = float(os.environ.get('PAGER_SURVEY_SAMPLE_INTERVAL', '2.0'))
|
||||
GPS_CACHE_SECONDS = 5.0
|
||||
|
||||
_survey_state = {'active': False, 'id': None, 'name': None, 'path': None,
|
||||
'started': 0, 'samples': 0, 'last_sample': 0}
|
||||
_survey_lock = threading.Lock()
|
||||
_gps_cache = {'updated': 0, 'data': None}
|
||||
_gps_lock = threading.Lock()
|
||||
_payload_runs = {}
|
||||
@@ -1067,8 +1061,10 @@ def band_of(freq):
|
||||
|
||||
|
||||
|
||||
def recon_scans_data(limit=50, _timeout=20):
|
||||
rows = _db_rows(RECON_DB,
|
||||
def recon_scans_data(limit=50, _timeout=20, db=None):
|
||||
if db is None:
|
||||
db = RECON_DB
|
||||
rows = _db_rows(db,
|
||||
'WITH recent AS (SELECT id, time, name FROM scan ORDER BY id DESC LIMIT %d), '
|
||||
'devices AS (SELECT scan, count(*) AS devices FROM wifi_device '
|
||||
'WHERE scan IN (SELECT id FROM recent) GROUP BY scan), '
|
||||
@@ -1090,15 +1086,17 @@ def recon_scans_data(limit=50, _timeout=20):
|
||||
'handshakes': r['handshakes']} for r in rows]}
|
||||
|
||||
|
||||
def recon_scan_data(scan_id, _timeout=20, _limit=None):
|
||||
def recon_scan_data(scan_id, _timeout=20, _limit=None, db=None):
|
||||
"""Scan detail with per-AP enrichment (band/vendor/first_seen/last_seen)
|
||||
plus an unassociated count.
|
||||
|
||||
With `_limit`, client rows are capped and the unassociated rows collapse to
|
||||
a count. The live survey view never renders the full client list, and the
|
||||
a count. The live Scanning view never renders the full client list, and the
|
||||
unassociated/device branches dominate query time on slow recon dbs, so the
|
||||
bounded mode keeps the 2s survey poll cheap.
|
||||
bounded mode keeps the poll and the report downloads cheap.
|
||||
"""
|
||||
if db is None:
|
||||
db = RECON_DB
|
||||
if _limit:
|
||||
sql = ("WITH dev AS (SELECT hash, time, mac, signal, freq, packets "
|
||||
"FROM wifi_device WHERE scan = %d LIMIT %d) "
|
||||
@@ -1116,12 +1114,12 @@ def recon_scan_data(scan_id, _timeout=20, _limit=None):
|
||||
"UNION ALL SELECT 'handshake', hash, time, NULL, NULL, NULL, NULL, NULL, "
|
||||
"NULL, NULL, NULL, NULL, NULL, stahash, aphash "
|
||||
"FROM handshake WHERE scan = %d" % (scan_id, _limit, scan_id, scan_id, scan_id))
|
||||
rows = _db_rows(RECON_DB, sql, timeout=_timeout)
|
||||
cnt = _db_rows(RECON_DB, 'SELECT count(*) AS c FROM ssid WHERE scan = %d AND type = 4'
|
||||
rows = _db_rows(db, sql, timeout=_timeout)
|
||||
cnt = _db_rows(db, 'SELECT count(*) AS c FROM ssid WHERE scan = %d AND type = 4'
|
||||
% scan_id, timeout=_timeout)
|
||||
unassociated = cnt[0]['c'] if cnt else 0
|
||||
else:
|
||||
rows = _db_rows(RECON_DB,
|
||||
rows = _db_rows(db,
|
||||
"SELECT 'scan' AS kind, id AS row_id, time, name, "
|
||||
"NULL AS mac, NULL AS bssid, NULL AS ssid, NULL AS hidden, "
|
||||
"NULL AS channel, NULL AS encryption, NULL AS signal, NULL AS freq, "
|
||||
@@ -1189,6 +1187,16 @@ def recon_scan_data(scan_id, _timeout=20, _limit=None):
|
||||
|
||||
|
||||
def h_recon_start(ctx):
|
||||
# Serialize starts: the firmware has no abort for a timed scan, so a
|
||||
# second /recon/new while one is running just stacks another empty scan
|
||||
# (the 3s-apart rows we saw). Refuse instead of piling on.
|
||||
scanning, remaining = _recon_scan_snapshot()
|
||||
if scanning:
|
||||
return 409, {
|
||||
'error': 'A recon scan is already running and cannot be '
|
||||
'interrupted; it will finish automatically.',
|
||||
'scan_remaining': remaining,
|
||||
}
|
||||
scan_time = (getattr(ctx, 'body', None) or {}).get(
|
||||
'scan_time', DEFAULT_RECON_DURATION)
|
||||
try:
|
||||
@@ -1237,15 +1245,42 @@ def _recon_watchdog_tick():
|
||||
|
||||
The Pager has no recon-stop operation. log/recon/stop controls the storage
|
||||
service and leaves recon.db locked, so timed scans must end natively.
|
||||
Also records a survey sample when a survey is active.
|
||||
"""
|
||||
st = _recon_scan_state
|
||||
if st['active'] and st['duration'] > 0 and time.time() - st['started'] >= st['duration']:
|
||||
st['active'] = False
|
||||
|
||||
|
||||
_hopper_cache = {'updated': 0, 'online': None}
|
||||
HOPPER_CACHE_SECONDS = 10.0
|
||||
HOPPER_IFACE = 'wlan2mon'
|
||||
|
||||
|
||||
def _hopper_online():
|
||||
"""Whether the fast-hopping radio (wlan2mon per pineapd config) exists.
|
||||
|
||||
iwinfo is a subprocess, so the answer is cached for a few seconds; the
|
||||
UI polls status every 5s.
|
||||
"""
|
||||
now = time.time()
|
||||
if now - _hopper_cache['updated'] < HOPPER_CACHE_SECONDS:
|
||||
return _hopper_cache['online']
|
||||
try:
|
||||
_survey_sample()
|
||||
online = HOPPER_IFACE in wifi_ifaces()
|
||||
except Exception:
|
||||
pass
|
||||
online = None
|
||||
_hopper_cache.update({'updated': now, 'online': online})
|
||||
return online
|
||||
|
||||
|
||||
def _recon_history_reset():
|
||||
"""True when pineapd rotated the live db (error-*-recon.db exists)."""
|
||||
directory = os.path.dirname(RECON_DB)
|
||||
try:
|
||||
names = os.listdir(directory)
|
||||
except OSError:
|
||||
return False
|
||||
return any(n.startswith('error-') and n.endswith('recon.db') for n in names)
|
||||
|
||||
|
||||
def h_recon_status(ctx):
|
||||
@@ -1271,7 +1306,9 @@ def h_recon_status(ctx):
|
||||
last_activity = last
|
||||
return 200, {'last_scan': last, 'last_activity': last_activity,
|
||||
'active': last_activity is not None and int(time.time()) - last_activity < 300,
|
||||
'scanning': scanning, 'scan_remaining': remaining, 'stale': stale}
|
||||
'scanning': scanning, 'scan_remaining': remaining, 'stale': stale,
|
||||
'hopper_online': _hopper_online(),
|
||||
'history_reset': _recon_history_reset()}
|
||||
|
||||
|
||||
def _db_write(db, sql):
|
||||
@@ -1369,13 +1406,20 @@ def h_recon_scans(ctx):
|
||||
|
||||
def h_recon_scan_detail(ctx):
|
||||
scan_id = int(ctx.args[0])
|
||||
data = recon_scan_data(scan_id)
|
||||
# Bounded + retried: the live view polls this every few seconds, and an
|
||||
# unbounded read across ssid/wifi_device can hold the DB long enough to
|
||||
# make pineapd rotate it (SQLITE_BUSY -> error-*-recon.db).
|
||||
try:
|
||||
data = _recon_read_retry(lambda: recon_scan_data(
|
||||
scan_id, _timeout=15, _limit=300))
|
||||
except RuntimeError:
|
||||
return 503, {'error': 'recon database is temporarily unavailable'}
|
||||
if data is None:
|
||||
return 404, {'error': 'scan not found'}
|
||||
return 200, data
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Recon report helpers (CSV / HTML) for scan and survey downloads.
|
||||
# Recon report helpers (CSV / HTML) for scan downloads.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _fmt_ts(ts):
|
||||
@@ -1406,16 +1450,6 @@ def _aps_csv(data):
|
||||
return '\r\n'.join(out) + '\r\n'
|
||||
|
||||
|
||||
def _survey_aps_csv(detail):
|
||||
out = ['bssid,ssid,band,channel,min_dbm,avg_dbm,max_dbm,samples,first_seen,last_seen']
|
||||
for a in (detail or {}).get('aps') or []:
|
||||
out.append(','.join(_csv_escape(x) for x in [
|
||||
a.get('bssid'), a.get('ssid'), a.get('band'), a.get('channel'),
|
||||
a.get('min'), a.get('avg'), a.get('max'), a.get('samples'),
|
||||
_fmt_ts(a.get('first_seen')), _fmt_ts(a.get('last_seen'))]))
|
||||
return '\r\n'.join(out) + '\r\n'
|
||||
|
||||
|
||||
def _esc_html(v):
|
||||
if v is None:
|
||||
return ''
|
||||
@@ -1425,42 +1459,97 @@ def _esc_html(v):
|
||||
|
||||
REPORT_CSS = """
|
||||
body { font-family: -apple-system, 'Segoe UI', Roboto, sans-serif; margin: 24px; color: #222; background: #fff; }
|
||||
h1 { font-size: 20px; margin: 0 0 4px; }
|
||||
.sub { color: #666; margin-bottom: 16px; }
|
||||
h1 { font-size: 22px; margin: 0 0 2px; }
|
||||
.sub { color: #666; margin-bottom: 2px; }
|
||||
.meta { color: #888; font-size: 13px; margin: 4px 0 12px; }
|
||||
.stat-cards { display: flex; flex-wrap: wrap; gap: 10px; margin: 14px 0 4px; }
|
||||
.stat-card { border: 1px solid #e0e0e0; border-radius: 6px; padding: 10px 16px; min-width: 108px; background: #fafafa; }
|
||||
.stat-value { font-size: 26px; font-weight: 700; line-height: 1.1; font-variant-numeric: tabular-nums; }
|
||||
.stat-label { font-size: 11px; text-transform: uppercase; letter-spacing: .05em; color: #888; }
|
||||
h2 { font-size: 15px; margin: 20px 0 8px; border-bottom: 2px solid #1a237e; padding-bottom: 4px; color: #1a237e; }
|
||||
table { border-collapse: collapse; width: 100%; font-size: 13px; }
|
||||
th, td { border: 1px solid #ddd; padding: 6px 10px; text-align: left; }
|
||||
th { background: #f4f4f4; }
|
||||
tr:nth-child(even) td { background: #fafafa; }
|
||||
.stats { margin: 12px 0; font-size: 13px; color: #333; }
|
||||
.sig-strong { color: #2e7d32; font-weight: 600; white-space: nowrap; }
|
||||
.sig-good { color: #f9a825; font-weight: 600; white-space: nowrap; }
|
||||
.sig-weak { color: #ef6c00; font-weight: 600; white-space: nowrap; }
|
||||
.sig-dead { color: #c62828; font-weight: 600; white-space: nowrap; }
|
||||
.empty { color: #888; font-style: italic; }
|
||||
@media print { body { margin: 12px; } .stat-card { border-color: #ccc; } }
|
||||
"""
|
||||
|
||||
|
||||
def _html_table(headers, rows):
|
||||
def _html_table(headers, rows, raw_columns=None):
|
||||
"""Render an HTML table. Cells are HTML-escaped unless their column index
|
||||
is listed in raw_columns (caller-provided safe markup, e.g. colored
|
||||
signal spans)."""
|
||||
raw = set(raw_columns or ())
|
||||
out = ['<table><thead><tr>']
|
||||
for header in headers:
|
||||
out.append('<th>%s</th>' % _esc_html(header))
|
||||
out.append('</tr></thead><tbody>')
|
||||
for row in rows:
|
||||
out.append('<tr>')
|
||||
for cell in row:
|
||||
out.append('<td>%s</td>' % _esc_html(cell))
|
||||
for i, cell in enumerate(row):
|
||||
out.append('<td>%s</td>' % (cell if i in raw else _esc_html(cell)))
|
||||
out.append('</tr>')
|
||||
out.append('</tbody></table>')
|
||||
return ''.join(out)
|
||||
|
||||
|
||||
def _html_doc(title, subtitle, body_html, stats=None):
|
||||
def _html_doc(title, subtitle, body_html, stats=None, meta=None):
|
||||
parts = ['<!DOCTYPE html><html><head><meta charset="utf-8"><title>%s</title>'
|
||||
'<style>%s</style></head><body>' % (_esc_html(title), REPORT_CSS)]
|
||||
parts.append('<h1>%s</h1>' % _esc_html(title))
|
||||
parts.append('<div class="sub">%s</div>' % _esc_html(subtitle))
|
||||
for label, value in (stats or []):
|
||||
parts.append('<div class="stats"><b>%s:</b> %s</div>' % (_esc_html(label), value))
|
||||
for line in (meta or []):
|
||||
parts.append('<div class="meta">%s</div>' % _esc_html(line))
|
||||
if stats:
|
||||
parts.append('<div class="stat-cards">')
|
||||
for label, value in stats:
|
||||
parts.append('<div class="stat-card"><div class="stat-value">%s</div>'
|
||||
'<div class="stat-label">%s</div></div>'
|
||||
% (_esc_html(value), _esc_html(label)))
|
||||
parts.append('</div>')
|
||||
parts.append(body_html)
|
||||
parts.append('</body></html>')
|
||||
return ''.join(parts)
|
||||
|
||||
|
||||
def _signal_html(dbm):
|
||||
"""Color-coded dBm cell matching the UI thresholds."""
|
||||
if dbm is None:
|
||||
return '<span class="sig-dead">--</span>'
|
||||
if dbm >= -50:
|
||||
cls = 'sig-strong'
|
||||
elif dbm >= -67:
|
||||
cls = 'sig-good'
|
||||
elif dbm >= -80:
|
||||
cls = 'sig-weak'
|
||||
else:
|
||||
cls = 'sig-dead'
|
||||
return '<span class="%s">%d dBm</span>' % (cls, dbm)
|
||||
|
||||
|
||||
def _enc_bucket(enc):
|
||||
"""Encryption display string -> coarse bucket (mirrors the UI chips)."""
|
||||
s = (enc or '').strip()
|
||||
if not s or s == 'Open':
|
||||
return 'Open'
|
||||
if 'Enterprise' in s:
|
||||
return 'Enterprise'
|
||||
if 'WEP' in s:
|
||||
return 'WEP'
|
||||
if 'WPA2' in s:
|
||||
return 'WPA2'
|
||||
if 'WPA3' in s:
|
||||
return 'WPA3'
|
||||
if 'WPA' in s:
|
||||
return 'WPA'
|
||||
return 'Unknown'
|
||||
|
||||
|
||||
def _recon_read_retry(fn, attempts=3, pause=1.0):
|
||||
"""Retry a recon.db read briefly; pineapd's write bursts hold the DB
|
||||
exclusive lock and reads can time out mid-burst."""
|
||||
@@ -1474,8 +1563,10 @@ def _recon_read_retry(fn, attempts=3, pause=1.0):
|
||||
raise last
|
||||
|
||||
|
||||
def _scan_client_count(scan_id, _timeout=12):
|
||||
rows = _db_rows(RECON_DB,
|
||||
def _scan_client_count(scan_id, _timeout=12, db=None):
|
||||
if db is None:
|
||||
db = RECON_DB
|
||||
rows = _db_rows(db,
|
||||
"SELECT count(*) AS c FROM wifi_device w WHERE w.scan = %d "
|
||||
"AND w.mac NOT IN (SELECT DISTINCT bssid FROM ssid "
|
||||
"WHERE scan = %d AND type = 8 AND bssid IS NOT NULL)"
|
||||
@@ -1495,6 +1586,79 @@ def h_recon_scan_download_csv(ctx):
|
||||
'scan-%d.csv' % scan_id)
|
||||
|
||||
|
||||
def _recon_html_download(scan_id, data, client_count, archive=None):
|
||||
"""Shared HTML report body for live and archived scans.
|
||||
|
||||
GPS metadata is only attached for live scans (an archive's data is
|
||||
historical, so a current fix would be misleading).
|
||||
"""
|
||||
aps = data.get('aps') or []
|
||||
scan = data.get('scan') or {}
|
||||
meta = []
|
||||
if archive is None:
|
||||
try:
|
||||
gps = _gps_status_data()
|
||||
if gps.get('lock'):
|
||||
meta.append('GPS: %.5f, %.5f%s' % (
|
||||
gps.get('lat') or 0, gps.get('lon') or 0,
|
||||
(' \u00b7 %d sats' % gps.get('satellites'))
|
||||
if gps.get('satellites') else ''))
|
||||
except Exception:
|
||||
pass
|
||||
stats = [('Access Points', len(aps)),
|
||||
('Clients', client_count),
|
||||
('Handshakes', len(data.get('handshakes') or [])),
|
||||
('Unassociated', data.get('unassociated') or 0)]
|
||||
body_parts = []
|
||||
# Band and encryption breakdowns.
|
||||
band_counts = {}
|
||||
enc_counts = {}
|
||||
for a in aps:
|
||||
band_counts[a.get('band') or '--'] = band_counts.get(a.get('band') or '--', 0) + 1
|
||||
b = _enc_bucket(a.get('encryption'))
|
||||
enc_counts[b] = enc_counts.get(b, 0) + 1
|
||||
body_parts.append('<h2>Band Breakdown</h2>')
|
||||
body_parts.append(_html_table(['Band', 'Access Points'], sorted(
|
||||
band_counts.items(), key=lambda kv: kv[1], reverse=True)))
|
||||
body_parts.append('<h2>Encryption Breakdown</h2>')
|
||||
body_parts.append(_html_table(['Encryption', 'Access Points'], sorted(
|
||||
enc_counts.items(), key=lambda kv: kv[1], reverse=True)))
|
||||
# Channel occupancy (count per channel, split by band).
|
||||
chan_counts = {}
|
||||
for a in aps:
|
||||
if a.get('channel') is None:
|
||||
continue
|
||||
key = (a.get('band') or '--', a.get('channel'))
|
||||
chan_counts[key] = chan_counts.get(key, 0) + 1
|
||||
body_parts.append('<h2>Channel Occupancy</h2>')
|
||||
if chan_counts:
|
||||
chan_rows = [[band + ' GHz', ch, n] for (band, ch), n in sorted(
|
||||
chan_counts.items(), key=lambda kv: (kv[0][0], kv[0][1]))]
|
||||
body_parts.append(_html_table(['Band', 'Channel', 'Access Points'], chan_rows))
|
||||
else:
|
||||
body_parts.append('<p class="empty">No access points with a known channel.</p>')
|
||||
# AP table with color-coded signal.
|
||||
ap_rows = []
|
||||
for a in aps:
|
||||
ap_rows.append([a.get('ssid') or '(hidden)', a.get('bssid'),
|
||||
a.get('band') or '--',
|
||||
a.get('channel') if a.get('channel') is not None else '--',
|
||||
_signal_html(a.get('signal')),
|
||||
a.get('encryption') or '--',
|
||||
a.get('vendor') or '--',
|
||||
_fmt_ts(a.get('first_seen')), _fmt_ts(a.get('last_seen'))])
|
||||
body_parts.append('<h2>Access Points</h2>')
|
||||
body_parts.append(_html_table(['SSID', 'BSSID', 'Band', 'Ch', 'Signal',
|
||||
'Encryption', 'Vendor', 'First seen', 'Last seen'],
|
||||
ap_rows, raw_columns=(4,)))
|
||||
subtitle = ('Pager recon capture report (archived history)'
|
||||
if archive else 'Pager recon capture report')
|
||||
return Download(_html_doc('Scan #%d' % scan.get('id'),
|
||||
subtitle, '\n'.join(body_parts),
|
||||
stats=stats, meta=meta).encode('utf-8'),
|
||||
'text/html', 'scan-%d.html' % scan_id)
|
||||
|
||||
|
||||
def h_recon_scan_download_html(ctx):
|
||||
scan_id = int(ctx.args[0])
|
||||
try:
|
||||
@@ -1504,24 +1668,153 @@ def h_recon_scan_download_html(ctx):
|
||||
return 503, {'error': 'recon database is temporarily unavailable'}
|
||||
if data is None:
|
||||
return 404, {'error': 'scan not found'}
|
||||
rows = []
|
||||
for a in data.get('aps') or []:
|
||||
rows.append([a.get('ssid') or '(hidden)', a.get('bssid'), a.get('band'),
|
||||
a.get('channel'), a.get('signal'), a.get('encryption'),
|
||||
a.get('vendor'), _fmt_ts(a.get('first_seen')),
|
||||
_fmt_ts(a.get('last_seen'))])
|
||||
scan = data.get('scan') or {}
|
||||
stats = [('Started', _fmt_ts(scan.get('time'))),
|
||||
('Access points', len(data.get('aps') or [])),
|
||||
('Clients', client_count),
|
||||
('Handshakes', len(data.get('handshakes') or [])),
|
||||
('Unassociated', data.get('unassociated') or 0)]
|
||||
body = _html_table(['SSID', 'BSSID', 'Band', 'Ch', 'Signal', 'Encryption',
|
||||
'Vendor', 'First seen', 'Last seen'], rows)
|
||||
return 200, Download(_html_doc('Scan #%d' % scan.get('id'),
|
||||
'Pager recon capture report', body,
|
||||
stats=stats).encode('utf-8'),
|
||||
'text/html', 'scan-%d.html' % scan_id)
|
||||
return 200, _recon_html_download(scan_id, data, client_count)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Recon archives: read-only history from pineapd-rotated databases.
|
||||
# pineapd renames recon.db to error-<ts>-recon.db when an INSERT hits
|
||||
# SQLITE_BUSY, then starts a fresh database at scan 1. These endpoints expose
|
||||
# the rotated files as read-only history. They never write and never change
|
||||
# the live RECON_DB.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _recon_archive_path(archive_id):
|
||||
"""Resolve a URL-encoded archive id (a rotated db filename) to an absolute
|
||||
path, or None if it is not a real archive in the recon directory."""
|
||||
if not archive_id:
|
||||
return None
|
||||
base = os.path.basename(archive_id)
|
||||
if base != archive_id or '/' in archive_id or '\\' in archive_id:
|
||||
return None
|
||||
if not (base.startswith('error-') or base.startswith('diagnostic-')):
|
||||
return None
|
||||
if not base.endswith('recon.db'):
|
||||
return None
|
||||
path = os.path.join(os.path.dirname(RECON_DB), base)
|
||||
return path if os.path.isfile(path) else None
|
||||
|
||||
|
||||
_recon_archives_cache = {'dir': None, 'updated': 0, 'data': None}
|
||||
RECON_ARCHIVES_CACHE_SECONDS = 15.0
|
||||
|
||||
|
||||
def recon_archives_data():
|
||||
now = time.time()
|
||||
directory = os.path.dirname(RECON_DB)
|
||||
cache = _recon_archives_cache
|
||||
if (cache['dir'] == directory and cache['data'] is not None
|
||||
and now - cache['updated'] < RECON_ARCHIVES_CACHE_SECONDS):
|
||||
return cache['data']
|
||||
archives = []
|
||||
try:
|
||||
names = sorted(os.listdir(directory))
|
||||
except OSError:
|
||||
return {'archives': []}
|
||||
for name in names:
|
||||
if not (name.startswith('error-') or name.startswith('diagnostic-')):
|
||||
continue
|
||||
if not name.endswith('recon.db'):
|
||||
continue
|
||||
path = os.path.join(directory, name)
|
||||
try:
|
||||
mtime = int(os.path.getmtime(path))
|
||||
except OSError:
|
||||
mtime = None
|
||||
scans = []
|
||||
try:
|
||||
scans = recon_scans_data(limit=500, _timeout=15, db=path).get('scans') or []
|
||||
except RuntimeError:
|
||||
pass
|
||||
lo = hi = n = None
|
||||
if scans:
|
||||
lo = min(s['id'] for s in scans)
|
||||
hi = max(s['id'] for s in scans)
|
||||
n = len(scans)
|
||||
archives.append({'id': name, 'label': name, 'mtime': mtime,
|
||||
'min_id': lo, 'max_id': hi, 'scans_count': n,
|
||||
'scans': scans})
|
||||
data = {'archives': archives}
|
||||
cache.update({'dir': directory, 'updated': time.time(), 'data': data})
|
||||
return data
|
||||
|
||||
|
||||
def h_recon_archives(ctx):
|
||||
return 200, recon_archives_data()
|
||||
|
||||
|
||||
def h_recon_archive_scans(ctx):
|
||||
path = _recon_archive_path(_unquote_plus(ctx.args[0]))
|
||||
if path is None:
|
||||
return 404, {'error': 'archive not found'}
|
||||
try:
|
||||
data = recon_scans_data(limit=500, db=path)
|
||||
except RuntimeError:
|
||||
return 503, {'error': 'archive database is temporarily unavailable'}
|
||||
return 200, dict(data, archive=ctx.args[0])
|
||||
|
||||
|
||||
def h_recon_archive_scan_detail(ctx):
|
||||
path = _recon_archive_path(_unquote_plus(ctx.args[0]))
|
||||
if path is None:
|
||||
return 404, {'error': 'archive not found'}
|
||||
scan_id = int(ctx.args[1])
|
||||
try:
|
||||
data = _recon_read_retry(lambda: recon_scan_data(
|
||||
scan_id, _timeout=15, _limit=300, db=path))
|
||||
except RuntimeError:
|
||||
return 503, {'error': 'archive database is temporarily unavailable'}
|
||||
if data is None:
|
||||
return 404, {'error': 'scan not found in archive'}
|
||||
return 200, data
|
||||
|
||||
|
||||
def h_recon_archive_scan_download(ctx):
|
||||
path = _recon_archive_path(_unquote_plus(ctx.args[0]))
|
||||
if path is None:
|
||||
return 404, {'error': 'archive not found'}
|
||||
scan_id = int(ctx.args[1])
|
||||
try:
|
||||
data = _recon_read_retry(lambda: recon_scan_data(
|
||||
scan_id, _timeout=15, _limit=300, db=path))
|
||||
except RuntimeError:
|
||||
return 503, {'error': 'archive database is temporarily unavailable'}
|
||||
if data is None:
|
||||
return 404, {'error': 'scan not found in archive'}
|
||||
return 200, Download(json.dumps(data, indent=2).encode('utf-8'),
|
||||
'application/json', 'scan-%d.json' % scan_id)
|
||||
|
||||
|
||||
def h_recon_archive_scan_download_csv(ctx):
|
||||
path = _recon_archive_path(_unquote_plus(ctx.args[0]))
|
||||
if path is None:
|
||||
return 404, {'error': 'archive not found'}
|
||||
scan_id = int(ctx.args[1])
|
||||
try:
|
||||
data = _recon_read_retry(lambda: recon_scan_data(
|
||||
scan_id, _timeout=15, _limit=300, db=path))
|
||||
except RuntimeError:
|
||||
return 503, {'error': 'archive database is temporarily unavailable'}
|
||||
if data is None:
|
||||
return 404, {'error': 'scan not found in archive'}
|
||||
return 200, Download(_aps_csv(data).encode('utf-8'), 'text/csv',
|
||||
'scan-%d.csv' % scan_id)
|
||||
|
||||
|
||||
def h_recon_archive_scan_download_html(ctx):
|
||||
path = _recon_archive_path(_unquote_plus(ctx.args[0]))
|
||||
if path is None:
|
||||
return 404, {'error': 'archive not found'}
|
||||
scan_id = int(ctx.args[1])
|
||||
try:
|
||||
data = _recon_read_retry(lambda: recon_scan_data(
|
||||
scan_id, _timeout=15, _limit=300, db=path))
|
||||
client_count = _scan_client_count(scan_id, _timeout=12, db=path)
|
||||
except RuntimeError:
|
||||
return 503, {'error': 'archive database is temporarily unavailable'}
|
||||
if data is None:
|
||||
return 404, {'error': 'scan not found in archive'}
|
||||
return 200, _recon_html_download(scan_id, data, client_count, archive=True)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -1640,7 +1933,9 @@ def _gps_status_data_nocache():
|
||||
'satellites': None, 'gpsd_running': running,
|
||||
'updated': None, 'wigle': _wigle_config().get('logwigle', False)}
|
||||
fix = _gps_from_gpspipe() if running else None
|
||||
if fix is None:
|
||||
# GPS_GET through the daemon can block for seconds when gpsd is down;
|
||||
# only fall back to it while gpsd is actually running.
|
||||
if fix is None and running:
|
||||
fix = _gps_from_hak5cmd()
|
||||
if fix:
|
||||
data.update(fix)
|
||||
@@ -1765,290 +2060,6 @@ def h_recon_wigle(ctx):
|
||||
return 200, resp
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Surveys: JSONL overlay on the device (meta line + one 'sample' line per
|
||||
# tick of the always-on recon). The recon.db itself is never modified.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _survey_path(sid):
|
||||
return _safe_join(SURVEY_DIR, '%s.jsonl' % sid)
|
||||
|
||||
|
||||
def _survey_slug(name):
|
||||
return re.sub(r'[^A-Za-z0-9]+', '-', (name or '').strip()).strip('-')
|
||||
|
||||
|
||||
def _survey_recording_state():
|
||||
st = _survey_state
|
||||
if not st['active']:
|
||||
return None
|
||||
return {'active': True, 'id': st['id'], 'name': st['name'],
|
||||
'started': st['started'], 'samples': st['samples']}
|
||||
|
||||
|
||||
def _survey_sample():
|
||||
with _survey_lock:
|
||||
st = _survey_state
|
||||
if not st['active'] or not st['path']:
|
||||
return
|
||||
now = time.time()
|
||||
if now - st['last_sample'] < SURVEY_SAMPLE_INTERVAL:
|
||||
return
|
||||
if st['samples'] >= SURVEY_MAX_SAMPLES:
|
||||
st['active'] = False
|
||||
return
|
||||
try:
|
||||
scans = recon_scans_data(1, _timeout=6).get('scans') or []
|
||||
detail = recon_scan_data(scans[0]['id'], _timeout=12, _limit=300) if scans else None
|
||||
gps = _gps_status_data()
|
||||
sample = {'t': int(now),
|
||||
'scan': (detail or {}).get('scan'),
|
||||
'aps': (detail or {}).get('aps') or [],
|
||||
'clients': (detail or {}).get('clients') or [],
|
||||
'handshakes': (detail or {}).get('handshakes') or [],
|
||||
'unassociated': (detail or {}).get('unassociated') or 0,
|
||||
'gps': {'fix': gps.get('fix'), 'lat': gps.get('lat'),
|
||||
'lon': gps.get('lon'),
|
||||
'satellites': gps.get('satellites')}}
|
||||
with open(st['path'], 'a') as fh:
|
||||
fh.write(json.dumps({'sample': sample}) + '\n')
|
||||
st['samples'] += 1
|
||||
st['last_sample'] = now
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def h_recon_survey_start(ctx):
|
||||
with _survey_lock:
|
||||
if _survey_state['active']:
|
||||
return 409, {'error': 'a survey is already recording'}
|
||||
name = ((getattr(ctx, 'body', None) or {}).get('name') or '').strip()
|
||||
sid = time.strftime('%Y%m%d-%H%M%S')
|
||||
slug = _survey_slug(name)
|
||||
if slug:
|
||||
sid += '-' + slug
|
||||
path = _survey_path(sid)
|
||||
if path is None:
|
||||
return 500, {'error': 'invalid survey id'}
|
||||
try:
|
||||
os.makedirs(SURVEY_DIR, exist_ok=True)
|
||||
except OSError:
|
||||
return 500, {'error': 'cannot create survey directory'}
|
||||
meta = {'id': sid, 'name': name, 'started': int(time.time()),
|
||||
'interval': SURVEY_SAMPLE_INTERVAL, 'max_samples': SURVEY_MAX_SAMPLES}
|
||||
try:
|
||||
with open(path, 'w') as fh:
|
||||
fh.write(json.dumps({'meta': meta}) + '\n')
|
||||
except OSError:
|
||||
return 500, {'error': 'cannot write survey file'}
|
||||
_survey_state.update({'active': True, 'id': sid, 'name': name, 'path': path,
|
||||
'started': meta['started'], 'samples': 0,
|
||||
'last_sample': 0})
|
||||
return 200, {'ok': True, 'id': sid, 'started': meta['started']}
|
||||
|
||||
|
||||
def h_recon_survey_stop(ctx):
|
||||
with _survey_lock:
|
||||
st = dict(_survey_state)
|
||||
if st['active']:
|
||||
_survey_state['active'] = False
|
||||
return 200, {'ok': True, 'samples': st['samples'], 'id': st['id']}
|
||||
|
||||
|
||||
def h_recon_survey_live(ctx):
|
||||
detail = None
|
||||
try:
|
||||
scans = recon_scans_data(1, _timeout=6).get('scans') or []
|
||||
if scans:
|
||||
detail = recon_scan_data(scans[0]['id'], _timeout=12, _limit=300)
|
||||
except RuntimeError:
|
||||
pass
|
||||
try:
|
||||
gps = _gps_status_data()
|
||||
except Exception:
|
||||
gps = {}
|
||||
return 200, {'scan': (detail or {}).get('scan'),
|
||||
'aps': (detail or {}).get('aps') or [],
|
||||
'clients': (detail or {}).get('clients') or [],
|
||||
'handshakes': (detail or {}).get('handshakes') or [],
|
||||
'unassociated': (detail or {}).get('unassociated') or 0,
|
||||
'gps': gps,
|
||||
'recording': _survey_recording_state()}
|
||||
|
||||
|
||||
def recon_surveys_data():
|
||||
surveys = []
|
||||
if os.path.isdir(SURVEY_DIR):
|
||||
try:
|
||||
names = sorted(os.listdir(SURVEY_DIR), reverse=True)
|
||||
except OSError:
|
||||
names = []
|
||||
for name in names:
|
||||
if not name.endswith('.jsonl'):
|
||||
continue
|
||||
path = os.path.join(SURVEY_DIR, name)
|
||||
sid = name[:-6]
|
||||
meta = None
|
||||
samples = 0
|
||||
try:
|
||||
size = os.path.getsize(path)
|
||||
with open(path, 'r', errors='replace') as fh:
|
||||
for line in fh:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
obj = json.loads(line)
|
||||
except ValueError:
|
||||
continue
|
||||
if 'meta' in obj:
|
||||
meta = obj['meta']
|
||||
elif 'sample' in obj:
|
||||
samples += 1
|
||||
except OSError:
|
||||
continue
|
||||
if meta is None:
|
||||
continue
|
||||
surveys.append({'id': sid, 'name': meta.get('name') or sid,
|
||||
'started': meta.get('started'),
|
||||
'samples': samples, 'size': size})
|
||||
return {'surveys': surveys}
|
||||
|
||||
|
||||
def h_recon_surveys(ctx):
|
||||
try:
|
||||
return 200, recon_surveys_data()
|
||||
except Exception:
|
||||
return 200, {'surveys': []}
|
||||
|
||||
|
||||
def recon_survey_data(sid):
|
||||
path = _survey_path(sid)
|
||||
if path is None or not os.path.isfile(path):
|
||||
return None
|
||||
agg = {}
|
||||
order = []
|
||||
gps_fixes = []
|
||||
try:
|
||||
with open(path, 'r', errors='replace') as fh:
|
||||
for line in fh:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
obj = json.loads(line)
|
||||
except ValueError:
|
||||
continue
|
||||
s = obj.get('sample')
|
||||
if not isinstance(s, dict):
|
||||
continue
|
||||
g = s.get('gps') or {}
|
||||
if g.get('lat') is not None and g.get('lon') is not None:
|
||||
gps_fixes.append({'t': s.get('t'), 'lat': g['lat'],
|
||||
'lon': g['lon']})
|
||||
for a in s.get('aps') or []:
|
||||
bssid = a.get('bssid')
|
||||
if not bssid:
|
||||
continue
|
||||
entry = agg.get(bssid)
|
||||
if entry is None:
|
||||
entry = {'ssid': a.get('ssid'), 'bssid': bssid,
|
||||
'band': a.get('band'), 'channel': a.get('channel'),
|
||||
'min': None, 'max': None, 'sum': 0, 'count': 0,
|
||||
'first_seen': None, 'last_seen': None}
|
||||
agg[bssid] = entry
|
||||
order.append(bssid)
|
||||
sig = a.get('signal')
|
||||
if sig is not None:
|
||||
entry['min'] = sig if entry['min'] is None else min(entry['min'], sig)
|
||||
entry['max'] = sig if entry['max'] is None else max(entry['max'], sig)
|
||||
entry['sum'] += sig
|
||||
entry['count'] += 1
|
||||
t = s.get('t')
|
||||
if t:
|
||||
entry['first_seen'] = (t if entry['first_seen'] is None
|
||||
else min(entry['first_seen'], t))
|
||||
entry['last_seen'] = (t if entry['last_seen'] is None
|
||||
else max(entry['last_seen'], t))
|
||||
except OSError:
|
||||
return None
|
||||
aps = []
|
||||
for bssid in order:
|
||||
e = agg[bssid]
|
||||
aps.append({'ssid': e['ssid'], 'bssid': e['bssid'], 'band': e['band'],
|
||||
'channel': e['channel'], 'min': e['min'],
|
||||
'avg': round(e['sum'] / e['count']) if e['count'] else None,
|
||||
'max': e['max'], 'samples': e['count'],
|
||||
'first_seen': e['first_seen'], 'last_seen': e['last_seen']})
|
||||
aps.sort(key=lambda a: a['avg'] if a['avg'] is not None else 0)
|
||||
return {'id': sid, 'aps': aps, 'gps_fixes': len(gps_fixes),
|
||||
'first_gps': gps_fixes[0] if gps_fixes else None,
|
||||
'last_gps': gps_fixes[-1] if gps_fixes else None}
|
||||
|
||||
|
||||
def h_recon_survey_detail(ctx):
|
||||
sid = ctx.args[0]
|
||||
data = recon_survey_data(sid)
|
||||
if data is None:
|
||||
return 404, {'error': 'survey not found'}
|
||||
return 200, data
|
||||
|
||||
|
||||
def h_recon_survey_download(ctx):
|
||||
sid = ctx.args[0]
|
||||
fmt = ctx.args[1]
|
||||
if fmt == 'json':
|
||||
data = recon_survey_data(sid)
|
||||
if data is None:
|
||||
return 404, {'error': 'survey not found'}
|
||||
return 200, Download(json.dumps(data, indent=2).encode('utf-8'),
|
||||
'application/json', 'survey-%s.json' % sid)
|
||||
data = recon_survey_data(sid)
|
||||
if data is None:
|
||||
return 404, {'error': 'survey not found'}
|
||||
if fmt == 'csv':
|
||||
return 200, Download(_survey_aps_csv(data).encode('utf-8'), 'text/csv',
|
||||
'survey-%s.csv' % sid)
|
||||
if fmt == 'html':
|
||||
rows = []
|
||||
for a in data.get('aps') or []:
|
||||
rows.append([a.get('ssid') or '(hidden)', a.get('bssid'),
|
||||
a.get('band'), a.get('channel'), a.get('min'),
|
||||
a.get('avg'), a.get('max'), a.get('samples'),
|
||||
_fmt_ts(a.get('first_seen')), _fmt_ts(a.get('last_seen'))])
|
||||
fixes = data.get('gps_fixes') or 0
|
||||
fg = data.get('first_gps') or {}
|
||||
lg = data.get('last_gps') or {}
|
||||
if fixes:
|
||||
gps = ('%d fixes · first %.5f, %.5f · last %.5f, %.5f'
|
||||
% (fixes, fg.get('lat') or 0, fg.get('lon') or 0,
|
||||
lg.get('lat') or 0, lg.get('lon') or 0))
|
||||
else:
|
||||
gps = 'No GPS fixes during this survey'
|
||||
body = _html_table(['SSID', 'BSSID', 'Band', 'Ch', 'Min', 'Avg', 'Max',
|
||||
'Samples', 'First', 'Last'], rows)
|
||||
return 200, Download(_html_doc('Survey %s' % sid,
|
||||
'AP signal aggregates', body,
|
||||
stats=[('GPS', gps)]).encode('utf-8'),
|
||||
'text/html', 'survey-%s.html' % sid)
|
||||
return 404, {'error': 'unknown format'}
|
||||
|
||||
|
||||
def h_recon_survey_delete(ctx):
|
||||
sid = ctx.args[0]
|
||||
path = _survey_path(sid)
|
||||
if path is None or not os.path.isfile(path):
|
||||
return 404, {'error': 'survey not found'}
|
||||
with _survey_lock:
|
||||
if _survey_state.get('id') == sid and _survey_state['active']:
|
||||
return 409, {'error': 'cannot delete the survey that is recording'}
|
||||
try:
|
||||
os.remove(path)
|
||||
except OSError:
|
||||
return 500, {'error': 'delete failed'}
|
||||
return 200, {'ok': True}
|
||||
|
||||
|
||||
HS_FILENAME_RE = re.compile(
|
||||
r'^(?:(\d+)_)?([0-9A-Fa-f]{2}(?:[:-][0-9A-Fa-f]{2}){5})_'
|
||||
r'([0-9A-Fa-f]{2}(?:[:-][0-9A-Fa-f]{2}){5})(?:_handshake)?'
|
||||
@@ -3757,13 +3768,12 @@ ROUTER.add('POST', r'/api/recon/gps/configure', h_recon_gps_configure)
|
||||
ROUTER.add('POST', r'/api/recon/wigle', h_recon_wigle)
|
||||
ROUTER.add('GET', r'/api/recon/wigle/files', h_recon_wigle_files)
|
||||
ROUTER.add('GET', r'/api/recon/wigle/files/([^/]+)', h_recon_wigle_file)
|
||||
ROUTER.add('GET', r'/api/recon/survey/live', h_recon_survey_live)
|
||||
ROUTER.add('POST', r'/api/recon/survey/start', h_recon_survey_start)
|
||||
ROUTER.add('POST', r'/api/recon/survey/stop', h_recon_survey_stop)
|
||||
ROUTER.add('GET', r'/api/recon/surveys', h_recon_surveys)
|
||||
ROUTER.add('GET', r'/api/recon/surveys/([^/]+)', h_recon_survey_detail)
|
||||
ROUTER.add('GET', r'/api/recon/surveys/([^/]+)/download/(csv|json|html)', h_recon_survey_download)
|
||||
ROUTER.add('DELETE', r'/api/recon/surveys/([^/]+)', h_recon_survey_delete)
|
||||
ROUTER.add('GET', r'/api/recon/archives', h_recon_archives)
|
||||
ROUTER.add('GET', r'/api/recon/archives/([^/]+)/scans', h_recon_archive_scans)
|
||||
ROUTER.add('GET', r'/api/recon/archives/([^/]+)/scans/(\d+)', h_recon_archive_scan_detail)
|
||||
ROUTER.add('GET', r'/api/recon/archives/([^/]+)/scans/(\d+)/download/json', h_recon_archive_scan_download)
|
||||
ROUTER.add('GET', r'/api/recon/archives/([^/]+)/scans/(\d+)/download/csv', h_recon_archive_scan_download_csv)
|
||||
ROUTER.add('GET', r'/api/recon/archives/([^/]+)/scans/(\d+)/download/html', h_recon_archive_scan_download_html)
|
||||
ROUTER.add('GET', r'/api/pineap/handshakes/location', h_handshakes_location)
|
||||
ROUTER.add('DELETE', r'/api/pineap/handshakes/all', h_handshakes_delete_all)
|
||||
ROUTER.add('GET', r'/api/pineap/handshakes', h_handshakes_get)
|
||||
|
||||
Reference in New Issue
Block a user