feat: recon surveys with OUI/vendor lookup and report views

Reconnaissance surveys (GPSD-tethered scan recording), OUI vendor
lookup for client/AP tables, survey/report pages, and the matching
test_recon.py suite. Co-authored with the recon-feature agent whose
work was finished in this checkout.
This commit is contained in:
2026-08-18 08:35:22 -05:00
parent f5cddb335a
commit 251b1f6261
7 changed files with 2095 additions and 45 deletions
+873 -25
View File
@@ -44,6 +44,21 @@ DEFAULT_RECON_DURATION = 30
_recon_scans_cache = {'db': None, 'updated': 0, 'data': {'scans': []}}
_recon_status_cache = {
'db': None, 'updated': 0, 'last_scan': None, 'last_activity': None}
SURVEY_DIR = os.environ.get('PAGER_SURVEY_DIR', '/root/loot/recon-surveys')
WIGLE_DIR = os.environ.get('PAGER_WIGLE_DIR', '/root/loot/wigle')
GPSD_CONFIG = os.environ.get('PAGER_GPSD_CONFIG', '/etc/config/gpsd')
GPSD_INIT = os.environ.get('PAGER_GPSD_INIT', '/etc/init.d/gpsd')
SERIAL_DIR = os.environ.get('PAGER_SERIAL_DIR', '/dev/serial/by-path')
SURVEY_MAX_SAMPLES = int(os.environ.get('PAGER_SURVEY_MAX_SAMPLES', '1800'))
SURVEY_SAMPLE_INTERVAL = float(os.environ.get('PAGER_SURVEY_SAMPLE_INTERVAL', '2.0'))
GPS_CACHE_SECONDS = 5.0
_survey_state = {'active': False, 'id': None, 'name': None, 'path': None,
'started': 0, 'samples': 0, 'last_sample': 0}
_survey_lock = threading.Lock()
_gps_cache = {'updated': 0, 'data': None}
_gps_lock = threading.Lock()
_payload_runs = {}
_payload_runs_lock = threading.Lock()
PAYLOAD_RUN_DIR = os.environ.get('PAGER_PAYLOAD_RUN_DIR', '/tmp/pagerwebui-payload-runs')
@@ -868,7 +883,7 @@ def h_deauth_client(ctx):
SQLITE_BUSY_MSGS = ('database is locked', 'database is busy')
def _db_rows(db, sql, _retries=1):
def _db_rows(db, sql, _retries=1, timeout=20):
if sqlite3 is not None:
try:
conn = sqlite3.connect('file:%s?mode=ro' % db, uri=True)
@@ -880,11 +895,13 @@ def _db_rows(db, sql, _retries=1):
conn.close()
except sqlite3.Error as exc:
raise RuntimeError('sqlite read failed: %s' % exc)
rc, out, err = device_run([SQLITE_CLI, '-json', '-cmd', '.timeout 500', db, sql])
rc, out, err = device_run([SQLITE_CLI, '-json', '-cmd', '.timeout 500', db, sql],
timeout=timeout)
attempt = 1
while rc != 0 and any(m in (err or '') for m in SQLITE_BUSY_MSGS) and attempt < _retries:
time.sleep(0.3)
rc, out, err = device_run([SQLITE_CLI, '-json', '-cmd', '.timeout 500', db, sql])
rc, out, err = device_run([SQLITE_CLI, '-json', '-cmd', '.timeout 500', db, sql],
timeout=timeout)
attempt += 1
st = _recon_scan_state
elapsed = time.time() - st['started'] if st['started'] else 0
@@ -895,7 +912,7 @@ def _db_rows(db, sql, _retries=1):
# The file is stable by this point, so bypass that stale lock read-only.
immutable_db = 'file:%s?immutable=1' % db
rc, out, err = device_run(
[SQLITE_CLI, '-json', immutable_db, sql])
[SQLITE_CLI, '-json', immutable_db, sql], timeout=timeout)
if rc != 0:
raise RuntimeError('sqlite read failed: %s' % (err or out).strip())
if out.strip():
@@ -961,7 +978,96 @@ def decode_encryption(v):
return ' '.join(parts) if parts else 'Open'
def recon_scans_data(limit=50):
# Compact OUI -> vendor table (24-bit prefix, hex without colons). Covers the
# vendors most commonly seen in the field; everything else resolves to
# 'Unknown'. Locally administered MACs resolve to 'Local'.
OUI_VENDORS = {
'00000C': 'Cisco', '000393': 'Apple', '000625': 'Linksys', '000C42': 'MikroTik',
'000DB9': 'Intel', '001376': 'MikroTik', '0016CB': 'Apple', '0017F2': 'Apple',
'001BFC': 'ASUS', '001E10': 'Huawei', '0025C7': 'Apple', '00500B': 'HP',
'0050F2': 'Micro-Star', '00606E': 'Xerox', '00A0C9': 'Intel', '00C0CA': 'Xerox',
'040CCE': 'Apple', '041854': 'Ubiquiti', '04ED33': 'Apple', '04F13E': 'Apple',
'04F938': 'Apple', '080007': 'Apple', '080028': 'Texas Instruments',
'080046': 'Sony', '083E8E': 'Apple', '089E01': 'Apple', '0C74C2': 'Apple',
'0C96BF': 'Huawei', '107BEF': 'Huawei', '10A2DC': 'Apple', '10BF48': 'Apple',
'1425BE': 'Huawei', '147A19': 'Apple', '1499E2': 'Apple', '14AC3C': 'Apple',
'14CC20': 'TP-Link', '181D86': 'Apple', '1C5C55': 'Apple', '1CE1A7': 'TP-Link',
'20010F': 'Apple', '20620B': 'Apple', '240AC4': 'Espressif', '241F4A': 'Apple',
'24A0DF': 'Apple', '24A43C': 'Ubiquiti', '24ABC0': 'Apple', '24B6FD': 'Espressif',
'247189': 'Espressif', '28CDC1': 'Raspberry Pi', '28CFE9': 'Apple',
'2C6E85': 'Apple', '2C7E81': 'Apple', '2CCF67': 'Raspberry Pi', '300C23': 'Apple',
'30720B': 'Apple', '3402E5': 'Apple', '3478D7': 'Apple', '381020': 'Apple',
'3C0754': 'Apple', '3C2177': 'TP-Link', '3C71BF': 'Espressif', '3C99F7': 'ASUS',
'3CD16E': 'Apple', '4006A0': 'Apple', '4083DE': 'Apple', '40B076': 'ASUS',
'40D3AE': 'Apple', '443212': 'Apple', '44C9A2': 'Apple', '44D9E7': 'Ubiquiti',
'48BF6B': 'Apple', '48C04E': 'Apple', '4C5E0C': 'MikroTik', '502D21': 'Apple',
'50C7BF': 'TP-Link', '54843B': 'Apple', '549392': 'ASUS', '54E43A': 'Apple',
'58B0D4': 'Apple', '586A97': 'TP-Link', '5C961D': 'Apple', '5CE1A1': 'Apple',
'603C07': 'Apple', '6083B2': 'Apple', '60D9C7': 'TP-Link', '640094': 'Apple',
'640980': 'TP-Link', '64167F': 'MikroTik', '649EF3': 'Apple', '6C0486': 'TP-Link',
'6C3B6B': 'MikroTik', '6C4008': 'Apple', '6CD68A': 'Apple', '70A2B3': 'Apple',
'742AF0': 'Apple', '748898': 'MikroTik', '74C46B': 'Apple', '782B1D': 'Apple',
'7847A6': 'Apple', '78CA39': 'Apple', '78E3B5': 'Ubiquiti', '7C0191': 'Apple',
'7CD1C3': 'Apple', '802AA8': 'Ubiquiti', '80BE05': 'Apple', '847C9B': 'Apple',
'84F3EB': 'Espressif', '88D42A': 'Apple', '8C3AF4': 'Apple', '8C7B9D': 'Apple',
'8CDEF9': 'Xiaomi', '90039F': 'Apple', '90B21F': 'Apple', '90F652': 'TP-Link',
'94099B': 'Apple', '98D6BB': 'Apple', '9CD24B': 'Ubiquiti', 'A01828': 'Apple',
'A020A6': 'Xiaomi', 'A05E6B': 'Apple', 'A07591': 'TP-Link', 'A41F72': 'Apple',
'A4B197': 'Apple', 'A4CF12': 'Espressif', 'A86484': 'Apple', 'ACBC32': 'Apple',
'B0DA00': 'Apple', 'B4E1EB': 'Apple', 'B827EB': 'Raspberry Pi',
'B8E45B': 'Raspberry Pi', 'BCA834': 'Apple', 'C03F0E': 'TP-Link',
'C04A00': 'Apple', 'C05E06': 'Apple', 'C08C60': 'Apple', 'C0E422': 'Apple',
'C45F5E': 'Espressif', 'C46516': 'Apple', 'C47D4F': 'Apple', 'C85B76': 'Apple',
'C8B5B7': 'Apple', 'C89A00': 'Apple', 'CC08E0': 'Apple', 'CC25EF': 'Apple',
'CC3D82': 'Apple', 'CCC73B': 'Apple', 'D0E140': 'Apple', 'D4154F': 'TP-Link',
'D4CA6D': 'Apple', 'D8A25E': 'Apple', 'DCA632': 'Raspberry Pi',
'DC6DCD': 'Apple', 'E03005': 'Apple', 'E45F01': 'Raspberry Pi',
'E45610': 'Apple', 'E4E4AB': 'Apple', 'E8802E': 'TP-Link', 'E8988F': 'Espressif',
'E8F2E2': 'Apple', 'EC8EB5': 'TP-Link', 'F0175E': 'Apple', 'F0D5BF': 'Apple',
'F4044C': 'Apple', 'F49BA0': 'Xiaomi', 'F4E97D': 'Apple', 'F8FFC2': 'Apple',
'FC633E': 'Google', 'FCF080': 'Apple',
}
def _oui_prefix(mac):
"""'C8:9E:43:64:80:80' / 'C89E43648080' -> 'C89E43' (uppercase, no colons)."""
mac = (mac or '').strip().upper().replace(':', '').replace('-', '').replace('.', '')
if len(mac) >= 6 and all(c in '0123456789ABCDEF' for c in mac[:6]):
return mac[:6]
return None
def oui_vendor(mac):
"""Best-effort vendor name for a MAC. Locally administered -> 'Local'."""
if not mac or mac == '--':
return 'Unknown'
prefix = _oui_prefix(mac)
if prefix is None:
return 'Unknown'
if int(prefix[1], 16) & 2: # locally administered (second hex digit bit 1)
return 'Local'
return OUI_VENDORS.get(prefix, 'Unknown')
def band_of(freq):
"""Channel frequency (MHz) -> '2.4' | '5' | '6' | '--'."""
if freq is None:
return '--'
try:
freq = int(freq)
except (TypeError, ValueError):
return '--'
if freq <= 0:
return '--'
if freq < 2500:
return '2.4'
if freq < 6000:
return '5'
return '6'
def recon_scans_data(limit=50, _timeout=20):
rows = _db_rows(RECON_DB,
'WITH recent AS (SELECT id, time, name FROM scan ORDER BY id DESC LIMIT %d), '
'devices AS (SELECT scan, count(*) AS devices FROM wifi_device '
@@ -978,42 +1084,90 @@ def recon_scans_data(limit=50):
'LEFT JOIN devices w ON w.scan = s.id '
'LEFT JOIN aps a ON a.scan = s.id '
'LEFT JOIN captures h ON h.scan = s.id '
'ORDER BY s.id DESC' % limit)
'ORDER BY s.id DESC' % limit, timeout=_timeout)
return {'scans': [{'id': r['id'], 'time': r['time'], 'name': r.get('name'),
'devices': r['devices'], 'aps': r['aps'],
'handshakes': r['handshakes']} for r in rows]}
def recon_scan_data(scan_id):
rows = _db_rows(RECON_DB,
"SELECT 'scan' AS kind, id AS row_id, time, name, "
"NULL AS mac, NULL AS bssid, NULL AS ssid, NULL AS hidden, "
"NULL AS channel, NULL AS encryption, NULL AS signal, NULL AS freq, "
"NULL AS packets, NULL AS stahash, NULL AS aphash "
"FROM scan WHERE id = %d "
"UNION ALL SELECT 'ap', hash, time, NULL, NULL, bssid, ssid, hidden, "
"channel, encryption, signal, freq, NULL, NULL, NULL "
"FROM ssid WHERE scan = %d AND type = 8 AND bssid IS NOT NULL "
"UNION ALL SELECT 'device', hash, time, NULL, mac, NULL, NULL, NULL, "
"NULL, NULL, signal, freq, packets, NULL, NULL "
"FROM wifi_device WHERE scan = %d "
"UNION ALL SELECT 'handshake', hash, time, NULL, NULL, NULL, NULL, NULL, "
"NULL, NULL, NULL, NULL, NULL, stahash, aphash "
"FROM handshake WHERE scan = %d" % (scan_id, scan_id, scan_id, scan_id))
def recon_scan_data(scan_id, _timeout=20, _limit=None):
"""Scan detail with per-AP enrichment (band/vendor/first_seen/last_seen)
plus an unassociated count.
With `_limit`, client rows are capped and the unassociated rows collapse to
a count. The live survey view never renders the full client list, and the
unassociated/device branches dominate query time on slow recon dbs, so the
bounded mode keeps the 2s survey poll cheap.
"""
if _limit:
sql = ("WITH dev AS (SELECT hash, time, mac, signal, freq, packets "
"FROM wifi_device WHERE scan = %d LIMIT %d) "
"SELECT 'scan' AS kind, id AS row_id, time, name, "
"NULL AS mac, NULL AS bssid, NULL AS ssid, NULL AS hidden, "
"NULL AS channel, NULL AS encryption, NULL AS signal, NULL AS freq, "
"NULL AS packets, NULL AS stahash, NULL AS aphash "
"FROM scan WHERE id = %d "
"UNION ALL SELECT 'ap', hash, time, NULL, NULL, bssid, ssid, hidden, "
"channel, encryption, signal, freq, NULL, NULL, NULL "
"FROM ssid WHERE scan = %d AND type = 8 AND bssid IS NOT NULL "
"UNION ALL SELECT 'device', hash, time, NULL, mac, NULL, NULL, NULL, "
"NULL, NULL, signal, freq, packets, NULL, NULL "
"FROM dev "
"UNION ALL SELECT 'handshake', hash, time, NULL, NULL, NULL, NULL, NULL, "
"NULL, NULL, NULL, NULL, NULL, stahash, aphash "
"FROM handshake WHERE scan = %d" % (scan_id, _limit, scan_id, scan_id, scan_id))
rows = _db_rows(RECON_DB, sql, timeout=_timeout)
cnt = _db_rows(RECON_DB, 'SELECT count(*) AS c FROM ssid WHERE scan = %d AND type = 4'
% scan_id, timeout=_timeout)
unassociated = cnt[0]['c'] if cnt else 0
else:
rows = _db_rows(RECON_DB,
"SELECT 'scan' AS kind, id AS row_id, time, name, "
"NULL AS mac, NULL AS bssid, NULL AS ssid, NULL AS hidden, "
"NULL AS channel, NULL AS encryption, NULL AS signal, NULL AS freq, "
"NULL AS packets, NULL AS stahash, NULL AS aphash "
"FROM scan WHERE id = %d "
"UNION ALL SELECT 'ap', hash, time, NULL, NULL, bssid, ssid, hidden, "
"channel, encryption, signal, freq, NULL, NULL, NULL "
"FROM ssid WHERE scan = %d AND type = 8 AND bssid IS NOT NULL "
"UNION ALL SELECT 'unassociated', hash, time, NULL, NULL, NULL, ssid, "
"hidden, channel, encryption, signal, freq, NULL, NULL, NULL "
"FROM ssid WHERE scan = %d AND type = 4 "
"UNION ALL SELECT 'device', hash, time, NULL, mac, NULL, NULL, NULL, "
"NULL, NULL, signal, freq, packets, NULL, NULL "
"FROM wifi_device WHERE scan = %d "
"UNION ALL SELECT 'handshake', hash, time, NULL, NULL, NULL, NULL, NULL, "
"NULL, NULL, NULL, NULL, NULL, stahash, aphash "
"FROM handshake WHERE scan = %d" % (scan_id, scan_id, scan_id, scan_id, scan_id),
timeout=_timeout)
unassociated = sum(1 for r in rows if r.get('kind') == 'unassociated')
scans = [r for r in rows if r.get('kind') == 'scan']
if not scans:
return None
# Per-bssid first/last sighting across the scan's ssid rows.
seen = {}
for r in (row for row in rows if row.get('kind') == 'ap'):
mac = (r.get('bssid') or '').strip().upper()
t = r.get('time') or 0
lo, hi = seen.get(mac, (None, None))
seen[mac] = (t if lo is None else min(lo, t), t if hi is None else max(hi, t))
aps = []
ap_macs = set()
for r in (row for row in rows if row.get('kind') == 'ap'):
ap_macs.add((r.get('bssid') or '').strip().upper())
mac = (r.get('bssid') or '').strip().upper()
ap_macs.add(mac)
lo, hi = seen.get(mac, (None, None))
aps.append({'bssid': fmt_mac(r.get('bssid')),
'ssid': decode_ssid(r.get('ssid')),
'hidden': bool(r.get('hidden')),
'channel': r.get('channel'),
'signal': r.get('signal'),
'freq': r.get('freq'),
'encryption': decode_encryption(r.get('encryption'))})
'encryption': decode_encryption(r.get('encryption')),
'band': band_of(r.get('freq')),
'vendor': oui_vendor(fmt_mac(r.get('bssid'))),
'first_seen': lo,
'last_seen': hi})
aps.sort(key=lambda row: row['signal'] if row['signal'] is not None else 0)
devices = [r for r in rows if r.get('kind') == 'device']
clients = []
@@ -1030,7 +1184,8 @@ def recon_scan_data(scan_id):
'time': r.get('time')})
return {'scan': {'id': scans[0]['row_id'], 'time': scans[0]['time'],
'name': scans[0].get('name')},
'aps': aps, 'clients': clients, 'handshakes': handshakes}
'aps': aps, 'clients': clients, 'handshakes': handshakes,
'unassociated': unassociated}
def h_recon_start(ctx):
@@ -1082,10 +1237,15 @@ def _recon_watchdog_tick():
The Pager has no recon-stop operation. log/recon/stop controls the storage
service and leaves recon.db locked, so timed scans must end natively.
Also records a survey sample when a survey is active.
"""
st = _recon_scan_state
if st['active'] and st['duration'] > 0 and time.time() - st['started'] >= st['duration']:
st['active'] = False
try:
_survey_sample()
except Exception:
pass
def h_recon_status(ctx):
@@ -1214,6 +1374,680 @@ def h_recon_scan_detail(ctx):
return 404, {'error': 'scan not found'}
return 200, data
# ---------------------------------------------------------------------------
# Recon report helpers (CSV / HTML) for scan and survey downloads.
# ---------------------------------------------------------------------------
def _fmt_ts(ts):
if not ts:
return '--'
try:
return time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(int(ts)))
except (ValueError, OSError, TypeError):
return str(ts)
def _csv_escape(v):
v = '' if v is None else str(v)
if any(c in v for c in ',"\n\r'):
return '"' + v.replace('"', '""') + '"'
return v
def _aps_csv(data):
out = ['bssid,ssid,hidden,band,channel,freq,encryption,signal,vendor,first_seen,last_seen']
for a in (data or {}).get('aps') or []:
out.append(','.join(_csv_escape(x) for x in [
a.get('bssid'), a.get('ssid'), int(bool(a.get('hidden'))),
a.get('band'), a.get('channel'), a.get('freq'),
a.get('encryption'), a.get('signal'), a.get('vendor'),
_fmt_ts(a.get('first_seen')), _fmt_ts(a.get('last_seen'))]))
out.append('unassociated,%d' % ((data or {}).get('unassociated') or 0))
return '\r\n'.join(out) + '\r\n'
def _survey_aps_csv(detail):
out = ['bssid,ssid,band,channel,min_dbm,avg_dbm,max_dbm,samples,first_seen,last_seen']
for a in (detail or {}).get('aps') or []:
out.append(','.join(_csv_escape(x) for x in [
a.get('bssid'), a.get('ssid'), a.get('band'), a.get('channel'),
a.get('min'), a.get('avg'), a.get('max'), a.get('samples'),
_fmt_ts(a.get('first_seen')), _fmt_ts(a.get('last_seen'))]))
return '\r\n'.join(out) + '\r\n'
def _esc_html(v):
if v is None:
return ''
return (str(v).replace('&', '&amp;').replace('<', '&lt;')
.replace('>', '&gt;').replace('"', '&quot;'))
REPORT_CSS = """
body { font-family: -apple-system, 'Segoe UI', Roboto, sans-serif; margin: 24px; color: #222; background: #fff; }
h1 { font-size: 20px; margin: 0 0 4px; }
.sub { color: #666; margin-bottom: 16px; }
table { border-collapse: collapse; width: 100%; font-size: 13px; }
th, td { border: 1px solid #ddd; padding: 6px 10px; text-align: left; }
th { background: #f4f4f4; }
tr:nth-child(even) td { background: #fafafa; }
.stats { margin: 12px 0; font-size: 13px; color: #333; }
"""
def _html_table(headers, rows):
out = ['<table><thead><tr>']
for header in headers:
out.append('<th>%s</th>' % _esc_html(header))
out.append('</tr></thead><tbody>')
for row in rows:
out.append('<tr>')
for cell in row:
out.append('<td>%s</td>' % _esc_html(cell))
out.append('</tr>')
out.append('</tbody></table>')
return ''.join(out)
def _html_doc(title, subtitle, body_html, stats=None):
parts = ['<!DOCTYPE html><html><head><meta charset="utf-8"><title>%s</title>'
'<style>%s</style></head><body>' % (_esc_html(title), REPORT_CSS)]
parts.append('<h1>%s</h1>' % _esc_html(title))
parts.append('<div class="sub">%s</div>' % _esc_html(subtitle))
for label, value in (stats or []):
parts.append('<div class="stats"><b>%s:</b> %s</div>' % (_esc_html(label), value))
parts.append(body_html)
parts.append('</body></html>')
return ''.join(parts)
def _recon_read_retry(fn, attempts=3, pause=1.0):
"""Retry a recon.db read briefly; pineapd's write bursts hold the DB
exclusive lock and reads can time out mid-burst."""
last = None
for _ in range(attempts):
try:
return fn()
except RuntimeError as exc:
last = exc
time.sleep(pause)
raise last
def _scan_client_count(scan_id, _timeout=12):
rows = _db_rows(RECON_DB,
"SELECT count(*) AS c FROM wifi_device w WHERE w.scan = %d "
"AND w.mac NOT IN (SELECT DISTINCT bssid FROM ssid "
"WHERE scan = %d AND type = 8 AND bssid IS NOT NULL)"
% (scan_id, scan_id), timeout=_timeout)
return rows[0]['c'] if rows else 0
def h_recon_scan_download_csv(ctx):
scan_id = int(ctx.args[0])
try:
data = _recon_read_retry(lambda: recon_scan_data(scan_id, _timeout=15, _limit=300))
except RuntimeError:
return 503, {'error': 'recon database is temporarily unavailable'}
if data is None:
return 404, {'error': 'scan not found'}
return 200, Download(_aps_csv(data).encode('utf-8'), 'text/csv',
'scan-%d.csv' % scan_id)
def h_recon_scan_download_html(ctx):
scan_id = int(ctx.args[0])
try:
data = _recon_read_retry(lambda: recon_scan_data(scan_id, _timeout=15, _limit=300))
client_count = _scan_client_count(scan_id, _timeout=12)
except RuntimeError:
return 503, {'error': 'recon database is temporarily unavailable'}
if data is None:
return 404, {'error': 'scan not found'}
rows = []
for a in data.get('aps') or []:
rows.append([a.get('ssid') or '(hidden)', a.get('bssid'), a.get('band'),
a.get('channel'), a.get('signal'), a.get('encryption'),
a.get('vendor'), _fmt_ts(a.get('first_seen')),
_fmt_ts(a.get('last_seen'))])
scan = data.get('scan') or {}
stats = [('Started', _fmt_ts(scan.get('time'))),
('Access points', len(data.get('aps') or [])),
('Clients', client_count),
('Handshakes', len(data.get('handshakes') or [])),
('Unassociated', data.get('unassociated') or 0)]
body = _html_table(['SSID', 'BSSID', 'Band', 'Ch', 'Signal', 'Encryption',
'Vendor', 'First seen', 'Last seen'], rows)
return 200, Download(_html_doc('Scan #%d' % scan.get('id'),
'Pager recon capture report', body,
stats=stats).encode('utf-8'),
'text/html', 'scan-%d.html' % scan_id)
# ---------------------------------------------------------------------------
# GPS: serial device discovery, gpsd control, TPV/SKY parsing, status cache.
# Tied to the Glytch GPS mod (gpsd + uci 'gpsd' config section).
# ---------------------------------------------------------------------------
def _gps_serial_candidates():
candidates = []
if os.path.isdir(SERIAL_DIR):
try:
names = sorted(os.listdir(SERIAL_DIR))
except OSError:
names = []
for name in names:
path = os.path.join(SERIAL_DIR, name)
# by-path entries are named like '1.3_1-1.3:1.0' and only reveal
# their ttyACM/ttyUSB target through the resolved symlink.
target = os.path.realpath(path)
if ('ttyACM' in name or 'ttyUSB' in name
or 'ttyACM' in target or 'ttyUSB' in target):
candidates.append((name, path))
return candidates
def _uci_gps_get():
rc, out, err = device_run(['uci', 'get', 'gpsd.core.device'])
return out.strip() or None
def _uci_gps_set(device):
device_run(['uci', 'set', 'gpsd.core.device=%s' % device])
device_run(['uci', 'commit', 'gpsd'])
def _gpsd_running():
rc, out, err = device_run(['pgrep', '-f', 'gpsd'])
return rc == 0
def _gpsd_restart():
device_run([GPSD_INIT, 'restart'], timeout=15)
def _gps_from_gpspipe():
try:
p = subprocess.Popen(['gpspipe', '-w', '-n', '3'],
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
out, _ = p.communicate(timeout=8)
except Exception:
return None
tpv = None
sky = None
for line in out.decode('utf-8', 'replace').splitlines():
obj = _json_or(line)
if not isinstance(obj, dict):
continue
cls = obj.get('class')
if cls == 'TPV' and tpv is None:
tpv = obj
elif cls == 'SKY' and sky is None:
sky = obj
if tpv is None:
return None
return {'fix': tpv.get('mode') or 0,
'lat': tpv.get('lat'),
'lon': tpv.get('lon'),
'alt': tpv.get('alt'),
'speed': tpv.get('speed'),
'satellites': (sky or {}).get('satellites') or None}
def _gps_from_hak5cmd():
out = hak5('GPS_GET', timeout=10)
obj = _json_or(out)
if not isinstance(obj, dict):
return None
return {'fix': obj.get('fix') or obj.get('mode') or 0,
'lat': obj.get('lat') or obj.get('latitude'),
'lon': obj.get('lon') or obj.get('longitude'),
'alt': obj.get('alt'),
'speed': obj.get('speed'),
'satellites': obj.get('satellites') or obj.get('satellites_used')}
def _wigle_config():
_, cur = daemon_sock_call('GET', '/api/pineap/get_config')
base = dict(PINEAP_CONFIG_DEFAULTS)
if isinstance(cur, dict) and 'reconpath' in cur:
base.update(cur)
return base
def _wigle_set(enabled):
base = _wigle_config()
base['logwigle'] = bool(enabled)
return _daemon_proxy('PUT', 'set_config', base)
def _gps_status_data_nocache():
try:
device = _uci_gps_get()
except Exception:
device = None
try:
present = any(os.path.exists(path)
for _, path in _gps_serial_candidates())
except Exception:
present = False
try:
running = _gpsd_running()
except Exception:
running = False
data = {'device': device, 'present': present, 'fix': 0,
'lat': None, 'lon': None, 'alt': None, 'speed': None,
'satellites': None, 'gpsd_running': running,
'updated': None, 'wigle': _wigle_config().get('logwigle', False)}
fix = _gps_from_gpspipe() if running else None
if fix is None:
fix = _gps_from_hak5cmd()
if fix:
data.update(fix)
data['updated'] = int(time.time())
return data
def _gps_status_data():
with _gps_lock:
if (_gps_cache['data'] is not None
and time.time() - _gps_cache['updated'] < GPS_CACHE_SECONDS):
return _gps_cache['data']
data = _gps_status_data_nocache()
_gps_cache.update({'updated': time.time(), 'data': data})
return data
def h_recon_gps(ctx):
return 200, _gps_status_data()
def h_recon_gps_configure(ctx):
try:
candidates = _gps_serial_candidates()
except Exception:
candidates = []
if not candidates:
return 200, {'present': False, 'error': 'No GPS serial device found'}
try:
current = _uci_gps_get()
except Exception:
current = None
ordered = sorted(candidates, key=lambda c: (c[0] != current, c[0]))
tried = []
for name, path in ordered[:3]:
tried.append(name)
try:
_uci_gps_set(name)
_gpsd_restart()
time.sleep(1.5)
fix = _gps_from_gpspipe()
except Exception:
fix = None
if fix is not None and fix.get('fix'):
data = _gps_status_data_nocache()
data.update({'configured': True, 'device': name, 'tried': tried,
'lock': True})
return 200, data
data = _gps_status_data_nocache()
data.update({'configured': True, 'device': ordered[0][0], 'tried': tried,
'note': 'GPS bound, waiting for a fix'})
return 200, data
# ---------------------------------------------------------------------------
# WiGLE logging: toggle the daemon 'logwigle' setting, list and download the
# CSV files the daemon writes under WIGLE_DIR.
# ---------------------------------------------------------------------------
def wigle_files_data():
files = []
if os.path.isdir(WIGLE_DIR):
try:
names = sorted(os.listdir(WIGLE_DIR))
except OSError:
names = []
for name in names:
path = os.path.join(WIGLE_DIR, name)
if not os.path.isfile(path):
continue
try:
size = os.path.getsize(path)
mtime = int(os.path.getmtime(path))
except OSError:
continue
rows = None
if size <= 2 * 1024 * 1024:
try:
with open(path, 'r', errors='replace') as fh:
lines = [line for line in fh if line.strip()]
# WiGLE CSVs lead with a meta line ('WigleWifi-1.6,...')
# followed by the column header; only count data rows.
header_offset = 2 if lines and lines[0].startswith('WigleWifi') else 1
rows = max(0, len(lines) - header_offset)
except OSError:
rows = None
files.append({'name': name, 'size': size, 'mtime': mtime, 'rows': rows})
return {'files': files}
def h_recon_wigle_files(ctx):
return 200, wigle_files_data()
def h_recon_wigle_file(ctx):
name = _unquote_plus(ctx.args[0])
path = _safe_join(WIGLE_DIR, name)
if path is None or not os.path.isfile(path):
return 404, {'error': 'file not found'}
try:
with open(path, 'rb') as fh:
body = fh.read()
except OSError:
return 404, {'error': 'file not found'}
return 200, Download(body, 'text/csv', os.path.basename(path))
def h_recon_wigle(ctx):
enable = bool((getattr(ctx, 'body', None) or {}).get('enable'))
status, data = _wigle_set(enable)
if status != 200:
return status, data
if enable:
hak5('WIGLE_START', timeout=10)
else:
hak5('WIGLE_STOP', timeout=10)
resp = {'ok': True, 'wigle': enable}
if enable:
files = wigle_files_data().get('files') or []
if files:
resp['filename'] = files[-1]['name']
return 200, resp
# ---------------------------------------------------------------------------
# Surveys: JSONL overlay on the device (meta line + one 'sample' line per
# tick of the always-on recon). The recon.db itself is never modified.
# ---------------------------------------------------------------------------
def _survey_path(sid):
return _safe_join(SURVEY_DIR, '%s.jsonl' % sid)
def _survey_slug(name):
return re.sub(r'[^A-Za-z0-9]+', '-', (name or '').strip()).strip('-')
def _survey_recording_state():
st = _survey_state
if not st['active']:
return None
return {'active': True, 'id': st['id'], 'name': st['name'],
'started': st['started'], 'samples': st['samples']}
def _survey_sample():
with _survey_lock:
st = _survey_state
if not st['active'] or not st['path']:
return
now = time.time()
if now - st['last_sample'] < SURVEY_SAMPLE_INTERVAL:
return
if st['samples'] >= SURVEY_MAX_SAMPLES:
st['active'] = False
return
try:
scans = recon_scans_data(1, _timeout=6).get('scans') or []
detail = recon_scan_data(scans[0]['id'], _timeout=12, _limit=300) if scans else None
gps = _gps_status_data()
sample = {'t': int(now),
'scan': (detail or {}).get('scan'),
'aps': (detail or {}).get('aps') or [],
'clients': (detail or {}).get('clients') or [],
'handshakes': (detail or {}).get('handshakes') or [],
'unassociated': (detail or {}).get('unassociated') or 0,
'gps': {'fix': gps.get('fix'), 'lat': gps.get('lat'),
'lon': gps.get('lon'),
'satellites': gps.get('satellites')}}
with open(st['path'], 'a') as fh:
fh.write(json.dumps({'sample': sample}) + '\n')
st['samples'] += 1
st['last_sample'] = now
except Exception:
pass
def h_recon_survey_start(ctx):
with _survey_lock:
if _survey_state['active']:
return 409, {'error': 'a survey is already recording'}
name = ((getattr(ctx, 'body', None) or {}).get('name') or '').strip()
sid = time.strftime('%Y%m%d-%H%M%S')
slug = _survey_slug(name)
if slug:
sid += '-' + slug
path = _survey_path(sid)
if path is None:
return 500, {'error': 'invalid survey id'}
try:
os.makedirs(SURVEY_DIR, exist_ok=True)
except OSError:
return 500, {'error': 'cannot create survey directory'}
meta = {'id': sid, 'name': name, 'started': int(time.time()),
'interval': SURVEY_SAMPLE_INTERVAL, 'max_samples': SURVEY_MAX_SAMPLES}
try:
with open(path, 'w') as fh:
fh.write(json.dumps({'meta': meta}) + '\n')
except OSError:
return 500, {'error': 'cannot write survey file'}
_survey_state.update({'active': True, 'id': sid, 'name': name, 'path': path,
'started': meta['started'], 'samples': 0,
'last_sample': 0})
return 200, {'ok': True, 'id': sid, 'started': meta['started']}
def h_recon_survey_stop(ctx):
with _survey_lock:
st = dict(_survey_state)
if st['active']:
_survey_state['active'] = False
return 200, {'ok': True, 'samples': st['samples'], 'id': st['id']}
def h_recon_survey_live(ctx):
detail = None
try:
scans = recon_scans_data(1, _timeout=6).get('scans') or []
if scans:
detail = recon_scan_data(scans[0]['id'], _timeout=12, _limit=300)
except RuntimeError:
pass
try:
gps = _gps_status_data()
except Exception:
gps = {}
return 200, {'scan': (detail or {}).get('scan'),
'aps': (detail or {}).get('aps') or [],
'clients': (detail or {}).get('clients') or [],
'handshakes': (detail or {}).get('handshakes') or [],
'unassociated': (detail or {}).get('unassociated') or 0,
'gps': gps,
'recording': _survey_recording_state()}
def recon_surveys_data():
surveys = []
if os.path.isdir(SURVEY_DIR):
try:
names = sorted(os.listdir(SURVEY_DIR), reverse=True)
except OSError:
names = []
for name in names:
if not name.endswith('.jsonl'):
continue
path = os.path.join(SURVEY_DIR, name)
sid = name[:-6]
meta = None
samples = 0
try:
size = os.path.getsize(path)
with open(path, 'r', errors='replace') as fh:
for line in fh:
line = line.strip()
if not line:
continue
try:
obj = json.loads(line)
except ValueError:
continue
if 'meta' in obj:
meta = obj['meta']
elif 'sample' in obj:
samples += 1
except OSError:
continue
if meta is None:
continue
surveys.append({'id': sid, 'name': meta.get('name') or sid,
'started': meta.get('started'),
'samples': samples, 'size': size})
return {'surveys': surveys}
def h_recon_surveys(ctx):
try:
return 200, recon_surveys_data()
except Exception:
return 200, {'surveys': []}
def recon_survey_data(sid):
path = _survey_path(sid)
if path is None or not os.path.isfile(path):
return None
agg = {}
order = []
gps_fixes = []
try:
with open(path, 'r', errors='replace') as fh:
for line in fh:
line = line.strip()
if not line:
continue
try:
obj = json.loads(line)
except ValueError:
continue
s = obj.get('sample')
if not isinstance(s, dict):
continue
g = s.get('gps') or {}
if g.get('lat') is not None and g.get('lon') is not None:
gps_fixes.append({'t': s.get('t'), 'lat': g['lat'],
'lon': g['lon']})
for a in s.get('aps') or []:
bssid = a.get('bssid')
if not bssid:
continue
entry = agg.get(bssid)
if entry is None:
entry = {'ssid': a.get('ssid'), 'bssid': bssid,
'band': a.get('band'), 'channel': a.get('channel'),
'min': None, 'max': None, 'sum': 0, 'count': 0,
'first_seen': None, 'last_seen': None}
agg[bssid] = entry
order.append(bssid)
sig = a.get('signal')
if sig is not None:
entry['min'] = sig if entry['min'] is None else min(entry['min'], sig)
entry['max'] = sig if entry['max'] is None else max(entry['max'], sig)
entry['sum'] += sig
entry['count'] += 1
t = s.get('t')
if t:
entry['first_seen'] = (t if entry['first_seen'] is None
else min(entry['first_seen'], t))
entry['last_seen'] = (t if entry['last_seen'] is None
else max(entry['last_seen'], t))
except OSError:
return None
aps = []
for bssid in order:
e = agg[bssid]
aps.append({'ssid': e['ssid'], 'bssid': e['bssid'], 'band': e['band'],
'channel': e['channel'], 'min': e['min'],
'avg': round(e['sum'] / e['count']) if e['count'] else None,
'max': e['max'], 'samples': e['count'],
'first_seen': e['first_seen'], 'last_seen': e['last_seen']})
aps.sort(key=lambda a: a['avg'] if a['avg'] is not None else 0)
return {'id': sid, 'aps': aps, 'gps_fixes': len(gps_fixes),
'first_gps': gps_fixes[0] if gps_fixes else None,
'last_gps': gps_fixes[-1] if gps_fixes else None}
def h_recon_survey_detail(ctx):
sid = ctx.args[0]
data = recon_survey_data(sid)
if data is None:
return 404, {'error': 'survey not found'}
return 200, data
def h_recon_survey_download(ctx):
sid = ctx.args[0]
fmt = ctx.args[1]
if fmt == 'json':
data = recon_survey_data(sid)
if data is None:
return 404, {'error': 'survey not found'}
return 200, Download(json.dumps(data, indent=2).encode('utf-8'),
'application/json', 'survey-%s.json' % sid)
data = recon_survey_data(sid)
if data is None:
return 404, {'error': 'survey not found'}
if fmt == 'csv':
return 200, Download(_survey_aps_csv(data).encode('utf-8'), 'text/csv',
'survey-%s.csv' % sid)
if fmt == 'html':
rows = []
for a in data.get('aps') or []:
rows.append([a.get('ssid') or '(hidden)', a.get('bssid'),
a.get('band'), a.get('channel'), a.get('min'),
a.get('avg'), a.get('max'), a.get('samples'),
_fmt_ts(a.get('first_seen')), _fmt_ts(a.get('last_seen'))])
fixes = data.get('gps_fixes') or 0
fg = data.get('first_gps') or {}
lg = data.get('last_gps') or {}
if fixes:
gps = ('%d fixes &middot; first %.5f, %.5f &middot; last %.5f, %.5f'
% (fixes, fg.get('lat') or 0, fg.get('lon') or 0,
lg.get('lat') or 0, lg.get('lon') or 0))
else:
gps = 'No GPS fixes during this survey'
body = _html_table(['SSID', 'BSSID', 'Band', 'Ch', 'Min', 'Avg', 'Max',
'Samples', 'First', 'Last'], rows)
return 200, Download(_html_doc('Survey %s' % sid,
'AP signal aggregates', body,
stats=[('GPS', gps)]).encode('utf-8'),
'text/html', 'survey-%s.html' % sid)
return 404, {'error': 'unknown format'}
def h_recon_survey_delete(ctx):
sid = ctx.args[0]
path = _survey_path(sid)
if path is None or not os.path.isfile(path):
return 404, {'error': 'survey not found'}
with _survey_lock:
if _survey_state.get('id') == sid and _survey_state['active']:
return 409, {'error': 'cannot delete the survey that is recording'}
try:
os.remove(path)
except OSError:
return 500, {'error': 'delete failed'}
return 200, {'ok': True}
HS_FILENAME_RE = re.compile(
r'^(?:(\d+)_)?([0-9A-Fa-f]{2}(?:[:-][0-9A-Fa-f]{2}){5})_'
@@ -2916,6 +3750,20 @@ ROUTER.add('GET', r'/api/recon/scans/(\d+)', h_recon_scan_detail)
ROUTER.add('DELETE', r'/api/recon/scans/(\d+)', h_recon_delete)
ROUTER.add('GET', r'/api/recon/events', h_recon_events)
ROUTER.add('POST', r'/api/recon/examine', h_recon_examine)
ROUTER.add('GET', r'/api/recon/scans/(\d+)/download/csv', h_recon_scan_download_csv)
ROUTER.add('GET', r'/api/recon/scans/(\d+)/download/html', h_recon_scan_download_html)
ROUTER.add('GET', r'/api/recon/gps', h_recon_gps)
ROUTER.add('POST', r'/api/recon/gps/configure', h_recon_gps_configure)
ROUTER.add('POST', r'/api/recon/wigle', h_recon_wigle)
ROUTER.add('GET', r'/api/recon/wigle/files', h_recon_wigle_files)
ROUTER.add('GET', r'/api/recon/wigle/files/([^/]+)', h_recon_wigle_file)
ROUTER.add('GET', r'/api/recon/survey/live', h_recon_survey_live)
ROUTER.add('POST', r'/api/recon/survey/start', h_recon_survey_start)
ROUTER.add('POST', r'/api/recon/survey/stop', h_recon_survey_stop)
ROUTER.add('GET', r'/api/recon/surveys', h_recon_surveys)
ROUTER.add('GET', r'/api/recon/surveys/([^/]+)', h_recon_survey_detail)
ROUTER.add('GET', r'/api/recon/surveys/([^/]+)/download/(csv|json|html)', h_recon_survey_download)
ROUTER.add('DELETE', r'/api/recon/surveys/([^/]+)', h_recon_survey_delete)
ROUTER.add('GET', r'/api/pineap/handshakes/location', h_handshakes_location)
ROUTER.add('DELETE', r'/api/pineap/handshakes/all', h_handshakes_delete_all)
ROUTER.add('GET', r'/api/pineap/handshakes', h_handshakes_get)