from datetime import datetime, timezone import csv import os import shutil import sqlite3 import tempfile import urllib.error import urllib.request import urllib.parse from pathlib import Path class VendorService: REGISTRIES = ( "MA-L", "MA-M", "MA-S", ) PREFIX_LENGTHS = ( 6, 7, 9, ) BACKUP_LIMIT = 3 def __init__( self, vendor_database, wifi_database, sources, mirrors=None ): self.vendor_database = Path(vendor_database) self.wifi_database = Path(wifi_database) self.sources = dict(sources) self.mirrors = dict( mirrors or {} ) self.vendor_database.parent.mkdir( parents=True, exist_ok=True ) self.backup_directory = ( self.vendor_database.parent / "backups" ) # ============================================================ # Current database status # ============================================================ def status(self): exists = self.vendor_database.exists() result = { "database": str( self.vendor_database ), "exists": exists, "healthy": False, "total": 0, "ma_l": 0, "ma_m": 0, "ma_s": 0, "schema_version": None, "last_import": None, "import_status": None, "size": ( self.vendor_database.stat().st_size if exists else 0 ) } if not exists: return result conn = None try: conn = sqlite3.connect( self.vendor_database ) self._validate_schema( conn ) result["total"] = conn.execute( "SELECT COUNT(*) FROM oui" ).fetchone()[0] rows = conn.execute( """ SELECT confidence, COUNT(*) FROM oui GROUP BY confidence """ ).fetchall() for confidence, count in rows: if confidence == "MA-L": result["ma_l"] = count elif confidence == "MA-M": result["ma_m"] = count elif confidence == "MA-S": result["ma_s"] = count result["schema_version"] = self._get_metadata( conn, "schema_version" ) result["last_import"] = self._get_metadata( conn, "last_import" ) result["import_status"] = self._get_metadata( conn, "import_status" ) result["healthy"] = True except ( OSError, sqlite3.Error, ValueError ): result["healthy"] = False finally: if conn is not None: conn.close() return result # ============================================================ # Prepare update # ============================================================ def prepare_update( self, progress_callback=None ): def progress( stage, source=None, message=None ): if progress_callback is not None: progress_callback( stage, source, message ) work_directory = Path( tempfile.mkdtemp( prefix="wifi-gps-mapper-vendor-" ) ) try: progress( "starting", message="Подготовка обновления" ) downloaded = self._download_sources( work_directory, progress_callback=progress_callback ) candidate = ( work_directory / "oui.sqlite3" ) progress( "building", message="Построение базы OUI" ) build_stats = self._build_candidate( downloaded, candidate ) progress( "analyzing", message="Анализ изменений" ) oui_diff = self._compare_databases( self.vendor_database, candidate ) ap_impact = self._calculate_ap_impact( candidate ) progress( "ready", message="Обновление готово к применению" ) return { "directory": str( work_directory ), "candidate": str( candidate ), "sources": downloaded, "build": build_stats, "oui": oui_diff, "ap": ap_impact } except Exception: shutil.rmtree( work_directory, ignore_errors=True ) raise # ============================================================ # Download # ============================================================ def _download_sources( self, work_directory, progress_callback=None ): downloaded = {} for name, url in self.sources.items(): if not isinstance( url, str ) or not url.strip(): raise ValueError( f"Invalid source URL for {name}" ) mirror = self.mirrors.get( name ) urls = [ ( "primary", url ) ] if ( isinstance( mirror, str ) and mirror.strip() ): urls.append( ( "mirror", mirror ) ) last_error = None for source_type, source_url in urls: parsed = urllib.parse.urlparse( source_url ) if parsed.scheme not in ( "http", "https" ): last_error = ValueError( f"Unsupported source URL " f"for {name}: {source_url}" ) continue if progress_callback is not None: progress_callback( "downloading", name, ( "Загрузка " f"{'зеркала' if source_type == 'mirror' else 'IEEE'}" ) ) target = ( work_directory / f"{name}.csv" ) request = urllib.request.Request( source_url, headers={ "User-Agent": "WiFi-GPS-Mapper/" "vendor-service" } ) try: with urllib.request.urlopen( request, timeout=60 ) as response: with open( target, "wb" ) as file: shutil.copyfileobj( response, file ) if progress_callback is not None: progress_callback( "validating", name, "Проверка загруженного источника" ) self._validate_csv( target, name ) downloaded[name] = str( target ) if progress_callback is not None: progress_callback( "validating", name, ( "Источник принят" if source_type == "primary" else "Резервное зеркало принято" ) ) break except ( urllib.error.URLError, OSError, ValueError ) as exc: last_error = exc if ( source_type == "primary" and len(urls) > 1 ): if progress_callback is not None: progress_callback( "fallback", name, ( "Основной источник недоступен, " "переход на зеркало" ) ) continue else: raise RuntimeError( f"Failed to download and validate " f"source {name}: {last_error}" ) from last_error return downloaded # ============================================================ # CSV validation # ============================================================ def _validate_csv( self, filename, source_name ): required = { "Registry", "Assignment", "Organization Name" } try: with open( filename, "r", encoding="utf-8", errors="ignore", newline="" ) as file: reader = csv.DictReader( file ) if reader.fieldnames is None: raise ValueError( "CSV has no header" ) fields = { field.strip() for field in reader.fieldnames if field is not None } missing = required - fields if missing: raise ValueError( f"missing columns: " f"{', '.join(sorted(missing))}" ) first_valid = False for row in reader: registry = row.get( "Registry" ) assignment = row.get( "Assignment" ) organization = row.get( "Organization Name" ) if ( registry and registry in self.REGISTRIES and assignment and organization ): first_valid = True break if not first_valid: raise ValueError( "CSV contains no valid IEEE records" ) except ( OSError, UnicodeError, csv.Error ) as exc: raise ValueError( f"Invalid CSV source {source_name}: {exc}" ) from exc # ============================================================ # CSV validation # ============================================================ def _validate_csv( self, filename, source_name ): required = { "Registry", "Assignment", "Organization Name" } try: with open( filename, "r", encoding="utf-8", errors="ignore", newline="" ) as file: reader = csv.DictReader( file ) if reader.fieldnames is None: raise ValueError( "CSV has no header" ) fields = { field.strip() for field in reader.fieldnames if field is not None } missing = required - fields if missing: raise ValueError( f"missing columns: " f"{', '.join(sorted(missing))}" ) first_valid = False for row in reader: registry = row.get( "Registry" ) assignment = row.get( "Assignment" ) organization = row.get( "Organization Name" ) if ( registry and registry in self.REGISTRIES and assignment and organization ): first_valid = True break if not first_valid: raise ValueError( "CSV contains no valid IEEE records" ) except ( OSError, UnicodeError, csv.Error ) as exc: raise ValueError( f"Invalid CSV source {source_name}: {exc}" ) from exc # ============================================================ # Candidate database # ============================================================ def _build_candidate( self, files, candidate ): conn = sqlite3.connect( candidate ) imported = 0 skipped = 0 stats = { "MA-L": 0, "MA-M": 0, "MA-S": 0 } try: self._create_schema( conn ) for source_name, filename in files.items(): with open( filename, "r", encoding="utf-8", errors="ignore", newline="" ) as file: reader = csv.DictReader( file ) for row in reader: registry = row.get( "Registry" ) assignment = row.get( "Assignment" ) organization = row.get( "Organization Name" ) if not registry: continue if registry not in self.REGISTRIES: continue if not assignment: skipped += 1 continue if not organization: skipped += 1 continue assignment = ( assignment .replace("-", "") .replace(":", "") .replace(".", "") .strip() .upper() ) if not assignment: skipped += 1 continue if not all( char in "0123456789ABCDEF" for char in assignment ): skipped += 1 continue if len( assignment ) not in self.PREFIX_LENGTHS: skipped += 1 continue try: conn.execute( """ INSERT OR REPLACE INTO oui ( prefix, vendor, confidence, prefix_length ) VALUES (?, ?, ?, ?) """, ( assignment, organization.strip(), registry, len(assignment) ) ) except sqlite3.Error: skipped += 1 continue imported += 1 stats[registry] += 1 conn.execute( """ INSERT OR REPLACE INTO metadata ( key, value ) VALUES (?, ?) """, ( "schema_version", "1" ) ) conn.execute( """ INSERT OR REPLACE INTO metadata ( key, value ) VALUES (?, ?) """, ( "import_status", "ok" ) ) conn.execute( """ INSERT OR REPLACE INTO metadata ( key, value ) VALUES (?, ?) """, ( "source_count", str( len(files) ) ) ) conn.execute( """ INSERT OR REPLACE INTO metadata ( key, value ) VALUES (?, ?) """, ( "last_import", datetime.now( timezone.utc ).isoformat().replace( "+00:00", "Z" ) ) ) conn.commit() return { "imported": imported, "skipped": skipped, "ma_l": stats["MA-L"], "ma_m": stats["MA-M"], "ma_s": stats["MA-S"] } except Exception: conn.rollback() raise finally: conn.close() # ============================================================ # Database schema # ============================================================ @staticmethod def _create_schema(conn): conn.execute( """ CREATE TABLE IF NOT EXISTS oui ( prefix TEXT PRIMARY KEY, vendor TEXT NOT NULL, confidence TEXT NOT NULL, prefix_length INTEGER NOT NULL ) """ ) conn.execute( """ CREATE TABLE IF NOT EXISTS metadata ( key TEXT PRIMARY KEY, value TEXT ) """ ) conn.execute( """ CREATE INDEX IF NOT EXISTS idx_prefix ON oui(prefix) """ ) @staticmethod def _validate_schema(conn): rows = conn.execute( """ SELECT name FROM sqlite_master WHERE type='table' """ ).fetchall() tables = { row[0] for row in rows } if "oui" not in tables: raise ValueError( "OUI table is missing" ) columns = { row[1] for row in conn.execute( "PRAGMA table_info(oui)" ).fetchall() } required = { "prefix", "vendor", "confidence", "prefix_length" } if not required.issubset( columns ): raise ValueError( "Invalid OUI database schema" ) # ============================================================ # OUI comparison # ============================================================ def _compare_databases( self, current, candidate ): current_rows = self._load_oui( current ) candidate_rows = self._load_oui( candidate ) current_keys = set( current_rows ) candidate_keys = set( candidate_rows ) added = ( candidate_keys - current_keys ) removed = ( current_keys - candidate_keys ) common = ( current_keys & candidate_keys ) changed = { prefix for prefix in common if current_rows[prefix] != candidate_rows[prefix] } unchanged = ( len(common) - len(changed) ) return { "current": len( current_rows ), "candidate": len( candidate_rows ), "added": len( added ), "changed": len( changed ), "removed": len( removed ), "unchanged": unchanged, "identical": ( not added and not changed and not removed ) } @staticmethod def _load_oui(filename): if not Path( filename ).exists(): return {} conn = sqlite3.connect( filename ) try: rows = conn.execute( """ SELECT prefix, vendor, confidence, prefix_length FROM oui """ ).fetchall() return { row[0]: ( row[1], row[2], row[3] ) for row in rows } finally: conn.close() # ============================================================ # AP impact # ============================================================ def _calculate_ap_impact( self, candidate ): result = { "total": 0, "unchanged": 0, "unknown_to_vendor": 0, "vendor_to_vendor": 0, "vendor_to_unknown": 0 } if not self.wifi_database.exists(): return result vendor_conn = sqlite3.connect( candidate ) wifi_conn = sqlite3.connect( self.wifi_database ) try: self._validate_wifi_schema( wifi_conn ) rows = wifi_conn.execute( """ SELECT bssid, vendor FROM access_points """ ).fetchall() result["total"] = len( rows ) for bssid, current_vendor in rows: candidate_vendor = self._lookup_vendor( vendor_conn, bssid ) current_unknown = ( current_vendor is None or current_vendor == "" or current_vendor == "Unknown" ) candidate_unknown = ( candidate_vendor is None or candidate_vendor == "" ) if ( current_unknown and candidate_unknown ): result["unchanged"] += 1 continue if ( not current_unknown and not candidate_unknown and current_vendor == candidate_vendor ): result["unchanged"] += 1 continue if ( current_unknown and not candidate_unknown ): result[ "unknown_to_vendor" ] += 1 elif ( not current_unknown and not candidate_unknown ): result[ "vendor_to_vendor" ] += 1 elif ( not current_unknown and candidate_unknown ): result[ "vendor_to_unknown" ] += 1 finally: vendor_conn.close() wifi_conn.close() return result @staticmethod def _lookup_vendor( conn, bssid ): if not bssid: return None normalized = ( str(bssid) .replace(":", "") .replace("-", "") .replace(".", "") .strip() .upper() ) for length in ( 9, 7, 6 ): prefix = normalized[ :length ] if len(prefix) != length: continue row = conn.execute( """ SELECT vendor FROM oui WHERE prefix=? """, ( prefix, ) ).fetchone() if row is not None: return row[0] return None @staticmethod def _validate_wifi_schema(conn): tables = { row[0] for row in conn.execute( """ SELECT name FROM sqlite_master WHERE type='table' """ ).fetchall() } if "access_points" not in tables: raise ValueError( "access_points table is missing" ) columns = { row[1] for row in conn.execute( "PRAGMA table_info(access_points)" ).fetchall() } required = { "bssid", "vendor" } if not required.issubset( columns ): raise ValueError( "Invalid WiFi database schema" ) # ============================================================ # Apply # ============================================================ def apply( self, preview, unknown_to_vendor=True, vendor_to_vendor=False, vendor_to_unknown=False ): candidate = Path( preview["candidate"] ) if not candidate.exists(): raise ValueError( "Vendor preview candidate does not exist" ) if not self._is_preview_candidate( preview ): raise ValueError( "Invalid vendor preview" ) oui_changed = not preview[ "oui" ][ "identical" ] selected = { "unknown_to_vendor": bool( unknown_to_vendor ), "vendor_to_vendor": bool( vendor_to_vendor ), "vendor_to_unknown": bool( vendor_to_unknown ) } selected_count = 0 if selected[ "unknown_to_vendor" ]: selected_count += preview[ "ap" ][ "unknown_to_vendor" ] if selected[ "vendor_to_vendor" ]: selected_count += preview[ "ap" ][ "vendor_to_vendor" ] if selected[ "vendor_to_unknown" ]: selected_count += preview[ "ap" ][ "vendor_to_unknown" ] result = { "oui_updated": False, "wifi_updated": 0, "selected": selected, "selected_count": selected_count } try: if oui_changed: self._backup_database( self.vendor_database, "oui" ) self._atomic_replace( candidate, self.vendor_database ) result[ "oui_updated" ] = True if selected_count: result[ "wifi_updated" ] = self._apply_wifi_changes( preview, selected ) return result finally: self._cleanup_preview( preview ) # ============================================================ # WiFi vendor synchronization # ============================================================ def _apply_wifi_changes( self, preview, selected ): candidate = Path( preview["candidate"] ) vendor_conn = sqlite3.connect( candidate ) wifi_conn = None try: wifi_conn = sqlite3.connect( self.wifi_database, timeout=30 ) self._validate_wifi_schema( wifi_conn ) changes = [] rows = wifi_conn.execute( """ SELECT id, bssid, vendor FROM access_points """ ).fetchall() for ap_id, bssid, current_vendor in rows: new_vendor = self._lookup_vendor( vendor_conn, bssid ) current_unknown = ( current_vendor is None or current_vendor == "" or current_vendor == "Unknown" ) new_unknown = ( new_vendor is None or new_vendor == "" ) category = None if ( current_unknown and not new_unknown ): category = ( "unknown_to_vendor" ) elif ( not current_unknown and not new_unknown and current_vendor != new_vendor ): category = ( "vendor_to_vendor" ) elif ( not current_unknown and new_unknown ): category = ( "vendor_to_unknown" ) if ( category is not None and selected[category] ): changes.append( ( new_vendor, ap_id ) ) if not changes: return 0 self._backup_database( self.wifi_database, "wifi" ) wifi_conn.execute( "BEGIN" ) for vendor, ap_id in changes: wifi_conn.execute( """ UPDATE access_points SET vendor=? WHERE id=? """, ( vendor, ap_id ) ) wifi_conn.commit() return len( changes ) except Exception: if wifi_conn is not None: wifi_conn.rollback() raise finally: vendor_conn.close() if wifi_conn is not None: wifi_conn.close() # ============================================================ # Backup # ============================================================ def _backup_database( self, database, database_type ): database = Path( database ) if not database.exists(): return None self.backup_directory.mkdir( parents=True, exist_ok=True ) prefix = ( f"{database_type}-" ) backups = sorted( self.backup_directory.glob( f"{prefix}*.sqlite*" ), key=lambda path: path.stat().st_mtime, reverse=True ) while len( backups ) >= self.BACKUP_LIMIT: oldest = backups.pop() try: oldest.unlink() except OSError: pass backup = ( self.backup_directory / ( f"{prefix}" f"{datetime.now(timezone.utc).strftime('%Y%m%d-%H%M%S')}" f".sqlite3" ) ) shutil.copy2( database, backup ) return backup # ============================================================ # Atomic replacement # ============================================================ @staticmethod def _atomic_replace( source, destination ): source = Path( source ) destination = Path( destination ) destination.parent.mkdir( parents=True, exist_ok=True ) temporary = destination.with_name( f".{destination.name}.tmp" ) try: shutil.copy2( source, temporary ) os.replace( temporary, destination ) finally: if temporary.exists(): try: temporary.unlink() except OSError: pass # ============================================================ # Preview validation / cleanup # ============================================================ def _is_preview_candidate( self, preview ): candidate = Path( preview.get( "candidate", "" ) ) directory = Path( preview.get( "directory", "" ) ) try: candidate.relative_to( directory ) except ValueError: return False return ( candidate.exists() and directory.exists() ) @staticmethod def _cleanup_preview( preview ): directory = Path( preview.get( "directory", "" ) ) if directory.exists(): shutil.rmtree( directory, ignore_errors=True ) # ============================================================ # Metadata # ============================================================ @staticmethod def _get_metadata( conn, key ): try: row = conn.execute( """ SELECT value FROM metadata WHERE key=? """, ( key, ) ).fetchone() except sqlite3.Error: return None if row is None: return None return row[0]