From feee8919b74dec2f5dd8398bd9eb24eff805ebfb Mon Sep 17 00:00:00 2001 From: selfdenial Date: Tue, 8 Aug 2023 22:37:24 -0600 Subject: [PATCH] Introduce Sqlite3 WAL mode --- ui/opensnitch/config.py | 12 +++-- ui/opensnitch/database/__init__.py | 70 +++++++++++++++++++++++----- ui/opensnitch/dialogs/preferences.py | 13 ++++++ ui/opensnitch/res/preferences.ui | 10 ++++ ui/opensnitch/service.py | 4 +- ui/opensnitch/utils/__init__.py | 3 +- 6 files changed, 94 insertions(+), 18 deletions(-) diff --git a/ui/opensnitch/config.py b/ui/opensnitch/config.py index c09110c487..168c680b8b 100644 --- a/ui/opensnitch/config.py +++ b/ui/opensnitch/config.py @@ -105,11 +105,12 @@ class Config: DEFAULT_SERVER_ADDR = "global/server_address" DEFAULT_SERVER_MAX_MESSAGE_LENGTH = "global/server_max_message_length" DEFAULT_HIDE_SYSTRAY_WARN = "global/hide_systray_warning" - DEFAULT_DB_TYPE_KEY = "database/type" - DEFAULT_DB_FILE_KEY = "database/file" - DEFAULT_DB_PURGE_OLDEST = "database/purge_oldest" - DEFAULT_DB_MAX_DAYS = "database/max_days" - DEFAULT_DB_PURGE_INTERVAL = "database/purge_interval" + DEFAULT_DB_TYPE_KEY = "database/type" + DEFAULT_DB_FILE_KEY = "database/file" + DEFAULT_DB_PURGE_OLDEST = "database/purge_oldest" + DEFAULT_DB_MAX_DAYS = "database/max_days" + DEFAULT_DB_PURGE_INTERVAL = "database/purge_interval" + DEFAULT_DB_JRNL_WAL = "database/jrnl_wal" DEFAULT_TIMEOUT = 30 @@ -169,6 +170,7 @@ def __init__(self): if self.settings.value(self.DEFAULT_DB_TYPE_KEY) == None: self.setSettings(self.DEFAULT_DB_TYPE_KEY, Database.DB_TYPE_MEMORY) self.setSettings(self.DEFAULT_DB_FILE_KEY, Database.DB_IN_MEMORY) + self.setSettings(self.DEFAULT_DB_JRNL_WAL, Database.DB_JRNL_WAL) self.setRulesDurationFilter( self.getBool(self.DEFAULT_IGNORE_RULES), diff --git a/ui/opensnitch/database/__init__.py b/ui/opensnitch/database/__init__.py index 3243b7ff2a..9e9446ea82 100644 --- a/ui/opensnitch/database/__init__.py +++ b/ui/opensnitch/database/__init__.py @@ -10,6 +10,17 @@ class Database: DB_IN_MEMORY = ":memory:" DB_TYPE_MEMORY = 0 DB_TYPE_FILE = 1 + DB_JRNL_WAL = False + + # Sqlite3 journal modes + DB_JOURNAL_MODE_LIST = { + 0: "DELETE", + 1: "TRUNCATE", + 2: "PERSIST", + 3: "MEMORY", + 4: "WAL", + 5: "OFF", + } # increase accordingly whenever the schema is updated DB_VERSION = 3 @@ -24,11 +35,16 @@ def __init__(self, dbname="db"): self._lock = threading.RLock() self.db = None self.db_file = Database.DB_IN_MEMORY + self.db_jrnl_wal = Database.DB_JRNL_WAL self.db_name = dbname - def initialize(self, dbtype=DB_TYPE_MEMORY, dbfile=DB_IN_MEMORY, db_name="db"): + def initialize(self, dbtype=DB_TYPE_MEMORY, dbfile=DB_IN_MEMORY, dbjrnl_wal=DB_JRNL_WAL, db_name="db"): if dbtype != Database.DB_TYPE_MEMORY: self.db_file = dbfile + self.db_jrnl_wal = dbjrnl_wal + else: + # Always disable under pure memory mode + self.db_jrnl_wal = False is_new_file = not os.path.isfile(self.db_file) @@ -87,19 +103,16 @@ def get_db_name(self): return self.db_name def _create_tables(self): - # https://www.sqlite.org/wal.html if self.db_file == Database.DB_IN_MEMORY: self.set_schema_version(self.DB_VERSION) - q = QSqlQuery("PRAGMA journal_mode = OFF", self.db) - q.exec_() - q = QSqlQuery("PRAGMA synchronous = OFF", self.db) - q.exec_() - q = QSqlQuery("PRAGMA cache_size=10000", self.db) - q.exec_() + # Disable journal (default) + self.set_journal_mode(5) + elif self.db_jrnl_wal is True: + # Set WAL mode (file+memory) + self.set_journal_mode(4) else: - q = QSqlQuery("PRAGMA synchronous = NORMAL", self.db) - q.exec_() - + # Set DELETE mode (file) + self.set_journal_mode(0) q = QSqlQuery("create table if not exists connections (" \ "time text, " \ "node text, " \ @@ -200,6 +213,41 @@ def set_schema_version(self, version): if q.exec_() == False: print("Error updating updating schema version:", q.lastError().text()) + def get_journal_mode(self): + q = QSqlQuery("PRAGMA journal_mode;", self.db) + q.exec_() + if q.next(): + return str(q.value(0)) + + return str("unknown") + + def set_journal_mode(self, mode): + # https://www.sqlite.org/wal.html + mode_str = Database.DB_JOURNAL_MODE_LIST[mode] + if self.get_journal_mode().lower() != mode_str.lower(): + print("Setting journal_mode: ", mode_str) + q = QSqlQuery("PRAGMA journal_mode = {modestr};".format(modestr = mode_str), self.db) + if q.exec_() == False: + print("Error updating PRAGMA journal_mode:", q.lastError().text()) + return False + if mode == 3 or mode == 5: + print("Setting DB memory optimizations") + q = QSqlQuery("PRAGMA synchronous = OFF;", self.db) + if q.exec_() == False: + print("Error updating PRAGMA synchronous:", q.lastError().text()) + return False + q = QSqlQuery("PRAGMA cache_size=10000;", self.db) + if q.exec_() == False: + print("Error updating PRAGMA cache_size:", q.lastError().text()) + return False + else: + print("Setting synchronous = NORMAL") + q = QSqlQuery("PRAGMA synchronous = NORMAL;", self.db) + if q.exec_() == False: + print("Error updating PRAGMA synchronous:", q.lastError().text()) + + return True + def _upgrade_db_schema(self): migrations_path = os.path.dirname(os.path.realpath(__file__)) + "/migrations" schema_version = self.get_schema_version() diff --git a/ui/opensnitch/dialogs/preferences.py b/ui/opensnitch/dialogs/preferences.py index 7bea24aa15..b07c0d51de 100644 --- a/ui/opensnitch/dialogs/preferences.py +++ b/ui/opensnitch/dialogs/preferences.py @@ -203,6 +203,7 @@ def showEvent(self, event): self.comboDBType.currentIndexChanged.connect(self._cb_db_type_changed) self.checkDBMaxDays.toggled.connect(self._cb_db_max_days_toggled) + self.checkDBJrnlWal.toggled.connect(self._cb_db_jrnl_wal_toggled) # True when any node option changes self._node_needs_update = False @@ -323,8 +324,10 @@ def _load_settings(self): self.dbLabel.setVisible(True) self.dbLabel.setText(self._cfg.getSettings(self._cfg.DEFAULT_DB_FILE_KEY)) dbMaxDays = self._cfg.getInt(self._cfg.DEFAULT_DB_MAX_DAYS, 1) + dbJrnlWal = self._cfg.getBool(self._cfg.DEFAULT_DB_JRNL_WAL) dbPurgeInterval = self._cfg.getInt(self._cfg.DEFAULT_DB_PURGE_INTERVAL, 5) self._enable_db_cleaner_options(self._cfg.getBool(Config.DEFAULT_DB_PURGE_OLDEST), dbMaxDays) + self._enable_db_jrnl_wal(self._cfg.getBool(Config.DEFAULT_DB_PURGE_OLDEST), dbJrnlWal) self.spinDBMaxDays.setValue(dbMaxDays) self.spinDBPurgeInterval.setValue(dbPurgeInterval) @@ -528,6 +531,7 @@ def _save_db_config(self): self._cfg.setSettings(Config.DEFAULT_DB_PURGE_OLDEST, bool(self.checkDBMaxDays.isChecked())) self._cfg.setSettings(Config.DEFAULT_DB_MAX_DAYS, int(self.spinDBMaxDays.value())) self._cfg.setSettings(Config.DEFAULT_DB_PURGE_INTERVAL, int(self.spinDBPurgeInterval.value())) + self._cfg.setSettings(Config.DEFAULT_DB_JRNL_WAL, bool(self.checkDBJrnlWal.isChecked())) self.dbType = self.comboDBType.currentIndex() return True @@ -781,6 +785,10 @@ def _enable_db_cleaner_options(self, enable, db_max_days): self.cmdDBPurgesUp.setEnabled(enable) self.cmdDBPurgesDown.setEnabled(enable) + def _enable_db_jrnl_wal(self, enable, db_jrnl_wal): + self.checkDBJrnlWal.setChecked(db_jrnl_wal) + self.checkDBJrnlWal.setEnabled(enable) + @QtCore.pyqtSlot(ui_pb2.NotificationReply) def _cb_notification_callback(self, reply): #print(self.LOG_TAG, "Config notification received: ", reply.id, reply.code) @@ -819,6 +827,8 @@ def _cb_db_type_changed(self): self.dbLabel.setVisible(not isDBMem) self.checkDBMaxDays.setEnabled(not isDBMem) self.checkDBMaxDays.setChecked(not isDBMem) + self.checkDBJrnlWal.setEnabled(not isDBMem) + self.checkDBJrnlWal.setChecked(False) def _cb_accept_button_clicked(self): self.accept() @@ -884,6 +894,9 @@ def _cb_combo_node_auth_type_changed(self, index): def _cb_db_max_days_toggled(self, state): self._enable_db_cleaner_options(state, 1) + def _cb_db_jrnl_wal_toggled(self, state): + self._changes_needs_restart = QC.translate("preferences", "DB journal_mode changed") + def _cb_cmd_spin_clicked(self, spinWidget, operation): if operation == self.SUM: spinWidget.setValue(spinWidget.value() + 1) diff --git a/ui/opensnitch/res/preferences.ui b/ui/opensnitch/res/preferences.ui index e6c56a5018..a2577e8ce7 100644 --- a/ui/opensnitch/res/preferences.ui +++ b/ui/opensnitch/res/preferences.ui @@ -1916,6 +1916,16 @@ Temporary rules will still be valid, and you can use them when prompted to allow + + + + false + + + Enable DB Write-Ahead Logging (WAL) + + + diff --git a/ui/opensnitch/service.py b/ui/opensnitch/service.py index 108ea9bac0..48687ad845 100644 --- a/ui/opensnitch/service.py +++ b/ui/opensnitch/service.py @@ -61,9 +61,11 @@ def __init__(self, app, on_exit, start_in_bg=False): self._cfg = Config.init() self._db = Database.instance() db_file=self._cfg.getSettings(self._cfg.DEFAULT_DB_FILE_KEY) + db_jrnl_wal=self._cfg.getBool(Config.DEFAULT_DB_JRNL_WAL) db_status, db_error = self._db.initialize( dbtype=self._cfg.getInt(self._cfg.DEFAULT_DB_TYPE_KEY), - dbfile=db_file + dbfile=db_file, + dbjrnl_wal=db_jrnl_wal ) if db_status is False: Message.ok( diff --git a/ui/opensnitch/utils/__init__.py b/ui/opensnitch/utils/__init__.py index 1c56a03938..1bd00bfb40 100644 --- a/ui/opensnitch/utils/__init__.py +++ b/ui/opensnitch/utils/__init__.py @@ -235,7 +235,8 @@ def __init__(self, _interval, _callback): self.db = Database("db-cleaner-connection") self.db_status, db_error = self.db.initialize( dbtype=self._cfg.getInt(self._cfg.DEFAULT_DB_TYPE_KEY), - dbfile=self._cfg.getSettings(self._cfg.DEFAULT_DB_FILE_KEY) + dbfile=self._cfg.getSettings(self._cfg.DEFAULT_DB_FILE_KEY), + dbjrnl_wal=self._cfg.getBool(self._cfg.DEFAULT_DB_JRNL_WAL) ) def run(self):