Merge pull request #1011 from lainedfles/db_wal

Introduce Sqlite WAL journal mode

ui/opensnitch/config.py

@@ -105,11 +105,12 @@ class Config:
     DEFAULT_SERVER_ADDR  = "global/server_address"
     DEFAULT_SERVER_MAX_MESSAGE_LENGTH  = "global/server_max_message_length"
     DEFAULT_HIDE_SYSTRAY_WARN  = "global/hide_systray_warning"
-    DEFAULT_DB_TYPE_KEY  = "database/type"
-    DEFAULT_DB_FILE_KEY  = "database/file"
-    DEFAULT_DB_PURGE_OLDEST  = "database/purge_oldest"
-    DEFAULT_DB_MAX_DAYS  = "database/max_days"
-    DEFAULT_DB_PURGE_INTERVAL  = "database/purge_interval"
+    DEFAULT_DB_TYPE_KEY       = "database/type"
+    DEFAULT_DB_FILE_KEY       = "database/file"
+    DEFAULT_DB_PURGE_OLDEST   = "database/purge_oldest"
+    DEFAULT_DB_MAX_DAYS       = "database/max_days"
+    DEFAULT_DB_PURGE_INTERVAL = "database/purge_interval"
+    DEFAULT_DB_JRNL_WAL       = "database/jrnl_wal"
 
     DEFAULT_TIMEOUT = 30
 
@@ -169,6 +170,7 @@ def __init__(self):
         if self.settings.value(self.DEFAULT_DB_TYPE_KEY) == None:
             self.setSettings(self.DEFAULT_DB_TYPE_KEY, Database.DB_TYPE_MEMORY)
             self.setSettings(self.DEFAULT_DB_FILE_KEY, Database.DB_IN_MEMORY)
+            self.setSettings(self.DEFAULT_DB_JRNL_WAL, Database.DB_JRNL_WAL)
 
         self.setRulesDurationFilter(
             self.getBool(self.DEFAULT_IGNORE_RULES),

ui/opensnitch/database/__init__.py

@@ -10,6 +10,17 @@ class Database:
     DB_IN_MEMORY   = ":memory:"
     DB_TYPE_MEMORY = 0
     DB_TYPE_FILE   = 1
+    DB_JRNL_WAL    = False
+
+    # Sqlite3 journal modes
+    DB_JOURNAL_MODE_LIST = {
+            0: "DELETE",
+            1: "TRUNCATE",
+            2: "PERSIST",
+            3: "MEMORY",
+            4: "WAL",
+            5: "OFF",
+            }
 
     # increase accordingly whenever the schema is updated
     DB_VERSION = 3
@@ -24,11 +35,16 @@ def __init__(self, dbname="db"):
         self._lock = threading.RLock()
         self.db = None
         self.db_file = Database.DB_IN_MEMORY
+        self.db_jrnl_wal = Database.DB_JRNL_WAL
         self.db_name = dbname
 
-    def initialize(self, dbtype=DB_TYPE_MEMORY, dbfile=DB_IN_MEMORY, db_name="db"):
+    def initialize(self, dbtype=DB_TYPE_MEMORY, dbfile=DB_IN_MEMORY, dbjrnl_wal=DB_JRNL_WAL, db_name="db"):
         if dbtype != Database.DB_TYPE_MEMORY:
             self.db_file = dbfile
+            self.db_jrnl_wal = dbjrnl_wal
+        else:
+            # Always disable under pure memory mode
+            self.db_jrnl_wal = False
 
         is_new_file = not os.path.isfile(self.db_file)
 
@@ -87,19 +103,16 @@ def get_db_name(self):
         return self.db_name
 
     def _create_tables(self):
-        # https://www.sqlite.org/wal.html
         if self.db_file == Database.DB_IN_MEMORY:
             self.set_schema_version(self.DB_VERSION)
-            q = QSqlQuery("PRAGMA journal_mode = OFF", self.db)
-            q.exec_()
-            q = QSqlQuery("PRAGMA synchronous = OFF", self.db)
-            q.exec_()
-            q = QSqlQuery("PRAGMA cache_size=10000", self.db)
-            q.exec_()
+            # Disable journal (default)
+            self.set_journal_mode(5)
+        elif self.db_jrnl_wal is True:
+            # Set WAL mode (file+memory)
+            self.set_journal_mode(4)
         else:
-            q = QSqlQuery("PRAGMA synchronous = NORMAL", self.db)
-            q.exec_()
-
+            # Set DELETE mode (file)
+            self.set_journal_mode(0)
         q = QSqlQuery("create table if not exists connections (" \
                 "time text, " \
                 "node text, " \
@@ -200,6 +213,41 @@ def set_schema_version(self, version):
         if q.exec_() == False:
             print("Error updating updating schema version:", q.lastError().text())
 
+    def get_journal_mode(self):
+        q = QSqlQuery("PRAGMA journal_mode;", self.db)
+        q.exec_()
+        if q.next():
+            return str(q.value(0))
+
+        return str("unknown")
+
+    def set_journal_mode(self, mode):
+        # https://www.sqlite.org/wal.html
+        mode_str = Database.DB_JOURNAL_MODE_LIST[mode]
+        if self.get_journal_mode().lower() != mode_str.lower():
+            print("Setting journal_mode: ", mode_str)
+            q = QSqlQuery("PRAGMA journal_mode = {modestr};".format(modestr = mode_str), self.db)
+            if q.exec_() == False:
+                print("Error updating PRAGMA journal_mode:", q.lastError().text())
+                return False
+        if mode == 3 or mode == 5:
+            print("Setting DB memory optimizations")
+            q = QSqlQuery("PRAGMA synchronous = OFF;", self.db)
+            if q.exec_() == False:
+                print("Error updating PRAGMA synchronous:", q.lastError().text())
+                return False
+            q = QSqlQuery("PRAGMA cache_size=10000;", self.db)
+            if q.exec_() == False:
+                print("Error updating PRAGMA cache_size:", q.lastError().text())
+                return False
+        else:
+            print("Setting synchronous = NORMAL")
+            q = QSqlQuery("PRAGMA synchronous = NORMAL;", self.db)
+            if q.exec_() == False:
+                print("Error updating PRAGMA synchronous:", q.lastError().text())
+
+        return True
+
     def _upgrade_db_schema(self):
         migrations_path = os.path.dirname(os.path.realpath(__file__)) + "/migrations"
         schema_version = self.get_schema_version()

ui/opensnitch/dialogs/preferences.py

@@ -203,6 +203,7 @@ def showEvent(self, event):
 
         self.comboDBType.currentIndexChanged.connect(self._cb_db_type_changed)
         self.checkDBMaxDays.toggled.connect(self._cb_db_max_days_toggled)
+        self.checkDBJrnlWal.toggled.connect(self._cb_db_jrnl_wal_toggled)
 
         # True when any node option changes
         self._node_needs_update = False
@@ -323,8 +324,10 @@ def _load_settings(self):
             self.dbLabel.setVisible(True)
             self.dbLabel.setText(self._cfg.getSettings(self._cfg.DEFAULT_DB_FILE_KEY))
         dbMaxDays = self._cfg.getInt(self._cfg.DEFAULT_DB_MAX_DAYS, 1)
+        dbJrnlWal = self._cfg.getBool(self._cfg.DEFAULT_DB_JRNL_WAL)
         dbPurgeInterval = self._cfg.getInt(self._cfg.DEFAULT_DB_PURGE_INTERVAL, 5)
         self._enable_db_cleaner_options(self._cfg.getBool(Config.DEFAULT_DB_PURGE_OLDEST), dbMaxDays)
+        self._enable_db_jrnl_wal(self._cfg.getBool(Config.DEFAULT_DB_PURGE_OLDEST), dbJrnlWal)
         self.spinDBMaxDays.setValue(dbMaxDays)
         self.spinDBPurgeInterval.setValue(dbPurgeInterval)
 
@@ -528,6 +531,7 @@ def _save_db_config(self):
         self._cfg.setSettings(Config.DEFAULT_DB_PURGE_OLDEST, bool(self.checkDBMaxDays.isChecked()))
         self._cfg.setSettings(Config.DEFAULT_DB_MAX_DAYS, int(self.spinDBMaxDays.value()))
         self._cfg.setSettings(Config.DEFAULT_DB_PURGE_INTERVAL, int(self.spinDBPurgeInterval.value()))
+        self._cfg.setSettings(Config.DEFAULT_DB_JRNL_WAL, bool(self.checkDBJrnlWal.isChecked()))
         self.dbType = self.comboDBType.currentIndex()
 
         return True
@@ -781,6 +785,10 @@ def _enable_db_cleaner_options(self, enable, db_max_days):
         self.cmdDBPurgesUp.setEnabled(enable)
         self.cmdDBPurgesDown.setEnabled(enable)
 
+    def _enable_db_jrnl_wal(self, enable, db_jrnl_wal):
+        self.checkDBJrnlWal.setChecked(db_jrnl_wal)
+        self.checkDBJrnlWal.setEnabled(enable)
+
     @QtCore.pyqtSlot(ui_pb2.NotificationReply)
     def _cb_notification_callback(self, reply):
         #print(self.LOG_TAG, "Config notification received: ", reply.id, reply.code)
@@ -819,6 +827,8 @@ def _cb_db_type_changed(self):
         self.dbLabel.setVisible(not isDBMem)
         self.checkDBMaxDays.setEnabled(not isDBMem)
         self.checkDBMaxDays.setChecked(not isDBMem)
+        self.checkDBJrnlWal.setEnabled(not isDBMem)
+        self.checkDBJrnlWal.setChecked(False)
 
     def _cb_accept_button_clicked(self):
         self.accept()
@@ -884,6 +894,9 @@ def _cb_combo_node_auth_type_changed(self, index):
     def _cb_db_max_days_toggled(self, state):
         self._enable_db_cleaner_options(state, 1)
 
+    def _cb_db_jrnl_wal_toggled(self, state):
+        self._changes_needs_restart = QC.translate("preferences", "DB journal_mode changed")
+
     def _cb_cmd_spin_clicked(self, spinWidget, operation):
         if operation == self.SUM:
             spinWidget.setValue(spinWidget.value() + 1)

ui/opensnitch/res/preferences.ui

@@ -1916,6 +1916,16 @@ Temporary rules will still be valid, and you can use them when prompted to allow
            </property>
           </widget>
          </item>
+         <item row="2" column="0">
+          <widget class="QCheckBox" name="checkDBJrnlWal">
+           <property name="enabled">
+            <bool>false</bool>
+           </property>
+           <property name="text">
+            <string>Enable DB Write-Ahead Logging (WAL)</string>
+           </property>
+          </widget>
+         </item>
         </layout>
        </item>
       </layout>

ui/opensnitch/service.py

@@ -61,9 +61,11 @@ def __init__(self, app, on_exit, start_in_bg=False):
         self._cfg = Config.init()
         self._db = Database.instance()
         db_file=self._cfg.getSettings(self._cfg.DEFAULT_DB_FILE_KEY)
+        db_jrnl_wal=self._cfg.getBool(Config.DEFAULT_DB_JRNL_WAL)
         db_status, db_error = self._db.initialize(
             dbtype=self._cfg.getInt(self._cfg.DEFAULT_DB_TYPE_KEY),
-            dbfile=db_file
+            dbfile=db_file,
+            dbjrnl_wal=db_jrnl_wal
         )
         if db_status is False:
             Message.ok(

ui/opensnitch/utils/__init__.py

@@ -235,7 +235,8 @@ def __init__(self, _interval, _callback):
         self.db = Database("db-cleaner-connection")
         self.db_status, db_error = self.db.initialize(
             dbtype=self._cfg.getInt(self._cfg.DEFAULT_DB_TYPE_KEY),
-            dbfile=self._cfg.getSettings(self._cfg.DEFAULT_DB_FILE_KEY)
+            dbfile=self._cfg.getSettings(self._cfg.DEFAULT_DB_FILE_KEY),
+            dbjrnl_wal=self._cfg.getBool(self._cfg.DEFAULT_DB_JRNL_WAL)
         )
 
     def run(self):