Merge branch 'feature/sqlite-db-v1' into develop

Author: stenya

.github/workflows/go.yml

@@ -38,9 +38,9 @@ jobs:
         cache: false
 
     - name: Run golangci-lint
-      uses: golangci/golangci-lint-action@v4
+      uses: golangci/golangci-lint-action@v6
       with:
-        version: v1.60.3
+        version: v1.64.6
         only-new-issues: true
         args: -c ./.golangci.yml --timeout 15m
 

.golangci.yml

@@ -8,20 +8,15 @@ linters:
     - contextcheck
     - cyclop
     - depguard
-    - exhaustivestruct
     - exhaustruct
     - forbidigo
     - funlen
     - gochecknoglobals
     - gochecknoinits
     - gocognit
     - gocyclo
-    - goerr113 
-    - gomnd
     - gomoddirectives
-    - ifshort
     - interfacebloat
-    - interfacer
     - ireturn
     - lll
     - mnd
@@ -32,7 +27,6 @@ linters:
     - noctx
     - nolintlint
     - nonamedreturns
-    - nosnakecase
     - perfsprint # TODO(ppacher): we should re-enanble this one to avoid costly fmt.* calls in the hot-path
     - revive
     - tagliatelle
@@ -42,15 +36,14 @@ linters:
     - whitespace
     - wrapcheck
     - wsl
+    - gci
+    - tenv # Deprecated
 
 linters-settings:
   revive:
     # See https://github.com/mgechev/revive#available-rules for details.
     enable-all-rules: true
-  gci:
-    # put imports beginning with prefix after 3rd-party packages;
-    # only support one prefix
-    # if not set, use goimports.local-prefixes
+  goimports:
     local-prefixes: github.com/safing
   godox:
     # report any comments starting with keywords, this is useful for TODO or FIXME comments that

Earthfile

@@ -1,9 +1,9 @@
 VERSION --arg-scope-and-set --global-cache 0.8
 
-ARG --global go_version = 1.22
+ARG --global go_version = 1.24
 ARG --global node_version = 18
 ARG --global rust_version = 1.79
-ARG --global golangci_lint_version = 1.57.1
+ARG --global golangci_lint_version = 1.64.6
 
 ARG --global go_builder_image = "golang:${go_version}-alpine"
 ARG --global node_builder_image = "node:${node_version}"

base/config/main.go

@@ -10,6 +10,7 @@ import (
 	"path/filepath"
 	"sort"
 
+	"github.com/safing/portmaster/base/database/dbmodule"
 	"github.com/safing/portmaster/base/dataroot"
 	"github.com/safing/portmaster/base/utils"
 	"github.com/safing/portmaster/base/utils/debug"
@@ -150,6 +151,7 @@ func InitializeUnitTestDataroot(testName string) (string, error) {
 	if err != nil {
 		return "", fmt.Errorf("failed to initialize dataroot: %w", err)
 	}
+	dbmodule.SetDatabaseLocation(dataroot.Root())
 
 	return basePath, nil
 }

base/database/controller.go

@@ -264,6 +264,20 @@ func (c *Controller) Purge(ctx context.Context, q *query.Query, local, internal
 	return 0, ErrNotImplemented
 }
 
+// PurgeOlderThan deletes all records last updated before the given time.
+// It returns the number of successful deletes and an error.
+func (c *Controller) PurgeOlderThan(ctx context.Context, prefix string, purgeBefore time.Time, local, internal bool) (int, error) {
+	if shuttingDown.IsSet() {
+		return 0, ErrShuttingDown
+	}
+
+	if purger, ok := c.storage.(storage.PurgeOlderThan); ok {
+		return purger.PurgeOlderThan(ctx, prefix, purgeBefore, local, internal, c.shadowDelete)
+	}
+
+	return 0, ErrNotImplemented
+}
+
 // Shutdown shuts down the storage.
 func (c *Controller) Shutdown() error {
 	return c.storage.Shutdown()

base/database/dbmodule/db.go

@@ -5,7 +5,6 @@ import (
 	"sync/atomic"
 
 	"github.com/safing/portmaster/base/database"
-	"github.com/safing/portmaster/base/dataroot"
 	"github.com/safing/portmaster/base/utils"
 	"github.com/safing/portmaster/service/mgr"
 )
@@ -37,7 +36,6 @@ func SetDatabaseLocation(dirStructureRoot *utils.DirStructure) {
 }
 
 func prep() error {
-	SetDatabaseLocation(dataroot.Root())
 	if databaseStructureRoot == nil {
 		return errors.New("database location not specified")
 	}

base/database/interface.go

@@ -562,6 +562,27 @@ func (i *Interface) Purge(ctx context.Context, q *query.Query) (int, error) {
 	return db.Purge(ctx, q, i.options.Local, i.options.Internal)
 }
 
+// PurgeOlderThan deletes all records last updated before the given time.
+// It returns the number of successful deletes and an error.
+func (i *Interface) PurgeOlderThan(ctx context.Context, prefix string, purgeBefore time.Time) (int, error) {
+	dbName, dbKeyPrefix := record.ParseKey(prefix)
+	if dbName == "" {
+		return 0, errors.New("unknown database")
+	}
+
+	db, err := getController(dbName)
+	if err != nil {
+		return 0, err
+	}
+
+	// Check if database is read only before we add to the cache.
+	if db.ReadOnly() {
+		return 0, ErrReadOnly
+	}
+
+	return db.PurgeOlderThan(ctx, dbKeyPrefix, purgeBefore, i.options.Local, i.options.Internal)
+}
+
 // Subscribe subscribes to updates matching the given query.
 func (i *Interface) Subscribe(q *query.Query) (*Subscription, error) {
 	_, err := q.Check()

base/database/main.go

@@ -9,9 +9,8 @@ import (
 	"github.com/safing/portmaster/base/utils"
 )
 
-const (
-	databasesSubDir = "databases"
-)
+// DatabasesSubDir defines the sub directory where the databases are stored.
+const DatabasesSubDir = "databases"
 
 var (
 	initialized = abool.NewBool(false)
@@ -34,7 +33,7 @@ func Initialize(dirStructureRoot *utils.DirStructure) error {
 		rootStructure = dirStructureRoot
 
 		// ensure root and databases dirs
-		databasesStructure = rootStructure.ChildDir(databasesSubDir, utils.AdminOnlyPermission)
+		databasesStructure = rootStructure.ChildDir(DatabasesSubDir, utils.AdminOnlyPermission)
 		err := databasesStructure.Ensure()
 		if err != nil {
 			return fmt.Errorf("could not create/open database directory (%s): %w", rootStructure.Path, err)

base/database/query/query.go

@@ -99,6 +99,11 @@ func (q *Query) MatchesKey(dbKey string) bool {
 	return strings.HasPrefix(dbKey, q.dbKeyPrefix)
 }
 
+// HasWhereCondition returns whether the query has a "where" condition set.
+func (q *Query) HasWhereCondition() bool {
+	return q.where != nil
+}
+
 // MatchesRecord checks whether the query matches the supplied database record (value only).
 func (q *Query) MatchesRecord(r record.Record) bool {
 	if q.where == nil {

base/database/record/base.go

@@ -102,7 +102,7 @@ func (b *Base) SetMeta(meta *Meta) {
 	b.meta = meta
 }
 
-// Marshal marshals the object, without the database key or metadata. It returns nil if the record is deleted.
+// Marshal marshals the format and data.
 func (b *Base) Marshal(self Record, format uint8) ([]byte, error) {
 	if b.Meta() == nil {
 		return nil, errors.New("missing meta")
@@ -119,7 +119,20 @@ func (b *Base) Marshal(self Record, format uint8) ([]byte, error) {
 	return dumped, nil
 }
 
-// MarshalRecord packs the object, including metadata, into a byte array for saving in a database.
+// MarshalDataOnly marshals the data only.
+func (b *Base) MarshalDataOnly(self Record, format uint8) ([]byte, error) {
+	if b.Meta() == nil {
+		return nil, errors.New("missing meta")
+	}
+
+	if b.Meta().Deleted > 0 {
+		return nil, nil
+	}
+
+	return dsd.DumpWithoutIdentifier(self, format, "")
+}
+
+// MarshalRecord marshals the data, format and metadata.
 func (b *Base) MarshalRecord(self Record) ([]byte, error) {
 	if b.Meta() == nil {
 		return nil, errors.New("missing meta")

base/database/record/meta.go

@@ -49,11 +49,21 @@ func (m *Meta) MakeCrownJewel() {
 	m.cronjewel = true
 }
 
+// IsCrownJewel returns whether the database record is marked as a crownjewel.
+func (m *Meta) IsCrownJewel() bool {
+	return m.cronjewel
+}
+
 // MakeSecret sets the database record as secret, meaning that it may only be used internally, and not by interfacing processes, such as the UI.
 func (m *Meta) MakeSecret() {
 	m.secret = true
 }
 
+// IsSecret returns whether the database record is marked as a secret.
+func (m *Meta) IsSecret() bool {
+	return m.secret
+}
+
 // Update updates the internal meta states and should be called before writing the record to the database.
 func (m *Meta) Update() {
 	now := time.Now().Unix()

base/database/record/record.go

@@ -20,6 +20,7 @@ type Record interface {
 
 	// Serialization.
 	Marshal(self Record, format uint8) ([]byte, error)
+	MarshalDataOnly(self Record, format uint8) ([]byte, error)
 	MarshalRecord(self Record) ([]byte, error)
 	GetAccessor(self Record) accessor.Accessor
 

base/database/record/wrapper.go

@@ -79,7 +79,21 @@ func NewWrapper(key string, meta *Meta, format uint8, data []byte) (*Wrapper, er
 	}, nil
 }
 
-// Marshal marshals the object, without the database key or metadata.
+// NewWrapperFromDatabase returns a new record wrapper for the given data.
+func NewWrapperFromDatabase(dbName, dbKey string, meta *Meta, format uint8, data []byte) (*Wrapper, error) {
+	return &Wrapper{
+		Base{
+			dbName: dbName,
+			dbKey:  dbKey,
+			meta:   meta,
+		},
+		sync.Mutex{},
+		format,
+		data,
+	}, nil
+}
+
+// Marshal marshals the format and data.
 func (w *Wrapper) Marshal(r Record, format uint8) ([]byte, error) {
 	if w.Meta() == nil {
 		return nil, errors.New("missing meta")
@@ -100,7 +114,24 @@ func (w *Wrapper) Marshal(r Record, format uint8) ([]byte, error) {
 	return data, nil
 }
 
-// MarshalRecord packs the object, including metadata, into a byte array for saving in a database.
+// MarshalDataOnly marshals the data only.
+func (w *Wrapper) MarshalDataOnly(self Record, format uint8) ([]byte, error) {
+	if w.Meta() == nil {
+		return nil, errors.New("missing meta")
+	}
+
+	if w.Meta().Deleted > 0 {
+		return nil, nil
+	}
+
+	if format != dsd.AUTO && format != w.Format {
+		return nil, errors.New("could not dump model, wrapped object format mismatch")
+	}
+
+	return w.Data, nil
+}
+
+// MarshalRecord marshals the data, format and metadata.
 func (w *Wrapper) MarshalRecord(r Record) ([]byte, error) {
 	// Duplication necessary, as the version from Base would call Base.Marshal instead of Wrapper.Marshal
 

base/database/storage/interface.go

@@ -46,3 +46,8 @@ type Batcher interface {
 type Purger interface {
 	Purge(ctx context.Context, q *query.Query, local, internal, shadowDelete bool) (int, error)
 }
+
+// PurgeOlderThan defines the database storage API for backends that support the PurgeOlderThan operation.
+type PurgeOlderThan interface {
+	PurgeOlderThan(ctx context.Context, prefix string, purgeBefore time.Time, local, internal, shadowDelete bool) (int, error)
+}

base/database/storage/sqlite/bobgen.yaml

@@ -0,0 +1,6 @@
+sqlite:
+  dsn: "testdata/schema.db"
+  except:
+    migrations:
+
+no_factory: true

base/database/storage/sqlite/migrations/0_settings.sql

@@ -0,0 +1,7 @@
+-- +migrate Up
+-- SQL in section 'Up' is executed when this migration is applied
+PRAGMA auto_vacuum = INCREMENTAL; -- https://sqlite.org/pragma.html#pragma_auto_vacuum
+
+-- +migrate Down
+-- SQL section 'Down' is executed when this migration is rolled back
+PRAGMA auto_vacuum = NONE; -- https://sqlite.org/pragma.html#pragma_auto_vacuum

base/database/storage/sqlite/migrations/1_initial.sql

@@ -0,0 +1,19 @@
+-- +migrate Up
+-- SQL in section 'Up' is executed when this migration is applied
+CREATE TABLE records (
+    key TEXT PRIMARY KEY,
+
+    format SMALLINT,
+    value  BLOB,
+
+    created    BIGINT NOT NULL,
+    modified   BIGINT NOT NULL,
+    expires    BIGINT DEFAULT 0 NOT NULL,
+    deleted    BIGINT DEFAULT 0 NOT NULL,
+    secret     BOOLEAN DEFAULT FALSE NOT NULL,
+    crownjewel BOOLEAN DEFAULT FALSE NOT NULL
+);
+
+-- +migrate Down
+-- SQL section 'Down' is executed when this migration is rolled back
+DROP TABLE records;

base/database/storage/sqlite/migrations_config.yml

@@ -0,0 +1,5 @@
+development:
+  dialect: sqlite3
+  datasource: testdata/schema.db
+  dir: migrations
+  table: migrations