[WIP] New updater first working prototype

service/intel/filterlists/database.go

@@ -14,15 +14,14 @@ import (
 	"github.com/safing/portmaster/base/database"
 	"github.com/safing/portmaster/base/database/record"
 	"github.com/safing/portmaster/base/log"
-	"github.com/safing/portmaster/base/updater"
-	"github.com/safing/portmaster/service/updates"
+	"github.com/safing/portmaster/service/updates/registry"
 )
 
 const (
-	baseListFilePath         = "intel/lists/base.dsdl"
-	intermediateListFilePath = "intel/lists/intermediate.dsdl"
-	urgentListFilePath       = "intel/lists/urgent.dsdl"
-	listIndexFilePath        = "intel/lists/index.dsd"
+	baseListFilePath         = "base.dsdl"
+	intermediateListFilePath = "intermediate.dsdl"
+	urgentListFilePath       = "urgent.dsdl"
+	listIndexFilePath        = "index.dsd"
 )
 
 // default bloomfilter element sizes (estimated).
@@ -40,9 +39,9 @@ var (
 	filterListLock sync.RWMutex
 
 	// Updater files for tracking upgrades.
-	baseFile         *updater.File
-	intermediateFile *updater.File
-	urgentFile       *updater.File
+	baseFile         *registry.File
+	intermediateFile *registry.File
+	urgentFile       *registry.File
 
 	filterListsLoaded chan struct{}
 )
@@ -56,11 +55,10 @@ var cache = database.NewInterface(&database.Options{
 // getFileFunc is the function used to get a file from
 // the updater. It's basically updates.GetFile and used
 // for unit testing.
-type getFileFunc func(string) (*updater.File, error)
 
 // getFile points to updates.GetFile but may be set to
 // something different during unit testing.
-var getFile getFileFunc = updates.GetFile
+// var getFile getFileFunc = registry.GetFile
 
 func init() {
 	filterListsLoaded = make(chan struct{})
@@ -79,7 +77,7 @@ func isLoaded() bool {
 
 // processListFile opens the latest version of file and decodes it's DSDL
 // content. It calls processEntry for each decoded filterlists entry.
-func processListFile(ctx context.Context, filter *scopedBloom, file *updater.File) error {
+func processListFile(ctx context.Context, filter *scopedBloom, file *registry.File) error {
 	f, err := os.Open(file.Path())
 	if err != nil {
 		return err

service/intel/filterlists/index.go

@@ -4,14 +4,12 @@ import (
 	"errors"
 	"fmt"
 	"os"
-	"strings"
 	"sync"
 
 	"github.com/safing/portmaster/base/database"
 	"github.com/safing/portmaster/base/database/record"
 	"github.com/safing/portmaster/base/log"
-	"github.com/safing/portmaster/base/updater"
-	"github.com/safing/portmaster/service/updates"
+	"github.com/safing/portmaster/service/updates/registry"
 	"github.com/safing/structures/dsd"
 )
 
@@ -164,7 +162,7 @@ func getListIndexFromCache() (*ListIndexFile, error) {
 
 var (
 	// listIndexUpdate must only be used by updateListIndex.
-	listIndexUpdate     *updater.File
+	listIndexUpdate     *registry.File
 	listIndexUpdateLock sync.Mutex
 )
 
@@ -177,24 +175,24 @@ func updateListIndex() error {
 	case listIndexUpdate == nil:
 		// This is the first time this function is run, get updater file for index.
 		var err error
-		listIndexUpdate, err = updates.GetFile(listIndexFilePath)
+		listIndexUpdate, err = module.instance.Updates().GetFile(listIndexFilePath)
 		if err != nil {
 			return err
 		}
 
 		// Check if the version in the cache is current.
-		index, err := getListIndexFromCache()
+		_, err = getListIndexFromCache()
 		switch {
 		case errors.Is(err, database.ErrNotFound):
 			log.Info("filterlists: index not in cache, starting update")
 		case err != nil:
 			log.Warningf("filterlists: failed to load index from cache, starting update: %s", err)
-		case !listIndexUpdate.EqualsVersion(strings.TrimPrefix(index.Version, "v")):
-			log.Infof(
-				"filterlists: index from cache is outdated, starting update (%s != %s)",
-				strings.TrimPrefix(index.Version, "v"),
-				listIndexUpdate.Version(),
-			)
+		// case !listIndexUpdate.EqualsVersion(strings.TrimPrefix(index.Version, "v")):
+		// 	log.Infof(
+		// 		"filterlists: index from cache is outdated, starting update (%s != %s)",
+		// 		strings.TrimPrefix(index.Version, "v"),
+		// 		listIndexUpdate.Version(),
+		// 	)
 		default:
 			// List is in cache and current, there is nothing to do.
 			log.Debug("filterlists: index is up to date")
@@ -204,8 +202,8 @@ func updateListIndex() error {
 
 			return nil
 		}
-	case listIndexUpdate.UpgradeAvailable():
-		log.Info("filterlists: index update available, starting update")
+	// case listIndexUpdate.UpgradeAvailable():
+	// 	log.Info("filterlists: index update available, starting update")
 	default:
 		// Index is loaded and no update is available, there is nothing to do.
 		return nil

service/intel/filterlists/updater.go

@@ -13,8 +13,8 @@ import (
 	"github.com/safing/portmaster/base/database"
 	"github.com/safing/portmaster/base/database/query"
 	"github.com/safing/portmaster/base/log"
-	"github.com/safing/portmaster/base/updater"
 	"github.com/safing/portmaster/service/mgr"
+	"github.com/safing/portmaster/service/updates/registry"
 )
 
 var updateInProgress = abool.New()
@@ -174,51 +174,51 @@ func removeAllObsoleteFilterEntries(wc *mgr.WorkerCtx) error {
 // getUpgradableFiles returns a slice of filterlists files
 // that should be updated. The files MUST be updated and
 // processed in the returned order!
-func getUpgradableFiles() ([]*updater.File, error) {
-	var updateOrder []*updater.File
+func getUpgradableFiles() ([]*registry.File, error) {
+	var updateOrder []*registry.File
 
-	cacheDBInUse := isLoaded()
+	// cacheDBInUse := isLoaded()
 
-	if baseFile == nil || baseFile.UpgradeAvailable() || !cacheDBInUse {
-		var err error
-		baseFile, err = getFile(baseListFilePath)
-		if err != nil {
-			return nil, err
-		}
-		log.Tracef("intel/filterlists: base file needs update, selected version %s", baseFile.Version())
-		updateOrder = append(updateOrder, baseFile)
-	}
+	// if baseFile == nil  || !cacheDBInUse { // TODO(vladimir): || baseFile.UpgradeAvailable()
+	// 	var err error
+	// 	baseFile, err = module.instance.Updates().GetFile(baseListFilePath)
+	// 	if err != nil {
+	// 		return nil, err
+	// 	}
+	// 	log.Tracef("intel/filterlists: base file needs update, selected version %s", baseFile.Version())
+	// 	updateOrder = append(updateOrder, baseFile)
+	// }
 
-	if intermediateFile == nil || intermediateFile.UpgradeAvailable() || !cacheDBInUse {
-		var err error
-		intermediateFile, err = getFile(intermediateListFilePath)
-		if err != nil && !errors.Is(err, updater.ErrNotFound) {
-			return nil, err
-		}
+	// if intermediateFile == nil || intermediateFile.UpgradeAvailable() || !cacheDBInUse {
+	// 	var err error
+	// 	intermediateFile, err = getFile(intermediateListFilePath)
+	// 	if err != nil && !errors.Is(err, updater.ErrNotFound) {
+	// 		return nil, err
+	// 	}
 
-		if err == nil {
-			log.Tracef("intel/filterlists: intermediate file needs update, selected version %s", intermediateFile.Version())
-			updateOrder = append(updateOrder, intermediateFile)
-		}
-	}
+	// 	if err == nil {
+	// 		log.Tracef("intel/filterlists: intermediate file needs update, selected version %s", intermediateFile.Version())
+	// 		updateOrder = append(updateOrder, intermediateFile)
+	// 	}
+	// }
 
-	if urgentFile == nil || urgentFile.UpgradeAvailable() || !cacheDBInUse {
-		var err error
-		urgentFile, err = getFile(urgentListFilePath)
-		if err != nil && !errors.Is(err, updater.ErrNotFound) {
-			return nil, err
-		}
+	// if urgentFile == nil || urgentFile.UpgradeAvailable() || !cacheDBInUse {
+	// 	var err error
+	// 	urgentFile, err = getFile(urgentListFilePath)
+	// 	if err != nil && !errors.Is(err, updater.ErrNotFound) {
+	// 		return nil, err
+	// 	}
 
-		if err == nil {
-			log.Tracef("intel/filterlists: urgent file needs update, selected version %s", urgentFile.Version())
-			updateOrder = append(updateOrder, urgentFile)
-		}
-	}
+	// 	if err == nil {
+	// 		log.Tracef("intel/filterlists: urgent file needs update, selected version %s", urgentFile.Version())
+	// 		updateOrder = append(updateOrder, urgentFile)
+	// 	}
+	// }
 
 	return resolveUpdateOrder(updateOrder)
 }
 
-func resolveUpdateOrder(updateOrder []*updater.File) ([]*updater.File, error) {
+func resolveUpdateOrder(updateOrder []*registry.File) ([]*registry.File, error) {
 	// sort the update order by ascending version
 	sort.Sort(byAscVersion(updateOrder))
 	log.Tracef("intel/filterlists: order of updates: %v", updateOrder)
@@ -258,7 +258,7 @@ func resolveUpdateOrder(updateOrder []*updater.File) ([]*updater.File, error) {
 	return updateOrder[startAtIdx:], nil
 }
 
-type byAscVersion []*updater.File
+type byAscVersion []*registry.File
 
 func (fs byAscVersion) Len() int { return len(fs) }
 

service/intel/geoip/database.go

@@ -8,9 +8,8 @@ import (
 	maxminddb "github.com/oschwald/maxminddb-golang"
 
 	"github.com/safing/portmaster/base/log"
-	"github.com/safing/portmaster/base/updater"
 	"github.com/safing/portmaster/service/mgr"
-	"github.com/safing/portmaster/service/updates"
+	"github.com/safing/portmaster/service/updates/registry"
 )
 
 var worker *updateWorker
@@ -22,13 +21,13 @@ func init() {
 }
 
 const (
-	v4MMDBResource = "intel/geoip/geoipv4.mmdb.gz"
-	v6MMDBResource = "intel/geoip/geoipv6.mmdb.gz"
+	v4MMDBResource = "geoipv4.mmdb"
+	v6MMDBResource = "geoipv6.mmdb"
 )
 
 type geoIPDB struct {
 	*maxminddb.Reader
-	file *updater.File
+	file *registry.File
 }
 
 // updateBroadcaster stores a geoIPDB and provides synchronized
@@ -47,7 +46,7 @@ func (ub *updateBroadcaster) NeedsUpdate() bool {
 	ub.rw.RLock()
 	defer ub.rw.RUnlock()
 
-	return ub.db == nil || ub.db.file.UpgradeAvailable()
+	return ub.db == nil // TODO(vladimir) is this needed: || ub.db.file.UpgradeAvailable()
 }
 
 // ReplaceDatabase replaces (or initially sets) the mmdb database.
@@ -181,12 +180,12 @@ func (upd *updateWorker) run(ctx *mgr.WorkerCtx) error {
 func getGeoIPDB(resource string) (*geoIPDB, error) {
 	log.Debugf("geoip: opening database %s", resource)
 
-	file, unpackedPath, err := openAndUnpack(resource)
+	file, err := open(resource)
 	if err != nil {
 		return nil, err
 	}
 
-	reader, err := maxminddb.Open(unpackedPath)
+	reader, err := maxminddb.Open(file.Path())
 	if err != nil {
 		return nil, fmt.Errorf("failed to open: %w", err)
 	}
@@ -198,16 +197,16 @@ func getGeoIPDB(resource string) (*geoIPDB, error) {
 	}, nil
 }
 
-func openAndUnpack(resource string) (*updater.File, string, error) {
-	f, err := updates.GetFile(resource)
+func open(resource string) (*registry.File, error) {
+	f, err := module.instance.Updates().GetFile(resource)
 	if err != nil {
-		return nil, "", fmt.Errorf("getting file: %w", err)
+		return nil, fmt.Errorf("getting file: %w", err)
 	}
 
-	unpacked, err := f.Unpack(".gz", updater.UnpackGZIP)
-	if err != nil {
-		return nil, "", fmt.Errorf("unpacking file: %w", err)
-	}
+	// unpacked, err := f.Unpack(".gz", updater.UnpackGZIP)
+	// if err != nil {
+	// 	return nil, "", fmt.Errorf("unpacking file: %w", err)
+	// }
 
-	return f, unpacked, nil
+	return f, nil
 }