From 563bff1d95fc52d04763098769fd683b8afd986f Mon Sep 17 00:00:00 2001
From: Daniel <dhaavi@users.noreply.github.com>
Date: Fri, 6 Nov 2020 17:29:58 +0100
Subject: [PATCH] Update filterlists to use new format

Also, add database reset flag
---
 intel/filterlists/cache_version.go | 10 ++++++++++
 intel/filterlists/database.go      |  4 ++--
 intel/filterlists/decoder.go       | 26 ++++++++++++++++++++++----
 3 files changed, 34 insertions(+), 6 deletions(-)

diff --git a/intel/filterlists/cache_version.go b/intel/filterlists/cache_version.go
index c48d8bd4..6b6dec9e 100644
--- a/intel/filterlists/cache_version.go
+++ b/intel/filterlists/cache_version.go
@@ -4,15 +4,20 @@ import (
 	"fmt"
 	"sync"
 
+	"github.com/safing/portbase/database"
+
 	"github.com/hashicorp/go-version"
 	"github.com/safing/portbase/database/record"
 )
 
+const resetVersion = "v0.6.0"
+
 type cacheVersionRecord struct {
 	record.Base
 	sync.Mutex
 
 	Version string
+	Reset   string
 }
 
 // getCacheDatabaseVersion reads and returns the cache
@@ -37,6 +42,10 @@ func getCacheDatabaseVersion() (*version.Version, error) {
 		}
 	}
 
+	if verRecord.Reset != resetVersion {
+		return nil, database.ErrNotFound
+	}
+
 	ver, err := version.NewSemver(verRecord.Version)
 	if err != nil {
 		return nil, err
@@ -50,6 +59,7 @@ func getCacheDatabaseVersion() (*version.Version, error) {
 func setCacheDatabaseVersion(ver string) error {
 	verRecord := &cacheVersionRecord{
 		Version: ver,
+		Reset:   resetVersion,
 	}
 
 	verRecord.SetKey(filterListCacheVersionKey)
diff --git a/intel/filterlists/database.go b/intel/filterlists/database.go
index c76fc10c..eb48e43c 100644
--- a/intel/filterlists/database.go
+++ b/intel/filterlists/database.go
@@ -200,14 +200,14 @@ func processEntry(ctx context.Context, filter *scopedBloom, entry *listEntry, re
 	normalizeEntry(entry)
 
 	// Only add the entry to the bloom filter if it has any sources.
-	if len(entry.Sources) > 0 {
+	if len(entry.Resources) > 0 {
 		filter.add(entry.Type, entry.Entity)
 	}
 
 	r := &entityRecord{
 		Value:     entry.Entity,
 		Type:      entry.Type,
-		Sources:   entry.Sources,
+		Sources:   entry.getSources(),
 		UpdatedAt: time.Now().Unix(),
 	}
 
diff --git a/intel/filterlists/decoder.go b/intel/filterlists/decoder.go
index 49133790..3e9c54b0 100644
--- a/intel/filterlists/decoder.go
+++ b/intel/filterlists/decoder.go
@@ -8,13 +8,31 @@ import (
 	"io"
 
 	"github.com/safing/portbase/formats/dsd"
+	"github.com/safing/portbase/utils"
 )
 
 type listEntry struct {
-	Entity    string   `json:"entity"`
-	Sources   []string `json:"sources"`
-	Whitelist bool     `json:"whitelist"`
-	Type      string   `json:"type"`
+	Type      string          `json:"type"`
+	Entity    string          `json:"entity"`
+	Whitelist bool            `json:"whitelist"`
+	Resources []entryResource `json:"resources"`
+}
+
+type entryResource struct {
+	SourceID   string `json:"sourceID"`
+	ResourceID string `json:"resourceID"`
+}
+
+func (entry *listEntry) getSources() (sourceIDs []string) {
+	sourceIDs = make([]string, 0, len(entry.Resources))
+
+	for _, resource := range entry.Resources {
+		if !utils.StringInSlice(sourceIDs, resource.SourceID) {
+			sourceIDs = append(sourceIDs, resource.SourceID)
+		}
+	}
+
+	return
 }
 
 // decodeFile decodes a DSDL filterlists file and sends decoded entities to