wip: migrate to mono-repo. SPN has already been moved to spn/

This commit is contained in:
Patrick Pacher
2024-03-15 11:55:13 +01:00
parent b30fd00ccf
commit 8579430db9
577 changed files with 35981 additions and 818 deletions

View File

@@ -0,0 +1,32 @@
package binmeta
import (
"bytes"
"fmt"
"image"
_ "image/png" // Register png support for image package
"github.com/fogleman/gg"
_ "github.com/mat/besticon/ico" // Register ico support for image package
)
// ConvertICOtoPNG converts a an .ico to a .png image.
func ConvertICOtoPNG(ico []byte) (png []byte, err error) {
// Decode the ICO.
icon, _, err := image.Decode(bytes.NewReader(ico))
if err != nil {
return nil, fmt.Errorf("failed to decode ICO: %w", err)
}
// Convert to raw image.
img := gg.NewContextForImage(icon)
// Convert to PNG.
imgBuf := &bytes.Buffer{}
err = img.EncodePNG(imgBuf)
if err != nil {
return nil, fmt.Errorf("failed to encode PNG: %w", err)
}
return imgBuf.Bytes(), nil
}

View File

@@ -0,0 +1,10 @@
//go:build !linux && !windows
package binmeta
import "context"
// GetIconAndName returns zero values for unsupported platforms.
func GetIconAndName(ctx context.Context, binPath string, homeDir string) (icon *Icon, name string, err error) {
return nil, "", nil
}

View File

@@ -0,0 +1,111 @@
package binmeta
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
)
// GetIconAndName returns an icon and name of the given binary path.
// Providing the home directory of the user running the process of that binary can improve results.
// Even if an error is returned, the other return values are valid, if set.
func GetIconAndName(ctx context.Context, binPath string, homeDir string) (icon *Icon, name string, err error) {
// Derive name from binary.
name = GenerateBinaryNameFromPath(binPath)
// Search for icon.
iconPath, err := searchForIcon(binPath, homeDir)
if iconPath == "" {
if err != nil {
return nil, name, fmt.Errorf("failed to find icon for %s: %w", binPath, err)
}
return nil, name, nil
}
// Save icon to internal storage.
icon, err = LoadAndSaveIcon(ctx, iconPath)
if err != nil {
return nil, name, fmt.Errorf("failed to store icon for %s: %w", binPath, err)
}
return icon, name, nil
}
func searchForIcon(binPath string, homeDir string) (iconPath string, err error) {
binPath = strings.ToLower(binPath)
// Search for icon path.
for _, iconLoc := range iconLocations {
basePath := iconLoc.GetPath(binPath, homeDir)
if basePath == "" {
continue
}
switch iconLoc.Type {
case FlatDir:
iconPath, err = searchDirectory(basePath, binPath)
case XDGIcons:
iconPath, err = searchXDGIconStructure(basePath, binPath)
}
if iconPath != "" {
return
}
}
return
}
func searchXDGIconStructure(baseDirectory string, binPath string) (iconPath string, err error) {
for _, xdgIconDir := range xdgIconPaths {
directory := filepath.Join(baseDirectory, xdgIconDir)
iconPath, err = searchDirectory(directory, binPath)
if iconPath != "" {
return
}
}
return
}
func searchDirectory(directory string, binPath string) (iconPath string, err error) {
entries, err := os.ReadDir(directory)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return "", nil
}
return "", fmt.Errorf("failed to read directory %s: %w", directory, err)
}
// DEBUG:
// fmt.Println(directory)
var (
bestMatch string
bestMatchExcessChars int
)
for _, entry := range entries {
// Skip dirs.
if entry.IsDir() {
continue
}
iconName := strings.ToLower(entry.Name())
iconName = strings.TrimSuffix(iconName, filepath.Ext(iconName))
switch {
case len(iconName) < len(binPath):
// Continue to next.
case iconName == binPath:
// Exact match, return immediately.
return filepath.Join(directory, entry.Name()), nil
case strings.HasPrefix(iconName, binPath):
excessChars := len(iconName) - len(binPath)
if bestMatch == "" || excessChars < bestMatchExcessChars {
bestMatch = entry.Name()
bestMatchExcessChars = excessChars
}
}
}
return bestMatch, nil
}

View File

@@ -0,0 +1,32 @@
package binmeta
import (
"os"
"testing"
)
func TestFindIcon(t *testing.T) {
if testing.Short() {
t.Skip("test depends on linux desktop environment")
}
t.Parallel()
home := os.Getenv("HOME")
testFindIcon(t, "evolution", home)
testFindIcon(t, "nextcloud", home)
}
func testFindIcon(t *testing.T, binName string, homeDir string) {
t.Helper()
iconPath, err := searchForIcon(binName, homeDir)
if err != nil {
t.Error(err)
return
}
if iconPath == "" {
t.Errorf("no icon found for %s", binName)
return
}
t.Logf("icon for %s found: %s", binName, iconPath)
}

View File

@@ -0,0 +1,119 @@
package binmeta
import (
"bytes"
"context"
"errors"
"fmt"
"os"
"github.com/tc-hib/winres"
"github.com/tc-hib/winres/version"
)
// GetIconAndName returns an icon and name of the given binary path.
// Providing the home directory of the user running the process of that binary can improve results.
// Even if an error is returned, the other return values are valid, if set.
func GetIconAndName(ctx context.Context, binPath string, homeDir string) (icon *Icon, name string, err error) {
// Get name and png from exe.
png, name, err := getIconAndNamefromRSS(ctx, binPath)
// Fall back to name generation if name is not set.
if name == "" {
name = GenerateBinaryNameFromPath(binPath)
}
// Handle previous error.
if err != nil {
return nil, name, err
}
// Update profile icon and return icon object.
filename, err := UpdateProfileIcon(png, "png")
if err != nil {
return nil, name, fmt.Errorf("failed to store icon: %w", err)
}
return &Icon{
Type: IconTypeAPI,
Value: filename,
Source: IconSourceCore,
}, name, nil
}
func getIconAndNamefromRSS(ctx context.Context, binPath string) (png []byte, name string, err error) {
// Open .exe file.
exeFile, err := os.Open(binPath)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return nil, "", nil
}
return nil, "", fmt.Errorf("failed to open exe %s to get icon: %w", binPath, err)
}
defer exeFile.Close() //nolint:errcheck
// Load .exe resources.
rss, err := winres.LoadFromEXE(exeFile)
if err != nil {
return nil, "", fmt.Errorf("failed to get rss: %w", err)
}
// DEBUG: Print all available resources:
// rss.Walk(func(typeID, resID winres.Identifier, langID uint16, data []byte) bool {
// fmt.Printf("typeID=%d resID=%d langID=%d\n", typeID, resID, langID)
// return true
// })
// Get name from version record.
var (
versionInfo *version.Info
versionInfoErr error
)
rss.WalkType(winres.RT_VERSION, func(resID winres.Identifier, langID uint16, data []byte) bool {
versionInfo, versionInfoErr = version.FromBytes(data)
switch {
case versionInfoErr != nil:
return true
case versionInfo == nil:
return true
}
// Get metadata table and main language.
table := versionInfo.Table().GetMainTranslation()
if table == nil {
return true
}
name = table[version.ProductName]
return name == ""
})
name = cleanFileDescription(name)
// Get first icon.
var (
icon *winres.Icon
iconErr error
)
rss.WalkType(winres.RT_GROUP_ICON, func(resID winres.Identifier, langID uint16, _ []byte) bool {
icon, iconErr = rss.GetIconTranslation(resID, langID)
return iconErr != nil
})
if iconErr != nil {
return nil, name, fmt.Errorf("failed to get icon: %w", err)
}
if icon == nil {
return nil, name, errors.New("no icon in resources")
}
// Convert icon, if it exists.
icoBuf := &bytes.Buffer{}
err = icon.SaveICO(icoBuf)
if err != nil {
return nil, name, fmt.Errorf("failed to save ico: %w", err)
}
png, err = ConvertICOtoPNG(icoBuf.Bytes())
if err != nil {
return nil, name, fmt.Errorf("failed to convert ico to png: %w", err)
}
return png, name, nil
}

View File

@@ -0,0 +1,27 @@
package binmeta
import (
"context"
"os"
"testing"
)
func TestFindIcon(t *testing.T) {
if testing.Short() {
t.Skip("test meant for compiling and running on desktop")
}
t.Parallel()
binName := os.Args[len(os.Args)-1]
t.Logf("getting name and icon for %s", binName)
png, name, err := getIconAndNamefromRSS(context.Background(), binName)
if err != nil {
t.Fatal(err)
}
t.Logf("name: %s", name)
err = os.WriteFile("icon.png", png, 0o0600)
if err != nil {
t.Fatal(err)
}
}

View File

@@ -0,0 +1,165 @@
package binmeta
import (
"errors"
"fmt"
"strings"
"sync"
"github.com/vincent-petithory/dataurl"
"golang.org/x/exp/slices"
"github.com/safing/portbase/database"
"github.com/safing/portbase/database/record"
)
// Icon describes an icon.
type Icon struct {
Type IconType
Value string
Source IconSource
}
// IconType describes the type of an Icon.
type IconType string
// Supported icon types.
const (
IconTypeFile IconType = "path"
IconTypeDatabase IconType = "database"
IconTypeAPI IconType = "api"
)
func (t IconType) sortOrder() int {
switch t {
case IconTypeAPI:
return 1
case IconTypeDatabase:
return 2
case IconTypeFile:
return 3
default:
return 9
}
}
// IconSource describes the source of an Icon.
type IconSource string
// Supported icon sources.
const (
IconSourceUser IconSource = "user"
IconSourceImport IconSource = "import"
IconSourceUI IconSource = "ui"
IconSourceCore IconSource = "core"
)
func (s IconSource) sortOrder() int {
switch s {
case IconSourceUser:
return 10
case IconSourceImport:
return 20
case IconSourceUI:
return 30
case IconSourceCore:
return 40
default:
return 90
}
}
func (icon Icon) sortOrder() int {
return icon.Source.sortOrder() + icon.Type.sortOrder()
}
// SortAndCompactIcons sorts and compacts a list of icons.
func SortAndCompactIcons(icons []Icon) []Icon {
// Sort.
slices.SortFunc[[]Icon, Icon](icons, func(a, b Icon) int {
aOrder := a.sortOrder()
bOrder := b.sortOrder()
switch {
case aOrder != bOrder:
return aOrder - bOrder
case a.Value != b.Value:
return strings.Compare(a.Value, b.Value)
default:
return 0
}
})
// De-duplicate.
icons = slices.CompactFunc[[]Icon, Icon](icons, func(a, b Icon) bool {
return a.Type == b.Type && a.Value == b.Value
})
return icons
}
// GetIconAsDataURL returns the icon data as a data URL.
func (icon Icon) GetIconAsDataURL() (bloburl string, err error) {
switch icon.Type {
case IconTypeFile:
return "", errors.New("getting icon from file is not supported")
case IconTypeDatabase:
if !strings.HasPrefix(icon.Value, "cache:icons/") {
return "", errors.New("invalid icon db key")
}
r, err := iconDB.Get(icon.Value)
if err != nil {
return "", err
}
dbIcon, err := EnsureIconInDatabase(r)
if err != nil {
return "", err
}
return dbIcon.IconData, nil
case IconTypeAPI:
data, err := GetProfileIcon(icon.Value)
if err != nil {
return "", err
}
return dataurl.EncodeBytes(data), nil
default:
return "", errors.New("unknown icon type")
}
}
var iconDB = database.NewInterface(&database.Options{
Local: true,
Internal: true,
})
// IconInDatabase represents an icon saved to the database.
type IconInDatabase struct {
sync.Mutex
record.Base
IconData string `json:"iconData,omitempty"` // DataURL
}
// EnsureIconInDatabase ensures that the given record is a *IconInDatabase, and returns it.
func EnsureIconInDatabase(r record.Record) (*IconInDatabase, error) {
// unwrap
if r.IsWrapped() {
// only allocate a new struct, if we need it
newIcon := &IconInDatabase{}
err := record.Unwrap(r, newIcon)
if err != nil {
return nil, err
}
return newIcon, nil
}
// or adjust type
newIcon, ok := r.(*IconInDatabase)
if !ok {
return nil, fmt.Errorf("record not of type *IconInDatabase, but %T", r)
}
return newIcon, nil
}

View File

@@ -0,0 +1,101 @@
package binmeta
import (
"context"
"crypto"
"encoding/hex"
"errors"
"fmt"
"net/http"
"os"
"path/filepath"
"strings"
"github.com/safing/portbase/api"
)
// ProfileIconStoragePath defines the location where profile icons are stored.
// Must be set before anything else from this package is called.
// Must not be changed once set.
var ProfileIconStoragePath = ""
// GetProfileIcon returns the profile icon with the given ID and extension.
func GetProfileIcon(name string) (data []byte, err error) {
// Check if enabled.
if ProfileIconStoragePath == "" {
return nil, errors.New("api icon storage not configured")
}
// Build storage path.
iconPath := filepath.Clean(
filepath.Join(ProfileIconStoragePath, name),
)
iconPath, err = filepath.Abs(iconPath)
if err != nil {
return nil, fmt.Errorf("failed to check icon path: %w", err)
}
// Do a quick check if we are still within the right directory.
// This check is not entirely correct, but is sufficient for this use case.
if filepath.Dir(iconPath) != ProfileIconStoragePath {
return nil, api.ErrorWithStatus(errors.New("invalid icon"), http.StatusBadRequest)
}
return os.ReadFile(iconPath)
}
// UpdateProfileIcon creates or updates the given icon.
func UpdateProfileIcon(data []byte, ext string) (filename string, err error) {
// Check icon size.
if len(data) > 1_000_000 {
return "", errors.New("icon too big")
}
// Calculate sha1 sum of icon.
h := crypto.SHA1.New()
if _, err := h.Write(data); err != nil {
return "", err
}
sum := hex.EncodeToString(h.Sum(nil))
// Check ext.
ext = strings.ToLower(ext)
switch ext {
case "gif":
case "jpeg":
ext = "jpg"
case "jpg":
case "png":
case "svg":
case "tiff":
case "webp":
default:
return "", errors.New("unsupported icon format")
}
// Save to disk.
filename = sum + "." + ext
return filename, os.WriteFile(filepath.Join(ProfileIconStoragePath, filename), data, 0o0644) //nolint:gosec
}
// LoadAndSaveIcon loads an icon from disk, updates it in the icon database
// and returns the icon object.
func LoadAndSaveIcon(ctx context.Context, iconPath string) (*Icon, error) {
// Load icon and save it.
data, err := os.ReadFile(iconPath)
if err != nil {
return nil, fmt.Errorf("failed to read icon %s: %w", iconPath, err)
}
filename, err := UpdateProfileIcon(data, filepath.Ext(iconPath))
if err != nil {
return nil, fmt.Errorf("failed to import icon %s: %w", iconPath, err)
}
return &Icon{
Type: IconTypeAPI,
Value: filename,
Source: IconSourceCore,
}, nil
}
// TODO: Clean up icons regularly.

View File

@@ -0,0 +1,68 @@
package binmeta
import (
"fmt"
)
// IconLocation describes an icon location.
type IconLocation struct {
Directory string
Type IconLocationType
PathArg PathArg
}
// IconLocationType describes an icon location type.
type IconLocationType uint8
// Icon Location Types.
const (
FlatDir IconLocationType = iota
XDGIcons
)
// PathArg describes an icon location path argument.
type PathArg uint8
// Path Args.
const (
NoPathArg PathArg = iota
Home
BinName
)
var (
iconLocations = []IconLocation{
{Directory: "/usr/share/pixmaps", Type: FlatDir},
{Directory: "/usr/share", Type: XDGIcons},
{Directory: "%s/.local/share", Type: XDGIcons, PathArg: Home},
{Directory: "%s/.local/share/flatpak/exports/share", Type: XDGIcons, PathArg: Home},
{Directory: "/usr/share/%s", Type: XDGIcons, PathArg: BinName},
}
xdgIconPaths = []string{
// UI currently uses 48x48, so 256x256 should suffice for the future, even at 2x. (12.2023)
"icons/hicolor/256x256/apps",
"icons/hicolor/192x192/apps",
"icons/hicolor/128x128/apps",
"icons/hicolor/96x96/apps",
"icons/hicolor/72x72/apps",
"icons/hicolor/64x64/apps",
"icons/hicolor/48x48/apps",
"icons/hicolor/512x512/apps",
}
)
// GetPath returns the path of an icon.
func (il IconLocation) GetPath(binName string, homeDir string) string {
switch il.PathArg {
case NoPathArg:
return il.Directory
case Home:
if homeDir != "" {
return fmt.Sprintf(il.Directory, homeDir)
}
case BinName:
return fmt.Sprintf(il.Directory, binName)
}
return ""
}

View File

@@ -0,0 +1,121 @@
package binmeta
import (
"path/filepath"
"regexp"
"strings"
)
var (
segmentsSplitter = regexp.MustCompile("[^A-Za-z0-9]*[A-Z]?[a-z0-9]*")
nameOnly = regexp.MustCompile("^[A-Za-z0-9]+$")
delimitersAtStart = regexp.MustCompile("^[^A-Za-z0-9]+")
delimitersOnly = regexp.MustCompile("^[^A-Za-z0-9]+$")
removeQuotes = strings.NewReplacer(`"`, ``, `'`, ``)
)
// GenerateBinaryNameFromPath generates a more human readable binary name from
// the given path. This function is used as fallback in the GetBinaryName
// functions.
func GenerateBinaryNameFromPath(path string) string {
// Get file name from path.
_, fileName := filepath.Split(path)
// Split up into segments.
segments := segmentsSplitter.FindAllString(fileName, -1)
// Remove last segment if it's an extension.
if len(segments) >= 2 {
switch strings.ToLower(segments[len(segments)-1]) {
case
".exe", // Windows Executable
".msi", // Windows Installer
".bat", // Windows Batch File
".cmd", // Windows Command Script
".ps1", // Windows Powershell Cmdlet
".run", // Linux Executable
".appimage", // Linux AppImage
".app", // MacOS Executable
".action", // MacOS Automator Action
".out": // Generic Compiled Executable
segments = segments[:len(segments)-1]
}
}
// Debugging snippet:
// fmt.Printf("segments: %s\n", segments)
// Go through segments and collect name parts.
nameParts := make([]string, 0, len(segments))
var fragments string
for _, segment := range segments {
// Group very short segments.
if len(delimitersAtStart.ReplaceAllString(segment, "")) <= 2 {
fragments += segment
continue
} else if fragments != "" {
nameParts = append(nameParts, fragments)
fragments = ""
}
// Add segment to name.
nameParts = append(nameParts, segment)
}
// Add last fragment.
if fragments != "" {
nameParts = append(nameParts, fragments)
}
// Debugging snippet:
// fmt.Printf("parts: %s\n", nameParts)
// Post-process name parts
for i := range nameParts {
// Remove any leading delimiters.
nameParts[i] = delimitersAtStart.ReplaceAllString(nameParts[i], "")
// Title-case name-only parts.
if nameOnly.MatchString(nameParts[i]) {
nameParts[i] = strings.Title(nameParts[i]) //nolint:staticcheck
}
}
// Debugging snippet:
// fmt.Printf("final: %s\n", nameParts)
return strings.Join(nameParts, " ")
}
func cleanFileDescription(fileDescr string) string {
fields := strings.Fields(fileDescr)
// Clean out and `"` and `'`.
for i := range fields {
fields[i] = removeQuotes.Replace(fields[i])
}
// If there is a 1 or 2 character delimiter field, only use fields before it.
endIndex := len(fields)
for i, field := range fields {
// Ignore the first field as well as fields with more than two characters.
if i >= 1 && len(field) <= 2 && !nameOnly.MatchString(field) {
endIndex = i
break
}
}
// Concatenate name
binName := strings.Join(fields[:endIndex], " ")
// If there are multiple sentences, only use the first.
if strings.Contains(binName, ". ") {
binName = strings.SplitN(binName, ". ", 2)[0]
}
// If does not have any characters or numbers, return an empty string.
if delimitersOnly.MatchString(binName) {
return ""
}
return strings.TrimSpace(binName)
}

View File

@@ -0,0 +1,48 @@
package binmeta
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestGenerateBinaryNameFromPath(t *testing.T) {
t.Parallel()
assert.Equal(t, "Nslookup", GenerateBinaryNameFromPath("nslookup.exe"))
assert.Equal(t, "System Settings", GenerateBinaryNameFromPath("SystemSettings.exe"))
assert.Equal(t, "One Drive Setup", GenerateBinaryNameFromPath("OneDriveSetup.exe"))
assert.Equal(t, "Msedge", GenerateBinaryNameFromPath("msedge.exe"))
assert.Equal(t, "SIH Client", GenerateBinaryNameFromPath("SIHClient.exe"))
assert.Equal(t, "Openvpn Gui", GenerateBinaryNameFromPath("openvpn-gui.exe"))
assert.Equal(t, "Portmaster Core v0-1-2", GenerateBinaryNameFromPath("portmaster-core_v0-1-2.exe"))
assert.Equal(t, "Win Store App", GenerateBinaryNameFromPath("WinStore.App.exe"))
assert.Equal(t, "Test Script", GenerateBinaryNameFromPath(".test-script"))
assert.Equal(t, "Browser Broker", GenerateBinaryNameFromPath("browser_broker.exe"))
assert.Equal(t, "Virtual Box VM", GenerateBinaryNameFromPath("VirtualBoxVM"))
assert.Equal(t, "Io Elementary Appcenter", GenerateBinaryNameFromPath("io.elementary.appcenter"))
assert.Equal(t, "Microsoft Windows Store", GenerateBinaryNameFromPath("Microsoft.WindowsStore"))
}
func TestCleanFileDescription(t *testing.T) {
t.Parallel()
assert.Equal(t, "Product Name", cleanFileDescription("Product Name"))
assert.Equal(t, "Product Name", cleanFileDescription("Product Name. Does this and that."))
assert.Equal(t, "Product Name", cleanFileDescription("Product Name - Does this and that."))
assert.Equal(t, "Product Name", cleanFileDescription("Product Name / Does this and that."))
assert.Equal(t, "Product Name", cleanFileDescription("Product Name :: Does this and that."))
assert.Equal(t, "/ Product Name", cleanFileDescription("/ Product Name"))
assert.Equal(t, "Product", cleanFileDescription("Product / Name"))
assert.Equal(t, "Software 2", cleanFileDescription("Software 2"))
assert.Equal(t, "Launcher for Software 2", cleanFileDescription("Launcher for 'Software 2'"))
assert.Equal(t, "", cleanFileDescription(". / Name"))
assert.Equal(t, "", cleanFileDescription(". "))
assert.Equal(t, "", cleanFileDescription("."))
assert.Equal(t, "N/A", cleanFileDescription("N/A"))
assert.Equal(t,
"Product Name a Does this and that.",
cleanFileDescription("Product Name a Does this and that."),
)
}