From 53cc498d9c26938fc5ab335f064c6604fda0c346 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de>
Date: Sat, 20 Jun 2020 21:41:59 +0200
Subject: [PATCH] improve config lookup when there are multiple possible
 locations

This specifically applies to all Mastodon extractors and all
extractors with a 'basecategory', i.e. 'booru', 'foolslide', etc.

Values inside those general config locations wouldn't be recognized
when a value with the same was set on the 'extractor' level.

For example 'extractor.mastodon.directory' should be used over
'extractor.directory' when both are set, but this was impossible
with the previous implementation.

(fixes #843)
---
 gallery_dl/config.py             | 32 ++++++++++++++++++++++++++++++++
 gallery_dl/extractor/common.py   | 11 +++++++----
 gallery_dl/extractor/mastodon.py | 11 ++++++-----
 test/test_config.py              | 28 ++++++++++++++++++++++++++++
 4 files changed, 73 insertions(+), 9 deletions(-)

diff --git a/gallery_dl/config.py b/gallery_dl/config.py
index 53036169..a3c71cde 100644
--- a/gallery_dl/config.py
+++ b/gallery_dl/config.py
@@ -108,6 +108,38 @@ def interpolate(path, key, default=None, *, conf=_config):
     return default
 
 
+def interpolate_common(common, paths, key, default=None, *, conf=_config):
+    """Interpolate the value of 'key'
+    using multiple 'paths' along a 'common' ancestor
+    """
+    if key in conf:
+        return conf[key]
+
+    # follow the common path
+    try:
+        for p in common:
+            conf = conf[p]
+            if key in conf:
+                default = conf[key]
+    except Exception:
+        return default
+
+    # try all paths until a value is found
+    value = util.SENTINEL
+    for path in paths:
+        c = conf
+        try:
+            for p in path:
+                c = c[p]
+                if key in c:
+                    value = c[key]
+        except Exception:
+            pass
+        if value is not util.SENTINEL:
+            return value
+    return default
+
+
 def set(path, key, value, *, conf=_config):
     """Set the value of property 'key' for this session"""
     for p in path:
diff --git a/gallery_dl/extractor/common.py b/gallery_dl/extractor/common.py
index 4d314c23..bbbd8a65 100644
--- a/gallery_dl/extractor/common.py
+++ b/gallery_dl/extractor/common.py
@@ -492,10 +492,13 @@ class SharedConfigMixin():
     """Enable sharing of config settings based on 'basecategory'"""
     basecategory = ""
 
-    def config(self, key, default=None, *, sentinel=util.SENTINEL):
-        value = Extractor.config(self, key, sentinel)
-        return value if value is not sentinel else config.interpolate(
-            ("extractor", self.basecategory, self.subcategory), key, default)
+    def config(self, key, default=None):
+        return config.interpolate_common(
+            ("extractor",), (
+                (self.category, self.subcategory),
+                (self.basecategory, self.subcategory),
+            ), key, default,
+        )
 
 
 def generate_extractors(extractor_data, symtable, classes):
diff --git a/gallery_dl/extractor/mastodon.py b/gallery_dl/extractor/mastodon.py
index f5e502b6..fa1fecc6 100644
--- a/gallery_dl/extractor/mastodon.py
+++ b/gallery_dl/extractor/mastodon.py
@@ -27,11 +27,12 @@ class MastodonExtractor(Extractor):
         Extractor.__init__(self, match)
         self.api = MastodonAPI(self)
 
-    def config(self, key, default=None, *, sentinel=util.SENTINEL):
-        value = Extractor.config(self, key, sentinel)
-        return value if value is not sentinel else config.interpolate(
-            ("extractor", "mastodon", self.instance, self.subcategory),
-            key, default,
+    def config(self, key, default=None):
+        return config.interpolate_common(
+            ("extractor",), (
+                (self.category, self.subcategory),
+                (self.basecategory, self.instance, self.subcategory),
+            ), key, default,
         )
 
     def items(self):
diff --git a/test/test_config.py b/test/test_config.py
index 4171435f..a9cefd40 100644
--- a/test/test_config.py
+++ b/test/test_config.py
@@ -68,6 +68,34 @@ class TestConfig(unittest.TestCase):
         self.assertEqual(config.interpolate(("b",), "d", 1) , 2)
         self.assertEqual(config.interpolate(("d",), "d", 1) , 2)
 
+    def test_interpolate_common(self):
+
+        def lookup():
+            return config.interpolate_common(
+                ("Z1", "Z2"), (
+                    ("A1", "A2"),
+                    ("B1",),
+                    ("C1", "C2", "C3"),
+                ), "KEY", "DEFAULT",
+            )
+
+        def test(path, value, expected=None):
+            config.set(path, "KEY", value)
+            self.assertEqual(lookup(), expected or value)
+
+        self.assertEqual(lookup(), "DEFAULT")
+        test(("Z1",), 1)
+        test(("Z1", "Z2"), 2)
+        test(("Z1", "Z2", "C1"), 3)
+        test(("Z1", "Z2", "C1", "C2"), 4)
+        test(("Z1", "Z2", "C1", "C2", "C3"), 5)
+        test(("Z1", "Z2", "B1"), 6)
+        test(("Z1", "Z2", "A1"), 7)
+        test(("Z1", "Z2", "A1", "A2"), 8)
+        test(("Z1", "A1", "A2"), 999, 8)
+        test(("Z1", "Z2", "A1", "A2", "A3"), 999, 8)
+        test((), 9)
+
     def test_set(self):
         config.set(()        , "c", [1, 2, 3])
         config.set(("b",)    , "c", [1, 2, 3])