[formatter] implement 'H' conversion (#4164)

to remove HTML tags and unescape HTML entities

docs/formatting.md

@@ -94,6 +94,18 @@ Conversion specifiers allow to *convert* the value to a different form or type.
     <td><code>{created!d}</code></td>
     <td><code>2010-01-01 00:00:00</code></td>
 </tr>
+<tr>
+    <td align="center"><code>U</code></td>
+    <td>Convert HTML entities</td>
+    <td><code>{html!U}</code></td>
+    <td><code>&lt;p&gt;foo &amp; bar&lt;/p&gt;</code></td>
+</tr>
+<tr>
+    <td align="center"><code>H</code></td>
+    <td>Convert HTML entities &amp; remove HTML tags</td>
+    <td><code>{html!H}</code></td>
+    <td><code>foo &amp; bar</code></td>
+</tr>
 <tr>
     <td align="center"><code>s</code></td>
     <td>Convert value to <a href="https://docs.python.org/3/library/stdtypes.html#text-sequence-type-str" rel="nofollow"><code>str</code></a></td>

gallery_dl/formatter.py

@@ -437,6 +437,7 @@ _CONVERSIONS = {
     "T": util.datetime_to_timestamp_string,
     "d": text.parse_timestamp,
     "U": text.unescape,
+    "H": lambda s: text.unescape(text.remove_html(s)),
     "g": text.slugify,
     "S": util.to_string,
     "s": str,

test/test_formatter.py

@@ -28,6 +28,7 @@ class TestFormatter(unittest.TestCase):
         "l": ["a", "b", "c"],
         "n": None,
         "s": " \n\r\tSPACE    ",
+        "h": "<p>foo </p> &amp; bar <p> </p>",
         "u": "&#x27;&lt; / &gt;&#x27;",
         "t": 1262304000,
         "dt": datetime.datetime(2010, 1, 1),
@@ -47,6 +48,10 @@ class TestFormatter(unittest.TestCase):
         self._run_test("{s!t}", "SPACE")
         self._run_test("{a!U}", self.kwdict["a"])
         self._run_test("{u!U}", "'< / >'")
+        self._run_test("{a!H}", self.kwdict["a"])
+        self._run_test("{h!H}", "foo & bar")
+        self._run_test("{u!H}", "'< / >'")
+        self._run_test("{n!H}", "")
         self._run_test("{a!s}", self.kwdict["a"])
         self._run_test("{a!r}", "'" + self.kwdict["a"] + "'")
         self._run_test("{a!a}", "'" + self.kwdict["a"] + "'")
@@ -434,10 +439,10 @@ def noarg():
             fmt4 = formatter.parse("\fM " + path + ":lengths")
 
         self.assertEqual(fmt1.format_map(self.kwdict), "'Title' by Name")
-        self.assertEqual(fmt2.format_map(self.kwdict), "96")
+        self.assertEqual(fmt2.format_map(self.kwdict), "126")
 
         self.assertEqual(fmt3.format_map(self.kwdict), "'Title' by Name")
-        self.assertEqual(fmt4.format_map(self.kwdict), "96")
+        self.assertEqual(fmt4.format_map(self.kwdict), "126")
 
         with self.assertRaises(TypeError):
             self.assertEqual(fmt0.format_map(self.kwdict), "")