[tests] implement expected failures

test/test_extractor.py

@@ -104,27 +104,39 @@ class TestExtractorModule(unittest.TestCase):
     @unittest.skipIf(not results, "no test data")
     def test_categories(self):
         for result in results.all():
-            url = result["#url"]
-            cls = result["#class"]
-            try:
-                extr = cls.from_url(url)
-            except ImportError as exc:
-                if exc.name in ("youtube_dl", "yt_dlp"):
-                    print("Skipping '{}' category checks".format(cls.category))
-                    continue
-                raise
-            self.assertTrue(extr, url)
-
-            categories = result.get("#category")
-            if categories:
-                base, cat, sub = categories
+            if result.get("#fail"):
+                try:
+                    self.assertCategories(result)
+                except AssertionError:
+                    pass
+                else:
+                    self.fail(result["#url"] + ": Test did not fail")
             else:
-                cat = cls.category
-                sub = cls.subcategory
-                base = cls.basecategory
-            self.assertEqual(extr.category, cat, url)
-            self.assertEqual(extr.subcategory, sub, url)
-            self.assertEqual(extr.basecategory, base, url)
+                self.assertCategories(result)
+
+    def assertCategories(self, result):
+        url = result["#url"]
+        cls = result["#class"]
+
+        try:
+            extr = cls.from_url(url)
+        except ImportError as exc:
+            if exc.name in ("youtube_dl", "yt_dlp"):
+                print("Skipping '{}' category checks".format(cls.category))
+                return
+            raise
+        self.assertTrue(extr, url)
+
+        categories = result.get("#category")
+        if categories:
+            base, cat, sub = categories
+        else:
+            cat = cls.category
+            sub = cls.subcategory
+            base = cls.basecategory
+        self.assertEqual(extr.category, cat, url)
+        self.assertEqual(extr.subcategory, sub, url)
+        self.assertEqual(extr.basecategory, base, url)
 
     @unittest.skipIf(not results, "no test data")
     def test_unique_pattern_matches(self):
@@ -133,7 +145,8 @@ class TestExtractorModule(unittest.TestCase):
         append = test_urls.append
 
         for result in results.all():
-            append((result["#url"], result["#class"]))
+            if not result.get("#fail"):
+                append((result["#url"], result["#class"]))
 
         # iterate over all testcase URLs
         for url, extr1 in test_urls:

test/test_results.py

@@ -92,6 +92,15 @@ class TestExtractorResults(unittest.TestCase):
             self.assertGreaterEqual(value, range.start, msg=msg)
 
     def _run_test(self, result):
+        if result.get("#fail"):
+            del result["#fail"]
+            try:
+                self._run_test(result)
+            except AssertionError:
+                return
+            else:
+                self.fail("Test did not fail")
+
         base, cat, sub = result_categories(result)
         result.pop("#comment", None)
         result.pop("#category", None)