[architizer] add 'firm' extractor (#1369)
This commit is contained in:
@@ -70,7 +70,7 @@ Consider all sites to be NSFW unless otherwise known.
|
|||||||
<tr>
|
<tr>
|
||||||
<td>Architizer</td>
|
<td>Architizer</td>
|
||||||
<td>https://architizer.com/</td>
|
<td>https://architizer.com/</td>
|
||||||
<td>Projects</td>
|
<td>Firms, Projects</td>
|
||||||
<td></td>
|
<td></td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr>
|
<tr>
|
||||||
|
|||||||
@@ -8,7 +8,7 @@
|
|||||||
|
|
||||||
"""Extractors for https://architizer.com/"""
|
"""Extractors for https://architizer.com/"""
|
||||||
|
|
||||||
from .common import GalleryExtractor
|
from .common import GalleryExtractor, Extractor, Message
|
||||||
from .. import text
|
from .. import text
|
||||||
|
|
||||||
|
|
||||||
@@ -72,3 +72,30 @@ class ArchitizerProjectExtractor(GalleryExtractor):
|
|||||||
for url in text.extract_iter(
|
for url in text.extract_iter(
|
||||||
page, "property='og:image:secure_url' content='", "?")
|
page, "property='og:image:secure_url' content='", "?")
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
class ArchitizerFirmExtractor(Extractor):
|
||||||
|
"""Extractor for all projects of a firm"""
|
||||||
|
category = "architizer"
|
||||||
|
subcategory = "firm"
|
||||||
|
root = "https://architizer.com"
|
||||||
|
pattern = r"(?:https?://)?architizer\.com/firms/([^/?#]+)"
|
||||||
|
test = ("https://architizer.com/firms/olson-kundig/", {
|
||||||
|
"pattern": ArchitizerProjectExtractor.pattern,
|
||||||
|
"count": ">= 90",
|
||||||
|
})
|
||||||
|
|
||||||
|
def __init__(self, match):
|
||||||
|
Extractor.__init__(self, match)
|
||||||
|
self.firm = match.group(1)
|
||||||
|
|
||||||
|
def items(self):
|
||||||
|
url = url = "{}/firms/{}/?requesting_merlin=pages".format(
|
||||||
|
self.root, self.firm)
|
||||||
|
page = self.request(url).text
|
||||||
|
data = {"_extractor": ArchitizerProjectExtractor}
|
||||||
|
|
||||||
|
for project in text.extract_iter(page, '<a href="/projects/', '"'):
|
||||||
|
if not project.startswith("q/"):
|
||||||
|
url = "{}/projects/{}".format(self.root, project)
|
||||||
|
yield Message.Queue, url, data
|
||||||
|
|||||||
Reference in New Issue
Block a user