github/endoflife-date-release-data
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[release-table] Implement render_js_click_selector parameter (#485)

Browse Source 
This parameter allows clicking on an element of the page after it was loaded when using `render_javascript`.
This commit is contained in:
Marc Wrobel
2025-08-03 21:43:16 +02:00
committed by GitHub
parent 6ab864d474
commit 1735954764
3 changed files with 18 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
src/common/http.py
View File

@@ -82,8 +82,9 @@ def fetch_markdown(url: str, data: any = None, user_agent: str = ENDOFLIFE_BOT_U
return mwparserfromhell.parse(response.text)
# This requires some setup, see https://playwright.dev/python/docs/intro#installing-playwright.
def fetch_javascript_url(url: str, user_agent: str = ENDOFLIFE_BOT_USER_AGENT, wait_until: str = None, wait_for: str = None, select_wait_for: bool = False) -> str:
logging.info(f"Fetching {url} with JavaScript (wait_until = {wait_until}, wait_for = {wait_for}, select_wait_for = {select_wait_for})")
def fetch_javascript_url(url: str, user_agent: str = ENDOFLIFE_BOT_USER_AGENT, wait_until: str = None, wait_for: str = None,
select_wait_for: bool = False, click_selector: str = None) -> str:
logging.info(f"Fetching {url} with JavaScript (wait_until = {wait_until}, wait_for = {wait_for}, select_wait_for = {select_wait_for}, click_selector = {click_selector})")
with sync_playwright() as p:
browser = p.chromium.launch()
context = browser.new_context()
@@ -94,22 +95,16 @@ def fetch_javascript_url(url: str, user_agent: str = ENDOFLIFE_BOT_USER_AGENT, w
page.goto(url, wait_until=wait_until)
logging.info(f"Fetched {url}")
element_to_wait_for = None
if wait_for:
try:
logging.info(f"Waiting for element with selector {wait_for}")
element = page.wait_for_selector(selector=wait_for)
logging.info(f"Waiting for element with selector {wait_for}")
element_to_wait_for = page.wait_for_selector(selector=wait_for)
if element:
logging.debug(f"Found element with selector {wait_for} on {url}")
return element.inner_html() if select_wait_for else page.content()
if click_selector:
logging.info(f"Clicking on element with selector {click_selector}")
page.click(selector=click_selector)
page.wait_for_timeout(1000) # Wait for 1 second to allow the page to update after the click.
logging.error(f"No element found with selector {wait_for} on {url}, will return full page content")
logging.debug(f"Full page content: {page.content()}")
except Exception as e: # noqa: BLE001
logging.error(f"Error while waiting for element with selector {wait_for} on {url}: {e}")
logging.debug(f"Full page content: {page.content()}")
return page.content()
return element_to_wait_for.inner_html() if select_wait_for else page.content()
finally:
browser.close()