github/endoflife-date-release-data
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[release_table] Support non-headless mode (#552)

Browse Source 
This may be useful for running scripts manually.
This commit is contained in:
Marc Wrobel
2026-02-01 11:10:15 +01:00
committed by GitHub
parent 23205b23ea
commit b9eafb3fd7
2 changed files with 8 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/common/http.py
View File

@@ -82,11 +82,11 @@ def fetch_markdown(url: str, data: any = None, user_agent: str = ENDOFLIFE_BOT_U
return mwparserfromhell.parse(response.text)
# This requires some setup, see https://playwright.dev/python/docs/intro#installing-playwright.
def fetch_javascript_url(url: str, user_agent: str = ENDOFLIFE_BOT_USER_AGENT, wait_until: str = None, wait_for: str = None,
select_wait_for: bool = False, click_selector: str = None) -> str:
def fetch_javascript_url(url: str, user_agent: str = ENDOFLIFE_BOT_USER_AGENT, headless: bool = True, wait_until: str = None,
wait_for: str = None, select_wait_for: bool = False, click_selector: str = None) -> str:
logging.info(f"Fetching {url} with JavaScript (wait_until = {wait_until}, wait_for = {wait_for}, select_wait_for = {select_wait_for}, click_selector = {click_selector})")
with sync_playwright() as p:
browser = p.chromium.launch()
browser = p.chromium.launch(headless=headless)
context = browser.new_context()
context.set_extra_http_headers({'User-Agent': user_agent})

7
src/release_table.py
View File

@@ -19,6 +19,7 @@ necessary information. Available configuration options are:
- user_agent (optional, default = <endoflife.date-bot User-Agent>): A user agent string to use when fetching the page.
Unused when render_javascript is true.
- render_javascript (optional, default = false): A boolean value indicating whether to render JavaScript on the page.
- render_javascript_headless (optional, default = true): Indicates whether to run the browser in headless mode.
- render_javascript_wait_for (optional, default = None): Wait until the given selector appear on the page. Only use when
render_javascript is true.
- render_javascript_wait_until (optional, default = None): Argument to pass to Playwright, one of "commit",
@@ -161,6 +162,7 @@ with ProductData(config.product) as product_data:
render_js_wait_until: str | None = config.data.get("render_javascript_wait_until", None)
render_js_wait_for: str | None = config.data.get("render_javascript_wait_for", None)
render_js_click_selector: str | None = config.data.get("render_javascript_click_selector", None)
render_js_headless: str | None = config.data.get("render_javascript_headless", None)
table_selector: str = config.data.get("selector", "table")
header_row_selector: str = config.data.get("header_selector", "thead tr")
@@ -173,8 +175,9 @@ with ProductData(config.product) as product_data:
fields = [Field(name, definition) for name, definition in config.data["fields"].items()]
if render_js:
response_text = http.fetch_javascript_url(config.url, user_agent=user_agent, wait_until=render_js_wait_until,
wait_for=render_js_wait_for, click_selector=render_js_click_selector)
response_text = http.fetch_javascript_url(config.url, user_agent=user_agent, headless=render_js_headless,
wait_until=render_js_wait_until, wait_for=render_js_wait_for,
click_selector=render_js_click_selector)
else:
response_text = http.fetch_url(config.url, user_agent=user_agent).text
soup = BeautifulSoup(response_text, features="html5lib")