Refactor HTTP common code (#207)

- move to common/http.py,
- make fetch_url return a Response.
This commit is contained in:
Marc Wrobel
2023-12-02 22:52:12 +01:00
committed by GitHub
parent aa3d15a3b5
commit 7cf62c2355
36 changed files with 138 additions and 113 deletions

View File

@@ -1,5 +1,6 @@
import re
from bs4 import BeautifulSoup
from common import http
from common import dates
from common import endoflife
@@ -12,7 +13,6 @@ URLS = [
"https://support.apple.com/kb/HT205759", # 2013
"https://support.apple.com/kb/HT204611", # 2011 to 2012
# Apple still links to the following articles, but they are 404:
# Disabled, too much timed out.
"http://web.archive.org/web/20230404214605_/https://support.apple.com/en-us/HT5165", # 2010
"http://web.archive.org/web/20230327200842_/https://support.apple.com/en-us/HT4218", # 2008-2009
"http://web.archive.org/web/20230204234533_/https://support.apple.com/en-us/HT1263", # 2005-2007
@@ -54,7 +54,7 @@ def parse_date(date_str):
print("::group::apple")
versions_by_product = {k: {} for k in CONFIG.keys()}
for response in endoflife.fetch_urls(URLS):
for response in http.fetch_urls(URLS):
soup = BeautifulSoup(response.text, features="html5lib")
versions_table = soup.find(id="tableWraper")
versions_table = versions_table if versions_table else soup.find('table', class_="gb-table")