Refactor HTTP common code (#207)

- move to common/http.py,
- make fetch_url return a Response.
This commit is contained in:
Marc Wrobel
2023-12-02 22:52:12 +01:00
committed by GitHub
parent aa3d15a3b5
commit 7cf62c2355
36 changed files with 138 additions and 113 deletions

View File

@@ -1,6 +1,7 @@
import re
import sys
from bs4 import BeautifulSoup
from common import http
from common import endoflife
from liquid import Template
@@ -27,8 +28,8 @@ def fetch_releases(distrowatch_id, regex, template):
releases = {}
l_template = Template(template)
url = f"https://distrowatch.com/index.php?distribution={distrowatch_id}"
response = endoflife.fetch_url(url)
soup = BeautifulSoup(response, features="html5lib")
response = http.fetch_url(url)
soup = BeautifulSoup(response.text, features="html5lib")
for table in soup.select("td.News1>table.News"):
headline = table.select_one("td.NewsHeadline a[href]").get_text().strip()
date = table.select_one("td.NewsDate").get_text()