Refactor HTTP URL fetching scripts
This creates a common function to fetch HTTP URLs, with enhanced capabilities (retry, use of a known User-Agent). It makes scripts that need those capabilities simpler, while improving other scripts. This commit also fixes some scripts that did not log properly (cos.py, eks.py, haproxy.py, palo-alto-networks.py, rhel.py, ros.py, unrealircd.py).
This commit is contained in:
@@ -1,16 +1,15 @@
|
||||
import json
|
||||
from datetime import datetime
|
||||
import urllib.request
|
||||
from bs4 import BeautifulSoup
|
||||
from common import endoflife
|
||||
from datetime import datetime
|
||||
|
||||
URL = "https://docs.plesk.com/release-notes/obsidian/change-log"
|
||||
PRODUCT = "plesk"
|
||||
|
||||
|
||||
def make_bs_request(url):
|
||||
req = urllib.request.Request(url)
|
||||
with urllib.request.urlopen(req, timeout=5) as response:
|
||||
return BeautifulSoup(response.read(), features="html5lib")
|
||||
response = endoflife.fetch_url(url)
|
||||
return BeautifulSoup(response, features="html5lib")
|
||||
|
||||
|
||||
# Only 18.0.20.3 and later will be picked up :
|
||||
|
||||
Reference in New Issue
Block a user