diff --git a/releases/apache-airflow.json b/releases/apache-airflow.json new file mode 100644 index 00000000..ee287caf --- /dev/null +++ b/releases/apache-airflow.json @@ -0,0 +1,42 @@ +{ + "1.10.0": "2018-08-27", + "1.10.1": "2018-11-21", + "1.10.10": "2020-04-09", + "1.10.11": "2020-07-10", + "1.10.12": "2020-08-25", + "1.10.13": "2020-11-25", + "1.10.14": "2020-12-10", + "1.10.15": "2021-03-17", + "1.10.2": "2019-01-23", + "1.10.3": "2019-04-10", + "1.10.4": "2019-08-06", + "1.10.5": "2019-09-04", + "1.10.6": "2019-10-28", + "1.10.7": "2019-12-24", + "1.10.8": "2020-02-07", + "1.10.9": "2020-02-07", + "1.8.1": "2017-05-09", + "1.8.2": "2017-09-04", + "1.9.0": "2018-01-02", + "2.0.0": "2020-12-17", + "2.0.1": "2021-02-08", + "2.0.2": "2021-04-19", + "2.1.0": "2021-05-21", + "2.1.1": "2021-07-02", + "2.1.2": "2021-07-14", + "2.1.3": "2021-08-23", + "2.1.4": "2021-09-18", + "2.2.0": "2021-10-11", + "2.2.1": "2021-10-29", + "2.2.2": "2021-11-15", + "2.2.3": "2021-12-21", + "2.2.4": "2022-02-22", + "2.2.5": "2022-04-04", + "2.3.0": "2022-04-30", + "2.3.1": "2022-05-25", + "2.3.2": "2022-06-04", + "2.3.3": "2022-07-09", + "2.3.4": "2022-08-23", + "2.4.0": "2022-09-19", + "2.4.1": "2022-09-30" +} \ No newline at end of file diff --git a/src/pypi.py b/src/pypi.py new file mode 100644 index 00000000..4a858926 --- /dev/null +++ b/src/pypi.py @@ -0,0 +1,65 @@ +from glob import glob +import os +import re +import sys +import json +import frontmatter +import urllib.request +from bs4 import BeautifulSoup +from datetime import datetime +from html.parser import HTMLParser +from liquid import Template + +# Same as used in Ruby (update.rb) +DEFAULT_TAG_TEMPLATE = "{{major}}{% if minor %}.{{minor}}{% if patch %}.{{patch}}{%endif%}{%endif%}" +REGEX = r'^(?:(\d+\.(?:\d+\.)*\d+))$' + +def fetch_releases(pypi_id, regex): + releases = {} + + if not isinstance(regex, list): + regex = [regex] + + url = "https://pypi.org/pypi/%s/json" % pypi_id + with urllib.request.urlopen(url, data=None, timeout=5) as response: + data = json.loads(response.read().decode("utf-8")) + for version in data['releases']: + R = data['releases'][version] + matches = False + for r in regex: + if re.match(r, version): + matches = True + if matches: + d = datetime.fromisoformat(R[0]['upload_time']).strftime("%Y-%m-%d") + releases[version] = d + print("%s: %s" % (version, d)) + + return releases + +def update_releases(product_filter=None): + for product_file in glob("website/products/*.md"): + product_name = os.path.splitext(os.path.basename(product_file))[0] + if product_filter and product_name != product_filter: + continue + with open(product_file, "r") as f: + data = frontmatter.load(f) + if "auto" in data: + for config in data["auto"]: + for key, d_id in config.items(): + if key == "pypi": + update_product(product_name, config) + +def update_product(product_name, config): + if "pypi" in config: + print("::group::%s" % product_name) + config = config | {"regex": REGEX} + r = fetch_releases(config['pypi'], config["regex"]) + with open("releases/%s.json" % product_name, "w") as f: + f.write(json.dumps(r, indent=2)) + print("::endgroup::") + +if __name__ == "__main__": + if len(sys.argv) > 1: + update_releases(sys.argv[1]) + else: + update_releases()