PyPi Automation

2022-10-08 19:31:51 +05:30
parent 79805be884
commit f8af1626c9
2 changed files with 107 additions and 0 deletions
--- a/releases/apache-airflow.json
+++ b/releases/apache-airflow.json
@@ -0,0 +1,42 @@
+{
+  "1.10.0": "2018-08-27",
+  "1.10.1": "2018-11-21",
+  "1.10.10": "2020-04-09",
+  "1.10.11": "2020-07-10",
+  "1.10.12": "2020-08-25",
+  "1.10.13": "2020-11-25",
+  "1.10.14": "2020-12-10",
+  "1.10.15": "2021-03-17",
+  "1.10.2": "2019-01-23",
+  "1.10.3": "2019-04-10",
+  "1.10.4": "2019-08-06",
+  "1.10.5": "2019-09-04",
+  "1.10.6": "2019-10-28",
+  "1.10.7": "2019-12-24",
+  "1.10.8": "2020-02-07",
+  "1.10.9": "2020-02-07",
+  "1.8.1": "2017-05-09",
+  "1.8.2": "2017-09-04",
+  "1.9.0": "2018-01-02",
+  "2.0.0": "2020-12-17",
+  "2.0.1": "2021-02-08",
+  "2.0.2": "2021-04-19",
+  "2.1.0": "2021-05-21",
+  "2.1.1": "2021-07-02",
+  "2.1.2": "2021-07-14",
+  "2.1.3": "2021-08-23",
+  "2.1.4": "2021-09-18",
+  "2.2.0": "2021-10-11",
+  "2.2.1": "2021-10-29",
+  "2.2.2": "2021-11-15",
+  "2.2.3": "2021-12-21",
+  "2.2.4": "2022-02-22",
+  "2.2.5": "2022-04-04",
+  "2.3.0": "2022-04-30",
+  "2.3.1": "2022-05-25",
+  "2.3.2": "2022-06-04",
+  "2.3.3": "2022-07-09",
+  "2.3.4": "2022-08-23",
+  "2.4.0": "2022-09-19",
+  "2.4.1": "2022-09-30"
+}
--- a/src/pypi.py
+++ b/src/pypi.py
@@ -0,0 +1,65 @@
+from glob import glob
+import os
+import re
+import sys
+import json
+import frontmatter
+import urllib.request
+from bs4 import BeautifulSoup
+from datetime import datetime
+from html.parser import HTMLParser
+from liquid import Template
+
+# Same as used in Ruby (update.rb)
+DEFAULT_TAG_TEMPLATE = "{{major}}{% if minor %}.{{minor}}{% if patch %}.{{patch}}{%endif%}{%endif%}"
+REGEX = r'^(?:(\d+\.(?:\d+\.)*\d+))$'
+
+def fetch_releases(pypi_id, regex):
+    releases = {}
+
+    if not isinstance(regex, list):
+      regex = [regex]
+
+    url = "https://pypi.org/pypi/%s/json" % pypi_id
+    with urllib.request.urlopen(url, data=None, timeout=5) as response:
+      data = json.loads(response.read().decode("utf-8"))
+      for version in data['releases']:
+        R = data['releases'][version]
+        matches = False
+        for r in regex:
+          if re.match(r, version):
+            matches = True
+        if matches:
+          d = datetime.fromisoformat(R[0]['upload_time']).strftime("%Y-%m-%d")
+          releases[version] = d
+          print("%s: %s" % (version, d))
+
+    return releases
+
+def update_releases(product_filter=None):
+  for product_file in glob("website/products/*.md"):
+      product_name = os.path.splitext(os.path.basename(product_file))[0]
+      if product_filter and product_name != product_filter:
+        continue
+      with open(product_file, "r") as f:
+          data = frontmatter.load(f)
+          if "auto" in data:
+              for config in data["auto"]:
+                  for key, d_id in config.items():
+                      if key == "pypi":
+                          update_product(product_name, config)
+
+def update_product(product_name, config):
+  if "pypi" in config:
+      print("::group::%s" % product_name)
+      config = config | {"regex": REGEX}
+      r = fetch_releases(config['pypi'], config["regex"])
+      with open("releases/%s.json" % product_name, "w") as f:
+          f.write(json.dumps(r, indent=2))
+      print("::endgroup::")
+
+if __name__ == "__main__":
+  if len(sys.argv) > 1:
+    update_releases(sys.argv[1])
+  else:
+    update_releases()