Merge pull request #7 from endoflife-date/distrowatch

[distrowatch] New Source

.github/workflows/update.yml

@@ -30,7 +30,17 @@ jobs:
       with:
         ref: ${{ github.head_ref }}
     - uses: actions/checkout@v3
-      name: Clone website
+      id: clone_same_branch
+      name: Clone website (Same Branch)
+      continue-on-error: true
+      with:
+        repository: endoflife-date/endoflife.date
+        path: website
+        submodules: false
+        ref: ${{github.ref_name}}
+    - uses: actions/checkout@v3
+      name: Clone website (Main)
+      if: steps.clone_same_branch.outcome != 'success'
       with:
         repository: endoflife-date/endoflife.date
         path: website
@@ -39,15 +49,15 @@ jobs:
       with:
         ruby-version: 3.1
         bundler-cache: true
-    - name: Update Release data
+    - name: Setup Release Script
       run: |
         git config --global init.defaultBranch main
         git config --global extensions.partialClone true
-
         pip install -r requirements.txt
-        for i in src/*.py; do python $i;done
-
-        bundle exec ruby update.rb ./website ~/.cache releases
+    - name: Custom Updates
+      run: for i in src/*.py; do python $i;done
+    - name: Git and DockerHub Updates
+      run: bundle exec ruby update.rb ./website ~/.cache releases
       id: update_releases
     - uses: stefanzweifel/git-auto-commit-action@v4
       name: Commit and update new releases

releases/almalinux.json

@@ -1,6 +1,7 @@
 {
-  "9.0": "2022-07-07",
-  "8.6": "2022-07-07",
-  "8.5": "2022-03-10",
-  "8.4": "2021-09-17"
+  "9.0": "2022-05-26",
+  "8.6": "2022-05-12",
+  "8.5": "2021-11-12",
+  "8.4": "2021-05-26",
+  "8.3": "2021-03-30"
 }

releases/debian.json

@@ -0,0 +1,33 @@
+{
+  "11": "2021-08-14",
+  "10": "2019-07-07",
+  "9": "2017-06-18",
+  "8.0": "2015-04-26",
+  "7.0": "2013-05-05",
+  "6.0": "2011-02-06",
+  "5.0": "2009-02-15",
+  "4.0.5": "2008-10-24",
+  "4.0.4": "2008-07-27",
+  "4.0.3": "2008-02-17",
+  "3.1.7": "2007-12-29",
+  "4.0.2": "2007-12-27",
+  "4.0.1": "2007-08-16",
+  "4.0": "2007-04-08",
+  "3.1.6": "2007-04-07",
+  "3.1.5": "2007-02-18",
+  "3.1.4": "2006-11-06",
+  "3.1.3": "2006-09-01",
+  "3.1.2": "2006-04-19",
+  "3.1.1": "2005-12-20",
+  "3.1": "2005-06-06",
+  "3.0.6": "2005-06-02",
+  "3.0.5": "2005-04-17",
+  "3.0.4": "2005-01-01",
+  "3.0.3": "2004-10-26",
+  "3.0.2": "2003-11-23",
+  "3.0.1": "2002-12-15",
+  "2.2.7": "2002-07-12",
+  "2.2.6": "2002-04-03",
+  "2.2.5": "2002-01-11",
+  "2.2.4": "2001-11-05"
+}

releases/oraclelinux.json

@@ -0,0 +1,32 @@
+{
+  "9.0": "2022-07-06",
+  "8.6": "2022-05-20",
+  "8.4": "2021-05-27",
+  "8.3": "2020-11-13",
+  "7.9": "2020-10-08",
+  "8.2": "2020-06-21",
+  "8.1": "2019-11-17",
+  "8.0": "2019-07-19",
+  "7.6": "2018-11-07",
+  "7.5": "2018-04-18",
+  "7.4": "2017-09-09",
+  "6.9": "2017-03-28",
+  "7.3": "2016-11-11",
+  "7.2": "2015-11-26",
+  "6.7": "2015-07-31",
+  "7.1": "2015-03-17",
+  "6.6": "2014-10-23",
+  "5.11": "2014-09-26",
+  "7.0": "2014-07-23",
+  "6.5": "2013-12-01",
+  "6.3": "2012-06-29",
+  "5.8": "2012-03-03",
+  "6.2": "2011-12-20",
+  "5.7": "2011-08-02",
+  "6.1": "2011-06-01",
+  "6": "2011-02-12",
+  "5.6": "2011-01-23",
+  "5.5": "2010-04-14",
+  "5.4": "2009-09-16",
+  "4.4": "2006-10-26"
+}

releases/rabbitmq.json

@@ -93,6 +93,7 @@
   "3.10.3": "2022-05-31",
   "3.10.4": "2022-05-31",
   "3.10.5": "2022-06-01",
+  "3.10.6": "2022-07-10",
   "3.7.0": "2017-11-28",
   "3.7.1": "2017-12-15",
   "3.7.10": "2019-01-07",

releases/rockylinux.json

@@ -0,0 +1,5 @@
+{
+  "8.6": "2022-05-17",
+  "8.5": "2021-11-16",
+  "8.4": "2021-06-21"
+}

releases/ubuntu.json

@@ -0,0 +1,76 @@
+{
+  "22.04": "2022-04-21",
+  "20.04.4": "2022-02-25",
+  "21.10": "2021-10-14",
+  "18.04.6": "2021-09-17",
+  "20.04.3": "2021-08-26",
+  "21.04": "2021-04-22",
+  "20.04.2": "2021-02-04",
+  "20.10": "2020-10-22",
+  "16.04.7": "2020-08-13",
+  "18.04.5": "2020-08-13",
+  "20.04.1": "2020-08-06",
+  "20.04": "2020-04-23",
+  "18.04.4": "2020-02-12",
+  "19.10": "2019-10-17",
+  "18.04.3": "2019-08-08",
+  "19.04": "2019-04-18",
+  "14.04.6": "2019-03-07",
+  "16.04.6": "2019-02-28",
+  "18.04.2": "2019-02-15",
+  "18.10": "2018-10-18",
+  "16.04.5": "2018-08-02",
+  "18.04.1": "2018-07-26",
+  "18.04": "2018-04-26",
+  "16.04.4": "2018-03-02",
+  "17.10": "2017-10-19",
+  "16.04.3": "2017-08-03",
+  "17.04": "2017-04-13",
+  "16.04.2": "2017-02-17",
+  "16.10": "2016-10-13",
+  "14.04.5": "2016-08-05",
+  "16.04.1": "2016-07-21",
+  "16.04": "2016-04-21",
+  "14.04.4": "2016-02-18",
+  "15.10": "2015-10-22",
+  "14.04.3": "2015-08-07",
+  "15.04": "2015-04-23",
+  "14.04.2": "2015-02-20",
+  "14.10": "2014-10-23",
+  "12.04.5": "2014-08-08",
+  "14.04.1": "2014-07-25",
+  "14.04": "2014-04-17",
+  "12.04.4": "2014-02-06",
+  "13.10": "2013-10-17",
+  "12.04.3": "2013-08-24",
+  "13.04": "2013-04-25",
+  "12.04.2": "2013-02-14",
+  "12.10": "2012-10-18",
+  "12.04.1": "2012-08-24",
+  "12.04": "2012-04-26",
+  "10.04.4": "2012-02-16",
+  "11.10": "2011-10-13",
+  "10.04.3": "2011-07-22",
+  "11.04": "2011-04-28",
+  "10.04.2": "2011-02-18",
+  "10.10": "2010-10-10",
+  "10.04.1": "2010-08-18",
+  "10.04": "2010-04-29",
+  "8.04.4": "2010-01-29",
+  "9.10": "2009-10-29",
+  "8.04.3": "2009-07-16",
+  "9.04": "2009-04-23",
+  "8.04.2": "2009-01-22",
+  "8.10": "2008-10-30",
+  "8.04.1": "2008-07-03",
+  "8.04": "2008-04-24",
+  "6.06.2": "2008-01-22",
+  "7.10": "2007-10-18",
+  "7.04": "2007-04-19",
+  "6.10": "2006-10-26",
+  "6.06.1": "2006-08-10",
+  "6.06": "2006-06-01",
+  "5.10": "2005-10-13",
+  "5.04": "2005-04-08",
+  "4.10": "2004-10-20"
+}

requirements.txt

@@ -6,6 +6,9 @@ pycparser==2.21
 pygit2==1.9.2
 python-frontmatter==1.0.0
 PyYAML==5.4
+python-dateutil==2.8.2
+python-liquid==1.2.1
 six==1.16.0
 soupsieve==2.3.2.post1
+typing_extensions==4.3.0
 webencodings==0.5.1

src/distowatch.py

@@ -0,0 +1,61 @@
+from glob import glob
+import os
+import re
+import json
+import frontmatter
+import urllib.request
+from bs4 import BeautifulSoup
+from html.parser import HTMLParser
+from liquid import Template
+
+# Same as used in Ruby (update.rb)
+DEFAULT_TAG_TEMPLATE = "{{major}}{% if minor %}.{{minor}}{% if patch %}.{{patch}}{%endif%}{%endif%}"
+
+def get_versions_from_headline(regex, headline, template):
+  if not isinstance(regex, list):
+    regex = [regex]
+  for r in regex:
+    matches = re.match(r.strip(), headline)
+    if matches:
+        match_data = matches.groupdict()
+        version_string = template.render(**match_data)
+        return version_string.split("\n")
+
+  return {}
+
+def fetch_releases(distrowatch_id, regex, template):
+    releases = {}
+    l_template = Template(template)
+    url = "https://distrowatch.com/index.php?distribution=%s" % distrowatch_id
+    with urllib.request.urlopen(url, data=None, timeout=5) as response:
+        soup = BeautifulSoup(response, features="html5lib")
+        for table in soup.select("td.News1>table.News"):
+            headline = table.select_one("td.NewsHeadline a[href]").get_text().strip()
+            date = table.select_one("td.NewsDate").get_text()
+            for v in get_versions_from_headline(regex, headline, l_template):
+              print("%s: %s" % (v, date))
+              releases[v] = date
+    return releases
+
+def update_releases():
+  for product_file in glob("website/products/*.md"):
+      product_name = os.path.splitext(os.path.basename(product_file))[0]
+      with open(product_file, "r") as f:
+          data = frontmatter.load(f)
+          if "auto" in data:
+              for config in data["auto"]:
+                  for key, d_id in config.items():
+                      if key == "distrowatch":
+                          update_product(product_name, config)
+
+def update_product(product_name, config):
+  t = config.get("template", DEFAULT_TAG_TEMPLATE)
+  if "regex" in config:
+      print("::group::%s" % product_name)
+      r = fetch_releases(config['distrowatch'], config["regex"], t)
+      with open("releases/%s.json" % product_name, "w") as f:
+          f.write(json.dumps(r, indent=2))
+      print("::endgroup::")
+
+if __name__ == "__main__":
+  update_releases()

update.rb

@@ -17,7 +17,7 @@ OPTIONAL_PRODUCT = ARGV[3]
 # This is more lenient from semver, but disallows MAJOR=0 as well
 # It also allows MAJOR.MINOR, which is quite common
 DEFAULT_VERSION_REGEX = '^v?(?<major>[1-9]\d*)\.(?<minor>0|[1-9]\d*)\.?(?<patch>0|[1-9]\d*)?$'
-DEFAULT_TAG_TEMPLATE = '{{major}}.{{minor}}{% if patch %}.{{patch}}{%endif%}'
+DEFAULT_TAG_TEMPLATE = "{{major}}{% if minor %}.{{minor}}{% if patch %}.{{patch}}{%endif%}{%endif%}"
 
 def fetch_git_releases(repo_dir, config)
   pwd = Dir.pwd