diff --git a/.github/workflows/update.yml b/.github/workflows/update.yml index 7c77a045..e8c60585 100644 --- a/.github/workflows/update.yml +++ b/.github/workflows/update.yml @@ -30,7 +30,17 @@ jobs: with: ref: ${{ github.head_ref }} - uses: actions/checkout@v3 - name: Clone website + id: clone_same_branch + name: Clone website (Same Branch) + continue-on-error: true + with: + repository: endoflife-date/endoflife.date + path: website + submodules: false + ref: ${{github.ref_name}} + - uses: actions/checkout@v3 + name: Clone website (Main) + if: steps.clone_same_branch.outcome != 'success' with: repository: endoflife-date/endoflife.date path: website @@ -39,15 +49,15 @@ jobs: with: ruby-version: 3.1 bundler-cache: true - - name: Update Release data + - name: Setup Release Script run: | git config --global init.defaultBranch main git config --global extensions.partialClone true - pip install -r requirements.txt - for i in src/*.py; do python $i;done - - bundle exec ruby update.rb ./website ~/.cache releases + - name: Custom Updates + run: for i in src/*.py; do python $i;done + - name: Git and DockerHub Updates + run: bundle exec ruby update.rb ./website ~/.cache releases id: update_releases - uses: stefanzweifel/git-auto-commit-action@v4 name: Commit and update new releases diff --git a/releases/almalinux.json b/releases/almalinux.json index 3fd24b9e..4f4c5d69 100644 --- a/releases/almalinux.json +++ b/releases/almalinux.json @@ -1,6 +1,7 @@ { - "9.0": "2022-07-07", - "8.6": "2022-07-07", - "8.5": "2022-03-10", - "8.4": "2021-09-17" + "9.0": "2022-05-26", + "8.6": "2022-05-12", + "8.5": "2021-11-12", + "8.4": "2021-05-26", + "8.3": "2021-03-30" } \ No newline at end of file diff --git a/releases/debian.json b/releases/debian.json new file mode 100644 index 00000000..b8226b6a --- /dev/null +++ b/releases/debian.json @@ -0,0 +1,33 @@ +{ + "11": "2021-08-14", + "10": "2019-07-07", + "9": "2017-06-18", + "8.0": "2015-04-26", + "7.0": "2013-05-05", + "6.0": "2011-02-06", + "5.0": "2009-02-15", + "4.0.5": "2008-10-24", + "4.0.4": "2008-07-27", + "4.0.3": "2008-02-17", + "3.1.7": "2007-12-29", + "4.0.2": "2007-12-27", + "4.0.1": "2007-08-16", + "4.0": "2007-04-08", + "3.1.6": "2007-04-07", + "3.1.5": "2007-02-18", + "3.1.4": "2006-11-06", + "3.1.3": "2006-09-01", + "3.1.2": "2006-04-19", + "3.1.1": "2005-12-20", + "3.1": "2005-06-06", + "3.0.6": "2005-06-02", + "3.0.5": "2005-04-17", + "3.0.4": "2005-01-01", + "3.0.3": "2004-10-26", + "3.0.2": "2003-11-23", + "3.0.1": "2002-12-15", + "2.2.7": "2002-07-12", + "2.2.6": "2002-04-03", + "2.2.5": "2002-01-11", + "2.2.4": "2001-11-05" +} \ No newline at end of file diff --git a/releases/oraclelinux.json b/releases/oraclelinux.json new file mode 100644 index 00000000..ee3ac68d --- /dev/null +++ b/releases/oraclelinux.json @@ -0,0 +1,32 @@ +{ + "9.0": "2022-07-06", + "8.6": "2022-05-20", + "8.4": "2021-05-27", + "8.3": "2020-11-13", + "7.9": "2020-10-08", + "8.2": "2020-06-21", + "8.1": "2019-11-17", + "8.0": "2019-07-19", + "7.6": "2018-11-07", + "7.5": "2018-04-18", + "7.4": "2017-09-09", + "6.9": "2017-03-28", + "7.3": "2016-11-11", + "7.2": "2015-11-26", + "6.7": "2015-07-31", + "7.1": "2015-03-17", + "6.6": "2014-10-23", + "5.11": "2014-09-26", + "7.0": "2014-07-23", + "6.5": "2013-12-01", + "6.3": "2012-06-29", + "5.8": "2012-03-03", + "6.2": "2011-12-20", + "5.7": "2011-08-02", + "6.1": "2011-06-01", + "6": "2011-02-12", + "5.6": "2011-01-23", + "5.5": "2010-04-14", + "5.4": "2009-09-16", + "4.4": "2006-10-26" +} \ No newline at end of file diff --git a/releases/rabbitmq.json b/releases/rabbitmq.json index 26e1c0ef..07f268ab 100644 --- a/releases/rabbitmq.json +++ b/releases/rabbitmq.json @@ -93,6 +93,7 @@ "3.10.3": "2022-05-31", "3.10.4": "2022-05-31", "3.10.5": "2022-06-01", + "3.10.6": "2022-07-10", "3.7.0": "2017-11-28", "3.7.1": "2017-12-15", "3.7.10": "2019-01-07", diff --git a/releases/rockylinux.json b/releases/rockylinux.json new file mode 100644 index 00000000..ce53b350 --- /dev/null +++ b/releases/rockylinux.json @@ -0,0 +1,5 @@ +{ + "8.6": "2022-05-17", + "8.5": "2021-11-16", + "8.4": "2021-06-21" +} \ No newline at end of file diff --git a/releases/ubuntu.json b/releases/ubuntu.json new file mode 100644 index 00000000..97792cef --- /dev/null +++ b/releases/ubuntu.json @@ -0,0 +1,76 @@ +{ + "22.04": "2022-04-21", + "20.04.4": "2022-02-25", + "21.10": "2021-10-14", + "18.04.6": "2021-09-17", + "20.04.3": "2021-08-26", + "21.04": "2021-04-22", + "20.04.2": "2021-02-04", + "20.10": "2020-10-22", + "16.04.7": "2020-08-13", + "18.04.5": "2020-08-13", + "20.04.1": "2020-08-06", + "20.04": "2020-04-23", + "18.04.4": "2020-02-12", + "19.10": "2019-10-17", + "18.04.3": "2019-08-08", + "19.04": "2019-04-18", + "14.04.6": "2019-03-07", + "16.04.6": "2019-02-28", + "18.04.2": "2019-02-15", + "18.10": "2018-10-18", + "16.04.5": "2018-08-02", + "18.04.1": "2018-07-26", + "18.04": "2018-04-26", + "16.04.4": "2018-03-02", + "17.10": "2017-10-19", + "16.04.3": "2017-08-03", + "17.04": "2017-04-13", + "16.04.2": "2017-02-17", + "16.10": "2016-10-13", + "14.04.5": "2016-08-05", + "16.04.1": "2016-07-21", + "16.04": "2016-04-21", + "14.04.4": "2016-02-18", + "15.10": "2015-10-22", + "14.04.3": "2015-08-07", + "15.04": "2015-04-23", + "14.04.2": "2015-02-20", + "14.10": "2014-10-23", + "12.04.5": "2014-08-08", + "14.04.1": "2014-07-25", + "14.04": "2014-04-17", + "12.04.4": "2014-02-06", + "13.10": "2013-10-17", + "12.04.3": "2013-08-24", + "13.04": "2013-04-25", + "12.04.2": "2013-02-14", + "12.10": "2012-10-18", + "12.04.1": "2012-08-24", + "12.04": "2012-04-26", + "10.04.4": "2012-02-16", + "11.10": "2011-10-13", + "10.04.3": "2011-07-22", + "11.04": "2011-04-28", + "10.04.2": "2011-02-18", + "10.10": "2010-10-10", + "10.04.1": "2010-08-18", + "10.04": "2010-04-29", + "8.04.4": "2010-01-29", + "9.10": "2009-10-29", + "8.04.3": "2009-07-16", + "9.04": "2009-04-23", + "8.04.2": "2009-01-22", + "8.10": "2008-10-30", + "8.04.1": "2008-07-03", + "8.04": "2008-04-24", + "6.06.2": "2008-01-22", + "7.10": "2007-10-18", + "7.04": "2007-04-19", + "6.10": "2006-10-26", + "6.06.1": "2006-08-10", + "6.06": "2006-06-01", + "5.10": "2005-10-13", + "5.04": "2005-04-08", + "4.10": "2004-10-20" +} \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 40759400..5c41018a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,6 +6,9 @@ pycparser==2.21 pygit2==1.9.2 python-frontmatter==1.0.0 PyYAML==5.4 +python-dateutil==2.8.2 +python-liquid==1.2.1 six==1.16.0 soupsieve==2.3.2.post1 +typing_extensions==4.3.0 webencodings==0.5.1 diff --git a/src/distowatch.py b/src/distowatch.py new file mode 100644 index 00000000..3d2c570d --- /dev/null +++ b/src/distowatch.py @@ -0,0 +1,61 @@ +from glob import glob +import os +import re +import json +import frontmatter +import urllib.request +from bs4 import BeautifulSoup +from html.parser import HTMLParser +from liquid import Template + +# Same as used in Ruby (update.rb) +DEFAULT_TAG_TEMPLATE = "{{major}}{% if minor %}.{{minor}}{% if patch %}.{{patch}}{%endif%}{%endif%}" + +def get_versions_from_headline(regex, headline, template): + if not isinstance(regex, list): + regex = [regex] + for r in regex: + matches = re.match(r.strip(), headline) + if matches: + match_data = matches.groupdict() + version_string = template.render(**match_data) + return version_string.split("\n") + + return {} + +def fetch_releases(distrowatch_id, regex, template): + releases = {} + l_template = Template(template) + url = "https://distrowatch.com/index.php?distribution=%s" % distrowatch_id + with urllib.request.urlopen(url, data=None, timeout=5) as response: + soup = BeautifulSoup(response, features="html5lib") + for table in soup.select("td.News1>table.News"): + headline = table.select_one("td.NewsHeadline a[href]").get_text().strip() + date = table.select_one("td.NewsDate").get_text() + for v in get_versions_from_headline(regex, headline, l_template): + print("%s: %s" % (v, date)) + releases[v] = date + return releases + +def update_releases(): + for product_file in glob("website/products/*.md"): + product_name = os.path.splitext(os.path.basename(product_file))[0] + with open(product_file, "r") as f: + data = frontmatter.load(f) + if "auto" in data: + for config in data["auto"]: + for key, d_id in config.items(): + if key == "distrowatch": + update_product(product_name, config) + +def update_product(product_name, config): + t = config.get("template", DEFAULT_TAG_TEMPLATE) + if "regex" in config: + print("::group::%s" % product_name) + r = fetch_releases(config['distrowatch'], config["regex"], t) + with open("releases/%s.json" % product_name, "w") as f: + f.write(json.dumps(r, indent=2)) + print("::endgroup::") + +if __name__ == "__main__": + update_releases() diff --git a/update.rb b/update.rb index 4e55e9fc..3187c9f4 100644 --- a/update.rb +++ b/update.rb @@ -17,7 +17,7 @@ OPTIONAL_PRODUCT = ARGV[3] # This is more lenient from semver, but disallows MAJOR=0 as well # It also allows MAJOR.MINOR, which is quite common DEFAULT_VERSION_REGEX = '^v?(?[1-9]\d*)\.(?0|[1-9]\d*)\.?(?0|[1-9]\d*)?$' -DEFAULT_TAG_TEMPLATE = '{{major}}.{{minor}}{% if patch %}.{{patch}}{%endif%}' +DEFAULT_TAG_TEMPLATE = "{{major}}{% if minor %}.{{minor}}{% if patch %}.{{patch}}{%endif%}{%endif%}" def fetch_git_releases(repo_dir, config) pwd = Dir.pwd