From df1466fe52c7f2d9c3990aee2e8968847a9af7b4 Mon Sep 17 00:00:00 2001 From: Nemo Date: Sun, 10 Jul 2022 15:20:03 +0530 Subject: [PATCH 1/6] [distrowatch] Release data from local runs --- releases/almalinux.json | 11 +++--- releases/debian.json | 33 +++++++++++++++++ releases/oraclelinux.json | 32 +++++++++++++++++ releases/rockylinux.json | 5 +++ releases/ubuntu.json | 76 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 152 insertions(+), 5 deletions(-) create mode 100644 releases/debian.json create mode 100644 releases/oraclelinux.json create mode 100644 releases/rockylinux.json create mode 100644 releases/ubuntu.json diff --git a/releases/almalinux.json b/releases/almalinux.json index 3fd24b9e..24dbda04 100644 --- a/releases/almalinux.json +++ b/releases/almalinux.json @@ -1,6 +1,7 @@ { - "9.0": "2022-07-07", - "8.6": "2022-07-07", - "8.5": "2022-03-10", - "8.4": "2021-09-17" -} \ No newline at end of file + "9.0": "2022-05-26", + "8.6": "2022-05-12", + "8.5": "2021-11-12", + "8.4": "2021-05-26", + "8.3": "2021-03-30" +} diff --git a/releases/debian.json b/releases/debian.json new file mode 100644 index 00000000..b8226b6a --- /dev/null +++ b/releases/debian.json @@ -0,0 +1,33 @@ +{ + "11": "2021-08-14", + "10": "2019-07-07", + "9": "2017-06-18", + "8.0": "2015-04-26", + "7.0": "2013-05-05", + "6.0": "2011-02-06", + "5.0": "2009-02-15", + "4.0.5": "2008-10-24", + "4.0.4": "2008-07-27", + "4.0.3": "2008-02-17", + "3.1.7": "2007-12-29", + "4.0.2": "2007-12-27", + "4.0.1": "2007-08-16", + "4.0": "2007-04-08", + "3.1.6": "2007-04-07", + "3.1.5": "2007-02-18", + "3.1.4": "2006-11-06", + "3.1.3": "2006-09-01", + "3.1.2": "2006-04-19", + "3.1.1": "2005-12-20", + "3.1": "2005-06-06", + "3.0.6": "2005-06-02", + "3.0.5": "2005-04-17", + "3.0.4": "2005-01-01", + "3.0.3": "2004-10-26", + "3.0.2": "2003-11-23", + "3.0.1": "2002-12-15", + "2.2.7": "2002-07-12", + "2.2.6": "2002-04-03", + "2.2.5": "2002-01-11", + "2.2.4": "2001-11-05" +} \ No newline at end of file diff --git a/releases/oraclelinux.json b/releases/oraclelinux.json new file mode 100644 index 00000000..ee3ac68d --- /dev/null +++ b/releases/oraclelinux.json @@ -0,0 +1,32 @@ +{ + "9.0": "2022-07-06", + "8.6": "2022-05-20", + "8.4": "2021-05-27", + "8.3": "2020-11-13", + "7.9": "2020-10-08", + "8.2": "2020-06-21", + "8.1": "2019-11-17", + "8.0": "2019-07-19", + "7.6": "2018-11-07", + "7.5": "2018-04-18", + "7.4": "2017-09-09", + "6.9": "2017-03-28", + "7.3": "2016-11-11", + "7.2": "2015-11-26", + "6.7": "2015-07-31", + "7.1": "2015-03-17", + "6.6": "2014-10-23", + "5.11": "2014-09-26", + "7.0": "2014-07-23", + "6.5": "2013-12-01", + "6.3": "2012-06-29", + "5.8": "2012-03-03", + "6.2": "2011-12-20", + "5.7": "2011-08-02", + "6.1": "2011-06-01", + "6": "2011-02-12", + "5.6": "2011-01-23", + "5.5": "2010-04-14", + "5.4": "2009-09-16", + "4.4": "2006-10-26" +} \ No newline at end of file diff --git a/releases/rockylinux.json b/releases/rockylinux.json new file mode 100644 index 00000000..ce53b350 --- /dev/null +++ b/releases/rockylinux.json @@ -0,0 +1,5 @@ +{ + "8.6": "2022-05-17", + "8.5": "2021-11-16", + "8.4": "2021-06-21" +} \ No newline at end of file diff --git a/releases/ubuntu.json b/releases/ubuntu.json new file mode 100644 index 00000000..97792cef --- /dev/null +++ b/releases/ubuntu.json @@ -0,0 +1,76 @@ +{ + "22.04": "2022-04-21", + "20.04.4": "2022-02-25", + "21.10": "2021-10-14", + "18.04.6": "2021-09-17", + "20.04.3": "2021-08-26", + "21.04": "2021-04-22", + "20.04.2": "2021-02-04", + "20.10": "2020-10-22", + "16.04.7": "2020-08-13", + "18.04.5": "2020-08-13", + "20.04.1": "2020-08-06", + "20.04": "2020-04-23", + "18.04.4": "2020-02-12", + "19.10": "2019-10-17", + "18.04.3": "2019-08-08", + "19.04": "2019-04-18", + "14.04.6": "2019-03-07", + "16.04.6": "2019-02-28", + "18.04.2": "2019-02-15", + "18.10": "2018-10-18", + "16.04.5": "2018-08-02", + "18.04.1": "2018-07-26", + "18.04": "2018-04-26", + "16.04.4": "2018-03-02", + "17.10": "2017-10-19", + "16.04.3": "2017-08-03", + "17.04": "2017-04-13", + "16.04.2": "2017-02-17", + "16.10": "2016-10-13", + "14.04.5": "2016-08-05", + "16.04.1": "2016-07-21", + "16.04": "2016-04-21", + "14.04.4": "2016-02-18", + "15.10": "2015-10-22", + "14.04.3": "2015-08-07", + "15.04": "2015-04-23", + "14.04.2": "2015-02-20", + "14.10": "2014-10-23", + "12.04.5": "2014-08-08", + "14.04.1": "2014-07-25", + "14.04": "2014-04-17", + "12.04.4": "2014-02-06", + "13.10": "2013-10-17", + "12.04.3": "2013-08-24", + "13.04": "2013-04-25", + "12.04.2": "2013-02-14", + "12.10": "2012-10-18", + "12.04.1": "2012-08-24", + "12.04": "2012-04-26", + "10.04.4": "2012-02-16", + "11.10": "2011-10-13", + "10.04.3": "2011-07-22", + "11.04": "2011-04-28", + "10.04.2": "2011-02-18", + "10.10": "2010-10-10", + "10.04.1": "2010-08-18", + "10.04": "2010-04-29", + "8.04.4": "2010-01-29", + "9.10": "2009-10-29", + "8.04.3": "2009-07-16", + "9.04": "2009-04-23", + "8.04.2": "2009-01-22", + "8.10": "2008-10-30", + "8.04.1": "2008-07-03", + "8.04": "2008-04-24", + "6.06.2": "2008-01-22", + "7.10": "2007-10-18", + "7.04": "2007-04-19", + "6.10": "2006-10-26", + "6.06.1": "2006-08-10", + "6.06": "2006-06-01", + "5.10": "2005-10-13", + "5.04": "2005-04-08", + "4.10": "2004-10-20" +} \ No newline at end of file From 31abee5c219ace664061c4672ff241a143ffdb04 Mon Sep 17 00:00:00 2001 From: Nemo Date: Sun, 10 Jul 2022 15:20:31 +0530 Subject: [PATCH 2/6] [update] Make minor releases optional in default template --- update.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/update.rb b/update.rb index 4e55e9fc..3187c9f4 100644 --- a/update.rb +++ b/update.rb @@ -17,7 +17,7 @@ OPTIONAL_PRODUCT = ARGV[3] # This is more lenient from semver, but disallows MAJOR=0 as well # It also allows MAJOR.MINOR, which is quite common DEFAULT_VERSION_REGEX = '^v?(?[1-9]\d*)\.(?0|[1-9]\d*)\.?(?0|[1-9]\d*)?$' -DEFAULT_TAG_TEMPLATE = '{{major}}.{{minor}}{% if patch %}.{{patch}}{%endif%}' +DEFAULT_TAG_TEMPLATE = "{{major}}{% if minor %}.{{minor}}{% if patch %}.{{patch}}{%endif%}{%endif%}" def fetch_git_releases(repo_dir, config) pwd = Dir.pwd From 5962146a1681f43af56a9239f6b026ee65d58431 Mon Sep 17 00:00:00 2001 From: Nemo Date: Sun, 10 Jul 2022 15:20:48 +0530 Subject: [PATCH 3/6] [distrowatch] Distrowatch update script --- .github/workflows/update.yml | 10 +++--- src/distowatch.py | 61 ++++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 5 deletions(-) create mode 100644 src/distowatch.py diff --git a/.github/workflows/update.yml b/.github/workflows/update.yml index 7c77a045..11bf6c30 100644 --- a/.github/workflows/update.yml +++ b/.github/workflows/update.yml @@ -39,15 +39,15 @@ jobs: with: ruby-version: 3.1 bundler-cache: true - - name: Update Release data + - name: Setup Release Script run: | git config --global init.defaultBranch main git config --global extensions.partialClone true - pip install -r requirements.txt - for i in src/*.py; do python $i;done - - bundle exec ruby update.rb ./website ~/.cache releases + - name: Custom Updates + run: for i in src/*.py; do python $i;done + - name: Git and DockerHub Updates + run: bundle exec ruby update.rb ./website ~/.cache releases id: update_releases - uses: stefanzweifel/git-auto-commit-action@v4 name: Commit and update new releases diff --git a/src/distowatch.py b/src/distowatch.py new file mode 100644 index 00000000..3d2c570d --- /dev/null +++ b/src/distowatch.py @@ -0,0 +1,61 @@ +from glob import glob +import os +import re +import json +import frontmatter +import urllib.request +from bs4 import BeautifulSoup +from html.parser import HTMLParser +from liquid import Template + +# Same as used in Ruby (update.rb) +DEFAULT_TAG_TEMPLATE = "{{major}}{% if minor %}.{{minor}}{% if patch %}.{{patch}}{%endif%}{%endif%}" + +def get_versions_from_headline(regex, headline, template): + if not isinstance(regex, list): + regex = [regex] + for r in regex: + matches = re.match(r.strip(), headline) + if matches: + match_data = matches.groupdict() + version_string = template.render(**match_data) + return version_string.split("\n") + + return {} + +def fetch_releases(distrowatch_id, regex, template): + releases = {} + l_template = Template(template) + url = "https://distrowatch.com/index.php?distribution=%s" % distrowatch_id + with urllib.request.urlopen(url, data=None, timeout=5) as response: + soup = BeautifulSoup(response, features="html5lib") + for table in soup.select("td.News1>table.News"): + headline = table.select_one("td.NewsHeadline a[href]").get_text().strip() + date = table.select_one("td.NewsDate").get_text() + for v in get_versions_from_headline(regex, headline, l_template): + print("%s: %s" % (v, date)) + releases[v] = date + return releases + +def update_releases(): + for product_file in glob("website/products/*.md"): + product_name = os.path.splitext(os.path.basename(product_file))[0] + with open(product_file, "r") as f: + data = frontmatter.load(f) + if "auto" in data: + for config in data["auto"]: + for key, d_id in config.items(): + if key == "distrowatch": + update_product(product_name, config) + +def update_product(product_name, config): + t = config.get("template", DEFAULT_TAG_TEMPLATE) + if "regex" in config: + print("::group::%s" % product_name) + r = fetch_releases(config['distrowatch'], config["regex"], t) + with open("releases/%s.json" % product_name, "w") as f: + f.write(json.dumps(r, indent=2)) + print("::endgroup::") + +if __name__ == "__main__": + update_releases() From 682995f1ec9cab4173251a5202cdaa55360f37b5 Mon Sep 17 00:00:00 2001 From: Nemo Date: Sun, 10 Jul 2022 15:35:08 +0530 Subject: [PATCH 4/6] [deps] Add liquid dependency --- requirements.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/requirements.txt b/requirements.txt index 40759400..5c41018a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,6 +6,9 @@ pycparser==2.21 pygit2==1.9.2 python-frontmatter==1.0.0 PyYAML==5.4 +python-dateutil==2.8.2 +python-liquid==1.2.1 six==1.16.0 soupsieve==2.3.2.post1 +typing_extensions==4.3.0 webencodings==0.5.1 From 32450c162cd8ebeeafd2f6e549cdf5b4ec0409d7 Mon Sep 17 00:00:00 2001 From: Nemo Date: Sun, 10 Jul 2022 15:35:26 +0530 Subject: [PATCH 5/6] Add a debugging workflow to allow testing If you push a branch to the main repo, and a branch here with the same name, then the update script here will use the same branch name instead of using `main`. This way, we can keep 2 branches in sync and then merge them. If a branch does not exist, we'll fallback to cloning master instead --- .github/workflows/update.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/update.yml b/.github/workflows/update.yml index 11bf6c30..e8c60585 100644 --- a/.github/workflows/update.yml +++ b/.github/workflows/update.yml @@ -30,7 +30,17 @@ jobs: with: ref: ${{ github.head_ref }} - uses: actions/checkout@v3 - name: Clone website + id: clone_same_branch + name: Clone website (Same Branch) + continue-on-error: true + with: + repository: endoflife-date/endoflife.date + path: website + submodules: false + ref: ${{github.ref_name}} + - uses: actions/checkout@v3 + name: Clone website (Main) + if: steps.clone_same_branch.outcome != 'success' with: repository: endoflife-date/endoflife.date path: website From 0a68bf285815f0924504fb6d4dd3652b03d9a6f6 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 10 Jul 2022 10:32:53 +0000 Subject: [PATCH 6/6] =?UTF-8?q?=F0=9F=A4=96:=20almalinux,=20rabbitmq?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit almalinux: rabbitmq: 3.10.6 --- releases/almalinux.json | 2 +- releases/rabbitmq.json | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/releases/almalinux.json b/releases/almalinux.json index 24dbda04..4f4c5d69 100644 --- a/releases/almalinux.json +++ b/releases/almalinux.json @@ -4,4 +4,4 @@ "8.5": "2021-11-12", "8.4": "2021-05-26", "8.3": "2021-03-30" -} +} \ No newline at end of file diff --git a/releases/rabbitmq.json b/releases/rabbitmq.json index 26e1c0ef..07f268ab 100644 --- a/releases/rabbitmq.json +++ b/releases/rabbitmq.json @@ -93,6 +93,7 @@ "3.10.3": "2022-05-31", "3.10.4": "2022-05-31", "3.10.5": "2022-06-01", + "3.10.6": "2022-07-10", "3.7.0": "2017-11-28", "3.7.1": "2017-12-15", "3.7.10": "2019-01-07",