Cleanup and fix update.rb (#261)
Cleanup unused code in update.rb: all methods are now handled by custom scripts. This should fix errors encountered in update.yml workflow (such as https://github.com/endoflife-date/release-data/actions/runs/7279373761) since the dependency to psych has been removed. This also reformat and update the update.yml workflow so that two workflows targeting the same branch cannot run concurrently. This change has been done because in such cases the last one always fail (changes cannot be commited because the previous workflow already update the branch). Fixes #260.
This commit is contained in:
Marc Wrobel
136
.github/workflows/update.yml
vendored
136
.github/workflows/update.yml
vendored
@@ -1,75 +1,83 @@
|
||||
name: Update Data
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
schedule:
|
||||
# Run 4 times a day (every 6 hours)
|
||||
# At minute 17 past hour 0, 6, 12, and 18
|
||||
# https://crontab.guru/#17_6,18_*_*_*
|
||||
# See https://crontab.guru/#17_6,18_*_*_*
|
||||
- cron: '17 0,6,12,18 * * *'
|
||||
|
||||
# Cancel previous runs for a given branch if they are still running when a new one starts.
|
||||
# This is useful to avoid errors as the same branch would be changed multiple times.
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
update:
|
||||
name: Update data
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Get current week
|
||||
uses: josStorer/get-current-time@v2
|
||||
id: current-time
|
||||
with:
|
||||
# 2022-01 to 2022-52 for eg
|
||||
format: YYYY-ww
|
||||
- name: Cache Repositories
|
||||
uses: actions/cache@v3
|
||||
with:
|
||||
path: ~/.cache
|
||||
# The cache is reset on the first build of every week this way.
|
||||
# Change the -1 part if you need to force reset the cache
|
||||
key: "${{ steps.current-time.outputs.formattedTime }}-2"
|
||||
- uses: actions/checkout@v4
|
||||
name: Clone self repository
|
||||
with:
|
||||
ref: ${{ github.head_ref }}
|
||||
- uses: actions/checkout@v4
|
||||
id: clone_same_branch
|
||||
name: Clone website (Same Branch)
|
||||
continue-on-error: true
|
||||
with:
|
||||
repository: endoflife-date/endoflife.date
|
||||
path: website
|
||||
submodules: false
|
||||
ref: ${{github.ref_name}}
|
||||
- uses: actions/checkout@v4
|
||||
name: Clone website (Main)
|
||||
if: steps.clone_same_branch.outcome != 'success'
|
||||
with:
|
||||
repository: endoflife-date/endoflife.date
|
||||
path: website
|
||||
submodules: false
|
||||
- uses: ruby/setup-ruby@v1
|
||||
with:
|
||||
ruby-version: 3.1
|
||||
bundler-cache: true
|
||||
- uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: '3.11'
|
||||
- name: Setup Release Script
|
||||
run: |
|
||||
git config --global init.defaultBranch main
|
||||
git config --global extensions.partialClone true
|
||||
pip install -r requirements.txt
|
||||
- name: Custom Updates
|
||||
env:
|
||||
# Add chromium downloaded by pyppeteer to the cache. See java.py for more information.
|
||||
# Note that using we had to use /home/runner because using ~ does not work, despite what's
|
||||
# explained in the https://github.com/actions/upload-artifact/tree/v2-preview#environment-variables-and-tilde-expansion.
|
||||
PYPPETEER_HOME: /home/runner/.cache/pyppeteer
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: for i in src/*.py; do python $i;done
|
||||
- name: Git and DockerHub Updates
|
||||
run: bundle exec ruby update.rb ./website ~/.cache releases
|
||||
id: update_releases
|
||||
- uses: stefanzweifel/git-auto-commit-action@v5
|
||||
name: Commit and update new releases
|
||||
with:
|
||||
commit_message: ${{ fromJSON(steps.update_releases.outputs.commit_message)}}
|
||||
commit_author: 'github-actions[bot] <github-actions[bot]@users.noreply.github.com>'
|
||||
- name: Get current week
|
||||
uses: josStorer/get-current-time@v2
|
||||
id: current-time
|
||||
with:
|
||||
format: YYYY-ww # 2022-01 to 2022-52 for eg
|
||||
|
||||
- name: Cache fetched repositories
|
||||
uses: actions/cache@v3
|
||||
with:
|
||||
path: ~/.cache
|
||||
# The cache is reset on the first build of every week this way.
|
||||
# Change the -1 part if you need to force reset the cache
|
||||
key: "${{ steps.current-time.outputs.formattedTime }}-2"
|
||||
|
||||
- uses: actions/checkout@v4
|
||||
name: Clone self repository
|
||||
with:
|
||||
ref: ${{ github.head_ref }}
|
||||
|
||||
- uses: actions/checkout@v4
|
||||
id: clone_same_branch
|
||||
name: Clone website (Same Branch)
|
||||
continue-on-error: true
|
||||
with:
|
||||
repository: endoflife-date/endoflife.date
|
||||
path: website
|
||||
submodules: false
|
||||
ref: ${{github.ref_name}}
|
||||
|
||||
- uses: actions/checkout@v4
|
||||
name: Clone website (Main)
|
||||
if: steps.clone_same_branch.outcome != 'success'
|
||||
with:
|
||||
repository: endoflife-date/endoflife.date
|
||||
path: website
|
||||
submodules: false
|
||||
|
||||
- uses: ruby/setup-ruby@v1
|
||||
with:
|
||||
ruby-version: '3.1'
|
||||
bundler-cache: true
|
||||
|
||||
- uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: '3.11'
|
||||
cache: 'pip'
|
||||
- run: pip install -r requirements.txt
|
||||
|
||||
- name: Custom Updates
|
||||
env:
|
||||
PYPPETEER_HOME: /home/runner/.cache/pyppeteer # Add chromium downloaded by pyppeteer to the cache.
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: for i in src/*.py; do python $i;done
|
||||
|
||||
- name: Build commit message
|
||||
id: update_releases
|
||||
run: bundle exec ruby update.rb
|
||||
|
||||
- name: Commit and update new releases
|
||||
uses: stefanzweifel/git-auto-commit-action@v5
|
||||
with:
|
||||
commit_message: ${{ fromJSON(steps.update_releases.outputs.commit_message)}}
|
||||
commit_author: 'github-actions[bot] <github-actions[bot]@users.noreply.github.com>'
|
||||
|
||||
4
Gemfile
4
Gemfile
@@ -3,7 +3,3 @@
|
||||
source "https://rubygems.org"
|
||||
|
||||
gem "rugged", "~> 1.5.1"
|
||||
gem "liquid", "~> 5.4"
|
||||
|
||||
gem "irb", "~> 1.9", :group => [:development]
|
||||
gem "rdoc", "~> 6.6", :group => [:development]
|
||||
|
||||
15
Gemfile.lock
15
Gemfile.lock
@@ -1,27 +1,12 @@
|
||||
GEM
|
||||
remote: https://rubygems.org/
|
||||
specs:
|
||||
io-console (0.6.0)
|
||||
irb (1.9.1)
|
||||
rdoc
|
||||
reline (>= 0.3.8)
|
||||
liquid (5.4.0)
|
||||
psych (5.1.1.1)
|
||||
stringio
|
||||
rdoc (6.6.0)
|
||||
psych (>= 4.0.0)
|
||||
reline (0.4.1)
|
||||
io-console (~> 0.5)
|
||||
rugged (1.5.1)
|
||||
stringio (3.1.0)
|
||||
|
||||
PLATFORMS
|
||||
x86_64-linux
|
||||
|
||||
DEPENDENCIES
|
||||
irb (~> 1.9)
|
||||
liquid (~> 5.4)
|
||||
rdoc (~> 6.6)
|
||||
rugged (~> 1.5.1)
|
||||
|
||||
BUNDLED WITH
|
||||
|
||||
135
update.rb
135
update.rb
@@ -1,92 +1,13 @@
|
||||
require 'yaml'
|
||||
require 'set'
|
||||
require 'date'
|
||||
require 'json'
|
||||
require 'rugged'
|
||||
require 'liquid'
|
||||
require 'digest'
|
||||
require 'net/http'
|
||||
require 'uri'
|
||||
|
||||
WEBSITE_DIR = ARGV[0]
|
||||
CACHE_DIR = ARGV[1]
|
||||
OUTPUT_DIR = ARGV[2]
|
||||
OPTIONAL_PRODUCT = ARGV[3]
|
||||
|
||||
# This regex is used in absence of anything else.
|
||||
# This is more lenient from semver, but disallows MAJOR=0 as well.
|
||||
# It allows MAJOR.MINOR, MAJOR.MINOR.PATCH and MAJOR.MINOR.PATCH.TINY versions,
|
||||
# with or without a 'v' prefix, which are quite common.
|
||||
DEFAULT_VERSION_REGEX = '^v?(?<major>[1-9]\d*)\.(?<minor>\d+)(\.(?<patch>\d+)(\.(?<tiny>\d+))?)?$'
|
||||
DEFAULT_TAG_TEMPLATE = "{{major}}.{{minor}}{% if patch %}.{{patch}}{% if tiny %}.{{tiny}}{%endif%}{%endif%}"
|
||||
|
||||
# extensions.partialClone=true is also set up in the workflow.
|
||||
# See also https://stackoverflow.com/a/65746233/374236
|
||||
def fetch_git_releases(repo_dir, config)
|
||||
pwd = Dir.pwd
|
||||
`git init --bare #{repo_dir}` unless Dir.exist? repo_dir
|
||||
Dir.chdir repo_dir
|
||||
`git fetch --quiet --tags --filter=blob:none --depth=1 "#{config['git']}"`
|
||||
Dir.chdir pwd
|
||||
end
|
||||
|
||||
def good_tag(tag, config)
|
||||
config['regex'] ||= DEFAULT_VERSION_REGEX
|
||||
tag.match?(config['regex'])
|
||||
end
|
||||
|
||||
def render_tag(tag, config)
|
||||
config['regex'] ||= DEFAULT_SEMVER_REGEX
|
||||
data = tag.match(config['regex']).named_captures
|
||||
|
||||
template = config['template'] ? config['template'] : DEFAULT_TAG_TEMPLATE
|
||||
Liquid::Template.parse(template).render(data)
|
||||
end
|
||||
|
||||
def get_releases_from_git(repo_dir, auto_config)
|
||||
data = {}
|
||||
repo = Rugged::Repository.bare repo_dir
|
||||
repo.tags.each do |tag|
|
||||
next unless good_tag(tag.name, auto_config)
|
||||
|
||||
tag_proper_name = render_tag(tag.name, auto_config)
|
||||
|
||||
# If the tag has an annotation, we get accurate time information
|
||||
# from the tag annotation "tagger"
|
||||
begin
|
||||
if tag.annotated?
|
||||
# We pick the data from the "tagger" which includes offset information
|
||||
t = tag.annotation.tagger[:time]
|
||||
data[tag_proper_name] = t.strftime('%F')
|
||||
puts "#{tag_proper_name}: #{t.strftime('%F %X %z')}"
|
||||
else
|
||||
# In other cases, we de-reference the tag to get the commit
|
||||
# and use the date of the commit itself
|
||||
t = tag.target.committer[:time]
|
||||
data[tag_proper_name] = t.strftime('%F')
|
||||
puts "#{tag_proper_name}: #{t.strftime('%F %X %z')}"
|
||||
end
|
||||
rescue StandardError
|
||||
puts "::warning No timestamp for #{tag.name}, ignoring"
|
||||
end
|
||||
end
|
||||
return data
|
||||
end
|
||||
|
||||
def get_cache_dir(ecosystem, product, config)
|
||||
h = Digest::SHA1.hexdigest config['git']
|
||||
"#{CACHE_DIR}/#{ecosystem}/#{product}_#{h}"
|
||||
end
|
||||
|
||||
def get_output_file(product)
|
||||
"#{OUTPUT_DIR}/#{product}.json"
|
||||
end
|
||||
|
||||
def generate_commit_message
|
||||
begin
|
||||
products = Set.new
|
||||
ret = nil
|
||||
msg = ""
|
||||
|
||||
r = Rugged::Repository.new '.'
|
||||
r.status() do |f, s|
|
||||
p = Pathname.new(f).dirname
|
||||
@@ -100,56 +21,12 @@ def generate_commit_message
|
||||
msg += "#{product}: #{new_versions.join(', ')}\n"
|
||||
end
|
||||
end
|
||||
ret ? "🤖: #{products.join(', ')}\n\n#{msg}": ""
|
||||
|
||||
commit_title = products.join(', ')
|
||||
return ret ? "🤖: #{commit_title}\n\n#{msg}": ""
|
||||
|
||||
rescue StandardError => e
|
||||
"🤖: Automatic Update"
|
||||
end
|
||||
end
|
||||
|
||||
def get_releases(product, config, i)
|
||||
type = get_update_type(config)
|
||||
if type == 'oldgit' # replaced by the git.py script, code kept for now to facilitate rollbacks
|
||||
dir = get_cache_dir('git', product, config)
|
||||
fetch_git_releases(dir, config)
|
||||
return get_releases_from_git(dir, config)
|
||||
elsif type != nil
|
||||
puts "#{product} handled by #{type} script, skipping"
|
||||
return {}
|
||||
else
|
||||
puts "Undetected method for #{product}"
|
||||
return {}
|
||||
end
|
||||
end
|
||||
|
||||
def get_update_type(config)
|
||||
for i in ['git', 'npm', 'docker_hub', 'distrowatch', 'custom', 'github_releases', 'pypi', 'maven']
|
||||
return i if config[i]
|
||||
end
|
||||
|
||||
return nil
|
||||
end
|
||||
|
||||
Dir.glob("#{WEBSITE_DIR}/products/*.md").each do |product_file|
|
||||
data = YAML.safe_load_file product_file, permitted_classes: [Date]
|
||||
next unless data['auto']
|
||||
|
||||
product = File.basename product_file, '.md'
|
||||
|
||||
# Only process one product
|
||||
next if OPTIONAL_PRODUCT && (OPTIONAL_PRODUCT != product)
|
||||
|
||||
if data['auto']
|
||||
release_data = {}
|
||||
|
||||
puts "::group::#{product}"
|
||||
data['auto'].each_with_index do |config, i|
|
||||
release_data.merge! get_releases(product, config, i)
|
||||
end
|
||||
|
||||
File.open(get_output_file(product), 'w') do |file|
|
||||
file.write(JSON.pretty_generate(release_data))
|
||||
end unless release_data.empty?
|
||||
puts "::endgroup::"
|
||||
return "🤖: Automatic Update"
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
Reference in New Issue
Block a user