Cleanup and fix update.rb (#261)

Cleanup unused code in update.rb: all methods are now handled by custom scripts. This should fix errors encountered in update.yml workflow (such as https://github.com/endoflife-date/release-data/actions/runs/7279373761) since the dependency to psych has been removed.

This also reformat and update the update.yml workflow so that two workflows targeting the same branch cannot run concurrently. This change has been done because in such cases the last one always fail (changes cannot be commited because the previous workflow already update the branch).

Fixes #260.
This commit is contained in:
Marc Wrobel
2023-12-22 22:29:20 +01:00
parent 09084ad500
commit 62e466a841
4 changed files with 78 additions and 212 deletions

135
update.rb
View File

@@ -1,92 +1,13 @@
require 'yaml'
require 'set'
require 'date'
require 'json'
require 'rugged'
require 'liquid'
require 'digest'
require 'net/http'
require 'uri'
WEBSITE_DIR = ARGV[0]
CACHE_DIR = ARGV[1]
OUTPUT_DIR = ARGV[2]
OPTIONAL_PRODUCT = ARGV[3]
# This regex is used in absence of anything else.
# This is more lenient from semver, but disallows MAJOR=0 as well.
# It allows MAJOR.MINOR, MAJOR.MINOR.PATCH and MAJOR.MINOR.PATCH.TINY versions,
# with or without a 'v' prefix, which are quite common.
DEFAULT_VERSION_REGEX = '^v?(?<major>[1-9]\d*)\.(?<minor>\d+)(\.(?<patch>\d+)(\.(?<tiny>\d+))?)?$'
DEFAULT_TAG_TEMPLATE = "{{major}}.{{minor}}{% if patch %}.{{patch}}{% if tiny %}.{{tiny}}{%endif%}{%endif%}"
# extensions.partialClone=true is also set up in the workflow.
# See also https://stackoverflow.com/a/65746233/374236
def fetch_git_releases(repo_dir, config)
pwd = Dir.pwd
`git init --bare #{repo_dir}` unless Dir.exist? repo_dir
Dir.chdir repo_dir
`git fetch --quiet --tags --filter=blob:none --depth=1 "#{config['git']}"`
Dir.chdir pwd
end
def good_tag(tag, config)
config['regex'] ||= DEFAULT_VERSION_REGEX
tag.match?(config['regex'])
end
def render_tag(tag, config)
config['regex'] ||= DEFAULT_SEMVER_REGEX
data = tag.match(config['regex']).named_captures
template = config['template'] ? config['template'] : DEFAULT_TAG_TEMPLATE
Liquid::Template.parse(template).render(data)
end
def get_releases_from_git(repo_dir, auto_config)
data = {}
repo = Rugged::Repository.bare repo_dir
repo.tags.each do |tag|
next unless good_tag(tag.name, auto_config)
tag_proper_name = render_tag(tag.name, auto_config)
# If the tag has an annotation, we get accurate time information
# from the tag annotation "tagger"
begin
if tag.annotated?
# We pick the data from the "tagger" which includes offset information
t = tag.annotation.tagger[:time]
data[tag_proper_name] = t.strftime('%F')
puts "#{tag_proper_name}: #{t.strftime('%F %X %z')}"
else
# In other cases, we de-reference the tag to get the commit
# and use the date of the commit itself
t = tag.target.committer[:time]
data[tag_proper_name] = t.strftime('%F')
puts "#{tag_proper_name}: #{t.strftime('%F %X %z')}"
end
rescue StandardError
puts "::warning No timestamp for #{tag.name}, ignoring"
end
end
return data
end
def get_cache_dir(ecosystem, product, config)
h = Digest::SHA1.hexdigest config['git']
"#{CACHE_DIR}/#{ecosystem}/#{product}_#{h}"
end
def get_output_file(product)
"#{OUTPUT_DIR}/#{product}.json"
end
def generate_commit_message
begin
products = Set.new
ret = nil
msg = ""
r = Rugged::Repository.new '.'
r.status() do |f, s|
p = Pathname.new(f).dirname
@@ -100,56 +21,12 @@ def generate_commit_message
msg += "#{product}: #{new_versions.join(', ')}\n"
end
end
ret ? "🤖: #{products.join(', ')}\n\n#{msg}": ""
commit_title = products.join(', ')
return ret ? "🤖: #{commit_title}\n\n#{msg}": ""
rescue StandardError => e
"🤖: Automatic Update"
end
end
def get_releases(product, config, i)
type = get_update_type(config)
if type == 'oldgit' # replaced by the git.py script, code kept for now to facilitate rollbacks
dir = get_cache_dir('git', product, config)
fetch_git_releases(dir, config)
return get_releases_from_git(dir, config)
elsif type != nil
puts "#{product} handled by #{type} script, skipping"
return {}
else
puts "Undetected method for #{product}"
return {}
end
end
def get_update_type(config)
for i in ['git', 'npm', 'docker_hub', 'distrowatch', 'custom', 'github_releases', 'pypi', 'maven']
return i if config[i]
end
return nil
end
Dir.glob("#{WEBSITE_DIR}/products/*.md").each do |product_file|
data = YAML.safe_load_file product_file, permitted_classes: [Date]
next unless data['auto']
product = File.basename product_file, '.md'
# Only process one product
next if OPTIONAL_PRODUCT && (OPTIONAL_PRODUCT != product)
if data['auto']
release_data = {}
puts "::group::#{product}"
data['auto'].each_with_index do |config, i|
release_data.merge! get_releases(product, config, i)
end
File.open(get_output_file(product), 'w') do |file|
file.write(JSON.pretty_generate(release_data))
end unless release_data.empty?
puts "::endgroup::"
return "🤖: Automatic Update"
end
end