From 62e466a84177270f3bb54610389aa665b4f15a97 Mon Sep 17 00:00:00 2001 From: Marc Wrobel Date: Fri, 22 Dec 2023 22:29:20 +0100 Subject: [PATCH] Cleanup and fix update.rb (#261) Cleanup unused code in update.rb: all methods are now handled by custom scripts. This should fix errors encountered in update.yml workflow (such as https://github.com/endoflife-date/release-data/actions/runs/7279373761) since the dependency to psych has been removed. This also reformat and update the update.yml workflow so that two workflows targeting the same branch cannot run concurrently. This change has been done because in such cases the last one always fail (changes cannot be commited because the previous workflow already update the branch). Fixes #260. --- .github/workflows/update.yml | 136 ++++++++++++++++++----------------- Gemfile | 4 -- Gemfile.lock | 15 ---- update.rb | 135 ++-------------------------------- 4 files changed, 78 insertions(+), 212 deletions(-) diff --git a/.github/workflows/update.yml b/.github/workflows/update.yml index 3531d304..8af507b8 100644 --- a/.github/workflows/update.yml +++ b/.github/workflows/update.yml @@ -1,75 +1,83 @@ name: Update Data + on: workflow_dispatch: push: schedule: - # Run 4 times a day (every 6 hours) - # At minute 17 past hour 0, 6, 12, and 18 - # https://crontab.guru/#17_6,18_*_*_* + # See https://crontab.guru/#17_6,18_*_*_* - cron: '17 0,6,12,18 * * *' + +# Cancel previous runs for a given branch if they are still running when a new one starts. +# This is useful to avoid errors as the same branch would be changed multiple times. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: update: name: Update data runs-on: ubuntu-latest steps: - - name: Get current week - uses: josStorer/get-current-time@v2 - id: current-time - with: - # 2022-01 to 2022-52 for eg - format: YYYY-ww - - name: Cache Repositories - uses: actions/cache@v3 - with: - path: ~/.cache - # The cache is reset on the first build of every week this way. - # Change the -1 part if you need to force reset the cache - key: "${{ steps.current-time.outputs.formattedTime }}-2" - - uses: actions/checkout@v4 - name: Clone self repository - with: - ref: ${{ github.head_ref }} - - uses: actions/checkout@v4 - id: clone_same_branch - name: Clone website (Same Branch) - continue-on-error: true - with: - repository: endoflife-date/endoflife.date - path: website - submodules: false - ref: ${{github.ref_name}} - - uses: actions/checkout@v4 - name: Clone website (Main) - if: steps.clone_same_branch.outcome != 'success' - with: - repository: endoflife-date/endoflife.date - path: website - submodules: false - - uses: ruby/setup-ruby@v1 - with: - ruby-version: 3.1 - bundler-cache: true - - uses: actions/setup-python@v4 - with: - python-version: '3.11' - - name: Setup Release Script - run: | - git config --global init.defaultBranch main - git config --global extensions.partialClone true - pip install -r requirements.txt - - name: Custom Updates - env: - # Add chromium downloaded by pyppeteer to the cache. See java.py for more information. - # Note that using we had to use /home/runner because using ~ does not work, despite what's - # explained in the https://github.com/actions/upload-artifact/tree/v2-preview#environment-variables-and-tilde-expansion. - PYPPETEER_HOME: /home/runner/.cache/pyppeteer - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: for i in src/*.py; do python $i;done - - name: Git and DockerHub Updates - run: bundle exec ruby update.rb ./website ~/.cache releases - id: update_releases - - uses: stefanzweifel/git-auto-commit-action@v5 - name: Commit and update new releases - with: - commit_message: ${{ fromJSON(steps.update_releases.outputs.commit_message)}} - commit_author: 'github-actions[bot] ' + - name: Get current week + uses: josStorer/get-current-time@v2 + id: current-time + with: + format: YYYY-ww # 2022-01 to 2022-52 for eg + + - name: Cache fetched repositories + uses: actions/cache@v3 + with: + path: ~/.cache + # The cache is reset on the first build of every week this way. + # Change the -1 part if you need to force reset the cache + key: "${{ steps.current-time.outputs.formattedTime }}-2" + + - uses: actions/checkout@v4 + name: Clone self repository + with: + ref: ${{ github.head_ref }} + + - uses: actions/checkout@v4 + id: clone_same_branch + name: Clone website (Same Branch) + continue-on-error: true + with: + repository: endoflife-date/endoflife.date + path: website + submodules: false + ref: ${{github.ref_name}} + + - uses: actions/checkout@v4 + name: Clone website (Main) + if: steps.clone_same_branch.outcome != 'success' + with: + repository: endoflife-date/endoflife.date + path: website + submodules: false + + - uses: ruby/setup-ruby@v1 + with: + ruby-version: '3.1' + bundler-cache: true + + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + cache: 'pip' + - run: pip install -r requirements.txt + + - name: Custom Updates + env: + PYPPETEER_HOME: /home/runner/.cache/pyppeteer # Add chromium downloaded by pyppeteer to the cache. + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: for i in src/*.py; do python $i;done + + - name: Build commit message + id: update_releases + run: bundle exec ruby update.rb + + - name: Commit and update new releases + uses: stefanzweifel/git-auto-commit-action@v5 + with: + commit_message: ${{ fromJSON(steps.update_releases.outputs.commit_message)}} + commit_author: 'github-actions[bot] ' diff --git a/Gemfile b/Gemfile index 56bd1243..6e005a1a 100644 --- a/Gemfile +++ b/Gemfile @@ -3,7 +3,3 @@ source "https://rubygems.org" gem "rugged", "~> 1.5.1" -gem "liquid", "~> 5.4" - -gem "irb", "~> 1.9", :group => [:development] -gem "rdoc", "~> 6.6", :group => [:development] diff --git a/Gemfile.lock b/Gemfile.lock index ddf1abb8..3916f0ac 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,27 +1,12 @@ GEM remote: https://rubygems.org/ specs: - io-console (0.6.0) - irb (1.9.1) - rdoc - reline (>= 0.3.8) - liquid (5.4.0) - psych (5.1.1.1) - stringio - rdoc (6.6.0) - psych (>= 4.0.0) - reline (0.4.1) - io-console (~> 0.5) rugged (1.5.1) - stringio (3.1.0) PLATFORMS x86_64-linux DEPENDENCIES - irb (~> 1.9) - liquid (~> 5.4) - rdoc (~> 6.6) rugged (~> 1.5.1) BUNDLED WITH diff --git a/update.rb b/update.rb index 40d1d2c9..0b5e631d 100644 --- a/update.rb +++ b/update.rb @@ -1,92 +1,13 @@ -require 'yaml' require 'set' -require 'date' require 'json' require 'rugged' -require 'liquid' -require 'digest' -require 'net/http' -require 'uri' - -WEBSITE_DIR = ARGV[0] -CACHE_DIR = ARGV[1] -OUTPUT_DIR = ARGV[2] -OPTIONAL_PRODUCT = ARGV[3] - -# This regex is used in absence of anything else. -# This is more lenient from semver, but disallows MAJOR=0 as well. -# It allows MAJOR.MINOR, MAJOR.MINOR.PATCH and MAJOR.MINOR.PATCH.TINY versions, -# with or without a 'v' prefix, which are quite common. -DEFAULT_VERSION_REGEX = '^v?(?[1-9]\d*)\.(?\d+)(\.(?\d+)(\.(?\d+))?)?$' -DEFAULT_TAG_TEMPLATE = "{{major}}.{{minor}}{% if patch %}.{{patch}}{% if tiny %}.{{tiny}}{%endif%}{%endif%}" - -# extensions.partialClone=true is also set up in the workflow. -# See also https://stackoverflow.com/a/65746233/374236 -def fetch_git_releases(repo_dir, config) - pwd = Dir.pwd - `git init --bare #{repo_dir}` unless Dir.exist? repo_dir - Dir.chdir repo_dir - `git fetch --quiet --tags --filter=blob:none --depth=1 "#{config['git']}"` - Dir.chdir pwd -end - -def good_tag(tag, config) - config['regex'] ||= DEFAULT_VERSION_REGEX - tag.match?(config['regex']) -end - -def render_tag(tag, config) - config['regex'] ||= DEFAULT_SEMVER_REGEX - data = tag.match(config['regex']).named_captures - - template = config['template'] ? config['template'] : DEFAULT_TAG_TEMPLATE - Liquid::Template.parse(template).render(data) -end - -def get_releases_from_git(repo_dir, auto_config) - data = {} - repo = Rugged::Repository.bare repo_dir - repo.tags.each do |tag| - next unless good_tag(tag.name, auto_config) - - tag_proper_name = render_tag(tag.name, auto_config) - - # If the tag has an annotation, we get accurate time information - # from the tag annotation "tagger" - begin - if tag.annotated? - # We pick the data from the "tagger" which includes offset information - t = tag.annotation.tagger[:time] - data[tag_proper_name] = t.strftime('%F') - puts "#{tag_proper_name}: #{t.strftime('%F %X %z')}" - else - # In other cases, we de-reference the tag to get the commit - # and use the date of the commit itself - t = tag.target.committer[:time] - data[tag_proper_name] = t.strftime('%F') - puts "#{tag_proper_name}: #{t.strftime('%F %X %z')}" - end - rescue StandardError - puts "::warning No timestamp for #{tag.name}, ignoring" - end - end - return data -end - -def get_cache_dir(ecosystem, product, config) - h = Digest::SHA1.hexdigest config['git'] - "#{CACHE_DIR}/#{ecosystem}/#{product}_#{h}" -end - -def get_output_file(product) - "#{OUTPUT_DIR}/#{product}.json" -end def generate_commit_message begin products = Set.new ret = nil msg = "" + r = Rugged::Repository.new '.' r.status() do |f, s| p = Pathname.new(f).dirname @@ -100,56 +21,12 @@ def generate_commit_message msg += "#{product}: #{new_versions.join(', ')}\n" end end - ret ? "🤖: #{products.join(', ')}\n\n#{msg}": "" + + commit_title = products.join(', ') + return ret ? "🤖: #{commit_title}\n\n#{msg}": "" + rescue StandardError => e - "🤖: Automatic Update" - end -end - -def get_releases(product, config, i) - type = get_update_type(config) - if type == 'oldgit' # replaced by the git.py script, code kept for now to facilitate rollbacks - dir = get_cache_dir('git', product, config) - fetch_git_releases(dir, config) - return get_releases_from_git(dir, config) - elsif type != nil - puts "#{product} handled by #{type} script, skipping" - return {} - else - puts "Undetected method for #{product}" - return {} - end -end - -def get_update_type(config) - for i in ['git', 'npm', 'docker_hub', 'distrowatch', 'custom', 'github_releases', 'pypi', 'maven'] - return i if config[i] - end - - return nil -end - -Dir.glob("#{WEBSITE_DIR}/products/*.md").each do |product_file| - data = YAML.safe_load_file product_file, permitted_classes: [Date] - next unless data['auto'] - - product = File.basename product_file, '.md' - - # Only process one product - next if OPTIONAL_PRODUCT && (OPTIONAL_PRODUCT != product) - - if data['auto'] - release_data = {} - - puts "::group::#{product}" - data['auto'].each_with_index do |config, i| - release_data.merge! get_releases(product, config, i) - end - - File.open(get_output_file(product), 'w') do |file| - file.write(JSON.pretty_generate(release_data)) - end unless release_data.empty? - puts "::endgroup::" + return "🤖: Automatic Update" end end