diff --git a/.github/workflows/update.yml b/.github/workflows/update.yml index 3531d304..8af507b8 100644 --- a/.github/workflows/update.yml +++ b/.github/workflows/update.yml @@ -1,75 +1,83 @@ name: Update Data + on: workflow_dispatch: push: schedule: - # Run 4 times a day (every 6 hours) - # At minute 17 past hour 0, 6, 12, and 18 - # https://crontab.guru/#17_6,18_*_*_* + # See https://crontab.guru/#17_6,18_*_*_* - cron: '17 0,6,12,18 * * *' + +# Cancel previous runs for a given branch if they are still running when a new one starts. +# This is useful to avoid errors as the same branch would be changed multiple times. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: update: name: Update data runs-on: ubuntu-latest steps: - - name: Get current week - uses: josStorer/get-current-time@v2 - id: current-time - with: - # 2022-01 to 2022-52 for eg - format: YYYY-ww - - name: Cache Repositories - uses: actions/cache@v3 - with: - path: ~/.cache - # The cache is reset on the first build of every week this way. - # Change the -1 part if you need to force reset the cache - key: "${{ steps.current-time.outputs.formattedTime }}-2" - - uses: actions/checkout@v4 - name: Clone self repository - with: - ref: ${{ github.head_ref }} - - uses: actions/checkout@v4 - id: clone_same_branch - name: Clone website (Same Branch) - continue-on-error: true - with: - repository: endoflife-date/endoflife.date - path: website - submodules: false - ref: ${{github.ref_name}} - - uses: actions/checkout@v4 - name: Clone website (Main) - if: steps.clone_same_branch.outcome != 'success' - with: - repository: endoflife-date/endoflife.date - path: website - submodules: false - - uses: ruby/setup-ruby@v1 - with: - ruby-version: 3.1 - bundler-cache: true - - uses: actions/setup-python@v4 - with: - python-version: '3.11' - - name: Setup Release Script - run: | - git config --global init.defaultBranch main - git config --global extensions.partialClone true - pip install -r requirements.txt - - name: Custom Updates - env: - # Add chromium downloaded by pyppeteer to the cache. See java.py for more information. - # Note that using we had to use /home/runner because using ~ does not work, despite what's - # explained in the https://github.com/actions/upload-artifact/tree/v2-preview#environment-variables-and-tilde-expansion. - PYPPETEER_HOME: /home/runner/.cache/pyppeteer - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: for i in src/*.py; do python $i;done - - name: Git and DockerHub Updates - run: bundle exec ruby update.rb ./website ~/.cache releases - id: update_releases - - uses: stefanzweifel/git-auto-commit-action@v5 - name: Commit and update new releases - with: - commit_message: ${{ fromJSON(steps.update_releases.outputs.commit_message)}} - commit_author: 'github-actions[bot] ' + - name: Get current week + uses: josStorer/get-current-time@v2 + id: current-time + with: + format: YYYY-ww # 2022-01 to 2022-52 for eg + + - name: Cache fetched repositories + uses: actions/cache@v3 + with: + path: ~/.cache + # The cache is reset on the first build of every week this way. + # Change the -1 part if you need to force reset the cache + key: "${{ steps.current-time.outputs.formattedTime }}-2" + + - uses: actions/checkout@v4 + name: Clone self repository + with: + ref: ${{ github.head_ref }} + + - uses: actions/checkout@v4 + id: clone_same_branch + name: Clone website (Same Branch) + continue-on-error: true + with: + repository: endoflife-date/endoflife.date + path: website + submodules: false + ref: ${{github.ref_name}} + + - uses: actions/checkout@v4 + name: Clone website (Main) + if: steps.clone_same_branch.outcome != 'success' + with: + repository: endoflife-date/endoflife.date + path: website + submodules: false + + - uses: ruby/setup-ruby@v1 + with: + ruby-version: '3.1' + bundler-cache: true + + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + cache: 'pip' + - run: pip install -r requirements.txt + + - name: Custom Updates + env: + PYPPETEER_HOME: /home/runner/.cache/pyppeteer # Add chromium downloaded by pyppeteer to the cache. + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: for i in src/*.py; do python $i;done + + - name: Build commit message + id: update_releases + run: bundle exec ruby update.rb + + - name: Commit and update new releases + uses: stefanzweifel/git-auto-commit-action@v5 + with: + commit_message: ${{ fromJSON(steps.update_releases.outputs.commit_message)}} + commit_author: 'github-actions[bot] ' diff --git a/Gemfile b/Gemfile index 56bd1243..6e005a1a 100644 --- a/Gemfile +++ b/Gemfile @@ -3,7 +3,3 @@ source "https://rubygems.org" gem "rugged", "~> 1.5.1" -gem "liquid", "~> 5.4" - -gem "irb", "~> 1.9", :group => [:development] -gem "rdoc", "~> 6.6", :group => [:development] diff --git a/Gemfile.lock b/Gemfile.lock index ddf1abb8..3916f0ac 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,27 +1,12 @@ GEM remote: https://rubygems.org/ specs: - io-console (0.6.0) - irb (1.9.1) - rdoc - reline (>= 0.3.8) - liquid (5.4.0) - psych (5.1.1.1) - stringio - rdoc (6.6.0) - psych (>= 4.0.0) - reline (0.4.1) - io-console (~> 0.5) rugged (1.5.1) - stringio (3.1.0) PLATFORMS x86_64-linux DEPENDENCIES - irb (~> 1.9) - liquid (~> 5.4) - rdoc (~> 6.6) rugged (~> 1.5.1) BUNDLED WITH diff --git a/update.rb b/update.rb index 40d1d2c9..0b5e631d 100644 --- a/update.rb +++ b/update.rb @@ -1,92 +1,13 @@ -require 'yaml' require 'set' -require 'date' require 'json' require 'rugged' -require 'liquid' -require 'digest' -require 'net/http' -require 'uri' - -WEBSITE_DIR = ARGV[0] -CACHE_DIR = ARGV[1] -OUTPUT_DIR = ARGV[2] -OPTIONAL_PRODUCT = ARGV[3] - -# This regex is used in absence of anything else. -# This is more lenient from semver, but disallows MAJOR=0 as well. -# It allows MAJOR.MINOR, MAJOR.MINOR.PATCH and MAJOR.MINOR.PATCH.TINY versions, -# with or without a 'v' prefix, which are quite common. -DEFAULT_VERSION_REGEX = '^v?(?[1-9]\d*)\.(?\d+)(\.(?\d+)(\.(?\d+))?)?$' -DEFAULT_TAG_TEMPLATE = "{{major}}.{{minor}}{% if patch %}.{{patch}}{% if tiny %}.{{tiny}}{%endif%}{%endif%}" - -# extensions.partialClone=true is also set up in the workflow. -# See also https://stackoverflow.com/a/65746233/374236 -def fetch_git_releases(repo_dir, config) - pwd = Dir.pwd - `git init --bare #{repo_dir}` unless Dir.exist? repo_dir - Dir.chdir repo_dir - `git fetch --quiet --tags --filter=blob:none --depth=1 "#{config['git']}"` - Dir.chdir pwd -end - -def good_tag(tag, config) - config['regex'] ||= DEFAULT_VERSION_REGEX - tag.match?(config['regex']) -end - -def render_tag(tag, config) - config['regex'] ||= DEFAULT_SEMVER_REGEX - data = tag.match(config['regex']).named_captures - - template = config['template'] ? config['template'] : DEFAULT_TAG_TEMPLATE - Liquid::Template.parse(template).render(data) -end - -def get_releases_from_git(repo_dir, auto_config) - data = {} - repo = Rugged::Repository.bare repo_dir - repo.tags.each do |tag| - next unless good_tag(tag.name, auto_config) - - tag_proper_name = render_tag(tag.name, auto_config) - - # If the tag has an annotation, we get accurate time information - # from the tag annotation "tagger" - begin - if tag.annotated? - # We pick the data from the "tagger" which includes offset information - t = tag.annotation.tagger[:time] - data[tag_proper_name] = t.strftime('%F') - puts "#{tag_proper_name}: #{t.strftime('%F %X %z')}" - else - # In other cases, we de-reference the tag to get the commit - # and use the date of the commit itself - t = tag.target.committer[:time] - data[tag_proper_name] = t.strftime('%F') - puts "#{tag_proper_name}: #{t.strftime('%F %X %z')}" - end - rescue StandardError - puts "::warning No timestamp for #{tag.name}, ignoring" - end - end - return data -end - -def get_cache_dir(ecosystem, product, config) - h = Digest::SHA1.hexdigest config['git'] - "#{CACHE_DIR}/#{ecosystem}/#{product}_#{h}" -end - -def get_output_file(product) - "#{OUTPUT_DIR}/#{product}.json" -end def generate_commit_message begin products = Set.new ret = nil msg = "" + r = Rugged::Repository.new '.' r.status() do |f, s| p = Pathname.new(f).dirname @@ -100,56 +21,12 @@ def generate_commit_message msg += "#{product}: #{new_versions.join(', ')}\n" end end - ret ? "🤖: #{products.join(', ')}\n\n#{msg}": "" + + commit_title = products.join(', ') + return ret ? "🤖: #{commit_title}\n\n#{msg}": "" + rescue StandardError => e - "🤖: Automatic Update" - end -end - -def get_releases(product, config, i) - type = get_update_type(config) - if type == 'oldgit' # replaced by the git.py script, code kept for now to facilitate rollbacks - dir = get_cache_dir('git', product, config) - fetch_git_releases(dir, config) - return get_releases_from_git(dir, config) - elsif type != nil - puts "#{product} handled by #{type} script, skipping" - return {} - else - puts "Undetected method for #{product}" - return {} - end -end - -def get_update_type(config) - for i in ['git', 'npm', 'docker_hub', 'distrowatch', 'custom', 'github_releases', 'pypi', 'maven'] - return i if config[i] - end - - return nil -end - -Dir.glob("#{WEBSITE_DIR}/products/*.md").each do |product_file| - data = YAML.safe_load_file product_file, permitted_classes: [Date] - next unless data['auto'] - - product = File.basename product_file, '.md' - - # Only process one product - next if OPTIONAL_PRODUCT && (OPTIONAL_PRODUCT != product) - - if data['auto'] - release_data = {} - - puts "::group::#{product}" - data['auto'].each_with_index do |config, i| - release_data.merge! get_releases(product, config, i) - end - - File.open(get_output_file(product), 'w') do |file| - file.write(JSON.pretty_generate(release_data)) - end unless release_data.empty? - puts "::endgroup::" + return "🤖: Automatic Update" end end