86860715ab
Reasons were: - one language make it easier for maintenance, - workflow simplification, - not stopping when a script fails anymore (it's better to have a partial update than nothing), - use `GITHUB_OUTPUT` instead of the deprecated `set-output` command, - display a workflow summary with statistics about scripts and information about updated products. The commit message is not as good as it used to be, but it makes the diff process agnostic from the file format (will be needed soon when it will change), and it handles versions updates / removal, which was not supported by the previous script.
115 lines
4.1 KiB
Python
115 lines
4.1 KiB
Python
import json
|
|
import logging
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from base64 import b64encode
|
|
from deepdiff import DeepDiff
|
|
from pathlib import Path
|
|
|
|
|
|
def github_output(name, value):
|
|
if "GITHUB_OUTPUT" not in os.environ:
|
|
logging.debug(f"GITHUB_OUTPUT does not exist, but would have written: {name}={value.strip()}")
|
|
return
|
|
|
|
if "\n" in value:
|
|
# https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings
|
|
delimiter = b64encode(os.urandom(16)).decode()
|
|
value = f"{delimiter}\n{value}\n{delimiter}"
|
|
command = f"{name}<<{value}"
|
|
else:
|
|
command = f"{name}={value}"
|
|
|
|
with open(os.environ["GITHUB_OUTPUT"], 'a') as github_output_var:
|
|
print(command, file=github_output_var)
|
|
logging.debug(f"Wrote to GITHUB_OUTPUT: {name}={value.strip()}")
|
|
|
|
|
|
def add_summary_line(line):
|
|
if "GITHUB_STEP_SUMMARY" not in os.environ:
|
|
logging.debug(f"GITHUB_STEP_SUMMARY does not exist, but would have written: {line}")
|
|
return
|
|
|
|
with open(os.environ["GITHUB_STEP_SUMMARY"], 'a') as github_step_summary:
|
|
print(line, file=github_step_summary)
|
|
|
|
|
|
SRC_DIR = 'src'
|
|
DATA_DIR = 'releases'
|
|
|
|
logging.basicConfig(format=logging.BASIC_FORMAT, level=logging.INFO)
|
|
|
|
# Run scripts
|
|
scripts = sorted([os.path.join(SRC_DIR, file) for file in os.listdir(SRC_DIR) if file.endswith('.py')])
|
|
some_script_failed = False
|
|
|
|
add_summary_line("## Script execution summary\n")
|
|
add_summary_line("| Name | Duration | Succeeded |")
|
|
add_summary_line("|------|----------|-----------|")
|
|
for script in scripts:
|
|
logging.info(f"start running {script}")
|
|
|
|
start = time.perf_counter()
|
|
child = subprocess.run([sys.executable, script], timeout=300)
|
|
elapsed_seconds = time.perf_counter() - start
|
|
|
|
if child.returncode != 0:
|
|
some_script_failed = True
|
|
add_summary_line(f"| {script} | {elapsed_seconds:.2f}s | ❌ |")
|
|
logging.error(f"Error while running {script} after {elapsed_seconds:.2f}s, update will only be partial")
|
|
else:
|
|
logging.info(f"Finished running {script}, took {elapsed_seconds:.2f}s")
|
|
add_summary_line(f"| {script} | {elapsed_seconds:.2f}s | ✅ |")
|
|
|
|
# Generate commit message
|
|
subprocess.run('git add --all', timeout=10, check=True, shell=True) # to also get new files in git diff
|
|
git_diff = subprocess.run('git diff --name-only --staged', capture_output=True, timeout=10, check=True, shell=True)
|
|
updated_files = sorted([Path(file) for file in git_diff.stdout.decode('utf-8').split('\n') if file.startswith(DATA_DIR)])
|
|
logging.info(f"Updated files: {updated_files}")
|
|
|
|
add_summary_line("## Update summary\n")
|
|
if updated_files:
|
|
# get modified files content
|
|
new_files_content = {}
|
|
for path in updated_files:
|
|
with open(path) as file:
|
|
new_files_content[path] = json.load(file)
|
|
|
|
# get original files content
|
|
old_files_content = {}
|
|
subprocess.run('git stash --all --quiet', timeout=10, check=True, shell=True)
|
|
for path in updated_files:
|
|
if path.exists():
|
|
with open(path) as file:
|
|
old_files_content[path] = json.load(file)
|
|
else: # new file
|
|
old_files_content[path] = {}
|
|
subprocess.run('git stash pop --quiet', timeout=10, check=True, shell=True)
|
|
|
|
# Generate commit message
|
|
product_names = ', '.join([path.stem for path in updated_files])
|
|
commit_message = f"🤖: {product_names}\n\n"
|
|
add_summary_line(f"Updated {len(updated_files)} products: {product_names}.")
|
|
|
|
for path in updated_files:
|
|
add_summary_line(f"### {path.stem}\n")
|
|
commit_message += f"{path.stem}:\n"
|
|
|
|
diff = DeepDiff(old_files_content[path], new_files_content[path], ignore_order=True)
|
|
for line in diff.pretty().split('\n'):
|
|
add_summary_line(f"- {line}")
|
|
commit_message += f"- {line}\n"
|
|
logging.info(f"{path.stem}: {line}")
|
|
|
|
commit_message += "\n"
|
|
add_summary_line("")
|
|
|
|
github_output('commit_message', commit_message)
|
|
|
|
else:
|
|
add_summary_line("No update")
|
|
|
|
sys.exit(1 if some_script_failed else 0)
|