From 686913031ed7d7f93c2caff801fccfcf73f2ae88 Mon Sep 17 00:00:00 2001 From: Cagri Yonca Date: Tue, 18 Mar 2025 12:35:33 +0100 Subject: [PATCH] currency: added new columns to currency report table Signed-off-by: Cagri Yonca --- .tekton/.currency/scripts/generate_report.py | 216 +++++++++++++++++-- 1 file changed, 194 insertions(+), 22 deletions(-) diff --git a/.tekton/.currency/scripts/generate_report.py b/.tekton/.currency/scripts/generate_report.py index 7582f045..42c56ebc 100644 --- a/.tekton/.currency/scripts/generate_report.py +++ b/.tekton/.currency/scripts/generate_report.py @@ -2,6 +2,7 @@ import glob import re from json import load +from datetime import datetime # Third Party from requests import get @@ -11,9 +12,10 @@ JSON_FILE = "resources/table.json" REPORT_FILE = "docs/report.md" API_V1_ENDPOINT = "https://rubygems.org/api/v1/versions/" +GEM_INFO_ENDPOINT = "https://rubygems.org/api/v1/gems/" -def get_bundle_install_output(): +def get_bundle_install_output(): libraries_from_logs = { "cuba": "cuba_40_ruby_3.3.", "excon": "excon_100_ruby_3.3.", @@ -28,22 +30,23 @@ def get_bundle_install_output(): "dalli": "dalli_32_ruby_3.3.", "resque": "resque_20_ruby_3.3.", "sidekiq": "sidekiq_70_ruby_3.3.", - "sequel": "sequel_58_ruby_3.3." + "sequel": "sequel_58_ruby_3.3.", } bundle_install_output = "" for library, pattern in libraries_from_logs.items(): glob_result = glob.glob(f"../../dep_{pattern}*") + if not glob_result: print(f"Could not find bundle install log for gem '{library}'.") continue - with open(glob_result[0], 'r') as file: - logs = file.read().replace('\n', ' ') + with open(glob_result[0], "r") as file: + logs = file.read().replace("\n", " ") if "Installing" not in logs: - print( f"Unable to retrieve logs from for gem '{library}'.") + print(f"Unable to retrieve logs from for gem '{library}'.") continue print(f"Retrieving currency for gem '{library}'.") @@ -53,27 +56,176 @@ def get_bundle_install_output(): return bundle_install_output +def estimate_days_behind(release_date, last_supported_release_date): + """Calculate days between release dates""" + latest_date = datetime.strptime(release_date, "%Y-%m-%d") + supported_date = datetime.strptime(last_supported_release_date, "%Y-%m-%d") + days_diff = (latest_date - supported_date).days + return max(0, days_diff) + + def get_upstream_version(dependency): - """get the latest version available upstream""" + """get the latest version available upstream and release date""" if dependency != "rails lts": - response = get(f"{API_V1_ENDPOINT}/{dependency}/latest.json") - response_json = response.json() - latest_version = response_json["version"] + try: + response = get(f"{API_V1_ENDPOINT}{dependency}/latest.json") + if not response: + return "Unknown", "Unknown" + + response_json = response.json() + latest_version = response_json["version"] + + gem_info_response = get(f"{GEM_INFO_ENDPOINT}{dependency}.json") + if not gem_info_response: + return latest_version, "Unknown" + gem_info = gem_info_response.json() + + version_info_response = get(f"{API_V1_ENDPOINT}{dependency}.json") + if not version_info_response: + return latest_version, "Unknown" + version_info = version_info_response.json() + + latest_version_release_date = None + for version_data in version_info: + if version_data["number"] == latest_version: + if "created_at" in version_data: + created_at = version_data["created_at"] + latest_version_release_date = datetime.strptime( + created_at, "%Y-%m-%dT%H:%M:%S.%fZ" + ).strftime("%Y-%m-%d") + break + + if not latest_version_release_date and "created_at" in gem_info: + created_at = gem_info["created_at"] + latest_version_release_date = datetime.strptime( + created_at, "%Y-%m-%dT%H:%M:%S.%fZ" + ).strftime("%Y-%m-%d") + + if not latest_version_release_date: + latest_version_release_date = "Unknown" + print(f"Could not find release date for {dependency}") + + return latest_version, latest_version_release_date + except Exception as e: + return "Unknown", "Unknown" else: - url = "https://makandracards.com/railslts/16137-installing-rails-lts/read" - page = get(url) - soup = BeautifulSoup(page.text, "html.parser") - text = soup.findAll("li")[-1].text - pattern = "(\d+\.\d+\.?\d*)" - latest_version = re.search(pattern, text)[1] - return latest_version + try: + initial_response = get( + "https://makandracards.com/railslts/16137-installing-rails-lts/read" + ) + if not initial_response: + return "Unknown", "Unknown" + + initial_soup = BeautifulSoup(initial_response.text, "html.parser") + + text = initial_soup.find_all("li")[-1].text + pattern = "(\d+\.\d+\.?\d*)" + latest_version_match = re.search(pattern, text) + + latest_version = "Unknown" + if latest_version_match: + latest_version = latest_version_match[1] + rails_lts_links = initial_soup.find_all("a", href=re.compile(r"railslts")) + latest_rails_lts_link = None + latest_version_num = 0.0 + + for link in rails_lts_links: + href = link.get("href", "") + link_text = link.text.strip() + + if "Installing Rails" in link_text: + version_match = re.search(r"Installing Rails (\d+\.\d+)", link_text) + if version_match: + version_str = version_match.group(1) + try: + version_num = float(version_str) + if version_num > latest_version_num: + latest_version_num = version_num + latest_rails_lts_link = href + latest_version = version_str + except ValueError: + pass + + latest_version_release_date = None + + if latest_rails_lts_link: + if latest_rails_lts_link.startswith("/"): + full_url = f"https://makandracards.com{latest_rails_lts_link}" + else: + full_url = latest_rails_lts_link + + detail_response = get(full_url) + if detail_response: + detail_soup = BeautifulSoup(detail_response.text, "html.parser") + + time_tags = detail_soup.find_all( + "time", attrs={"data-relative": "true"} + ) + for time_tag in time_tags: + if "Updated" in time_tag.parent.text: + if "datetime" in time_tag.attrs: + datetime_str = time_tag["datetime"] + try: + date_obj = datetime.strptime( + datetime_str, "%Y-%m-%dT%H:%M:%SZ" + ) + latest_version_release_date = date_obj.strftime( + "%Y-%m-%d" + ) + break + except ValueError: + try: + date_obj = datetime.strptime( + datetime_str, "%Y-%m-%dT%H:%M:%S.%fZ" + ) + latest_version_release_date = date_obj.strftime( + "%Y-%m-%d" + ) + break + except ValueError as e: + print(f"Error parsing datetime: {e}") + + if not latest_version_release_date: + latest_version_release_date = datetime.now().strftime("%Y-%m-%d") + return latest_version, latest_version_release_date + + except Exception as e: + print(f"Error getting Rails LTS information: {str(e)}") + return "Unknown", datetime.now().strftime("%Y-%m-%d") + + +def get_version_release_date(dependency, version): + if dependency == "rails lts": + _, release_date = get_upstream_version(dependency) + return release_date + + try: + response = get(f"{API_V1_ENDPOINT}/{dependency}.json") + if not response: + print(f"Failed to get version info for {dependency}") + return "Unknown" + + version_info = response.json() + for version_data in version_info: + if version_data["number"] == version and "created_at" in version_data: + created_at = version_data["created_at"] + release_date = datetime.strptime( + created_at, "%Y-%m-%dT%H:%M:%S.%fZ" + ).strftime("%Y-%m-%d") + return release_date + except Exception as e: + print(f"Error getting release date for {dependency} {version}: {str(e)}") + + return "Unknown" def get_last_supported_version(bundle_install_output, dependency): """get up-to-date supported version""" pattern = r" ([^\s]+)" - last_supported_version = re.search(dependency + pattern, bundle_install_output, flags=re.I | re.M) + last_supported_version = re.search( + dependency + pattern, bundle_install_output, flags=re.I | re.M + ) return last_supported_version[1] @@ -86,6 +238,7 @@ def isUptodate(last_supported_version, latest_version): return up_to_date + def main(): # Read the JSON file with open(JSON_FILE) as file: @@ -99,23 +252,41 @@ def main(): package = item["Package name"] package = package.lower().replace("::", "-") - latest_version = get_upstream_version(package) - - if not package in ["rails lts", "rails-api"]: - last_supported_version = get_last_supported_version(bundle_install_output, package) + latest_version, release_date = get_upstream_version(package) + if package not in ["rails lts", "rails-api"]: + last_supported_version = get_last_supported_version( + bundle_install_output, package + ) else: last_supported_version = latest_version + last_supported_version_release_date = get_version_release_date( + package, last_supported_version + ) + up_to_date = isUptodate(last_supported_version, latest_version) + days_behind = "0 day/s" + if ( + up_to_date == "No" + and release_date != "Unknown" + and last_supported_version_release_date != "Unknown" + ): + days = estimate_days_behind( + release_date, last_supported_version_release_date + ) + days_behind = f"{days} day/s" + item.update( { "Last Supported Version": last_supported_version, "Latest version": latest_version, "Up-to-date": up_to_date, + "Release date": release_date, + "Latest Version Published At": last_supported_version_release_date, + "Days behind": days_behind, }, ) - # Create a DataFrame from the list of dictionaries df = DataFrame(items) df.insert(len(df.columns) - 1, "Cloud Native", df.pop("Cloud Native")) @@ -132,5 +303,6 @@ def main(): with open(REPORT_FILE, "w") as file: file.write(final_markdown) + if __name__ == "__main__": main()