From 4a84ecd4dd2e5c459def9b069e0decc68c861ede Mon Sep 17 00:00:00 2001 From: Glenn Snyder Date: Thu, 11 Aug 2022 15:57:49 -0400 Subject: [PATCH 1/2] added exception handling to deal with the situation where a project is deleted during the loop to iterate over the list of projects and gather data about them --- sage.py | 14 +++++++++++++- test_data.py | 4 ++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/sage.py b/sage.py index 0096771..07c46cd 100644 --- a/sage.py +++ b/sage.py @@ -8,6 +8,7 @@ import logging import os from pathlib import Path +from requests.exceptions import HTTPError import sys # TODO: Find scans (code locations) whose scan frequency is higher than we recommend @@ -135,11 +136,21 @@ def _get_data(self): logging.info("Fetched %i projects", len(projects)) total_versions = 0 project_count = 0 + # + # When Sage runs for a long time there is a chance someone deletes a project while it is gathering data + # about the projects. To defend against that we catch the error and build a list of errant projects + # and remove them from the list (below) + errant_projects = list() for project in projects: project_count += 1 project_name = project['name'] print("Project ({}/{}): {}; versions:".format(project_count, len(projects), project_name), end='', flush=True) - versions = list(hub.get_resource('versions', project, headers={'accept': "application/vnd.blackducksoftware.project-detail-5+json"})) + try: + versions = list(hub.get_resource('versions', project, headers={'accept': "application/vnd.blackducksoftware.project-detail-5+json"})) + except HTTPError: + logging.warning(f"HTTPError when attempting retrieve versions for project {project['name']}, removing project") + errant_projects.append(project['name']) + continue print(len(versions)) for version in versions: version_name = version['versionName'] @@ -160,6 +171,7 @@ def _get_data(self): project['versions'] = versions project['num_versions'] = len(versions) total_versions += len(versions) + projects = [p for p in projects if p['name'] not in errant_projects] projects = [self._copy_common_attributes(p) for p in projects] self.data['projects'] = projects diff --git a/test_data.py b/test_data.py index bddac54..7297b1d 100644 --- a/test_data.py +++ b/test_data.py @@ -20,8 +20,8 @@ def create_project_versions(hub_instance, project_name, versions=[], project_par if response.status_code == 201: project = hub_instance.get_project_by_name(project_name) - for version in range(20): - response = hub_instance.create_project_version(project, version) + for version in versions: + response = hub_instance.create_project_version(project, str(version)) if response.status_code == 201: logging.debug("Created version {}".format(version)) else: From 01bd14749c8ddb53bbe17c281279fedeb559da95 Mon Sep 17 00:00:00 2001 From: Glenn Snyder Date: Fri, 9 Sep 2022 09:54:35 -0400 Subject: [PATCH 2/2] better error handling --- sage.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sage.py b/sage.py index 07c46cd..824adf5 100644 --- a/sage.py +++ b/sage.py @@ -171,6 +171,7 @@ def _get_data(self): project['versions'] = versions project['num_versions'] = len(versions) total_versions += len(versions) + projects = [p for p in projects if p['name'] not in errant_projects] projects = [self._copy_common_attributes(p) for p in projects] self.data['projects'] = projects