From 2654c9f5e109d2a2bec1d405a18909a87b20937c Mon Sep 17 00:00:00 2001 From: Crinibus <57172157+Crinibus@users.noreply.github.com> Date: Sun, 24 Dec 2023 01:09:13 +0100 Subject: [PATCH 1/3] Add function 'log_exception' and use it as a decorator for database functions --- scraper/database/functions.py | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/scraper/database/functions.py b/scraper/database/functions.py index a5a68200..a9f06d9f 100644 --- a/scraper/database/functions.py +++ b/scraper/database/functions.py @@ -1,29 +1,43 @@ from sqlmodel import Session, select, col +import logging from scraper.models.product import ProductInfo from .db import engine from .models import Product, DataPoint +def log_exception(func): + def inner(*args, **kwargs): + try: + return func(*args, **kwargs) + except Exception as ex: + logging.getLogger(func.__name__).exception(f"Function {func.__name__} raised an exception") + + return inner + + +@log_exception def delete_all(elements: list[Product | DataPoint]) -> None: with Session(engine) as session: for element in elements: session.delete(element) session.commit() - +@log_exception def add(element: Product | DataPoint) -> None: with Session(engine) as session: session.add(element) session.commit() +@log_exception def add_all(elements: list[Product | DataPoint]) -> None: with Session(engine) as session: session.add_all(elements) session.commit() +@log_exception def get_all_products(select_only_active: bool = False) -> list[Product]: with Session(engine) as session: query = select(Product) @@ -34,41 +48,49 @@ def get_all_products(select_only_active: bool = False) -> list[Product]: return session.exec(query).all() +@log_exception def get_all_datapoints() -> list[DataPoint]: with Session(engine) as session: return session.exec(select(DataPoint)).all() +@log_exception def get_all_unique_categories() -> list[str]: with Session(engine) as session: return session.exec(select(Product.category).distinct()).all() +@log_exception def get_all_unique_domains() -> list[str]: with Session(engine) as session: return session.exec(select(Product.domain).distinct()).all() +@log_exception def get_product_by_product_code(product_code: str) -> Product | None: with Session(engine) as session: return session.exec(select(Product).where(Product.product_code == product_code)).first() +@log_exception def get_products_by_product_codes(product_codes: list[str]) -> list[Product]: with Session(engine) as session: return session.exec(select(Product).where(col(Product.product_code).in_(product_codes))).all() +@log_exception def get_products_by_categories(categories: list[str]) -> list[Product]: with Session(engine) as session: return session.exec(select(Product).where(col(Product.category).in_(categories))).all() +@log_exception def get_products_by_names(names: list[str]) -> list[Product]: with Session(engine) as session: return session.exec(select(Product).where(col(Product.name).in_(names))).all() +@log_exception def get_products_by_names_fuzzy(names: list[str]) -> list[Product]: with Session(engine) as session: matched_products = [] @@ -81,6 +103,7 @@ def get_products_by_names_fuzzy(names: list[str]) -> list[Product]: return matched_products +@log_exception def get_products_by_domains(domains: list[str], select_only_active: bool = False) -> list[Product]: with Session(engine) as session: query = select(Product).where(col(Product.domain).in_(domains)) @@ -91,6 +114,7 @@ def get_products_by_domains(domains: list[str], select_only_active: bool = False return session.exec(query).all() +@log_exception def get_datapoints_by_categories(categories: list[str]) -> list[DataPoint]: with Session(engine) as session: products = session.exec(select(Product).where(col(Product.category).in_(categories))).all() @@ -99,6 +123,7 @@ def get_datapoints_by_categories(categories: list[str]) -> list[DataPoint]: return datapoints +@log_exception def get_datapoints_by_names(names: list[str]) -> list[DataPoint]: with Session(engine) as session: products = session.exec(select(Product).where(col(Product.name).in_(names))).all() @@ -107,6 +132,7 @@ def get_datapoints_by_names(names: list[str]) -> list[DataPoint]: return datapoints +@log_exception def get_datapoints_by_product_codes(product_codes: list[str]) -> list[DataPoint]: with Session(engine) as session: products = session.exec(select(Product).where(col(Product.product_code).in_(product_codes))).all() @@ -115,11 +141,13 @@ def get_datapoints_by_product_codes(product_codes: list[str]) -> list[DataPoint] return datapoints +@log_exception def get_all_products_with_datapoints(select_only_active: bool = False) -> list[ProductInfo]: products = get_all_products(select_only_active=select_only_active) return get_product_infos_from_products(products) +@log_exception def get_product_infos_from_products(products: list[Product]) -> list[ProductInfo]: with Session(engine) as session: product_infos: list[ProductInfo] = [] @@ -144,11 +172,13 @@ def get_product_infos_from_products(products: list[Product]) -> list[ProductInfo return product_infos +@log_exception def get_all_products_grouped_by_domains(select_only_active: bool = False) -> list[list[Product]]: all_products = get_all_products(select_only_active=select_only_active) return group_products_by_domains(all_products) +@log_exception def group_products_by_domains(products: list[Product]) -> list[list[Product]]: grouped_products = [] @@ -165,6 +195,7 @@ def group_products_by_domains(products: list[Product]) -> list[list[Product]]: return grouped_products +@log_exception def group_products_by_names(products: list[Product]) -> list[list[Product]]: grouped_products = [] From 3088e45855bebcb1e05e522d997fdc05b08861e8 Mon Sep 17 00:00:00 2001 From: Crinibus <57172157+Crinibus@users.noreply.github.com> Date: Fri, 29 Dec 2023 22:57:09 +0100 Subject: [PATCH 2/3] Move function 'log_exception' decorator to exceptions.py and re-raise error in decorator --- scraper/database/functions.py | 13 ++----------- scraper/exceptions.py | 13 +++++++++++++ 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/scraper/database/functions.py b/scraper/database/functions.py index a9f06d9f..13a60a1c 100644 --- a/scraper/database/functions.py +++ b/scraper/database/functions.py @@ -1,21 +1,11 @@ from sqlmodel import Session, select, col -import logging +from scraper.exceptions import log_exception from scraper.models.product import ProductInfo from .db import engine from .models import Product, DataPoint -def log_exception(func): - def inner(*args, **kwargs): - try: - return func(*args, **kwargs) - except Exception as ex: - logging.getLogger(func.__name__).exception(f"Function {func.__name__} raised an exception") - - return inner - - @log_exception def delete_all(elements: list[Product | DataPoint]) -> None: with Session(engine) as session: @@ -23,6 +13,7 @@ def delete_all(elements: list[Product | DataPoint]) -> None: session.delete(element) session.commit() + @log_exception def add(element: Product | DataPoint) -> None: with Session(engine) as session: diff --git a/scraper/exceptions.py b/scraper/exceptions.py index c9545bc3..6ac5ba66 100644 --- a/scraper/exceptions.py +++ b/scraper/exceptions.py @@ -1,6 +1,19 @@ +from typing import Callable +import logging from scraper.constants import URL_SCHEMES +def log_exception(func: Callable): + def inner(*args, **kwargs): + try: + return func(*args, **kwargs) + except Exception as ex: + logging.getLogger(func.__name__).exception(f"Function '{func.__name__}' raised an exception") + raise ex + + return inner + + class WebsiteNotSupported(Exception): def __init__(self, website_name: str, *args: object) -> None: super().__init__(*args) From a7db9596a051356dce17c4d76d386d5f8d27a9f9 Mon Sep 17 00:00:00 2001 From: Crinibus <57172157+Crinibus@users.noreply.github.com> Date: Fri, 29 Dec 2023 22:59:09 +0100 Subject: [PATCH 3/3] Splitting one line in exceptions.py into two lines --- scraper/exceptions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scraper/exceptions.py b/scraper/exceptions.py index 6ac5ba66..d5732025 100644 --- a/scraper/exceptions.py +++ b/scraper/exceptions.py @@ -38,4 +38,5 @@ def __init__(self, url, *args: object) -> None: self.url = url def __str__(self) -> str: - return f"Missing schema in url '{self.url}'. Consider prefixing the url with one of following schemes: {', '.join(URL_SCHEMES)}" + url_schemes = ", ".join(URL_SCHEMES) + return f"Missing schema in url '{self.url}'. Consider prefixing the url with one of following schemes: {url_schemes}"