Feature/harvesting metadata from a provided repository url via cloning

src/hermes/commands/base.py

@@ -104,6 +104,19 @@ def init_common_parser(self, parser: argparse.ArgumentParser) -> None:
             type=pathlib.Path,
             help="Configuration file in TOML format",
         )
+        # Add a new argument to accept a URL for harvesting (in harvest command)
+        parser.add_argument(
+            "--url",
+            type=str,
+            help="URL from which to extract metadata (GitHub or GitLab))"
+        )
+        # Add a new argument to accept a token (from GitHub or GitLab) for harvesting (in harvest command)
+        parser.add_argument(
+            "--token",
+            type=str,
+            required=False,
+            help="Access token for GitHub/GitLab (optional, only needed for private repos or GitHub/GitLab API plugin)"
+        )
 
         plugin_args = parser.add_argument_group("Extra options")
         plugin_args.add_argument(

src/hermes/commands/clean/base.py

@@ -6,6 +6,7 @@
 
 import argparse
 import shutil
+import logging
 
 from pydantic import BaseModel
 
@@ -25,6 +26,7 @@ class HermesCleanCommand(HermesCommand):
 
     def __call__(self, args: argparse.Namespace) -> None:
         self.log.info("Removing HERMES caches...")
+        logging.shutdown()
 
         # Naive implementation for now... check errors, validate directory, don't construct the path ourselves, etc.
         shutil.rmtree(args.path / '.hermes')

src/hermes/commands/harvest/base.py

@@ -7,13 +7,17 @@
 import argparse
 import typing as t
 from datetime import datetime
+import tempfile
+import pathlib
+from hermes import logger
 
 from pydantic import BaseModel
 
 from hermes.commands.base import HermesCommand, HermesPlugin
 from hermes.model.context import HermesContext, HermesHarvestContext
 from hermes.model.errors import HermesValidationError, MergeError
-
+from hermes.commands.harvest.util.token import update_token_to_toml, remove_token_from_toml
+from hermes.commands.harvest.util.clone import clone_repository
 
 class HermesHarvestPlugin(HermesPlugin):
     """Base plugin that does harvesting.
@@ -44,6 +48,30 @@ def __call__(self, args: argparse.Namespace) -> None:
         # Initialize the harvest cache directory here to indicate the step ran
         ctx.init_cache("harvest")
 
+        logger.init_logging()
+        log = logger.getLogger("hermes.cli")
+
+        if args.url:
+            with tempfile.TemporaryDirectory(dir=".") as temp_dir:
+                temp_path = pathlib.Path(temp_dir)
+                log.info(f"Cloning repository {args.url} into {temp_path}")
+
+                try:
+                    clone_repository(args.url, temp_path, recursive=True, depth=1, filter_blobs=True, sparse=False, verbose=True)
+                except Exception as exc:
+                    print("ERROR:", exc)
+                args.path = temp_path  # Overwrite args.path to temp directory
+
+                if args.token:
+                    update_token_to_toml(args.token)
+                self._harvest(ctx)
+                if args.token:
+                    remove_token_from_toml('hermes.toml')
+        else:
+            self._harvest(ctx)
+
+    def _harvest(self, ctx: HermesContext) -> None:
+        """Harvest metadata from configured sources using plugins."""
         for plugin_name in self.settings.sources:
             try:
                 plugin_func = self.plugins[plugin_name]()