feat: add support for cloning GitLab repositories

scripts/dev_scripts/integration_tests.sh

@@ -36,6 +36,15 @@ echo "micronaut-projects/micronaut-core: Analyzing the repo path and the branch
 echo -e "----------------------------------------------------------------------------------\n"
 $RUN_MACARON analyze -rp https://github.com/micronaut-projects/micronaut-core -b 3.5.x --skip-deps || log_fail
 
+echo -e "\n----------------------------------------------------------------------------------"
+echo "gitlab.com/tinyMediaManager/tinyMediaManager: Analyzing the repo path and the branch name when automatic dependency resolution is skipped."
+echo -e "----------------------------------------------------------------------------------\n"
+JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/tinyMediaManager/tinyMediaManager.json
+JSON_RESULT=$WORKSPACE/output/reports/gitlab_com/tinyMediaManager/tinyMediaManager/tinyMediaManager.json
+$RUN_MACARON analyze -rp https://gitlab.com/tinyMediaManager/tinyMediaManager -b main -d cca6b67a335074eca42136556f0a321f75dc4f48 --skip-deps || log_fail
+
+python $COMPARE_JSON_OUT $JSON_RESULT $JSON_EXPECTED || log_fail
+
 echo -e "\n----------------------------------------------------------------------------------"
 echo "jenkinsci/plot-plugin: Analyzing the repo path, the branch name and the commit digest when automatic dependency resolution is skipped."
 echo -e "----------------------------------------------------------------------------------\n"

scripts/release_scripts/run_macaron.sh

@@ -413,6 +413,8 @@ docker run \
     -e "USER_UID=${USER_UID}" \
     -e "USER_GID=${USER_GID}" \
     -e "GITHUB_TOKEN=${GITHUB_TOKEN}" \
+    -e "MCN_PUBLIC_GITLAB_TOKEN=${MCN_PUBLIC_GITLAB_TOKEN}" \
+    -e "MCN_PRIVATE_GITLAB_TOKEN=${MCN_PRIVATE_GITLAB_TOKEN}" \
     "${proxy_vars[@]}" \
     "${prod_vars[@]}" \
     "${mounts[@]}" \

src/macaron/__main__.py

@@ -16,10 +16,12 @@
 from macaron.config.defaults import create_defaults, load_defaults
 from macaron.config.global_config import global_config
 from macaron.config.target_config import TARGET_CONFIG_SCHEMA
+from macaron.errors import ConfigurationError
 from macaron.output_reporter.reporter import HTMLReporter, JSONReporter, PolicyReporter
 from macaron.parsers.yaml.loader import YamlLoader
 from macaron.policy_engine.policy_engine import run_policy_engine, show_prelude
 from macaron.slsa_analyzer.analyzer import Analyzer
+from macaron.slsa_analyzer.git_service import GIT_SERVICES
 
 logger: logging.Logger = logging.getLogger(__name__)
 
@@ -134,6 +136,16 @@ def perform_action(action_args: argparse.Namespace) -> None:
                 logger.error("GitHub access token not set.")
                 sys.exit(os.EX_USAGE)
             global_config.gh_token = gh_token
+
+            # TODO: Here we should try to statically analyze the config before
+            # actually running the analysis.
+            try:
+                for git_service in GIT_SERVICES:
+                    git_service.load_defaults()
+            except ConfigurationError as error:
+                logger.error(error)
+                sys.exit(os.EX_USAGE)
+
             analyze_slsa_levels_single(action_args)
         case _:
             logger.error("Macaron does not support command option %s.", action_args.action)

src/macaron/config/defaults.ini

@@ -58,14 +58,26 @@ parent_limit = 10
 # E.g. com.oracle.coherence.ce:coherence
 artifact_ignore_list =
 
-[git]
-# The list of allowed git hosts.
-# Host names are separated by spaces and they can be defined in multiple lines.
-# Duplicated host names are ignored.
-allowed_hosts =
-    github.com
-    ol-bitbucket.us.oracle.com
-    gitlab.com
+# Git services that Macaron has access to clone repositories.
+# For security purposes, Macaron will only clone repositories from the domains specified.
+
+# Access to GitHub is required in most case for Macaron to analyse not only the main
+# repo but also its dependencies.
+[git_service.github]
+domain = github.com
+
+# Access to public GitLab (gitlab.com).
+# An optional access token can be provided through the `MCN_PUBLIC_GITLAB_TOKEN` environment variable.
+# This access token is optional, only necessary when you need to clone private repositories.
+# The `read_repository` permission is required for this token.
+[git_service.gitlab.public]
+domain = gitlab.com
+
+# Access to a private GitLab instance (e.g. your organization's self-hosted GitLab instance).
+# If this section is enabled, an access token must be provided through the `MCN_PRIVATE_GITLAB_TOKEN` environment variable.
+# The `read_repository` permission is required for this token.
+# [git_service.gitlab.private]
+# domain = example.org
 
 # This is the spec for trusted Maven build tools.
 [builder.maven]

src/macaron/errors.py

@@ -22,3 +22,11 @@ class CUEExpectationError(MacaronError):
 
 class CUERuntimeError(MacaronError):
     """Happens when there are errors in CUE expectation validation."""
+
+
+class ConfigurationError(MacaronError):
+    """Happens when there is an error in the configuration (.ini) file."""
+
+
+class CloneError(MacaronError):
+    """Happens when cannot clone a git repository."""

src/macaron/slsa_analyzer/analyzer.py

@@ -27,6 +27,7 @@
     NoneDependencyAnalyzer,
 )
 from macaron.dependency_analyzer.cyclonedx import get_deps_from_sbom
+from macaron.errors import CloneError
 from macaron.output_reporter.reporter import FileReporter
 from macaron.output_reporter.results import Record, Report, SCMStatus
 from macaron.slsa_analyzer import git_url
@@ -527,13 +528,13 @@ def _prepare_repo(
                 return None
 
             git_service = self.get_git_service(resolved_remote_path)
-            if not git_service.can_clone_remote_repo(resolved_remote_path):
-                logger.error("Cannot clone the remote repo at %s", resolved_remote_path)
-                return None
-
             repo_unique_path = git_url.get_repo_dir_name(resolved_remote_path)
             resolved_local_path = os.path.join(target_dir, repo_unique_path)
-            git_url.clone_remote_repo(resolved_local_path, resolved_remote_path)
+            try:
+                git_service.clone_repo(resolved_local_path, resolved_remote_path)
+            except CloneError as error:
+                logger.error("Cannot clone %s: %s", resolved_remote_path, str(error))
+                return None
         else:
             logger.info("The path to repo %s is a local path.", repo_path)
             resolved_local_path = self._resolve_local_path(self.local_repos_path, repo_path)
@@ -577,7 +578,6 @@ def get_git_service(remote_path: str) -> BaseGitService:
             The git service derived from the remote path.
         """
         for git_service in GIT_SERVICES:
-            git_service.load_defaults()
             if git_service.is_detected(remote_path):
                 return git_service
 

src/macaron/slsa_analyzer/git_service/__init__.py

@@ -1,13 +1,13 @@
-# Copyright (c) 2022 - 2022, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2022 - 2023, Oracle and/or its affiliates. All rights reserved.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/.
 
 """The git_service package contains the supported git services for Macaron."""
 
 from .base_git_service import BaseGitService
 from .bitbucket import BitBucket
 from .github import GitHub
-from .gitlab import GitLab
+from .gitlab import PrivateGitLab, PublicGitLab
 
 # The list of supported git services. The order of the list determines the order
 # in which each git service is checked against the target repository.
-GIT_SERVICES: list[BaseGitService] = [GitHub(), GitLab(), BitBucket()]
+GIT_SERVICES: list[BaseGitService] = [GitHub(), PublicGitLab(), PrivateGitLab(), BitBucket()]

src/macaron/slsa_analyzer/git_service/base_git_service.py

@@ -1,10 +1,14 @@
-# Copyright (c) 2022 - 2022, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2022 - 2023, Oracle and/or its affiliates. All rights reserved.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/.
 
 """This module contains the BaseGitService class to be inherited by a git service."""
 
 from abc import abstractmethod
 
+from macaron.config.defaults import defaults
+from macaron.errors import CloneError, ConfigurationError
+from macaron.slsa_analyzer import git_url
+
 
 class BaseGitService:
     """This abstract class is used to implement git services."""
@@ -18,15 +22,56 @@ def __init__(self, name: str) -> None:
             The name of the git service.
         """
         self.name = name
+        self.domain: str | None = None
 
     @abstractmethod
     def load_defaults(self) -> None:
-        """Load the default values from defaults.ini."""
+        """Load the values for this git service from the ini configuration."""
         raise NotImplementedError
 
-    @abstractmethod
+    def load_domain(self, section_name: str) -> str | None:
+        """Load the domain of the git service from the ini configuration section ``section_name``.
+
+        The section may or may not be available in the configuration. In both cases,
+        the method should not raise ``ConfigurationError``.
+
+        Meanwhile, if the section is present but there is a schema violation (e.g. a key such as
+        ``domain`` is missing), this method will raise a ``ConfigurationError``.
+
+        Parameters
+        ----------
+        section_name : str
+            The name of the git service section in the ini configuration file.
+
+        Returns
+        -------
+        str | None
+            The domain. This can be ``None`` if the git service section is not found in
+            the ini configuration file, meaning the user does not enable the
+            corresponding git service.
+
+        Raises
+        ------
+        ConfigurationError
+            If there is a schema violation in the git service section.
+        """
+        if not defaults.has_section(section_name):
+            # We do not raise ConfigurationError here because it is not compulsory
+            # to have all available git services in the ini config.
+            return None
+        section = defaults[section_name]
+        domain = section.get("domain")
+        if not domain:
+            raise ConfigurationError(
+                f'The "domain" key is missing in section [{section_name}] of the .ini configuration file.'
+            )
+        return domain
+
     def is_detected(self, url: str) -> bool:
-        """Return True if the remote repo is using this git service.
+        """Check if the remote repo at the given ``url`` is hosted on this git service.
+
+        This check is done by checking the URL of the repo against the domain of this
+        git service.
 
         Parameters
         ----------
@@ -36,25 +81,36 @@ def is_detected(self, url: str) -> bool:
         Returns
         -------
         bool
-            True if this git service is detected else False.
+            True if the repo is indeed hosted on this git service.
         """
-        raise NotImplementedError
+        if self.domain is None:
+            return False
+        return (
+            git_url.parse_remote_url(
+                url,
+                allowed_git_service_domains=[self.domain],
+            )
+            is not None
+        )
 
     @abstractmethod
-    def can_clone_remote_repo(self, url: str) -> bool:
-        """Return True if the remote repository can be cloned.
+    def clone_repo(self, clone_dir: str, url: str) -> None:
+        """Clone a repository.
 
         Parameters
         ----------
+        clone_dir: str
+            The name of the directory to clone into.
+            This is equivalent to the <directory> argument of ``git clone``.
         url : str
-            The remote url.
+            The url to the repository.
 
-        Returns
-        -------
-        bool
-            True if the repo can be cloned, else False.
+        Raises
+        ------
+        CloneError
+            If there is an error cloning the repo.
         """
-        raise NotImplementedError
+        raise NotImplementedError()
 
 
 class NoneGitService(BaseGitService):
@@ -65,7 +121,11 @@ def __init__(self) -> None:
         super().__init__("")
 
     def load_defaults(self) -> None:
-        """Load the default values from defaults.ini."""
+        """Load the values for this git service from the ini configuration.
+
+        In this particular case, since this class represents a ``None`` git service,
+        we do nothing.
+        """
 
     def is_detected(self, url: str) -> bool:
         """Return True if the remote repo is using this git service.
@@ -82,17 +142,15 @@ def is_detected(self, url: str) -> bool:
         """
         return False
 
-    def can_clone_remote_repo(self, url: str) -> bool:
-        """Return True if the remote repository can be cloned.
+    def clone_repo(self, _clone_dir: str, url: str) -> None:
+        """Clone a repo.
 
-        Parameters
-        ----------
-        url : str
-            The remote url.
+        In this particular case, since this class represents a ``None`` git service,
+        we do nothing but raise a ``CloneError``.
 
-        Returns
-        -------
-        bool
-            True if the repo can be cloned, else False.
+        Raises
+        ------
+        CloneError
+            Always raise, since this method should not be used to clone any repository.
         """
-        return False
+        raise CloneError(f"Internal error encountered when cloning the repo '{url}'.")

src/macaron/slsa_analyzer/git_service/bitbucket.py

@@ -1,11 +1,10 @@
-# Copyright (c) 2022 - 2022, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2022 - 2023, Oracle and/or its affiliates. All rights reserved.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/.
 
 """This module contains the spec for the BitBucket service."""
 
 import logging
 
-from macaron.slsa_analyzer import git_url
 from macaron.slsa_analyzer.git_service.base_git_service import BaseGitService
 
 logger: logging.Logger = logging.getLogger(__name__)
@@ -19,39 +18,11 @@ def __init__(self) -> None:
         super().__init__("bitbucket")
 
     def load_defaults(self) -> None:
-        """Load the default values from defaults.ini."""
+        """Load the values for this git service from the ini configuration."""
+        # TODO: implement this once support for BitBucket is added.
+        return None
 
-    def can_clone_remote_repo(self, url: str) -> bool:
-        """Return True if the remote repository can be cloned.
-
-        Parameters
-        ----------
-        url : str
-            The remote url.
-
-        Returns
-        -------
-        bool
-            True if the repo can be cloned, else False.
-        """
+    def clone_repo(self, _clone_dir: str, _url: str) -> None:
+        """Clone a BitBucket repo."""
+        # TODO: implement this once support for BitBucket is added.
         logger.info("Cloning BitBucket repositories is not supported yet. Please clone the repository manually.")
-        return False
-
-    def is_detected(self, url: str) -> bool:
-        """Return True if the remote repo is using this git service.
-
-        Parameters
-        ----------
-        url : str
-            The url of the remote repo.
-
-        Returns
-        -------
-        bool
-            True if this git service is detected else False.
-        """
-        parsed_url = git_url.parse_remote_url(url)
-        if not parsed_url or self.name not in parsed_url.netloc:
-            return False
-
-        return True