feat: support validating CI lint results

nejch · JohnVillalovos · commit 3b1ede4a27cd · 2022-07-09T08:22:20.000-07:00
diff --git a/docs/cli-examples.rst b/docs/cli-examples.rst
@@ -15,6 +15,9 @@ Lint a CI YAML configuration from a string:
 
    To see output, you will need to use the ``-v``/``--verbose`` flag.
 
+   To exit with non-zero on YAML lint failures instead, use the ``validate``
+   subcommand shown below.
+
 .. code-block:: console
 
    $ gitlab --verbose ci-lint create --content \
@@ -30,12 +33,24 @@ Lint a CI YAML configuration from a file (see :ref:`cli_from_files`):
 
    $ gitlab --verbose ci-lint create --content @.gitlab-ci.yml
 
+Validate a CI YAML configuration from a file (lints and exits with non-zero on failure):
+
+.. code-block:: console
+
+   $ gitlab ci-lint validate --content @.gitlab-ci.yml
+
 Lint a project's CI YAML configuration:
 
 .. code-block:: console
 
    $ gitlab --verbose project-ci-lint create --project-id group/my-project --content @.gitlab-ci.yml
 
+Validate a project's CI YAML configuration (lints and exits with non-zero on failure):
+
+.. code-block:: console
+
+   $ gitlab project-ci-lint validate --project-id group/my-project --content @.gitlab-ci.yml
+
 Lint a project's current CI YAML configuration:
 
 .. code-block:: console
diff --git a/docs/gl_objects/ci_lint.rst b/docs/gl_objects/ci_lint.rst
@@ -19,7 +19,7 @@ Reference
 Examples
 ---------
 
-Validate a CI YAML configuration::
+Lint a CI YAML configuration::
 
     gitlab_ci_yml = """.api_test:
       rules:
@@ -40,14 +40,30 @@ Validate a CI YAML configuration::
     print(lint_result.status)  # Print the status of the CI YAML
     print(lint_result.merged_yaml)  # Print the merged YAML file
 
-Validate a project's CI configuration::
+Lint a project's CI configuration::
 
     lint_result = project.ci_lint.get()
     assert lint_result.valid is True  # Test that the .gitlab-ci.yml is valid
     print(lint_result.merged_yaml)    # Print the merged YAML file
 
-Validate a CI YAML configuration with a namespace::
+Lint a CI YAML configuration with a namespace::
 
     lint_result = project.ci_lint.create({"content": gitlab_ci_yml})
     assert lint_result.valid is True  # Test that the .gitlab-ci.yml is valid
     print(lint_result.merged_yaml)    # Print the merged YAML file
+
+Validate a CI YAML configuration (raises ``GitlabCiLintError`` on failures)::
+
+    # returns None
+    gl.ci_lint.validate({"content": gitlab_ci_yml})
+
+    # raises GitlabCiLintError
+    gl.ci_lint.validate({"content": "invalid"})
+
+Validate a CI YAML configuration with a namespace::
+
+    # returns None
+    project.ci_lint.validate({"content": gitlab_ci_yml})
+
+    # raises GitlabCiLintError
+    project.ci_lint.validate({"content": "invalid"})
diff --git a/gitlab/exceptions.py b/gitlab/exceptions.py
@@ -62,6 +62,10 @@ class GitlabParsingError(GitlabError):
     pass
 
 
+class GitlabCiLintError(GitlabError):
+    pass
+
+
 class GitlabConnectionError(GitlabError):
     pass
 
diff --git a/gitlab/v4/cli.py b/gitlab/v4/cli.py
@@ -25,6 +25,7 @@
 import gitlab.base
 import gitlab.v4.objects
 from gitlab import cli
+from gitlab.exceptions import GitlabCiLintError
 
 
 class GitlabCLI:
@@ -133,6 +134,16 @@ def do_project_export_download(self) -> None:
         except Exception as e:  # pragma: no cover, cli.die is unit-tested
             cli.die("Impossible to download the export", e)
 
+    def do_validate(self) -> None:
+        if TYPE_CHECKING:
+            assert isinstance(self.mgr, gitlab.v4.objects.CiLintManager)
+        try:
+            self.mgr.validate(self.args)
+        except GitlabCiLintError as e:  # pragma: no cover, cli.die is unit-tested
+            cli.die("CI YAML Lint failed", e)
+        except Exception as e:  # pragma: no cover, cli.die is unit-tested
+            cli.die("Cannot validate CI YAML", e)
+
     def do_create(self) -> gitlab.base.RESTObject:
         if TYPE_CHECKING:
             assert isinstance(self.mgr, gitlab.mixins.CreateMixin)
diff --git a/gitlab/v4/objects/ci_lint.py b/gitlab/v4/objects/ci_lint.py
@@ -6,6 +6,8 @@
 from typing import Any, cast
 
 from gitlab.base import RESTManager, RESTObject
+from gitlab.cli import register_custom_action
+from gitlab.exceptions import GitlabCiLintError
 from gitlab.mixins import CreateMixin, GetWithoutIdMixin
 from gitlab.types import RequiredOptional
 
@@ -28,9 +30,24 @@ class CiLintManager(CreateMixin, RESTManager):
         required=("content",), optional=("include_merged_yaml", "include_jobs")
     )
 
+    @register_custom_action(
+        "CiLintManager",
+        ("content",),
+        optional=("include_merged_yaml", "include_jobs"),
+    )
+    def validate(self, *args: Any, **kwargs: Any) -> None:
+        """Raise an error if the CI Lint results are not valid.
+
+        This is a custom python-gitlab method to wrap lint endpoints."""
+        result = self.create(*args, **kwargs)
+
+        if result.status != "valid":
+            message = ",\n".join(result.errors)
+            raise GitlabCiLintError(message)
+
 
 class ProjectCiLint(RESTObject):
-    pass
+    _id_attr = None
 
 
 class ProjectCiLintManager(GetWithoutIdMixin, CreateMixin, RESTManager):
@@ -43,3 +60,18 @@ class ProjectCiLintManager(GetWithoutIdMixin, CreateMixin, RESTManager):
 
     def get(self, **kwargs: Any) -> ProjectCiLint:
         return cast(ProjectCiLint, super().get(**kwargs))
+
+    @register_custom_action(
+        "ProjectCiLintManager",
+        ("content",),
+        optional=("dry_run", "include_jobs", "ref"),
+    )
+    def validate(self, *args: Any, **kwargs: Any) -> None:
+        """Raise an error if the Project CI Lint results are not valid.
+
+        This is a custom python-gitlab method to wrap lint endpoints."""
+        result = self.create(*args, **kwargs)
+
+        if not result.valid:
+            message = ",\n".join(result.errors)
+            raise GitlabCiLintError(message)
diff --git a/tests/conftest.py b/tests/conftest.py
@@ -4,3 +4,16 @@
 @pytest.fixture(scope="session")
 def test_dir(pytestconfig):
     return pytestconfig.rootdir / "tests"
+
+
+@pytest.fixture
+def valid_gitlab_ci_yml():
+    return """---
+:test_job:
+  :script: echo 1
+"""
+
+
+@pytest.fixture
+def invalid_gitlab_ci_yml():
+    return "invalid"
diff --git a/tests/functional/cli/test_cli_v4.py b/tests/functional/cli/test_cli_v4.py
@@ -22,6 +22,59 @@ def test_update_project(gitlab_cli, project):
     assert description in ret.stdout
 
 
+def test_create_ci_lint(gitlab_cli, valid_gitlab_ci_yml):
+    cmd = ["ci-lint", "create", "--content", valid_gitlab_ci_yml]
+    ret = gitlab_cli(cmd)
+
+    assert ret.success
+
+
+def test_validate_ci_lint(gitlab_cli, valid_gitlab_ci_yml):
+    cmd = ["ci-lint", "validate", "--content", valid_gitlab_ci_yml]
+    ret = gitlab_cli(cmd)
+
+    assert ret.success
+
+
+def test_validate_ci_lint_invalid_exits_non_zero(gitlab_cli, invalid_gitlab_ci_yml):
+    cmd = ["ci-lint", "validate", "--content", invalid_gitlab_ci_yml]
+    ret = gitlab_cli(cmd)
+
+    assert not ret.success
+    assert "CI YAML Lint failed (Invalid configuration format)" in ret.stderr
+
+
+def test_validate_project_ci_lint(gitlab_cli, project, valid_gitlab_ci_yml):
+    cmd = [
+        "project-ci-lint",
+        "validate",
+        "--project-id",
+        project.id,
+        "--content",
+        valid_gitlab_ci_yml,
+    ]
+    ret = gitlab_cli(cmd)
+
+    assert ret.success
+
+
+def test_validate_project_ci_lint_invalid_exits_non_zero(
+    gitlab_cli, project, invalid_gitlab_ci_yml
+):
+    cmd = [
+        "project-ci-lint",
+        "validate",
+        "--project-id",
+        project.id,
+        "--content",
+        invalid_gitlab_ci_yml,
+    ]
+    ret = gitlab_cli(cmd)
+
+    assert not ret.success
+    assert "CI YAML Lint failed (Invalid configuration format)" in ret.stderr
+
+
 def test_create_group(gitlab_cli):
     name = "test-group1"
     path = "group1"
diff --git a/tests/unit/objects/test_ci_lint.py b/tests/unit/objects/test_ci_lint.py
@@ -1,12 +1,14 @@
 import pytest
 import responses
 
-gitlab_ci_yml = """---
-:test_job:
-  :script: echo 1
-"""
+from gitlab import exceptions
 
 ci_lint_create_content = {"status": "valid", "errors": [], "warnings": []}
+ci_lint_create_invalid_content = {
+    "status": "invalid",
+    "errors": ["invalid format"],
+    "warnings": [],
+}
 
 
 project_ci_lint_content = {
@@ -30,6 +32,19 @@ def resp_create_ci_lint():
         yield rsps
 
 
+@pytest.fixture
+def resp_create_ci_lint_invalid():
+    with responses.RequestsMock() as rsps:
+        rsps.add(
+            method=responses.POST,
+            url="http://localhost/api/v4/ci/lint",
+            json=ci_lint_create_invalid_content,
+            content_type="application/json",
+            status=200,
+        )
+        yield rsps
+
+
 @pytest.fixture
 def resp_get_project_ci_lint():
     with responses.RequestsMock() as rsps:
@@ -56,16 +71,29 @@ def resp_create_project_ci_lint():
         yield rsps
 
 
-def test_ci_lint_create(gl, resp_create_ci_lint):
-    lint_result = gl.ci_lint.create({"content": gitlab_ci_yml})
+def test_ci_lint_create(gl, resp_create_ci_lint, valid_gitlab_ci_yml):
+    lint_result = gl.ci_lint.create({"content": valid_gitlab_ci_yml})
     assert lint_result.status == "valid"
 
 
+def test_ci_lint_validate(gl, resp_create_ci_lint, valid_gitlab_ci_yml):
+    gl.ci_lint.validate({"content": valid_gitlab_ci_yml})
+
+
+def test_ci_lint_validate_invalid_raises(
+    gl, resp_create_ci_lint_invalid, invalid_gitlab_ci_yml
+):
+    with pytest.raises(exceptions.GitlabCiLintError, match="invalid format"):
+        gl.ci_lint.validate({"content": invalid_gitlab_ci_yml})
+
+
 def test_project_ci_lint_get(project, resp_get_project_ci_lint):
     lint_result = project.ci_lint.get()
     assert lint_result.valid is True
 
 
-def test_project_ci_lint_create(project, resp_create_project_ci_lint):
-    lint_result = project.ci_lint.create({"content": gitlab_ci_yml})
+def test_project_ci_lint_create(
+    project, resp_create_project_ci_lint, valid_gitlab_ci_yml
+):
+    lint_result = project.ci_lint.create({"content": valid_gitlab_ci_yml})
     assert lint_result.valid is True
