google-cloud-python/.generator/cli.py at bazel-any-user · draftcode/google-cloud-python

498 lines (418 loc) · 17.2 KB
# Copyright 2025 Google LLC
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#     http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import argparse
import glob
import json
import logging
import shutil
import subprocess
from typing import Dict, List
    import synthtool
    from synthtool.languages import python_mono_repo
    SYNTHTOOL_INSTALLED = True
    SYNTHTOOL_IMPORT_ERROR = None
except ImportError as e:  # pragma: NO COVER
    SYNTHTOOL_IMPORT_ERROR = e
    SYNTHTOOL_INSTALLED = False
logger = logging.getLogger()
LIBRARIAN_DIR = "librarian"
GENERATE_REQUEST_FILE = "generate-request.json"
INPUT_DIR = "input"
BUILD_REQUEST_FILE = "build-request.json"
SOURCE_DIR = "source"
OUTPUT_DIR = "output"
REPO_DIR = "repo"
def _read_json_file(path: str) -> Dict:
    """Helper function that reads a json file path and returns the loaded json content.
        path(str): The file path to read.
    Returns:
        dict: The parsed JSON content.
    Raises:
        FileNotFoundError: If the file is not found at the specified path.
        json.JSONDecodeError: If the file does not contain valid JSON.
        IOError: If there is an issue reading the file.
    with open(path, "r") as f:
        return json.load(f)
def handle_configure():
    # TODO(https://github.com/googleapis/librarian/issues/466): Implement configure command and update docstring.
    logger.info("'configure' command executed.")
def _determine_bazel_rule(api_path: str, source: str) -> str:
    """Finds a Bazel rule by parsing the BUILD.bazel file directly.
        api_path (str): The API path, e.g., 'google/cloud/language/v1'.
        source(str): The path to the root of the Bazel workspace.
    Returns:
        str: The discovered Bazel rule, e.g., '//google/cloud/language/v1:language-v1-py'.
    Raises:
        ValueError: If the file can't be processed or no matching rule is found.
    logger.info(f"Determining Bazel rule for api_path: '{api_path}' by parsing file.")
        build_file_path = os.path.join(source, api_path, "BUILD.bazel")
        with open(build_file_path, "r") as f:
            content = f.read()
        match = re.search(r'name\s*=\s*"([^"]+-py)"', content)
        # This check is for a logical failure (no match), not a runtime exception.
        # It's good to keep it for clear error messaging.
        if not match:  # pragma: NO COVER
            raise ValueError(
                f"No Bazel rule with a name ending in '-py' found in {build_file_path}"
        rule_name = match.group(1)
        bazel_rule = f"//{api_path}:{rule_name}"
        logger.info(f"Found Bazel rule: {bazel_rule}")
        return bazel_rule
    except Exception as e:
        raise ValueError(
            f"Failed to determine Bazel rule for '{api_path}' by parsing."
        ) from e
def _get_library_id(request_data: Dict) -> str:
    """Retrieve the library id from the given request dictionary
        request_data(Dict): The contents `generate-request.json`.
    Raises:
        ValueError: If the key `id` does not exist in `request_data`.
    Returns:
        str: The id of the library in `generate-request.json`
    library_id = request_data.get("id")
    if not library_id:
        raise ValueError("Request file is missing required 'id' field.")
    return library_id
def _build_bazel_target(bazel_rule: str, source: str):
    """Executes `bazelisk build` on a given Bazel rule.
        bazel_rule(str): The Bazel rule to build.
        source(str): The path to the root of the Bazel workspace.
    Raises:
        ValueError: If the subprocess call fails.
    logger.info(f"Executing build for rule: {bazel_rule}")
        # We're using the prewarmed bazel cache from the docker image to speed up the bazelisk commands.
        # Previously built artifacts are stored in `/bazel_cache/_bazel_ubuntu/output_base` and will be
        # used to speed up the build. `disk_cache` is used as the 'remote cache' and is also prewarmed as part of
        # the docker image.
        # See https://bazel.build/remote/caching#disk-cache which explains using a file system as a 'remote cache'.
        command = [
            "bazelisk",
            # "--output_base=/bazel_cache/_bazel_ubuntu/output_base",
            "build",
            # "--disk_cache=/bazel_cache/_bazel_ubuntu/cache/repos",
            # "--incompatible_strict_action_env",
            bazel_rule,
        subprocess.run(
            command,
            cwd=source,
            text=True,
            check=True,
        logger.info(f"Bazel build for {bazel_rule} rule completed successfully.")
    except Exception as e:
        raise ValueError(f"Bazel build for {bazel_rule} rule failed.") from e
def _locate_and_extract_artifact(
    bazel_rule: str,
    library_id: str,
    source: str,
    output: str,
    api_path: str,
    """Finds and extracts the tarball artifact from a Bazel build.
        bazel_rule(str): The Bazel rule that was built.
        library_id(str): The ID of the library being generated.
        source(str): The path to the root of the Bazel workspace.
        output(str): The path to the location where generated output
            should be stored.
        api_path(str): The API path for the artifact
    Raises:
        ValueError: If failed to locate or extract artifact.
        # 1. Find the bazel-bin output directory.
        logger.info("Locating Bazel output directory...")
        # Previously built artifacts are stored in `/bazel_cache/_bazel_ubuntu/output_base`.
        # See `--output_base` in `_build_bazel_target`
        info_command = [
            "bazelisk",
            "--output_base=/bazel_cache/_bazel_ubuntu/output_base",
            "info",
            "bazel-bin",
        result = subprocess.run(
            info_command,
            cwd=source,
            text=True,
            check=True,
            capture_output=True,
        bazel_bin_path = result.stdout.strip()
        # 2. Construct the path to the generated tarball.
        rule_path, rule_name = bazel_rule.split(":")
        tarball_name = f"{rule_name}.tar.gz"
        tarball_path = os.path.join(bazel_bin_path, rule_path.strip("/"), tarball_name)
        logger.info(f"Found artifact at: {tarball_path}")
        # 3. Create a staging directory.
        api_version = api_path.split("/")[-1]
        staging_dir = os.path.join(output, "owl-bot-staging", library_id, api_version)
        os.makedirs(staging_dir, exist_ok=True)
        logger.info(f"Preparing staging directory: {staging_dir}")
        # 4. Extract the artifact.
        extract_command = ["tar", "-xvf", tarball_path, "--strip-components=1"]
        subprocess.run(
            extract_command, cwd=staging_dir, capture_output=True, text=True, check=True
        logger.info(f"Artifact {tarball_path} extracted successfully.")
    except Exception as e:
        raise ValueError(
            f"Failed to locate or extract artifact for {bazel_rule} rule"
        ) from e
def _run_post_processor(output: str, library_id: str):
    """Runs the synthtool post-processor on the output directory.
        output(str): Path to the directory in the container where code
            should be generated.
        library_id(str): The library id to be used for post processing.
    os.chdir(output)
    path_to_library = f"packages/{library_id}"
    logger.info("Running Python post-processor...")
    if SYNTHTOOL_INSTALLED:
        python_mono_repo.owlbot_main(path_to_library)
        raise SYNTHTOOL_IMPORT_ERROR  # pragma: NO COVER
    logger.info("Python post-processor ran successfully.")
def _copy_files_needed_for_post_processing(output: str, input: str, library_id: str):
    """Copy files to the output directory whcih are needed during the post processing
    step, such as .repo-metadata.json and script/client-post-processing, using
    the input directory as the source.
        output(str): Path to the directory in the container where code
            should be generated.
        input(str): The path to the directory in the container
            which contains additional generator input.
        library_id(str): The library id to be used for post processing.
    path_to_library = f"packages/{library_id}"
    # We need to create these directories so that we can copy files necessary for post-processing.
    os.makedirs(f"{output}/{path_to_library}")
    os.makedirs(f"{output}/{path_to_library}/scripts/client-post-processing")
    shutil.copy(
        f"{input}/{path_to_library}/.repo-metadata.json",
        f"{output}/{path_to_library}/.repo-metadata.json",
    # copy post-procesing files
    for post_processing_file in glob.glob(
        f"{input}/client-post-processing/*.yaml"
    ):  # pragma: NO COVER
        with open(post_processing_file, "r") as post_processing:
            if f"{path_to_library}/" in post_processing.read():
                shutil.copy(
                    post_processing_file,
                    f"{output}/{path_to_library}/scripts/client-post-processing",
def _clean_up_files_after_post_processing(output: str, library_id: str):
    Clean up files which should not be included in the generated client
        output(str): Path to the directory in the container where code
            should be generated.
        library_id(str): The library id to be used for post processing.
    path_to_library = f"packages/{library_id}"
    shutil.rmtree(f"{output}/{path_to_library}/.nox")
    os.remove(f"{output}/{path_to_library}/CHANGELOG.md")
    os.remove(f"{output}/{path_to_library}/docs/CHANGELOG.md")
    os.remove(f"{output}/{path_to_library}/docs/README.rst")
    for post_processing_file in glob.glob(
        f"{output}/{path_to_library}/scripts/client-post-processing/*.yaml"
    ):  # pragma: NO COVER
        os.remove(post_processing_file)
    for gapic_version_file in glob.glob(
        f"{output}/{path_to_library}/**/gapic_version.py", recursive=True
    ):  # pragma: NO COVER
        os.remove(gapic_version_file)
    for snippet_metadata_file in glob.glob(
        f"{output}/{path_to_library}/samples/generated_samples/snippet_metadata*.json"
    ):  # pragma: NO COVER
        os.remove(snippet_metadata_file)
    shutil.rmtree(f"{output}/owl-bot-staging")
def handle_generate(
    librarian: str = LIBRARIAN_DIR,
    source: str = SOURCE_DIR,
    output: str = OUTPUT_DIR,
    input: str = INPUT_DIR,
    """The main coordinator for the code generation process.
    This function orchestrates the generation of a client library by reading a
    `librarian/generate-request.json` file, determining the necessary Bazel rule for each API, and
    (in future steps) executing the build.
    See https://github.com/googleapis/librarian/blob/main/doc/container-contract.md#generate-container-command
        librarian(str): Path to the directory in the container which contains
            the librarian configuration.
        source(str): Path to the directory in the container which contains
            API protos.
        output(str): Path to the directory in the container where code
            should be generated.
        input(str): The path to the directory in the container
            which contains additional generator input.
    Raises:
        ValueError: If the `generate-request.json` file is not found or read.
        # Read a generate-request.json file
        request_data = _read_json_file(f"{librarian}/{GENERATE_REQUEST_FILE}")
        library_id = _get_library_id(request_data)
        for api in request_data.get("apis", []):
            api_path = api.get("path")
            if api_path:
                bazel_rule = _determine_bazel_rule(api_path, source)
                _build_bazel_target(bazel_rule, source)
                _locate_and_extract_artifact(
                    bazel_rule, library_id, source, output, api_path
        _copy_files_needed_for_post_processing(output, input, library_id)
        _run_post_processor(output, library_id)
        _clean_up_files_after_post_processing(output, library_id)
        # Write the `generate-response.json` using `generate-request.json` as the source
        with open(f"{librarian}/generate-response.json", "w") as f:
            json.dump(request_data, f, indent=4)
            f.write("\n")
    except Exception as e:
        raise ValueError("Generation failed.") from e
    # TODO(https://github.com/googleapis/librarian/issues/448): Implement generate command and update docstring.
    logger.info("'generate' command executed.")
def _run_nox_sessions(sessions: List[str], librarian: str, repo: str):
    """Calls nox for all specified sessions.
        sessions(List[str]): The list of nox sessions to run.
        librarian(str): The path to the librarian build configuration directory
    # Read a build-request.json file
    current_session = None
        request_data = _read_json_file(f"{librarian}/{BUILD_REQUEST_FILE}")
        library_id = _get_library_id(request_data)
        for nox_session in sessions:
            _run_individual_session(nox_session, library_id, repo)
        # Write the `build-response.json` using `build-request.json` as the source
        with open(f"{librarian}/build-response.json", "w") as f:
            json.dump(request_data, f, indent=4)
            f.write("\n")
    except Exception as e:
        raise ValueError(f"Failed to run the nox session: {current_session}") from e
def _run_individual_session(nox_session: str, library_id: str, repo: str):
    Calls nox with the specified sessions.
        nox_session(str): The nox session to run
        library_id(str): The library id under test
    command = [
        "nox",
        "-s",
        nox_session,
        "-f",
        f"{repo}/packages/{library_id}/noxfile.py",
    result = subprocess.run(command, text=True, check=True)
    logger.info(result)
def handle_build(librarian: str = LIBRARIAN_DIR, repo: str = REPO_DIR):
    """The main coordinator for validating client library generation."""
    sessions = [
        "unit-3.9",
        "unit-3.10",
        "unit-3.11",
        "unit-3.12",
        "unit-3.13",
        "docs",
        "system-3.13",
        "lint",
        "lint_setup_py",
        "mypy-3.13",
    _run_nox_sessions(sessions, librarian, repo)
    logger.info("'build' command executed.")
if __name__ == "__main__":  # pragma: NO COVER
    parser = argparse.ArgumentParser(description="A simple CLI tool.")
    subparsers = parser.add_subparsers(
        dest="command", required=True, help="Available commands"
    # Define commands
    handler_map = {
        "configure": handle_configure,
        "generate": handle_generate,
        "build": handle_build,
    for command_name, help_text in [
        ("configure", "Onboard a new library or an api path to Librarian workflow."),
        ("generate", "generate a python client for an API."),
        ("build", "Run unit tests via nox for the generated library."),
        parser_cmd = subparsers.add_parser(command_name, help=help_text)
        parser_cmd.set_defaults(func=handler_map[command_name])
        parser_cmd.add_argument(
            "--librarian",
            type=str,
            help="Path to the directory in the container which contains the librarian configuration",
            default=LIBRARIAN_DIR,
        parser_cmd.add_argument(
            "--input",
            type=str,
            help="Path to the directory in the container which contains additional generator input",
            default=INPUT_DIR,
        parser_cmd.add_argument(
            "--output",
            type=str,
            help="Path to the directory in the container where code should be generated",
            default=OUTPUT_DIR,
        parser_cmd.add_argument(
            "--source",
            type=str,
            help="Path to the directory in the container which contains API protos",
            default=SOURCE_DIR,
        parser_cmd.add_argument(
            "--repo",
            type=str,
            help="Path to the directory in the container which contains google-cloud-python repository",
            default=SOURCE_DIR,
    if len(sys.argv) == 1:
        parser.print_help(sys.stderr)
        sys.exit(1)
    args = parser.parse_args()
    # Pass specific arguments to the handler functions for generate/build
    if args.command == "generate":
        args.func(
            librarian=args.librarian,
            source=args.source,
            output=args.output,
            input=args.input,
    elif args.command == "build":
        args.func(librarian=args.librarian, repo=args.repo)
        args.func()
Provide feedback

Saved searches

Use saved searches to filter your results more quickly

FilesExpand file tree

cli.py

Latest commit

History

cli.py

File metadata and controls
