Skip to content

MaD: add script to convert YAML data extensions to JSON #23102

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
nickrolfe wants to merge 1 commit into
base: main
Choose a base branch
from
Open

MaD: add script to convert YAML data extensions to JSON #23102

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
388 changes: 388 additions & 0 deletions misc/scripts/convert-yaml-data-extensions-to-json.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,388 @@
#!/usr/bin/env python3

"""Convert YAML data extension files to JSON while preserving comments.

USAGE:

Pass files directly to convert a specific set of data extensions:

convert-yaml-data-extensions-to-json.py [--delete-yaml] FILE [FILE ...]

Each file will be converted to a JSON file with the same base name. By default, the original YAML
file is retained; pass --delete-yaml to remove it after successful conversion.

Alternatively, pass a qlpack.yml manifest:

convert-yaml-data-extensions-to-json.py [--delete-yaml] --qlpack PATH/qlpack.yml

In this mode, files will be converted as above, but the qlpack.yml manifest itself will be updated:
the existing YAML patterns will be preserved (so that they will continue to match if any new YAML
extensions are added), and new file patterns matching the JSON files will be added.

WARNING:

This script was generated by an LLM and has not been reviewed by a human; always check its output
for correctness.
"""

import argparse
import json
import sys
from collections import defaultdict
from io import StringIO
from pathlib import Path
from typing import Any

from ruamel.yaml import YAML

YAML_PARSER = YAML(typ="rt")
YAML_PARSER.allow_duplicate_keys = False
YAML_PARSER.indent(mapping=2, sequence=4, offset=2)
YAML_PARSER.width = 4096


def load_yaml(path: Path) -> tuple[Any, str]:
"""Load a YAML file and return both its parsed value and source text."""
text = path.read_text(encoding="utf-8")
return YAML_PARSER.load(text), text


def encode_scalar(value: Any) -> str:
"""Encode a scalar value as compact JSON."""
return json.dumps(value, ensure_ascii=False, separators=(",", ":"))


def yaml_comment_start(line: str) -> int | None:
"""Find the start of a YAML comment outside quoted strings."""
single_quoted = False
double_quoted = False
escaped = False
index = 0
while index < len(line):
character = line[index]
if double_quoted:
if escaped:
escaped = False
elif character == "\\":
escaped = True
elif character == '"':
double_quoted = False
elif single_quoted:
if character == "'" and index + 1 < len(line) and line[index + 1] == "'":
index += 1
elif character == "'":
single_quoted = False
elif character == '"':
double_quoted = True
elif character == "'":
single_quoted = True
elif character == "#" and (index == 0 or line[index - 1].isspace()):
return index
index += 1
return None


def node_anchors(
value: Any, path: tuple[Any, ...] = ()
) -> list[tuple[int, tuple[Any, ...]]]:
"""Map parsed YAML nodes to their source lines and object paths."""
anchors = []
if path == () and hasattr(value, "lc"):
anchors.append((value.lc.line, path))
if isinstance(value, dict):
for key, child in value.items():
child_path = path + (key,)
anchors.append((value.lc.key(key)[0], child_path))
anchors.extend(node_anchors(child, child_path))
elif isinstance(value, list):
for index, child in enumerate(value):
child_path = path + (index,)
anchors.append((value.lc.item(index)[0], child_path))
if not path or path[-1] != "data":
anchors.extend(node_anchors(child, child_path))
return anchors


def associate_comments(
value: Any, source: str
) -> tuple[
dict[tuple[Any, ...], list[str]],
dict[tuple[Any, ...], str],
list[str],
]:
"""Associate YAML comments with nearby parsed nodes."""
anchors_by_line: dict[int, set[tuple[Any, ...]]] = defaultdict(set)
for line, path in node_anchors(value):
anchors_by_line[line].add(path)

pre_comments: dict[tuple[Any, ...], list[str]] = defaultdict(list)
inline_comments: dict[tuple[Any, ...], str] = {}
trailing_comments: list[str] = []
pending: list[str] = []

for line_number, line in enumerate(source.splitlines()):
comment_start = yaml_comment_start(line)
if comment_start is None:
continue
comment = line[comment_start + 1 :].strip()
if line[:comment_start].strip():
candidates = anchors_by_line.get(line_number)
if candidates:
path = min(
candidates, key=lambda candidate: (len(candidate), repr(candidate))
)
inline_comments[path] = comment
else:
pending.append(comment)
later_lines = [
anchor_line
for anchor_line in anchors_by_line
if anchor_line > line_number
]
if later_lines:
next_line = min(later_lines)
path = min(
anchors_by_line[next_line],
key=lambda candidate: (len(candidate), repr(candidate)),
)
pre_comments[path].extend(pending)
pending = []

trailing_comments.extend(pending)
return pre_comments, inline_comments, trailing_comments


def format_json(
value: Any,
pre_comments: dict[tuple[Any, ...], list[str]],
inline_comments: dict[tuple[Any, ...], str],
level: int = 0,
inline_rows: bool = False,
path: tuple[Any, ...] = (),
) -> str:
"""Format a parsed value as JSON while restoring associated comments."""
indent = " " * level
child_indent = " " * (level + 1)

if not isinstance(value, (dict, list)):
return encode_scalar(value)

if isinstance(value, dict):
if not value:
return "{}"
entries = []
items = list(value.items())
for index, (key, child) in enumerate(items):
child_path = path + (key,)
encoded_key = json.dumps(key, ensure_ascii=False)
encoded_child = format_json(
child,
pre_comments,
inline_comments,
level + 1,
inline_rows=(key == "data"),
path=child_path,
)
entry = f"{child_indent}{encoded_key}: {encoded_child}"
if index + 1 < len(items):
entry += ","
if comment := inline_comments.get(child_path):
entry += f" // {comment}"
comment_lines = [
f"{child_indent}// {comment}".rstrip()
for comment in pre_comments.get(child_path, [])
]
entries.append("\n".join(comment_lines + [entry]))
return "{\n" + "\n".join(entries) + f"\n{indent}}}"

if not value:
return "[]"
if inline_rows:
entries = []
for index, row in enumerate(value):
child_path = path + (index,)
entry = (
f"{child_indent}"
f"{json.dumps(row, ensure_ascii=False, separators=(', ', ': '))}"
)
if index + 1 < len(value):
entry += ","
if comment := inline_comments.get(child_path):
entry += f" // {comment}"
comment_lines = [
f"{child_indent}// {comment}".rstrip()
for comment in pre_comments.get(child_path, [])
]
entries.append("\n".join(comment_lines + [entry]))
return "[\n" + "\n".join(entries) + f"\n{indent}]"

entries = []
for index, child in enumerate(value):
child_path = path + (index,)
entry = (
f"{child_indent}"
f"{format_json(child, pre_comments, inline_comments, level + 1, path=child_path)}"
)
if index + 1 < len(value):
entry += ","
if comment := inline_comments.get(child_path):
entry += f" // {comment}"
comment_lines = [
f"{child_indent}// {comment}".rstrip()
for comment in pre_comments.get(child_path, [])
]
entries.append("\n".join(comment_lines + [entry]))
return "[\n" + "\n".join(entries) + f"\n{indent}]"


def format_json_with_comments(value: Any, source: str) -> str:
"""Format a parsed YAML value as JSON with its original comments."""
pre_comments, inline_comments, trailing_comments = associate_comments(value, source)
lines = [f"// {comment}".rstrip() for comment in pre_comments.pop((), [])]
lines.append(format_json(value, pre_comments, inline_comments))
lines.extend(f"// {comment}".rstrip() for comment in trailing_comments)
return "\n".join(lines)


def write_text_atomically(path: Path, contents: str) -> None:
"""Write text through a temporary file and atomically replace the target."""
temporary = path.with_name(f".{path.name}.tmp")
try:
temporary.write_text(contents, encoding="utf-8")
temporary.replace(path)
finally:
temporary.unlink(missing_ok=True)
Comment on lines +249 to +254


def convert_data_extension(source: Path, delete_yaml: bool = False) -> Path:
"""Convert a YAML data extension to JSON, optionally removing the source."""
target = source.with_suffix(".json")
model, source_text = load_yaml(source)
write_text_atomically(target, format_json_with_comments(model, source_text) + "\n")
if delete_yaml:
source.unlink()
return target


def data_extensions_from_qlpack(path: Path) -> list[Path]:
"""Find YAML data extensions matched by a qlpack manifest."""
manifest, _ = load_yaml(path)
if not isinstance(manifest, dict):
raise ValueError(f"qlpack manifest is not a mapping: {path}")

patterns = manifest.get("dataExtensions")
if not isinstance(patterns, list):
raise ValueError(f"qlpack manifest has no dataExtensions list: {path}")

files = set()
for pattern in patterns:
if not isinstance(pattern, str):
raise ValueError(f"dataExtensions pattern is not a string: {pattern!r}")
files.update(
match.resolve()
for match in path.parent.glob(pattern.lstrip("/"))
if match.is_file() and match.suffix in {".yml", ".yaml"}
)
return sorted(files)
Comment on lines +277 to +286


def add_json_data_extension_patterns(path: Path) -> None:
"""Add JSON equivalents of YAML data-extension patterns to a manifest."""
manifest, _ = load_yaml(path)
if not isinstance(manifest, dict):
raise ValueError(f"qlpack manifest is not a mapping: {path}")

patterns = manifest.get("dataExtensions")
if not isinstance(patterns, list):
raise ValueError(f"qlpack manifest has no dataExtensions list: {path}")

existing_patterns = set()
for pattern in patterns:
if not isinstance(pattern, str):
raise ValueError(f"dataExtensions pattern is not a string: {pattern!r}")
existing_patterns.add(pattern)

json_patterns = []
for pattern in patterns:
if pattern.endswith(".yml"):
json_pattern = pattern.removesuffix(".yml") + ".json"
elif pattern.endswith(".yaml"):
json_pattern = pattern.removesuffix(".yaml") + ".json"
else:
continue
if json_pattern not in existing_patterns:
json_patterns.append(json_pattern)
existing_patterns.add(json_pattern)

if not json_patterns:
return

patterns.extend(json_patterns)
output = StringIO()
YAML_PARSER.dump(manifest, output)
write_text_atomically(path, output.getvalue())


def validate_yaml_file(path: Path) -> Path:
"""Resolve and validate a YAML input file."""
path = path.expanduser().resolve()
if not path.is_file():
raise ValueError(f"YAML file does not exist: {path}")
if path.suffix not in {".yml", ".yaml"}:
raise ValueError(f"not a YAML file: {path}")
return path


def main() -> None:
"""Parse command-line arguments and convert the selected data extensions."""
parser = argparse.ArgumentParser(description=__doc__)
inputs = parser.add_mutually_exclusive_group(required=True)
inputs.add_argument(
"files",
metavar="YAML_FILE",
nargs="*",
type=Path,
help="YAML data extension file to convert",
)
inputs.add_argument(
"--qlpack",
metavar="QLPACK_FILE",
type=Path,
help="convert data extensions matched by a qlpack.yml manifest",
)
parser.add_argument(
"--delete-yaml",
action="store_true",
help="delete YAML files after successful conversion",
)
args = parser.parse_args()

if args.qlpack is not None:
qlpack = args.qlpack.expanduser().resolve()
if not qlpack.is_file():
raise ValueError(f"qlpack manifest does not exist: {qlpack}")
files = data_extensions_from_qlpack(qlpack)
else:
files = []
for file in args.files:
if file.name == "qlpack.yml":
print(
f"Warning: skipping qlpack.yml passed as a data extension. Try running with --qlpack {file}",
file=sys.stderr,
flush=True,
)
else:
files.append(file)

files = [validate_yaml_file(file) for file in files]
for file in files:
convert_data_extension(file, delete_yaml=args.delete_yaml)

if args.qlpack is not None:
add_json_data_extension_patterns(qlpack)

print(f"Converted {len(files)} data extensions.", flush=True)


if __name__ == "__main__":
main()
Loading