Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/cowork-auto-pr.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ jobs:
# without this step every run failed with "not a git repository" and no
# PR was ever opened (fleet-wide defect: 11/11 seeded copies lacked it).
- name: Check out the pushed branch
uses: actions/checkout@v4
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.2.2
with:
ref: ${{ github.ref_name }}
fetch-depth: 0
Expand Down
1 change: 1 addition & 0 deletions conftest.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
"""pytest configuration — add project src to Python path and skip rate limits."""

import os
import sys
from pathlib import Path
Expand Down
19 changes: 5 additions & 14 deletions src/json2sql/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,7 @@
except ImportError:
import warnings

warnings.warn(
"revenueholdings-license not installed; license checks skipped", stacklevel=2
)
warnings.warn("revenueholdings-license not installed; license checks skipped", stacklevel=2)

def require_license(product: str) -> None: # type: ignore[misc]
pass
Expand Down Expand Up @@ -45,9 +43,7 @@ def _app_callback(
) -> None:
"""Convert JSON files/datasets to SQL INSERT statements."""
global _require_license_strict
_require_license_strict = require_license_flag or bool(
os.environ.get("REVENUEHOLDINGS_REQUIRE_LICENSE")
)
_require_license_strict = require_license_flag or bool(os.environ.get("REVENUEHOLDINGS_REQUIRE_LICENSE"))


def _check_license(tool_name: str) -> None:
Expand All @@ -61,8 +57,7 @@ def _check_license(tool_name: str) -> None:
except ImportError:
if _require_license_strict:
typer.echo(
"Error: revenueholdings-license is not installed. "
"Install it with: pip install revenueholdings-license",
"Error: revenueholdings-license is not installed. Install it with: pip install revenueholdings-license",
err=True,
)
raise typer.Exit(code=1) from None
Expand Down Expand Up @@ -120,9 +115,7 @@ def convert(
dialect_enum = Dialect(dialect)
except ValueError:
valid = ", ".join(d.value for d in Dialect)
typer.echo(
f"Error: Unknown dialect '{dialect}'. Choose from: {valid}", err=True
)
typer.echo(f"Error: Unknown dialect '{dialect}'. Choose from: {valid}", err=True)
raise typer.Exit(code=1) from None

# Read input
Expand Down Expand Up @@ -165,8 +158,7 @@ def mcp() -> None:
from click_to_mcp import run # type: ignore[import-untyped]
except ImportError:
typer.echo(
"Error: click_to_mcp is required for MCP mode. "
"Install it with: pip install click-to-mcp",
"Error: click_to_mcp is required for MCP mode. Install it with: pip install click-to-mcp",
err=True,
)
raise typer.Exit(code=1) from None
Expand All @@ -183,4 +175,3 @@ def version() -> None:

if __name__ == "__main__":
app()

61 changes: 29 additions & 32 deletions src/json2sql/converter.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,18 +43,12 @@ def convert(self, json_text: str, table_name: str = "data") -> str:
# Add any extra tables from flattening
for name, columns, rows in self._extra_tables:
statements.insert(0, create_table_sql(name, columns, self.dialect))
statements.append(
insert_sql(name, list(columns.keys()), rows, self.dialect)
)
statements.append(insert_sql(name, list(columns.keys()), rows, self.dialect))

result = "\n\n".join(s for s in statements if s)
# An empty object / nested-only root legitimately produces no SQL; say
# so explicitly instead of returning "" (avoids a silent green no-op).
return (
result
if result
else "-- No columns to generate (empty or nested-only object)."
)
return result if result else "-- No columns to generate (empty or nested-only object)."

def generate_schema(self, json_text: str, table_name: str = "data") -> str:
"""Generate only CREATE TABLE statements from JSON data."""
Expand All @@ -74,7 +68,9 @@ def generate_schema(self, json_text: str, table_name: str = "data") -> str:
else:
columns = self._infer_columns(objects)
else:
columns = {"value": "TEXT"}
# Primitive array — scan all elements to merge types safely
col_type = self._infer_primitive_column_type(data if isinstance(data, list) else [])
columns = {"value": col_type}

statements = []
if columns:
Expand All @@ -96,11 +92,7 @@ def _convert_objects(self, objects: list[dict], table_name: str) -> str:
# Process nested arrays into child tables
for obj in objects:
for key, value in obj.items():
if (
isinstance(value, list)
and value
and all(isinstance(v, dict) for v in value)
):
if isinstance(value, list) and value and all(isinstance(v, dict) for v in value):
self._flatten_nested(table_name, key, value, obj)
else:
columns = self._infer_columns(objects)
Expand Down Expand Up @@ -134,14 +126,30 @@ def _convert_objects(self, objects: list[dict], table_name: str) -> str:
return ""
parts = [create_table_sql(table_name, columns, self.dialect)]
if rows:
parts.append(
insert_sql(table_name, list(columns.keys()), rows, self.dialect)
)
parts.append(insert_sql(table_name, list(columns.keys()), rows, self.dialect))
return "\n\n".join(parts)

def _infer_primitive_column_type(self, values: list) -> str:
"""Infer the column type for a primitive array by scanning all elements.

Uses the same merge logic as object-column inference: if any two
non-NULL values have incompatible SQL types, the column collapses
to TEXT. This prevents declaring INTEGER for ``[1, "hello", 3]``
which would make the INSERT fail.
"""
if not values:
return "TEXT"
resolved: str | None = None
for v in values:
inferred = self._infer_type(v)
if inferred is None:
continue
resolved = inferred if resolved is None else self._merge_type(resolved, inferred)
return resolved if resolved is not None else "TEXT"

def _convert_primitives(self, values: list, table_name: str) -> str:
"""Convert a list of primitive values to SQL."""
col_type = sql_type_for(values[0] if values else None, self.dialect)
col_type = self._infer_primitive_column_type(values)
columns = {"value": col_type}
rows = [[format_value(v, self.dialect)] for v in values]
parts = [create_table_sql(table_name, columns, self.dialect)]
Expand Down Expand Up @@ -215,15 +223,8 @@ def _infer_columns_flattened(
columns[flat_key] = inferred
flat_map[flat_key] = (key, sub_key)
elif inferred is not None:
columns[flat_key] = self._merge_type(
columns[flat_key], inferred
)
elif (
isinstance(value, list)
and value
and self.flatten
and all(isinstance(v, dict) for v in value)
):
columns[flat_key] = self._merge_type(columns[flat_key], inferred)
elif isinstance(value, list) and value and self.flatten and all(isinstance(v, dict) for v in value):
# Skip - goes to separate table
pass
else:
Expand Down Expand Up @@ -279,9 +280,5 @@ def _process_flatten(self, objects: list, table_name: str) -> None:
return
for obj in objects:
for key, value in obj.items():
if (
isinstance(value, list)
and value
and all(isinstance(v, dict) for v in value)
):
if isinstance(value, list) and value and all(isinstance(v, dict) for v in value):
self._flatten_nested(table_name, key, value, obj)
73 changes: 73 additions & 0 deletions tests/test_converter.py
Original file line number Diff line number Diff line change
Expand Up @@ -535,3 +535,76 @@ def test_version_in_init_matches_pyproject(self):
assert data["project"]["version"] == __version__, (
f"pyproject.toml version ({data['project']['version']}) != __init__.__version__ ({__version__})"
)


class TestGenerateSchemaPrimitiveTypeInference:
"""Regression: generate_schema must infer primitive column types, not always TEXT."""

def test_schema_primitive_int_array(self):
conv = JSONToSQLConverter(dialect=Dialect.POSTGRES)
data = json.dumps([1, 2, 3])
result = conv.generate_schema(data, table_name="nums")
assert "INTEGER" in result
assert "TEXT" not in result.split("CREATE TABLE")[1].split(")")[0]

def test_schema_primitive_float_array_mysql(self):
conv = JSONToSQLConverter(dialect=Dialect.MYSQL)
data = json.dumps([1.5, 2.5])
result = conv.generate_schema(data, table_name="vals")
assert "DOUBLE" in result

def test_schema_primitive_bool_array_sqlite(self):
conv = JSONToSQLConverter(dialect=Dialect.SQLITE)
data = json.dumps([True, False])
result = conv.generate_schema(data, table_name="flags")
# SQLite bool -> INTEGER
assert "INTEGER" in result


class TestMixedPrimitiveArrays:
"""Regression: mixed-type primitive arrays must fall back to TEXT.

When a primitive array contains values of incompatible types (e.g.
[1, "hello", 3]), the column type must be TEXT so that every value
can be inserted without SQL errors. Sampling only the first element
would declare INTEGER and fail on the string.
"""

def test_convert_mixed_int_and_string_falls_back_to_text(self, converter_postgres):
data = json.dumps([1, "hello", 3])
result = converter_postgres.convert(data, table_name="mixed")
assert "CREATE TABLE" in result
# Column type must be TEXT, not INTEGER
schema_part = result.split("INSERT INTO")[0]
assert "TEXT" in schema_part
assert "INTEGER" not in schema_part

def test_convert_mixed_int_and_float_falls_back_to_text(self, converter_postgres):
"""int and float are different SQL types; mixed should be TEXT."""
data = json.dumps([1, 2.5, 3])
result = converter_postgres.convert(data, table_name="mixed")
schema_part = result.split("INSERT INTO")[0]
assert "TEXT" in schema_part

def test_generate_schema_mixed_primitives_falls_back_to_text(self, converter_postgres):
data = json.dumps([1, "two", 3.0])
result = converter_postgres.generate_schema(data, table_name="mixed")
assert "CREATE TABLE" in result
assert "TEXT" in result
assert "INTEGER" not in result

def test_convert_all_same_type_stays_specific(self, converter_postgres):
"""Homogeneous arrays should still get specific types, not TEXT."""
data = json.dumps([1, 2, 3])
result = converter_postgres.convert(data, table_name="nums")
schema_part = result.split("INSERT INTO")[0]
assert "INTEGER" in schema_part
assert "TEXT" not in schema_part

def test_convert_mixed_with_nulls_skips_null_for_type(self, converter_postgres):
"""NULLs should not force TEXT when all non-null values agree."""
data = json.dumps([1, None, 3])
result = converter_postgres.convert(data, table_name="nums")
schema_part = result.split("INSERT INTO")[0]
assert "INTEGER" in schema_part
assert "TEXT" not in schema_part
Loading