Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
166 changes: 157 additions & 9 deletions .github/workflows/tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -63,13 +63,17 @@ jobs:
run: ruby -Ilib -Itest -e 'Dir["test/*_test.rb"].sort.each { |file| require File.expand_path(file) }'

end2end-pg:
name: End-to-End (PostgreSQL, Ruby ${{ matrix.ruby-version }})
name: End-to-End (PostgreSQL, Ruby ${{ matrix.ruby-version }}, ${{ matrix.variant }})
runs-on: ubuntu-latest

strategy:
fail-fast: false
matrix:
ruby-version: ['3.3', '4.0']
ruby-version: ['3.1', '3.2', '3.3', '3.4', '4.0']
variant: ['default']
include:
- ruby-version: '4.0'
variant: 'module-no-prepared'

services:
postgres:
Expand All @@ -88,14 +92,43 @@ jobs:

env:
POSTGRES_HOST: 127.0.0.1
EXAMPLES_DRIVER: pg
EXAMPLES_MODULE: ${{ matrix.variant == 'module-no-prepared' && 'MyApp::Db' || 'PgCodegen' }}
PREPARED_STATEMENTS: ${{ matrix.variant == 'module-no-prepared' && 'false' || 'true' }}
BUNDLE_GEMFILE: ${{ (matrix.ruby-version == '3.1' || matrix.ruby-version == '3.2') && 'Gemfile.e2e' || 'Gemfile' }}

steps:
- uses: actions/checkout@v7

- uses: ruby/setup-ruby@v1
with:
ruby-version: ${{ matrix.ruby-version }}
bundler-cache: true
bundler-cache: ${{ matrix.ruby-version != '3.1' && matrix.ruby-version != '3.2' }}

- name: Load tool versions
if: matrix.ruby-version == '3.1' || matrix.ruby-version == '3.2' || matrix.variant != 'default'
run: grep -E '^[A-Z_]+=' scripts/tool-versions.env >> "$GITHUB_ENV"

- uses: sqlc-dev/setup-sqlc@v5
if: matrix.ruby-version == '3.1' || matrix.ruby-version == '3.2' || matrix.variant != 'default'
with:
sqlc-version: '${{ env.SQLC_VERSION }}'

- name: Generate version-specific examples
if: matrix.ruby-version == '3.1' || matrix.ruby-version == '3.2'
run: |
ruby scripts/tests/generate_examples.rb "${{ matrix.ruby-version }}" sqlc.local.yaml --generate-gemfile
echo "EXAMPLES_DIR=$PWD/.examples-ruby-${{ matrix.ruby-version }}" >> "$GITHUB_ENV"

- name: Generate option variant
if: matrix.variant == 'module-no-prepared'
run: |
ruby scripts/tests/generate_examples.rb 4.0 sqlc.local.yaml --driver pg --options '{"moduleName":"MyApp::Db","preparedStatements":false}'
echo "EXAMPLES_DIR=$PWD/.examples-ruby-4.0" >> "$GITHUB_ENV"

- name: Install E2E dependencies
if: matrix.ruby-version == '3.1' || matrix.ruby-version == '3.2'
run: bundle install

- name: Create the schema
run: psql -v ON_ERROR_STOP=1 -h 127.0.0.1 -U "${DB_USER}" -d "${DB_NAME}" -f examples/authors/postgresql/schema.sql
Expand All @@ -106,13 +139,17 @@ jobs:
run: bundle exec rake end2end_pg

end2end-mysql:
name: End-to-End (MySQL, Ruby ${{ matrix.ruby-version }})
name: End-to-End (MySQL, Ruby ${{ matrix.ruby-version }}, ${{ matrix.variant }})
runs-on: ubuntu-latest

strategy:
fail-fast: false
matrix:
ruby-version: ['3.3', '4.0']
ruby-version: ['3.1', '3.2', '3.3', '3.4', '4.0']
variant: ['default']
include:
- ruby-version: '4.0'
variant: 'module-name'

services:
mysql:
Expand All @@ -130,14 +167,42 @@ jobs:

env:
MYSQL_HOST: 127.0.0.1
EXAMPLES_DRIVER: mysql2
EXAMPLES_MODULE: ${{ matrix.variant == 'module-name' && 'MyApp::Db' || 'Mysql2Codegen' }}
BUNDLE_GEMFILE: ${{ (matrix.ruby-version == '3.1' || matrix.ruby-version == '3.2') && 'Gemfile.e2e' || 'Gemfile' }}

steps:
- uses: actions/checkout@v7

- uses: ruby/setup-ruby@v1
with:
ruby-version: ${{ matrix.ruby-version }}
bundler-cache: true
bundler-cache: ${{ matrix.ruby-version != '3.1' && matrix.ruby-version != '3.2' }}

- name: Load tool versions
if: matrix.ruby-version == '3.1' || matrix.ruby-version == '3.2' || matrix.variant != 'default'
run: grep -E '^[A-Z_]+=' scripts/tool-versions.env >> "$GITHUB_ENV"

- uses: sqlc-dev/setup-sqlc@v5
if: matrix.ruby-version == '3.1' || matrix.ruby-version == '3.2' || matrix.variant != 'default'
with:
sqlc-version: '${{ env.SQLC_VERSION }}'

- name: Generate version-specific examples
if: matrix.ruby-version == '3.1' || matrix.ruby-version == '3.2'
run: |
ruby scripts/tests/generate_examples.rb "${{ matrix.ruby-version }}" sqlc.local.yaml --generate-gemfile
echo "EXAMPLES_DIR=$PWD/.examples-ruby-${{ matrix.ruby-version }}" >> "$GITHUB_ENV"

- name: Generate option variant
if: matrix.variant == 'module-name'
run: |
ruby scripts/tests/generate_examples.rb 4.0 sqlc.local.yaml --driver mysql2 --options '{"moduleName":"MyApp::Db"}'
echo "EXAMPLES_DIR=$PWD/.examples-ruby-4.0" >> "$GITHUB_ENV"

- name: Install E2E dependencies
if: matrix.ruby-version == '3.1' || matrix.ruby-version == '3.2'
run: bundle install

- name: Create the schema (and allow LOAD DATA LOCAL INFILE for :copyfrom)
run: |
Expand All @@ -148,21 +213,104 @@ jobs:
run: bundle exec rake end2end_mysql2

end2end-sqlite:
name: End-to-End (SQLite, Ruby ${{ matrix.ruby-version }})
name: End-to-End (SQLite, Ruby ${{ matrix.ruby-version }}, ${{ matrix.variant }})
runs-on: ubuntu-latest

strategy:
fail-fast: false
matrix:
ruby-version: ['3.3', '4.0']
ruby-version: ['3.1', '3.2', '3.3', '3.4', '4.0']
variant: ['default']
include:
- ruby-version: '4.0'
variant: 'module-name'

env:
EXAMPLES_DRIVER: sqlite3
EXAMPLES_MODULE: ${{ matrix.variant == 'module-name' && 'MyApp::Db' || 'Sqlite3Codegen' }}
BUNDLE_GEMFILE: ${{ (matrix.ruby-version == '3.1' || matrix.ruby-version == '3.2') && 'Gemfile.e2e' || 'Gemfile' }}

steps:
- uses: actions/checkout@v7

- uses: ruby/setup-ruby@v1
with:
ruby-version: ${{ matrix.ruby-version }}
bundler-cache: true
bundler-cache: ${{ matrix.ruby-version != '3.1' && matrix.ruby-version != '3.2' }}

- name: Load tool versions
if: matrix.ruby-version == '3.1' || matrix.ruby-version == '3.2' || matrix.variant != 'default'
run: grep -E '^[A-Z_]+=' scripts/tool-versions.env >> "$GITHUB_ENV"

- uses: sqlc-dev/setup-sqlc@v5
if: matrix.ruby-version == '3.1' || matrix.ruby-version == '3.2' || matrix.variant != 'default'
with:
sqlc-version: '${{ env.SQLC_VERSION }}'

- name: Generate version-specific examples
if: matrix.ruby-version == '3.1' || matrix.ruby-version == '3.2'
run: |
ruby scripts/tests/generate_examples.rb "${{ matrix.ruby-version }}" sqlc.local.yaml --generate-gemfile
echo "EXAMPLES_DIR=$PWD/.examples-ruby-${{ matrix.ruby-version }}" >> "$GITHUB_ENV"

- name: Generate option variant
if: matrix.variant == 'module-name'
run: |
ruby scripts/tests/generate_examples.rb 4.0 sqlc.local.yaml --driver sqlite3 --options '{"moduleName":"MyApp::Db"}'
echo "EXAMPLES_DIR=$PWD/.examples-ruby-4.0" >> "$GITHUB_ENV"

- name: Install E2E dependencies
if: matrix.ruby-version == '3.1' || matrix.ruby-version == '3.2'
run: bundle install

- name: End-to-End tests
run: bundle exec rake end2end_sqlite3

end2end-options:
name: Generator options E2E (Ruby ${{ matrix.ruby-version }})
runs-on: ubuntu-latest

strategy:
fail-fast: false
matrix:
ruby-version: ['3.1', '4.0']

services:
postgres:
image: postgres:16.2
env:
POSTGRES_DB: tests
POSTGRES_USER: root
POSTGRES_PASSWORD: pass
ports:
- 5432:5432
options: >-
--health-cmd "pg_isready -U root -d tests"
--health-interval 5s
--health-timeout 5s
--health-retries 10

env:
POSTGRES_HOST: 127.0.0.1
BUNDLE_GEMFILE: Gemfile.options.e2e

steps:
- uses: actions/checkout@v7

- uses: ruby/setup-ruby@v1
with:
ruby-version: ${{ matrix.ruby-version }}
bundler-cache: false

- name: Install options test dependencies
run: bundle install

- name: Load tool versions
run: grep -E '^[A-Z_]+=' scripts/tool-versions.env >> "$GITHUB_ENV"

- uses: sqlc-dev/setup-sqlc@v5
with:
sqlc-version: '${{ env.SQLC_VERSION }}'

- name: End-to-end option tests
run: bundle exec ruby tests/end2end_options.rb
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -144,3 +144,4 @@ plugin.wasm
plugins
# scratch output of scripts/tests/run_codegen.rb
.codegen-test*
.examples-ruby-*
13 changes: 10 additions & 3 deletions CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,14 +31,21 @@ UPDATE_GOLDEN=1 bundle exec ruby -Itest test/extended_golden_test.rb # rewrite
* **Generated code must stay valid for Ruby 3.1–4.0** (`Struct` below 3.2, `Data` from 3.2) and RuboCop/Steep-clean.
* Generated code is plain, idiomatic Ruby (keyword args, `Data` rows, implicit returns, 2-space indent); do not mirror
the C# sibling plugin's style. Parameter *values* are never logged.
* Prepared statements are per connection (`ConnectionState`) because the query object is shared by pooled connections.
* PostgreSQL prepared statements are per connection (`ConnectionState`) because the query object is shared by pooled
connections. MySQL uses client-side quoting and `Mysql2::Client#query` instead of server-side prepared statements.
* Every README claim or supported feature needs a real-database E2E test for each applicable engine, listed in
`docs/Coverage.md`. A bug fix starts with an E2E test that fails before the fix; use unit tests only where E2E cannot
reach the behavior (for example option validation or generation errors).
* E2E coverage runs on every supported Ruby and includes full-suite option variants. Update `docs/Coverage.md` and
`docs/Testing.md` with new supported behavior and its test names.
* sqlc runs process plugins with an empty env (`env: [PATH]` is needed in `sqlc.local.yaml`) and resolves `out:`
relative to the config file.

## Git, CI and releases
* Develop on a feature branch, small commits; PRs target `main`. Label PRs (`bug`, `enhancement`, `documentation`; `breaking` for incompatible changes, see `.github/release.yml`).
* CI: `build.yml` (WASM, lint, `sqlc diff`, Steep, option matrix) and `tests.yml` (unit, Ruby 3.1/3.2/3.4/4.0, E2E on real
PostgreSQL/MySQL/SQLite). Both must be green.
* CI: `build.yml` (WASM, lint, `sqlc diff`, Steep, option matrix) and `tests.yml` (unit, compatibility Ruby
3.1/3.2/3.4/4.0, full E2E Ruby 3.1–4.0, Ruby 4.0 module-name variants and PostgreSQL without prepared statements,
plus generator-options E2E on Ruby 3.1/4.0 against PostgreSQL 16, MySQL 8.3 and SQLite). Both must be green.
* **Never create tags or releases by hand and never add release markers to commit messages.** Releases are started by a
maintainer with the *Release* workflow (`docs/Releasing.md`); do not trigger it unless explicitly asked.
* Tool versions (sqlc, ruby.wasm) are pinned in `scripts/tool-versions.env`; bump them there only.
11 changes: 11 additions & 0 deletions Gemfile.e2e
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
# frozen_string_literal: true

source 'https://rubygems.org'

examples_dir = ENV.fetch('EXAMPLES_DIR') { raise 'EXAMPLES_DIR must point at generated scratch examples' }
driver = ENV.fetch('EXAMPLES_DRIVER') { raise 'EXAMPLES_DRIVER must be set for E2E bundling' }
eval_gemfile File.join(examples_dir, driver, 'Gemfile')

gem 'async', RUBY_VERSION.start_with?('3.1') ? '< 2.7' : '>= 2.7'
gem 'minitest'
gem 'rake'
6 changes: 6 additions & 0 deletions Gemfile.options.e2e
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
source 'https://rubygems.org'

gem 'connection_pool', '~> 2.5'
gem 'minitest', RUBY_VERSION.start_with?('3.1') ? '< 6' : '>= 5'
gem 'pg', '1.7.0'
gem 'sqlite3'
23 changes: 17 additions & 6 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -93,15 +93,18 @@ per table, one module of constants per enum), `connection.rb` (shared helpers) a
* **Arguments** are keywords named after the parameters (`get_author(id:)`); `sqlc.arg(name)` names them yourself.
Missing or unknown ones raise `ArgumentError`. Nullable parameters named with `sqlc.arg` / `sqlc.narg` default
to `nil`; other nullable parameters (`VALUES ($1, $2)`) must be passed, even as `nil`.
* **Rows** are frozen `Data` objects (keyword `Struct`s below Ruby 3.2), so pattern matching works: `case row in { id:, name: }`.
* **Rows** are frozen `Data` objects from Ruby 3.2 (with `with` and `to_h`) or mutable keyword `Struct`s below Ruby 3.2;
both support pattern matching: `case row in { id:, name: }`.
* **Bang finders:** every `:one` query also gets a `get_author!` that raises `QuerySql::NotFound` instead of returning `nil`.
* **Connections:** `QuerySql.connect(params, **pool_options)` builds a [`connection_pool`](https://github.com/mperham/connection_pool).
To use a pool you already have (or a fake in tests) pass it to `QuerySql.new(pool)`; anything with `#with` works.
For SQLite pass a `SQLite3::Database`: `QuerySql.new(db)`.
* **Logging:** `connect(params, logger: Rails.logger)` / `new(pool, logger:)` logs `QuerySql#get_author (0.4ms)` at
debug level for each query and transaction. Parameter values are never logged.
* **Async / fibers:** it is plain synchronous Ruby and works with the Fiber scheduler ([`async`](https://github.com/socketry/async)):
inside `Sync { ... }`, pooled `pg` and `mysql2` queries run concurrently. SQLite queries run one at a time.
inside `Sync { ... }`, pooled `pg` and `mysql2` queries run concurrently. MySQL parameters are quoted client-side
with `Mysql2::Client#escape` and sent with `Client#query`, not server-side prepared statements. SQLite queries run
one at a time.
* **Style:** SQL lives in frozen heredoc constants (`GET_AUTHOR_SQL`), comments above a query become its doc comment,
table / column / enum comments of the schema become doc comments of the models. The code is RuboCop-clean; where a
name from your schema breaks a RuboCop naming rule (`"UserId"`, `naïve`), that class or method gets a `rubocop:disable`.
Expand All @@ -115,7 +118,11 @@ queries.transaction(isolation: :serializable) do |tx|
end
```
* Everything in the block runs on one connection. It commits when the block ends (also on `break`, `return`, `throw`)
and rolls back when it raises or the thread is killed. A nested `transaction` rolls back only its own part.
and rolls back when it raises or the thread is killed. On Ruby <3.3 with timeout <0.4, `break`, `return`, and
`throw` also roll back because Timeout cannot be told apart; add `gem 'timeout', '>= 0.4'` to the generated Gemfile
to keep those exits committing. A nested `transaction` rolls back only its own part. With `connection_pool` or
SQLite, parent-object calls from the same thread reuse the transaction connection; injected `#with` objects must do
the same.
* `isolation:` (`pg`, `mysql2`) takes `:read_uncommitted`, `:read_committed`, `:repeatable_read`, `:serializable`;
`mode:` (`sqlite3`) takes `:deferred`, `:immediate`, `:exclusive`. Other values raise before anything runs, and so
do `isolation:` / `mode:` on a nested `transaction` (a savepoint cannot change them).
Expand Down Expand Up @@ -170,7 +177,11 @@ server; duplicate keys and invalid values raise `Mysql2::Error` instead of being
The generated SQLite code uses `Monitor`: if you run Steep, add `library "monitor"` to your `Steepfile`.

### Names
* Names are snake_cased, acronyms kept together (`MIN_ID` → `min_id`, `HTTPServer` → `http_server`). Ruby keywords and names a row or method already uses get a trailing `_` (`class_`, `hash_`).
* Query methods and keyword arguments are snake_cased, with acronyms kept together (`MIN_ID` → `min_id`,
`HTTPServer` → `http_server`); Ruby keywords get a trailing `_` (`end_:`). Row members use the column spelling
reported by sqlc for the engine, except names that conflict with generated row methods (`class_`, `hash_`):
PostgreSQL and SQLite keep `"camelCase"` as `:camelCase`, while MySQL reports it as `:camelcase`. A column named
`end` is the row member `:end` but the argument `end_:`.
* A parameter used twice is one keyword when named with `sqlc.arg`; repeated `?` / `$n` of one column become `id:`, `id_2:`.
* A table `authors` next to a table `author` keeps its plural model name (`Authors`). Query names that would replace
a generated method (`Transaction`, `Initialize`, ...) are errors.
Expand All @@ -179,8 +190,8 @@ server; duplicate keys and invalid values raise `Mysql2::Error` instead of being
* Two result columns with the same name (e.g. `a.id, b.id`) are an error: alias one (`b.id AS book_id`) or use `sqlc.embed`.
* SQLite's `NUMERIC` affinity can return an `Integer` where `Float` is declared.
* `:batchexec` / `:batchone` / `:batchmany` are not atomic on their own: wrap them in `transaction` for all-or-nothing.
* SQLite: a transaction belongs to the thread that opened it; a Fiber / `Enumerator#next` inside it that uses the same
database waits for it (with `async`, run the work in the transaction's own fiber).
* SQLite transactions belong to the thread that opened them; fibers sharing the same database take turns, and a
transaction holds it until it ends.
* Not applicable to Ruby (they exist in the C# plugin): `withAsyncSuffix` / `withCancellationToken` (use the Fiber
scheduler), `useDapper`, target framework / csproj options (`moduleName` replaces the namespace).

Expand Down
10 changes: 6 additions & 4 deletions docs/Architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ stdout ◄─ Proto (encode) ◄─ GenerateResponse (query_sql.rb, query_sql.rb
| `type_resolver.rb` | RBS type of a column / parameter: driver type table, arrays, nullability (`T?`), `overrides:` (a `db_type` override types array elements too) |
| `catalog.rb` | the schema's tables and enums, and their Ruby names |
| `drivers/base.rb` | the driver interface and what all drivers share: `instrument` logging, `NotFound`, transaction RBS |
| `drivers/{pg,mysql2,sqlite3}.rb` | the per-library parts: type mapping, connecting / pooling, preparing and executing statements, transactions, `:copyfrom` |
| `drivers/{pg,mysql2,sqlite3}.rb` | the per-library parts: type mapping, connecting / pooling, executing statements, transactions, `:copyfrom` |
| `code.rb` | text helpers: indentation, `snake_case`, `camelize`, Ruby-keyword-safe parameter names, `rubocop:disable` pairs for schema names that break a naming cop |

**Rule of thumb:** `MethodGen` / `Generator` decide *what* is generated and are the same for every database; a driver only
Expand Down Expand Up @@ -62,11 +62,13 @@ Look at [`examples/pg/query_sql.rb`](../examples/pg/query_sql.rb) for a complete

Design decisions worth knowing (the *why* is in the comments of `drivers/base.rb` and `drivers/pg.rb`):

* **Prepared statements are per connection.** A pooled query object is shared by many connections, and a prepared
statement exists only on the connection that prepared it, so each connection remembers its own (`ConnectionState`).
* **PostgreSQL prepared statements are per connection.** A pooled query object is shared by many connections, and a
prepared statement exists only on the connection that prepared it, so each connection remembers its own
(`ConnectionState`). MySQL quotes values client-side and uses `Mysql2::Client#query` so queries cooperate with the
Fiber scheduler.
* **Pool injection.** `QuerySql.new(pool)` accepts anything that responds to `#with`, so an app can share its pool
and tests can pass a fake. `connect(...)` is only a convenience that builds a `connection_pool`.
* **Statements are closed.** SQLite/MySQL statements are closed in `ensure` (an open SQLite statement blocks `COMMIT`).
* **SQLite statements are closed.** They are closed in `ensure` because an open statement blocks `COMMIT`.
* **Parameters are encoded by the driver.** `Driver#encode_param` (e.g. `bytea`, `json` and arrays on `pg`) is used by
every command, batches included; add a type's encoding there, not in `MethodGen`.
* **`:copyfrom` is all or nothing** on every driver (a savepoint / transaction around the load), and MySQL's
Expand Down
Loading
Loading