Skip to content

Commit 5161ba1

Browse files
claudeclaude
committed
Load posts in chunks in wp post list
`wp post list` loaded every matching post, including its full content, in a single query, primed their meta and term caches, and computed the permalink of every post even when the `url` field was not requested. On a site with ~100k posts this used ~900 MB of memory regardless of the requested fields. Query the IDs first, then load the posts in chunks with the same query arguments, clearing the object cache after each chunk, and only compute `url` when it is requested. The posts are passed to the formatter as a generator, which newer WP-CLI versions stream for CSV and JSON. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01D26yjkN2BiqCXT6p6o1WqS
1 parent 64e9d34 commit 5161ba1

2 files changed

Lines changed: 80 additions & 12 deletions

File tree

‎features/post.feature‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1152,3 +1152,24 @@ Feature: Manage WordPress posts
11521152
2021-02-02
11531153
"""
11541154
And STDOUT should not be empty
1155+
1156+
Scenario: List more posts than are loaded from the database at a time
1157+
Given a WP install
1158+
And I run `wp post generate --count=1200 --post_date="2020-01-01 00:00:00"`
1159+
And I run `wp post list --post_status=publish --orderby=ID --order=asc --format=ids`
1160+
And save STDOUT as {IDS}
1161+
1162+
When I run `wp post list --post_status=publish --orderby=ID --order=asc --field=ID | tr '\n' ' ' | sed 's/ $//'`
1163+
Then STDOUT should be:
1164+
"""
1165+
{IDS}
1166+
"""
1167+
1168+
When I run `wp post list --post_status=publish --orderby=ID --order=asc --fields=ID,post_title --format=csv | wc -l`
1169+
Then STDOUT should contain:
1170+
"""
1171+
1202
1172+
"""
1173+
1174+
When I run `wp post list --post_status=publish --orderby=ID --order=desc --posts_per_page=2 --fields=ID,url --format=csv`
1175+
Then STDOUT should match /^ID,url\n(\d+),https?:\/\/example\.com\/\?p=\1\n(\d+),https?:\/\/example\.com\/\?p=\2$/

‎src/Post_Command.php‎

Lines changed: 59 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,11 @@
2727
*/
2828
class Post_Command extends CommandWithDBObject {
2929

30+
/**
31+
* Number of posts `wp post list` loads from the database at a time.
32+
*/
33+
const LIST_CHUNK_SIZE = 500;
34+
3035
protected $obj_type = 'post';
3136
protected $obj_fields = [
3237
'ID',
@@ -1057,20 +1062,62 @@ public function list_( $args, $assoc_args ) {
10571062
$query = new WP_Query( $query_args );
10581063
$formatter->display_items( $query->posts ?? [] );
10591064
} else {
1060-
$query = new WP_Query( $query_args );
1061-
$posts = array_map(
1062-
function ( $post ) {
1063-
/**
1064-
* @var \WP_Post $post
1065-
*/
1065+
$need_url = 'url' === $formatter->field || in_array( 'url', (array) $formatter->fields, true );
1066+
$formatter->display_items( $this->query_posts_in_chunks( $query_args, $need_url ) );
1067+
}
1068+
}
10661069

1070+
/**
1071+
* Query posts in chunks, so that they don't all have to be held in memory at once.
1072+
*
1073+
* The IDs of all matching posts are queried first. The posts are then loaded a chunk
1074+
* at a time with the same query arguments, and the object cache is cleared after each
1075+
* chunk. Formats that can be written item by item are then streamed by the formatter.
1076+
*
1077+
* @param array<string, mixed> $query_args WP_Query arguments.
1078+
* @param bool $need_url Whether to add the `url` property to each post.
1079+
* @return \Generator<int, \WP_Post>
1080+
*/
1081+
private function query_posts_in_chunks( $query_args, $need_url ) {
1082+
/** @var int[] $ids */
1083+
$ids = ( new WP_Query( array_merge( $query_args, [ 'fields' => 'ids' ] ) ) )->posts;
1084+
1085+
$chunk_args = $query_args;
1086+
unset( $chunk_args['offset'], $chunk_args['paged'], $chunk_args['page'], $chunk_args['nopaging'] );
1087+
// The order is restored from $ids below. Ordering by `post__in` in SQL would
1088+
// need a FIELD() call with one argument per post, which SQLite does not allow.
1089+
$chunk_args['orderby'] = 'none';
1090+
$chunk_args['no_found_rows'] = true;
1091+
$chunk_args['ignore_sticky_posts'] = true;
1092+
1093+
foreach ( array_chunk( $ids, self::LIST_CHUNK_SIZE ) as $chunk ) {
1094+
$chunk_args['post__in'] = $chunk;
1095+
$chunk_args['posts_per_page'] = count( $chunk );
1096+
1097+
$query = new WP_Query( $chunk_args );
1098+
1099+
$posts = [];
1100+
foreach ( $query->posts as $post ) {
1101+
if ( $post instanceof \WP_Post ) {
1102+
$posts[ $post->ID ] = $post;
1103+
}
1104+
}
1105+
1106+
foreach ( $chunk as $id ) {
1107+
if ( ! isset( $posts[ $id ] ) ) {
1108+
continue;
1109+
}
1110+
1111+
$post = $posts[ $id ];
1112+
if ( $need_url ) {
10671113
// @phpstan-ignore property.notFound
1068-
$post->url = get_permalink( $post->ID );
1069-
return $post;
1070-
},
1071-
$query->posts ?? []
1072-
);
1073-
$formatter->display_items( $posts );
1114+
$post->url = get_permalink( $post );
1115+
}
1116+
yield $post;
1117+
}
1118+
1119+
unset( $posts, $query );
1120+
Utils\wp_clear_object_cache(); // phpcs:ignore PHPCompatibility.FunctionUse.RemovedFunctions.wp_clear_object_cacheDeprecatedRemoved @phpstan-ignore-line
10741121
}
10751122
}
10761123

0 commit comments

Comments
 (0)