Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 70 additions & 4 deletions phpBB/phpbb/search/fulltext_sphinx.php
Original file line number Diff line number Diff line change
Expand Up @@ -528,6 +528,10 @@ public function split_keywords(&$keywords, $terms)
* By default, only $, %, & and @ characters are indexed and searchable.
* String transformation is in backend only and not visible to the end user
* nor reflected in the results page URL or keyword highlighting.
* 6. Unmatched quotation marks and parentheses as well as parenthesized groups
* and operators without any search term left are removed, as their incomplete
* syntax would otherwise make the whole query fail with a syntax error.
* Balanced groups like (either | or) are preserved.
*
* @param string $search_string
* @return string
Expand Down Expand Up @@ -555,6 +559,57 @@ public function sphinx_clean_search_string($search_string)
*/
// $search_string = preg_replace('#[0-9]{1,3}\K,(?=[0-9]{3})#', '', $search_string);

// An unmatched quotation mark makes the whole query fail with a syntax error, remove it
if (substr_count($search_string, '"') % 2 !== 0)
{
$search_string = substr_replace($search_string, '', strrpos($search_string, '"'), 1);
}

// Remove parenthesized groups that contain no search term, eg "()" or "( +)"
do
{
$search_string = preg_replace('#\([^\p{L}\p{N}*"()]*\)#u', ' ', $search_string, -1, $count);
}
while ($count);

// Remove unmatched parentheses, ignoring those within quotation marks where they carry no meaning
$in_quotes = false;
$open_positions = [];
$unmatched_positions = [];
for ($i = 0; $i < strlen($search_string); $i++)
{
$char = $search_string[$i];
if ($char === '"')
{
$in_quotes = !$in_quotes;
}
else if (!$in_quotes && $char === '(')
{
$open_positions[] = $i;
}
else if (!$in_quotes && $char === ')')
{
if (count($open_positions))
{
array_pop($open_positions);
}
else
{
$unmatched_positions[] = $i;
}
}
}

$unmatched_positions = array_merge($unmatched_positions, $open_positions);
rsort($unmatched_positions);
foreach ($unmatched_positions as $position)
{
$search_string = substr_replace($search_string, '', $position, 1);
}

// Remove operators that lost their operand in the cleanup above, eg "+test -" or "(word |)"
$search_string = preg_replace('#(?<![^\s(])[+\-|]+(?![^\s)])#', '', $search_string);

return $search_string;
}

Expand Down Expand Up @@ -748,16 +803,27 @@ public function keyword_search($type, $fields, $terms, $sort_by_sql, $sort_key,

$this->sphinx->SetFilter('deleted', array(0));

$clean_search_query = $this->sphinx_clean_search_string(str_replace('&quot;', '"', $this->search_query));

if (strlen($this->search_query) && trim($clean_search_query) === '')
{
// No searchable terms are left after cleaning the search query,
// eg the user searched for a single parenthesis. An empty query
// would match all documents, so return no results instead. Author
// searches intentionally run with an empty query and are not affected.
return false;
}

$this->sphinx->SetLimits((int) $start, (int) $per_page, max(SPHINX_MAX_MATCHES, (int) $start + $per_page));
$result = $this->sphinx->Query($search_query_prefix . $this->sphinx_clean_search_string(str_replace('&quot;', '"', $this->search_query)), $this->indexes);
$result = $this->sphinx->Query($search_query_prefix . $clean_search_query, $this->indexes);

// Could be connection to localhost:9312 failed (errno=111,
// msg=Connection refused) during rotate, retry if so
$retries = SPHINX_CONNECT_RETRIES;
while (!$result && (strpos($this->sphinx->GetLastError(), "errno=111,") !== false) && $retries--)
{
usleep(SPHINX_CONNECT_WAIT_TIME);
$result = $this->sphinx->Query($search_query_prefix . $this->sphinx_clean_search_string(str_replace('&quot;', '"', $this->search_query)), $this->indexes);
$result = $this->sphinx->Query($search_query_prefix . $clean_search_query, $this->indexes);
}

if ($this->sphinx->GetLastError())
Expand All @@ -780,15 +846,15 @@ public function keyword_search($type, $fields, $terms, $sort_by_sql, $sort_key,
$start = floor(($result_count - 1) / $per_page) * $per_page;

$this->sphinx->SetLimits((int) $start, (int) $per_page, max(SPHINX_MAX_MATCHES, (int) $start + $per_page));
$result = $this->sphinx->Query($search_query_prefix . $this->sphinx_clean_search_string(str_replace('&quot;', '"', $this->search_query)), $this->indexes);
$result = $this->sphinx->Query($search_query_prefix . $clean_search_query, $this->indexes);

// Could be connection to localhost:9312 failed (errno=111,
// msg=Connection refused) during rotate, retry if so
$retries = SPHINX_CONNECT_RETRIES;
while (!$result && (strpos($this->sphinx->GetLastError(), "errno=111,") !== false) && $retries--)
{
usleep(SPHINX_CONNECT_WAIT_TIME);
$result = $this->sphinx->Query($search_query_prefix . $this->sphinx_clean_search_string(str_replace('&quot;', '"', $this->search_query)), $this->indexes);
$result = $this->sphinx->Query($search_query_prefix . $clean_search_query, $this->indexes);
}
}

Expand Down
77 changes: 77 additions & 0 deletions tests/search/sphinx_clean_search_string_test.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
<?php
/**
*
* This file is part of the phpBB Forum Software package.
*
* @copyright (c) phpBB Limited <https://www.phpbb.com>
* @license GNU General Public License, version 2 (GPL-2.0)
*
* For full copyright and license information, please see
* the docs/CREDITS.txt file.
*
*/

class phpbb_search_sphinx_clean_search_string_test extends phpbb_test_case
{
/** @var \phpbb\search\fulltext_sphinx */
protected $search;

protected function setUp(): void
{
global $phpbb_container, $phpbb_root_path, $phpEx;

parent::setUp();

$phpbb_container = new phpbb_mock_container_builder();
$phpbb_container->set('dbal.tools', new \stdClass());

$config = new \phpbb\config\config(['fulltext_sphinx_id' => 'test1234test5678']);
$lang_loader = new \phpbb\language\language_file_loader($phpbb_root_path, $phpEx);
$lang = new \phpbb\language\language($lang_loader);
$user = new \phpbb\user($lang, '\phpbb\datetime');
$phpbb_dispatcher = new phpbb_mock_event_dispatcher();
$error = null;

$this->search = new \phpbb\search\fulltext_sphinx($error, $phpbb_root_path, $phpEx, null, $config, null, $user, $phpbb_dispatcher);
}

public function clean_search_string_data()
{
return [
// Unmatched closing parenthesis is removed together with its operator (PHPBB-17614)
['+test +)', '+test '],
['+test +) ', '+test '],
['+test)', '+test'],
['+) ', ' '],
// Unmatched opening parenthesis is removed
['+test +(word ', '+test +word '],
['+a +)) +(( ', '+a '],
// Empty groups are removed together with their operator
['+test +() ', '+test '],
['+test +( +) ', '+test '],
// Operators without an operand are removed
['+test -) ', '+test '],
['+(word |) ', '+(word ) '],
// Unmatched quotation mark is removed
['+test +" ', '+test '],
// Balanced groups and phrases are preserved
['+(either +or) ', '+(either +or) '],
['+((nested)) ', '+((nested)) '],
['+test +"exact phrase" ', '+test +"exact phrase" '],
// Parentheses within quotation marks carry no meaning and do not affect balancing
['+(a +"b )" +c) ', '+(a +"b )" +c) '],
// Hyphenated words keep their special treatment
['know-it-all', '("know it all"|knowitall*)'],
// Plain queries are left alone
['+test +word ', '+test +word '],
];
}

/**
* @dataProvider clean_search_string_data
*/
public function test_clean_search_string($input, $expected)
{
$this->assertEquals($expected, $this->search->sphinx_clean_search_string($input));
}
}
Loading