Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions src/CondFormat.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,28 @@ CondFormat::CondFormat(const QString& filter,
m_font.fromString(model->data(index, Qt::FontRole).toString());
}

namespace
{

// Checks whether a string is a number which does not change when it is converted to a number and written out again, i.e.
// that it is the same number no matter whether it is read as text or as a number. This is not the case for things like
// 007, 1.50, 1e5 or +5. It also excludes values like inf and nan which SQLite would not read as numbers.
bool isCanonicalNumber(const QString& str)
{
bool ok = false;
if(!str.contains('.'))
{
const qlonglong integer = str.toLongLong(&ok);
return ok && QString::number(integer) == str;
}

// Real numbers are converted to text by SQLite using 15 significant digits
const double real = str.toDouble(&ok);
return ok && QString::number(real, 'g', 15) == str;
}

}

std::string CondFormat::filterToSqlCondition(const QString& value, const QString& encoding)
{
if(value.isEmpty())
Expand Down Expand Up @@ -111,6 +133,11 @@ std::string CondFormat::filterToSqlCondition(const QString& value, const QString
{
// It's not, so just compare normally to the value, whatever it is.
op = "=";

// Don't quote numbers. SQLite never considers a number equal to a quoted string if the column has no type affinity,
// e.g. the result of COUNT(*). Numbers which can be written differently, like 007 or 1.50, are kept as text because they
// would not match the text stored in a text column anymore.
numeric = isCanonicalNumber(val);
} else {
// It is NULL. Override the comparison operator to search for NULL values in this column. Also treat search value (NULL) as number,
// in order to avoid putting quotes around it.
Expand Down
23 changes: 23 additions & 0 deletions src/Data.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
#include <QTextCodec>
#include <QFile>
#include <QRegularExpression>
#include <QIODevice>
#include <QTextStream>

#include <algorithm>

Expand Down Expand Up @@ -307,3 +309,24 @@ std::vector<BufferRow> parseClipboard(QString clipboard)

return result;
}

bool writeAll(QIODevice& device, const char* data, qint64 size)
{
while(size > 0)
{
const qint64 written = device.write(data, size);
if(written <= 0)
return false;

data += written;
size -= written;
}

return true;
}

bool flushStream(QTextStream& stream)
{
stream.flush();
return stream.status() == QTextStream::Ok;
}
13 changes: 13 additions & 0 deletions src/Data.h
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@
#include <QByteArray>
#include <vector>

class QIODevice;
class QTextStream;

// This returns false if the data in the data parameter contains binary data. If it is text only, the function returns
// true. If the second parameter is specified, it will be used to convert the data from the given encoding to Unicode
// before doing the check. The third parameter can be used to only check the first couple of bytes which speeds up the
Expand Down Expand Up @@ -44,4 +47,14 @@ void removeCommentsFromQuery(QString& query);
// Split tab-separated clipboard data into rows, preserving quoted fields.
std::vector<std::vector<QByteArray>> parseClipboard(QString clipboard);

// Write all of the data to the device. Returns false if any of it could not be written. QIODevice::write() reports a partial write, which
// is what happens when the disk is full, only by returning a number smaller than the size and not as an error of the device. Writing the
// rest of the data then gives the error, so that the device can tell what went wrong.
bool writeAll(QIODevice& device, const char* data, qint64 size);

// Write the data which is still buffered in the stream to its device. Returns false if this or any earlier write to the stream failed.
// Without flushing the stream, errors in writing the last part of the data would only show up when the device is closed, after the
// status of the stream was last checked.
bool flushStream(QTextStream& stream);

#endif
16 changes: 10 additions & 6 deletions src/ExportDataDialog.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -185,8 +185,12 @@ bool ExportDataDialog::exportQueryCsv(const std::string& sQuery, const QString&
QApplication::restoreOverrideCursor();
qApp->processEvents();

// Done writing the file
if(!file.commit()) {
// The stream still holds the last part of the data. If the disk is full, writing it is where that shows up when the file is small.
if(!flushStream(stream))
writeError = true;

// Done writing the file. Only replace the original file if everything has been written, not with an incomplete file.
if(!writeError && !file.commit()) {
writeError = true;
}

Expand Down Expand Up @@ -284,14 +288,14 @@ bool ExportDataDialog::exportQueryJson(const std::string& sQuery, const QString&
sqlite3_finalize(stmt);

// Create JSON document
file.write(json_table.dump(ui->checkPrettyPrint->isChecked() ? 4 : -1).c_str());
bool writeError = file.error() != QFileDevice::NoError;
const std::string json_text = json_table.dump(ui->checkPrettyPrint->isChecked() ? 4 : -1);
bool writeError = !writeAll(file, json_text.c_str(), static_cast<qint64>(json_text.size()));

QApplication::restoreOverrideCursor();
qApp->processEvents();

// Done writing the file
if(!file.commit()) {
// Done writing the file. Only replace the original file if everything has been written, not with an incomplete file.
if(!writeError && !file.commit()) {
writeError = true;
}

Expand Down
6 changes: 5 additions & 1 deletion src/ExtendedTableWidget.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -796,7 +796,11 @@ void ExtendedTableWidget::paste()
int column = firstColumn;
for(const QByteArray& source_cell : source_row)
{
m->setData(m->index(row, column), source_cell);
// Binary data must be stored as binary data and not as text, just like in the single cell cases above. Like setData() does,
// we don't change anything if the model is not editable.
const QModelIndex index = m->index(row, column);
if(m->isEditable(index))
m->setTypedData(index, !isTextOnly(source_cell), source_cell);

column++;
if (column > lastColumn)
Expand Down
71 changes: 71 additions & 0 deletions src/tests/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -123,3 +123,74 @@ set(TESTCACHE_HDR
add_executable(test-cache ${TESTCACHE_HDR} ${TESTCACHE_SRC})
target_link_libraries(test-cache ${QT_MAJOR}::Test)
add_test(NAME test-cache COMMAND $<TARGET_FILE:test-cache>)

# test paste

set(TESTPASTE_SRC
TestPaste.cpp
../ExtendedTableWidget.cpp
../FilterTableHeader.cpp
../FilterLineEdit.cpp
../sqlitedb.cpp
../sqlitetablemodel.cpp
../RowLoader.cpp
../CipherDialog.cpp
../CipherSettings.cpp
../CondFormat.cpp
../Data.cpp
../Settings.cpp
../sql/sqlitetypes.cpp
../sql/Query.cpp
../sql/ObjectIdentifier.cpp
../sql/parser/ParserDriver.cpp
../sql/parser/sqlite3_lexer.cpp
../sql/parser/sqlite3_parser.cpp
)

set(TESTPASTE_HDR
TestPaste.h
../ExtendedTableWidget.h
../FilterTableHeader.h
../FilterLineEdit.h
../sqlitedb.h
../sqlitetablemodel.h
../RowLoader.h
../RowCache.h
../CipherDialog.h
../CipherSettings.h
../CondFormat.h
../Data.h
../Settings.h
../sql/sqlitetypes.h
../sql/Query.h
../sql/ObjectIdentifier.h
../sql/parser/ParserDriver.h
../sql/parser/sqlite3_lexer.h
../sql/parser/sqlite3_location.h
../sql/parser/sqlite3_parser.hpp
)

add_executable(test-paste ${TESTPASTE_HDR} ${TESTPASTE_SRC} ../CipherDialog.ui)
target_include_directories(test-paste SYSTEM PRIVATE ${CMAKE_SOURCE_DIR}/libs/json)
if(QT_MAJOR STREQUAL "Qt6")
set_target_properties(test-paste PROPERTIES AUTOUIC_OPTIONS "--connections=string")
endif()
target_link_libraries(test-paste ${QT_MAJOR}::Test ${QT_MAJOR}::Widgets ${QT_MAJOR}::Concurrent ${QT_MAJOR}::PrintSupport ${LIBSQLITE_NAME} ${QT5_COMPAT})
add_test(NAME test-paste COMMAND $<TARGET_FILE:test-paste>)
if(WIN32)
set(_paste_extra_paths "")
foreach(_p IN LISTS CMAKE_PREFIX_PATH)
list(APPEND _paste_extra_paths "${_p}" "${_p}/bin")
# OpenSSL 3 Universal installer layout: bin/x86, bin/x64
if(CMAKE_SIZEOF_VOID_P EQUAL 8)
list(APPEND _paste_extra_paths "${_p}/bin/x64")
else()
list(APPEND _paste_extra_paths "${_p}/bin/x86")
endif()
endforeach()
# SQLite and SQLCipher are built as DLLs in the CMake prefix by the Windows CI jobs.
string(REPLACE ";" "\\;" TESTPASTE_PATH "${_paste_extra_paths};$ENV{PATH}")
set_tests_properties(test-paste PROPERTIES ENVIRONMENT "QT_QPA_PLATFORM=offscreen;PATH=${TESTPASTE_PATH}")
else()
set_tests_properties(test-paste PROPERTIES ENVIRONMENT "QT_QPA_PLATFORM=offscreen")
endif()
116 changes: 116 additions & 0 deletions src/tests/TestGlobalFilter.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,28 @@ QStringList executeIssueQuery(const std::vector<QString>& filters)
return rows;
}

// Runs the statements in a fresh in-memory database and returns the first column of all rows of the last query,
// joined by commas
QString runQuery(const char* setup, const std::string& query)
{
sqlite3* db = nullptr;
if(sqlite3_open(":memory:", &db) != SQLITE_OK)
qFatal("Could not open the SQLite fixture");
if(sqlite3_exec(db, setup, nullptr, nullptr, nullptr) != SQLITE_OK)
qFatal("Could not create the SQLite fixture: %s", sqlite3_errmsg(db));

QStringList rows;
const int result = sqlite3_exec(db, query.c_str(), [](void* data, int count, char** values, char**) {
if(count > 0 && values[0])
static_cast<QStringList*>(data)->push_back(QString::fromUtf8(values[0]));
return 0;
}, &rows, nullptr);
if(result != SQLITE_OK)
qFatal("Fixture query failed: %s", sqlite3_errmsg(db));
sqlite3_close(db);
return rows.join(',');
}

}

void TestGlobalFilter::tokenizes_data()
Expand Down Expand Up @@ -139,3 +161,97 @@ void TestGlobalFilter::projectRoundTripPreservesRegexTerm()
QCOMPARE(asStringList(restored), QStringList{"/New York/"});
QCOMPARE(executeIssueQuery(restored), QStringList{"New York"});
}

void TestGlobalFilter::exactFilterMatches_data()
{
QTest::addColumn<QString>("setup");
QTest::addColumn<QString>("filter");
QTest::addColumn<QString>("expected");

// The result of COUNT(*) has no type affinity, so SQLite never considers it equal to a quoted string (#3608)
QTest::newRow("count column")
<< "CREATE TABLE d(g); INSERT INTO d VALUES (1),(2),(2),(3),(3),(3);"
"CREATE VIEW t AS SELECT COUNT(*) AS c FROM d GROUP BY g;"
<< "=2" << "2";
QTest::newRow("count column zero")
<< "CREATE TABLE d(g); CREATE TABLE e(g); INSERT INTO d VALUES (1),(2),(2);"
"INSERT INTO e VALUES (1),(2),(3);"
"CREATE VIEW t AS SELECT COUNT(d.g) AS c FROM e LEFT JOIN d ON d.g = e.g GROUP BY e.g;"
<< "=0" << "0";
QTest::newRow("integer column")
<< "CREATE TABLE t(c INTEGER); INSERT INTO t VALUES (4),(5),(50);"
<< "=5" << "5";
QTest::newRow("negative integer column")
<< "CREATE TABLE t(c INTEGER); INSERT INTO t VALUES (-3),(3),(-30);"
<< "=-3" << "-3";
QTest::newRow("real column")
<< "CREATE TABLE t(c REAL); INSERT INTO t VALUES (2.5),(25),(2);"
<< "=2.5" << "2.5";
QTest::newRow("computed real")
<< "CREATE TABLE d(x); INSERT INTO d VALUES (1),(2);"
"CREATE VIEW t AS SELECT x / 2.0 AS c FROM d;"
<< "=0.5" << "0.5";

// Text columns must keep matching the text which is stored in them, even if it looks like a number
QTest::newRow("text column")
<< "CREATE TABLE t(c TEXT); INSERT INTO t VALUES ('5'),('50'),('05');"
<< "=5" << "5";
QTest::newRow("text column leading zeros")
<< "CREATE TABLE t(c TEXT); INSERT INTO t VALUES ('007'),('7'),('70');"
<< "=007" << "007";
QTest::newRow("text column trailing zero")
<< "CREATE TABLE t(c TEXT); INSERT INTO t VALUES ('9.90'),('9.9'),('9');"
<< "=9.90" << "9.90";
QTest::newRow("text column too long for a number")
<< "CREATE TABLE t(c TEXT); INSERT INTO t VALUES ('12345678901234567890123'),('1');"
<< "=12345678901234567890123" << "12345678901234567890123";
QTest::newRow("text column word")
<< "CREATE TABLE t(c TEXT); INSERT INTO t VALUES ('abc'),('abcd');"
<< "=abc" << "abc";
}

void TestGlobalFilter::exactFilterMatches()
{
QFETCH(QString, setup);
QFETCH(QString, filter);
QFETCH(QString, expected);

const std::string query = "SELECT group_concat(c) FROM (SELECT c FROM t WHERE c " + CondFormat::filterToSqlCondition(filter) + " ORDER BY c)";
QCOMPARE(runQuery(setup.toUtf8().constData(), query), expected);
}

void TestGlobalFilter::exactFilterQuoting_data()
{
QTest::addColumn<QString>("filter");
QTest::addColumn<QString>("expected");

QTest::newRow("integer") << "=2" << "= 2 ";
QTest::newRow("zero") << "=0" << "= 0 ";
QTest::newRow("negative") << "=-3" << "= -3 ";
QTest::newRow("decimal") << "=2.5" << "= 2.5 ";
QTest::newRow("small decimal") << "=0.5" << "= 0.5 ";
QTest::newRow("largest integer") << "=9223372036854775807" << "= 9223372036854775807 ";

// These would not be the same value after being converted to a number and back, or are no numbers at all.
// They must stay quoted because SQLite would read them differently or not at all, e.g. inf as a column name.
QTest::newRow("leading zeros") << "=007" << "= '007' ";
QTest::newRow("trailing zero") << "=1.50" << "= '1.50' ";
QTest::newRow("exponent") << "=1e5" << "= '1e5' ";
QTest::newRow("explicit plus") << "=+5" << "= '+5' ";
QTest::newRow("negative zero") << "=-0" << "= '-0' ";
QTest::newRow("no integer part") << "=.5" << "= '.5' ";
QTest::newRow("no fraction part") << "=5." << "= '5.' ";
QTest::newRow("hexadecimal") << "=0x10" << "= '0x10' ";
QTest::newRow("infinity") << "=inf" << "= 'inf' ";
QTest::newRow("not a number") << "=nan" << "= 'nan' ";
QTest::newRow("beyond 64 bit integers") << "=9223372036854775808" << "= '9223372036854775808' ";
QTest::newRow("text") << "=abc" << "= 'abc' ";
QTest::newRow("null") << "=NULL" << "IS NULL ";
}

void TestGlobalFilter::exactFilterQuoting()
{
QFETCH(QString, filter);
QFETCH(QString, expected);
QCOMPARE(QString::fromStdString(CondFormat::filterToSqlCondition(filter)), expected);
}
4 changes: 4 additions & 0 deletions src/tests/TestGlobalFilter.h
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,10 @@ private slots:
void issueFixtureReturnsNewYork();
void ordinaryTermsRemainAndConnected();
void projectRoundTripPreservesRegexTerm();
void exactFilterMatches_data();
void exactFilterMatches();
void exactFilterQuoting_data();
void exactFilterQuoting();
};

#endif // TESTGLOBALFILTER_H
Loading
Loading