Skip to content

Commit 97d5037

Browse files
Qup42Qup42
authored
Use DeltaTriplesManager to process update request (#1608)
Add the missing code in `Server.cpp` and `Server.h` to process a parsed update request using the `DeltaTriplesManager` from #1603 . As of this commit, QLever has an initial beta support for a subset of SPARQL UPDATE with the following limitations (all of which will be added in the future, and the list is far from being complete): 1. Updates are not yet persistent, so a restart or crash of the QLever server will delete all updates. 2. Only a single update request per query is allowed, not the syntax that atomically chains an arbitrary sequence of updates. 3. Only INSERT and DELETE queries are supported, the support for queries that drop or add a complete graph will be added in the future.
1 parent daf2c30 commit 97d5037

File tree

6 files changed

+292
-90
lines changed

6 files changed

+292
-90
lines changed

‎src/engine/Server.cpp‎

Lines changed: 169 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
#include <string>
1212
#include <vector>
1313

14+
#include "engine/ExecuteUpdate.h"
1415
#include "engine/ExportQueryExecutionTrees.h"
1516
#include "engine/QueryPlanner.h"
1617
#include "global/RuntimeParameters.h"
@@ -43,7 +44,7 @@ Server::Server(unsigned short port, size_t numThreads,
4344
enablePatternTrick_(usePatternTrick),
4445
// The number of server threads currently also is the number of queries
4546
// that can be processed simultaneously.
46-
threadPool_{numThreads} {
47+
queryThreadPool_{numThreads} {
4748
// This also directly triggers the update functions and propagates the
4849
// values of the parameters to the cache.
4950
RuntimeParameters().setOnUpdateAction<"cache-max-num-entries">(
@@ -394,6 +395,25 @@ Awaitable<void> Server::process(
394395
logCommand(cmd, "clear cache completely (including unpinned elements)");
395396
cache_.clearAll();
396397
response = createJsonResponse(composeCacheStatsJson(), request);
398+
} else if (auto cmd = checkParameter("cmd", "clear-delta-triples")) {
399+
requireValidAccessToken("clear-delta-triples");
400+
logCommand(cmd, "clear delta triples");
401+
// The function requires a SharedCancellationHandle, but the operation is
402+
// not cancellable.
403+
auto handle = std::make_shared<ad_utility::CancellationHandle<>>();
404+
// We don't directly `co_await` because of lifetime issues (bugs) in the
405+
// Conan setup.
406+
auto coroutine = computeInNewThread(
407+
updateThreadPool_,
408+
[this] {
409+
// Use `this` explicitly to silence false-positive errors on the
410+
// captured `this` being unused.
411+
this->index_.deltaTriplesManager().clear();
412+
},
413+
handle);
414+
co_await std::move(coroutine);
415+
response = createOkResponse("Delta triples have been cleared", request,
416+
MediaType::textPlain);
397417
} else if (auto cmd = checkParameter("cmd", "get-settings")) {
398418
logCommand(cmd, "get server settings");
399419
response = createJsonResponse(RuntimeParameters().toMap(), request);
@@ -470,9 +490,10 @@ Awaitable<void> Server::process(
470490
};
471491
auto visitUpdate =
472492
[&checkParameter, &accessTokenOk, &request, &send, &parameters,
473-
&requestTimer,
474-
this](const ad_utility::url_parser::sparqlOperation::Update& update)
493+
&requestTimer, this, &requireValidAccessToken](
494+
const ad_utility::url_parser::sparqlOperation::Update& update)
475495
-> Awaitable<void> {
496+
requireValidAccessToken("SPARQL Update");
476497
if (auto timeLimit = co_await verifyUserSubmittedQueryTimeout(
477498
checkParameter("timeout", std::nullopt), accessTokenOk, request,
478499
send)) {
@@ -521,17 +542,17 @@ std::pair<bool, bool> Server::determineResultPinning(
521542
checkParameter(params, "pinresult", "true").has_value();
522543
return {pinSubtrees, pinResult};
523544
}
545+
524546
// ____________________________________________________________________________
525-
Awaitable<Server::PlannedQuery> Server::setupPlannedQuery(
547+
Server::PlannedQuery Server::setupPlannedQuery(
526548
const ad_utility::url_parser::ParamValueMap& params,
527549
const std::string& operation, QueryExecutionContext& qec,
528550
SharedCancellationHandle handle, TimeLimit timeLimit,
529-
const ad_utility::Timer& requestTimer) {
551+
const ad_utility::Timer& requestTimer) const {
530552
auto queryDatasets = ad_utility::url_parser::parseDatasetClauses(params);
531-
std::optional<PlannedQuery> plannedQuery =
532-
co_await parseAndPlan(operation, queryDatasets, qec, handle, timeLimit);
533-
AD_CORRECTNESS_CHECK(plannedQuery.has_value());
534-
auto& qet = plannedQuery.value().queryExecutionTree_;
553+
PlannedQuery plannedQuery =
554+
parseAndPlan(operation, queryDatasets, qec, handle, timeLimit);
555+
auto& qet = plannedQuery.queryExecutionTree_;
535556
qet.isRoot() = true; // allow pinning of the final result
536557
auto timeForQueryPlanning = requestTimer.msecs();
537558
auto& runtimeInfoWholeQuery =
@@ -541,7 +562,7 @@ Awaitable<Server::PlannedQuery> Server::setupPlannedQuery(
541562
<< " ms" << std::endl;
542563
LOG(TRACE) << qet.getCacheKey() << std::endl;
543564

544-
co_return std::move(plannedQuery.value());
565+
return plannedQuery;
545566
}
546567
// _____________________________________________________________________________
547568
json Server::composeErrorResponseJson(
@@ -751,6 +772,18 @@ MediaType Server::determineMediaType(
751772
return mediaType.value();
752773
}
753774

775+
// ____________________________________________________________________________
776+
ad_utility::websocket::MessageSender Server::createMessageSender(
777+
const std::weak_ptr<ad_utility::websocket::QueryHub>& queryHub,
778+
const ad_utility::httpUtils::HttpRequest auto& request,
779+
const string& operation) {
780+
auto queryHubLock = queryHub.lock();
781+
AD_CORRECTNESS_CHECK(queryHubLock);
782+
ad_utility::websocket::MessageSender messageSender{
783+
getQueryId(request, operation), *queryHubLock};
784+
return messageSender;
785+
}
786+
754787
// ____________________________________________________________________________
755788
Awaitable<void> Server::processQuery(
756789
const ad_utility::url_parser::ParamValueMap& params, const string& query,
@@ -761,11 +794,8 @@ Awaitable<void> Server::processQuery(
761794
LOG(INFO) << "Requested media type of result is \""
762795
<< ad_utility::toString(mediaType) << "\"" << std::endl;
763796

764-
auto queryHub = queryHub_.lock();
765-
AD_CORRECTNESS_CHECK(queryHub);
766-
ad_utility::websocket::MessageSender messageSender{getQueryId(request, query),
767-
*queryHub};
768-
797+
ad_utility::websocket::MessageSender messageSender =
798+
createMessageSender(queryHub_, request, query);
769799
auto [cancellationHandle, cancelTimeoutOnDestruction] =
770800
setupCancellationHandle(messageSender.getQueryId(), timeLimit);
771801

@@ -779,15 +809,35 @@ Awaitable<void> Server::processQuery(
779809
QueryExecutionContext qec(index_, &cache_, allocator_,
780810
sortPerformanceEstimator_, std::ref(messageSender),
781811
pinSubtrees, pinResult);
782-
auto plannedQuery = co_await setupPlannedQuery(
783-
params, query, qec, cancellationHandle, timeLimit, requestTimer);
812+
813+
// The usage of an `optional` here is required because of a limitation in
814+
// Boost::Asio which forces us to use default-constructible result types with
815+
// `computeInNewThread`. We also can't unwrap the optional directly in this
816+
// function, because then the conan build fails in a very strange way,
817+
// probably related to issues in GCC's coroutine implementation.
818+
// For the same reason (crashes in the conanbuild) we store the coroutine in
819+
// an explicit variable instead of directly `co_await`-ing it.
820+
auto coroutine = computeInNewThread(
821+
queryThreadPool_,
822+
[this, &params, &query, &qec, cancellationHandle, &timeLimit,
823+
&requestTimer]() -> std::optional<PlannedQuery> {
824+
return setupPlannedQuery(params, query, qec, cancellationHandle,
825+
timeLimit, requestTimer);
826+
},
827+
cancellationHandle);
828+
auto plannedQueryOpt = co_await std::move(coroutine);
829+
AD_CORRECTNESS_CHECK(plannedQueryOpt.has_value());
830+
auto plannedQuery = std::move(plannedQueryOpt).value();
784831
auto qet = plannedQuery.queryExecutionTree_;
785832

786833
if (plannedQuery.parsedQuery_.hasUpdateClause()) {
787834
// This may be caused by a bug (the code is not yet tested well) or by an
788835
// attack which tries to circumvent (not yet existing) access controls for
789836
// Update.
790-
throw std::runtime_error("Expected normal query but received update query");
837+
throw std::runtime_error(
838+
absl::StrCat("SPARQL QUERY was request via the HTTP request, but the "
839+
"following update was sent instead of an update: ",
840+
plannedQuery.parsedQuery_._originalString));
791841
}
792842

793843
// Read the export limit from the send` parameter (historical name). This
@@ -833,6 +883,80 @@ Awaitable<void> Server::processQuery(
833883
co_return;
834884
}
835885

886+
// ____________________________________________________________________________
887+
void Server::processUpdateImpl(
888+
const ad_utility::url_parser::ParamValueMap& params, const string& update,
889+
ad_utility::Timer& requestTimer, TimeLimit timeLimit, auto& messageSender,
890+
ad_utility::SharedCancellationHandle cancellationHandle,
891+
DeltaTriples& deltaTriples) {
892+
auto [pinSubtrees, pinResult] = determineResultPinning(params);
893+
LOG(INFO) << "Processing the following SPARQL update:"
894+
<< (pinResult ? " [pin result]" : "")
895+
<< (pinSubtrees ? " [pin subresults]" : "") << "\n"
896+
<< update << std::endl;
897+
QueryExecutionContext qec(index_, &cache_, allocator_,
898+
sortPerformanceEstimator_, std::ref(messageSender),
899+
pinSubtrees, pinResult);
900+
auto plannedQuery = setupPlannedQuery(params, update, qec, cancellationHandle,
901+
timeLimit, requestTimer);
902+
auto qet = plannedQuery.queryExecutionTree_;
903+
904+
if (!plannedQuery.parsedQuery_.hasUpdateClause()) {
905+
throw std::runtime_error(
906+
absl::StrCat("SPARQL UPDATE was request via the HTTP request, but the "
907+
"following query was sent instead of an update: ",
908+
plannedQuery.parsedQuery_._originalString));
909+
}
910+
ExecuteUpdate::executeUpdate(index_, plannedQuery.parsedQuery_, qet,
911+
deltaTriples, cancellationHandle);
912+
913+
LOG(INFO) << "Done processing update"
914+
<< ", total time was " << requestTimer.msecs().count() << " ms"
915+
<< std::endl;
916+
LOG(DEBUG) << "Runtime Info:\n"
917+
<< qet.getRootOperation()->runtimeInfo().toString() << std::endl;
918+
}
919+
920+
// ____________________________________________________________________________
921+
Awaitable<void> Server::processUpdate(
922+
const ad_utility::url_parser::ParamValueMap& params, const string& update,
923+
ad_utility::Timer& requestTimer,
924+
const ad_utility::httpUtils::HttpRequest auto& request, auto&& send,
925+
TimeLimit timeLimit) {
926+
auto messageSender = createMessageSender(queryHub_, request, update);
927+
928+
auto [cancellationHandle, cancelTimeoutOnDestruction] =
929+
setupCancellationHandle(messageSender.getQueryId(), timeLimit);
930+
931+
// Update the delta triples.
932+
933+
// Note: We don't directly `co_await` because of lifetime issues (probably
934+
// bugs in GCC or Boost) that occur in the Conan build.
935+
auto coroutine = computeInNewThread(
936+
updateThreadPool_,
937+
[this, &params, &update, &requestTimer, &timeLimit, &messageSender,
938+
&cancellationHandle] {
939+
index_.deltaTriplesManager().modify(
940+
[this, &params, &update, &requestTimer, &timeLimit, &messageSender,
941+
&cancellationHandle](auto& deltaTriples) {
942+
// Use `this` explicitly to silence false-positive errors on
943+
// captured `this` being unused.
944+
this->processUpdateImpl(params, update, requestTimer, timeLimit,
945+
messageSender, cancellationHandle,
946+
deltaTriples);
947+
});
948+
},
949+
cancellationHandle);
950+
co_await std::move(coroutine);
951+
952+
// TODO<qup42> send a proper response
953+
// SPARQL 1.1 Protocol 2.2.4 Successful Responses: "The response body of a
954+
// successful update request is implementation defined."
955+
co_await send(ad_utility::httpUtils::createOkResponse(
956+
"Update successful", request, MediaType::textPlain));
957+
co_return;
958+
}
959+
836960
// ____________________________________________________________________________
837961
template <Server::OperationType type>
838962
Awaitable<void> Server::processQueryOrUpdate(
@@ -858,8 +982,8 @@ Awaitable<void> Server::processQueryOrUpdate(
858982
co_await processQuery(params, queryOrUpdate, requestTimer, request, send,
859983
timeLimit);
860984
} else {
861-
throw std::runtime_error(
862-
"SPARQL 1.1 Update is currently not supported by QLever.");
985+
co_await processUpdate(params, queryOrUpdate, requestTimer, request, send,
986+
timeLimit);
863987
}
864988
} catch (const ParseException& e) {
865989
responseStatus = http::status::bad_request;
@@ -894,7 +1018,7 @@ Awaitable<void> Server::processQueryOrUpdate(
8941018
exceptionErrorMsg.value().append(absl::StrCat(
8951019
" Highlighting an error for the command line log failed: ",
8961020
e.what()));
897-
LOG(ERROR) << "Failed to highlight error in query. " << e.what()
1021+
LOG(ERROR) << "Failed to highlight error in operation. " << e.what()
8981022
<< std::endl;
8991023
LOG(ERROR) << metadata.value().query_ << std::endl;
9001024
}
@@ -915,7 +1039,8 @@ Awaitable<void> Server::processQueryOrUpdate(
9151039

9161040
// _____________________________________________________________________________
9171041
template <std::invocable Function, typename T>
918-
Awaitable<T> Server::computeInNewThread(Function function,
1042+
Awaitable<T> Server::computeInNewThread(net::static_thread_pool& threadPool,
1043+
Function function,
9191044
SharedCancellationHandle handle) {
9201045
// `interruptible` will set the shared state of this promise
9211046
// with a function that can be used to cancel the timer.
@@ -935,48 +1060,35 @@ Awaitable<T> Server::computeInNewThread(Function function,
9351060
// this might still block. However it will make the code check the
9361061
// cancellation handle while waiting for a thread in the pool to become ready.
9371062
return ad_utility::interruptible(
938-
ad_utility::runFunctionOnExecutor(threadPool_.get_executor(),
1063+
ad_utility::runFunctionOnExecutor(threadPool.get_executor(),
9391064
std::move(inner), net::use_awaitable),
9401065
std::move(handle), std::move(cancelTimerPromise));
9411066
}
9421067

9431068
// _____________________________________________________________________________
944-
net::awaitable<std::optional<Server::PlannedQuery>> Server::parseAndPlan(
1069+
Server::PlannedQuery Server::parseAndPlan(
9451070
const std::string& query, const vector<DatasetClause>& queryDatasets,
9461071
QueryExecutionContext& qec, SharedCancellationHandle handle,
947-
TimeLimit timeLimit) {
948-
auto handleCopy = handle;
949-
950-
// The usage of an `optional` here is required because of a limitation in
951-
// Boost::Asio which forces us to use default-constructible result types with
952-
// `computeInNewThread`. We also can't unwrap the optional directly in this
953-
// function, because then the conan build fails in a very strange way,
954-
// probably related to issues in GCC's coroutine implementation.
955-
return computeInNewThread(
956-
[&query, &qec, enablePatternTrick = enablePatternTrick_,
957-
handle = std::move(handle), timeLimit,
958-
&queryDatasets]() mutable -> std::optional<PlannedQuery> {
959-
auto pq = SparqlParser::parseQuery(query);
960-
handle->throwIfCancelled();
961-
// SPARQL Protocol 2.1.4 specifies that the dataset from the query
962-
// parameters overrides the dataset from the query itself.
963-
if (!queryDatasets.empty()) {
964-
pq.datasetClauses_ =
965-
parsedQuery::DatasetClauses::fromClauses(queryDatasets);
966-
}
967-
QueryPlanner qp(&qec, handle);
968-
qp.setEnablePatternTrick(enablePatternTrick);
969-
auto qet = qp.createExecutionTree(pq);
970-
handle->throwIfCancelled();
971-
PlannedQuery plannedQuery{std::move(pq), std::move(qet)};
972-
973-
plannedQuery.queryExecutionTree_.getRootOperation()
974-
->recursivelySetCancellationHandle(std::move(handle));
975-
plannedQuery.queryExecutionTree_.getRootOperation()
976-
->recursivelySetTimeConstraint(timeLimit);
977-
return plannedQuery;
978-
},
979-
std::move(handleCopy));
1072+
TimeLimit timeLimit) const {
1073+
auto pq = SparqlParser::parseQuery(query);
1074+
handle->throwIfCancelled();
1075+
// SPARQL Protocol 2.1.4 specifies that the dataset from the query
1076+
// parameters overrides the dataset from the query itself.
1077+
if (!queryDatasets.empty()) {
1078+
pq.datasetClauses_ =
1079+
parsedQuery::DatasetClauses::fromClauses(queryDatasets);
1080+
}
1081+
QueryPlanner qp(&qec, handle);
1082+
qp.setEnablePatternTrick(enablePatternTrick_);
1083+
auto qet = qp.createExecutionTree(pq);
1084+
handle->throwIfCancelled();
1085+
PlannedQuery plannedQuery{std::move(pq), std::move(qet)};
1086+
1087+
plannedQuery.queryExecutionTree_.getRootOperation()
1088+
->recursivelySetCancellationHandle(std::move(handle));
1089+
plannedQuery.queryExecutionTree_.getRootOperation()
1090+
->recursivelySetTimeConstraint(timeLimit);
1091+
return plannedQuery;
9801092
}
9811093

9821094
// _____________________________________________________________________________

0 commit comments

Comments
 (0)