Skip to content

Commit df180d9

Browse files
zhicwuzhicwu
authored
Merge pull request ClickHouse#498 from infovista/result-summary
Give access to stats returned by server in X-ClickHouse-Summary
2 parents f933823 + e44a78c commit df180d9

File tree

8 files changed

+323
-7
lines changed

8 files changed

+323
-7
lines changed

‎src/main/java/ru/yandex/clickhouse/ClickHouseStatement.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package ru.yandex.clickhouse;
22

33
import ru.yandex.clickhouse.response.ClickHouseResponse;
4+
import ru.yandex.clickhouse.response.ClickHouseResponseSummary;
45
import ru.yandex.clickhouse.settings.ClickHouseQueryParam;
56
import ru.yandex.clickhouse.util.ClickHouseRowBinaryInputStream;
67
import ru.yandex.clickhouse.util.ClickHouseStreamCallback;
@@ -105,4 +106,6 @@ ResultSet executeQuery(String sql,
105106
* Returns extended write-API
106107
*/
107108
Writer write();
109+
110+
ClickHouseResponseSummary getResponseSummary();
108111
}

‎src/main/java/ru/yandex/clickhouse/ClickHouseStatementImpl.java‎

Lines changed: 56 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package ru.yandex.clickhouse;
22

33
import com.google.common.base.Strings;
4+
import org.apache.http.Header;
45
import org.apache.http.HttpEntity;
56
import org.apache.http.HttpResponse;
67
import org.apache.http.NameValuePair;
@@ -38,7 +39,7 @@
3839
import java.util.*;
3940

4041

41-
public class ClickHouseStatementImpl implements ClickHouseStatement {
42+
public class ClickHouseStatementImpl extends ConfigurableApi<ClickHouseStatement> implements ClickHouseStatement {
4243

4344
private static final Logger log = LoggerFactory.getLogger(ClickHouseStatementImpl.class);
4445

@@ -52,6 +53,8 @@ public class ClickHouseStatementImpl implements ClickHouseStatement {
5253

5354
private ClickHouseRowBinaryInputStream currentRowBinaryResult;
5455

56+
private ClickHouseResponseSummary currentSummary;
57+
5558
private int currentUpdateCount = -1;
5659

5760
private int queryTimeout;
@@ -78,6 +81,7 @@ public class ClickHouseStatementImpl implements ClickHouseStatement {
7881

7982
public ClickHouseStatementImpl(CloseableHttpClient client, ClickHouseConnection connection,
8083
ClickHouseProperties properties, int resultSetType) {
84+
super(null);
8185
this.client = client;
8286
this.connection = connection;
8387
this.properties = properties == null ? new ClickHouseProperties() : properties;
@@ -217,7 +221,8 @@ public int executeUpdate(String sql) throws SQLException {
217221
} finally {
218222
StreamUtils.close(is);
219223
}
220-
return 1;
224+
225+
return currentSummary != null ? (int) currentSummary.getWrittenRows() : 1;
221226
}
222227

223228
@Override
@@ -444,6 +449,11 @@ public boolean isWrapperFor(Class<?> iface) throws SQLException {
444449
return iface.isAssignableFrom(getClass());
445450
}
446451

452+
@Override
453+
public ClickHouseResponseSummary getResponseSummary() {
454+
return currentSummary;
455+
}
456+
447457
static String clickhousifySql(String sql) {
448458
return addFormatIfAbsent(sql, ClickHouseFormat.TabSeparatedWithNamesAndTypes);
449459
}
@@ -623,6 +633,13 @@ private InputStream getInputStream(
623633
entity.writeTo(baos);
624634
is = baos.convertToInputStream();
625635
}
636+
637+
// retrieve response summary
638+
if (isQueryParamSet(ClickHouseQueryParam.SEND_PROGRESS_IN_HTTP_HEADERS, additionalClickHouseDBParams, additionalRequestParams)) {
639+
Header summaryHeader = response.getFirstHeader("X-ClickHouse-Summary");
640+
currentSummary = summaryHeader != null ? Jackson.getObjectMapper().readValue(summaryHeader.getValue(), ClickHouseResponseSummary.class) : null;
641+
}
642+
626643
return is;
627644
} catch (ClickHouseException e) {
628645
throw e;
@@ -700,6 +717,8 @@ private List<NameValuePair> getUrlQueryParams(
700717
params.put(ClickHouseQueryParam.DATABASE, initialDatabase);
701718
}
702719

720+
params.putAll(getAdditionalDBParams());
721+
703722
if (additionalClickHouseDBParams != null && !additionalClickHouseDBParams.isEmpty()) {
704723
params.putAll(additionalClickHouseDBParams);
705724
}
@@ -712,6 +731,12 @@ private List<NameValuePair> getUrlQueryParams(
712731
}
713732
}
714733

734+
for (Map.Entry<String, String> entry : getRequestParams().entrySet()) {
735+
if (!Strings.isNullOrEmpty(entry.getValue())) {
736+
result.add(new BasicNameValuePair(entry.getKey(), entry.getValue()));
737+
}
738+
}
739+
715740
if (additionalRequestParams != null) {
716741
for (Map.Entry<String, String> entry : additionalRequestParams.entrySet()) {
717742
if (!Strings.isNullOrEmpty(entry.getValue())) {
@@ -720,10 +745,31 @@ private List<NameValuePair> getUrlQueryParams(
720745
}
721746
}
722747

723-
724748
return result;
725749
}
726750

751+
private boolean isQueryParamSet(ClickHouseQueryParam param, Map<ClickHouseQueryParam, String> additionalClickHouseDBParams, Map<String, String> additionalRequestParams) {
752+
String value = getQueryParamValue(param, additionalClickHouseDBParams, additionalRequestParams);
753+
754+
return "true".equals(value) || "1".equals(value);
755+
}
756+
757+
private String getQueryParamValue(ClickHouseQueryParam param, Map<ClickHouseQueryParam, String> additionalClickHouseDBParams, Map<String, String> additionalRequestParams) {
758+
if (additionalRequestParams != null && additionalRequestParams.containsKey(param.getKey()) && !Strings.isNullOrEmpty(additionalRequestParams.get(param.getKey())))
759+
return additionalRequestParams.get(param.getKey());
760+
761+
if (getRequestParams().containsKey(param.getKey()) && !Strings.isNullOrEmpty(getRequestParams().get(param.getKey())))
762+
return getRequestParams().get(param.getKey());
763+
764+
if (additionalClickHouseDBParams != null && additionalClickHouseDBParams.containsKey(param) && !Strings.isNullOrEmpty(additionalClickHouseDBParams.get(param)))
765+
return additionalClickHouseDBParams.get(param);
766+
767+
if (getAdditionalDBParams().containsKey(param) && !Strings.isNullOrEmpty(getAdditionalDBParams().get(param)))
768+
return getAdditionalDBParams().get(param);
769+
770+
return properties.asProperties().getProperty(param.getKey());
771+
}
772+
727773
private URI followRedirects(URI uri) throws IOException, URISyntaxException {
728774
if (properties.isCheckForRedirects()) {
729775
int redirects = 0;
@@ -850,6 +896,12 @@ void sendStream(Writer writer, HttpEntity content) throws ClickHouseException {
850896
HttpResponse response = client.execute(httpPost);
851897
entity = response.getEntity();
852898
checkForErrorAndThrow(entity, response);
899+
900+
// retrieve response summary
901+
if (isQueryParamSet(ClickHouseQueryParam.SEND_PROGRESS_IN_HTTP_HEADERS, writer.getAdditionalDBParams(), writer.getRequestParams())) {
902+
Header summaryHeader = response.getFirstHeader("X-ClickHouse-Summary");
903+
currentSummary = summaryHeader != null ? Jackson.getObjectMapper().readValue(summaryHeader.getValue(), ClickHouseResponseSummary.class) : null;
904+
}
853905
} catch (ClickHouseException e) {
854906
throw e;
855907
} catch (Exception e) {
@@ -919,6 +971,6 @@ private Map<ClickHouseQueryParam, String> addQueryIdTo(Map<ClickHouseQueryParam,
919971

920972
@Override
921973
public Writer write() {
922-
return new Writer(this);
974+
return new Writer(this).withDbParams(getAdditionalDBParams()).options(getRequestParams());
923975
}
924976
}

‎src/main/java/ru/yandex/clickhouse/ConfigurableApi.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,11 @@ public T addDbParam(ClickHouseQueryParam param, String value) {
2828
return (T) this;
2929
}
3030

31+
public T removeDbParam(ClickHouseQueryParam param) {
32+
additionalDBParams.remove(param);
33+
return (T) this;
34+
}
35+
3136
public T withDbParams(Map<ClickHouseQueryParam, String> dbParams) {
3237
this.additionalDBParams = new HashMap<ClickHouseQueryParam, String>();
3338
if (null != dbParams) {
@@ -49,4 +54,9 @@ public T option(String key, String value) {
4954
return (T) this;
5055
}
5156

57+
public T removeOption(String key) {
58+
additionalRequestParams.remove(key);
59+
return (T) this;
60+
}
61+
5262
}

‎src/main/java/ru/yandex/clickhouse/response/ClickHouseResponseSummary.java‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
package ru.yandex.clickhouse.response;
2+
3+
import com.fasterxml.jackson.annotation.JsonProperty;
4+
5+
public class ClickHouseResponseSummary {
6+
final private long readRows; // number of read rows for selects (may be more than rows in result set)
7+
final private long writtenRows; // number of written rows for inserts
8+
final private long readBytes;
9+
final private long writtenBytes;
10+
final private long totalRowsToRead;
11+
12+
public ClickHouseResponseSummary(@JsonProperty("read_rows") long readRows, @JsonProperty("written_rows") long writtenRows, @JsonProperty("read_bytes") long readBytes,
13+
@JsonProperty("written_bytes") long writtenBytes, @JsonProperty("total_rows_to_read") long totalRowsToRead) {
14+
this.readRows = readRows;
15+
this.writtenRows = writtenRows;
16+
this.readBytes = readBytes;
17+
this.writtenBytes = writtenBytes;
18+
this.totalRowsToRead = totalRowsToRead;
19+
}
20+
21+
public long getReadRows() {
22+
return readRows;
23+
}
24+
25+
public long getWrittenRows() {
26+
return writtenRows;
27+
}
28+
29+
public long getReadBytes() {
30+
return readBytes;
31+
}
32+
33+
public long getWrittenBytes() {
34+
return writtenBytes;
35+
}
36+
37+
public long getTotalRowsToRead() {
38+
return totalRowsToRead;
39+
}
40+
}

‎src/main/java/ru/yandex/clickhouse/settings/ClickHouseProperties.java‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,8 @@ public class ClickHouseProperties {
9494
private Boolean insertDeduplicate;
9595
private Boolean insertDistributedSync;
9696
private Boolean anyJoinDistinctRightTableKeys;
97-
97+
private Boolean sendProgressInHttpHeaders;
98+
private Boolean waitEndOfQuery;
9899

99100
public ClickHouseProperties() {
100101
this(new Properties());
@@ -162,6 +163,8 @@ public ClickHouseProperties(Properties info) {
162163
this.insertDeduplicate = getSetting(info, ClickHouseQueryParam.INSERT_DEDUPLICATE);
163164
this.insertDistributedSync = getSetting(info, ClickHouseQueryParam.INSERT_DISTRIBUTED_SYNC);
164165
this.anyJoinDistinctRightTableKeys = getSetting(info, ClickHouseQueryParam.ANY_JOIN_DISTINCT_RIGHT_TABLE_KEYS);
166+
this.sendProgressInHttpHeaders = (Boolean)getSetting(info, ClickHouseQueryParam.SEND_PROGRESS_IN_HTTP_HEADERS);
167+
this.waitEndOfQuery = (Boolean)getSetting(info, ClickHouseQueryParam.WAIT_END_OF_QUERY);
165168
}
166169

167170
public Properties asProperties() {
@@ -226,6 +229,8 @@ public Properties asProperties() {
226229
ret.put(ClickHouseQueryParam.INSERT_DEDUPLICATE.getKey(), insertDeduplicate);
227230
ret.put(ClickHouseQueryParam.INSERT_DISTRIBUTED_SYNC.getKey(), insertDistributedSync);
228231
ret.put(ClickHouseQueryParam.ANY_JOIN_DISTINCT_RIGHT_TABLE_KEYS.getKey(), anyJoinDistinctRightTableKeys);
232+
ret.put(ClickHouseQueryParam.SEND_PROGRESS_IN_HTTP_HEADERS.getKey(), sendProgressInHttpHeaders);
233+
ret.put(ClickHouseQueryParam.WAIT_END_OF_QUERY.getKey(), waitEndOfQuery);
229234

230235
return ret.getProperties();
231236
}
@@ -292,6 +297,8 @@ public ClickHouseProperties(ClickHouseProperties properties) {
292297
setInsertDeduplicate(properties.insertDeduplicate);
293298
setInsertDistributedSync(properties.insertDistributedSync);
294299
setAnyJoinDistinctRightTableKeys(properties.anyJoinDistinctRightTableKeys);
300+
setSendProgressInHttpHeaders(properties.sendProgressInHttpHeaders);
301+
setWaitEndOfQuery(properties.waitEndOfQuery);
295302
}
296303

297304
public Map<ClickHouseQueryParam, String> buildQueryParams(boolean ignoreDatabase){
@@ -379,6 +386,9 @@ public Map<ClickHouseQueryParam, String> buildQueryParams(boolean ignoreDatabase
379386
params.put(ClickHouseQueryParam.ENABLE_OPTIMIZE_PREDICATE_EXPRESSION, enableOptimizePredicateExpression ? "1" : "0");
380387
}
381388

389+
addQueryParam(sendProgressInHttpHeaders, ClickHouseQueryParam.SEND_PROGRESS_IN_HTTP_HEADERS, params);
390+
addQueryParam(waitEndOfQuery, ClickHouseQueryParam.WAIT_END_OF_QUERY, params);
391+
382392
return params;
383393
}
384394

@@ -907,6 +917,22 @@ public Boolean getAnyJoinDistinctRightTableKeys() {
907917
return anyJoinDistinctRightTableKeys;
908918
}
909919

920+
public Boolean getSendProgressInHttpHeaders() {
921+
return sendProgressInHttpHeaders;
922+
}
923+
924+
public void setSendProgressInHttpHeaders(Boolean sendProgressInHttpHeaders) {
925+
this.sendProgressInHttpHeaders = sendProgressInHttpHeaders;
926+
}
927+
928+
public Boolean getWaitEndOfQuery() {
929+
return waitEndOfQuery;
930+
}
931+
932+
public void setWaitEndOfQuery(Boolean waitEndOfQuery) {
933+
this.waitEndOfQuery = waitEndOfQuery;
934+
}
935+
910936
private static class PropertiesBuilder {
911937
private final Properties properties;
912938
public PropertiesBuilder() {

‎src/main/java/ru/yandex/clickhouse/settings/ClickHouseQueryParam.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -217,6 +217,8 @@ public enum ClickHouseQueryParam implements DriverPropertyCreator {
217217

218218
SELECT_SEQUENTIAL_CONSISTENCY("select_sequential_consistency", null, Long.class, ""),
219219

220+
SEND_PROGRESS_IN_HTTP_HEADERS("send_progress_in_http_headers", null, Boolean.class, "Allow to populate summary in ClickHouseStatement with read/written rows/bytes"),
221+
220222
SEND_TIMEOUT("send_timeout", null, Integer.class, ""),
221223

222224
SESSION_CHECK("session_check", false, Boolean.class, ""),
@@ -253,6 +255,8 @@ public enum ClickHouseQueryParam implements DriverPropertyCreator {
253255
PREFERRED_BLOCK_SIZE_BYTES("preferred_block_size_bytes", null, Long.class, "Adaptively estimates number of required rows in a block."),
254256

255257
ENABLE_OPTIMIZE_PREDICATE_EXPRESSION("enable_optimize_predicate_expression", null, Boolean.class, "See Clickhouse server description for this parameter. Default value is null so that server setting is taken."),
258+
259+
WAIT_END_OF_QUERY("wait_end_of_query", null, Boolean.class, "Buffer the response server-side before sending to client. Useful when using SEND_PROGRESS_IN_HTTP_HEADERS to get accurate stats."),
256260
;
257261

258262
private final String key;

‎src/test/java/ru/yandex/clickhouse/ClickHouseStatementTest.java‎

Lines changed: 47 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import java.net.URISyntaxException;
66
import java.sql.ResultSet;
77
import java.sql.SQLException;
8+
import java.util.Collections;
89
import java.util.Properties;
910

1011
import org.apache.http.impl.client.HttpClientBuilder;
@@ -13,6 +14,7 @@
1314
import com.google.common.collect.ImmutableMap;
1415

1516
import ru.yandex.clickhouse.settings.ClickHouseProperties;
17+
import ru.yandex.clickhouse.settings.ClickHouseQueryParam;
1618

1719
import static org.testng.Assert.assertEquals;
1820
import static org.testng.Assert.assertFalse;
@@ -109,7 +111,7 @@ public void testMaxMemoryUsage() throws Exception {
109111
}
110112

111113
@Test
112-
public void testAdditionalRequestParams() throws Exception {
114+
public void testAdditionalRequestParams() {
113115
ClickHouseProperties properties = new ClickHouseProperties();
114116
ClickHouseStatementImpl statement = new ClickHouseStatementImpl(
115117
HttpClientBuilder.create().build(),
@@ -118,15 +120,58 @@ public void testAdditionalRequestParams() throws Exception {
118120
ResultSet.TYPE_FORWARD_ONLY
119121
);
120122

123+
statement.option("cache_namespace", "aaaa");
121124
URI uri = statement.buildRequestUri(
122125
null,
123126
null,
124127
null,
125-
ImmutableMap.of("cache_namespace", "aaaa"),
128+
null,
126129
false
127130
);
128131
String query = uri.getQuery();
129132
assertTrue(query.contains("cache_namespace=aaaa"), "cache_namespace param is missing in URL");
133+
134+
uri = statement.buildRequestUri(
135+
null,
136+
null,
137+
null,
138+
ImmutableMap.of("cache_namespace", "bbbb"),
139+
false
140+
);
141+
query = uri.getQuery();
142+
assertTrue(query.contains("cache_namespace=bbbb"), "cache_namespace param is missing in URL");
143+
144+
// check that statement level params are given to Writer
145+
assertEquals(statement.write().getRequestParams().get("cache_namespace"), "aaaa");
146+
}
147+
148+
@Test
149+
public void testAdditionalDBParams() {
150+
ClickHouseProperties properties = new ClickHouseProperties();
151+
properties.setMaxThreads(1);
152+
153+
ClickHouseStatementImpl statement = new ClickHouseStatementImpl(
154+
HttpClientBuilder.create().build(),
155+
null,
156+
properties,
157+
ResultSet.TYPE_FORWARD_ONLY
158+
);
159+
160+
URI uri = statement.buildRequestUri(null, null, null, null, false);
161+
assertTrue(uri.getQuery().contains("max_threads=1"));
162+
163+
// override on statement level
164+
statement.addDbParam(ClickHouseQueryParam.MAX_THREADS, "2");
165+
166+
uri = statement.buildRequestUri(null, null, null, null, false);
167+
assertTrue(uri.getQuery().contains("max_threads=2"));
168+
169+
// override on method level
170+
uri = statement.buildRequestUri(null, null, Collections.singletonMap(ClickHouseQueryParam.MAX_THREADS, "3"), null, false);
171+
assertTrue(uri.getQuery().contains("max_threads=3"));
172+
173+
// check that statement level params are given to Writer
174+
assertEquals(statement.write().getAdditionalDBParams().get(ClickHouseQueryParam.MAX_THREADS), "2");
130175
}
131176

132177
@Test

0 commit comments

Comments
 (0)