Skip to content

Commit bbb2feb

Browse files
JanCizmarJanCizmar
authored
fix: Add metrics for big meta storing (tolgee#2476)
1 parent b2533ca commit bbb2feb

File tree

4 files changed

+39
-8
lines changed

4 files changed

+39
-8
lines changed

‎backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/BigMetaControllerTest.kt‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@ import io.tolgee.util.logger
1414
import org.junit.jupiter.api.BeforeEach
1515
import org.junit.jupiter.api.Test
1616
import org.springframework.beans.factory.annotation.Autowired
17-
import kotlin.time.ExperimentalTime
1817
import kotlin.time.measureTime
1918

2019
class BigMetaControllerTest : ProjectAuthControllerTest("/v2/projects/"), Logging {
@@ -57,11 +56,11 @@ class BigMetaControllerTest : ProjectAuthControllerTest("/v2/projects/"), Loggin
5756
bigMetaService.findExistingKeysDistancesDtosByIds(listOf(testData.yepKey.id)).assert.hasSize(1)
5857
}
5958

60-
@OptIn(ExperimentalTime::class)
6159
@Test
6260
@ProjectJWTAuthTestMethod
6361
fun `it performs well`() {
6462
val keys = testData.addLotOfData()
63+
testData.addLotOfReferences(keys)
6564
saveTestDataAndPrepare()
6665

6766
logger.infoMeasureTime("it performs well time 1") {
@@ -87,6 +86,16 @@ class BigMetaControllerTest : ProjectAuthControllerTest("/v2/projects/"), Loggin
8786
bigMetaService.findExistingKeysDistancesDtosByIds(keys.map { it.id }).assert.hasSize(104790)
8887
}
8988

89+
@Test
90+
@ProjectJWTAuthTestMethod
91+
fun `it performs well (large)`() {
92+
val keys = testData.addLotOfData()
93+
testData.addLotOfReferences(keys)
94+
saveTestDataAndPrepare()
95+
96+
storeLogOfBigMeta(keys, 0, 200)
97+
}
98+
9099
private fun storeLogOfBigMeta(
91100
keys: List<Key>,
92101
drop: Int,
@@ -96,7 +105,7 @@ class BigMetaControllerTest : ProjectAuthControllerTest("/v2/projects/"), Loggin
96105
"big-meta",
97106
mapOf(
98107
"relatedKeysInOrder" to
99-
keys.drop(drop).take(take).map {
108+
keys.drop(drop).take(take).reversed().map {
100109
mapOf(
101110
"namespace" to it.namespace,
102111
"keyName" to it.name,
@@ -110,6 +119,7 @@ class BigMetaControllerTest : ProjectAuthControllerTest("/v2/projects/"), Loggin
110119
@ProjectJWTAuthTestMethod
111120
fun `it returns correct data`() {
112121
val keys = testData.addLotOfData()
122+
testData.addLotOfReferences(keys)
113123
saveTestDataAndPrepare()
114124

115125
performProjectAuthGet("keys/${keys.first().id}/big-meta").andIsOk.andAssertThatJson {

‎backend/data/src/main/kotlin/io/tolgee/Metrics.kt‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package io.tolgee
33
import io.micrometer.core.instrument.Counter
44
import io.micrometer.core.instrument.Gauge
55
import io.micrometer.core.instrument.MeterRegistry
6+
import io.micrometer.core.instrument.Timer
67
import org.springframework.stereotype.Component
78
import java.util.concurrent.ConcurrentLinkedQueue
89

@@ -27,4 +28,16 @@ class Metrics(
2728
.description("Total number of failures when trying to store data about batch job execution (execution failed)")
2829
.register(meterRegistry)
2930
}
31+
32+
val bigMetaStoringTimer: Timer by lazy {
33+
Timer.builder("tolgee.big_meta.storing.timer")
34+
.description("Time spent storing big meta data")
35+
.register(meterRegistry)
36+
}
37+
38+
val bigMetaNewDistancesComputeTimer: Timer by lazy {
39+
Timer.builder("tolgee.big_meta.new_distances.compute.timer")
40+
.description("Time spent computing new distances for big meta data")
41+
.register(meterRegistry)
42+
}
3043
}

‎backend/data/src/main/kotlin/io/tolgee/development/testDataBuilder/data/BigMetaTestData.kt‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,10 @@ class BigMetaTestData {
3636
(0..5000).map {
3737
projectBuilder.addKey(null, "key$it").self
3838
}
39+
return keys
40+
}
3941

42+
fun addLotOfReferences(keys: List<Key>) {
4043
keys.forEachIndexed forEach1@{ idx1, key1 ->
4144
keys.forEachIndexed forEach2@{ idx2, key2 ->
4245
if (idx1 >= idx2 || abs(idx1 - idx2) > (BigMetaService.MAX_ORDER_DISTANCE + 1)) return@forEach2
@@ -46,7 +49,5 @@ class BigMetaTestData {
4649
}
4750
}
4851
}
49-
50-
return keys
5152
}
5253
}

‎backend/data/src/main/kotlin/io/tolgee/service/bigMeta/BigMetaService.kt‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package io.tolgee.service.bigMeta
22

3+
import io.tolgee.Metrics
34
import io.tolgee.component.CurrentDateProvider
45
import io.tolgee.dtos.BigMetaDto
56
import io.tolgee.dtos.RelatedKeyDto
@@ -36,6 +37,7 @@ class BigMetaService(
3637
private val transactionManager: PlatformTransactionManager,
3738
private val jdbcTemplate: JdbcTemplate,
3839
private val currentDateProvider: CurrentDateProvider,
40+
private val metrics: Metrics,
3941
) : Logging {
4042
companion object {
4143
const val MAX_DISTANCE_SCORE = 10000L
@@ -64,9 +66,14 @@ class BigMetaService(
6466
return
6567
}
6668

67-
val util = KeysDistanceUtil(relatedKeysInOrder, project, this)
69+
val util =
70+
metrics.bigMetaNewDistancesComputeTimer.recordCallable {
71+
KeysDistanceUtil(relatedKeysInOrder, project, this)
72+
}!!
6873

69-
insertNewDistances(util.newDistances)
74+
metrics.bigMetaStoringTimer.recordCallable {
75+
insertNewDistances(util.newDistances)
76+
}
7077
}
7178

7279
private fun insertNewDistances(toInsert: List<KeysDistanceDto>) {
@@ -78,7 +85,7 @@ class BigMetaService(
7885
on conflict (key1id, key2id) do update set score = excluded.score, hits = excluded.hits, updated_at = ?
7986
""",
8087
toInsert,
81-
100,
88+
10000,
8289
) { ps, dto ->
8390
ps.setLong(1, dto.key1Id)
8491
ps.setLong(2, dto.key2Id)

0 commit comments

Comments
 (0)