Skip to content

Commit f54b557

Browse files
doleyzidoleyzi
authored
[INLONG-12009][Audit] Audit support for managing storage multiple clusters through routing (#12013)
* [INLONG-12009][Audit] Audit support for managing storage multiple clusters through routing * [INLONG-11993][Audit] Optimize the SQL * [INLONG-11993][Audit] Optimize the name of local indicators
1 parent 8fb2b49 commit f54b557

File tree

22 files changed

+857
-30
lines changed

22 files changed

+857
-30
lines changed

‎inlong-audit/audit-common/src/main/java/org/apache/inlong/audit/consts/OpenApiConstants.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,8 @@ public class OpenApiConstants {
3737
public static final String DEFAULT_API_GET_AUDIT_PROXY_PATH = "/audit/query/getAuditProxy";
3838
public static final String KEY_API_RECONCILIATION_PATH = "api.reconciliation.path";
3939
public static final String DEFAULT_API_RECONCILIATION_PATH = "/audit/query/reconciliation";
40+
public static final String KEY_API_GET_AUDIT_ROUTE_PATH = "api.get.audit.route.path";
41+
public static final String DEFAULT_API_GET_AUDIT_ROUTE_PATH = "/audit/query/getAuditRoute";
4042
public static final String KEY_API_THREAD_POOL_SIZE = "api.thread.pool.size";
4143
public static final int DEFAULT_API_THREAD_POOL_SIZE = 10;
4244
public static final String KEY_API_BACKLOG_SIZE = "api.backlog.size";
@@ -69,4 +71,5 @@ public class OpenApiConstants {
6971
public static final int DEFAULT_HTTP_SERVER_BIND_PORT = 10080;
7072
public static final int HTTP_RESPOND_CODE = 200;
7173
public static final String PARAMS_AUDIT_COMPONENT = "component";
74+
public static final String PARAMS_AUDIT_HOST = "host";
7275
}

‎inlong-audit/audit-common/src/main/java/org/apache/inlong/audit/entity/AuditRoute.java‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.inlong.audit.entity;
19+
20+
import lombok.AllArgsConstructor;
21+
import lombok.Data;
22+
import lombok.NoArgsConstructor;
23+
24+
@Data
25+
@NoArgsConstructor
26+
@AllArgsConstructor
27+
public class AuditRoute {
28+
29+
private String address;
30+
private String auditId;
31+
private String inlongGroupIdsInclude;
32+
private String inlongGroupIdsExclude;
33+
}

‎inlong-audit/audit-common/src/main/java/org/apache/inlong/audit/utils/HttpUtils.java‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,4 +104,45 @@ public static String httpGet(String component, String url, String secretId, Stri
104104
}
105105
return null;
106106
}
107+
108+
public static String httpGet(String url, Map<String, String> queryParams, int timeoutMs) {
109+
if (httpClient == null) {
110+
LOGGER.error("httpClient is null");
111+
return null;
112+
}
113+
114+
try {
115+
RequestConfig requestConfig = RequestConfig.custom()
116+
.setConnectTimeout(timeoutMs)
117+
.setConnectionRequestTimeout(timeoutMs)
118+
.setSocketTimeout(timeoutMs)
119+
.build();
120+
121+
URIBuilder uriBuilder = new URIBuilder(url);
122+
123+
if (queryParams != null) {
124+
for (Map.Entry<String, String> entry : queryParams.entrySet()) {
125+
uriBuilder.addParameter(entry.getKey(), entry.getValue());
126+
}
127+
}
128+
129+
String finalUrl = uriBuilder.build().toString();
130+
LOGGER.info("Http URL: {}", finalUrl);
131+
132+
HttpGet request = new HttpGet(finalUrl);
133+
request.setConfig(requestConfig);
134+
135+
try (CloseableHttpResponse response = (CloseableHttpResponse) httpClient.execute(request)) {
136+
String responseStr = EntityUtils.toString(response.getEntity());
137+
LOGGER.info("Http response: {}", responseStr);
138+
if (responseStr != null && !responseStr.isEmpty()
139+
&& response.getStatusLine().getStatusCode() == 200) {
140+
return responseStr;
141+
}
142+
}
143+
} catch (Throwable e) {
144+
LOGGER.error("Http request url = {} has exception!", url, e);
145+
}
146+
return null;
147+
}
107148
}

‎inlong-audit/audit-common/src/main/java/org/apache/inlong/audit/utils/RouteUtils.java‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.inlong.audit.utils;
19+
20+
import org.apache.inlong.audit.entity.AuditRoute;
21+
22+
import java.util.List;
23+
import java.util.regex.Matcher;
24+
import java.util.regex.Pattern;
25+
import java.util.regex.PatternSyntaxException;
26+
27+
public class RouteUtils {
28+
29+
public static boolean isValidRegex(String regex) {
30+
if (regex == null || regex.isEmpty()) {
31+
return false;
32+
}
33+
try {
34+
Pattern.compile(regex);
35+
return true;
36+
} catch (PatternSyntaxException e) {
37+
return false;
38+
}
39+
}
40+
41+
public static boolean matchesAuditRoute(String auditId, String inlongGroupId, List<AuditRoute> auditRouteList) {
42+
if (auditRouteList == null || auditRouteList.isEmpty()) {
43+
return true;
44+
}
45+
try {
46+
for (AuditRoute route : auditRouteList) {
47+
boolean auditIdMatch = Pattern.matches(route.getAuditId(), auditId);
48+
boolean inlongGroupIdIsEmpty = (inlongGroupId == null || inlongGroupId.trim().isEmpty());
49+
50+
boolean includeGroupId =
51+
inlongGroupIdIsEmpty || Pattern.matches(route.getInlongGroupIdsInclude(), inlongGroupId);
52+
boolean excludeGroupId = !inlongGroupIdIsEmpty && route.getInlongGroupIdsExclude() != null
53+
&& !route.getInlongGroupIdsExclude().trim().isEmpty()
54+
&& Pattern.matches(route.getInlongGroupIdsExclude(), inlongGroupId);
55+
56+
if (auditIdMatch && includeGroupId && !excludeGroupId) {
57+
return true;
58+
}
59+
}
60+
} catch (Exception e) {
61+
return false;
62+
}
63+
return false;
64+
}
65+
66+
public static String extractAddress(String jdbcUrl) {
67+
if (jdbcUrl == null || jdbcUrl.trim().isEmpty()) {
68+
return null;
69+
}
70+
Pattern pattern = Pattern.compile("^jdbc:\\w+://([\\d\\.]+):(\\d+)");
71+
Matcher matcher = pattern.matcher(jdbcUrl);
72+
if (matcher.find()) {
73+
return matcher.group(1) + ":" + matcher.group(2);
74+
}
75+
return null;
76+
}
77+
}

‎inlong-audit/audit-common/src/test/java/org/apache/inlong/audit/utils/RouteUtilsTest.java‎

Lines changed: 184 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,184 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.inlong.audit.utils;
19+
20+
import org.apache.inlong.audit.entity.AuditRoute;
21+
22+
import org.junit.Test;
23+
24+
import java.util.ArrayList;
25+
import java.util.List;
26+
27+
import static org.junit.Assert.assertEquals;
28+
import static org.junit.Assert.assertFalse;
29+
import static org.junit.Assert.assertNull;
30+
import static org.junit.Assert.assertTrue;
31+
32+
public class RouteUtilsTest {
33+
34+
@Test
35+
public void extractAddress_ValidJdbcUrl() {
36+
String jdbcUrl = "jdbc:mysql://127.0.0.1:3306/testdb";
37+
String result = RouteUtils.extractAddress(jdbcUrl);
38+
assertEquals("127.0.0.1:3306", result);
39+
}
40+
41+
@Test
42+
public void extractAddress_InvalidJdbcUrl_NoPort() {
43+
String jdbcUrl = "jdbc:mysql://127.0.0.1/testdb";
44+
String result = RouteUtils.extractAddress(jdbcUrl);
45+
assertNull(result);
46+
}
47+
48+
@Test
49+
public void extractAddress_InvalidJdbcUrl_NoProtocol() {
50+
String jdbcUrl = "127.0.0.1:3306/testdb";
51+
String result = RouteUtils.extractAddress(jdbcUrl);
52+
assertNull(result);
53+
}
54+
55+
@Test
56+
public void extractAddress_EmptyJdbcUrl() {
57+
String jdbcUrl = "";
58+
String result = RouteUtils.extractAddress(jdbcUrl);
59+
assertNull(result);
60+
}
61+
62+
@Test
63+
public void extractAddress_NullJdbcUrl() {
64+
String result = RouteUtils.extractAddress(null);
65+
assertNull(result);
66+
}
67+
68+
@Test
69+
public void extractAddress_ValidJdbcUrlWithDifferentProtocol() {
70+
String jdbcUrl = "jdbc:postgresql://192.168.1.100:5432/mydb";
71+
String result = RouteUtils.extractAddress(jdbcUrl);
72+
assertEquals("192.168.1.100:5432", result);
73+
}
74+
75+
@Test
76+
public void matchesAuditRoute_EmptyAuditRouteList() {
77+
List<AuditRoute> auditRouteList = new ArrayList<>();
78+
boolean result = RouteUtils.matchesAuditRoute("auditId1", "groupId1", auditRouteList);
79+
assertTrue(result);
80+
}
81+
82+
@Test
83+
public void matchesAuditRoute_NullAuditRouteList() {
84+
boolean result = RouteUtils.matchesAuditRoute("auditId1", "groupId1", null);
85+
assertTrue(result);
86+
}
87+
88+
@Test
89+
public void matchesAuditRoute_MatchingIncludeAndExclude() {
90+
List<AuditRoute> auditRouteList = new ArrayList<>();
91+
AuditRoute route = new AuditRoute();
92+
route.setAuditId("1");
93+
route.setInlongGroupIdsInclude("groupId.*");
94+
route.setInlongGroupIdsExclude("groupId2");
95+
auditRouteList.add(route);
96+
97+
boolean result = RouteUtils.matchesAuditRoute("1", "groupId1", auditRouteList);
98+
assertTrue(result);
99+
}
100+
101+
@Test
102+
public void matchesAuditRoute_ExcludeRegexMatches() {
103+
List<AuditRoute> auditRouteList = new ArrayList<>();
104+
AuditRoute route = new AuditRoute();
105+
route.setAuditId("1");
106+
route.setInlongGroupIdsInclude("groupId1");
107+
route.setInlongGroupIdsExclude("groupId1");
108+
auditRouteList.add(route);
109+
110+
boolean result = RouteUtils.matchesAuditRoute("1", "groupId1", auditRouteList);
111+
assertFalse(result);
112+
}
113+
114+
@Test
115+
public void matchesAuditRoute_NoMatchingAuditId() {
116+
List<AuditRoute> auditRouteList = new ArrayList<>();
117+
AuditRoute route = new AuditRoute();
118+
route.setAuditId("1");
119+
route.setInlongGroupIdsInclude("groupId1");
120+
auditRouteList.add(route);
121+
122+
boolean result = RouteUtils.matchesAuditRoute("2", "groupId1", auditRouteList);
123+
assertFalse(result);
124+
}
125+
126+
@Test
127+
public void matchesAuditRoute_NoMatchingIncludeRegex() {
128+
List<AuditRoute> auditRouteList = new ArrayList<>();
129+
AuditRoute route = new AuditRoute();
130+
route.setAuditId("1");
131+
route.setInlongGroupIdsInclude("groupId2");
132+
auditRouteList.add(route);
133+
134+
boolean result = RouteUtils.matchesAuditRoute("1", "groupId1", auditRouteList);
135+
assertFalse(result);
136+
}
137+
138+
@Test
139+
public void matchesAuditRoute_IncludeRegex() {
140+
List<AuditRoute> auditRouteList = new ArrayList<>();
141+
AuditRoute route = new AuditRoute();
142+
route.setAuditId("1");
143+
route.setInlongGroupIdsInclude("groupId[0-9]+");
144+
route.setInlongGroupIdsExclude(null);
145+
auditRouteList.add(route);
146+
147+
boolean result = RouteUtils.matchesAuditRoute("1", "groupId123", auditRouteList);
148+
assertTrue(result);
149+
150+
result = RouteUtils.matchesAuditRoute("1", "groupIdx", auditRouteList);
151+
assertFalse(result);
152+
}
153+
154+
@Test
155+
public void matchesAuditRoute_ExcludeRegex() {
156+
List<AuditRoute> auditRouteList = new ArrayList<>();
157+
AuditRoute route = new AuditRoute();
158+
route.setAuditId("1");
159+
route.setInlongGroupIdsInclude("groupId.*");
160+
route.setInlongGroupIdsExclude("groupId[0-9]+");
161+
auditRouteList.add(route);
162+
163+
boolean result = RouteUtils.matchesAuditRoute("1", "groupId123", auditRouteList);
164+
assertFalse(result);
165+
166+
result = RouteUtils.matchesAuditRoute("1", "groupIdABC", auditRouteList);
167+
assertTrue(result);
168+
}
169+
@Test
170+
public void matchesAuditRoute_AllRegex() {
171+
List<AuditRoute> auditRouteList = new ArrayList<>();
172+
AuditRoute route = new AuditRoute();
173+
route.setAuditId("*");
174+
route.setInlongGroupIdsInclude("groupId.*");
175+
route.setInlongGroupIdsExclude("groupId[0-9]+");
176+
auditRouteList.add(route);
177+
178+
boolean result = RouteUtils.matchesAuditRoute("1", "groupId123", auditRouteList);
179+
assertFalse(result);
180+
181+
result = RouteUtils.matchesAuditRoute("1", "groupIdABC", auditRouteList);
182+
assertFalse(result);
183+
}
184+
}

0 commit comments

Comments
 (0)