Skip to content

Commit a466f64

Browse files
mrzzyOleksii Moskalenko
mrzzy
authored and
Oleksii Moskalenko
committed
Add Structured Audit Logging (#891)
1 parent bf24595 commit a466f64

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

45 files changed

+1153
-389
lines changed

‎common/pom.xml‎

Lines changed: 49 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -44,11 +44,15 @@
4444
</build>
4545

4646
<dependencies>
47-
<dependency>
47+
<dependency>
4848
<groupId>dev.feast</groupId>
4949
<artifactId>datatypes-java</artifactId>
5050
<version>${project.version}</version>
5151
<scope>compile</scope>
52+
</dependency>
53+
<dependency>
54+
<groupId>com.google.protobuf</groupId>
55+
<artifactId>protobuf-java-util</artifactId>
5256
</dependency>
5357
<!--compileOnly 'org.projectlombok:lombok:1.18.12'-->
5458
<dependency>
@@ -58,13 +62,54 @@
5862
<dependency>
5963
<groupId>javax.validation</groupId>
6064
<artifactId>validation-api</artifactId>
61-
<version>2.0.0.Final</version>
6265
</dependency>
66+
<dependency>
67+
<groupId>com.google.auto.value</groupId>
68+
<artifactId>auto-value-annotations</artifactId>
69+
</dependency>
70+
<dependency>
71+
<groupId>com.google.auto.value</groupId>
72+
<artifactId>auto-value</artifactId>
73+
</dependency>
74+
<dependency>
75+
<groupId>com.google.code.gson</groupId>
76+
<artifactId>gson</artifactId>
77+
</dependency>
78+
<dependency>
79+
<groupId>net.devh</groupId>
80+
<artifactId>grpc-server-spring-boot-starter</artifactId>
81+
<exclusions>
82+
<exclusion>
83+
<groupId>org.springframework.boot</groupId>
84+
<artifactId>spring-boot-starter-logging</artifactId>
85+
</exclusion>
86+
</exclusions>
87+
</dependency>
88+
<dependency>
89+
<groupId>org.springframework.security</groupId>
90+
<artifactId>spring-security-core</artifactId>
91+
</dependency>
92+
<dependency>
93+
<groupId>org.springframework.boot</groupId>
94+
<artifactId>spring-boot-starter-data-jpa</artifactId>
95+
</dependency>
96+
97+
<!-- Logging -->
98+
<dependency>
99+
<groupId>org.slf4j</groupId>
100+
<artifactId>slf4j-api</artifactId>
101+
</dependency>
102+
103+
<!-- Testing -->
63104
<dependency>
64105
<groupId>junit</groupId>
65106
<artifactId>junit</artifactId>
66-
<version>4.12</version>
107+
<scope>test</scope>
108+
</dependency>
109+
<dependency>
110+
<groupId>org.hamcrest</groupId>
111+
<artifactId>hamcrest-library</artifactId>
67112
<scope>test</scope>
68113
</dependency>
69114
</dependencies>
70-
</project>
115+
</project>

‎common/src/main/java/feast/common/interceptors/GrpcMessageInterceptor.java‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
* Copyright 2018-2019 The Feast Authors
4+
*
5+
* Licensed under the Apache License, Version 2.0 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
*
9+
* https://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package feast.common.interceptors;
18+
19+
import com.google.protobuf.Empty;
20+
import com.google.protobuf.Message;
21+
import feast.common.logging.AuditLogger;
22+
import feast.common.logging.entry.MessageAuditLogEntry;
23+
import io.grpc.ForwardingServerCall.SimpleForwardingServerCall;
24+
import io.grpc.ForwardingServerCallListener.SimpleForwardingServerCallListener;
25+
import io.grpc.Metadata;
26+
import io.grpc.ServerCall;
27+
import io.grpc.ServerCall.Listener;
28+
import io.grpc.ServerCallHandler;
29+
import io.grpc.ServerInterceptor;
30+
import io.grpc.Status;
31+
import org.slf4j.event.Level;
32+
import org.springframework.security.core.Authentication;
33+
import org.springframework.security.core.context.SecurityContextHolder;
34+
35+
/**
36+
* GrpcMessageInterceptor intercepts a GRPC calls to log handling of GRPC messages to the Audit Log.
37+
* Intercepts the incoming and outgoing messages logs them to the audit log, together with method
38+
* name and assumed authenticated identity (if authentication is enabled). NOTE:
39+
* GrpcMessageInterceptor assumes that all service calls are unary (ie single request/response).
40+
*/
41+
public class GrpcMessageInterceptor implements ServerInterceptor {
42+
@Override
43+
public <ReqT, RespT> Listener<ReqT> interceptCall(
44+
ServerCall<ReqT, RespT> call, Metadata headers, ServerCallHandler<ReqT, RespT> next) {
45+
MessageAuditLogEntry.Builder entryBuilder = MessageAuditLogEntry.newBuilder();
46+
// default response message to empty proto in log entry.
47+
entryBuilder.setResponse(Empty.newBuilder().build());
48+
49+
// Unpack service & method name from call
50+
// full method name is in format <classpath>.<Service>/<Method>
51+
String fullMethodName = call.getMethodDescriptor().getFullMethodName();
52+
entryBuilder.setService(
53+
fullMethodName.substring(fullMethodName.lastIndexOf(".") + 1, fullMethodName.indexOf("/")));
54+
entryBuilder.setMethod(fullMethodName.substring(fullMethodName.indexOf("/") + 1));
55+
56+
// Attempt Extract current authenticated identity.
57+
Authentication authentication = SecurityContextHolder.getContext().getAuthentication();
58+
String identity = (authentication == null) ? "" : authentication.getName();
59+
entryBuilder.setIdentity(identity);
60+
61+
// Register forwarding call to intercept outgoing response and log to audit log
62+
call =
63+
new SimpleForwardingServerCall<ReqT, RespT>(call) {
64+
@Override
65+
public void sendMessage(RespT message) {
66+
// 2. Track the response & Log entry to audit logger
67+
super.sendMessage(message);
68+
entryBuilder.setResponse((Message) message);
69+
}
70+
71+
@Override
72+
public void close(Status status, Metadata trailers) {
73+
super.close(status, trailers);
74+
// 3. Log the message log entry to the audit log
75+
Level logLevel = (status.isOk()) ? Level.INFO : Level.ERROR;
76+
entryBuilder.setStatusCode(status.getCode());
77+
AuditLogger.logMessage(logLevel, entryBuilder);
78+
}
79+
};
80+
81+
ServerCall.Listener<ReqT> listener = next.startCall(call, headers);
82+
return new SimpleForwardingServerCallListener<ReqT>(listener) {
83+
@Override
84+
// Register listener to intercept incoming request messages and log to audit log
85+
public void onMessage(ReqT message) {
86+
super.onMessage(message);
87+
// 1. Track the request.
88+
entryBuilder.setRequest((Message) message);
89+
}
90+
};
91+
}
92+
}

‎common/src/main/java/feast/common/logging/AuditLogger.java‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
* Copyright 2018-2020 The Feast Authors
4+
*
5+
* Licensed under the Apache License, Version 2.0 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
*
9+
* https://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package feast.common.logging;
18+
19+
import feast.common.logging.config.LoggingProperties;
20+
import feast.common.logging.config.LoggingProperties.AuditLogProperties;
21+
import feast.common.logging.entry.ActionAuditLogEntry;
22+
import feast.common.logging.entry.AuditLogEntry;
23+
import feast.common.logging.entry.AuditLogEntryKind;
24+
import feast.common.logging.entry.LogResource;
25+
import feast.common.logging.entry.LogResource.ResourceType;
26+
import feast.common.logging.entry.MessageAuditLogEntry;
27+
import feast.common.logging.entry.TransitionAuditLogEntry;
28+
import lombok.extern.slf4j.Slf4j;
29+
import org.slf4j.Marker;
30+
import org.slf4j.MarkerFactory;
31+
import org.slf4j.event.Level;
32+
import org.springframework.beans.factory.annotation.Autowired;
33+
import org.springframework.boot.info.BuildProperties;
34+
import org.springframework.stereotype.Component;
35+
36+
@Slf4j
37+
@Component
38+
public class AuditLogger {
39+
private static final Marker AUDIT_MARKER = MarkerFactory.getMarker("AUDIT_MARK");
40+
private static AuditLogProperties properties;
41+
private static BuildProperties buildProperties;
42+
43+
@Autowired
44+
public AuditLogger(LoggingProperties loggingProperties, BuildProperties buildProperties) {
45+
// Spring runs this constructor when creating the AuditLogger bean,
46+
// which allows us to populate the AuditLogger class with dependencies.
47+
// This allows us to use the dependencies in the AuditLogger's static methods
48+
AuditLogger.properties = loggingProperties.getAudit();
49+
AuditLogger.buildProperties = buildProperties;
50+
}
51+
52+
/**
53+
* Log the handling of a Protobuf message by a service call.
54+
*
55+
* @param entryBuilder with all fields set except instance.
56+
*/
57+
public static void logMessage(Level level, MessageAuditLogEntry.Builder entryBuilder) {
58+
log(
59+
level,
60+
entryBuilder
61+
.setComponent(buildProperties.getArtifact())
62+
.setVersion(buildProperties.getVersion())
63+
.build());
64+
}
65+
66+
/**
67+
* Log an action being taken on a specific resource
68+
*
69+
* @param level describing the severity of the log.
70+
* @param action name of the action being taken on specific resource.
71+
* @param resourceType the type of resource being logged.
72+
* @param resourceId resource specific identifier identifing the instance of the resource.
73+
*/
74+
public static void logAction(
75+
Level level, String action, ResourceType resourceType, String resourceId) {
76+
log(
77+
level,
78+
ActionAuditLogEntry.of(
79+
buildProperties.getArtifact(),
80+
buildProperties.getArtifact(),
81+
LogResource.of(resourceType, resourceId),
82+
action));
83+
}
84+
85+
/**
86+
* Log a transition in state/status in a specific resource.
87+
*
88+
* @param level describing the severity of the log.
89+
* @param status name of end status which the resource transition to.
90+
* @param resourceType the type of resource being logged.
91+
* @param resourceId resource specific identifier identifing the instance of the resource.
92+
*/
93+
public static void logTransition(
94+
Level level, String status, ResourceType resourceType, String resourceId) {
95+
log(
96+
level,
97+
TransitionAuditLogEntry.of(
98+
buildProperties.getArtifact(),
99+
buildProperties.getArtifact(),
100+
LogResource.of(resourceType, resourceId),
101+
status));
102+
}
103+
104+
/**
105+
* Log given {@link AuditLogEntry} at the given logging {@link Level} to the Audit log.
106+
*
107+
* @param level describing the severity of the log.
108+
* @param entry the {@link AuditLogEntry} to push to the audit log.
109+
*/
110+
private static void log(Level level, AuditLogEntry entry) {
111+
// Check if audit logging is of this specific log entry enabled.
112+
if (!properties.isEnabled()) {
113+
return;
114+
}
115+
if (entry.getKind().equals(AuditLogEntryKind.MESSAGE)
116+
&& !properties.isMessageLoggingEnabled()) {
117+
return;
118+
}
119+
120+
// Log event to audit log through enabled formats
121+
String entryJSON = entry.toJSON();
122+
switch (level) {
123+
case TRACE:
124+
log.trace(AUDIT_MARKER, entryJSON);
125+
break;
126+
case DEBUG:
127+
log.debug(AUDIT_MARKER, entryJSON);
128+
break;
129+
case INFO:
130+
log.info(AUDIT_MARKER, entryJSON);
131+
break;
132+
case WARN:
133+
log.warn(AUDIT_MARKER, entryJSON);
134+
break;
135+
case ERROR:
136+
log.error(AUDIT_MARKER, entryJSON);
137+
break;
138+
}
139+
}
140+
}

core/src/main/java/feast/core/job/TerminateJobTask.java renamed to common/src/main/java/feast/common/logging/config/LoggingProperties.java

Lines changed: 13 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
/*
22
* SPDX-License-Identifier: Apache-2.0
3-
* Copyright 2018-2020 The Feast Authors
3+
* Copyright 2018-2019 The Feast Authors
44
*
55
* Licensed under the Apache License, Version 2.0 (the "License");
66
* you may not use this file except in compliance with the License.
@@ -14,30 +14,24 @@
1414
* See the License for the specific language governing permissions and
1515
* limitations under the License.
1616
*/
17-
package feast.core.job;
17+
package feast.common.logging.config;
1818

19-
import feast.core.log.Action;
20-
import feast.core.model.Job;
21-
import lombok.Builder;
19+
import javax.validation.constraints.NotNull;
2220
import lombok.Getter;
2321
import lombok.Setter;
2422

25-
/** Task to terminate given {@link Job} by using {@link JobManager} */
2623
@Getter
2724
@Setter
28-
@Builder(setterPrefix = "set")
29-
public class TerminateJobTask implements JobTask {
30-
private Job job;
31-
private JobManager jobManager;
25+
public class LoggingProperties {
26+
@NotNull private AuditLogProperties audit;
3227

33-
@Override
34-
public Job call() {
35-
JobTask.logAudit(
36-
Action.ABORT,
37-
job,
38-
"Aborting job %s for runner %s",
39-
job.getId(),
40-
jobManager.getRunnerType().toString());
41-
return jobManager.abortJob(job);
28+
@Getter
29+
@Setter
30+
public static class AuditLogProperties {
31+
// Whether to enable/disable audit logging entirely.
32+
private boolean enabled;
33+
34+
// Whether to enable/disable message level (ie request/response) audit logging.
35+
private boolean messageLoggingEnabled;
4236
}
4337
}

0 commit comments

Comments
 (0)