Skip to content

Commit a7f505d

Browse files
coderbruiscoderbruis
committed
添加架构方案伪代码
1 parent 7cf7a1b commit a7f505d

38 files changed

Lines changed: 1598 additions & 0 deletions

‎architecture/README.md‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
# Policy snapshot lifecycle
2+
3+
This module implements version-isolated policy snapshots. Each snapshot owns a RocksDB detail
4+
store, a bitmap index, and the incremental-message position at which both were built.
5+
6+
## Integration flow
7+
8+
1. Implement `FullPolicyLoader` to stream the full-data cut into `PolicySnapshot.upsert` and
9+
return the cut's message position.
10+
2. Implement `IncrementalReplayer` to apply upserts/deletes after that position. The builder
11+
repeats replay until it reaches a stable latest position.
12+
3. Supply `DefaultSnapshotValidator` (or a stricter domain validator), a
13+
`FileSystemSnapshotDirectory`, and construct `SnapshotBuilder`.
14+
4. Call `PolicySnapshotService.start(version)` during startup. Keep the application's readiness
15+
probe bound to `service.isReady()`; failures remain unready and retry.
16+
5. Call `refresh(newVersion)` at runtime. A failed candidate is discarded. A valid candidate is
17+
atomically activated while the old snapshot continues serving existing leases.
18+
6. Every search must use `try (SnapshotLease lease = registry.acquire())`. Releasing the last old
19+
lease closes RocksDB and deletes that retired version's directory.
20+
21+
The module intentionally leaves message-broker and full-data-source clients behind interfaces so
22+
the snapshot consistency rules are independent of Kafka, HTTP, database, or framework choices.
23+
24+
## Package layout
25+
26+
- `api`: serializable Dubbo contract and request/response DTOs.
27+
- `rpc`: Dubbo search provider and supplier callback provider.
28+
- `aggregation`: Redis-backed fan-out/fan-in search coordination and local waiters.
29+
- `redis`: atomic Lua state transitions and Pub/Sub early wake-up.
30+
- `demo`: asynchronous downstream supplier simulation.
31+
- `kafka`: JSON policy-change consumer and message DTO.
32+
- `application`: startup and incremental-update use cases.
33+
- root `policy` package: versioned RocksDB/Bitmap snapshot domain and lifecycle.
34+
35+
Kafka messages use a globally monotonic `position` so duplicate/out-of-order delivery is ignored.
36+
If the topic has multiple partitions, the producer must supply this global sequence; otherwise the
37+
position model should be replaced with a per-partition offset map.
38+
39+
## Async supplier search
40+
41+
`asyncSearch` initializes pending suppliers, state and result TTL atomically in Redis, registers a
42+
local waiter, double-checks Redis, then dispatches all supplier tasks. Supplier callbacks append
43+
result chunks and remove a supplier from the pending set only on its final callback. The Lua script
44+
sets `COMPLETED` and publishes `search-finished` when the last supplier finishes. Pub/Sub only wakes
45+
the local waiter early; Redis remains the source of truth and is checked every 200 ms. Timeout is
46+
also a Lua state transition and returns all partial results already recorded.

‎architecture/pom.xml‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<project xmlns="http://maven.apache.org/POM/4.0.0"
3+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
4+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
5+
<modelVersion>4.0.0</modelVersion>
6+
7+
<groupId>com.arch</groupId>
8+
<artifactId>architecture</artifactId>
9+
<version>1.0-SNAPSHOT</version>
10+
11+
<properties>
12+
<maven.compiler.source>8</maven.compiler.source>
13+
<maven.compiler.target>8</maven.compiler.target>
14+
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
15+
<junit.version>5.10.2</junit.version>
16+
<spring-boot.version>2.7.18</spring-boot.version>
17+
<dubbo.version>3.2.15</dubbo.version>
18+
</properties>
19+
20+
<dependencies>
21+
<dependency>
22+
<groupId>org.rocksdb</groupId>
23+
<artifactId>rocksdbjni</artifactId>
24+
<version>8.11.3</version>
25+
</dependency>
26+
<dependency>
27+
<groupId>org.roaringbitmap</groupId>
28+
<artifactId>RoaringBitmap</artifactId>
29+
<version>0.9.47</version>
30+
</dependency>
31+
<dependency>
32+
<groupId>org.junit.jupiter</groupId>
33+
<artifactId>junit-jupiter</artifactId>
34+
<version>${junit.version}</version>
35+
<scope>test</scope>
36+
</dependency>
37+
<dependency>
38+
<groupId>org.springframework.boot</groupId>
39+
<artifactId>spring-boot-starter</artifactId>
40+
<version>${spring-boot.version}</version>
41+
</dependency>
42+
<dependency>
43+
<groupId>org.springframework.boot</groupId>
44+
<artifactId>spring-boot-starter-data-redis</artifactId>
45+
<version>${spring-boot.version}</version>
46+
</dependency>
47+
<dependency>
48+
<groupId>org.springframework.kafka</groupId>
49+
<artifactId>spring-kafka</artifactId>
50+
<version>2.9.13</version>
51+
</dependency>
52+
<dependency>
53+
<groupId>com.fasterxml.jackson.core</groupId>
54+
<artifactId>jackson-databind</artifactId>
55+
<version>2.13.5</version>
56+
</dependency>
57+
<dependency>
58+
<groupId>org.apache.dubbo</groupId>
59+
<artifactId>dubbo-spring-boot-starter</artifactId>
60+
<version>${dubbo.version}</version>
61+
</dependency>
62+
</dependencies>
63+
64+
<build>
65+
<plugins>
66+
<plugin>
67+
<groupId>org.apache.maven.plugins</groupId>
68+
<artifactId>maven-surefire-plugin</artifactId>
69+
<version>3.2.5</version>
70+
</plugin>
71+
<plugin>
72+
<groupId>org.springframework.boot</groupId>
73+
<artifactId>spring-boot-maven-plugin</artifactId>
74+
<version>${spring-boot.version}</version>
75+
</plugin>
76+
</plugins>
77+
</build>
78+
</project>

‎architecture/src/main/java/com/arch/policy/ActiveSnapshotRegistry.java‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
package com.arch.policy;
2+
3+
import java.util.concurrent.atomic.AtomicBoolean;
4+
5+
/** Atomic activation plus draining of queries that still hold the old version. */
6+
public final class ActiveSnapshotRegistry implements AutoCloseable {
7+
private Entry active;
8+
9+
public synchronized boolean isReady() { return active != null; }
10+
11+
public synchronized SnapshotLease acquire() {
12+
if (active == null) throw new IllegalStateException("policy snapshot is not ready");
13+
active.references++;
14+
return new SnapshotLease(active);
15+
}
16+
17+
public synchronized void activate(PolicySnapshot snapshot) {
18+
Entry previous = active;
19+
active = new Entry(snapshot);
20+
if (previous != null) retire(previous);
21+
}
22+
23+
@Override public synchronized void close() {
24+
Entry previous = active;
25+
active = null;
26+
if (previous != null) retire(previous);
27+
}
28+
29+
private void release(Entry entry) {
30+
synchronized (this) {
31+
entry.references--;
32+
closeWhenDrained(entry);
33+
}
34+
}
35+
36+
private void retire(Entry entry) {
37+
entry.retired = true;
38+
closeWhenDrained(entry);
39+
}
40+
41+
private void closeWhenDrained(Entry entry) {
42+
if (entry.retired && entry.references == 0) entry.snapshot.closeAndDelete();
43+
}
44+
45+
private static final class Entry {
46+
private final PolicySnapshot snapshot;
47+
private int references;
48+
private boolean retired;
49+
private Entry(PolicySnapshot snapshot) { this.snapshot = snapshot; }
50+
}
51+
52+
public final class SnapshotLease implements AutoCloseable {
53+
private final Entry entry;
54+
private final AtomicBoolean released = new AtomicBoolean();
55+
private SnapshotLease(Entry entry) { this.entry = entry; }
56+
public PolicySnapshot snapshot() { return entry.snapshot; }
57+
@Override public void close() {
58+
if (released.compareAndSet(false, true)) release(entry);
59+
}
60+
}
61+
}

‎architecture/src/main/java/com/arch/policy/DefaultSnapshotValidator.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
package com.arch.policy;
2+
3+
import static com.arch.policy.SnapshotPorts.SnapshotValidator;
4+
5+
/** Baseline invariants; domain-specific checks can be supplied through SnapshotValidator. */
6+
public final class DefaultSnapshotValidator implements SnapshotValidator {
7+
@Override public void validate(PolicySnapshot candidate, MessagePosition expectedPosition) throws Exception {
8+
if (!candidate.getPosition().equals(expectedPosition)) {
9+
throw new IllegalStateException("message position mismatch");
10+
}
11+
if (candidate.policyCount() != candidate.indexedPolicyCount()) {
12+
throw new IllegalStateException("RocksDB and bitmap policy counts differ");
13+
}
14+
}
15+
}

‎architecture/src/main/java/com/arch/policy/FileSystemSnapshotDirectory.java‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
package com.arch.policy;
2+
3+
import java.io.IOException;
4+
import java.nio.file.Files;
5+
import java.nio.file.Path;
6+
import java.util.Comparator;
7+
import java.util.stream.Stream;
8+
9+
import static com.arch.policy.SnapshotPorts.SnapshotDirectory;
10+
11+
/** Keeps every version in an isolated directory and removes abandoned candidates. */
12+
public final class FileSystemSnapshotDirectory implements SnapshotDirectory {
13+
private final Path root;
14+
15+
public FileSystemSnapshotDirectory(Path root) { this.root = root; }
16+
17+
@Override public Path create(String version) throws IOException {
18+
Path directory = root.resolve(safeVersion(version));
19+
delete(directory);
20+
return Files.createDirectories(directory);
21+
}
22+
23+
@Override public void delete(Path directory) throws IOException {
24+
if (!Files.exists(directory)) return;
25+
try (Stream<Path> paths = Files.walk(directory)) {
26+
paths.sorted(Comparator.reverseOrder()).forEach(path -> {
27+
try { Files.deleteIfExists(path); }
28+
catch (IOException failure) { throw new DeleteFailure(failure); }
29+
});
30+
} catch (DeleteFailure failure) {
31+
throw (IOException) failure.getCause();
32+
}
33+
}
34+
35+
private static String safeVersion(String version) {
36+
if (version == null || !version.matches("[A-Za-z0-9._-]+")) {
37+
throw new IllegalArgumentException("invalid snapshot version: " + version);
38+
}
39+
return version;
40+
}
41+
42+
private static final class DeleteFailure extends RuntimeException {
43+
private DeleteFailure(IOException cause) { super(cause); }
44+
}
45+
}

‎architecture/src/main/java/com/arch/policy/MessagePosition.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
package com.arch.policy;
2+
3+
public final class MessagePosition implements Comparable<MessagePosition> {
4+
public static final MessagePosition BEGINNING = new MessagePosition(0);
5+
private final long value;
6+
7+
public MessagePosition(long value) {
8+
if (value < 0) throw new IllegalArgumentException("position must be non-negative");
9+
this.value = value;
10+
}
11+
12+
public long getValue() { return value; }
13+
14+
@Override public int compareTo(MessagePosition other) { return Long.compare(value, other.value); }
15+
@Override public boolean equals(Object other) {
16+
return other instanceof MessagePosition && value == ((MessagePosition) other).value;
17+
}
18+
@Override public int hashCode() { return Long.valueOf(value).hashCode(); }
19+
@Override public String toString() { return Long.toString(value); }
20+
}

‎architecture/src/main/java/com/arch/policy/PolicyChange.java‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
package com.arch.policy;
2+
3+
public final class PolicyChange {
4+
public enum Type { UPSERT, DELETE }
5+
6+
private final Type type;
7+
private final int policyId;
8+
private final PolicyRecord policy;
9+
private final MessagePosition position;
10+
11+
private PolicyChange(Type type, int policyId, PolicyRecord policy, MessagePosition position) {
12+
this.type = type;
13+
this.policyId = policyId;
14+
this.policy = policy;
15+
this.position = position;
16+
}
17+
18+
public static PolicyChange upsert(PolicyRecord policy, MessagePosition position) {
19+
return new PolicyChange(Type.UPSERT, policy.getId(), policy, position);
20+
}
21+
22+
public static PolicyChange delete(int policyId, MessagePosition position) {
23+
return new PolicyChange(Type.DELETE, policyId, null, position);
24+
}
25+
26+
public Type getType() { return type; }
27+
public int getPolicyId() { return policyId; }
28+
public PolicyRecord getPolicy() { return policy; }
29+
public MessagePosition getPosition() { return position; }
30+
}

‎architecture/src/main/java/com/arch/policy/PolicyRecord.java‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
package com.arch.policy;
2+
3+
import java.util.Arrays;
4+
import java.util.Collections;
5+
import java.util.HashSet;
6+
import java.util.Set;
7+
8+
public final class PolicyRecord {
9+
private final int id;
10+
private final byte[] detail;
11+
private final Set<String> indexTerms;
12+
13+
public PolicyRecord(int id, byte[] detail, Set<String> indexTerms) {
14+
if (id < 0) {
15+
throw new IllegalArgumentException("policy id must be non-negative");
16+
}
17+
this.id = id;
18+
this.detail = Arrays.copyOf(detail, detail.length);
19+
this.indexTerms = Collections.unmodifiableSet(new HashSet<String>(indexTerms));
20+
}
21+
22+
public int getId() { return id; }
23+
24+
public byte[] getDetail() { return Arrays.copyOf(detail, detail.length); }
25+
26+
public Set<String> getIndexTerms() { return indexTerms; }
27+
}

‎architecture/src/main/java/com/arch/policy/PolicySearchApplication.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
package com.arch.policy;
2+
3+
import org.apache.dubbo.config.spring.context.annotation.EnableDubbo;
4+
import org.springframework.boot.SpringApplication;
5+
import org.springframework.boot.autoconfigure.SpringBootApplication;
6+
import org.springframework.kafka.annotation.EnableKafka;
7+
8+
@EnableKafka
9+
@EnableDubbo
10+
@SpringBootApplication
11+
public class PolicySearchApplication {
12+
public static void main(String[] args) {
13+
SpringApplication.run(PolicySearchApplication.class, args);
14+
}
15+
}

0 commit comments

Comments
 (0)