zyr/FT546MWS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: run shadow v3 without legacy waiting

Browse Source
This commit is contained in:
zgw
2026-09-03 16:24:39 +08:00
parent 9617038e4c
commit 87700246d8
11 changed files with 1189 additions and 148 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
ruoyi-admin/src/main/java/com/ruoyi/algorithm/shadow/InboundShadowCompletion.java
View File

@@ -1,29 +1,51 @@
package com.ruoyi.algorithm.shadow; package com.ruoyi.algorithm.shadow;
import java.util.Objects;
import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException; import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
public final class InboundShadowCompletion { public final class InboundShadowCompletion {
private static final InboundShadowCompletion REJECTED = private static final InboundShadowCompletion REJECTED =
new InboundShadowCompletion(false, 0L); new InboundShadowCompletion(false, false, 0L, null, null);
private final boolean tracked;
private final boolean accepted; private final boolean accepted;
private final long startedNanos;
private final long deadlineNanos; private final long deadlineNanos;
private final CompletableFuture<LegacyResult> legacyResult = new CompletableFuture<>(); private final Runnable onLegacyExit;
private final ShadowRuntimeMetrics metrics;
private final CompletableFuture<LegacyResult> legacyResult;
private final AtomicBoolean finished = new AtomicBoolean();
static InboundShadowCompletion accepted(int timeoutSeconds) { static InboundShadowCompletion tracked(boolean accepted, int timeoutSeconds,
return new InboundShadowCompletion( Runnable onLegacyExit, ShadowRuntimeMetrics metrics) {
true, System.nanoTime() + TimeUnit.SECONDS.toNanos(timeoutSeconds)); long startedNanos = System.nanoTime();
return new InboundShadowCompletion(true, accepted, startedNanos,
startedNanos + TimeUnit.SECONDS.toNanos(timeoutSeconds),
Objects.requireNonNull(onLegacyExit, "onLegacyExit"),
Objects.requireNonNull(metrics, "metrics"));
} }
public static InboundShadowCompletion rejected() { public static InboundShadowCompletion rejected() {
return REJECTED; return REJECTED;
} }
private InboundShadowCompletion(boolean accepted, long deadlineNanos) { private InboundShadowCompletion(boolean tracked, boolean accepted, long deadlineNanos,
Runnable onLegacyExit, ShadowRuntimeMetrics metrics) {
this(tracked, accepted, 0L, deadlineNanos, onLegacyExit, metrics);
}
private InboundShadowCompletion(boolean tracked, boolean accepted, long startedNanos,
long deadlineNanos, Runnable onLegacyExit, ShadowRuntimeMetrics metrics) {
this.tracked = tracked;
this.accepted = accepted; this.accepted = accepted;
this.startedNanos = startedNanos;
this.deadlineNanos = deadlineNanos; this.deadlineNanos = deadlineNanos;
this.onLegacyExit = onLegacyExit;
this.metrics = metrics;
this.legacyResult = accepted ? new CompletableFuture<>() : null;
} }
public boolean accepted() { public boolean accepted() {
@@ -31,19 +53,44 @@ public final class InboundShadowCompletion {
} }
public void complete(String resultCode, String spaceId) { public void complete(String resultCode, String spaceId) {
if (accepted) { finish(new LegacyResult(resultCode, spaceId, null));
legacyResult.complete(new LegacyResult(resultCode, spaceId)); }
}
public void fail(Throwable failure) {
finish(new LegacyResult(null, null, failure));
} }
LegacyResult awaitLegacyResult() throws Exception { LegacyResult awaitLegacyResult() throws Exception {
if (!accepted) {
throw new IllegalStateException("Skipped shadow completion has no Legacy result");
}
long remainingNanos = deadlineNanos - System.nanoTime(); long remainingNanos = deadlineNanos - System.nanoTime();
if (remainingNanos <= 0) { if (remainingNanos <= 0L) {
throw new TimeoutException("Shadow inbound observation deadline expired"); throw new TimeoutException("Shadow inbound observation deadline expired");
} }
return legacyResult.get(remainingNanos, TimeUnit.NANOSECONDS); return legacyResult.get(remainingNanos, TimeUnit.NANOSECONDS);
} }
record LegacyResult(String resultCode, String spaceId) { private void finish(LegacyResult result) {
if (!tracked || !finished.compareAndSet(false, true)) {
return;
}
try {
if (accepted) {
legacyResult.complete(result);
}
metrics.recordLegacyDuration(System.nanoTime() - startedNanos);
} catch (RuntimeException ignored) {
// Shadow bookkeeping must never escape into a Legacy request.
} finally {
try {
onLegacyExit.run();
} catch (RuntimeException ignored) {
// Shadow bookkeeping must never escape into a Legacy request.
}
}
}
record LegacyResult(String resultCode, String spaceId, Throwable failure) {
} }
} }

72
ruoyi-admin/src/main/java/com/ruoyi/algorithm/shadow/OutboundShadowCompletion.java
View File

@@ -1,29 +1,51 @@
package com.ruoyi.algorithm.shadow; package com.ruoyi.algorithm.shadow;
import java.util.Objects;
import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException; import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
public final class OutboundShadowCompletion { public final class OutboundShadowCompletion {
private static final OutboundShadowCompletion REJECTED = private static final OutboundShadowCompletion REJECTED =
new OutboundShadowCompletion(false, 0L); new OutboundShadowCompletion(false, false, 0L, null, null);
private final boolean tracked;
private final boolean accepted; private final boolean accepted;
private final long startedNanos;
private final long deadlineNanos; private final long deadlineNanos;
private final CompletableFuture<String> legacyResult = new CompletableFuture<>(); private final Runnable onLegacyExit;
private final ShadowRuntimeMetrics metrics;
private final CompletableFuture<LegacyResult> legacyResult;
private final AtomicBoolean finished = new AtomicBoolean();
static OutboundShadowCompletion accepted(int timeoutSeconds) { static OutboundShadowCompletion tracked(boolean accepted, int timeoutSeconds,
return new OutboundShadowCompletion( Runnable onLegacyExit, ShadowRuntimeMetrics metrics) {
true, System.nanoTime() + TimeUnit.SECONDS.toNanos(timeoutSeconds)); long startedNanos = System.nanoTime();
return new OutboundShadowCompletion(true, accepted, startedNanos,
startedNanos + TimeUnit.SECONDS.toNanos(timeoutSeconds),
Objects.requireNonNull(onLegacyExit, "onLegacyExit"),
Objects.requireNonNull(metrics, "metrics"));
} }
public static OutboundShadowCompletion rejected() { public static OutboundShadowCompletion rejected() {
return REJECTED; return REJECTED;
} }
private OutboundShadowCompletion(boolean accepted, long deadlineNanos) { private OutboundShadowCompletion(boolean tracked, boolean accepted, long deadlineNanos,
Runnable onLegacyExit, ShadowRuntimeMetrics metrics) {
this(tracked, accepted, 0L, deadlineNanos, onLegacyExit, metrics);
}
private OutboundShadowCompletion(boolean tracked, boolean accepted, long startedNanos,
long deadlineNanos, Runnable onLegacyExit, ShadowRuntimeMetrics metrics) {
this.tracked = tracked;
this.accepted = accepted; this.accepted = accepted;
this.startedNanos = startedNanos;
this.deadlineNanos = deadlineNanos; this.deadlineNanos = deadlineNanos;
this.onLegacyExit = onLegacyExit;
this.metrics = metrics;
this.legacyResult = accepted ? new CompletableFuture<>() : null;
} }
public boolean accepted() { public boolean accepted() {
@@ -31,16 +53,44 @@ public final class OutboundShadowCompletion {
} }
public void complete(String resultCode) { public void complete(String resultCode) {
if (accepted) { finish(new LegacyResult(resultCode, null));
legacyResult.complete(resultCode);
}
} }
String awaitLegacyResult() throws Exception { public void fail(Throwable failure) {
finish(new LegacyResult(null, failure));
}
LegacyResult awaitLegacyResult() throws Exception {
if (!accepted) {
throw new IllegalStateException("Skipped shadow completion has no Legacy result");
}
long remainingNanos = deadlineNanos - System.nanoTime(); long remainingNanos = deadlineNanos - System.nanoTime();
if (remainingNanos <= 0) { if (remainingNanos <= 0L) {
throw new TimeoutException("Shadow outbound observation deadline expired"); throw new TimeoutException("Shadow outbound observation deadline expired");
} }
return legacyResult.get(remainingNanos, TimeUnit.NANOSECONDS); return legacyResult.get(remainingNanos, TimeUnit.NANOSECONDS);
} }
private void finish(LegacyResult result) {
if (!tracked || !finished.compareAndSet(false, true)) {
return;
}
try {
if (accepted) {
legacyResult.complete(result);
}
metrics.recordLegacyDuration(System.nanoTime() - startedNanos);
} catch (RuntimeException ignored) {
// Shadow bookkeeping must never escape into a Legacy request.
} finally {
try {
onLegacyExit.run();
} catch (RuntimeException ignored) {
// Shadow bookkeeping must never escape into a Legacy request.
}
}
}
record LegacyResult(String resultCode, Throwable failure) {
}
} }

10
ruoyi-admin/src/main/java/com/ruoyi/algorithm/shadow/ShadowFailureStage.java Normal file
View File

@@ -0,0 +1,10 @@
package com.ruoyi.algorithm.shadow;
public enum ShadowFailureStage {
STARTUP,
PLAN,
LEGACY_WAIT,
COMPARE,
REFRESH,
METRIC_FLUSH
}

388
ruoyi-admin/src/main/java/com/ruoyi/algorithm/shadow/ShadowObservationGateway.java
View File

@@ -1,102 +1,360 @@
package com.ruoyi.algorithm.shadow; package com.ruoyi.algorithm.shadow;
import com.ruoyi.algorithm.integration.SchedulingFacade;
import com.ruoyi.algorithm.shadow.mapper.ShadowInputMapper;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy; import jakarta.annotation.PreDestroy;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
import java.util.Objects; import java.util.Objects;
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
@Component @Component
public class ShadowObservationGateway { public class ShadowObservationGateway {
private static final Logger LOG = LoggerFactory.getLogger(ShadowObservationGateway.class); private static final Logger LOG = LoggerFactory.getLogger(ShadowObservationGateway.class);
private static final Duration RECOVERY_COOLDOWN = Duration.ofMinutes(1);
private static final Duration[] RETRY_DELAYS = {
Duration.ofSeconds(1), Duration.ofSeconds(2), Duration.ofSeconds(4)
};
private final ShadowObservationProperties properties; private final ShadowObservationProperties properties;
private final ShadowPlanningEngine engine; private final ShadowPlanningEngine engine;
private final ShadowInputMapper inputMapper;
private final ShadowSnapshotCoordinator coordinator;
private final ShadowRuntimeMetrics metrics;
private final Clock clock;
private final Sleeper sleeper;
private final ThreadPoolExecutor executor; private final ThreadPoolExecutor executor;
private final String instanceId;
private final String warehouseId;
private final AtomicBoolean started = new AtomicBoolean();
private final AtomicBoolean recoveryQueued = new AtomicBoolean();
private final AtomicLong lastGeneration = new AtomicLong();
private final AtomicReference<Instant> recoveryNotBefore =
new AtomicReference<>(Instant.MIN);
public ShadowObservationGateway(ShadowObservationProperties properties, ShadowPlanningEngine engine) { @Autowired
public ShadowObservationGateway(ShadowObservationProperties properties,
ShadowPlanningEngine engine, ShadowInputMapper inputMapper,
SchedulingFacade schedulingFacade, ShadowRuntimeMetrics metrics) {
this(properties, engine, inputMapper,
new ShadowSnapshotCoordinator(resolveWarehouseId(properties, schedulingFacade),
Duration.ofSeconds(properties.getSnapshotMaxAgeSeconds()), Clock.systemUTC()),
metrics, Clock.systemUTC(), ShadowObservationGateway::sleep,
newExecutor(), UUID.randomUUID().toString());
}
ShadowObservationGateway(ShadowObservationProperties properties,
ShadowPlanningEngine engine, ShadowInputMapper inputMapper,
ShadowSnapshotCoordinator coordinator, ShadowRuntimeMetrics metrics,
Clock clock, Sleeper sleeper, ThreadPoolExecutor executor, String instanceId) {
this.properties = Objects.requireNonNull(properties, "properties"); this.properties = Objects.requireNonNull(properties, "properties");
this.engine = engine; this.engine = engine;
this.executor = new ThreadPoolExecutor( this.inputMapper = inputMapper;
1, 1, 0L, TimeUnit.MILLISECONDS, new SynchronousQueue<>(), runnable -> { this.coordinator = Objects.requireNonNull(coordinator, "coordinator");
Thread thread = new Thread(runnable, "ft546-shadow-observer"); this.metrics = Objects.requireNonNull(metrics, "metrics");
thread.setDaemon(true); this.clock = Objects.requireNonNull(clock, "clock");
return thread; this.sleeper = Objects.requireNonNull(sleeper, "sleeper");
}, new ThreadPoolExecutor.AbortPolicy()); this.executor = Objects.requireNonNull(executor, "executor");
this.instanceId = Objects.requireNonNull(instanceId, "instanceId");
this.warehouseId = coordinator.configuredWarehouseId();
}
@PostConstruct
public void start() {
if (available() && started.compareAndSet(false, true)) {
requestRecovery(true, ShadowFailureStage.STARTUP);
}
} }
public InboundShadowCompletion observeInbound(String taskNo, String trayNo) { public InboundShadowCompletion observeInbound(String taskNo, String trayNo) {
if (!available()) { if (!available()) {
return InboundShadowCompletion.rejected(); return InboundShadowCompletion.rejected();
} }
InboundShadowCompletion completion = long startedNanos = System.nanoTime();
InboundShadowCompletion.accepted(properties.getTimeoutSeconds()); ShadowSnapshotCoordinator.Admission admission = coordinator.enter(clock.instant());
Runnable onLegacyExit = () -> finishLegacy(admission);
if (!admission.accepted()) {
metrics.recordAdmission(admission.skipReason(), System.nanoTime() - startedNanos);
return InboundShadowCompletion.tracked(
false, properties.getTimeoutSeconds(), onLegacyExit, metrics);
}
InboundShadowCompletion completion = InboundShadowCompletion.tracked(
true, properties.getTimeoutSeconds(), onLegacyExit, metrics);
try { try {
executor.execute(() -> observeInbound(taskNo, trayNo, completion)); executor.execute(() -> observeInbound(taskNo, trayNo, admission, completion));
metrics.recordAdmission(null, System.nanoTime() - startedNanos);
return completion; return completion;
} catch (RejectedExecutionException exception) { } catch (RejectedExecutionException rejected) {
LOG.debug("Shadow inbound observation skipped because worker is busy for task {}", taskNo); coordinator.finishObservation(admission);
return InboundShadowCompletion.rejected(); metrics.recordAdmission(ShadowSkipReason.BUSY, System.nanoTime() - startedNanos);
return InboundShadowCompletion.tracked(
false, properties.getTimeoutSeconds(), onLegacyExit, metrics);
} }
} }
public OutboundShadowCompletion observeProjectOutbound(String stockoutDetailId) { public OutboundShadowCompletion observeProjectOutbound(String stockoutDetailId) {
return observeOutbound(stockoutDetailId, return observeOutbound(stockoutDetailId,
() -> engine.planProjectOutbound(stockoutDetailId)); snapshot -> engine.planProjectOutbound(snapshot, stockoutDetailId));
} }
public OutboundShadowCompletion observeExactStockOutbound(String stockoutDetailId, String stockId) { public OutboundShadowCompletion observeExactStockOutbound(String stockoutDetailId, String stockId) {
return observeOutbound(stockoutDetailId, return observeOutbound(stockoutDetailId,
() -> engine.planExactStockOutbound(stockoutDetailId, stockId)); snapshot -> engine.planExactStockOutbound(snapshot, stockoutDetailId, stockId));
} }
private void observeInbound(String taskNo, String trayNo, InboundShadowCompletion completion) { private OutboundShadowCompletion observeOutbound(String stockoutDetailId, OutboundPlanner planner) {
try {
InboundShadowObservation observation = engine.planInbound(taskNo, trayNo);
InboundShadowCompletion.LegacyResult legacy = completion.awaitLegacyResult();
engine.compareInbound(observation, legacy.resultCode(), legacy.spaceId());
} catch (InterruptedException exception) {
Thread.currentThread().interrupt();
LOG.warn("Shadow inbound observation interrupted for task {}", taskNo);
} catch (Exception exception) {
LOG.warn("Shadow inbound observation failed open for task {}: {}", taskNo, exception.getMessage());
}
}
private OutboundShadowCompletion observeOutbound(String stockoutDetailId,
OutboundPlanner planner) {
if (!available()) { if (!available()) {
return OutboundShadowCompletion.rejected(); return OutboundShadowCompletion.rejected();
} }
OutboundShadowCompletion completion = long startedNanos = System.nanoTime();
OutboundShadowCompletion.accepted(properties.getTimeoutSeconds()); ShadowSnapshotCoordinator.Admission admission = coordinator.enter(clock.instant());
Runnable onLegacyExit = () -> finishLegacy(admission);
if (!admission.accepted()) {
metrics.recordAdmission(admission.skipReason(), System.nanoTime() - startedNanos);
return OutboundShadowCompletion.tracked(
false, properties.getTimeoutSeconds(), onLegacyExit, metrics);
}
OutboundShadowCompletion completion = OutboundShadowCompletion.tracked(
true, properties.getTimeoutSeconds(), onLegacyExit, metrics);
try { try {
executor.execute(() -> { executor.execute(() -> observeOutbound(
try { stockoutDetailId, admission, completion, planner));
OutboundShadowObservation observation = planner.plan(); metrics.recordAdmission(null, System.nanoTime() - startedNanos);
engine.compareOutbound(observation, completion.awaitLegacyResult());
} catch (InterruptedException exception) {
Thread.currentThread().interrupt();
LOG.warn("Shadow outbound observation interrupted for detail {}", stockoutDetailId);
} catch (Exception exception) {
LOG.warn("Shadow outbound observation failed open for detail {}: {}",
stockoutDetailId, exception.getMessage());
}
});
return completion; return completion;
} catch (RejectedExecutionException exception) { } catch (RejectedExecutionException rejected) {
LOG.debug("Shadow outbound observation skipped because worker is busy for detail {}", stockoutDetailId); coordinator.finishObservation(admission);
return OutboundShadowCompletion.rejected(); metrics.recordAdmission(ShadowSkipReason.BUSY, System.nanoTime() - startedNanos);
return OutboundShadowCompletion.tracked(
false, properties.getTimeoutSeconds(), onLegacyExit, metrics);
} }
} }
private void observeInbound(String taskNo, String trayNo,
ShadowSnapshotCoordinator.Admission admission, InboundShadowCompletion completion) {
long startedNanos = System.nanoTime();
ShadowFailureStage stage = ShadowFailureStage.PLAN;
try {
InboundShadowObservation observation =
engine.planInbound(admission.snapshot(), taskNo, trayNo);
stage = ShadowFailureStage.LEGACY_WAIT;
InboundShadowCompletion.LegacyResult legacy = completion.awaitLegacyResult();
if (legacy.failure() != null) {
return;
}
if (!coordinator.isComparable(admission)) {
metrics.increment(ShadowSkipReason.CONCURRENT);
return;
}
stage = ShadowFailureStage.COMPARE;
engine.compareInbound(observation, legacy.resultCode(), legacy.spaceId());
} catch (InterruptedException interrupted) {
Thread.currentThread().interrupt();
recordWorkerFailure(taskNo, admission, stage, interrupted, startedNanos);
} catch (Exception shadowFailure) {
recordWorkerFailure(taskNo, admission, stage, shadowFailure, startedNanos);
} finally {
coordinator.finishObservation(admission);
requestRecovery(false, ShadowFailureStage.REFRESH);
}
}
private void observeOutbound(String stockoutDetailId,
ShadowSnapshotCoordinator.Admission admission, OutboundShadowCompletion completion,
OutboundPlanner planner) {
long startedNanos = System.nanoTime();
ShadowFailureStage stage = ShadowFailureStage.PLAN;
try {
OutboundShadowObservation observation = planner.plan(admission.snapshot());
stage = ShadowFailureStage.LEGACY_WAIT;
OutboundShadowCompletion.LegacyResult legacy = completion.awaitLegacyResult();
if (legacy.failure() != null) {
return;
}
if (!coordinator.isComparable(admission)) {
metrics.increment(ShadowSkipReason.CONCURRENT);
return;
}
stage = ShadowFailureStage.COMPARE;
engine.compareOutbound(observation, legacy.resultCode());
} catch (InterruptedException interrupted) {
Thread.currentThread().interrupt();
recordWorkerFailure(stockoutDetailId, admission, stage, interrupted, startedNanos);
} catch (Exception shadowFailure) {
recordWorkerFailure(stockoutDetailId, admission, stage, shadowFailure, startedNanos);
} finally {
coordinator.finishObservation(admission);
requestRecovery(false, ShadowFailureStage.REFRESH);
}
}
private void finishLegacy(ShadowSnapshotCoordinator.Admission admission) {
coordinator.exit(admission);
requestRecovery(false, ShadowFailureStage.REFRESH);
}
private void requestRecovery(boolean ignoreCooldown, ShadowFailureStage failureStage) {
if (!available()) {
return;
}
Instant now = clock.instant();
if (!ignoreCooldown && now.isBefore(recoveryNotBefore.get())) {
return;
}
if (!recoveryQueued.compareAndSet(false, true)) {
return;
}
try {
executor.execute(() -> runRecoveryTask(failureStage));
} catch (RejectedExecutionException rejected) {
recoveryQueued.set(false);
}
}
private void runRecoveryTask(ShadowFailureStage failureStage) {
try {
refreshWithBoundedRecovery(failureStage);
} finally {
flushMetrics();
recoveryQueued.set(false);
if (available() && coordinator.activeLegacyCount() == 0
&& coordinator.phase() != ShadowSnapshotCoordinator.Phase.READY
&& !clock.instant().isBefore(recoveryNotBefore.get())) {
requestRecovery(false, ShadowFailureStage.REFRESH);
}
}
}
private void refreshWithBoundedRecovery(ShadowFailureStage failureStage) {
for (int attempt = 0; attempt <= RETRY_DELAYS.length; attempt++) {
if (attempt > 0) {
try {
sleeper.sleep(RETRY_DELAYS[attempt - 1]);
} catch (InterruptedException interrupted) {
Thread.currentThread().interrupt();
metrics.recordError(failureStage);
warnFailure("CONTROL", 0L, failureStage, interrupted, 0L);
return;
}
}
Optional<ShadowSnapshotCoordinator.RefreshTicket> optionalTicket;
try {
optionalTicket = tryBeginRefreshAfterQuietWindow();
} catch (InterruptedException interrupted) {
Thread.currentThread().interrupt();
metrics.recordError(failureStage);
warnFailure("CONTROL", 0L, failureStage, interrupted, 0L);
return;
}
if (optionalTicket.isEmpty()) {
return;
}
ShadowSnapshotCoordinator.RefreshTicket ticket = optionalTicket.orElseThrow();
long generation = 0L;
long startedNanos = System.nanoTime();
try {
generation = nextGeneration();
PublishedShadowSnapshot snapshot = engine.refreshSnapshot(generation, instanceId);
if (coordinator.publish(ticket, snapshot, clock.instant())) {
recoveryNotBefore.set(Instant.MIN);
return;
}
metrics.increment(ShadowSkipReason.CONCURRENT);
} catch (Exception shadowFailure) {
coordinator.failRefresh(ticket, failureDescription(failureStage, shadowFailure));
metrics.recordError(failureStage);
warnFailure("CONTROL", generation, failureStage, shadowFailure, startedNanos);
}
}
recoveryNotBefore.set(clock.instant().plus(RECOVERY_COOLDOWN));
}
private Optional<ShadowSnapshotCoordinator.RefreshTicket> tryBeginRefreshAfterQuietWindow()
throws InterruptedException {
if (coordinator.activeLegacyCount() != 0) {
return Optional.empty();
}
Instant quietSince = coordinator.quietSince();
if (quietSince != null) {
Instant quietUntil = quietSince.plusMillis(properties.getQuietIntervalMillis());
Instant now = clock.instant();
if (now.isBefore(quietUntil)) {
sleeper.sleep(Duration.between(now, quietUntil));
}
}
return coordinator.tryBeginRefresh(clock.instant());
}
private long nextGeneration() {
ShadowSnapshotDescriptor descriptor = inputMapper.selectPublishedSnapshot(warehouseId);
long databaseGeneration = descriptor == null ? 0L : descriptor.getSnapshotGeneration();
if (databaseGeneration == Long.MAX_VALUE) {
throw new IllegalStateException("shadow snapshot generation is exhausted");
}
return lastGeneration.updateAndGet(current ->
Math.max(current + 1L, Math.max(1L, databaseGeneration + 1L)));
}
private void flushMetrics() {
ShadowRuntimeMetrics.Batch batch = metrics.drain();
if (batch.totalCount() == 0L) {
return;
}
Instant flushedAt = clock.instant();
try {
for (ShadowRuntimeMetrics.Row row : batch.rows()) {
inputMapper.upsertRuntimeMetric(warehouseId, row.metricName(),
row.bucketUpperMicros(), row.count(), row.totalMicros(),
row.maxMicros(), flushedAt);
}
} catch (Exception shadowFailure) {
metrics.recordError(ShadowFailureStage.METRIC_FLUSH);
warnFailure("CONTROL", 0L, ShadowFailureStage.METRIC_FLUSH,
shadowFailure, 0L);
}
}
private void recordWorkerFailure(String requestId,
ShadowSnapshotCoordinator.Admission admission, ShadowFailureStage stage,
Exception failure, long startedNanos) {
metrics.recordError(stage);
warnFailure(requestId, admission.snapshot().generation(), stage, failure, startedNanos);
}
private void warnFailure(String requestId, long generation,
ShadowFailureStage stage, Throwable failure, long startedNanos) {
long elapsedMicros = startedNanos == 0L ? 0L
: TimeUnit.NANOSECONDS.toMicros(Math.max(0L, System.nanoTime() - startedNanos));
LOG.warn("shadow_failure requestId={} warehouse={} snapshotGeneration={} stage={} "
+ "exceptionClass={} elapsedMicros={}",
requestId, warehouseId, generation, stage,
failure.getClass().getName(), elapsedMicros);
}
private String failureDescription(ShadowFailureStage stage, Throwable failure) {
return stage.name() + ':' + failure.getClass().getName();
}
private boolean available() { private boolean available() {
return properties.isEnabled() && engine != null && !executor.isShutdown(); return properties.isEnabled() && engine != null && inputMapper != null && !executor.isShutdown();
} }
@PreDestroy @PreDestroy
@@ -104,8 +362,38 @@ public class ShadowObservationGateway {
executor.shutdownNow(); executor.shutdownNow();
} }
private static String resolveWarehouseId(ShadowObservationProperties properties,
SchedulingFacade schedulingFacade) {
String warehouseId = properties.getWarehouseId();
if (warehouseId == null || warehouseId.isBlank()) {
warehouseId = Objects.requireNonNull(schedulingFacade, "schedulingFacade").defaultWarehouseId();
}
if (warehouseId == null || warehouseId.isBlank()) {
throw new IllegalStateException("shadow warehouse is not configured");
}
return warehouseId.trim();
}
private static ThreadPoolExecutor newExecutor() {
return new ThreadPoolExecutor(
1, 1, 0L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(1), runnable -> {
Thread thread = new Thread(runnable, "ft546-shadow-observer");
thread.setDaemon(true);
return thread;
}, new ThreadPoolExecutor.AbortPolicy());
}
private static void sleep(Duration duration) throws InterruptedException {
TimeUnit.NANOSECONDS.sleep(duration.toNanos());
}
@FunctionalInterface
interface Sleeper {
void sleep(Duration delay) throws InterruptedException;
}
@FunctionalInterface @FunctionalInterface
private interface OutboundPlanner { private interface OutboundPlanner {
OutboundShadowObservation plan(); OutboundShadowObservation plan(PublishedShadowSnapshot snapshot);
} }
} }

157
ruoyi-admin/src/main/java/com/ruoyi/algorithm/shadow/ShadowRuntimeMetrics.java Normal file
View File

@@ -0,0 +1,157 @@
package com.ruoyi.algorithm.shadow;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReferenceArray;
@Component
public final class ShadowRuntimeMetrics {
private static final long[] ADMISSION_BUCKETS = {
100L, 250L, 500L, 1000L, 2500L, 5000L, 10000L, Long.MAX_VALUE
};
private static final long[] LEGACY_DURATION_BUCKETS = {
100L, 250L, 500L, 1000L, 2500L, 5000L, 10000L,
25000L, 50000L, 100000L, 250000L, 500000L, 1000000L, Long.MAX_VALUE
};
private static final String[] COUNTER_NAMES = counterNames();
private static final int ADMISSION_OFFSET = COUNTER_NAMES.length;
private static final int LEGACY_OFFSET = ADMISSION_OFFSET + ADMISSION_BUCKETS.length;
private final AtomicReferenceArray<MetricValue> values =
new AtomicReferenceArray<>(LEGACY_OFFSET + LEGACY_DURATION_BUCKETS.length);
public ShadowRuntimeMetrics() {
for (int index = 0; index < values.length(); index++) {
values.set(index, MetricValue.ZERO);
}
}
public void recordAdmission(ShadowSkipReason reason, long elapsedNanos) {
record(reason == null ? 0 : 1 + reason.ordinal(), 0L);
record(ADMISSION_OFFSET + bucketIndex(ADMISSION_BUCKETS, toMicros(elapsedNanos)),
toMicros(elapsedNanos));
}
public void recordLegacyDuration(long elapsedNanos) {
long elapsedMicros = toMicros(elapsedNanos);
record(LEGACY_OFFSET + bucketIndex(LEGACY_DURATION_BUCKETS, elapsedMicros), elapsedMicros);
}
public void increment(ShadowSkipReason reason) {
if (reason != null) {
record(1 + reason.ordinal(), 0L);
}
}
public void recordError(ShadowFailureStage stage) {
Objects.requireNonNull(stage, "stage");
record(1 + ShadowSkipReason.values().length + stage.ordinal(), 0L);
}
public Batch drain() {
List<Row> rows = new ArrayList<>(values.length());
for (int index = 0; index < values.length(); index++) {
MetricValue value = values.getAndSet(index, MetricValue.ZERO);
if (value.count() == 0) {
continue;
}
if (index < COUNTER_NAMES.length) {
rows.add(new Row(COUNTER_NAMES[index], 0L,
value.count(), value.totalMicros(), value.maxMicros()));
} else if (index < LEGACY_OFFSET) {
int bucket = index - ADMISSION_OFFSET;
rows.add(new Row("ADMISSION_US", ADMISSION_BUCKETS[bucket],
value.count(), value.totalMicros(), value.maxMicros()));
} else {
int bucket = index - LEGACY_OFFSET;
rows.add(new Row("LEGACY_DURATION_US", LEGACY_DURATION_BUCKETS[bucket],
value.count(), value.totalMicros(), value.maxMicros()));
}
}
return new Batch(rows);
}
private void record(int index, long elapsedMicros) {
while (true) {
MetricValue current = values.get(index);
MetricValue next = current.add(elapsedMicros);
if (values.compareAndSet(index, current, next)) {
return;
}
}
}
private static int bucketIndex(long[] buckets, long value) {
for (int index = 0; index < buckets.length; index++) {
if (value <= buckets[index]) {
return index;
}
}
return buckets.length - 1;
}
private static long toMicros(long elapsedNanos) {
return TimeUnit.NANOSECONDS.toMicros(Math.max(0L, elapsedNanos));
}
private static String[] counterNames() {
List<String> names = new ArrayList<>();
names.add("ADMITTED");
Arrays.stream(ShadowSkipReason.values())
.map(reason -> "SKIP_" + reason.name())
.forEach(names::add);
Arrays.stream(ShadowFailureStage.values())
.map(stage -> "ERROR_" + stage.name())
.forEach(names::add);
return names.toArray(String[]::new);
}
private record MetricValue(long count, long totalMicros, long maxMicros) {
private static final MetricValue ZERO = new MetricValue(0L, 0L, 0L);
private MetricValue add(long elapsedMicros) {
return new MetricValue(count + 1L, totalMicros + elapsedMicros,
Math.max(maxMicros, elapsedMicros));
}
}
public record Row(String metricName, long bucketUpperMicros, long count,
long totalMicros, long maxMicros) {
}
public static final class Batch {
private final List<Row> rows;
private Batch(List<Row> rows) {
this.rows = List.copyOf(rows);
}
public List<Row> rows() {
return rows;
}
public long count(String metricName) {
return rows.stream()
.filter(row -> row.metricName().equals(metricName) && row.bucketUpperMicros() == 0L)
.mapToLong(Row::count)
.sum();
}
public long bucketCount(String metricName, long bucketUpperMicros) {
return rows.stream()
.filter(row -> row.metricName().equals(metricName)
&& row.bucketUpperMicros() == bucketUpperMicros)
.mapToLong(Row::count)
.sum();
}
public long totalCount() {
return rows.stream().mapToLong(Row::count).sum();
}
}
}

32
ruoyi-admin/src/main/java/com/ruoyi/algorithm/shadow/ShadowSnapshotCoordinator.java
View File

@@ -6,6 +6,7 @@ import java.time.Clock;
import java.util.Objects; import java.util.Objects;
import java.util.Optional; import java.util.Optional;
import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.atomic.AtomicReference;
/** /**
@@ -14,11 +15,10 @@ import java.util.concurrent.atomic.AtomicReference;
public final class ShadowSnapshotCoordinator { public final class ShadowSnapshotCoordinator {
private static final Duration DEFAULT_SNAPSHOT_MAX_AGE = Duration.ofSeconds(60); private static final Duration DEFAULT_SNAPSHOT_MAX_AGE = Duration.ofSeconds(60);
private static final long REFRESH_LEASE = Long.MIN_VALUE;
private final String configuredWarehouseId; private final String configuredWarehouseId;
private final Duration snapshotMaxAge; private final Duration snapshotMaxAge;
private final Clock clock; private final Clock clock;
private final AtomicLong refreshLeaseSequence = new AtomicLong(-1L);
private final AtomicReference<State> state = new AtomicReference<>( private final AtomicReference<State> state = new AtomicReference<>(
new State(Phase.INITIALIZING, 0L, 0, null, null, null, null)); new State(Phase.INITIALIZING, 0L, 0, null, null, null, null));
@@ -126,8 +126,9 @@ public final class ShadowSnapshotCoordinator {
if (current.activeLegacyCount() != 0 || current.leasedEpoch() != null) { if (current.activeLegacyCount() != 0 || current.leasedEpoch() != null) {
return Optional.empty(); return Optional.empty();
} }
long refreshLease = refreshLeaseSequence.getAndDecrement();
State next = new State(Phase.BUSY, current.epoch(), current.activeLegacyCount(), current.published(), State next = new State(Phase.BUSY, current.epoch(), current.activeLegacyCount(), current.published(),
REFRESH_LEASE, current.quietSince() == null ? now : current.quietSince(), current.lastFailure()); refreshLease, current.quietSince() == null ? now : current.quietSince(), current.lastFailure());
if (state.compareAndSet(current, next)) { if (state.compareAndSet(current, next)) {
return Optional.of(new RefreshTicket(this, next)); return Optional.of(new RefreshTicket(this, next));
} }
@@ -141,17 +142,28 @@ public final class ShadowSnapshotCoordinator {
State expected = ticket.state; State expected = ticket.state;
State next = new State(Phase.READY, expected.epoch(), expected.activeLegacyCount(), snapshot, State next = new State(Phase.READY, expected.epoch(), expected.activeLegacyCount(), snapshot,
null, expected.quietSince() == null ? now : expected.quietSince(), null); null, expected.quietSince() == null ? now : expected.quietSince(), null);
return state.compareAndSet(expected, next); if (state.compareAndSet(expected, next)) {
return true;
}
failRefresh(ticket, "refresh invalidated by concurrent Legacy activity");
return false;
} }
public void failRefresh(RefreshTicket ticket, String failure) { public void failRefresh(RefreshTicket ticket, String failure) {
if (ticket == null || ticket.owner != this) { if (ticket == null || ticket.owner != this) {
return; return;
} }
State expected = ticket.state; while (true) {
State next = new State(Phase.UNAVAILABLE, expected.epoch(), expected.activeLegacyCount(), State current = state.get();
expected.published(), null, expected.quietSince(), failure); if (!Objects.equals(current.leasedEpoch(), ticket.state.leasedEpoch())) {
state.compareAndSet(expected, next); return;
}
State next = new State(Phase.UNAVAILABLE, current.epoch(), current.activeLegacyCount(),
current.published(), null, current.quietSince(), failure);
if (state.compareAndSet(current, next)) {
return;
}
}
} }
public boolean isComparable(Admission admission) { public boolean isComparable(Admission admission) {
@@ -186,6 +198,10 @@ public final class ShadowSnapshotCoordinator {
return state.get().lastFailure(); return state.get().lastFailure();
} }
String configuredWarehouseId() {
return configuredWarehouseId;
}
public static final class Admission { public static final class Admission {
private final ShadowSnapshotCoordinator owner; private final ShadowSnapshotCoordinator owner;
private final boolean accepted; private final boolean accepted;

8
ruoyi-admin/src/main/java/com/ruoyi/algorithm/shadow/mapper/ShadowInputMapper.java
View File

@@ -37,4 +37,12 @@ public interface ShadowInputMapper {
@Param("sourceSignature") String sourceSignature, @Param("sourceSignature") String sourceSignature,
@Param("resultCode") String resultCode, @Param("resultCode") String resultCode,
@Param("reason") String reason); @Param("reason") String reason);
void upsertRuntimeMetric(@Param("warehouseId") String warehouseId,
@Param("metricName") String metricName,
@Param("bucketUpperMicros") long bucketUpperMicros,
@Param("metricCount") long metricCount,
@Param("metricTotalMicros") long metricTotalMicros,
@Param("metricMaxMicros") long metricMaxMicros,
@Param("flushedAt") Instant flushedAt);
} }

22
ruoyi-admin/src/main/resources/mapper/algorithm/ShadowInputMapper.xml
View File

@@ -112,4 +112,26 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
and state.synchronized_at = #{snapshotSynchronizedAt} and state.synchronized_at = #{snapshotSynchronizedAt}
and convert(varchar(64), state.source_signature, 2) = #{sourceSignature} and convert(varchar(64), state.source_signature, 2) = #{sourceSignature}
</insert> </insert>
<update id="upsertRuntimeMetric">
merge alg.shadow_runtime_metric with (holdlock) as target
using (select #{warehouseId} as warehouse_id,
#{metricName} as metric_name,
#{bucketUpperMicros} as bucket_upper_micros) as source
on target.warehouse_id = source.warehouse_id
and target.metric_name = source.metric_name
and target.bucket_upper_micros = source.bucket_upper_micros
when matched then update set
metric_count = target.metric_count + #{metricCount},
metric_total_micros = target.metric_total_micros + #{metricTotalMicros},
metric_max_micros = case when target.metric_max_micros &gt; #{metricMaxMicros}
then target.metric_max_micros else #{metricMaxMicros} end,
flushed_at = #{flushedAt}
when not matched then insert (
warehouse_id, metric_name, bucket_upper_micros,
metric_count, metric_total_micros, metric_max_micros, flushed_at)
values (
source.warehouse_id, source.metric_name, source.bucket_upper_micros,
#{metricCount}, #{metricTotalMicros}, #{metricMaxMicros}, #{flushedAt});
</update>
</mapper> </mapper>

484
ruoyi-admin/src/test/java/com/ruoyi/algorithm/shadow/ShadowObservationGatewayTest.java
View File

@@ -1,15 +1,30 @@
package com.ruoyi.algorithm.shadow; package com.ruoyi.algorithm.shadow;
import com.ruoyi.algorithm.shadow.mapper.ShadowInputMapper;
import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
import java.time.ZoneId;
import java.time.ZoneOffset;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.CountDownLatch; import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatCode;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.after;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never; import static org.mockito.Mockito.never;
import static org.mockito.Mockito.timeout; import static org.mockito.Mockito.timeout;
@@ -17,100 +32,433 @@ import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
class ShadowObservationGatewayTest { class ShadowObservationGatewayTest {
private ShadowObservationGateway gateway; private static final Instant NOW = Instant.parse("2026-09-03T00:00:00Z");
private static final PublishedShadowSnapshot SNAPSHOT =
new PublishedShadowSnapshot(42L, "01", NOW, "ABC123");
private final List<ShadowObservationGateway> gateways = new ArrayList<>();
private final List<ThreadPoolExecutor> executors = new ArrayList<>();
@AfterEach @AfterEach
void shutDownGateway() { void shutDownGateways() {
if (gateway != null) { gateways.forEach(ShadowObservationGateway::shutdown);
gateway.shutdown(); executors.forEach(ThreadPoolExecutor::shutdownNow);
}
} }
@Test @Test
void disabledShadowNeverInvokesThePlanningEngine() { void completionTokensExitLegacyActivityExactlyOnceOnSuccessFailureAndDuplicates() throws Exception {
ShadowObservationProperties properties = new ShadowObservationProperties(); ShadowRuntimeMetrics metrics = new ShadowRuntimeMetrics();
AtomicInteger inboundExits = new AtomicInteger();
InboundShadowCompletion inbound = InboundShadowCompletion.tracked(
true, 3, inboundExits::incrementAndGet, metrics);
inbound.complete("10", "SPACE-1");
inbound.complete("10", "SPACE-2");
inbound.fail(new IllegalStateException("late failure"));
assertThat(inboundExits).hasValue(1);
assertThat(inbound.awaitLegacyResult().spaceId()).isEqualTo("SPACE-1");
AtomicInteger outboundExits = new AtomicInteger();
OutboundShadowCompletion outbound = OutboundShadowCompletion.tracked(
true, 3, outboundExits::incrementAndGet, metrics);
IllegalStateException failure = new IllegalStateException("legacy failed");
outbound.fail(failure);
outbound.fail(new IllegalStateException("again"));
outbound.complete("10");
assertThat(outboundExits).hasValue(1);
assertThat(outbound.awaitLegacyResult().failure()).isSameAs(failure);
}
@Test
void enabledSkippedTokenStillExitsActivityWhileDisabledTokenIsAReusableNoOp() {
ShadowRuntimeMetrics metrics = new ShadowRuntimeMetrics();
AtomicInteger exits = new AtomicInteger();
InboundShadowCompletion skipped = InboundShadowCompletion.tracked(
false, 3, exits::incrementAndGet, metrics);
skipped.complete("10", "SPACE-1");
skipped.fail(new IllegalStateException("ignored"));
assertThat(skipped.accepted()).isFalse();
assertThat(exits).hasValue(1);
assertThat(InboundShadowCompletion.rejected())
.isSameAs(InboundShadowCompletion.rejected());
assertThat(OutboundShadowCompletion.rejected())
.isSameAs(OutboundShadowCompletion.rejected());
OutboundShadowCompletion failingCallback = OutboundShadowCompletion.tracked(
false, 3, () -> { throw new IllegalStateException("callback failed"); }, metrics);
assertThatCode(() -> failingCallback.fail(new IllegalStateException("legacy failed")))
.doesNotThrowAnyException();
}
@Test
void startRefreshesGenerationOneAndPublishesReadyWithoutBlockingTheCaller() throws Exception {
ShadowPlanningEngine engine = mock(ShadowPlanningEngine.class); ShadowPlanningEngine engine = mock(ShadowPlanningEngine.class);
gateway = new ShadowObservationGateway(properties, engine); ShadowInputMapper mapper = mock(ShadowInputMapper.class);
CountDownLatch refreshCalled = new CountDownLatch(1);
when(engine.refreshSnapshot(1L, "INSTANCE-1")).thenAnswer(invocation -> {
refreshCalled.countDown();
return new PublishedShadowSnapshot(1L, "01", NOW, "ABC001");
});
ShadowSnapshotCoordinator coordinator = coordinator();
ShadowObservationGateway gateway = gateway(engine, mapper, coordinator,
new ShadowRuntimeMetrics(), Clock.fixed(NOW, ZoneOffset.UTC), delay -> { });
gateway.start();
assertThat(refreshCalled.await(1, TimeUnit.SECONDS)).isTrue();
verify(engine, timeout(1000)).refreshSnapshot(1L, "INSTANCE-1");
assertThat(waitForPhase(coordinator, ShadowSnapshotCoordinator.Phase.READY)).isTrue();
}
@Test
void validInboundPlansFromLeaseWaitsOnlyOnWorkerAndComparesLegacyResult() throws Exception {
ShadowPlanningEngine engine = mock(ShadowPlanningEngine.class);
ShadowInputMapper mapper = mock(ShadowInputMapper.class);
ShadowSnapshotCoordinator coordinator = coordinatorWithSnapshot();
InboundShadowObservation observation = inboundObservation();
when(engine.planInbound(SNAPSHOT, "TASK-1", "TRAY-1")).thenReturn(observation);
CountDownLatch compared = new CountDownLatch(1);
doAnswer(invocation -> {
compared.countDown();
return null;
}).when(engine).compareInbound(observation, "10", "LEGACY-SPACE");
stubSuccessfulRecovery(engine, mapper, 43L);
ShadowObservationGateway gateway = gateway(engine, mapper, coordinator,
new ShadowRuntimeMetrics(), Clock.fixed(NOW.plusSeconds(2), ZoneOffset.UTC), delay -> { });
InboundShadowCompletion completion = gateway.observeInbound("TASK-1", "TRAY-1"); InboundShadowCompletion completion = gateway.observeInbound("TASK-1", "TRAY-1");
completion.complete("10", "SPACE-1");
verify(engine, never()).planInbound("TASK-1", "TRAY-1"); assertThat(completion.accepted()).isTrue();
verify(mapper, never()).upsertRuntimeMetric(anyString(), anyString(), anyLong(), anyLong(),
anyLong(), anyLong(), any(Instant.class));
completion.complete("10", "LEGACY-SPACE");
assertThat(compared.await(1, TimeUnit.SECONDS)).isTrue();
verify(engine).planInbound(SNAPSHOT, "TASK-1", "TRAY-1");
verify(engine).compareInbound(observation, "10", "LEGACY-SPACE");
} }
@Test @Test
void planningFailureIsContainedOnTheBackgroundWorker() { void concurrentLegacyCallReturnsImmediatelyAndInvalidatesTheFirstComparison() throws Exception {
ShadowObservationProperties properties = enabledProperties();
ShadowPlanningEngine engine = mock(ShadowPlanningEngine.class);
when(engine.planInbound("TASK-2", "TRAY-2"))
.thenThrow(new IllegalStateException("algorithm objects are unavailable"));
gateway = new ShadowObservationGateway(properties, engine);
InboundShadowCompletion completion = gateway.observeInbound("TASK-2", "TRAY-2");
completion.complete("10", "SPACE-2");
verify(engine, timeout(1000)).planInbound("TASK-2", "TRAY-2");
verify(engine, never()).compareInbound(
org.mockito.ArgumentMatchers.any(),
org.mockito.ArgumentMatchers.any(),
org.mockito.ArgumentMatchers.any());
}
@Test
void missingEngineFailsOpenForOutboundObservation() {
gateway = new ShadowObservationGateway(enabledProperties(), null);
OutboundShadowCompletion completion =
gateway.observeExactStockOutbound("DETAIL-1", "STOCK-1");
completion.complete("10");
assertThat(completion.accepted()).isFalse();
}
@Test
void planningSubmissionNeverWaitsAndConcurrentObservationIsDropped() throws Exception {
ShadowPlanningEngine engine = mock(ShadowPlanningEngine.class); ShadowPlanningEngine engine = mock(ShadowPlanningEngine.class);
ShadowInputMapper mapper = mock(ShadowInputMapper.class);
ShadowSnapshotCoordinator coordinator = coordinatorWithSnapshot();
CountDownLatch planningStarted = new CountDownLatch(1); CountDownLatch planningStarted = new CountDownLatch(1);
CountDownLatch releasePlanning = new CountDownLatch(1); CountDownLatch releasePlanning = new CountDownLatch(1);
InboundShadowObservation observation = new InboundShadowObservation( when(engine.planInbound(SNAPSHOT, "TASK-2", "TRAY-2")).thenAnswer(invocation -> {
true, "W01", "TASK-3", "READY", "SPACE-SHADOW", "proposal");
when(engine.planInbound("TASK-3", "TRAY-3")).thenAnswer(invocation -> {
planningStarted.countDown(); planningStarted.countDown();
if (!releasePlanning.await(5, TimeUnit.SECONDS)) { releasePlanning.await(1, TimeUnit.SECONDS);
throw new IllegalStateException("test planning was not released"); return inboundObservation();
}
return observation;
}); });
gateway = new ShadowObservationGateway(enabledProperties(), engine); stubSuccessfulRecovery(engine, mapper, 43L);
ExecutorService legacyCaller = Executors.newSingleThreadExecutor(); ShadowObservationGateway gateway = gateway(engine, mapper, coordinator,
new ShadowRuntimeMetrics(), Clock.fixed(NOW.plusSeconds(2), ZoneOffset.UTC), delay -> { });
try { InboundShadowCompletion first = gateway.observeInbound("TASK-2", "TRAY-2");
Future<InboundShadowCompletion> submitted = assertThat(planningStarted.await(1, TimeUnit.SECONDS)).isTrue();
legacyCaller.submit(() -> gateway.observeInbound("TASK-3", "TRAY-3")); long started = System.nanoTime();
InboundShadowCompletion first = submitted.get(1, TimeUnit.SECONDS); OutboundShadowCompletion concurrent = gateway.observeProjectOutbound("DETAIL-2");
assertThat(first.accepted()).isTrue(); long elapsedMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - started);
assertThat(planningStarted.await(1, TimeUnit.SECONDS)).isTrue();
OutboundShadowCompletion concurrent = gateway.observeProjectOutbound("DETAIL-3"); assertThat(concurrent.accepted()).isFalse();
assertThat(elapsedMillis).isLessThan(1000L);
concurrent.complete("10");
first.complete("10", "LEGACY-SPACE");
releasePlanning.countDown();
assertThat(concurrent.accepted()).isFalse(); verify(engine, after(500).never()).compareInbound(any(), anyString(), anyString());
concurrent.complete("10"); verify(engine, never()).planProjectOutbound(any(), anyString());
verify(engine, never()).planProjectOutbound("DETAIL-3"); assertThat(coordinator.activeLegacyCount()).isZero();
}
first.complete("10", "SPACE-LEGACY"); @Test
releasePlanning.countDown(); void rejectedWorkerSubmissionReturnsTrackedSkipAndReleasesAdmission() throws Exception {
ThreadPoolExecutor fullExecutor = executor();
CountDownLatch holdWorker = new CountDownLatch(1);
fullExecutor.execute(() -> await(holdWorker));
fullExecutor.execute(() -> { });
ShadowPlanningEngine engine = mock(ShadowPlanningEngine.class);
ShadowSnapshotCoordinator coordinator = coordinatorWithSnapshot();
ShadowObservationGateway gateway = gateway(engine, mock(ShadowInputMapper.class), coordinator,
new ShadowRuntimeMetrics(), Clock.fixed(NOW, ZoneOffset.UTC), delay -> { }, fullExecutor);
verify(engine, timeout(2000)).compareInbound(observation, "10", "SPACE-LEGACY"); InboundShadowCompletion completion = gateway.observeInbound("TASK-3", "TRAY-3");
} finally { completion.complete("10", "SPACE-3");
releasePlanning.countDown();
legacyCaller.shutdownNow(); assertThat(completion.accepted()).isFalse();
assertThat(coordinator.activeLegacyCount()).isZero();
verify(engine, never()).planInbound(any(), anyString(), anyString());
holdWorker.countDown();
}
@Test
void planningFailureIsWorkerOnlyAndRecordedByFixedStage() throws Exception {
ShadowPlanningEngine engine = mock(ShadowPlanningEngine.class);
ShadowInputMapper mapper = mock(ShadowInputMapper.class);
ShadowSnapshotCoordinator coordinator = coordinatorWithSnapshot();
when(engine.planInbound(SNAPSHOT, "TASK-4", "TRAY-4"))
.thenThrow(new IllegalStateException("sensitive business payload"));
CountDownLatch planMetric = metricLatch(mapper, "ERROR_PLAN");
stubSuccessfulRecovery(engine, mapper, 43L);
ShadowObservationGateway gateway = gateway(engine, mapper, coordinator,
new ShadowRuntimeMetrics(), Clock.fixed(NOW.plusSeconds(2), ZoneOffset.UTC), delay -> { });
InboundShadowCompletion completion = gateway.observeInbound("TASK-4", "TRAY-4");
completion.complete("10", "SPACE-4");
assertThat(planMetric.await(1, TimeUnit.SECONDS)).isTrue();
verify(engine, never()).compareInbound(any(), anyString(), anyString());
}
@Test
void comparisonFailureIsWorkerOnlyAndRecordedByFixedStage() throws Exception {
ShadowPlanningEngine engine = mock(ShadowPlanningEngine.class);
ShadowInputMapper mapper = mock(ShadowInputMapper.class);
ShadowSnapshotCoordinator coordinator = coordinatorWithSnapshot();
InboundShadowObservation observation = inboundObservation();
when(engine.planInbound(SNAPSHOT, "TASK-8", "TRAY-8")).thenReturn(observation);
org.mockito.Mockito.doThrow(new IllegalStateException("comparison failed"))
.when(engine).compareInbound(observation, "10", "SPACE-8");
CountDownLatch compareMetric = metricLatch(mapper, "ERROR_COMPARE");
stubSuccessfulRecovery(engine, mapper, 43L);
ShadowObservationGateway gateway = gateway(engine, mapper, coordinator,
new ShadowRuntimeMetrics(), Clock.fixed(NOW.plusSeconds(2), ZoneOffset.UTC), delay -> { });
InboundShadowCompletion completion = gateway.observeInbound("TASK-8", "TRAY-8");
completion.complete("10", "SPACE-8");
assertThat(compareMetric.await(1, TimeUnit.SECONDS)).isTrue();
verify(engine).compareInbound(observation, "10", "SPACE-8");
}
@Test
void legacyWaitTimeoutIsRecordedWithoutClosingCallerOwnedActivity() throws Exception {
ShadowPlanningEngine engine = mock(ShadowPlanningEngine.class);
ShadowInputMapper mapper = mock(ShadowInputMapper.class);
ShadowSnapshotCoordinator coordinator = coordinatorWithSnapshot();
when(engine.planInbound(SNAPSHOT, "TASK-5", "TRAY-5")).thenReturn(inboundObservation());
CountDownLatch waitMetric = metricLatch(mapper, "ERROR_LEGACY_WAIT");
ShadowObservationProperties properties = enabledProperties();
properties.setTimeoutSeconds(1);
ShadowObservationGateway gateway = gateway(properties, engine, mapper, coordinator,
new ShadowRuntimeMetrics(), Clock.fixed(NOW, ZoneOffset.UTC), delay -> { }, executor());
InboundShadowCompletion completion = gateway.observeInbound("TASK-5", "TRAY-5");
assertThat(waitMetric.await(2, TimeUnit.SECONDS)).isTrue();
assertThat(coordinator.activeLegacyCount()).isEqualTo(1);
completion.fail(new IllegalStateException("legacy failed later"));
assertThat(coordinator.activeLegacyCount()).isZero();
}
@Test
void projectAndExactStockOutboundUseTheLeasedGeneration() throws Exception {
assertOutboundUsesLease(false);
assertOutboundUsesLease(true);
}
@Test
void refreshRetriesWithOneTwoFourSecondBackoffAndCoalescesRecoveryForOneMinute() throws Exception {
ShadowPlanningEngine engine = mock(ShadowPlanningEngine.class);
ShadowInputMapper mapper = mock(ShadowInputMapper.class);
AtomicInteger attempts = new AtomicInteger();
CountDownLatch failedCycle = new CountDownLatch(1);
when(engine.refreshSnapshot(anyLong(), eq("INSTANCE-1"))).thenAnswer(invocation -> {
int attempt = attempts.incrementAndGet();
if (attempt <= 4) {
if (attempt == 4) {
failedCycle.countDown();
}
throw new IllegalStateException("refresh failed");
}
return new PublishedShadowSnapshot(invocation.getArgument(0), "01",
NOW.plusSeconds(61), "ABC999");
});
MutableClock clock = new MutableClock(NOW);
List<Duration> delays = new ArrayList<>();
ShadowObservationGateway gateway = gateway(engine, mapper, coordinator(),
new ShadowRuntimeMetrics(), clock, delay -> {
delays.add(delay);
clock.advance(delay);
});
gateway.start();
assertThat(failedCycle.await(1, TimeUnit.SECONDS)).isTrue();
verify(engine, timeout(1000).times(4)).refreshSnapshot(anyLong(), eq("INSTANCE-1"));
assertThat(delays).containsExactly(Duration.ofSeconds(1), Duration.ofSeconds(2), Duration.ofSeconds(4));
InboundShadowCompletion beforeMinute = gateway.observeInbound("TASK-6", "TRAY-6");
beforeMinute.complete("10", "SPACE-6");
verify(engine, after(200).times(4)).refreshSnapshot(anyLong(), eq("INSTANCE-1"));
clock.advance(Duration.ofSeconds(61));
CountDownLatch recovered = metricLatch(mapper, "SKIP_NOT_READY");
InboundShadowCompletion afterMinute = gateway.observeInbound("TASK-7", "TRAY-7");
afterMinute.complete("10", "SPACE-7");
verify(engine, timeout(1000).times(5)).refreshSnapshot(anyLong(), eq("INSTANCE-1"));
assertThat(recovered.await(1, TimeUnit.SECONDS)).isTrue();
}
private void assertOutboundUsesLease(boolean exactStock) throws Exception {
ShadowPlanningEngine engine = mock(ShadowPlanningEngine.class);
ShadowInputMapper mapper = mock(ShadowInputMapper.class);
ShadowSnapshotCoordinator coordinator = coordinatorWithSnapshot();
OutboundShadowObservation observation = new OutboundShadowObservation(
true, 42L, "01", NOW, "ABC123", "DETAIL-7", "READY",
"STOCK-7", "SPACE-7", "diagnostic");
if (exactStock) {
when(engine.planExactStockOutbound(SNAPSHOT, "DETAIL-7", "STOCK-7")).thenReturn(observation);
} else {
when(engine.planProjectOutbound(SNAPSHOT, "DETAIL-7")).thenReturn(observation);
} }
CountDownLatch compared = new CountDownLatch(1);
doAnswer(invocation -> {
compared.countDown();
return null;
}).when(engine).compareOutbound(observation, "10");
stubSuccessfulRecovery(engine, mapper, 43L);
ShadowObservationGateway gateway = gateway(engine, mapper, coordinator,
new ShadowRuntimeMetrics(), Clock.fixed(NOW.plusSeconds(2), ZoneOffset.UTC), delay -> { });
OutboundShadowCompletion completion = exactStock
? gateway.observeExactStockOutbound("DETAIL-7", "STOCK-7")
: gateway.observeProjectOutbound("DETAIL-7");
completion.complete("10");
assertThat(completion.accepted()).isTrue();
assertThat(compared.await(1, TimeUnit.SECONDS)).isTrue();
if (exactStock) {
verify(engine).planExactStockOutbound(SNAPSHOT, "DETAIL-7", "STOCK-7");
} else {
verify(engine).planProjectOutbound(SNAPSHOT, "DETAIL-7");
}
}
private ShadowObservationGateway gateway(ShadowPlanningEngine engine, ShadowInputMapper mapper,
ShadowSnapshotCoordinator coordinator, ShadowRuntimeMetrics metrics, Clock clock,
ShadowObservationGateway.Sleeper sleeper) {
return gateway(enabledProperties(), engine, mapper, coordinator, metrics, clock, sleeper, executor());
}
private ShadowObservationGateway gateway(ShadowPlanningEngine engine, ShadowInputMapper mapper,
ShadowSnapshotCoordinator coordinator, ShadowRuntimeMetrics metrics, Clock clock,
ShadowObservationGateway.Sleeper sleeper, ThreadPoolExecutor executor) {
return gateway(enabledProperties(), engine, mapper, coordinator, metrics, clock, sleeper, executor);
}
private ShadowObservationGateway gateway(ShadowObservationProperties properties,
ShadowPlanningEngine engine, ShadowInputMapper mapper, ShadowSnapshotCoordinator coordinator,
ShadowRuntimeMetrics metrics, Clock clock, ShadowObservationGateway.Sleeper sleeper,
ThreadPoolExecutor executor) {
ShadowObservationGateway gateway = new ShadowObservationGateway(properties, engine, mapper,
coordinator, metrics, clock, sleeper, executor, "INSTANCE-1");
gateways.add(gateway);
return gateway;
}
private ThreadPoolExecutor executor() {
ThreadPoolExecutor executor = new ThreadPoolExecutor(
1, 1, 0L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(1),
new ThreadPoolExecutor.AbortPolicy());
executors.add(executor);
return executor;
}
private ShadowSnapshotCoordinator coordinator() {
return new ShadowSnapshotCoordinator("01", Duration.ofSeconds(60), Clock.fixed(NOW, ZoneOffset.UTC));
}
private ShadowSnapshotCoordinator coordinatorWithSnapshot() {
ShadowSnapshotCoordinator coordinator = coordinator();
ShadowSnapshotCoordinator.RefreshTicket ticket = coordinator.tryBeginRefresh(NOW).orElseThrow();
assertThat(coordinator.publish(ticket, SNAPSHOT, NOW)).isTrue();
return coordinator;
}
private void stubSuccessfulRecovery(ShadowPlanningEngine engine, ShadowInputMapper mapper, long generation) {
ShadowSnapshotDescriptor descriptor = new ShadowSnapshotDescriptor();
descriptor.setSnapshotGeneration(generation - 1);
when(mapper.selectPublishedSnapshot("01")).thenReturn(descriptor);
when(engine.refreshSnapshot(generation, "INSTANCE-1")).thenReturn(
new PublishedShadowSnapshot(generation, "01", NOW.plusSeconds(2), "ABC124"));
}
private CountDownLatch metricLatch(ShadowInputMapper mapper, String metricName) {
CountDownLatch latch = new CountDownLatch(1);
doAnswer(invocation -> {
if (metricName.equals(invocation.getArgument(1))) {
latch.countDown();
}
return null;
}).when(mapper).upsertRuntimeMetric(anyString(), anyString(), anyLong(), anyLong(),
anyLong(), anyLong(), any(Instant.class));
return latch;
}
private InboundShadowObservation inboundObservation() {
return new InboundShadowObservation(true, 42L, "01", NOW, "ABC123",
"TASK-1", "READY", "SHADOW-SPACE", "diagnostic");
} }
private ShadowObservationProperties enabledProperties() { private ShadowObservationProperties enabledProperties() {
ShadowObservationProperties properties = new ShadowObservationProperties(); ShadowObservationProperties properties = new ShadowObservationProperties();
properties.setEnabled(true); properties.setEnabled(true);
properties.setWarehouseId("01");
properties.setTimeoutSeconds(3); properties.setTimeoutSeconds(3);
properties.setQuietIntervalMillis(1000);
return properties; return properties;
} }
private boolean waitForPhase(ShadowSnapshotCoordinator coordinator,
ShadowSnapshotCoordinator.Phase expected) throws InterruptedException {
long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(1);
while (System.nanoTime() < deadline) {
if (coordinator.phase() == expected) {
return true;
}
Thread.sleep(5L);
}
return coordinator.phase() == expected;
}
private static void await(CountDownLatch latch) {
try {
latch.await(5, TimeUnit.SECONDS);
} catch (InterruptedException interrupted) {
Thread.currentThread().interrupt();
}
}
private static final class MutableClock extends Clock {
private Instant instant;
private MutableClock(Instant instant) {
this.instant = instant;
}
private void advance(Duration duration) {
instant = instant.plus(duration);
}
@Override
public ZoneId getZone() {
return ZoneOffset.UTC;
}
@Override
public Clock withZone(ZoneId zone) {
return this;
}
@Override
public Instant instant() {
return instant;
}
}
} }

81
ruoyi-admin/src/test/java/com/ruoyi/algorithm/shadow/ShadowRuntimeMetricsTest.java Normal file
View File

@@ -0,0 +1,81 @@
package com.ruoyi.algorithm.shadow;
import org.junit.jupiter.api.Test;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
class ShadowRuntimeMetricsTest {
@Test
void recordsFixedCountersAndHistogramBucketsThenDrainsAtomically() {
ShadowRuntimeMetrics metrics = new ShadowRuntimeMetrics();
metrics.recordAdmission(ShadowSkipReason.BUSY, 750_000L);
metrics.recordAdmission(null, 1_500_000L);
metrics.recordLegacyDuration(1_500_000L);
metrics.recordError(ShadowFailureStage.PLAN);
ShadowRuntimeMetrics.Batch batch = metrics.drain();
assertThat(batch.count("SKIP_BUSY")).isEqualTo(1);
assertThat(batch.count("ADMITTED")).isEqualTo(1);
assertThat(batch.count("ERROR_PLAN")).isEqualTo(1);
assertThat(batch.bucketCount("ADMISSION_US", 1000L)).isEqualTo(1);
assertThat(batch.bucketCount("ADMISSION_US", 2500L)).isEqualTo(1);
assertThat(batch.bucketCount("LEGACY_DURATION_US", 2500L)).isEqualTo(1);
assertThat(metrics.drain().totalCount()).isZero();
assertThatThrownBy(() -> batch.rows().clear())
.isInstanceOf(UnsupportedOperationException.class);
}
@Test
void recordsOverflowAndDoesNotLoseConcurrentCounterUpdates() throws Exception {
ShadowRuntimeMetrics metrics = new ShadowRuntimeMetrics();
int workers = 4;
int perWorker = 1000;
ExecutorService executor = Executors.newFixedThreadPool(workers);
try {
for (int worker = 0; worker < workers; worker++) {
executor.submit(() -> {
for (int index = 0; index < perWorker; index++) {
metrics.increment(ShadowSkipReason.CONCURRENT);
}
});
}
} finally {
executor.shutdown();
assertThat(executor.awaitTermination(2, TimeUnit.SECONDS)).isTrue();
}
metrics.recordAdmission(null, TimeUnit.SECONDS.toNanos(2));
metrics.recordLegacyDuration(TimeUnit.SECONDS.toNanos(2));
ShadowRuntimeMetrics.Batch batch = metrics.drain();
assertThat(batch.count("SKIP_CONCURRENT")).isEqualTo((long) workers * perWorker);
assertThat(batch.bucketCount("ADMISSION_US", Long.MAX_VALUE)).isEqualTo(1);
assertThat(batch.bucketCount("LEGACY_DURATION_US", Long.MAX_VALUE)).isEqualTo(1);
}
@Test
void metricPersistenceUsesOneFixedShadowOwnedSqlStatement() throws Exception {
String xml;
try (InputStream stream = Thread.currentThread().getContextClassLoader()
.getResourceAsStream("mapper/algorithm/ShadowInputMapper.xml")) {
assertThat(stream).isNotNull();
xml = new String(stream.readAllBytes(), StandardCharsets.UTF_8).toLowerCase();
}
int start = xml.indexOf("<update id=\"upsertruntimemetric\">");
int end = xml.indexOf("</update>", start);
String statement = xml.substring(start, end);
assertThat(statement).contains("merge alg.shadow_runtime_metric")
.doesNotContain("dbo.", "${");
}
}

14
ruoyi-admin/src/test/java/com/ruoyi/algorithm/shadow/ShadowSnapshotCoordinatorTest.java
View File

@@ -61,6 +61,7 @@ class ShadowSnapshotCoordinatorTest {
assertThat(coordinator.publish(refresh, snapshot(2, "01", NOW), NOW)).isFalse(); assertThat(coordinator.publish(refresh, snapshot(2, "01", NOW), NOW)).isFalse();
assertThat(coordinator.phase()).isNotEqualTo(ShadowSnapshotCoordinator.Phase.READY); assertThat(coordinator.phase()).isNotEqualTo(ShadowSnapshotCoordinator.Phase.READY);
coordinator.exit(concurrent); coordinator.exit(concurrent);
assertThat(coordinator.tryBeginRefresh(NOW)).isPresent();
} }
@Test @Test
@@ -130,6 +131,19 @@ class ShadowSnapshotCoordinatorTest {
assertThat(coordinator.phase()).isEqualTo(ShadowSnapshotCoordinator.Phase.READY); assertThat(coordinator.phase()).isEqualTo(ShadowSnapshotCoordinator.Phase.READY);
} }
@Test
void staleRefreshTicketCannotCancelANewerRefreshLease() {
ShadowSnapshotCoordinator coordinator = coordinatorAt("01");
ShadowSnapshotCoordinator.RefreshTicket first = coordinator.tryBeginRefresh(NOW).orElseThrow();
coordinator.failRefresh(first, "first failed");
ShadowSnapshotCoordinator.RefreshTicket second = coordinator.tryBeginRefresh(NOW).orElseThrow();
coordinator.failRefresh(first, "late duplicate failure");
assertThat(coordinator.phase()).isEqualTo(ShadowSnapshotCoordinator.Phase.BUSY);
assertThat(coordinator.publish(second, snapshot(9, "01", NOW), NOW)).isTrue();
}
private ShadowSnapshotCoordinator coordinatorAt(String warehouseId) { private ShadowSnapshotCoordinator coordinatorAt(String warehouseId) {
return new ShadowSnapshotCoordinator(warehouseId, Duration.ofSeconds(60)); return new ShadowSnapshotCoordinator(warehouseId, Duration.ofSeconds(60));
} }