feat: run shadow v3 without legacy waiting
This commit is contained in:
@@ -1,15 +1,30 @@
|
||||
package com.ruoyi.algorithm.shadow;
|
||||
|
||||
import com.ruoyi.algorithm.shadow.mapper.ShadowInputMapper;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.ZoneId;
|
||||
import java.time.ZoneOffset;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.ArrayBlockingQueue;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.ThreadPoolExecutor;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatCode;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.after;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.timeout;
|
||||
@@ -17,100 +32,433 @@ import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
class ShadowObservationGatewayTest {
|
||||
private ShadowObservationGateway gateway;
|
||||
private static final Instant NOW = Instant.parse("2026-09-03T00:00:00Z");
|
||||
private static final PublishedShadowSnapshot SNAPSHOT =
|
||||
new PublishedShadowSnapshot(42L, "01", NOW, "ABC123");
|
||||
|
||||
private final List<ShadowObservationGateway> gateways = new ArrayList<>();
|
||||
private final List<ThreadPoolExecutor> executors = new ArrayList<>();
|
||||
|
||||
@AfterEach
|
||||
void shutDownGateway() {
|
||||
if (gateway != null) {
|
||||
gateway.shutdown();
|
||||
}
|
||||
void shutDownGateways() {
|
||||
gateways.forEach(ShadowObservationGateway::shutdown);
|
||||
executors.forEach(ThreadPoolExecutor::shutdownNow);
|
||||
}
|
||||
|
||||
@Test
|
||||
void disabledShadowNeverInvokesThePlanningEngine() {
|
||||
ShadowObservationProperties properties = new ShadowObservationProperties();
|
||||
void completionTokensExitLegacyActivityExactlyOnceOnSuccessFailureAndDuplicates() throws Exception {
|
||||
ShadowRuntimeMetrics metrics = new ShadowRuntimeMetrics();
|
||||
AtomicInteger inboundExits = new AtomicInteger();
|
||||
InboundShadowCompletion inbound = InboundShadowCompletion.tracked(
|
||||
true, 3, inboundExits::incrementAndGet, metrics);
|
||||
|
||||
inbound.complete("10", "SPACE-1");
|
||||
inbound.complete("10", "SPACE-2");
|
||||
inbound.fail(new IllegalStateException("late failure"));
|
||||
|
||||
assertThat(inboundExits).hasValue(1);
|
||||
assertThat(inbound.awaitLegacyResult().spaceId()).isEqualTo("SPACE-1");
|
||||
|
||||
AtomicInteger outboundExits = new AtomicInteger();
|
||||
OutboundShadowCompletion outbound = OutboundShadowCompletion.tracked(
|
||||
true, 3, outboundExits::incrementAndGet, metrics);
|
||||
IllegalStateException failure = new IllegalStateException("legacy failed");
|
||||
|
||||
outbound.fail(failure);
|
||||
outbound.fail(new IllegalStateException("again"));
|
||||
outbound.complete("10");
|
||||
|
||||
assertThat(outboundExits).hasValue(1);
|
||||
assertThat(outbound.awaitLegacyResult().failure()).isSameAs(failure);
|
||||
}
|
||||
|
||||
@Test
|
||||
void enabledSkippedTokenStillExitsActivityWhileDisabledTokenIsAReusableNoOp() {
|
||||
ShadowRuntimeMetrics metrics = new ShadowRuntimeMetrics();
|
||||
AtomicInteger exits = new AtomicInteger();
|
||||
InboundShadowCompletion skipped = InboundShadowCompletion.tracked(
|
||||
false, 3, exits::incrementAndGet, metrics);
|
||||
|
||||
skipped.complete("10", "SPACE-1");
|
||||
skipped.fail(new IllegalStateException("ignored"));
|
||||
|
||||
assertThat(skipped.accepted()).isFalse();
|
||||
assertThat(exits).hasValue(1);
|
||||
assertThat(InboundShadowCompletion.rejected())
|
||||
.isSameAs(InboundShadowCompletion.rejected());
|
||||
assertThat(OutboundShadowCompletion.rejected())
|
||||
.isSameAs(OutboundShadowCompletion.rejected());
|
||||
|
||||
OutboundShadowCompletion failingCallback = OutboundShadowCompletion.tracked(
|
||||
false, 3, () -> { throw new IllegalStateException("callback failed"); }, metrics);
|
||||
assertThatCode(() -> failingCallback.fail(new IllegalStateException("legacy failed")))
|
||||
.doesNotThrowAnyException();
|
||||
}
|
||||
|
||||
@Test
|
||||
void startRefreshesGenerationOneAndPublishesReadyWithoutBlockingTheCaller() throws Exception {
|
||||
ShadowPlanningEngine engine = mock(ShadowPlanningEngine.class);
|
||||
gateway = new ShadowObservationGateway(properties, engine);
|
||||
ShadowInputMapper mapper = mock(ShadowInputMapper.class);
|
||||
CountDownLatch refreshCalled = new CountDownLatch(1);
|
||||
when(engine.refreshSnapshot(1L, "INSTANCE-1")).thenAnswer(invocation -> {
|
||||
refreshCalled.countDown();
|
||||
return new PublishedShadowSnapshot(1L, "01", NOW, "ABC001");
|
||||
});
|
||||
ShadowSnapshotCoordinator coordinator = coordinator();
|
||||
ShadowObservationGateway gateway = gateway(engine, mapper, coordinator,
|
||||
new ShadowRuntimeMetrics(), Clock.fixed(NOW, ZoneOffset.UTC), delay -> { });
|
||||
|
||||
gateway.start();
|
||||
|
||||
assertThat(refreshCalled.await(1, TimeUnit.SECONDS)).isTrue();
|
||||
verify(engine, timeout(1000)).refreshSnapshot(1L, "INSTANCE-1");
|
||||
assertThat(waitForPhase(coordinator, ShadowSnapshotCoordinator.Phase.READY)).isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
void validInboundPlansFromLeaseWaitsOnlyOnWorkerAndComparesLegacyResult() throws Exception {
|
||||
ShadowPlanningEngine engine = mock(ShadowPlanningEngine.class);
|
||||
ShadowInputMapper mapper = mock(ShadowInputMapper.class);
|
||||
ShadowSnapshotCoordinator coordinator = coordinatorWithSnapshot();
|
||||
InboundShadowObservation observation = inboundObservation();
|
||||
when(engine.planInbound(SNAPSHOT, "TASK-1", "TRAY-1")).thenReturn(observation);
|
||||
CountDownLatch compared = new CountDownLatch(1);
|
||||
doAnswer(invocation -> {
|
||||
compared.countDown();
|
||||
return null;
|
||||
}).when(engine).compareInbound(observation, "10", "LEGACY-SPACE");
|
||||
stubSuccessfulRecovery(engine, mapper, 43L);
|
||||
ShadowObservationGateway gateway = gateway(engine, mapper, coordinator,
|
||||
new ShadowRuntimeMetrics(), Clock.fixed(NOW.plusSeconds(2), ZoneOffset.UTC), delay -> { });
|
||||
|
||||
InboundShadowCompletion completion = gateway.observeInbound("TASK-1", "TRAY-1");
|
||||
completion.complete("10", "SPACE-1");
|
||||
|
||||
verify(engine, never()).planInbound("TASK-1", "TRAY-1");
|
||||
assertThat(completion.accepted()).isTrue();
|
||||
verify(mapper, never()).upsertRuntimeMetric(anyString(), anyString(), anyLong(), anyLong(),
|
||||
anyLong(), anyLong(), any(Instant.class));
|
||||
completion.complete("10", "LEGACY-SPACE");
|
||||
assertThat(compared.await(1, TimeUnit.SECONDS)).isTrue();
|
||||
verify(engine).planInbound(SNAPSHOT, "TASK-1", "TRAY-1");
|
||||
verify(engine).compareInbound(observation, "10", "LEGACY-SPACE");
|
||||
}
|
||||
|
||||
@Test
|
||||
void planningFailureIsContainedOnTheBackgroundWorker() {
|
||||
ShadowObservationProperties properties = enabledProperties();
|
||||
ShadowPlanningEngine engine = mock(ShadowPlanningEngine.class);
|
||||
when(engine.planInbound("TASK-2", "TRAY-2"))
|
||||
.thenThrow(new IllegalStateException("algorithm objects are unavailable"));
|
||||
gateway = new ShadowObservationGateway(properties, engine);
|
||||
|
||||
InboundShadowCompletion completion = gateway.observeInbound("TASK-2", "TRAY-2");
|
||||
completion.complete("10", "SPACE-2");
|
||||
|
||||
verify(engine, timeout(1000)).planInbound("TASK-2", "TRAY-2");
|
||||
verify(engine, never()).compareInbound(
|
||||
org.mockito.ArgumentMatchers.any(),
|
||||
org.mockito.ArgumentMatchers.any(),
|
||||
org.mockito.ArgumentMatchers.any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void missingEngineFailsOpenForOutboundObservation() {
|
||||
gateway = new ShadowObservationGateway(enabledProperties(), null);
|
||||
|
||||
OutboundShadowCompletion completion =
|
||||
gateway.observeExactStockOutbound("DETAIL-1", "STOCK-1");
|
||||
completion.complete("10");
|
||||
|
||||
assertThat(completion.accepted()).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void planningSubmissionNeverWaitsAndConcurrentObservationIsDropped() throws Exception {
|
||||
void concurrentLegacyCallReturnsImmediatelyAndInvalidatesTheFirstComparison() throws Exception {
|
||||
ShadowPlanningEngine engine = mock(ShadowPlanningEngine.class);
|
||||
ShadowInputMapper mapper = mock(ShadowInputMapper.class);
|
||||
ShadowSnapshotCoordinator coordinator = coordinatorWithSnapshot();
|
||||
CountDownLatch planningStarted = new CountDownLatch(1);
|
||||
CountDownLatch releasePlanning = new CountDownLatch(1);
|
||||
InboundShadowObservation observation = new InboundShadowObservation(
|
||||
true, "W01", "TASK-3", "READY", "SPACE-SHADOW", "proposal");
|
||||
when(engine.planInbound("TASK-3", "TRAY-3")).thenAnswer(invocation -> {
|
||||
when(engine.planInbound(SNAPSHOT, "TASK-2", "TRAY-2")).thenAnswer(invocation -> {
|
||||
planningStarted.countDown();
|
||||
if (!releasePlanning.await(5, TimeUnit.SECONDS)) {
|
||||
throw new IllegalStateException("test planning was not released");
|
||||
}
|
||||
return observation;
|
||||
releasePlanning.await(1, TimeUnit.SECONDS);
|
||||
return inboundObservation();
|
||||
});
|
||||
gateway = new ShadowObservationGateway(enabledProperties(), engine);
|
||||
ExecutorService legacyCaller = Executors.newSingleThreadExecutor();
|
||||
stubSuccessfulRecovery(engine, mapper, 43L);
|
||||
ShadowObservationGateway gateway = gateway(engine, mapper, coordinator,
|
||||
new ShadowRuntimeMetrics(), Clock.fixed(NOW.plusSeconds(2), ZoneOffset.UTC), delay -> { });
|
||||
|
||||
try {
|
||||
Future<InboundShadowCompletion> submitted =
|
||||
legacyCaller.submit(() -> gateway.observeInbound("TASK-3", "TRAY-3"));
|
||||
InboundShadowCompletion first = submitted.get(1, TimeUnit.SECONDS);
|
||||
assertThat(first.accepted()).isTrue();
|
||||
assertThat(planningStarted.await(1, TimeUnit.SECONDS)).isTrue();
|
||||
InboundShadowCompletion first = gateway.observeInbound("TASK-2", "TRAY-2");
|
||||
assertThat(planningStarted.await(1, TimeUnit.SECONDS)).isTrue();
|
||||
long started = System.nanoTime();
|
||||
OutboundShadowCompletion concurrent = gateway.observeProjectOutbound("DETAIL-2");
|
||||
long elapsedMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - started);
|
||||
|
||||
OutboundShadowCompletion concurrent = gateway.observeProjectOutbound("DETAIL-3");
|
||||
assertThat(concurrent.accepted()).isFalse();
|
||||
assertThat(elapsedMillis).isLessThan(1000L);
|
||||
concurrent.complete("10");
|
||||
first.complete("10", "LEGACY-SPACE");
|
||||
releasePlanning.countDown();
|
||||
|
||||
assertThat(concurrent.accepted()).isFalse();
|
||||
concurrent.complete("10");
|
||||
verify(engine, never()).planProjectOutbound("DETAIL-3");
|
||||
verify(engine, after(500).never()).compareInbound(any(), anyString(), anyString());
|
||||
verify(engine, never()).planProjectOutbound(any(), anyString());
|
||||
assertThat(coordinator.activeLegacyCount()).isZero();
|
||||
}
|
||||
|
||||
first.complete("10", "SPACE-LEGACY");
|
||||
releasePlanning.countDown();
|
||||
@Test
|
||||
void rejectedWorkerSubmissionReturnsTrackedSkipAndReleasesAdmission() throws Exception {
|
||||
ThreadPoolExecutor fullExecutor = executor();
|
||||
CountDownLatch holdWorker = new CountDownLatch(1);
|
||||
fullExecutor.execute(() -> await(holdWorker));
|
||||
fullExecutor.execute(() -> { });
|
||||
ShadowPlanningEngine engine = mock(ShadowPlanningEngine.class);
|
||||
ShadowSnapshotCoordinator coordinator = coordinatorWithSnapshot();
|
||||
ShadowObservationGateway gateway = gateway(engine, mock(ShadowInputMapper.class), coordinator,
|
||||
new ShadowRuntimeMetrics(), Clock.fixed(NOW, ZoneOffset.UTC), delay -> { }, fullExecutor);
|
||||
|
||||
verify(engine, timeout(2000)).compareInbound(observation, "10", "SPACE-LEGACY");
|
||||
} finally {
|
||||
releasePlanning.countDown();
|
||||
legacyCaller.shutdownNow();
|
||||
InboundShadowCompletion completion = gateway.observeInbound("TASK-3", "TRAY-3");
|
||||
completion.complete("10", "SPACE-3");
|
||||
|
||||
assertThat(completion.accepted()).isFalse();
|
||||
assertThat(coordinator.activeLegacyCount()).isZero();
|
||||
verify(engine, never()).planInbound(any(), anyString(), anyString());
|
||||
holdWorker.countDown();
|
||||
}
|
||||
|
||||
@Test
|
||||
void planningFailureIsWorkerOnlyAndRecordedByFixedStage() throws Exception {
|
||||
ShadowPlanningEngine engine = mock(ShadowPlanningEngine.class);
|
||||
ShadowInputMapper mapper = mock(ShadowInputMapper.class);
|
||||
ShadowSnapshotCoordinator coordinator = coordinatorWithSnapshot();
|
||||
when(engine.planInbound(SNAPSHOT, "TASK-4", "TRAY-4"))
|
||||
.thenThrow(new IllegalStateException("sensitive business payload"));
|
||||
CountDownLatch planMetric = metricLatch(mapper, "ERROR_PLAN");
|
||||
stubSuccessfulRecovery(engine, mapper, 43L);
|
||||
ShadowObservationGateway gateway = gateway(engine, mapper, coordinator,
|
||||
new ShadowRuntimeMetrics(), Clock.fixed(NOW.plusSeconds(2), ZoneOffset.UTC), delay -> { });
|
||||
|
||||
InboundShadowCompletion completion = gateway.observeInbound("TASK-4", "TRAY-4");
|
||||
completion.complete("10", "SPACE-4");
|
||||
|
||||
assertThat(planMetric.await(1, TimeUnit.SECONDS)).isTrue();
|
||||
verify(engine, never()).compareInbound(any(), anyString(), anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void comparisonFailureIsWorkerOnlyAndRecordedByFixedStage() throws Exception {
|
||||
ShadowPlanningEngine engine = mock(ShadowPlanningEngine.class);
|
||||
ShadowInputMapper mapper = mock(ShadowInputMapper.class);
|
||||
ShadowSnapshotCoordinator coordinator = coordinatorWithSnapshot();
|
||||
InboundShadowObservation observation = inboundObservation();
|
||||
when(engine.planInbound(SNAPSHOT, "TASK-8", "TRAY-8")).thenReturn(observation);
|
||||
org.mockito.Mockito.doThrow(new IllegalStateException("comparison failed"))
|
||||
.when(engine).compareInbound(observation, "10", "SPACE-8");
|
||||
CountDownLatch compareMetric = metricLatch(mapper, "ERROR_COMPARE");
|
||||
stubSuccessfulRecovery(engine, mapper, 43L);
|
||||
ShadowObservationGateway gateway = gateway(engine, mapper, coordinator,
|
||||
new ShadowRuntimeMetrics(), Clock.fixed(NOW.plusSeconds(2), ZoneOffset.UTC), delay -> { });
|
||||
|
||||
InboundShadowCompletion completion = gateway.observeInbound("TASK-8", "TRAY-8");
|
||||
completion.complete("10", "SPACE-8");
|
||||
|
||||
assertThat(compareMetric.await(1, TimeUnit.SECONDS)).isTrue();
|
||||
verify(engine).compareInbound(observation, "10", "SPACE-8");
|
||||
}
|
||||
|
||||
@Test
|
||||
void legacyWaitTimeoutIsRecordedWithoutClosingCallerOwnedActivity() throws Exception {
|
||||
ShadowPlanningEngine engine = mock(ShadowPlanningEngine.class);
|
||||
ShadowInputMapper mapper = mock(ShadowInputMapper.class);
|
||||
ShadowSnapshotCoordinator coordinator = coordinatorWithSnapshot();
|
||||
when(engine.planInbound(SNAPSHOT, "TASK-5", "TRAY-5")).thenReturn(inboundObservation());
|
||||
CountDownLatch waitMetric = metricLatch(mapper, "ERROR_LEGACY_WAIT");
|
||||
ShadowObservationProperties properties = enabledProperties();
|
||||
properties.setTimeoutSeconds(1);
|
||||
ShadowObservationGateway gateway = gateway(properties, engine, mapper, coordinator,
|
||||
new ShadowRuntimeMetrics(), Clock.fixed(NOW, ZoneOffset.UTC), delay -> { }, executor());
|
||||
|
||||
InboundShadowCompletion completion = gateway.observeInbound("TASK-5", "TRAY-5");
|
||||
|
||||
assertThat(waitMetric.await(2, TimeUnit.SECONDS)).isTrue();
|
||||
assertThat(coordinator.activeLegacyCount()).isEqualTo(1);
|
||||
completion.fail(new IllegalStateException("legacy failed later"));
|
||||
assertThat(coordinator.activeLegacyCount()).isZero();
|
||||
}
|
||||
|
||||
@Test
|
||||
void projectAndExactStockOutboundUseTheLeasedGeneration() throws Exception {
|
||||
assertOutboundUsesLease(false);
|
||||
assertOutboundUsesLease(true);
|
||||
}
|
||||
|
||||
@Test
|
||||
void refreshRetriesWithOneTwoFourSecondBackoffAndCoalescesRecoveryForOneMinute() throws Exception {
|
||||
ShadowPlanningEngine engine = mock(ShadowPlanningEngine.class);
|
||||
ShadowInputMapper mapper = mock(ShadowInputMapper.class);
|
||||
AtomicInteger attempts = new AtomicInteger();
|
||||
CountDownLatch failedCycle = new CountDownLatch(1);
|
||||
when(engine.refreshSnapshot(anyLong(), eq("INSTANCE-1"))).thenAnswer(invocation -> {
|
||||
int attempt = attempts.incrementAndGet();
|
||||
if (attempt <= 4) {
|
||||
if (attempt == 4) {
|
||||
failedCycle.countDown();
|
||||
}
|
||||
throw new IllegalStateException("refresh failed");
|
||||
}
|
||||
return new PublishedShadowSnapshot(invocation.getArgument(0), "01",
|
||||
NOW.plusSeconds(61), "ABC999");
|
||||
});
|
||||
MutableClock clock = new MutableClock(NOW);
|
||||
List<Duration> delays = new ArrayList<>();
|
||||
ShadowObservationGateway gateway = gateway(engine, mapper, coordinator(),
|
||||
new ShadowRuntimeMetrics(), clock, delay -> {
|
||||
delays.add(delay);
|
||||
clock.advance(delay);
|
||||
});
|
||||
|
||||
gateway.start();
|
||||
|
||||
assertThat(failedCycle.await(1, TimeUnit.SECONDS)).isTrue();
|
||||
verify(engine, timeout(1000).times(4)).refreshSnapshot(anyLong(), eq("INSTANCE-1"));
|
||||
assertThat(delays).containsExactly(Duration.ofSeconds(1), Duration.ofSeconds(2), Duration.ofSeconds(4));
|
||||
|
||||
InboundShadowCompletion beforeMinute = gateway.observeInbound("TASK-6", "TRAY-6");
|
||||
beforeMinute.complete("10", "SPACE-6");
|
||||
verify(engine, after(200).times(4)).refreshSnapshot(anyLong(), eq("INSTANCE-1"));
|
||||
|
||||
clock.advance(Duration.ofSeconds(61));
|
||||
CountDownLatch recovered = metricLatch(mapper, "SKIP_NOT_READY");
|
||||
InboundShadowCompletion afterMinute = gateway.observeInbound("TASK-7", "TRAY-7");
|
||||
afterMinute.complete("10", "SPACE-7");
|
||||
|
||||
verify(engine, timeout(1000).times(5)).refreshSnapshot(anyLong(), eq("INSTANCE-1"));
|
||||
assertThat(recovered.await(1, TimeUnit.SECONDS)).isTrue();
|
||||
}
|
||||
|
||||
private void assertOutboundUsesLease(boolean exactStock) throws Exception {
|
||||
ShadowPlanningEngine engine = mock(ShadowPlanningEngine.class);
|
||||
ShadowInputMapper mapper = mock(ShadowInputMapper.class);
|
||||
ShadowSnapshotCoordinator coordinator = coordinatorWithSnapshot();
|
||||
OutboundShadowObservation observation = new OutboundShadowObservation(
|
||||
true, 42L, "01", NOW, "ABC123", "DETAIL-7", "READY",
|
||||
"STOCK-7", "SPACE-7", "diagnostic");
|
||||
if (exactStock) {
|
||||
when(engine.planExactStockOutbound(SNAPSHOT, "DETAIL-7", "STOCK-7")).thenReturn(observation);
|
||||
} else {
|
||||
when(engine.planProjectOutbound(SNAPSHOT, "DETAIL-7")).thenReturn(observation);
|
||||
}
|
||||
CountDownLatch compared = new CountDownLatch(1);
|
||||
doAnswer(invocation -> {
|
||||
compared.countDown();
|
||||
return null;
|
||||
}).when(engine).compareOutbound(observation, "10");
|
||||
stubSuccessfulRecovery(engine, mapper, 43L);
|
||||
ShadowObservationGateway gateway = gateway(engine, mapper, coordinator,
|
||||
new ShadowRuntimeMetrics(), Clock.fixed(NOW.plusSeconds(2), ZoneOffset.UTC), delay -> { });
|
||||
|
||||
OutboundShadowCompletion completion = exactStock
|
||||
? gateway.observeExactStockOutbound("DETAIL-7", "STOCK-7")
|
||||
: gateway.observeProjectOutbound("DETAIL-7");
|
||||
completion.complete("10");
|
||||
|
||||
assertThat(completion.accepted()).isTrue();
|
||||
assertThat(compared.await(1, TimeUnit.SECONDS)).isTrue();
|
||||
if (exactStock) {
|
||||
verify(engine).planExactStockOutbound(SNAPSHOT, "DETAIL-7", "STOCK-7");
|
||||
} else {
|
||||
verify(engine).planProjectOutbound(SNAPSHOT, "DETAIL-7");
|
||||
}
|
||||
}
|
||||
|
||||
private ShadowObservationGateway gateway(ShadowPlanningEngine engine, ShadowInputMapper mapper,
|
||||
ShadowSnapshotCoordinator coordinator, ShadowRuntimeMetrics metrics, Clock clock,
|
||||
ShadowObservationGateway.Sleeper sleeper) {
|
||||
return gateway(enabledProperties(), engine, mapper, coordinator, metrics, clock, sleeper, executor());
|
||||
}
|
||||
|
||||
private ShadowObservationGateway gateway(ShadowPlanningEngine engine, ShadowInputMapper mapper,
|
||||
ShadowSnapshotCoordinator coordinator, ShadowRuntimeMetrics metrics, Clock clock,
|
||||
ShadowObservationGateway.Sleeper sleeper, ThreadPoolExecutor executor) {
|
||||
return gateway(enabledProperties(), engine, mapper, coordinator, metrics, clock, sleeper, executor);
|
||||
}
|
||||
|
||||
private ShadowObservationGateway gateway(ShadowObservationProperties properties,
|
||||
ShadowPlanningEngine engine, ShadowInputMapper mapper, ShadowSnapshotCoordinator coordinator,
|
||||
ShadowRuntimeMetrics metrics, Clock clock, ShadowObservationGateway.Sleeper sleeper,
|
||||
ThreadPoolExecutor executor) {
|
||||
ShadowObservationGateway gateway = new ShadowObservationGateway(properties, engine, mapper,
|
||||
coordinator, metrics, clock, sleeper, executor, "INSTANCE-1");
|
||||
gateways.add(gateway);
|
||||
return gateway;
|
||||
}
|
||||
|
||||
private ThreadPoolExecutor executor() {
|
||||
ThreadPoolExecutor executor = new ThreadPoolExecutor(
|
||||
1, 1, 0L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(1),
|
||||
new ThreadPoolExecutor.AbortPolicy());
|
||||
executors.add(executor);
|
||||
return executor;
|
||||
}
|
||||
|
||||
private ShadowSnapshotCoordinator coordinator() {
|
||||
return new ShadowSnapshotCoordinator("01", Duration.ofSeconds(60), Clock.fixed(NOW, ZoneOffset.UTC));
|
||||
}
|
||||
|
||||
private ShadowSnapshotCoordinator coordinatorWithSnapshot() {
|
||||
ShadowSnapshotCoordinator coordinator = coordinator();
|
||||
ShadowSnapshotCoordinator.RefreshTicket ticket = coordinator.tryBeginRefresh(NOW).orElseThrow();
|
||||
assertThat(coordinator.publish(ticket, SNAPSHOT, NOW)).isTrue();
|
||||
return coordinator;
|
||||
}
|
||||
|
||||
private void stubSuccessfulRecovery(ShadowPlanningEngine engine, ShadowInputMapper mapper, long generation) {
|
||||
ShadowSnapshotDescriptor descriptor = new ShadowSnapshotDescriptor();
|
||||
descriptor.setSnapshotGeneration(generation - 1);
|
||||
when(mapper.selectPublishedSnapshot("01")).thenReturn(descriptor);
|
||||
when(engine.refreshSnapshot(generation, "INSTANCE-1")).thenReturn(
|
||||
new PublishedShadowSnapshot(generation, "01", NOW.plusSeconds(2), "ABC124"));
|
||||
}
|
||||
|
||||
private CountDownLatch metricLatch(ShadowInputMapper mapper, String metricName) {
|
||||
CountDownLatch latch = new CountDownLatch(1);
|
||||
doAnswer(invocation -> {
|
||||
if (metricName.equals(invocation.getArgument(1))) {
|
||||
latch.countDown();
|
||||
}
|
||||
return null;
|
||||
}).when(mapper).upsertRuntimeMetric(anyString(), anyString(), anyLong(), anyLong(),
|
||||
anyLong(), anyLong(), any(Instant.class));
|
||||
return latch;
|
||||
}
|
||||
|
||||
private InboundShadowObservation inboundObservation() {
|
||||
return new InboundShadowObservation(true, 42L, "01", NOW, "ABC123",
|
||||
"TASK-1", "READY", "SHADOW-SPACE", "diagnostic");
|
||||
}
|
||||
|
||||
private ShadowObservationProperties enabledProperties() {
|
||||
ShadowObservationProperties properties = new ShadowObservationProperties();
|
||||
properties.setEnabled(true);
|
||||
properties.setWarehouseId("01");
|
||||
properties.setTimeoutSeconds(3);
|
||||
properties.setQuietIntervalMillis(1000);
|
||||
return properties;
|
||||
}
|
||||
|
||||
private boolean waitForPhase(ShadowSnapshotCoordinator coordinator,
|
||||
ShadowSnapshotCoordinator.Phase expected) throws InterruptedException {
|
||||
long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(1);
|
||||
while (System.nanoTime() < deadline) {
|
||||
if (coordinator.phase() == expected) {
|
||||
return true;
|
||||
}
|
||||
Thread.sleep(5L);
|
||||
}
|
||||
return coordinator.phase() == expected;
|
||||
}
|
||||
|
||||
private static void await(CountDownLatch latch) {
|
||||
try {
|
||||
latch.await(5, TimeUnit.SECONDS);
|
||||
} catch (InterruptedException interrupted) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
}
|
||||
|
||||
private static final class MutableClock extends Clock {
|
||||
private Instant instant;
|
||||
|
||||
private MutableClock(Instant instant) {
|
||||
this.instant = instant;
|
||||
}
|
||||
|
||||
private void advance(Duration duration) {
|
||||
instant = instant.plus(duration);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ZoneId getZone() {
|
||||
return ZoneOffset.UTC;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Clock withZone(ZoneId zone) {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Instant instant() {
|
||||
return instant;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
package com.ruoyi.algorithm.shadow;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
class ShadowRuntimeMetricsTest {
|
||||
|
||||
@Test
|
||||
void recordsFixedCountersAndHistogramBucketsThenDrainsAtomically() {
|
||||
ShadowRuntimeMetrics metrics = new ShadowRuntimeMetrics();
|
||||
|
||||
metrics.recordAdmission(ShadowSkipReason.BUSY, 750_000L);
|
||||
metrics.recordAdmission(null, 1_500_000L);
|
||||
metrics.recordLegacyDuration(1_500_000L);
|
||||
metrics.recordError(ShadowFailureStage.PLAN);
|
||||
|
||||
ShadowRuntimeMetrics.Batch batch = metrics.drain();
|
||||
|
||||
assertThat(batch.count("SKIP_BUSY")).isEqualTo(1);
|
||||
assertThat(batch.count("ADMITTED")).isEqualTo(1);
|
||||
assertThat(batch.count("ERROR_PLAN")).isEqualTo(1);
|
||||
assertThat(batch.bucketCount("ADMISSION_US", 1000L)).isEqualTo(1);
|
||||
assertThat(batch.bucketCount("ADMISSION_US", 2500L)).isEqualTo(1);
|
||||
assertThat(batch.bucketCount("LEGACY_DURATION_US", 2500L)).isEqualTo(1);
|
||||
assertThat(metrics.drain().totalCount()).isZero();
|
||||
assertThatThrownBy(() -> batch.rows().clear())
|
||||
.isInstanceOf(UnsupportedOperationException.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
void recordsOverflowAndDoesNotLoseConcurrentCounterUpdates() throws Exception {
|
||||
ShadowRuntimeMetrics metrics = new ShadowRuntimeMetrics();
|
||||
int workers = 4;
|
||||
int perWorker = 1000;
|
||||
ExecutorService executor = Executors.newFixedThreadPool(workers);
|
||||
try {
|
||||
for (int worker = 0; worker < workers; worker++) {
|
||||
executor.submit(() -> {
|
||||
for (int index = 0; index < perWorker; index++) {
|
||||
metrics.increment(ShadowSkipReason.CONCURRENT);
|
||||
}
|
||||
});
|
||||
}
|
||||
} finally {
|
||||
executor.shutdown();
|
||||
assertThat(executor.awaitTermination(2, TimeUnit.SECONDS)).isTrue();
|
||||
}
|
||||
|
||||
metrics.recordAdmission(null, TimeUnit.SECONDS.toNanos(2));
|
||||
metrics.recordLegacyDuration(TimeUnit.SECONDS.toNanos(2));
|
||||
ShadowRuntimeMetrics.Batch batch = metrics.drain();
|
||||
|
||||
assertThat(batch.count("SKIP_CONCURRENT")).isEqualTo((long) workers * perWorker);
|
||||
assertThat(batch.bucketCount("ADMISSION_US", Long.MAX_VALUE)).isEqualTo(1);
|
||||
assertThat(batch.bucketCount("LEGACY_DURATION_US", Long.MAX_VALUE)).isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void metricPersistenceUsesOneFixedShadowOwnedSqlStatement() throws Exception {
|
||||
String xml;
|
||||
try (InputStream stream = Thread.currentThread().getContextClassLoader()
|
||||
.getResourceAsStream("mapper/algorithm/ShadowInputMapper.xml")) {
|
||||
assertThat(stream).isNotNull();
|
||||
xml = new String(stream.readAllBytes(), StandardCharsets.UTF_8).toLowerCase();
|
||||
}
|
||||
int start = xml.indexOf("<update id=\"upsertruntimemetric\">");
|
||||
int end = xml.indexOf("</update>", start);
|
||||
String statement = xml.substring(start, end);
|
||||
|
||||
assertThat(statement).contains("merge alg.shadow_runtime_metric")
|
||||
.doesNotContain("dbo.", "${");
|
||||
}
|
||||
}
|
||||
@@ -61,6 +61,7 @@ class ShadowSnapshotCoordinatorTest {
|
||||
assertThat(coordinator.publish(refresh, snapshot(2, "01", NOW), NOW)).isFalse();
|
||||
assertThat(coordinator.phase()).isNotEqualTo(ShadowSnapshotCoordinator.Phase.READY);
|
||||
coordinator.exit(concurrent);
|
||||
assertThat(coordinator.tryBeginRefresh(NOW)).isPresent();
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -130,6 +131,19 @@ class ShadowSnapshotCoordinatorTest {
|
||||
assertThat(coordinator.phase()).isEqualTo(ShadowSnapshotCoordinator.Phase.READY);
|
||||
}
|
||||
|
||||
@Test
|
||||
void staleRefreshTicketCannotCancelANewerRefreshLease() {
|
||||
ShadowSnapshotCoordinator coordinator = coordinatorAt("01");
|
||||
ShadowSnapshotCoordinator.RefreshTicket first = coordinator.tryBeginRefresh(NOW).orElseThrow();
|
||||
coordinator.failRefresh(first, "first failed");
|
||||
ShadowSnapshotCoordinator.RefreshTicket second = coordinator.tryBeginRefresh(NOW).orElseThrow();
|
||||
|
||||
coordinator.failRefresh(first, "late duplicate failure");
|
||||
|
||||
assertThat(coordinator.phase()).isEqualTo(ShadowSnapshotCoordinator.Phase.BUSY);
|
||||
assertThat(coordinator.publish(second, snapshot(9, "01", NOW), NOW)).isTrue();
|
||||
}
|
||||
|
||||
private ShadowSnapshotCoordinator coordinatorAt(String warehouseId) {
|
||||
return new ShadowSnapshotCoordinator(warehouseId, Duration.ofSeconds(60));
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user