fix: observe legacy failures in shadow worker
This commit is contained in:
@@ -156,6 +156,7 @@ public class ShadowObservationGateway {
|
||||
stage = ShadowFailureStage.LEGACY_WAIT;
|
||||
InboundShadowCompletion.LegacyResult legacy = completion.awaitLegacyResult();
|
||||
if (legacy.failure() != null) {
|
||||
recordWorkerFailure(taskNo, admission, stage, legacy.failure(), startedNanos);
|
||||
return;
|
||||
}
|
||||
if (!coordinator.isComparable(admission)) {
|
||||
@@ -185,6 +186,7 @@ public class ShadowObservationGateway {
|
||||
stage = ShadowFailureStage.LEGACY_WAIT;
|
||||
OutboundShadowCompletion.LegacyResult legacy = completion.awaitLegacyResult();
|
||||
if (legacy.failure() != null) {
|
||||
recordWorkerFailure(stockoutDetailId, admission, stage, legacy.failure(), startedNanos);
|
||||
return;
|
||||
}
|
||||
if (!coordinator.isComparable(admission)) {
|
||||
@@ -334,7 +336,7 @@ public class ShadowObservationGateway {
|
||||
|
||||
private void recordWorkerFailure(String requestId,
|
||||
ShadowSnapshotCoordinator.Admission admission, ShadowFailureStage stage,
|
||||
Exception failure, long startedNanos) {
|
||||
Throwable failure, long startedNanos) {
|
||||
metrics.recordError(stage);
|
||||
warnFailure(requestId, admission.snapshot().generation(), stage, failure, startedNanos);
|
||||
}
|
||||
|
||||
@@ -255,6 +255,51 @@ class ShadowObservationGatewayTest {
|
||||
assertThat(coordinator.activeLegacyCount()).isZero();
|
||||
}
|
||||
|
||||
@Test
|
||||
void inboundLegacyFailureIsRecordedAtLegacyWaitStage() throws Exception {
|
||||
ShadowPlanningEngine engine = mock(ShadowPlanningEngine.class);
|
||||
ShadowInputMapper mapper = mock(ShadowInputMapper.class);
|
||||
ShadowSnapshotCoordinator coordinator = coordinatorWithSnapshot();
|
||||
when(engine.planInbound(SNAPSHOT, "TASK-FAIL", "TRAY-FAIL")).thenReturn(inboundObservation());
|
||||
CountDownLatch waitMetric = metricLatch(mapper, "ERROR_LEGACY_WAIT");
|
||||
stubSuccessfulRecovery(engine, mapper, 43L);
|
||||
ShadowObservationGateway gateway = gateway(engine, mapper, coordinator,
|
||||
new ShadowRuntimeMetrics(), Clock.fixed(NOW, ZoneOffset.UTC), delay -> { });
|
||||
|
||||
InboundShadowCompletion completion = gateway.observeInbound("TASK-FAIL", "TRAY-FAIL");
|
||||
completion.fail(new IllegalStateException("business payload must not be logged"));
|
||||
|
||||
assertThat(waitMetric.await(1, TimeUnit.SECONDS)).isTrue();
|
||||
assertThat(coordinator.activeLegacyCount()).isZero();
|
||||
completion.fail(new IllegalStateException("duplicate"));
|
||||
completion.complete("10", "SPACE-FAIL");
|
||||
assertThat(coordinator.activeLegacyCount()).isZero();
|
||||
}
|
||||
|
||||
@Test
|
||||
void outboundLegacyFailureIsRecordedAtLegacyWaitStage() throws Exception {
|
||||
ShadowPlanningEngine engine = mock(ShadowPlanningEngine.class);
|
||||
ShadowInputMapper mapper = mock(ShadowInputMapper.class);
|
||||
ShadowSnapshotCoordinator coordinator = coordinatorWithSnapshot();
|
||||
OutboundShadowObservation observation = new OutboundShadowObservation(
|
||||
true, 42L, "01", NOW, "ABC123", "DETAIL-FAIL", "READY",
|
||||
"STOCK-FAIL", "SPACE-FAIL", "diagnostic");
|
||||
when(engine.planProjectOutbound(SNAPSHOT, "DETAIL-FAIL")).thenReturn(observation);
|
||||
CountDownLatch waitMetric = metricLatch(mapper, "ERROR_LEGACY_WAIT");
|
||||
stubSuccessfulRecovery(engine, mapper, 43L);
|
||||
ShadowObservationGateway gateway = gateway(engine, mapper, coordinator,
|
||||
new ShadowRuntimeMetrics(), Clock.fixed(NOW, ZoneOffset.UTC), delay -> { });
|
||||
|
||||
OutboundShadowCompletion completion = gateway.observeProjectOutbound("DETAIL-FAIL");
|
||||
completion.fail(new IllegalStateException("business payload must not be logged"));
|
||||
|
||||
assertThat(waitMetric.await(1, TimeUnit.SECONDS)).isTrue();
|
||||
assertThat(coordinator.activeLegacyCount()).isZero();
|
||||
completion.fail(new IllegalStateException("duplicate"));
|
||||
completion.complete("10");
|
||||
assertThat(coordinator.activeLegacyCount()).isZero();
|
||||
}
|
||||
|
||||
@Test
|
||||
void projectAndExactStockOutboundUseTheLeasedGeneration() throws Exception {
|
||||
assertOutboundUsesLease(false);
|
||||
|
||||
Reference in New Issue
Block a user