lhq/FT546-WCS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(shadow): harden production observation cycle

Browse Source
This commit is contained in:
zgw
2026-09-03 11:10:22 +08:00
parent 0080befff4
commit 9829b9c0fa
7 changed files with 325 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

223
ruoyi-wcs-shadow/src/test/java/com/ruoyi/shadow/Ft546ShadowFaultIsolationTest.java
View File

@@ -1,5 +1,8 @@
package com.ruoyi.shadow;
import ch.qos.logback.classic.Logger;
import ch.qos.logback.classic.spi.ILoggingEvent;
import ch.qos.logback.core.read.ListAppender;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.ruoyi.ft546.protocol.Ft546DeviceProfile;
import com.ruoyi.ft546.protocol.Ft546SrgvCommandEncoder;
@@ -11,6 +14,7 @@ import com.ruoyi.shadow.evidence.ShadowEvidenceException;
import com.ruoyi.shadow.evidence.ShadowEvidenceRepository;
import com.ruoyi.shadow.evidence.ShadowEvidenceStorageHealth;
import com.ruoyi.shadow.facts.ShadowFactBatchValidator;
import com.ruoyi.shadow.facts.ShadowFactReadException;
import com.ruoyi.shadow.facts.ShadowMapFact;
import com.ruoyi.shadow.facts.ShadowQueueFact;
import com.ruoyi.shadow.facts.ShadowReadCursor;
@@ -28,10 +32,13 @@ import org.springframework.core.io.ClassPathResource;
import org.springframework.core.env.EnumerablePropertySource;
import org.springframework.core.env.PropertySource;
import org.springframework.boot.env.YamlPropertySourceLoader;
import org.springframework.dao.DataAccessResourceFailureException;
import org.springframework.dao.QueryTimeoutException;
import org.springframework.dao.TransientDataAccessResourceException;
import org.springframework.jdbc.datasource.DataSourceTransactionManager;
import org.springframework.jdbc.datasource.DriverManagerDataSource;
import org.springframework.transaction.support.TransactionTemplate;
import org.slf4j.LoggerFactory;
import java.time.Clock;
import java.time.Duration;
@@ -51,6 +58,7 @@ import java.util.UUID;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Supplier;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -61,7 +69,7 @@ class Ft546ShadowFaultIsolationTest {
private static final Instant NOW = Instant.parse("2026-08-14T06:00:00Z");
@Test
void sourceTimeoutNeverEntersCycleOrRecordsACommand() {
void sourceQueryTimeoutPermanentlyStopsCollectionBeforeCycle() {
MutableClock clock = new MutableClock(NOW);
AtomicInteger sourceCalls = new AtomicInteger();
SqlServerShadowFactReader reader = reader(timeoutSource(sourceCalls), clock);
@@ -72,15 +80,172 @@ class Ft546ShadowFaultIsolationTest {
evidenceCycle(report, evidence, gateway,
() -> ShadowEvidenceStorageHealth.Check.available(
Long.MAX_VALUE)), report, clock);
Logger schedulerLogger = (Logger) LoggerFactory.getLogger(
Ft546ShadowScheduler.class);
ListAppender<ILoggingEvent> logAppender =
new ListAppender<ILoggingEvent>();
logAppender.start();
schedulerLogger.addAppender(logAppender);
assertFalse(scheduler.runOnce());
try {
assertFalse(scheduler.runOnce());
clock.advance(Duration.ofMinutes(1));
assertEquals(1, sourceCalls.get());
assertEquals(0, evidence.transactionCalls.get());
assertEquals(Collections.emptyList(), gateway.getRecorded());
assertEquals(ShadowStartupReport.Status.DEGRADED, report.getStatus());
assertEquals("SOURCE_QUERY_TIMEOUT", report.getReasonCode());
assertEquals(Duration.ofSeconds(1), scheduler.getRemainingBackoff());
assertFalse(scheduler.runOnce());
assertEquals(1, sourceCalls.get());
assertEquals(0, evidence.transactionCalls.get());
assertEquals(Collections.emptyList(), gateway.getRecorded());
assertEquals(ShadowStartupReport.Status.DEGRADED,
report.getStatus());
assertEquals("SOURCE_QUERY_TIMEOUT", report.getReasonCode());
assertTrue(scheduler.isPermanentlyStopped());
assertTrue(logAppender.list.stream().anyMatch(event ->
event.getFormattedMessage()
.contains("stage=SOURCE_READ")),
"source timeout log must identify the blocked stage");
} finally {
schedulerLogger.detachAppender(logAppender);
logAppender.stop();
scheduler.close();
}
}
@Test
void sourceBatchDeadlinePermanentlyStopsCollection() {
MutableClock clock = new MutableClock(NOW);
AtomicInteger sourceCalls = new AtomicInteger();
SqlServerShadowFactReader.FactSource source = new FactSourceAdapter() {
@Override
public Instant readDatabaseClock(Instant deadline) {
sourceCalls.incrementAndGet();
throw new ShadowFactReadException(
"SOURCE_BATCH_DEADLINE_EXCEEDED", Duration.ZERO);
}
};
ShadowStartupReport report = report();
Ft546ShadowScheduler scheduler = new Ft546ShadowScheduler(
reader(source, clock),
evidenceCycle(report, new RecordingEvidenceRepository(),
gateway(),
() -> ShadowEvidenceStorageHealth.Check.available(
Long.MAX_VALUE)),
report, clock);
try {
assertFalse(scheduler.runOnce());
clock.advance(Duration.ofMinutes(1));
assertFalse(scheduler.runOnce());
assertEquals(1, sourceCalls.get());
assertEquals("SOURCE_BATCH_DEADLINE_EXCEEDED",
report.getReasonCode());
assertTrue(scheduler.isPermanentlyStopped());
} finally {
scheduler.close();
}
}
@Test
void wrappedSourceQueryTimeoutPermanentlyStopsCollection() {
MutableClock clock = new MutableClock(NOW);
AtomicInteger sourceCalls = new AtomicInteger();
SqlServerShadowFactReader.FactSource source = new FactSourceAdapter() {
@Override
public Instant readDatabaseClock(Instant deadline) {
sourceCalls.incrementAndGet();
throw new DataAccessResourceFailureException(
"wrapped source timeout",
new QueryTimeoutException("query timed out"));
}
};
ShadowStartupReport report = report();
Ft546ShadowScheduler scheduler = new Ft546ShadowScheduler(
reader(source, clock),
evidenceCycle(report, new RecordingEvidenceRepository(),
gateway(),
() -> ShadowEvidenceStorageHealth.Check.available(
Long.MAX_VALUE)),
report, clock);
try {
assertFalse(scheduler.runOnce());
clock.advance(Duration.ofMinutes(1));
assertFalse(scheduler.runOnce());
assertEquals(1, sourceCalls.get());
assertEquals("SOURCE_QUERY_TIMEOUT", report.getReasonCode());
assertTrue(scheduler.isPermanentlyStopped());
} finally {
scheduler.close();
}
}
@Test
void nonTimeoutSourceFailureStillUsesBackoff() {
MutableClock clock = new MutableClock(NOW);
AtomicInteger sourceCalls = new AtomicInteger();
SqlServerShadowFactReader.FactSource source = new FactSourceAdapter() {
@Override
public Instant readDatabaseClock(Instant deadline) {
sourceCalls.incrementAndGet();
throw new DataAccessResourceFailureException(
"simulated source outage");
}
};
ShadowStartupReport report = report();
Ft546ShadowScheduler scheduler = new Ft546ShadowScheduler(
reader(source, clock),
evidenceCycle(report, new RecordingEvidenceRepository(),
gateway(),
() -> ShadowEvidenceStorageHealth.Check.available(
Long.MAX_VALUE)),
report, clock);
try {
assertFalse(scheduler.runOnce());
assertEquals(1, sourceCalls.get());
assertEquals("SOURCE_UNAVAILABLE", report.getReasonCode());
assertFalse(scheduler.isPermanentlyStopped());
assertEquals(Duration.ofSeconds(1),
scheduler.getRemainingBackoff());
} finally {
scheduler.close();
}
}
@Test
void transientSourceFailureStillUsesBackoff() {
MutableClock clock = new MutableClock(NOW);
AtomicInteger sourceCalls = new AtomicInteger();
SqlServerShadowFactReader.FactSource source = new FactSourceAdapter() {
@Override
public Instant readDatabaseClock(Instant deadline) {
sourceCalls.incrementAndGet();
throw new TransientDataAccessResourceException(
"simulated transient source failure");
}
};
ShadowStartupReport report = report();
Ft546ShadowScheduler scheduler = new Ft546ShadowScheduler(
reader(source, clock),
evidenceCycle(report, new RecordingEvidenceRepository(),
gateway(),
() -> ShadowEvidenceStorageHealth.Check.available(
Long.MAX_VALUE)),
report, clock);
try {
assertFalse(scheduler.runOnce());
assertEquals(1, sourceCalls.get());
assertEquals("SOURCE_UNAVAILABLE", report.getReasonCode());
assertFalse(scheduler.isPermanentlyStopped());
assertEquals(Duration.ofSeconds(1),
scheduler.getRemainingBackoff());
} finally {
scheduler.close();
}
}
@Test
@@ -174,6 +339,12 @@ class Ft546ShadowFaultIsolationTest {
}, "shadow-cycle-test-release");
delayedRelease.setDaemon(true);
delayedRelease.start();
Logger schedulerLogger = (Logger) LoggerFactory.getLogger(
Ft546ShadowScheduler.class);
ListAppender<ILoggingEvent> logAppender =
new ListAppender<ILoggingEvent>();
logAppender.start();
schedulerLogger.addAppender(logAppender);
long startedAt = System.nanoTime();
try {
@@ -189,6 +360,10 @@ class Ft546ShadowFaultIsolationTest {
assertEquals(ShadowStartupReport.Status.DEGRADED,
report.getStatus());
assertEquals("SHADOW_CYCLE_TIMEOUT", report.getReasonCode());
assertTrue(logAppender.list.stream().anyMatch(event ->
event.getFormattedMessage()
.contains("stage=SHADOW_CYCLE")),
"timeout log must identify the blocked stage");
assertFalse(scheduler.runOnce());
assertEquals(callsAfterTimeout, sourceCalls.get());
assertEquals(Collections.emptyList(), gateway.getRecorded());
@@ -198,6 +373,8 @@ class Ft546ShadowFaultIsolationTest {
report.getStatus());
assertEquals("SHADOW_CYCLE_TIMEOUT", report.getReasonCode());
} finally {
schedulerLogger.detachAppender(logAppender);
logAppender.stop();
releaseCycle.countDown();
scheduler.close();
}
@@ -219,10 +396,40 @@ class Ft546ShadowFaultIsolationTest {
assertTrue(configuredKeys.contains("shadow.source.jdbc-url"));
assertTrue(configuredKeys.contains("shadow.evidence.jdbc-url"));
assertTrue(configuredKeys.contains("shadow.cycle.deadline-ms"));
assertFalse(configuredKeys.stream().anyMatch(
Ft546ShadowFaultIsolationTest::isOutboundEndpointKey));
}
@Test
void longCycleDeadlineKeepsSourceReadDeadlineCappedAtFourSeconds() {
MutableClock clock = new MutableClock(NOW);
AtomicReference<Instant> observedDeadline =
new AtomicReference<Instant>();
SqlServerShadowFactReader.FactSource source = new FactSourceAdapter() {
@Override
public Instant readDatabaseClock(Instant deadline) {
observedDeadline.set(deadline);
return NOW;
}
};
RecordingEvidenceRepository evidence = new RecordingEvidenceRepository();
ShadowStartupReport report = report();
Ft546ShadowScheduler scheduler = new Ft546ShadowScheduler(
reader(source, clock),
evidenceCycle(report, evidence, gateway(),
() -> ShadowEvidenceStorageHealth.Check.available(
Long.MAX_VALUE)),
report, clock, Duration.ofSeconds(10));
try {
assertTrue(scheduler.runOnce());
assertEquals(NOW.plusSeconds(4), observedDeadline.get());
} finally {
scheduler.close();
}
}
@Test
void activeQueueCannotAdvanceTheFinishHistoryCursor() {
MutableClock clock = new MutableClock(NOW);

27
ruoyi-wcs-shadow/src/test/java/com/ruoyi/shadow/facts/ShadowFactBatchValidatorTest.java
View File

@@ -28,6 +28,33 @@ class ShadowFactBatchValidatorTest {
assertEquals("FACTS_COMPLETE", result.getReasonCode());
}
@Test
void databaseClockOneMillisecondAheadIsComparable() {
ShadowFactValidation result = validator.validate(batch(
Collections.singletonList(queue("Q-1", "070501", "070502", "1")),
Collections.singletonList(vehicle("1", "070501")),
Arrays.asList(map("070501"), map("070502")),
NOW.minusSeconds(1), NOW.plusMillis(1)), NOW);
assertEquals(ShadowFactValidation.Status.COMPARABLE,
result.getStatus());
}
@Test
void oldLastTaskTimeDoesNotMakeVehicleFactStale() {
ShadowVehicleFact idleVehicle = completeVehicleBuilder("1", "070501")
.capturedAt(NOW.minus(Duration.ofHours(2)))
.build();
ShadowFactValidation result = validator.validate(batch(
Collections.singletonList(queue("Q-1", "070501", "070502", "1")),
Collections.singletonList(idleVehicle),
Arrays.asList(map("070501"), map("070502")),
NOW.minusSeconds(1), NOW.minusMillis(500)), NOW);
assertEquals(ShadowFactValidation.Status.COMPARABLE,
result.getStatus());
}
@Test
void contradictoryVehiclePositionsAreInsufficient() {
ShadowFactValidation result = validator.validate(batch(