lhq/FT546-WCS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(shadow): recover completed read timeouts and pair issued observations

Browse Source
This commit is contained in:
zgw
2026-09-09 10:00:43 +08:00
parent af8b5f1830
commit edb13450a6
14 changed files with 948 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
ruoyi-wcs-shadow/src/main/java/com/ruoyi/shadow/kernel/Ft546ShadowScheduler.java
View File

@@ -114,6 +114,7 @@ public final class Ft546ShadowScheduler implements AutoCloseable {
}
public boolean runOnce() {
long startedNanos = System.nanoTime();
Instant now = clock.instant();
if (permanentlyStopped.get() || now.isBefore(nextAttemptAt)
|| !inFlight.compareAndSet(false, true)) {
@@ -121,6 +122,7 @@ public final class Ft546ShadowScheduler implements AutoCloseable {
}
ShadowCycleDiagnostics diagnostics = new ShadowCycleDiagnostics();
lastAttemptAt = now;
diagnostics.enter("RUNTIME_PUBLICATION");
if (!publishRuntimeState()) {
inFlight.set(false);
return false;
@@ -130,9 +132,13 @@ public final class Ft546ShadowScheduler implements AutoCloseable {
Future<CycleAttempt> future = null;
try {
if (permanentlyStopped.get()) return false;
future = cycleExecutor.submit(() -> executeCycle(now, diagnostics));
CycleAttempt attempt = future.get(cycleDeadline.toMillis(),
TimeUnit.MILLISECONDS);
Instant overallDeadline = now.plus(cycleDeadline);
long remainingNanos = Math.min(cycleDeadline.toNanos() - (System.nanoTime() - startedNanos),
Duration.between(clock.instant(), overallDeadline).toNanos());
if (remainingNanos <= 0L) throw new TimeoutException("runtime publication exhausted cycle budget");
diagnostics.enter("QUEUED");
future = cycleExecutor.submit(() -> executeCycle(overallDeadline, diagnostics));
CycleAttempt attempt = future.get(remainingNanos, TimeUnit.NANOSECONDS);
outcome = attempt.result.getStatusCode();
statusReport.recordCycleDiagnostics(diagnostics.finish(outcome, null));
if ("DEGRADED".equals(attempt.result.getStatusCode())) {
@@ -212,32 +218,37 @@ public final class Ft546ShadowScheduler implements AutoCloseable {
}
}
private CycleAttempt executeCycle(Instant startedAt, ShadowCycleDiagnostics diagnostics) {
private CycleAttempt executeCycle(Instant overallDeadline, ShadowCycleDiagnostics diagnostics) {
cycleWorker = Thread.currentThread();
activeStage = CycleStage.SOURCE_READ;
diagnostics.enter("SOURCE_READ");
ShadowFactBatch batch = factReader.read(cursor,
startedAt.plus(sourceReadDeadline));
Instant sourceDeadline = clock.instant().plus(sourceReadDeadline);
if (sourceDeadline.isAfter(overallDeadline)) sourceDeadline = overallDeadline;
ShadowFactBatch batch = factReader.read(cursor, sourceDeadline);
activeStage = CycleStage.SHADOW_CYCLE;
return new CycleAttempt(batch,
shadowCycle.run(batch, clock.instant(), diagnostics));
}
private boolean handleFailure(Throwable failure) {
shadowCycle.invalidatePendingComparisons();
if (failure instanceof ShadowFactReadException) {
ShadowFactReadException readFailure =
(ShadowFactReadException) failure;
String timeoutReason = sourceTimeoutReason(readFailure);
if (timeoutReason != null) {
return stopPermanently(timeoutReason, readFailure,
CycleStage.SOURCE_READ);
if ("SOURCE_TRANSACTION_FAILED".equals(readFailure.getReasonCode())
|| "SOURCE_CYCLE_ALREADY_IN_FLIGHT".equals(readFailure.getReasonCode())) {
return stopPermanently(readFailure.getReasonCode(), readFailure, CycleStage.SOURCE_READ);
}
// ExecutionException is delivered only after the reader and its transaction have unwound.
// A watchdog timeout still stops permanently because cancellation does not prove termination.
String timeoutReason = sourceTimeoutReason(readFailure);
String reason = timeoutReason == null ? readFailure.getReasonCode() : timeoutReason;
Instant failedAt = clock.instant();
statusReport.markDegraded(readFailure.getReasonCode(), failedAt);
nextAttemptAt = failedAt.plus(readFailure.getRetryAfter());
Duration backoff = readFailure.getRetryAfter().isZero() ? CRASH_BACKOFF : readFailure.getRetryAfter();
statusReport.markDegraded(reason, failedAt);
nextAttemptAt = failedAt.plus(backoff);
LOGGER.warn("FT546 shadow source read failed: {}; retry after {} ms",
readFailure.getReasonCode(),
readFailure.getRetryAfter().toMillis());
reason, backoff.toMillis());
if (readFailure.getCause() != null) {
LOGGER.warn("FT546 shadow source read root cause",
readFailure.getCause());
@@ -278,6 +289,7 @@ public final class Ft546ShadowScheduler implements AutoCloseable {
private boolean stopPermanently(String reasonCode, Exception failure,
CycleStage stage) {
shadowCycle.invalidatePendingComparisons();
permanentlyStopped.set(true);
nextAttemptAt = Instant.EPOCH;
statusReport.markPermanentlyDegraded(reasonCode, clock.instant());