lhq/FT546-WCS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(shadow): recover confirmed late evidence commits

Browse Source
This commit is contained in:
zgw
2026-09-14 09:57:57 +08:00
parent 0d53f41f1a
commit 5d5668b04e
11 changed files with 549 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

145
ruoyi-wcs-shadow/src/main/java/com/ruoyi/shadow/kernel/Ft546ShadowScheduler.java
View File

@@ -31,6 +31,7 @@ public final class Ft546ShadowScheduler implements AutoCloseable {
private static final Duration CYCLE_DEADLINE = Duration.ofSeconds(10);
private static final Duration SOURCE_READ_DEADLINE = Duration.ofSeconds(4);
private static final Duration CRASH_BACKOFF = Duration.ofSeconds(1);
private static final Duration COMMIT_RECOVERY_WINDOW = Duration.ofSeconds(30);
private static final String FINISH_QUEUE_VIEW =
"shadow_ro.v_wcs_srgv_queue_finish";
@@ -52,6 +53,9 @@ public final class Ft546ShadowScheduler implements AutoCloseable {
private volatile Instant nextAttemptAt = Instant.EPOCH;
private volatile Thread cycleWorker;
private long lastTimingLogNanos;
private long lastStoppedWarningNanos;
private boolean runtimeInitialized;
private volatile PendingCommit pendingCommit;
public Ft546ShadowScheduler(SqlServerShadowFactReader factReader,
Ft546ShadowCycle shadowCycle,
@@ -107,6 +111,7 @@ public final class Ft546ShadowScheduler implements AutoCloseable {
fixedDelayString = "${shadow.poll.fixed-delay-ms:5000}")
public void runScheduledCycle() {
if (permanentlyStopped.get() || clock.instant().isBefore(nextAttemptAt)) {
warnIfStopped();
publishRuntimeState();
} else {
runOnce();
@@ -116,16 +121,26 @@ public final class Ft546ShadowScheduler implements AutoCloseable {
public boolean runOnce() {
long startedNanos = System.nanoTime();
Instant now = clock.instant();
if (permanentlyStopped.get() || now.isBefore(nextAttemptAt)
if (closed.get() || permanentlyStopped.get() || now.isBefore(nextAttemptAt)
|| !inFlight.compareAndSet(false, true)) {
return false;
}
if (pendingCommit != null) {
try {
return reconcilePendingCommit();
} finally {
try { publishRuntimeState(); } finally { inFlight.set(false); }
}
}
ShadowCycleDiagnostics diagnostics = new ShadowCycleDiagnostics();
lastAttemptAt = now;
diagnostics.enter("RUNTIME_PUBLICATION");
if (!publishRuntimeState()) {
inFlight.set(false);
return false;
if (!runtimeInitialized) {
diagnostics.enter("RUNTIME_PUBLICATION");
if (!publishRuntimeState()) {
inFlight.set(false);
return false;
}
runtimeInitialized = true;
}
cycleWorker = null;
String outcome = "FAILED";
@@ -154,6 +169,19 @@ public final class Ft546ShadowScheduler implements AutoCloseable {
return true;
} catch (TimeoutException timeout) {
CycleStage timedOutStage = activeStage;
Map<String, Object> snapshot = diagnostics.snapshot("TIMEOUT", cycleWorker);
String phase = String.valueOf(snapshot.get("phase"));
if (future != null && ("TRANSACTION_COMMIT".equals(phase) || "RECORDING".equals(phase))) {
// Retain the future. cancel(true) loses the terminal result and can itself
// block inside JDBC cancellation. No new batch starts while this is pending.
pendingCommit = new PendingCommit(future, diagnostics, clock.instant());
statusReport.recordCycleDiagnostics(snapshot);
statusReport.awaitCommitCompletion(clock.instant());
statusReport.recordCollectionFailure("SHADOW_COMMIT_AWAITING_COMPLETION", phase,
timeout, clock.instant(), null);
LOGGER.warn("FT546 shadow commit exceeded cycle budget; awaiting worker completion; diagnostics={}", snapshot);
return false;
}
statusReport.recordCycleDiagnostics(diagnostics.cancel("TIMEOUT", cycleWorker));
if (future != null) future.cancel(true);
return stopPermanently("SHADOW_CYCLE_TIMEOUT", timeout,
@@ -171,17 +199,73 @@ public final class Ft546ShadowScheduler implements AutoCloseable {
} catch (java.util.concurrent.RejectedExecutionException shutdown) {
return stopPermanently("SHADOW_EXECUTOR_STOPPED", shutdown, activeStage);
} finally {
Map<String, Object> completed = diagnostics.finish(outcome, null);
statusReport.recordCycleDiagnostics(completed);
long logTime = System.nanoTime();
if (!permanentlyStopped.get() && (lastTimingLogNanos == 0L
|| logTime - lastTimingLogNanos >= TimeUnit.MINUTES.toNanos(1))) {
LOGGER.info("FT546 shadow cycle diagnostics: {}", completed);
lastTimingLogNanos = logTime;
if (pendingCommit == null) {
Map<String, Object> completed = diagnostics.finish(outcome, null);
statusReport.recordCycleDiagnostics(completed);
long logTime = System.nanoTime();
if (!permanentlyStopped.get() && (lastTimingLogNanos == 0L
|| logTime - lastTimingLogNanos >= TimeUnit.MINUTES.toNanos(1))) {
LOGGER.info("FT546 shadow cycle diagnostics: {}", completed);
lastTimingLogNanos = logTime;
}
activeStage = CycleStage.IDLE;
}
inFlight.set(false);
try { publishRuntimeState(); } finally { inFlight.set(false); }
}
}
private boolean reconcilePendingCommit() {
PendingCommit pending = pendingCommit;
if (!pending.future.isDone()) {
if (pending.expired(clock.instant())) {
statusReport.recordCycleDiagnostics(pending.diagnostics.cancel("COMMIT_UNCONFIRMED", cycleWorker));
return stopPermanently("SHADOW_COMMIT_COMPLETION_UNCONFIRMED",
new TimeoutException("worker did not finish within commit recovery window"), CycleStage.SHADOW_CYCLE);
}
statusReport.recordCycleDiagnostics(pending.diagnostics.snapshot("AWAITING_COMMIT", cycleWorker));
return false;
}
try {
CycleAttempt attempt = pending.future.get();
pendingCommit = null;
activeStage = CycleStage.IDLE;
publishRuntimeState();
if (closed.get() || permanentlyStopped.get()) return false;
if ("DEGRADED".equals(attempt.result.getStatusCode())) {
permanentlyStopped.set(true);
statusReport.markPermanentlyDegraded(attempt.result.getReasonCode(), clock.instant());
statusReport.recordCycleDiagnostics(pending.diagnostics.finish("COMMIT_FAILED", null));
LOGGER.error("FT546 shadow late commit did not complete successfully; collection stopped: {}",
attempt.result.getReasonCode());
return false;
}
// The future has completed normally, including JDBC cleanup and after-commit
// callbacks. Do not repeat the transaction or compare across this delayed gap.
shadowCycle.invalidatePendingComparisons();
cursor = advanceFinishCursor(cursor, attempt.batch);
Long lag = attempt.batch.getDatabaseCapturedAt() == null ? null : Long.valueOf(Math.max(0L,
Duration.between(attempt.batch.getDatabaseCapturedAt(), attempt.completedAt).toMillis()));
statusReport.confirmCommitCompletion(attempt.completedAt, lag);
statusReport.recordCycleDiagnostics(pending.diagnostics.finish("COMMIT_CONFIRMED", null));
nextAttemptAt = Instant.EPOCH;
LOGGER.info("FT546 shadow late commit confirmed; collection can resume from the next batch; completedAt={}",
attempt.completedAt);
return true;
} catch (InterruptedException interrupted) {
Thread.currentThread().interrupt();
return stopPermanently("SHADOW_CYCLE_INTERRUPTED", interrupted, CycleStage.SHADOW_CYCLE);
} catch (ExecutionException | java.util.concurrent.CancellationException failure) {
statusReport.recordCycleDiagnostics(pending.diagnostics.finish("COMMIT_FAILED", null));
return stopPermanently("SHADOW_COMMIT_COMPLETION_FAILED", failure, CycleStage.SHADOW_CYCLE);
}
}
private void warnIfStopped() {
if (!permanentlyStopped.get() || closed.get()) return;
long now = System.nanoTime();
if (lastStoppedWarningNanos == 0L || now - lastStoppedWarningNanos >= TimeUnit.MINUTES.toNanos(1)) {
LOGGER.warn("FT546 shadow collection remains stopped; reason={}; lastCompletedCycle={}; heartbeat is not collection",
statusReport.getReasonCode(), statusReport.getLastCompletedCycle());
lastStoppedWarningNanos = now;
}
}
@@ -199,7 +283,13 @@ public final class Ft546ShadowScheduler implements AutoCloseable {
if (!closed.compareAndSet(false, true)) return;
permanentlyStopped.set(true);
statusReport.markStopped(clock.instant());
cycleExecutor.shutdownNow();
PendingCommit pending = pendingCommit;
if (pending != null) {
pending.diagnostics.cancel("STOPPED", cycleWorker);
cycleExecutor.shutdown();
} else {
cycleExecutor.shutdownNow();
}
publishRuntimeState();
}
@@ -226,8 +316,8 @@ public final class Ft546ShadowScheduler implements AutoCloseable {
if (sourceDeadline.isAfter(overallDeadline)) sourceDeadline = overallDeadline;
ShadowFactBatch batch = factReader.read(cursor, sourceDeadline);
activeStage = CycleStage.SHADOW_CYCLE;
return new CycleAttempt(batch,
shadowCycle.run(batch, clock.instant(), diagnostics));
ShadowCycleResult result = shadowCycle.run(batch, clock.instant(), diagnostics);
return new CycleAttempt(batch, result, clock.instant());
}
private boolean handleFailure(Throwable failure) {
@@ -353,10 +443,29 @@ public final class Ft546ShadowScheduler implements AutoCloseable {
private static final class CycleAttempt {
private final ShadowFactBatch batch;
private final ShadowCycleResult result;
private final Instant completedAt;
private CycleAttempt(ShadowFactBatch batch, ShadowCycleResult result) {
private CycleAttempt(ShadowFactBatch batch, ShadowCycleResult result, Instant completedAt) {
this.batch = batch;
this.result = result;
this.completedAt = completedAt;
}
}
private static final class PendingCommit {
private final Future<CycleAttempt> future;
private final ShadowCycleDiagnostics diagnostics;
private final Instant deadline;
private final long startedNanos = System.nanoTime();
private PendingCommit(Future<CycleAttempt> future, ShadowCycleDiagnostics diagnostics, Instant startedAt) {
this.future = future;
this.diagnostics = diagnostics;
this.deadline = startedAt.plus(COMMIT_RECOVERY_WINDOW);
}
private boolean expired(Instant now) {
return !now.isBefore(deadline) || System.nanoTime() - startedNanos >= COMMIT_RECOVERY_WINDOW.toNanos();
}
}
}