lhq/FT546-WCS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(shadow): harden production observation cycle

Browse Source
This commit is contained in:
zgw
2026-09-03 11:10:22 +08:00
parent 0080befff4
commit 9829b9c0fa
7 changed files with 325 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

75
ruoyi-wcs-shadow/src/main/java/com/ruoyi/shadow/kernel/Ft546ShadowScheduler.java
View File

@@ -26,7 +26,8 @@ import java.util.concurrent.atomic.AtomicBoolean;
public final class Ft546ShadowScheduler implements AutoCloseable {
private static final Logger LOGGER =
LoggerFactory.getLogger(Ft546ShadowScheduler.class);
private static final Duration CYCLE_DEADLINE = Duration.ofSeconds(4);
private static final Duration CYCLE_DEADLINE = Duration.ofSeconds(10);
private static final Duration SOURCE_READ_DEADLINE = Duration.ofSeconds(4);
private static final Duration CRASH_BACKOFF = Duration.ofSeconds(1);
private static final String FINISH_QUEUE_VIEW =
"shadow_ro.v_wcs_srgv_queue_finish";
@@ -36,9 +37,11 @@ public final class Ft546ShadowScheduler implements AutoCloseable {
private final ShadowStartupReport statusReport;
private final Clock clock;
private final Duration cycleDeadline;
private final Duration sourceReadDeadline;
private final ExecutorService cycleExecutor;
private final AtomicBoolean inFlight = new AtomicBoolean();
private final AtomicBoolean permanentlyStopped = new AtomicBoolean();
private volatile CycleStage activeStage = CycleStage.IDLE;
private volatile ShadowReadCursor cursor = ShadowReadCursor.initial();
private volatile Instant nextAttemptAt = Instant.EPOCH;
@@ -54,12 +57,23 @@ public final class Ft546ShadowScheduler implements AutoCloseable {
ShadowStartupReport statusReport,
Clock clock,
Duration cycleDeadline) {
this(factReader, shadowCycle, statusReport, clock, cycleDeadline,
shorter(cycleDeadline, SOURCE_READ_DEADLINE));
}
Ft546ShadowScheduler(SqlServerShadowFactReader factReader,
Ft546ShadowCycle shadowCycle,
ShadowStartupReport statusReport,
Clock clock,
Duration cycleDeadline,
Duration sourceReadDeadline) {
this.factReader = Objects.requireNonNull(factReader, "factReader");
this.shadowCycle = Objects.requireNonNull(shadowCycle, "shadowCycle");
this.statusReport = Objects.requireNonNull(statusReport,
"statusReport");
this.clock = Objects.requireNonNull(clock, "clock");
this.cycleDeadline = requirePositive(cycleDeadline);
this.sourceReadDeadline = requirePositive(sourceReadDeadline);
this.cycleExecutor = Executors.newSingleThreadExecutor(
shadowCycleThreadFactory());
}
@@ -94,16 +108,21 @@ public final class Ft546ShadowScheduler implements AutoCloseable {
nextAttemptAt = Instant.EPOCH;
return true;
} catch (TimeoutException timeout) {
CycleStage timedOutStage = activeStage;
future.cancel(true);
return stopPermanently("SHADOW_CYCLE_TIMEOUT", timeout);
return stopPermanently("SHADOW_CYCLE_TIMEOUT", timeout,
timedOutStage);
} catch (InterruptedException interrupted) {
CycleStage interruptedStage = activeStage;
future.cancel(true);
Thread.currentThread().interrupt();
return stopPermanently("SHADOW_CYCLE_INTERRUPTED", interrupted);
return stopPermanently("SHADOW_CYCLE_INTERRUPTED", interrupted,
interruptedStage);
} catch (ExecutionException wrapped) {
return handleFailure(wrapped.getCause());
} finally {
inFlight.set(false);
activeStage = CycleStage.IDLE;
}
}
@@ -123,15 +142,23 @@ public final class Ft546ShadowScheduler implements AutoCloseable {
}
private CycleAttempt executeCycle(Instant startedAt) {
activeStage = CycleStage.SOURCE_READ;
ShadowFactBatch batch = factReader.read(cursor,
startedAt.plus(cycleDeadline));
return new CycleAttempt(batch, shadowCycle.run(batch, clock.instant()));
startedAt.plus(sourceReadDeadline));
activeStage = CycleStage.SHADOW_CYCLE;
return new CycleAttempt(batch,
shadowCycle.run(batch, clock.instant()));
}
private boolean handleFailure(Throwable failure) {
if (failure instanceof ShadowFactReadException) {
ShadowFactReadException readFailure =
(ShadowFactReadException) failure;
String timeoutReason = sourceTimeoutReason(readFailure);
if (timeoutReason != null) {
return stopPermanently(timeoutReason, readFailure,
CycleStage.SOURCE_READ);
}
Instant failedAt = clock.instant();
statusReport.markDegraded(readFailure.getReasonCode(), failedAt);
nextAttemptAt = failedAt.plus(readFailure.getRetryAfter());
@@ -157,15 +184,41 @@ public final class Ft546ShadowScheduler implements AutoCloseable {
return false;
}
private boolean stopPermanently(String reasonCode, Exception failure) {
private static String sourceTimeoutReason(
ShadowFactReadException failure) {
String reasonCode = failure.getReasonCode();
if ("SOURCE_QUERY_TIMEOUT".equals(reasonCode)
|| "SOURCE_BATCH_DEADLINE_EXCEEDED".equals(reasonCode)) {
return reasonCode;
}
Throwable cause = failure.getCause();
while (cause != null) {
String type = cause.getClass().getSimpleName()
.toUpperCase(java.util.Locale.ROOT);
if (type.contains("TIMEOUT")) {
return "SOURCE_QUERY_TIMEOUT";
}
cause = cause.getCause();
}
return null;
}
private boolean stopPermanently(String reasonCode, Exception failure,
CycleStage stage) {
permanentlyStopped.set(true);
nextAttemptAt = Instant.EPOCH;
statusReport.markPermanentlyDegraded(reasonCode, clock.instant());
LOGGER.error("FT546 shadow collection stopped: {}", reasonCode,
failure);
LOGGER.error("FT546 shadow collection stopped: {}; stage={}",
reasonCode, stage, failure);
return false;
}
private static Duration shorter(Duration left, Duration right) {
Duration first = requirePositive(left);
Duration second = requirePositive(right);
return first.compareTo(second) <= 0 ? first : second;
}
private static Duration requirePositive(Duration value) {
Duration duration = Objects.requireNonNull(value, "cycleDeadline");
if (duration.isZero() || duration.isNegative()
@@ -184,6 +237,12 @@ public final class Ft546ShadowScheduler implements AutoCloseable {
};
}
private enum CycleStage {
IDLE,
SOURCE_READ,
SHADOW_CYCLE
}
private static ShadowReadCursor advanceFinishCursor(
ShadowReadCursor current, ShadowFactBatch batch) {
Instant latestDate = current.getLastCreateDate();