fix(shadow): repair field routing and harden evidence collection
This commit is contained in:
@@ -62,21 +62,31 @@ public class ShadowEvidenceConfiguration {
|
||||
}
|
||||
|
||||
@Bean
|
||||
public ShadowEvidenceRepository shadowEvidenceRepository(
|
||||
public JdbcShadowEvidenceRepository shadowEvidenceRepository(
|
||||
@Qualifier("shadowEvidenceJdbc") JdbcTemplate evidenceJdbc,
|
||||
@Qualifier("shadowEvidenceTx")
|
||||
DataSourceTransactionManager evidenceTx,
|
||||
ObjectMapper objectMapper, ShadowRunEvidence run) {
|
||||
ObjectMapper objectMapper, ShadowRunEvidence run, ShadowProperties properties) {
|
||||
return new JdbcShadowEvidenceRepository(evidenceJdbc,
|
||||
new TransactionTemplate(evidenceTx), objectMapper, run);
|
||||
new TransactionTemplate(evidenceTx), objectMapper, run,
|
||||
properties.getEvidence().getMaximumBatchPayloadBytes());
|
||||
}
|
||||
|
||||
@Bean
|
||||
public ShadowEvidenceStorageHealth shadowEvidenceStorageHealth(
|
||||
ShadowEvidenceRepository repository,
|
||||
@Qualifier("shadowEvidenceJdbc") JdbcTemplate evidenceJdbc,
|
||||
ShadowProperties properties) {
|
||||
return new JdbcShadowEvidenceStorageHealth(repository,
|
||||
Paths.get(properties.getEvidence().getOutputDirectory()),
|
||||
properties.getEvidence().getMinimumFreeBytes().longValue());
|
||||
properties.getEvidence().getMinimumFreeBytes().longValue(),
|
||||
() -> {
|
||||
Long allocated = evidenceJdbc.queryForObject(
|
||||
"SELECT SUM(CAST(size AS bigint)) * 8192 FROM sys.database_files", Long.class);
|
||||
if (allocated == null || allocated <= 0L) {
|
||||
throw new IllegalStateException("evidence database allocation unavailable");
|
||||
}
|
||||
return allocated;
|
||||
}, properties.getEvidence().getMaximumDatabaseBytes());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import com.ruoyi.shadow.evidence.ShadowCycleEvidenceCollector;
|
||||
import com.ruoyi.shadow.evidence.ShadowEvidenceRepository;
|
||||
import com.ruoyi.shadow.evidence.ShadowEvidenceStorageHealth;
|
||||
import com.ruoyi.shadow.evidence.ShadowRunEvidence;
|
||||
import com.ruoyi.shadow.evidence.JdbcShadowEvidenceRepository;
|
||||
import com.ruoyi.shadow.facts.ShadowFactBatchValidator;
|
||||
import com.ruoyi.shadow.kernel.Ft546ShadowCycle;
|
||||
import com.ruoyi.shadow.kernel.Ft546ShadowScheduler;
|
||||
@@ -43,6 +44,7 @@ import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
@Configuration(proxyBeanMethods = false)
|
||||
@@ -69,7 +71,8 @@ public class ShadowKernelConfiguration {
|
||||
@Bean
|
||||
public IdGenerator shadowIdGenerator() {
|
||||
AtomicLong sequence = new AtomicLong();
|
||||
return () -> "SHADOW-" + sequence.incrementAndGet();
|
||||
String runPrefix = "SHADOW-" + UUID.randomUUID() + "-";
|
||||
return () -> runPrefix + sequence.incrementAndGet();
|
||||
}
|
||||
|
||||
@Bean(name = "shadowClock")
|
||||
@@ -173,11 +176,12 @@ public class ShadowKernelConfiguration {
|
||||
ShadowEvidenceStorageHealth shadowEvidenceStorageHealth,
|
||||
ShadowEvidenceRepository shadowEvidenceRepository,
|
||||
ShadowCycleEvidenceCollector shadowCycleEvidenceCollector,
|
||||
ShadowStartupReport shadowStartupReport) {
|
||||
ShadowStartupReport shadowStartupReport,
|
||||
@Qualifier("shadowClock") Clock clock) {
|
||||
return new Ft546ShadowCycle(validator, assembler::assemble,
|
||||
dispatchCycle, commandGateway, shadowEvidenceStorageHealth,
|
||||
shadowEvidenceRepository, shadowCycleEvidenceCollector,
|
||||
shadowStartupReport);
|
||||
shadowStartupReport, clock);
|
||||
}
|
||||
|
||||
@Bean(destroyMethod = "close")
|
||||
@@ -185,11 +189,12 @@ public class ShadowKernelConfiguration {
|
||||
SqlServerShadowFactReader factReader,
|
||||
Ft546ShadowCycle shadowCycle,
|
||||
ShadowStartupReport shadowStartupReport,
|
||||
JdbcShadowEvidenceRepository evidenceRepository,
|
||||
@Qualifier("shadowClock") Clock clock,
|
||||
@Value("${shadow.cycle.deadline-ms:10000}")
|
||||
long cycleDeadlineMs) {
|
||||
return new Ft546ShadowScheduler(factReader, shadowCycle,
|
||||
shadowStartupReport, clock,
|
||||
Duration.ofMillis(cycleDeadlineMs));
|
||||
Duration.ofMillis(cycleDeadlineMs), evidenceRepository::saveRuntimeState);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -29,6 +29,8 @@ public class ShadowProperties {
|
||||
public static final class EvidenceEndpoint extends JdbcEndpoint {
|
||||
private String outputDirectory;
|
||||
private Long minimumFreeBytes;
|
||||
private long maximumDatabaseBytes = 16L * 1024L * 1024L * 1024L;
|
||||
private long maximumBatchPayloadBytes = 32L * 1024L * 1024L;
|
||||
private final RunMetadata run = new RunMetadata();
|
||||
|
||||
public String getOutputDirectory() { return outputDirectory; }
|
||||
@@ -40,6 +42,10 @@ public class ShadowProperties {
|
||||
this.minimumFreeBytes = minimumFreeBytes;
|
||||
}
|
||||
public RunMetadata getRun() { return run; }
|
||||
public long getMaximumDatabaseBytes() { return maximumDatabaseBytes; }
|
||||
public void setMaximumDatabaseBytes(long value) { maximumDatabaseBytes = value; }
|
||||
public long getMaximumBatchPayloadBytes() { return maximumBatchPayloadBytes; }
|
||||
public void setMaximumBatchPayloadBytes(long value) { maximumBatchPayloadBytes = value; }
|
||||
}
|
||||
|
||||
public static final class RunMetadata {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package com.ruoyi.shadow.evidence;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.ruoyi.ft546.protocol.Ft546RegisterWrite;
|
||||
import org.springframework.dao.DataAccessException;
|
||||
import org.springframework.dao.DuplicateKeyException;
|
||||
@@ -28,16 +29,27 @@ public final class JdbcShadowEvidenceRepository
|
||||
private final TransactionTemplate transaction;
|
||||
private final ObjectMapper objectMapper;
|
||||
private final ShadowRunEvidence run;
|
||||
private final long maximumBatchPayloadBytes;
|
||||
|
||||
public JdbcShadowEvidenceRepository(JdbcTemplate jdbc,
|
||||
TransactionTemplate transaction,
|
||||
ObjectMapper objectMapper,
|
||||
ShadowRunEvidence run) {
|
||||
this(jdbc, transaction, objectMapper, run, 32L * 1024L * 1024L);
|
||||
}
|
||||
|
||||
public JdbcShadowEvidenceRepository(JdbcTemplate jdbc, TransactionTemplate transaction,
|
||||
ObjectMapper objectMapper, ShadowRunEvidence run,
|
||||
long maximumBatchPayloadBytes) {
|
||||
this.jdbc = Objects.requireNonNull(jdbc, "jdbc");
|
||||
this.transaction = Objects.requireNonNull(transaction, "transaction");
|
||||
this.objectMapper = Objects.requireNonNull(objectMapper,
|
||||
"objectMapper");
|
||||
this.run = Objects.requireNonNull(run, "run");
|
||||
if (maximumBatchPayloadBytes <= 0L) {
|
||||
throw new IllegalArgumentException("maximumBatchPayloadBytes must be positive");
|
||||
}
|
||||
this.maximumBatchPayloadBytes = maximumBatchPayloadBytes;
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -174,6 +186,55 @@ public final class JdbcShadowEvidenceRepository
|
||||
}
|
||||
}
|
||||
|
||||
public void saveRuntimeState(Map<String, Object> state) {
|
||||
Objects.requireNonNull(state, "state");
|
||||
String status = Objects.toString(state.get("status"), "");
|
||||
if (!java.util.Arrays.asList("STARTING", "READY", "DEGRADED", "STOPPED").contains(status)
|
||||
|| !"RUN_HEARTBEAT".equals(state.get("kind"))
|
||||
|| !(state.get("collectionStopped") instanceof Boolean)) {
|
||||
throw new ShadowEvidenceException("EVIDENCE_RUNTIME_STATE_INVALID");
|
||||
}
|
||||
Instant heartbeatAt = Instant.parse(Objects.toString(state.get("heartbeatAt")));
|
||||
Map<String, Object> values = new LinkedHashMap<String, Object>(state);
|
||||
values.put("runId", run.getRunId().toString());
|
||||
String json = ShadowEvidenceDigest.canonicalJson(objectMapper, values);
|
||||
if (json.length() > 32768) {
|
||||
throw new ShadowEvidenceException("EVIDENCE_RUNTIME_PAYLOAD_LIMIT");
|
||||
}
|
||||
String key = "FT546:RUN:" + run.getRunId();
|
||||
transaction.execute(tx -> {
|
||||
ensureRun();
|
||||
List<Map<String, Object>> rows = jdbc.queryForList(
|
||||
"SELECT snapshot_version,state_json FROM shadow_traffic_state WHERE state_key=?", key);
|
||||
if (rows.isEmpty()) {
|
||||
jdbc.update("INSERT INTO shadow_traffic_state "
|
||||
+ "(state_key,snapshot_version,state_json,updated_at) VALUES (?,?,?,?)",
|
||||
key, 1L, json, Timestamp.from(heartbeatAt));
|
||||
} else {
|
||||
Map<String, Object> previous = rows.get(0);
|
||||
JsonNode previousState;
|
||||
try {
|
||||
previousState = objectMapper.readTree(Objects.toString(previous.get("state_json")));
|
||||
} catch (Exception failure) {
|
||||
throw new ShadowEvidenceException("EVIDENCE_RUNTIME_STATE_INVALID", failure);
|
||||
}
|
||||
if (previousState.path("collectionStopped").asBoolean()
|
||||
&& !Boolean.TRUE.equals(state.get("collectionStopped"))) {
|
||||
throw new ShadowEvidenceException("EVIDENCE_RUNTIME_STATE_REGRESSION");
|
||||
}
|
||||
long version = ((Number) previous.get("snapshot_version")).longValue();
|
||||
int updated = jdbc.update("UPDATE shadow_traffic_state "
|
||||
+ "SET snapshot_version=?,state_json=?,updated_at=? "
|
||||
+ "WHERE state_key=? AND snapshot_version=?",
|
||||
Math.addExact(version, 1L), json, Timestamp.from(heartbeatAt), key, version);
|
||||
if (updated != 1) throw new ShadowEvidenceException("EVIDENCE_RUNTIME_STATE_CHANGED");
|
||||
}
|
||||
jdbc.update("UPDATE shadow_run SET status=? WHERE run_id=?",
|
||||
"READY".equals(status) ? "RUNNING" : status, run.getRunId());
|
||||
return null;
|
||||
});
|
||||
}
|
||||
|
||||
private List<SavedCycle> saveAllInTransaction(
|
||||
List<ShadowCycleEvidence> evidence) {
|
||||
ensureRun();
|
||||
@@ -189,6 +250,7 @@ public final class JdbcShadowEvidenceRepository
|
||||
result.add(verifyReplay(existing, item));
|
||||
}
|
||||
}
|
||||
verifyPayloadBudget(inserts);
|
||||
for (ShadowCycleEvidence item : inserts) {
|
||||
insert(item);
|
||||
}
|
||||
@@ -200,6 +262,33 @@ public final class JdbcShadowEvidenceRepository
|
||||
return Collections.unmodifiableList(result);
|
||||
}
|
||||
|
||||
private void verifyPayloadBudget(List<ShadowCycleEvidence> inserts) {
|
||||
long remaining = maximumBatchPayloadBytes;
|
||||
for (ShadowCycleEvidence item : inserts) {
|
||||
remaining = consumePayload(remaining, item.getInputJson());
|
||||
remaining = consumePayload(remaining, item.getLegacyJson());
|
||||
remaining = consumePayload(remaining, item.getLegacyPathInfo());
|
||||
remaining = consumePayload(remaining, item.getDecisionJson());
|
||||
for (Ft546RegisterWrite write : item.getEncodedWrites()) {
|
||||
remaining = consumePayload(remaining,
|
||||
ShadowEvidenceDigest.canonicalJson(objectMapper, write.getValues()));
|
||||
}
|
||||
for (ShadowDifference difference : item.getDifferences()) {
|
||||
remaining = consumePayload(remaining, difference.getExpectedJson());
|
||||
remaining = consumePayload(remaining, difference.getActualJson());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static long consumePayload(long remaining, String value) {
|
||||
// SQL Server nvarchar stores UTF-16 code units; page/index/log overhead is separate.
|
||||
long bytes = value == null ? 0L : 2L * value.length();
|
||||
if (bytes > remaining) {
|
||||
throw new ShadowEvidenceException("EVIDENCE_BATCH_PAYLOAD_LIMIT");
|
||||
}
|
||||
return remaining - bytes;
|
||||
}
|
||||
|
||||
private void insert(ShadowCycleEvidence evidence) {
|
||||
jdbc.update("INSERT INTO shadow_cycle "
|
||||
+ "(cycle_id,run_id,input_batch_hash,parent_task_id,"
|
||||
|
||||
@@ -4,16 +4,25 @@ import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.Objects;
|
||||
import java.util.function.LongSupplier;
|
||||
|
||||
public final class JdbcShadowEvidenceStorageHealth
|
||||
implements ShadowEvidenceStorageHealth {
|
||||
private final ShadowEvidenceRepository repository;
|
||||
private final Path outputDirectory;
|
||||
private final long minimumFreeBytes;
|
||||
private final LongSupplier allocatedDatabaseBytes;
|
||||
private final long maximumDatabaseBytes;
|
||||
|
||||
public JdbcShadowEvidenceStorageHealth(
|
||||
ShadowEvidenceRepository repository, Path outputDirectory,
|
||||
long minimumFreeBytes) {
|
||||
this(repository, outputDirectory, minimumFreeBytes, () -> 0L, Long.MAX_VALUE);
|
||||
}
|
||||
|
||||
public JdbcShadowEvidenceStorageHealth(
|
||||
ShadowEvidenceRepository repository, Path outputDirectory,
|
||||
long minimumFreeBytes, LongSupplier allocatedDatabaseBytes, long maximumDatabaseBytes) {
|
||||
this.repository = Objects.requireNonNull(repository, "repository");
|
||||
this.outputDirectory = Objects.requireNonNull(outputDirectory,
|
||||
"outputDirectory").toAbsolutePath().normalize();
|
||||
@@ -22,6 +31,11 @@ public final class JdbcShadowEvidenceStorageHealth
|
||||
"minimumFreeBytes must not be negative");
|
||||
}
|
||||
this.minimumFreeBytes = minimumFreeBytes;
|
||||
this.allocatedDatabaseBytes = Objects.requireNonNull(allocatedDatabaseBytes, "allocatedDatabaseBytes");
|
||||
if (maximumDatabaseBytes <= 0L) {
|
||||
throw new IllegalArgumentException("maximumDatabaseBytes must be positive");
|
||||
}
|
||||
this.maximumDatabaseBytes = maximumDatabaseBytes;
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -29,6 +43,17 @@ public final class JdbcShadowEvidenceStorageHealth
|
||||
if (!repository.isHealthy()) {
|
||||
return Check.unavailable("EVIDENCE_DATABASE_UNAVAILABLE");
|
||||
}
|
||||
try {
|
||||
long allocated = allocatedDatabaseBytes.getAsLong();
|
||||
if (allocated < 0L) {
|
||||
return Check.unavailable("EVIDENCE_DATABASE_SIZE_CHECK_FAILED");
|
||||
}
|
||||
if (allocated >= maximumDatabaseBytes) {
|
||||
return Check.unavailable("EVIDENCE_DATABASE_SIZE_LIMIT");
|
||||
}
|
||||
} catch (RuntimeException failure) {
|
||||
return Check.unavailable("EVIDENCE_DATABASE_SIZE_CHECK_FAILED");
|
||||
}
|
||||
try {
|
||||
long usableBytes = Files.getFileStore(outputDirectory)
|
||||
.getUsableSpace();
|
||||
|
||||
@@ -195,6 +195,7 @@ public final class ShadowCycleEvidence {
|
||||
row.put("tableName", value.getTableName());
|
||||
row.put("taskNo", value.getTaskNo());
|
||||
row.put("taskType", value.getTaskType());
|
||||
row.put("theFloor", value.getTheFloor());
|
||||
row.put("turnPath", value.getTurnPath());
|
||||
result.add(row);
|
||||
}
|
||||
|
||||
@@ -121,13 +121,21 @@ public final class LegacyQueueProtocolProjector {
|
||||
}
|
||||
|
||||
private static List<Integer> ordinaryPath(ShadowQueueFact queue) {
|
||||
List<Integer> executedPath = executedPath(queue.getPathInfo());
|
||||
if (executedPath == null || executedPath.size() < 2) {
|
||||
List<String> executedPath = executedPath(queue.getPathInfo());
|
||||
if (executedPath == null || executedPath.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
if (executedPath.size() == 1
|
||||
&& !("3".equals(trim(queue.getTaskType()))
|
||||
&& "3".equals(trim(queue.getDefault02()))
|
||||
&& trim(queue.getDefault08()).isEmpty()
|
||||
&& site(executedPath.get(0)).equals(site(queue.getStartSite()))
|
||||
&& site(executedPath.get(0)).equals(site(queue.getEndSite())))) {
|
||||
return null;
|
||||
}
|
||||
List<Integer> result = new ArrayList<Integer>();
|
||||
result.add(executedPath.get(0));
|
||||
result.add(executedPath.get(executedPath.size() - 1));
|
||||
result.add(site(executedPath.get(0)));
|
||||
result.add(site(executedPath.get(executedPath.size() - 1)));
|
||||
if (notBlank(queue.getTurnPath())) {
|
||||
for (String token : queue.getTurnPath().split(",")) {
|
||||
if (!notBlank(token)) {
|
||||
@@ -149,21 +157,23 @@ public final class LegacyQueueProtocolProjector {
|
||||
return result;
|
||||
}
|
||||
|
||||
private static List<Integer> executedPath(String value) {
|
||||
private static List<String> executedPath(String value) {
|
||||
if (!notBlank(value)) {
|
||||
return null;
|
||||
}
|
||||
List<Integer> result = new ArrayList<Integer>();
|
||||
List<String> result = new ArrayList<String>();
|
||||
for (String token : value.split(",")) {
|
||||
if (!notBlank(token)) {
|
||||
continue;
|
||||
}
|
||||
Integer point = site(token);
|
||||
if (point == null) {
|
||||
if (point == null && !token.trim().matches("(?:TD[0-9]{3}|WY[0-9]{4})")) {
|
||||
return null;
|
||||
}
|
||||
result.add(point);
|
||||
result.add(token.trim());
|
||||
}
|
||||
if (result.isEmpty() || site(result.get(0)) == null
|
||||
|| site(result.get(result.size() - 1)) == null) return null;
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
@@ -47,6 +47,11 @@ public final class ShadowFactBatch {
|
||||
public List<ShadowMapFact> getMaps() { return maps; }
|
||||
|
||||
public ShadowQueueFact queue(String guid) {
|
||||
for (ShadowQueueFact fact : queues) {
|
||||
if (fact.isActive() && fact.getGuid().equals(guid)) {
|
||||
return fact;
|
||||
}
|
||||
}
|
||||
for (ShadowQueueFact fact : queues) {
|
||||
if (fact.getGuid().equals(guid)) {
|
||||
return fact;
|
||||
|
||||
@@ -110,14 +110,23 @@ public final class ShadowFactBatchValidator {
|
||||
return ShadowFactValidation.insufficient("MISSING_MAP_FACTS");
|
||||
}
|
||||
for (ShadowQueueFact queue : batch.getQueues()) {
|
||||
// Finished rows are historical versions, not current dispatch inputs.
|
||||
if (!queue.isActive()) {
|
||||
continue;
|
||||
}
|
||||
if (queue.getCreatedAt() == null) {
|
||||
return ShadowFactValidation.insufficient(
|
||||
"MISSING_QUEUE_CREATED_AT:" + queue.getTableName()
|
||||
+ ':' + queue.getGuid());
|
||||
}
|
||||
String missing = firstMissingSite(sites, queue.getStartSite(),
|
||||
queue.getEndSite(), queue.getCargoPickSite(),
|
||||
queue.getCargoPlaceSite());
|
||||
String missing;
|
||||
try {
|
||||
missing = firstMissingSite(sites, ShadowQueueEndpoints.start(queue, batch),
|
||||
ShadowQueueEndpoints.end(queue, batch), queue.getCargoPickSite(),
|
||||
queue.getCargoPlaceSite());
|
||||
} catch (IllegalArgumentException unresolved) {
|
||||
return ShadowFactValidation.insufficient(unresolved.getMessage());
|
||||
}
|
||||
if (missing != null) {
|
||||
return ShadowFactValidation.insufficient("MISSING_MAP_SITE:" + missing);
|
||||
}
|
||||
@@ -141,6 +150,9 @@ public final class ShadowFactBatchValidator {
|
||||
private static ShadowFactValidation queueMutations(ShadowFactBatch batch) {
|
||||
Map<String, String> hashes = new HashMap<String, String>();
|
||||
for (ShadowQueueFact queue : batch.getSourceQueues()) {
|
||||
if (!queue.isActive()) {
|
||||
continue;
|
||||
}
|
||||
String key = queue.getTableName() + ':' + queue.getGuid();
|
||||
String existing = hashes.putIfAbsent(key, queue.getContentHash());
|
||||
if (existing != null && !existing.equals(queue.getContentHash())) {
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
package com.ruoyi.shadow.facts;
|
||||
|
||||
/** Resolves legacy business placeholders without modifying captured evidence. */
|
||||
public final class ShadowQueueEndpoints {
|
||||
private ShadowQueueEndpoints() { }
|
||||
|
||||
public static String start(ShadowQueueFact queue, ShadowFactBatch batch) {
|
||||
String site = trimmed(queue.getStartSite());
|
||||
if (!"tsj".equals(site)) return site;
|
||||
if ("4".equals(trimmed(queue.getDefault02()))
|
||||
&& ("1".equals(queue.getTaskType()) || "2".equals(queue.getTaskType()))) {
|
||||
int floor = sourceFloor(queue);
|
||||
// sendPointToPointTask starts at the assigned vehicle's live currSite.
|
||||
for (ShadowVehicleFact vehicle : batch.getVehicles()) {
|
||||
if (vehicle.getDeviceId().equals(queue.getAgvId())
|
||||
&& Integer.valueOf(floor).equals(vehicle.getCurrentFloor())) {
|
||||
return requireFloorSite(vehicle.getCurrentSite(), floor, batch);
|
||||
}
|
||||
}
|
||||
}
|
||||
throw new IllegalArgumentException("UNRESOLVED_TSJ_START:" + queue.getGuid());
|
||||
}
|
||||
|
||||
public static String end(ShadowQueueFact queue, ShadowFactBatch batch) {
|
||||
String site = trimmed(queue.getEndSite());
|
||||
if (!"tsj".equals(site)) return site;
|
||||
String type = trimmed(queue.getDefault02());
|
||||
String prefix;
|
||||
if ("8".equals(type)) prefix = "07050";
|
||||
else if ("1".equals(type) || "2".equals(type)) prefix = "09050";
|
||||
else throw new IllegalArgumentException("UNRESOLVED_TSJ_END:" + queue.getGuid());
|
||||
int floor = sourceFloor(queue);
|
||||
return requireFloorSite(prefix + floor, floor, batch);
|
||||
}
|
||||
|
||||
private static int sourceFloor(ShadowQueueFact queue) {
|
||||
String floor = trimmed(queue.getTheFloor());
|
||||
if (floor == null || !floor.matches("[1-9]")) {
|
||||
throw new IllegalArgumentException("INVALID_TSJ_SOURCE_FLOOR:" + queue.getGuid());
|
||||
}
|
||||
return Integer.parseInt(floor);
|
||||
}
|
||||
|
||||
private static String requireFloorSite(String site, int floor, ShadowFactBatch batch) {
|
||||
for (ShadowMapFact map : batch.getMaps()) {
|
||||
if (map.getSiteId().equals(site) && map.getZ() == floor
|
||||
&& (map.getValue() == 1 || map.getValue() == 2)) return site;
|
||||
}
|
||||
throw new IllegalArgumentException("MISSING_TSJ_MAP_SITE:" + site);
|
||||
}
|
||||
|
||||
private static String trimmed(String value) {
|
||||
return value == null ? null : value.trim();
|
||||
}
|
||||
}
|
||||
@@ -12,6 +12,7 @@ public final class ShadowQueueFact {
|
||||
private final String cargoPlaceSite;
|
||||
private final String startSite;
|
||||
private final String endSite;
|
||||
private final String theFloor;
|
||||
private final Instant createdAt;
|
||||
private final String agvId;
|
||||
private final String pathInfo;
|
||||
@@ -35,6 +36,7 @@ public final class ShadowQueueFact {
|
||||
this.cargoPlaceSite = builder.cargoPlaceSite;
|
||||
this.startSite = builder.startSite;
|
||||
this.endSite = builder.endSite;
|
||||
this.theFloor = builder.theFloor;
|
||||
this.createdAt = builder.createdAt;
|
||||
this.agvId = builder.agvId;
|
||||
this.pathInfo = builder.pathInfo;
|
||||
@@ -49,7 +51,7 @@ public final class ShadowQueueFact {
|
||||
this.contentHash = FactHash.sha256(tableName, guid, taskNo, taskType,
|
||||
priority, cargoPickSite, cargoPlaceSite, startSite, endSite,
|
||||
createdAt, agvId, pathInfo, turnPath, default02, default03,
|
||||
default04, default05, default06, default07, default08);
|
||||
default04, default05, default06, default07, default08, theFloor);
|
||||
}
|
||||
|
||||
public static Builder builder(String tableName, String guid) {
|
||||
@@ -57,6 +59,7 @@ public final class ShadowQueueFact {
|
||||
}
|
||||
|
||||
public String getTableName() { return tableName; }
|
||||
public boolean isActive() { return "shadow_ro.v_wcs_srgv_queue".equals(tableName); }
|
||||
public String getGuid() { return guid; }
|
||||
public String getTaskNo() { return taskNo; }
|
||||
public String getTaskType() { return taskType; }
|
||||
@@ -65,6 +68,7 @@ public final class ShadowQueueFact {
|
||||
public String getCargoPlaceSite() { return cargoPlaceSite; }
|
||||
public String getStartSite() { return startSite; }
|
||||
public String getEndSite() { return endSite; }
|
||||
public String getTheFloor() { return theFloor; }
|
||||
public Instant getCreatedAt() { return createdAt; }
|
||||
public String getAgvId() { return agvId; }
|
||||
public String getPathInfo() { return pathInfo; }
|
||||
@@ -95,6 +99,7 @@ public final class ShadowQueueFact {
|
||||
private String cargoPlaceSite;
|
||||
private String startSite;
|
||||
private String endSite;
|
||||
private String theFloor;
|
||||
private Instant createdAt;
|
||||
private String agvId;
|
||||
private String pathInfo;
|
||||
@@ -119,6 +124,7 @@ public final class ShadowQueueFact {
|
||||
public Builder cargoPlaceSite(String value) { cargoPlaceSite = value; return this; }
|
||||
public Builder startSite(String value) { startSite = value; return this; }
|
||||
public Builder endSite(String value) { endSite = value; return this; }
|
||||
public Builder theFloor(String value) { theFloor = value; return this; }
|
||||
public Builder createdAt(Instant value) { createdAt = value; return this; }
|
||||
public Builder agvId(String value) { agvId = value; return this; }
|
||||
public Builder pathInfo(String value) { pathInfo = value; return this; }
|
||||
|
||||
@@ -181,7 +181,7 @@ public final class SqlServerShadowFactReader {
|
||||
"SELECT database_utc_time FROM shadow_ro.v_current_database_clock";
|
||||
static final String QUEUE_SQL_PREFIX =
|
||||
"SELECT TOP (?) guid, task_no, task_type, priority, "
|
||||
+ "cargo_pick_site, cargo_place_site, start_site, end_site, "
|
||||
+ "cargo_pick_site, cargo_place_site, start_site, end_site, the_floor, "
|
||||
+ "create_date_utc, agv_id, path_info, trun_path, default02, "
|
||||
+ "default03, default04, default05, default06, default07, default08 ";
|
||||
static final String QUEUE_SQL_SUFFIX =
|
||||
@@ -345,6 +345,7 @@ public final class SqlServerShadowFactReader {
|
||||
.cargoPlaceSite(result.getString("cargo_place_site"))
|
||||
.startSite(result.getString("start_site"))
|
||||
.endSite(result.getString("end_site"))
|
||||
.theFloor(result.getString("the_floor"))
|
||||
.createdAt(instant(result, "create_date_utc"))
|
||||
.agvId(result.getString("agv_id"))
|
||||
.pathInfo(result.getString("path_info"))
|
||||
|
||||
@@ -16,6 +16,7 @@ import com.ruoyi.wcs.application.ShadowDispatchResult;
|
||||
import com.ruoyi.wcs.application.dispatch.DispatchSnapshot;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.time.Clock;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
@@ -30,6 +31,7 @@ public final class Ft546ShadowCycle {
|
||||
private final ShadowEvidenceRepository evidenceRepository;
|
||||
private final ShadowCycleEvidenceCollector evidenceCollector;
|
||||
private final ShadowStartupReport statusReport;
|
||||
private final Clock clock;
|
||||
private final AtomicReference<String> degradedReason =
|
||||
new AtomicReference<String>();
|
||||
|
||||
@@ -48,6 +50,7 @@ public final class Ft546ShadowCycle {
|
||||
this.evidenceRepository = null;
|
||||
this.evidenceCollector = null;
|
||||
this.statusReport = null;
|
||||
this.clock = Clock.systemUTC();
|
||||
}
|
||||
|
||||
public Ft546ShadowCycle(
|
||||
@@ -59,6 +62,19 @@ public final class Ft546ShadowCycle {
|
||||
ShadowEvidenceRepository evidenceRepository,
|
||||
ShadowCycleEvidenceCollector evidenceCollector,
|
||||
ShadowStartupReport statusReport) {
|
||||
this(validator, assembler, dispatchCycle, commandGateway, storageHealth,
|
||||
evidenceRepository, evidenceCollector, statusReport, Clock.systemUTC());
|
||||
}
|
||||
|
||||
public Ft546ShadowCycle(
|
||||
ShadowFactBatchValidator validator,
|
||||
Function<ShadowFactBatch, DispatchSnapshot> assembler,
|
||||
ReadOnlyDispatchCycle<Ft546SrgvCommandIntent> dispatchCycle,
|
||||
RecordingFt546CommandGateway commandGateway,
|
||||
ShadowEvidenceStorageHealth storageHealth,
|
||||
ShadowEvidenceRepository evidenceRepository,
|
||||
ShadowCycleEvidenceCollector evidenceCollector,
|
||||
ShadowStartupReport statusReport, Clock clock) {
|
||||
this.validator = Objects.requireNonNull(validator, "validator");
|
||||
this.assembler = Objects.requireNonNull(assembler, "assembler");
|
||||
this.dispatchCycle = Objects.requireNonNull(dispatchCycle,
|
||||
@@ -73,86 +89,103 @@ public final class Ft546ShadowCycle {
|
||||
"evidenceCollector");
|
||||
this.statusReport = Objects.requireNonNull(statusReport,
|
||||
"statusReport");
|
||||
this.clock = Objects.requireNonNull(clock, "clock");
|
||||
}
|
||||
|
||||
public ShadowCycleResult run(ShadowFactBatch batch, Instant now) {
|
||||
return run(batch, now, new ShadowCycleDiagnostics());
|
||||
}
|
||||
|
||||
ShadowCycleResult run(ShadowFactBatch batch, Instant now, ShadowCycleDiagnostics progress) {
|
||||
if (evidenceRepository == null) {
|
||||
return runWithoutEvidence(batch, now);
|
||||
return runWithoutEvidence(batch, now, progress);
|
||||
}
|
||||
String stoppedReason = degradedReason.get();
|
||||
if (stoppedReason != null) {
|
||||
return ShadowCycleResult.degraded(stoppedReason);
|
||||
}
|
||||
progress.enter("STORAGE_HEALTH");
|
||||
ShadowEvidenceStorageHealth.Check health = storageHealth.verify();
|
||||
if (!health.isAvailable()) {
|
||||
return degrade(now);
|
||||
return degrade(health.getReasonCode(), clock.instant());
|
||||
}
|
||||
try {
|
||||
progress.enter("TRANSACTION_BEGIN");
|
||||
ShadowCycleResult result = evidenceRepository.inTransaction(
|
||||
() -> runAndPersist(batch, now));
|
||||
() -> {
|
||||
ShadowCycleResult saved = runAndPersist(batch, now, progress);
|
||||
progress.enter("TRANSACTION_COMMIT");
|
||||
return saved;
|
||||
});
|
||||
progress.enter("RECORDING");
|
||||
commandGateway.rememberAll(result.getEncodedCommands());
|
||||
Instant completedAt = clock.instant();
|
||||
Long lag = batch.getDatabaseCapturedAt() == null ? null
|
||||
: Long.valueOf(Math.max(0L,
|
||||
java.time.Duration.between(
|
||||
batch.getDatabaseCapturedAt(), now).toMillis()));
|
||||
statusReport.markReady(now, lag);
|
||||
batch.getDatabaseCapturedAt(), completedAt).toMillis()));
|
||||
statusReport.markReady(completedAt, lag);
|
||||
return result;
|
||||
} catch (RuntimeException evidenceFailure) {
|
||||
return degrade(failureReason(evidenceFailure), now);
|
||||
return degrade(failureReason(evidenceFailure), clock.instant());
|
||||
}
|
||||
}
|
||||
|
||||
private ShadowCycleResult runAndPersist(ShadowFactBatch batch,
|
||||
Instant now) {
|
||||
Instant now, ShadowCycleDiagnostics progress) {
|
||||
progress.enter("VALIDATION");
|
||||
ShadowFactValidation validation = validator.validate(batch, now);
|
||||
if (validation.getStatus()
|
||||
!= ShadowFactValidation.Status.COMPARABLE) {
|
||||
return persistInsufficient(batch, validation.getReasonCode(), now);
|
||||
return persistInsufficient(batch, validation.getReasonCode(), progress);
|
||||
}
|
||||
final DispatchSnapshot snapshot;
|
||||
progress.enter("ASSEMBLY_AND_STATE_LOAD");
|
||||
try {
|
||||
snapshot = assembler.apply(batch);
|
||||
} catch (RuntimeException failure) {
|
||||
return persistInsufficient(batch,
|
||||
"INVALID_KERNEL_FACTS:" + message(failure), now);
|
||||
"INVALID_KERNEL_FACTS:" + message(failure), progress);
|
||||
}
|
||||
ShadowDispatchResult<Ft546SrgvCommandIntent> result;
|
||||
progress.enter("DISPATCH_AND_STATE_SAVE");
|
||||
try {
|
||||
result = dispatchCycle.run(snapshot);
|
||||
} catch (IllegalArgumentException failure) {
|
||||
return persistInsufficient(batch,
|
||||
"INVALID_COMMAND_FACTS:" + message(failure), now);
|
||||
"INVALID_COMMAND_FACTS:" + message(failure), progress);
|
||||
}
|
||||
if (result.getDecisions().isRetrySnapshotChanged()) {
|
||||
List<ShadowCycleEvidence> evidence = evidenceCollector.insufficient(
|
||||
batch, "SHADOW_TRAFFIC_STATE_CHANGED", now);
|
||||
evidenceRepository.saveAll(evidence);
|
||||
persistInsufficient(batch, "SHADOW_TRAFFIC_STATE_CHANGED", progress);
|
||||
return ShadowCycleResult.retry(result.getDecisions());
|
||||
}
|
||||
final List<Ft546EncodedCommand> encoded;
|
||||
progress.enter("ENCODING");
|
||||
try {
|
||||
encoded = commandGateway.previewAll(result.getIntents());
|
||||
} catch (RuntimeException failure) {
|
||||
return persistInsufficient(batch,
|
||||
"INVALID_COMMAND_FACTS:" + message(failure), now);
|
||||
"INVALID_COMMAND_FACTS:" + message(failure), progress);
|
||||
}
|
||||
evidenceRepository.saveAll(evidenceCollector.collect(batch,
|
||||
result.getDecisions(), encoded, now));
|
||||
progress.enter("EVIDENCE_BUILD");
|
||||
List<ShadowCycleEvidence> evidence = evidenceCollector.collect(batch,
|
||||
result.getDecisions(), encoded, clock.instant());
|
||||
progress.enter("EVIDENCE_SAVE");
|
||||
evidenceRepository.saveAll(evidence);
|
||||
return ShadowCycleResult.comparable(result.getDecisions(),
|
||||
result.getIntents(), encoded);
|
||||
}
|
||||
|
||||
private ShadowCycleResult persistInsufficient(
|
||||
ShadowFactBatch batch, String reasonCode, Instant now) {
|
||||
evidenceRepository.saveAll(evidenceCollector.insufficient(batch,
|
||||
reasonCode, now));
|
||||
ShadowFactBatch batch, String reasonCode, ShadowCycleDiagnostics progress) {
|
||||
progress.enter("EVIDENCE_BUILD");
|
||||
List<ShadowCycleEvidence> evidence = evidenceCollector.insufficient(batch,
|
||||
reasonCode, clock.instant());
|
||||
progress.enter("EVIDENCE_SAVE");
|
||||
evidenceRepository.saveAll(evidence);
|
||||
return ShadowCycleResult.insufficient(reasonCode);
|
||||
}
|
||||
|
||||
private ShadowCycleResult degrade(Instant now) {
|
||||
return degrade("EVIDENCE_STORAGE_UNAVAILABLE", now);
|
||||
}
|
||||
|
||||
private ShadowCycleResult degrade(String reasonCode, Instant now) {
|
||||
degradedReason.compareAndSet(null, reasonCode);
|
||||
String stoppedReason = degradedReason.get();
|
||||
@@ -165,7 +198,8 @@ public final class Ft546ShadowCycle {
|
||||
String reasonCode = ((ShadowEvidenceException) failure)
|
||||
.getReasonCode();
|
||||
if ("EVIDENCE_KEY_COLLISION".equals(reasonCode)
|
||||
|| "EVIDENCE_RUN_COLLISION".equals(reasonCode)) {
|
||||
|| "EVIDENCE_RUN_COLLISION".equals(reasonCode)
|
||||
|| "EVIDENCE_BATCH_PAYLOAD_LIMIT".equals(reasonCode)) {
|
||||
return reasonCode;
|
||||
}
|
||||
}
|
||||
@@ -173,13 +207,15 @@ public final class Ft546ShadowCycle {
|
||||
}
|
||||
|
||||
private ShadowCycleResult runWithoutEvidence(ShadowFactBatch batch,
|
||||
Instant now) {
|
||||
Instant now, ShadowCycleDiagnostics progress) {
|
||||
progress.enter("VALIDATION");
|
||||
ShadowFactValidation validation = validator.validate(batch, now);
|
||||
if (validation.getStatus()
|
||||
!= ShadowFactValidation.Status.COMPARABLE) {
|
||||
return ShadowCycleResult.insufficient(validation.getReasonCode());
|
||||
}
|
||||
final DispatchSnapshot snapshot;
|
||||
progress.enter("ASSEMBLY_AND_STATE_LOAD");
|
||||
try {
|
||||
snapshot = assembler.apply(batch);
|
||||
} catch (RuntimeException failure) {
|
||||
@@ -187,6 +223,7 @@ public final class Ft546ShadowCycle {
|
||||
"INVALID_KERNEL_FACTS:" + message(failure));
|
||||
}
|
||||
ShadowDispatchResult<Ft546SrgvCommandIntent> result;
|
||||
progress.enter("DISPATCH_AND_STATE_SAVE");
|
||||
try {
|
||||
result = dispatchCycle.run(snapshot);
|
||||
} catch (IllegalArgumentException failure) {
|
||||
@@ -197,6 +234,7 @@ public final class Ft546ShadowCycle {
|
||||
return ShadowCycleResult.retry(result.getDecisions());
|
||||
}
|
||||
final List<Ft546EncodedCommand> encoded;
|
||||
progress.enter("ENCODING");
|
||||
try {
|
||||
encoded = commandGateway.recordAll(result.getIntents());
|
||||
} catch (RuntimeException failure) {
|
||||
|
||||
@@ -14,6 +14,7 @@ import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.Objects;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
@@ -22,6 +23,7 @@ import java.util.concurrent.ThreadFactory;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.function.Consumer;
|
||||
|
||||
public final class Ft546ShadowScheduler implements AutoCloseable {
|
||||
private static final Logger LOGGER =
|
||||
@@ -39,11 +41,17 @@ public final class Ft546ShadowScheduler implements AutoCloseable {
|
||||
private final Duration cycleDeadline;
|
||||
private final Duration sourceReadDeadline;
|
||||
private final ExecutorService cycleExecutor;
|
||||
private final Consumer<Map<String, Object>> runtimeWriter;
|
||||
private final Object runtimeLock = new Object();
|
||||
private final AtomicBoolean closed = new AtomicBoolean();
|
||||
private volatile Instant lastAttemptAt;
|
||||
private final AtomicBoolean inFlight = new AtomicBoolean();
|
||||
private final AtomicBoolean permanentlyStopped = new AtomicBoolean();
|
||||
private volatile CycleStage activeStage = CycleStage.IDLE;
|
||||
private volatile ShadowReadCursor cursor = ShadowReadCursor.initial();
|
||||
private volatile Instant nextAttemptAt = Instant.EPOCH;
|
||||
private volatile Thread cycleWorker;
|
||||
private long lastTimingLogNanos;
|
||||
|
||||
public Ft546ShadowScheduler(SqlServerShadowFactReader factReader,
|
||||
Ft546ShadowCycle shadowCycle,
|
||||
@@ -67,6 +75,21 @@ public final class Ft546ShadowScheduler implements AutoCloseable {
|
||||
Clock clock,
|
||||
Duration cycleDeadline,
|
||||
Duration sourceReadDeadline) {
|
||||
this(factReader, shadowCycle, statusReport, clock, cycleDeadline, sourceReadDeadline, state -> { });
|
||||
}
|
||||
|
||||
public Ft546ShadowScheduler(SqlServerShadowFactReader factReader,
|
||||
Ft546ShadowCycle shadowCycle, ShadowStartupReport statusReport,
|
||||
Clock clock, Duration cycleDeadline,
|
||||
Consumer<Map<String, Object>> runtimeWriter) {
|
||||
this(factReader, shadowCycle, statusReport, clock, cycleDeadline,
|
||||
shorter(cycleDeadline, SOURCE_READ_DEADLINE), runtimeWriter);
|
||||
}
|
||||
|
||||
private Ft546ShadowScheduler(SqlServerShadowFactReader factReader,
|
||||
Ft546ShadowCycle shadowCycle, ShadowStartupReport statusReport,
|
||||
Clock clock, Duration cycleDeadline, Duration sourceReadDeadline,
|
||||
Consumer<Map<String, Object>> runtimeWriter) {
|
||||
this.factReader = Objects.requireNonNull(factReader, "factReader");
|
||||
this.shadowCycle = Objects.requireNonNull(shadowCycle, "shadowCycle");
|
||||
this.statusReport = Objects.requireNonNull(statusReport,
|
||||
@@ -74,6 +97,7 @@ public final class Ft546ShadowScheduler implements AutoCloseable {
|
||||
this.clock = Objects.requireNonNull(clock, "clock");
|
||||
this.cycleDeadline = requirePositive(cycleDeadline);
|
||||
this.sourceReadDeadline = requirePositive(sourceReadDeadline);
|
||||
this.runtimeWriter = Objects.requireNonNull(runtimeWriter, "runtimeWriter");
|
||||
this.cycleExecutor = Executors.newSingleThreadExecutor(
|
||||
shadowCycleThreadFactory());
|
||||
}
|
||||
@@ -82,7 +106,11 @@ public final class Ft546ShadowScheduler implements AutoCloseable {
|
||||
initialDelayString = "${shadow.poll.initial-delay-ms:5000}",
|
||||
fixedDelayString = "${shadow.poll.fixed-delay-ms:5000}")
|
||||
public void runScheduledCycle() {
|
||||
runOnce();
|
||||
if (permanentlyStopped.get() || clock.instant().isBefore(nextAttemptAt)) {
|
||||
publishRuntimeState();
|
||||
} else {
|
||||
runOnce();
|
||||
}
|
||||
}
|
||||
|
||||
public boolean runOnce() {
|
||||
@@ -91,11 +119,22 @@ public final class Ft546ShadowScheduler implements AutoCloseable {
|
||||
|| !inFlight.compareAndSet(false, true)) {
|
||||
return false;
|
||||
}
|
||||
Future<CycleAttempt> future = cycleExecutor.submit(
|
||||
() -> executeCycle(now));
|
||||
ShadowCycleDiagnostics diagnostics = new ShadowCycleDiagnostics();
|
||||
lastAttemptAt = now;
|
||||
if (!publishRuntimeState()) {
|
||||
inFlight.set(false);
|
||||
return false;
|
||||
}
|
||||
cycleWorker = null;
|
||||
String outcome = "FAILED";
|
||||
Future<CycleAttempt> future = null;
|
||||
try {
|
||||
if (permanentlyStopped.get()) return false;
|
||||
future = cycleExecutor.submit(() -> executeCycle(now, diagnostics));
|
||||
CycleAttempt attempt = future.get(cycleDeadline.toMillis(),
|
||||
TimeUnit.MILLISECONDS);
|
||||
outcome = attempt.result.getStatusCode();
|
||||
statusReport.recordCycleDiagnostics(diagnostics.finish(outcome, null));
|
||||
if ("DEGRADED".equals(attempt.result.getStatusCode())) {
|
||||
permanentlyStopped.set(true);
|
||||
statusReport.markPermanentlyDegraded(
|
||||
@@ -109,20 +148,34 @@ public final class Ft546ShadowScheduler implements AutoCloseable {
|
||||
return true;
|
||||
} catch (TimeoutException timeout) {
|
||||
CycleStage timedOutStage = activeStage;
|
||||
future.cancel(true);
|
||||
statusReport.recordCycleDiagnostics(diagnostics.cancel("TIMEOUT", cycleWorker));
|
||||
if (future != null) future.cancel(true);
|
||||
return stopPermanently("SHADOW_CYCLE_TIMEOUT", timeout,
|
||||
timedOutStage);
|
||||
} catch (InterruptedException interrupted) {
|
||||
CycleStage interruptedStage = activeStage;
|
||||
future.cancel(true);
|
||||
statusReport.recordCycleDiagnostics(diagnostics.cancel("INTERRUPTED", cycleWorker));
|
||||
if (future != null) future.cancel(true);
|
||||
Thread.currentThread().interrupt();
|
||||
return stopPermanently("SHADOW_CYCLE_INTERRUPTED", interrupted,
|
||||
interruptedStage);
|
||||
} catch (ExecutionException wrapped) {
|
||||
statusReport.recordCycleDiagnostics(diagnostics.finish("FAILED", null));
|
||||
return handleFailure(wrapped.getCause());
|
||||
} catch (java.util.concurrent.RejectedExecutionException shutdown) {
|
||||
return stopPermanently("SHADOW_EXECUTOR_STOPPED", shutdown, activeStage);
|
||||
} finally {
|
||||
Map<String, Object> completed = diagnostics.finish(outcome, null);
|
||||
statusReport.recordCycleDiagnostics(completed);
|
||||
long logTime = System.nanoTime();
|
||||
if (!permanentlyStopped.get() && (lastTimingLogNanos == 0L
|
||||
|| logTime - lastTimingLogNanos >= TimeUnit.MINUTES.toNanos(1))) {
|
||||
LOGGER.info("FT546 shadow cycle diagnostics: {}", completed);
|
||||
lastTimingLogNanos = logTime;
|
||||
}
|
||||
inFlight.set(false);
|
||||
activeStage = CycleStage.IDLE;
|
||||
publishRuntimeState();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -137,17 +190,37 @@ public final class Ft546ShadowScheduler implements AutoCloseable {
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
if (!closed.compareAndSet(false, true)) return;
|
||||
permanentlyStopped.set(true);
|
||||
statusReport.markStopped(clock.instant());
|
||||
cycleExecutor.shutdownNow();
|
||||
publishRuntimeState();
|
||||
}
|
||||
|
||||
private CycleAttempt executeCycle(Instant startedAt) {
|
||||
private boolean publishRuntimeState() {
|
||||
synchronized (runtimeLock) {
|
||||
try {
|
||||
runtimeWriter.accept(statusReport.snapshotRuntimeState(
|
||||
clock.instant(), lastAttemptAt, permanentlyStopped.get()));
|
||||
return true;
|
||||
} catch (RuntimeException failure) {
|
||||
permanentlyStopped.set(true);
|
||||
statusReport.markPermanentlyDegraded("EVIDENCE_RUNTIME_WRITE_FAILED", clock.instant());
|
||||
LOGGER.error("FT546 shadow runtime state unavailable; collection stopped", failure);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private CycleAttempt executeCycle(Instant startedAt, ShadowCycleDiagnostics diagnostics) {
|
||||
cycleWorker = Thread.currentThread();
|
||||
activeStage = CycleStage.SOURCE_READ;
|
||||
diagnostics.enter("SOURCE_READ");
|
||||
ShadowFactBatch batch = factReader.read(cursor,
|
||||
startedAt.plus(sourceReadDeadline));
|
||||
activeStage = CycleStage.SHADOW_CYCLE;
|
||||
return new CycleAttempt(batch,
|
||||
shadowCycle.run(batch, clock.instant()));
|
||||
shadowCycle.run(batch, clock.instant(), diagnostics));
|
||||
}
|
||||
|
||||
private boolean handleFailure(Throwable failure) {
|
||||
@@ -208,8 +281,8 @@ public final class Ft546ShadowScheduler implements AutoCloseable {
|
||||
permanentlyStopped.set(true);
|
||||
nextAttemptAt = Instant.EPOCH;
|
||||
statusReport.markPermanentlyDegraded(reasonCode, clock.instant());
|
||||
LOGGER.error("FT546 shadow collection stopped: {}; stage={}",
|
||||
reasonCode, stage, failure);
|
||||
LOGGER.error("FT546 shadow collection stopped: {}; stage={}; diagnostics={}",
|
||||
reasonCode, stage, statusReport.getCycleDiagnostics(), failure);
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
@@ -88,8 +88,33 @@ public final class JdbcShadowTrafficStateRepository
|
||||
throw new IllegalStateException(
|
||||
"persisted traffic resources do not match topology");
|
||||
}
|
||||
return new ShadowTrafficState(ledger,
|
||||
WaitForGraph.restore(persisted.waitForEdges), row.version,
|
||||
// Only explicitly tagged predictions expire; untagged state is preserved conservatively.
|
||||
List<ResourceReservationState> protectedReservations = new ArrayList<ResourceReservationState>();
|
||||
Set<String> predictedVehicles = new LinkedHashSet<String>();
|
||||
Set<String> protectedVehicles = new LinkedHashSet<String>();
|
||||
for (ResourceReservationState reservation : persisted.ledger.getReservations()) {
|
||||
if (!persisted.predictionIds.contains(reservation.getReservationId())
|
||||
|| reservation.getState() == ReservationState.OCCUPIED
|
||||
|| reservation.isCommandUnknown() || reservation.isRecoveryUnresolved()) {
|
||||
protectedReservations.add(reservation);
|
||||
protectedVehicles.add(reservation.getVehicleId());
|
||||
} else {
|
||||
predictedVehicles.add(reservation.getVehicleId());
|
||||
}
|
||||
}
|
||||
predictedVehicles.removeAll(protectedVehicles);
|
||||
Map<String, Set<String>> remainingWaits = new LinkedHashMap<String, Set<String>>();
|
||||
for (Map.Entry<String, Set<String>> entry : persisted.waitForEdges.entrySet()) {
|
||||
if (!predictedVehicles.contains(entry.getKey())) {
|
||||
Set<String> holders = new LinkedHashSet<String>(entry.getValue());
|
||||
holders.removeAll(predictedVehicles);
|
||||
if (!holders.isEmpty()) remainingWaits.put(entry.getKey(), holders);
|
||||
}
|
||||
}
|
||||
WaitForGraph graph = WaitForGraph.restore(remainingWaits);
|
||||
ledger = ResourceLedger.restore(new ResourceLedgerState(persisted.ledger.getCapacities(),
|
||||
protectedReservations, persisted.ledger.getNextFencingToken()));
|
||||
return new ShadowTrafficState(ledger, graph, row.version,
|
||||
persisted.holdCounts);
|
||||
}
|
||||
|
||||
@@ -110,8 +135,10 @@ public final class JdbcShadowTrafficStateRepository
|
||||
Map<String, Integer> nextHolds = nextHoldCounts(previousHolds,
|
||||
commit.getHeld());
|
||||
long nextVersion = increment(commit.getSnapshotVersion());
|
||||
Set<String> predictionIds = new TreeSet<String>();
|
||||
commit.getGranted().forEach(grant -> predictionIds.addAll(grant.getReservationIds()));
|
||||
String stateJson = encode(commit.getResultingLedger().snapshotState(),
|
||||
commit.getResultingWaitForGraph().snapshotEdges(), nextHolds);
|
||||
commit.getResultingWaitForGraph().snapshotEdges(), nextHolds, predictionIds);
|
||||
Timestamp now = Timestamp.from(clock.instant());
|
||||
if (current == null) {
|
||||
jdbc.update("INSERT INTO shadow_traffic_state "
|
||||
@@ -197,9 +224,12 @@ public final class JdbcShadowTrafficStateRepository
|
||||
|
||||
private String encode(ResourceLedgerState ledger,
|
||||
Map<String, Set<String>> waitForEdges,
|
||||
Map<String, Integer> holdCounts) {
|
||||
Map<String, Integer> holdCounts,
|
||||
Set<String> predictionIds) {
|
||||
Map<String, Object> root = new LinkedHashMap<String, Object>();
|
||||
root.put("schemaVersion", 1);
|
||||
root.put("evaluationMode", "PER_SNAPSHOT_PREDICTION");
|
||||
root.put("predictionReservationIds", new ArrayList<String>(predictionIds));
|
||||
root.put("ledger", ledgerMap(ledger));
|
||||
Map<String, Object> edges = new TreeMap<String, Object>();
|
||||
for (Map.Entry<String, Set<String>> entry : waitForEdges.entrySet()) {
|
||||
@@ -267,7 +297,13 @@ public final class JdbcShadowTrafficStateRepository
|
||||
requiredObject(root, "waitForEdges"));
|
||||
Map<String, Integer> holds = integerMap(
|
||||
requiredObject(root, "holdCounts"));
|
||||
return new PersistedState(ledgerState, edges, holds);
|
||||
Set<String> predictionIds = new LinkedHashSet<String>();
|
||||
if (root.has("predictionReservationIds")) {
|
||||
for (JsonNode id : requiredArray(root, "predictionReservationIds")) {
|
||||
predictionIds.add(requiredNodeText(id, "predictionReservationId"));
|
||||
}
|
||||
}
|
||||
return new PersistedState(ledgerState, edges, holds, predictionIds);
|
||||
} catch (RuntimeException failure) {
|
||||
throw failure;
|
||||
} catch (Exception failure) {
|
||||
@@ -426,13 +462,16 @@ public final class JdbcShadowTrafficStateRepository
|
||||
private final ResourceLedgerState ledger;
|
||||
private final Map<String, Set<String>> waitForEdges;
|
||||
private final Map<String, Integer> holdCounts;
|
||||
private final Set<String> predictionIds;
|
||||
|
||||
private PersistedState(ResourceLedgerState ledger,
|
||||
Map<String, Set<String>> waitForEdges,
|
||||
Map<String, Integer> holdCounts) {
|
||||
Map<String, Integer> holdCounts,
|
||||
Set<String> predictionIds) {
|
||||
this.ledger = ledger;
|
||||
this.waitForEdges = waitForEdges;
|
||||
this.holdCounts = holdCounts;
|
||||
this.predictionIds = predictionIds;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ package com.ruoyi.shadow.kernel;
|
||||
|
||||
import com.ruoyi.ft546.protocol.Ft546SrgvCommandIntent;
|
||||
import com.ruoyi.wcs.application.dispatch.GrantedDispatch;
|
||||
import com.ruoyi.wcs.domain.traffic.RoutePlan;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
@@ -20,7 +21,8 @@ public final class ShadowCommandIntentFactory {
|
||||
short taskType = signedShort(parameters, "taskType");
|
||||
Integer stage = optionalStage(parameters.get("stage"));
|
||||
Short crossFloorTarget = crossFloorTarget(parameters, taskType, stage);
|
||||
List<Integer> path = path(grant);
|
||||
List<Integer> path = path(grant, taskType == 3 && stage == null
|
||||
&& "LEGACY_PHASE_3".equals(phase));
|
||||
String commandKey = parentTaskId + ':' + phase + ':' + routeVersion
|
||||
+ ':' + requiredText(grant.getStepId(), "stepId");
|
||||
return new Ft546SrgvCommandIntent(
|
||||
@@ -29,20 +31,33 @@ public final class ShadowCommandIntentFactory {
|
||||
taskType, path, crossFloorTarget, stage);
|
||||
}
|
||||
|
||||
private static List<Integer> path(GrantedDispatch grant) {
|
||||
private static List<Integer> path(GrantedDispatch grant, boolean sameSiteCharge) {
|
||||
RoutePlan route = grant.getProtectedRoute();
|
||||
List<String> nodes = route.getNodeIds();
|
||||
if (!route.hasTurnGeometry()) {
|
||||
throw new IllegalArgumentException("SHADOW_ROUTE_GEOMETRY_UNAVAILABLE");
|
||||
}
|
||||
if (nodes.size() == 1 && !sameSiteCharge) {
|
||||
throw new IllegalArgumentException("SHADOW_SINGLE_NODE_TRANSPORT_ROUTE");
|
||||
}
|
||||
List<Integer> points = new ArrayList<Integer>();
|
||||
for (String nodeId : grant.getProtectedRoute().getNodeIds()) {
|
||||
try {
|
||||
points.add(Integer.valueOf(requiredText(nodeId, "route node")));
|
||||
} catch (NumberFormatException failure) {
|
||||
throw new IllegalArgumentException(
|
||||
"route node must be a numeric FT546 site: " + nodeId,
|
||||
failure);
|
||||
}
|
||||
points.add(site(nodes.get(0)));
|
||||
points.add(site(nodes.get(nodes.size() - 1)));
|
||||
for (String turn : route.getTurnNodeIds()) points.add(site(turn));
|
||||
if (points.size() > 8) {
|
||||
throw new IllegalArgumentException("SHADOW_ROUTE_EXCEEDS_SIX_TURNS");
|
||||
}
|
||||
return points;
|
||||
}
|
||||
|
||||
private static Integer site(String node) {
|
||||
String value = requiredText(node, "command point");
|
||||
if (!value.matches("[0-9]{1,6}") || Integer.parseInt(value) == 0) {
|
||||
throw new IllegalArgumentException("SHADOW_COMMAND_POINT_NOT_NUMERIC:" + value);
|
||||
}
|
||||
return Integer.valueOf(value);
|
||||
}
|
||||
|
||||
private static Short crossFloorTarget(Map<String, String> parameters,
|
||||
short taskType, Integer stage) {
|
||||
String value = parameters.get("crossFloorTarget");
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
package com.ruoyi.shadow.kernel;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.CancellationException;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.function.LongSupplier;
|
||||
|
||||
/** One bounded diagnostic snapshot per poll, including polls with no task evidence. */
|
||||
final class ShadowCycleDiagnostics {
|
||||
private final LongSupplier ticker;
|
||||
private final long started;
|
||||
private long phaseStarted;
|
||||
private String phase = "QUEUED";
|
||||
private final Map<String, Long> phaseNanos = new LinkedHashMap<String, Long>();
|
||||
private Map<String, Object> completed;
|
||||
private boolean cancelled;
|
||||
|
||||
ShadowCycleDiagnostics() { this(System::nanoTime); }
|
||||
|
||||
ShadowCycleDiagnostics(LongSupplier ticker) {
|
||||
this.ticker = ticker;
|
||||
this.started = ticker.getAsLong();
|
||||
this.phaseStarted = started;
|
||||
}
|
||||
|
||||
synchronized void enter(String next) {
|
||||
if (cancelled || Thread.currentThread().isInterrupted()) {
|
||||
throw new CancellationException("SHADOW_CYCLE_CANCELLED");
|
||||
}
|
||||
if (completed != null) return;
|
||||
long now = ticker.getAsLong();
|
||||
accumulate(now);
|
||||
phase = next;
|
||||
phaseStarted = now;
|
||||
}
|
||||
|
||||
synchronized Map<String, Object> finish(String outcome, Thread worker) {
|
||||
if (completed != null) return completed;
|
||||
long now = ticker.getAsLong();
|
||||
accumulate(now);
|
||||
Map<String, Object> snapshot = new LinkedHashMap<String, Object>();
|
||||
snapshot.put("outcome", outcome);
|
||||
snapshot.put("phase", phase);
|
||||
snapshot.put("elapsedMillis", TimeUnit.NANOSECONDS.toMillis(now - started));
|
||||
Map<String, Long> timings = new LinkedHashMap<String, Long>();
|
||||
phaseNanos.forEach((key, value) -> timings.put(key, TimeUnit.NANOSECONDS.toMillis(value)));
|
||||
snapshot.put("phaseMillis", Collections.unmodifiableMap(timings));
|
||||
if (worker != null) {
|
||||
snapshot.put("workerState", worker.getState().name());
|
||||
List<String> stack = new ArrayList<String>();
|
||||
StackTraceElement[] frames = worker.getStackTrace();
|
||||
for (int i = 0; i < Math.min(16, frames.length); i++) stack.add(frames[i].toString());
|
||||
snapshot.put("workerStack", Collections.unmodifiableList(stack));
|
||||
}
|
||||
completed = Collections.unmodifiableMap(snapshot);
|
||||
return completed;
|
||||
}
|
||||
|
||||
synchronized Map<String, Object> cancel(String outcome, Thread worker) {
|
||||
cancelled = true;
|
||||
return finish(outcome, worker);
|
||||
}
|
||||
|
||||
private void accumulate(long now) {
|
||||
phaseNanos.merge(phase, now - phaseStarted, Long::sum);
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package com.ruoyi.shadow.kernel;
|
||||
import com.ruoyi.shadow.facts.ShadowFactBatch;
|
||||
import com.ruoyi.shadow.facts.ShadowMapFact;
|
||||
import com.ruoyi.shadow.facts.ShadowQueueFact;
|
||||
import com.ruoyi.shadow.facts.ShadowQueueEndpoints;
|
||||
import com.ruoyi.shadow.facts.ShadowVehicleFact;
|
||||
import com.ruoyi.wcs.application.dispatch.DispatchSnapshot;
|
||||
import com.ruoyi.wcs.domain.dispatch.DispatchTask;
|
||||
@@ -106,8 +107,8 @@ public final class ShadowDispatchSnapshotAssembler {
|
||||
if (!ACTIVE_QUEUE.equals(fact.getTableName())) {
|
||||
continue;
|
||||
}
|
||||
String source = required(fact.getStartSite(), "startSite");
|
||||
String destination = required(fact.getEndSite(), "endSite");
|
||||
String source = required(ShadowQueueEndpoints.start(fact, batch), "startSite");
|
||||
String destination = required(ShadowQueueEndpoints.end(fact, batch), "endSite");
|
||||
if (!topology.containsNode(source)
|
||||
|| !topology.containsNode(destination)) {
|
||||
throw new IllegalArgumentException(
|
||||
@@ -174,8 +175,8 @@ public final class ShadowDispatchSnapshotAssembler {
|
||||
EnumSet.of(DeviceCapability.FOUR_WAY_MOVE,
|
||||
DeviceCapability.LOAD_HANDLING,
|
||||
DeviceCapability.LIFT_TRANSFER));
|
||||
boolean connectedAndSendable = "1".equals(optional(
|
||||
fact.getAgvStatus()))
|
||||
// Read-only projection of the legacy PLC gate; agvStatus is not refreshed by PLC polling.
|
||||
boolean connectedAndSendable = Integer.valueOf(5).equals(fact.getWorkMode())
|
||||
&& Integer.valueOf(1).equals(fact.getAllowSend());
|
||||
boolean healthy = zeroOrMissing(fact.getAlarm())
|
||||
&& zeroOrMissing(fact.getErrorCode());
|
||||
|
||||
@@ -16,6 +16,7 @@ public final class ShadowStartupReport {
|
||||
private volatile String reasonCode;
|
||||
private volatile String permanentDegradedReason;
|
||||
private final Map<String, String> startupChecks;
|
||||
private volatile Map<String, Object> cycleDiagnostics = Collections.emptyMap();
|
||||
|
||||
private ShadowStartupReport(Status status, Instant checkedAt,
|
||||
Instant lastCompletedCycle,
|
||||
@@ -44,10 +45,15 @@ public final class ShadowStartupReport {
|
||||
public long getEvidenceBacklog() { return evidenceBacklog; }
|
||||
public String getReasonCode() { return reasonCode; }
|
||||
public Map<String, String> getStartupChecks() { return startupChecks; }
|
||||
public Map<String, Object> getCycleDiagnostics() { return cycleDiagnostics; }
|
||||
|
||||
public void recordCycleDiagnostics(Map<String, Object> diagnostics) {
|
||||
cycleDiagnostics = Collections.unmodifiableMap(new LinkedHashMap<String, Object>(diagnostics));
|
||||
}
|
||||
|
||||
public synchronized void markReady(Instant completedAt,
|
||||
Long sourceLagMillis) {
|
||||
if (permanentDegradedReason != null) {
|
||||
if (permanentDegradedReason != null || status == Status.STOPPED) {
|
||||
return;
|
||||
}
|
||||
this.status = Status.READY;
|
||||
@@ -60,7 +66,7 @@ public final class ShadowStartupReport {
|
||||
public synchronized void markDegraded(String reasonCode,
|
||||
Instant checkedAt) {
|
||||
String normalized = requireReasonCode(reasonCode);
|
||||
if (permanentDegradedReason != null) {
|
||||
if (permanentDegradedReason != null || status == Status.STOPPED) {
|
||||
return;
|
||||
}
|
||||
this.status = Status.DEGRADED;
|
||||
@@ -71,6 +77,7 @@ public final class ShadowStartupReport {
|
||||
public synchronized void markPermanentlyDegraded(String reasonCode,
|
||||
Instant checkedAt) {
|
||||
String normalized = requireReasonCode(reasonCode);
|
||||
if (status == Status.STOPPED) return;
|
||||
if (permanentDegradedReason == null) {
|
||||
permanentDegradedReason = normalized;
|
||||
this.checkedAt = checkedAt;
|
||||
@@ -79,6 +86,30 @@ public final class ShadowStartupReport {
|
||||
this.reasonCode = permanentDegradedReason;
|
||||
}
|
||||
|
||||
public synchronized void markStopped(Instant stoppedAt) {
|
||||
status = Status.STOPPED;
|
||||
checkedAt = stoppedAt;
|
||||
if (reasonCode == null) reasonCode = "SHADOW_APPLICATION_STOPPED";
|
||||
}
|
||||
|
||||
public synchronized Map<String, Object> snapshotRuntimeState(
|
||||
Instant heartbeatAt, Instant lastAttemptAt, boolean collectionStopped) {
|
||||
Map<String, Object> state = new LinkedHashMap<String, Object>();
|
||||
state.put("schemaVersion", 1);
|
||||
state.put("kind", "RUN_HEARTBEAT");
|
||||
state.put("heartbeatAt", heartbeatAt.toString());
|
||||
state.put("lastAttemptAt", lastAttemptAt == null ? null : lastAttemptAt.toString());
|
||||
state.put("lastCompletedCycle", lastCompletedCycle == null ? null : lastCompletedCycle.toString());
|
||||
state.put("checkedAt", checkedAt == null ? null : checkedAt.toString());
|
||||
state.put("status", status.name());
|
||||
state.put("reasonCode", reasonCode);
|
||||
state.put("collectionStopped", collectionStopped || status == Status.STOPPED
|
||||
|| permanentDegradedReason != null);
|
||||
state.put("sourceLagMillis", sourceLagMillis);
|
||||
state.put("cycleDiagnostics", cycleDiagnostics);
|
||||
return Collections.unmodifiableMap(state);
|
||||
}
|
||||
|
||||
private static String requireReasonCode(String reasonCode) {
|
||||
if (reasonCode == null || reasonCode.trim().isEmpty()) {
|
||||
throw new IllegalArgumentException(
|
||||
|
||||
Reference in New Issue
Block a user