222 lines
9.9 KiB
Java
222 lines
9.9 KiB
Java
|
package com.common.service;
|
||
|
|
|
||
|
|
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||
|
|
|
||
|
|
import java.time.LocalDate;
|
||
|
|
import java.time.LocalDateTime;
|
||
|
|
import java.util.ArrayList;
|
||
|
|
import java.util.List;
|
||
|
|
import java.util.Map;
|
||
|
|
import java.util.UUID;
|
||
|
|
|
||
|
|
/**
|
||
|
|
* A durable, batch-aggregated statistics delta written beside the fact rows.
|
||
|
|
*
|
||
|
|
* <p>The event is additive/max-min only, so events from different application nodes may be
|
||
|
|
* published in a different order without changing the final projection. Kafka duplicates are
|
||
|
|
* eliminated by {@code eventId} in the projection consumer transaction.</p>
|
||
|
|
*/
|
||
|
|
@JsonIgnoreProperties(ignoreUnknown = true)
|
||
|
|
public class StatsProjectionOutboxEvent {
|
||
|
|
public static final String SCHEMA = "SYSLOG_STATS_PROJECTION_V1";
|
||
|
|
public static final String KAFKA_KEY = "SYSLOG_STATS_PROJECTION";
|
||
|
|
|
||
|
|
private String schema = SCHEMA;
|
||
|
|
private String eventId = UUID.randomUUID().toString();
|
||
|
|
private List<DeviceDelta> devices = new ArrayList<>();
|
||
|
|
private List<RuleDelta> rules = new ArrayList<>();
|
||
|
|
private List<CollectDelta> collects = new ArrayList<>();
|
||
|
|
|
||
|
|
public static StatsProjectionOutboxEvent fromAccumulator(DirectStatsAccumulator accumulator) {
|
||
|
|
StatsProjectionOutboxEvent event = new StatsProjectionOutboxEvent();
|
||
|
|
for (Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.DeviceDelta> entry
|
||
|
|
: accumulator.getDevices().entrySet()) {
|
||
|
|
DirectStatsAccumulator.DeviceDelta value = entry.getValue();
|
||
|
|
if (value.getParseDelta() != 0L || value.getNonLogDelta() != 0L) {
|
||
|
|
event.devices.add(DeviceDelta.counts(entry.getKey().getDate(),
|
||
|
|
entry.getKey().getDimensionId(), value.getParseDelta(), value.getNonLogDelta()));
|
||
|
|
}
|
||
|
|
}
|
||
|
|
for (Map.Entry<DirectStatsAccumulator.DailyKey, DirectStatsAccumulator.RuleDelta> entry
|
||
|
|
: accumulator.getRules().entrySet()) {
|
||
|
|
if (entry.getValue().getMaxLogTime() != null) {
|
||
|
|
event.rules.add(RuleDelta.max(entry.getKey().getDate(),
|
||
|
|
entry.getKey().getDimensionId(), entry.getValue().getMaxLogTime()));
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return event;
|
||
|
|
}
|
||
|
|
|
||
|
|
static StatsProjectionOutboxEvent fromIngress(IngressStatsProjectionWriter.Aggregation aggregation) {
|
||
|
|
StatsProjectionOutboxEvent event = new StatsProjectionOutboxEvent();
|
||
|
|
for (Map.Entry<IngressStatsProjectionWriter.DeviceKey, LocalDateTime> entry
|
||
|
|
: aggregation.devices.entrySet()) {
|
||
|
|
event.devices.add(DeviceDelta.received(entry.getKey().date,
|
||
|
|
entry.getKey().deviceId, entry.getValue()));
|
||
|
|
}
|
||
|
|
for (Map.Entry<Long, IngressStatsProjectionWriter.CollectTimes> entry
|
||
|
|
: aggregation.collects.entrySet()) {
|
||
|
|
event.collects.add(CollectDelta.range(entry.getKey(),
|
||
|
|
entry.getValue().min, entry.getValue().max));
|
||
|
|
}
|
||
|
|
return event;
|
||
|
|
}
|
||
|
|
|
||
|
|
public void validate() {
|
||
|
|
if (!SCHEMA.equals(schema)) {
|
||
|
|
throw new IllegalArgumentException("Unsupported stats projection schema: " + schema);
|
||
|
|
}
|
||
|
|
if (eventId == null || eventId.trim().isEmpty()) {
|
||
|
|
throw new IllegalArgumentException("Stats projection event is missing eventId");
|
||
|
|
}
|
||
|
|
if (devices == null || rules == null || collects == null) {
|
||
|
|
throw new IllegalArgumentException("Stats projection event contains null collections");
|
||
|
|
}
|
||
|
|
if (devices.isEmpty() && rules.isEmpty() && collects.isEmpty()) {
|
||
|
|
throw new IllegalArgumentException("Stats projection event has no changes");
|
||
|
|
}
|
||
|
|
for (DeviceDelta delta : devices) delta.validate();
|
||
|
|
for (RuleDelta delta : rules) delta.validate();
|
||
|
|
for (CollectDelta delta : collects) delta.validate();
|
||
|
|
}
|
||
|
|
|
||
|
|
public boolean isEmpty() {
|
||
|
|
return devices.isEmpty() && rules.isEmpty() && collects.isEmpty();
|
||
|
|
}
|
||
|
|
|
||
|
|
public String getSchema() { return schema; }
|
||
|
|
public void setSchema(String schema) { this.schema = schema; }
|
||
|
|
public String getEventId() { return eventId; }
|
||
|
|
public void setEventId(String eventId) { this.eventId = eventId; }
|
||
|
|
public List<DeviceDelta> getDevices() { return devices; }
|
||
|
|
public void setDevices(List<DeviceDelta> devices) { this.devices = devices; }
|
||
|
|
public List<RuleDelta> getRules() { return rules; }
|
||
|
|
public void setRules(List<RuleDelta> rules) { this.rules = rules; }
|
||
|
|
public List<CollectDelta> getCollects() { return collects; }
|
||
|
|
public void setCollects(List<CollectDelta> collects) { this.collects = collects; }
|
||
|
|
|
||
|
|
@JsonIgnoreProperties(ignoreUnknown = true)
|
||
|
|
public static class DeviceDelta {
|
||
|
|
private String statDate;
|
||
|
|
private long deviceId;
|
||
|
|
private long parseDelta;
|
||
|
|
private long nonLogDelta;
|
||
|
|
private String lastReceiveTime;
|
||
|
|
|
||
|
|
public DeviceDelta() {}
|
||
|
|
|
||
|
|
static DeviceDelta counts(LocalDate date, long deviceId, long parseDelta, long nonLogDelta) {
|
||
|
|
DeviceDelta result = new DeviceDelta();
|
||
|
|
result.statDate = date.toString();
|
||
|
|
result.deviceId = deviceId;
|
||
|
|
result.parseDelta = parseDelta;
|
||
|
|
result.nonLogDelta = nonLogDelta;
|
||
|
|
return result;
|
||
|
|
}
|
||
|
|
|
||
|
|
static DeviceDelta received(LocalDate date, long deviceId, LocalDateTime receiveTime) {
|
||
|
|
DeviceDelta result = counts(date, deviceId, 0L, 0L);
|
||
|
|
result.lastReceiveTime = receiveTime.toString();
|
||
|
|
return result;
|
||
|
|
}
|
||
|
|
|
||
|
|
void validate() {
|
||
|
|
date();
|
||
|
|
if (deviceId <= 0L) throw new IllegalArgumentException("Invalid device projection id");
|
||
|
|
if (parseDelta == 0L && nonLogDelta == 0L && receiveTime() == null) {
|
||
|
|
throw new IllegalArgumentException("Device projection delta is empty");
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
public LocalDate date() { return LocalDate.parse(statDate); }
|
||
|
|
public LocalDateTime receiveTime() {
|
||
|
|
return lastReceiveTime == null || lastReceiveTime.trim().isEmpty()
|
||
|
|
? null : LocalDateTime.parse(lastReceiveTime);
|
||
|
|
}
|
||
|
|
public String getStatDate() { return statDate; }
|
||
|
|
public void setStatDate(String statDate) { this.statDate = statDate; }
|
||
|
|
public long getDeviceId() { return deviceId; }
|
||
|
|
public void setDeviceId(long deviceId) { this.deviceId = deviceId; }
|
||
|
|
public long getParseDelta() { return parseDelta; }
|
||
|
|
public void setParseDelta(long parseDelta) { this.parseDelta = parseDelta; }
|
||
|
|
public long getNonLogDelta() { return nonLogDelta; }
|
||
|
|
public void setNonLogDelta(long nonLogDelta) { this.nonLogDelta = nonLogDelta; }
|
||
|
|
public String getLastReceiveTime() { return lastReceiveTime; }
|
||
|
|
public void setLastReceiveTime(String lastReceiveTime) { this.lastReceiveTime = lastReceiveTime; }
|
||
|
|
}
|
||
|
|
|
||
|
|
@JsonIgnoreProperties(ignoreUnknown = true)
|
||
|
|
public static class RuleDelta {
|
||
|
|
private String statDate;
|
||
|
|
private long normalizeRuleId;
|
||
|
|
private String maxLogTime;
|
||
|
|
|
||
|
|
public RuleDelta() {}
|
||
|
|
|
||
|
|
static RuleDelta max(LocalDate date, long ruleId, LocalDateTime maxLogTime) {
|
||
|
|
RuleDelta result = new RuleDelta();
|
||
|
|
result.statDate = date.toString();
|
||
|
|
result.normalizeRuleId = ruleId;
|
||
|
|
result.maxLogTime = maxLogTime.toString();
|
||
|
|
return result;
|
||
|
|
}
|
||
|
|
|
||
|
|
void validate() {
|
||
|
|
date();
|
||
|
|
if (normalizeRuleId <= 0L || time() == null) {
|
||
|
|
throw new IllegalArgumentException("Invalid rule projection delta");
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
public LocalDate date() { return LocalDate.parse(statDate); }
|
||
|
|
public LocalDateTime time() {
|
||
|
|
return maxLogTime == null || maxLogTime.trim().isEmpty()
|
||
|
|
? null : LocalDateTime.parse(maxLogTime);
|
||
|
|
}
|
||
|
|
public String getStatDate() { return statDate; }
|
||
|
|
public void setStatDate(String statDate) { this.statDate = statDate; }
|
||
|
|
public long getNormalizeRuleId() { return normalizeRuleId; }
|
||
|
|
public void setNormalizeRuleId(long normalizeRuleId) { this.normalizeRuleId = normalizeRuleId; }
|
||
|
|
public String getMaxLogTime() { return maxLogTime; }
|
||
|
|
public void setMaxLogTime(String maxLogTime) { this.maxLogTime = maxLogTime; }
|
||
|
|
}
|
||
|
|
|
||
|
|
@JsonIgnoreProperties(ignoreUnknown = true)
|
||
|
|
public static class CollectDelta {
|
||
|
|
private long deviceCollectId;
|
||
|
|
private String firstSuccessTime;
|
||
|
|
private String lastSuccessTime;
|
||
|
|
|
||
|
|
public CollectDelta() {}
|
||
|
|
|
||
|
|
static CollectDelta range(long collectId, LocalDateTime first, LocalDateTime last) {
|
||
|
|
CollectDelta result = new CollectDelta();
|
||
|
|
result.deviceCollectId = collectId;
|
||
|
|
result.firstSuccessTime = first == null ? null : first.toString();
|
||
|
|
result.lastSuccessTime = last == null ? null : last.toString();
|
||
|
|
return result;
|
||
|
|
}
|
||
|
|
|
||
|
|
void validate() {
|
||
|
|
if (deviceCollectId <= 0L || (firstTime() == null && lastTime() == null)) {
|
||
|
|
throw new IllegalArgumentException("Invalid collector projection delta");
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
public LocalDateTime firstTime() {
|
||
|
|
return firstSuccessTime == null || firstSuccessTime.trim().isEmpty()
|
||
|
|
? null : LocalDateTime.parse(firstSuccessTime);
|
||
|
|
}
|
||
|
|
public LocalDateTime lastTime() {
|
||
|
|
return lastSuccessTime == null || lastSuccessTime.trim().isEmpty()
|
||
|
|
? null : LocalDateTime.parse(lastSuccessTime);
|
||
|
|
}
|
||
|
|
public long getDeviceCollectId() { return deviceCollectId; }
|
||
|
|
public void setDeviceCollectId(long deviceCollectId) { this.deviceCollectId = deviceCollectId; }
|
||
|
|
public String getFirstSuccessTime() { return firstSuccessTime; }
|
||
|
|
public void setFirstSuccessTime(String firstSuccessTime) { this.firstSuccessTime = firstSuccessTime; }
|
||
|
|
public String getLastSuccessTime() { return lastSuccessTime; }
|
||
|
|
public void setLastSuccessTime(String lastSuccessTime) { this.lastSuccessTime = lastSuccessTime; }
|
||
|
|
}
|
||
|
|
}
|