metric optimization
This commit is contained in:
parent
f4b783f8af
commit
7bdf108532
2 changed files with 171 additions and 125 deletions
|
|
@ -1,11 +1,8 @@
|
|||
package com.alexanderlogistics.metrics;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.logging.Logger;
|
||||
|
||||
import org.imixs.marty.team.TeamService;
|
||||
import org.imixs.workflow.ItemCollection;
|
||||
import org.imixs.workflow.engine.DocumentService;
|
||||
import org.imixs.workflow.exceptions.PluginException;
|
||||
|
|
@ -33,9 +30,6 @@ public class MetricDebitorRestService {
|
|||
@Inject
|
||||
MetricDebitorService metricService;
|
||||
|
||||
@Inject
|
||||
TeamService teamService;
|
||||
|
||||
@GET
|
||||
@Path("/ping")
|
||||
@Produces({ MediaType.TEXT_PLAIN })
|
||||
|
|
@ -83,15 +77,16 @@ public class MetricDebitorRestService {
|
|||
@Path("/init")
|
||||
@Produces({ MediaType.TEXT_PLAIN })
|
||||
public Response initMetrics() {
|
||||
Map<String, ItemCollection> metricCache = new HashMap<String, ItemCollection>();
|
||||
// Map<String, ItemCollection> metricCache = new HashMap<String,
|
||||
// ItemCollection>();
|
||||
long l = System.currentTimeMillis();
|
||||
logger.info("├── init metrics...");
|
||||
try {
|
||||
groupInvoicesByDebitor(metricCache);
|
||||
groupInvoicesByDebitor();
|
||||
logger.info("│ ├── grouping invoices finished in " + (System.currentTimeMillis() - l) + "ms");
|
||||
|
||||
rebuildMetrics(metricCache);
|
||||
String message = "├── init " + metricCache.size() + " metrics completed in "
|
||||
rebuildMetrics();
|
||||
String message = "├── init metrics completed in "
|
||||
+ (System.currentTimeMillis() - l)
|
||||
+ "ms";
|
||||
logger.info(message);
|
||||
|
|
@ -105,11 +100,13 @@ public class MetricDebitorRestService {
|
|||
}
|
||||
}
|
||||
|
||||
private void rebuildMetrics(Map<String, ItemCollection> metricCache) throws PluginException {
|
||||
private void rebuildMetrics() throws PluginException {
|
||||
logger.info("│ ├── rebuild metrics...");
|
||||
|
||||
for (String dbtrNumber : metricCache.keySet()) {
|
||||
ItemCollection metricData = metricCache.get(dbtrNumber);
|
||||
List<String> keys = metricService.getMetricKeys();
|
||||
for (String hashKey : keys) {
|
||||
ItemCollection metricData = metricService.getMetric(hashKey);
|
||||
// metricCache.get(dbtrNumber);
|
||||
documentService.save(metricData);
|
||||
metricService.initMetric(metricData);
|
||||
}
|
||||
|
|
@ -125,8 +122,7 @@ public class MetricDebitorRestService {
|
|||
*
|
||||
*
|
||||
*/
|
||||
private void groupInvoicesByDebitor(
|
||||
Map<String, ItemCollection> metricCache) {
|
||||
private void groupInvoicesByDebitor() {
|
||||
|
||||
logger.info("│ │ ├── group invoices by debitor...");
|
||||
try {
|
||||
|
|
@ -135,35 +131,18 @@ public class MetricDebitorRestService {
|
|||
9999, 0,
|
||||
"invoice.number", false);
|
||||
|
||||
logger.fine(" found " + invoices.size() + " open invoices");
|
||||
logger.info("│ │ ├──found " + invoices.size() + " open invoices");
|
||||
for (ItemCollection invoice : invoices) {
|
||||
|
||||
String debitorName = invoice.getItemValueString("dbtr.name");
|
||||
String debitorNumber = invoice.getItemValueString("dbtr.number");
|
||||
|
||||
ItemCollection metricData = metricCache.get(debitorNumber);
|
||||
if (metricData == null) {
|
||||
try {
|
||||
metricData = metricService.loadMetric(debitorNumber, debitorName);
|
||||
// reset old metrics now!
|
||||
for (String itemName : metricData.getItemNames()) {
|
||||
if (itemName.startsWith(MetricDebitorService.ITEM_SALDO + ".")) {
|
||||
metricData.setItemValue(itemName, 0.0);
|
||||
}
|
||||
}
|
||||
} catch (PluginException e) {
|
||||
logger.severe(
|
||||
"Unable to load metric for invoice " + invoice.getUniqueID() + " : " + e.getMessage());
|
||||
continue;
|
||||
}
|
||||
}
|
||||
ItemCollection metricData = metricService.getMetricByInvoice(invoice);
|
||||
// Jetzt Rechnung addieren
|
||||
addInvoice(metricData, invoice);
|
||||
// invoiceData.add(invoice);
|
||||
metricCache.put(debitorNumber, metricData);
|
||||
logger.fine("....put invoice " + invoice.getUniqueID());
|
||||
metricService.putMetric(metricData);
|
||||
// metricCache.put(metricKey, metricData);
|
||||
}
|
||||
logger.info("│ │ ├── grouped " + invoices.size() + " invoices.");
|
||||
} catch (QueryException e) {
|
||||
} catch (QueryException | PluginException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
|
|
@ -176,11 +155,9 @@ public class MetricDebitorRestService {
|
|||
* @param invoice
|
||||
*/
|
||||
public void addInvoice(ItemCollection metricData, ItemCollection invoice) {
|
||||
String currency = invoice.getItemValueString("invoice.currency");
|
||||
double saldo = invoice.getItemValueDouble(MetricDebitorService.ITEM_SALDO);
|
||||
double saldoOld = metricData.getItemValueDouble(MetricDebitorService.ITEM_SALDO + "." + currency);
|
||||
double saldoOld = metricData.getItemValueDouble(MetricDebitorService.ITEM_SALDO);
|
||||
double saldoNew = InvoiceUtil.round(saldoOld + saldo);
|
||||
|
||||
metricData.setItemValue(MetricDebitorService.ITEM_SALDO + "." + currency, saldoNew);
|
||||
metricData.setItemValue(MetricDebitorService.ITEM_SALDO, saldoNew);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,9 @@
|
|||
package com.alexanderlogistics.metrics;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.logging.Logger;
|
||||
|
|
@ -30,7 +32,7 @@ import jakarta.inject.Inject;
|
|||
|
||||
/**
|
||||
* Dieser Service reagiert auch ProcessingEvents und speichert/aktualisiert die
|
||||
* zugehörige Debitoren Metric Entity.
|
||||
* zugehörige Debitoren Metric Entity (type=metric.debitor).
|
||||
* <p>
|
||||
* Der Service liest in einem AFTER_PROCESS Event den alten invoice.saldo aus.
|
||||
* Hierzu wird die Rechnung in einer neuen Transaktion geladen was einem
|
||||
|
|
@ -74,7 +76,7 @@ public class MetricDebitorService {
|
|||
* @param processingEvent
|
||||
* @throws PluginException
|
||||
*/
|
||||
public void onProcessingEvent(@Observes ProcessingEvent processingEvent) throws PluginException {
|
||||
public void onProcessingEvent(@Observes ProcessingEvent processingEvent) {
|
||||
|
||||
if (!metricsEnabled) {
|
||||
return;
|
||||
|
|
@ -86,45 +88,124 @@ public class MetricDebitorService {
|
|||
return;
|
||||
}
|
||||
|
||||
// verify if saldo has changed.....
|
||||
double invoiceSaldo = invoice.getItemValueDouble(ITEM_SALDO);
|
||||
if (!"workitem".equals(invoice.getType())) {
|
||||
// vorgang ist archiviert oder gelöscht worden => saldo = 0!
|
||||
invoiceSaldo = 0.0;
|
||||
try {
|
||||
// update metric and the metric cache
|
||||
if (processingEvent.getEventType() == ProcessingEvent.AFTER_PROCESS) {
|
||||
// verify if saldo has changed.....
|
||||
double invoiceSaldo = invoice.getItemValueDouble(ITEM_SALDO);
|
||||
if (!"workitem".equals(invoice.getType())) {
|
||||
// vorgang ist archiviert oder gelöscht worden => saldo = 0!
|
||||
invoiceSaldo = 0.0;
|
||||
}
|
||||
|
||||
// load last metric...
|
||||
double lastInvoiceSaldo = readDirtySaldo(invoice.getUniqueID());
|
||||
if (invoiceSaldo == lastInvoiceSaldo) {
|
||||
// no change - no metric update!
|
||||
return;
|
||||
}
|
||||
|
||||
ItemCollection metricData = getMetricByInvoice(invoice);
|
||||
|
||||
// Saldo-Berechnung
|
||||
debitorSaldo = metricData.getItemValueDouble(ITEM_SALDO);
|
||||
debitorSaldo = debitorSaldo - lastInvoiceSaldo;
|
||||
|
||||
// update debitor saldo
|
||||
debitorSaldo = InvoiceUtil.round(debitorSaldo + invoiceSaldo);
|
||||
metricData.setItemValue(ITEM_SALDO, debitorSaldo);
|
||||
putMetric(metricData);
|
||||
documentService.save(metricData);
|
||||
|
||||
// Gauge registrieren
|
||||
updateGauge(metricData);
|
||||
}
|
||||
} catch (PluginException e) {
|
||||
logger.warning("unable to process metric: " + e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the corresponding metric data object for an invoice. The method uses
|
||||
* an internal cache. If the metric was not yet cached the method loads the
|
||||
* metric from the database. If not metric exists in the database the method
|
||||
* automatically creates a new metric data object.
|
||||
*
|
||||
* @param invoice
|
||||
* @return
|
||||
* @throws PluginException
|
||||
*/
|
||||
public ItemCollection getMetricByInvoice(ItemCollection invoice) throws PluginException {
|
||||
String metricKey = buildKeyByInvoice(invoice);
|
||||
ItemCollection metricData = metricCache.get(metricKey);
|
||||
if (metricData == null) {
|
||||
metricData = loadMetric(invoice);
|
||||
}
|
||||
// if metric still null we create a new metric data object.
|
||||
if (metricData == null) {
|
||||
metricData = createMetaData(invoice);
|
||||
}
|
||||
return metricData;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a metric data object by key
|
||||
*
|
||||
* @param key
|
||||
* @return
|
||||
*/
|
||||
public ItemCollection getMetric(String key) {
|
||||
return metricCache.get(key);
|
||||
}
|
||||
|
||||
/**
|
||||
* Puts a metric data object into the cache.
|
||||
*
|
||||
* @param metricData
|
||||
*/
|
||||
public void putMetric(ItemCollection metricData) {
|
||||
metricCache.put(metricData.getItemValueString("name"), metricData);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a list with all cached metric keys.
|
||||
* The method returns an unmodifiable list to prevent modifications.
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
public List<String> getMetricKeys() {
|
||||
return Collections.unmodifiableList(new ArrayList<>(metricCache.keySet()));
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the metric hash key by the invoice attributes. The returned key can be
|
||||
* used for caching the metric.
|
||||
*
|
||||
* @param invoice - The invoice ItemCollection containing the necessary
|
||||
* attributes
|
||||
* @return A unique hash string based on the invoice attributes
|
||||
* @throws IllegalArgumentException if the invoice is null
|
||||
*/
|
||||
private static String buildKeyByInvoice(ItemCollection invoice) {
|
||||
// Validate input
|
||||
Objects.requireNonNull(invoice, "Invoice must not be null");
|
||||
|
||||
String dbtrNumber = invoice.getItemValueString("dbtr.number");
|
||||
String dbtrName = invoice.getItemValueString("dbtr.name");
|
||||
String currency = invoice.getItemValueString("invoice.currency");
|
||||
// load last metric...
|
||||
double lastInvoiceSaldo = readDirtySaldo(invoice.getUniqueID());
|
||||
if (invoiceSaldo == lastInvoiceSaldo) {
|
||||
// no change - no metric update!
|
||||
return;
|
||||
}
|
||||
String department = invoice.getItemValueString("space.name");
|
||||
|
||||
// update metric and the metric cache
|
||||
if (processingEvent.getEventType() == ProcessingEvent.AFTER_PROCESS) {
|
||||
// Concatenate the values and create a hash
|
||||
String combinedValue = String.format("%s::%s::%s::%s",
|
||||
dbtrNumber,
|
||||
dbtrName,
|
||||
currency,
|
||||
department);
|
||||
|
||||
ItemCollection metricData = metricCache.get(dbtrNumber);
|
||||
if (metricData == null) {
|
||||
metricData = loadMetric(dbtrNumber, dbtrName);
|
||||
}
|
||||
// Saldo-Berechnung
|
||||
debitorSaldo = metricData.getItemValueDouble(ITEM_SALDO + "." + currency); // totals.get(currency);
|
||||
debitorSaldo = debitorSaldo - lastInvoiceSaldo;
|
||||
|
||||
// update debitor saldo
|
||||
debitorSaldo = InvoiceUtil.round(debitorSaldo + invoiceSaldo);
|
||||
|
||||
metricData.setItemValue(ITEM_SALDO + "." + currency, debitorSaldo);
|
||||
metricCache.put(dbtrNumber, metricData);
|
||||
// Speichern in der Datenbank
|
||||
logger.info("Update metric for debitor: " + dbtrNumber + " -> " + debitorSaldo);
|
||||
documentService.save(metricData);
|
||||
|
||||
// Gauge registrieren
|
||||
updateGauge(metricData, currency);
|
||||
}
|
||||
String hash = String.valueOf(combinedValue.hashCode());
|
||||
// a hash can start with '-' which we need to avoid and create a alphanumeric
|
||||
// key instead!
|
||||
return "HASH" + hash;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -146,34 +227,26 @@ public class MetricDebitorService {
|
|||
}
|
||||
|
||||
/**
|
||||
* This method loads a metric entity. If no metric entity exits, the method
|
||||
* creates a new one.
|
||||
* This method loads a metric entity for a given invoice workitem. If no metric
|
||||
* entity exits, the method
|
||||
* creates a new metric entity.
|
||||
*
|
||||
* @param dbtrID
|
||||
* @param invoice
|
||||
* @return
|
||||
* @throws PluginException
|
||||
*/
|
||||
public ItemCollection loadMetric(String dbtrNumber, String dbtrName) throws PluginException {
|
||||
|
||||
private ItemCollection loadMetric(ItemCollection invoice) throws PluginException {
|
||||
ItemCollection debitorMetric = null;
|
||||
if (dbtrNumber == null || dbtrNumber.isEmpty()) {
|
||||
throw new PluginException(PluginException.class.getName(), "QUERY ERROR",
|
||||
"missing debitor number");
|
||||
}
|
||||
try {
|
||||
String query = "(type:" + TYPE_METRIC_DEBITOR + ") AND (name:" + dbtrNumber + ")";
|
||||
String metricKey = buildKeyByInvoice(invoice);
|
||||
String query = "(type:" + TYPE_METRIC_DEBITOR + ") AND (name:" + metricKey + ")";
|
||||
List<ItemCollection> result = documentService.find(query, 1, 0, "$modified", true);
|
||||
if (result.size() > 0) {
|
||||
debitorMetric = result.get(0);
|
||||
}
|
||||
if (debitorMetric == null) {
|
||||
// create a new instance
|
||||
// logger.info("creating new debitor metric");
|
||||
debitorMetric = createMetaData(dbtrNumber, dbtrName);
|
||||
|
||||
}
|
||||
} catch (QueryException e) {
|
||||
throw new PluginException(PluginException.class.getName(), "QUERY ERROR", e.getMessage(), e);
|
||||
} catch (IllegalArgumentException | QueryException e) {
|
||||
throw new PluginException(PluginException.class.getName(),
|
||||
"Failed to load metric object for invoice " + invoice.getUniqueID() + ": ", e.getMessage(), e);
|
||||
}
|
||||
|
||||
return debitorMetric;
|
||||
|
|
@ -185,18 +258,18 @@ public class MetricDebitorService {
|
|||
* The ItemCollection stores the name and number and also all saldos for all
|
||||
* currencies
|
||||
*
|
||||
* @param dbtrNumber
|
||||
* @param dbtrName
|
||||
* @param invoice - invoice ItemCollection
|
||||
* @return
|
||||
*/
|
||||
private ItemCollection createMetaData(String dbtrNumber, String dbtrName) {
|
||||
private ItemCollection createMetaData(ItemCollection invoice) {
|
||||
String key = buildKeyByInvoice(invoice);
|
||||
ItemCollection metricData = new ItemCollection();
|
||||
|
||||
metricData = new ItemCollection();
|
||||
metricData.setType(TYPE_METRIC_DEBITOR);
|
||||
metricData.setItemValue("name", dbtrNumber);
|
||||
metricData.setItemValue("dbtr.number", dbtrNumber);
|
||||
metricData.setItemValue("dbtr.name", dbtrName);
|
||||
metricData.setItemValue("name", key);
|
||||
metricData.setItemValue("dbtr.number", invoice.getItemValueString("dbtr.number"));
|
||||
metricData.setItemValue("dbtr.name", invoice.getItemValueString("dbtr.name"));
|
||||
metricData.setItemValue("invoice.currency", invoice.getItemValueString("invoice.currency"));
|
||||
metricData.setItemValue("space.name", invoice.getItemValueString("space.name"));
|
||||
return metricData;
|
||||
}
|
||||
|
||||
|
|
@ -206,28 +279,32 @@ public class MetricDebitorService {
|
|||
* @param dbtrNumber - the debitor number
|
||||
* @param dbtrName - the debitor name
|
||||
*/
|
||||
private void updateGauge(ItemCollection metricData, String currency) {
|
||||
private void updateGauge(ItemCollection metricData) {
|
||||
String dbtrNumber = metricData.getItemValueString("dbtr.number");
|
||||
String dbtrName = metricData.getItemValueString("dbtr.name");
|
||||
String metricKey = "dbtr_" + dbtrNumber + "_" + dbtrName + "_" + currency;
|
||||
String metricKey = metricData.getItemValueString("name");
|
||||
// Prüfen ob Gauge bereits registriert ist
|
||||
if (registeredGauges.add(metricKey)) { // returns true newly added
|
||||
List<Tag> tags = new ArrayList<>();
|
||||
tags.add(new Tag("type", "dbtr"));
|
||||
tags.add(new Tag("number", dbtrNumber));
|
||||
tags.add(new Tag("name", dbtrName));
|
||||
tags.add(new Tag("currency", currency));
|
||||
logger.info("register new metric for debitor: " + dbtrNumber + " -> " + currency);
|
||||
tags.add(new Tag("currency", metricData.getItemValueString("invoice.currency")));
|
||||
tags.add(new Tag("department", metricData.getItemValueString("space.name")));
|
||||
logger.info("register new metric for debitor: " + dbtrNumber +
|
||||
", " + metricData.getItemValueString(ITEM_SALDO) +
|
||||
" " + metricData.getItemValueString("invoice.currency"));
|
||||
Metadata metadata = Metadata.builder()
|
||||
.withName("dbtr.invoice.saldo")
|
||||
.withDescription("Debitor Balance by Currency")
|
||||
.withName("dbtr.balance")
|
||||
.withDescription("Debitor Balance")
|
||||
.build();
|
||||
|
||||
metricCache.get(metricKey);
|
||||
metricRegistry.gauge(metadata,
|
||||
() -> metricCache.getOrDefault(dbtrNumber, createMetaData(dbtrNumber, dbtrName))
|
||||
.getItemValueDouble(ITEM_SALDO + "." + currency),
|
||||
() -> metricCache.get(metricKey).getItemValueDouble(ITEM_SALDO),
|
||||
tags.toArray(new Tag[0]));
|
||||
} else {
|
||||
logger.fine("Gauge already registered for debitor: " + dbtrNumber + " -> " + currency);
|
||||
logger.fine("Gauge already registered for debitor: " + dbtrNumber);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -238,19 +315,11 @@ public class MetricDebitorService {
|
|||
*
|
||||
*/
|
||||
protected void initMetric(ItemCollection metric) {
|
||||
String dbtrNumber = metric.getItemValueString("dbtr.number");
|
||||
// Aktuellen Saldo in Map speichern
|
||||
metricCache.put(dbtrNumber, metric);
|
||||
// Gauge für jede Währung registrieren
|
||||
for (String itemName : metric.getItemNames()) {
|
||||
if (itemName.startsWith(ITEM_SALDO + ".")) {
|
||||
// currency value found
|
||||
int pos = (ITEM_SALDO + ".").length();
|
||||
String currency = itemName.substring(pos).toUpperCase();
|
||||
logger.info("│ │ ├──init metric for debitor " + dbtrNumber + " -> " + currency);
|
||||
updateGauge(metric, currency);
|
||||
}
|
||||
}
|
||||
String key = metric.getItemValueString("name");
|
||||
// Aktuelle metric cachen
|
||||
metricCache.put(key, metric);
|
||||
logger.info("│ │ ├──init metric for debitor " + key);
|
||||
updateGauge(metric);
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Reference in a new issue