From 7bdf10853263502c8648c653767b431edd41c2b6 Mon Sep 17 00:00:00 2001 From: Ralph Soika Date: Tue, 3 Dec 2024 16:02:12 +0100 Subject: [PATCH] metric optimization --- .../metrics/MetricDebitorRestService.java | 61 ++--- .../metrics/MetricDebitorService.java | 235 +++++++++++------- 2 files changed, 171 insertions(+), 125 deletions(-) diff --git a/office-alexander-logistics-app/src/main/java/com/alexanderlogistics/metrics/MetricDebitorRestService.java b/office-alexander-logistics-app/src/main/java/com/alexanderlogistics/metrics/MetricDebitorRestService.java index a7a8f3e..0b4129d 100644 --- a/office-alexander-logistics-app/src/main/java/com/alexanderlogistics/metrics/MetricDebitorRestService.java +++ b/office-alexander-logistics-app/src/main/java/com/alexanderlogistics/metrics/MetricDebitorRestService.java @@ -1,11 +1,8 @@ package com.alexanderlogistics.metrics; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.logging.Logger; -import org.imixs.marty.team.TeamService; import org.imixs.workflow.ItemCollection; import org.imixs.workflow.engine.DocumentService; import org.imixs.workflow.exceptions.PluginException; @@ -33,9 +30,6 @@ public class MetricDebitorRestService { @Inject MetricDebitorService metricService; - @Inject - TeamService teamService; - @GET @Path("/ping") @Produces({ MediaType.TEXT_PLAIN }) @@ -83,15 +77,16 @@ public class MetricDebitorRestService { @Path("/init") @Produces({ MediaType.TEXT_PLAIN }) public Response initMetrics() { - Map metricCache = new HashMap(); + // Map metricCache = new HashMap(); long l = System.currentTimeMillis(); logger.info("├── init metrics..."); try { - groupInvoicesByDebitor(metricCache); + groupInvoicesByDebitor(); logger.info("│   ├── grouping invoices finished in " + (System.currentTimeMillis() - l) + "ms"); - rebuildMetrics(metricCache); - String message = "├── init " + metricCache.size() + " metrics completed in " + rebuildMetrics(); + String message = "├── init metrics completed in " + (System.currentTimeMillis() - l) + "ms"; logger.info(message); @@ -105,11 +100,13 @@ public class MetricDebitorRestService { } } - private void rebuildMetrics(Map metricCache) throws PluginException { + private void rebuildMetrics() throws PluginException { logger.info("│   ├── rebuild metrics..."); - for (String dbtrNumber : metricCache.keySet()) { - ItemCollection metricData = metricCache.get(dbtrNumber); + List keys = metricService.getMetricKeys(); + for (String hashKey : keys) { + ItemCollection metricData = metricService.getMetric(hashKey); + // metricCache.get(dbtrNumber); documentService.save(metricData); metricService.initMetric(metricData); } @@ -125,8 +122,7 @@ public class MetricDebitorRestService { * * */ - private void groupInvoicesByDebitor( - Map metricCache) { + private void groupInvoicesByDebitor() { logger.info("│   │   ├── group invoices by debitor..."); try { @@ -135,35 +131,18 @@ public class MetricDebitorRestService { 9999, 0, "invoice.number", false); - logger.fine(" found " + invoices.size() + " open invoices"); + logger.info("│   │   ├──found " + invoices.size() + " open invoices"); for (ItemCollection invoice : invoices) { - - String debitorName = invoice.getItemValueString("dbtr.name"); - String debitorNumber = invoice.getItemValueString("dbtr.number"); - - ItemCollection metricData = metricCache.get(debitorNumber); - if (metricData == null) { - try { - metricData = metricService.loadMetric(debitorNumber, debitorName); - // reset old metrics now! - for (String itemName : metricData.getItemNames()) { - if (itemName.startsWith(MetricDebitorService.ITEM_SALDO + ".")) { - metricData.setItemValue(itemName, 0.0); - } - } - } catch (PluginException e) { - logger.severe( - "Unable to load metric for invoice " + invoice.getUniqueID() + " : " + e.getMessage()); - continue; - } - } + ItemCollection metricData = metricService.getMetricByInvoice(invoice); // Jetzt Rechnung addieren addInvoice(metricData, invoice); // invoiceData.add(invoice); - metricCache.put(debitorNumber, metricData); + logger.fine("....put invoice " + invoice.getUniqueID()); + metricService.putMetric(metricData); + // metricCache.put(metricKey, metricData); } logger.info("│   │   ├── grouped " + invoices.size() + " invoices."); - } catch (QueryException e) { + } catch (QueryException | PluginException e) { e.printStackTrace(); } @@ -176,11 +155,9 @@ public class MetricDebitorRestService { * @param invoice */ public void addInvoice(ItemCollection metricData, ItemCollection invoice) { - String currency = invoice.getItemValueString("invoice.currency"); double saldo = invoice.getItemValueDouble(MetricDebitorService.ITEM_SALDO); - double saldoOld = metricData.getItemValueDouble(MetricDebitorService.ITEM_SALDO + "." + currency); + double saldoOld = metricData.getItemValueDouble(MetricDebitorService.ITEM_SALDO); double saldoNew = InvoiceUtil.round(saldoOld + saldo); - - metricData.setItemValue(MetricDebitorService.ITEM_SALDO + "." + currency, saldoNew); + metricData.setItemValue(MetricDebitorService.ITEM_SALDO, saldoNew); } } diff --git a/office-alexander-logistics-app/src/main/java/com/alexanderlogistics/metrics/MetricDebitorService.java b/office-alexander-logistics-app/src/main/java/com/alexanderlogistics/metrics/MetricDebitorService.java index 009daa2..2e9edd2 100644 --- a/office-alexander-logistics-app/src/main/java/com/alexanderlogistics/metrics/MetricDebitorService.java +++ b/office-alexander-logistics-app/src/main/java/com/alexanderlogistics/metrics/MetricDebitorService.java @@ -1,7 +1,9 @@ package com.alexanderlogistics.metrics; import java.util.ArrayList; +import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.logging.Logger; @@ -30,7 +32,7 @@ import jakarta.inject.Inject; /** * Dieser Service reagiert auch ProcessingEvents und speichert/aktualisiert die - * zugehörige Debitoren Metric Entity. + * zugehörige Debitoren Metric Entity (type=metric.debitor). *

* Der Service liest in einem AFTER_PROCESS Event den alten invoice.saldo aus. * Hierzu wird die Rechnung in einer neuen Transaktion geladen was einem @@ -74,7 +76,7 @@ public class MetricDebitorService { * @param processingEvent * @throws PluginException */ - public void onProcessingEvent(@Observes ProcessingEvent processingEvent) throws PluginException { + public void onProcessingEvent(@Observes ProcessingEvent processingEvent) { if (!metricsEnabled) { return; @@ -86,45 +88,124 @@ public class MetricDebitorService { return; } - // verify if saldo has changed..... - double invoiceSaldo = invoice.getItemValueDouble(ITEM_SALDO); - if (!"workitem".equals(invoice.getType())) { - // vorgang ist archiviert oder gelöscht worden => saldo = 0! - invoiceSaldo = 0.0; + try { + // update metric and the metric cache + if (processingEvent.getEventType() == ProcessingEvent.AFTER_PROCESS) { + // verify if saldo has changed..... + double invoiceSaldo = invoice.getItemValueDouble(ITEM_SALDO); + if (!"workitem".equals(invoice.getType())) { + // vorgang ist archiviert oder gelöscht worden => saldo = 0! + invoiceSaldo = 0.0; + } + + // load last metric... + double lastInvoiceSaldo = readDirtySaldo(invoice.getUniqueID()); + if (invoiceSaldo == lastInvoiceSaldo) { + // no change - no metric update! + return; + } + + ItemCollection metricData = getMetricByInvoice(invoice); + + // Saldo-Berechnung + debitorSaldo = metricData.getItemValueDouble(ITEM_SALDO); + debitorSaldo = debitorSaldo - lastInvoiceSaldo; + + // update debitor saldo + debitorSaldo = InvoiceUtil.round(debitorSaldo + invoiceSaldo); + metricData.setItemValue(ITEM_SALDO, debitorSaldo); + putMetric(metricData); + documentService.save(metricData); + + // Gauge registrieren + updateGauge(metricData); + } + } catch (PluginException e) { + logger.warning("unable to process metric: " + e.getMessage()); } + } + + /** + * Returns the corresponding metric data object for an invoice. The method uses + * an internal cache. If the metric was not yet cached the method loads the + * metric from the database. If not metric exists in the database the method + * automatically creates a new metric data object. + * + * @param invoice + * @return + * @throws PluginException + */ + public ItemCollection getMetricByInvoice(ItemCollection invoice) throws PluginException { + String metricKey = buildKeyByInvoice(invoice); + ItemCollection metricData = metricCache.get(metricKey); + if (metricData == null) { + metricData = loadMetric(invoice); + } + // if metric still null we create a new metric data object. + if (metricData == null) { + metricData = createMetaData(invoice); + } + return metricData; + } + + /** + * Returns a metric data object by key + * + * @param key + * @return + */ + public ItemCollection getMetric(String key) { + return metricCache.get(key); + } + + /** + * Puts a metric data object into the cache. + * + * @param metricData + */ + public void putMetric(ItemCollection metricData) { + metricCache.put(metricData.getItemValueString("name"), metricData); + } + + /** + * Returns a list with all cached metric keys. + * The method returns an unmodifiable list to prevent modifications. + * + * @return + */ + public List getMetricKeys() { + return Collections.unmodifiableList(new ArrayList<>(metricCache.keySet())); + } + + /** + * Builds the metric hash key by the invoice attributes. The returned key can be + * used for caching the metric. + * + * @param invoice - The invoice ItemCollection containing the necessary + * attributes + * @return A unique hash string based on the invoice attributes + * @throws IllegalArgumentException if the invoice is null + */ + private static String buildKeyByInvoice(ItemCollection invoice) { + // Validate input + Objects.requireNonNull(invoice, "Invoice must not be null"); + String dbtrNumber = invoice.getItemValueString("dbtr.number"); String dbtrName = invoice.getItemValueString("dbtr.name"); String currency = invoice.getItemValueString("invoice.currency"); - // load last metric... - double lastInvoiceSaldo = readDirtySaldo(invoice.getUniqueID()); - if (invoiceSaldo == lastInvoiceSaldo) { - // no change - no metric update! - return; - } + String department = invoice.getItemValueString("space.name"); - // update metric and the metric cache - if (processingEvent.getEventType() == ProcessingEvent.AFTER_PROCESS) { + // Concatenate the values and create a hash + String combinedValue = String.format("%s::%s::%s::%s", + dbtrNumber, + dbtrName, + currency, + department); - ItemCollection metricData = metricCache.get(dbtrNumber); - if (metricData == null) { - metricData = loadMetric(dbtrNumber, dbtrName); - } - // Saldo-Berechnung - debitorSaldo = metricData.getItemValueDouble(ITEM_SALDO + "." + currency); // totals.get(currency); - debitorSaldo = debitorSaldo - lastInvoiceSaldo; - - // update debitor saldo - debitorSaldo = InvoiceUtil.round(debitorSaldo + invoiceSaldo); - - metricData.setItemValue(ITEM_SALDO + "." + currency, debitorSaldo); - metricCache.put(dbtrNumber, metricData); - // Speichern in der Datenbank - logger.info("Update metric for debitor: " + dbtrNumber + " -> " + debitorSaldo); - documentService.save(metricData); - - // Gauge registrieren - updateGauge(metricData, currency); - } + String hash = String.valueOf(combinedValue.hashCode()); + // a hash can start with '-' which we need to avoid and create a alphanumeric + // key instead! + return "HASH" + hash; } /** @@ -146,34 +227,26 @@ public class MetricDebitorService { } /** - * This method loads a metric entity. If no metric entity exits, the method - * creates a new one. + * This method loads a metric entity for a given invoice workitem. If no metric + * entity exits, the method + * creates a new metric entity. * - * @param dbtrID + * @param invoice * @return * @throws PluginException */ - public ItemCollection loadMetric(String dbtrNumber, String dbtrName) throws PluginException { - + private ItemCollection loadMetric(ItemCollection invoice) throws PluginException { ItemCollection debitorMetric = null; - if (dbtrNumber == null || dbtrNumber.isEmpty()) { - throw new PluginException(PluginException.class.getName(), "QUERY ERROR", - "missing debitor number"); - } try { - String query = "(type:" + TYPE_METRIC_DEBITOR + ") AND (name:" + dbtrNumber + ")"; + String metricKey = buildKeyByInvoice(invoice); + String query = "(type:" + TYPE_METRIC_DEBITOR + ") AND (name:" + metricKey + ")"; List result = documentService.find(query, 1, 0, "$modified", true); if (result.size() > 0) { debitorMetric = result.get(0); } - if (debitorMetric == null) { - // create a new instance - // logger.info("creating new debitor metric"); - debitorMetric = createMetaData(dbtrNumber, dbtrName); - - } - } catch (QueryException e) { - throw new PluginException(PluginException.class.getName(), "QUERY ERROR", e.getMessage(), e); + } catch (IllegalArgumentException | QueryException e) { + throw new PluginException(PluginException.class.getName(), + "Failed to load metric object for invoice " + invoice.getUniqueID() + ": ", e.getMessage(), e); } return debitorMetric; @@ -185,18 +258,18 @@ public class MetricDebitorService { * The ItemCollection stores the name and number and also all saldos for all * currencies * - * @param dbtrNumber - * @param dbtrName + * @param invoice - invoice ItemCollection * @return */ - private ItemCollection createMetaData(String dbtrNumber, String dbtrName) { + private ItemCollection createMetaData(ItemCollection invoice) { + String key = buildKeyByInvoice(invoice); ItemCollection metricData = new ItemCollection(); - - metricData = new ItemCollection(); metricData.setType(TYPE_METRIC_DEBITOR); - metricData.setItemValue("name", dbtrNumber); - metricData.setItemValue("dbtr.number", dbtrNumber); - metricData.setItemValue("dbtr.name", dbtrName); + metricData.setItemValue("name", key); + metricData.setItemValue("dbtr.number", invoice.getItemValueString("dbtr.number")); + metricData.setItemValue("dbtr.name", invoice.getItemValueString("dbtr.name")); + metricData.setItemValue("invoice.currency", invoice.getItemValueString("invoice.currency")); + metricData.setItemValue("space.name", invoice.getItemValueString("space.name")); return metricData; } @@ -206,28 +279,32 @@ public class MetricDebitorService { * @param dbtrNumber - the debitor number * @param dbtrName - the debitor name */ - private void updateGauge(ItemCollection metricData, String currency) { + private void updateGauge(ItemCollection metricData) { String dbtrNumber = metricData.getItemValueString("dbtr.number"); String dbtrName = metricData.getItemValueString("dbtr.name"); - String metricKey = "dbtr_" + dbtrNumber + "_" + dbtrName + "_" + currency; + String metricKey = metricData.getItemValueString("name"); // Prüfen ob Gauge bereits registriert ist if (registeredGauges.add(metricKey)) { // returns true newly added List tags = new ArrayList<>(); tags.add(new Tag("type", "dbtr")); tags.add(new Tag("number", dbtrNumber)); tags.add(new Tag("name", dbtrName)); - tags.add(new Tag("currency", currency)); - logger.info("register new metric for debitor: " + dbtrNumber + " -> " + currency); + tags.add(new Tag("currency", metricData.getItemValueString("invoice.currency"))); + tags.add(new Tag("department", metricData.getItemValueString("space.name"))); + logger.info("register new metric for debitor: " + dbtrNumber + + ", " + metricData.getItemValueString(ITEM_SALDO) + + " " + metricData.getItemValueString("invoice.currency")); Metadata metadata = Metadata.builder() - .withName("dbtr.invoice.saldo") - .withDescription("Debitor Balance by Currency") + .withName("dbtr.balance") + .withDescription("Debitor Balance") .build(); + + metricCache.get(metricKey); metricRegistry.gauge(metadata, - () -> metricCache.getOrDefault(dbtrNumber, createMetaData(dbtrNumber, dbtrName)) - .getItemValueDouble(ITEM_SALDO + "." + currency), + () -> metricCache.get(metricKey).getItemValueDouble(ITEM_SALDO), tags.toArray(new Tag[0])); } else { - logger.fine("Gauge already registered for debitor: " + dbtrNumber + " -> " + currency); + logger.fine("Gauge already registered for debitor: " + dbtrNumber); } } @@ -238,19 +315,11 @@ public class MetricDebitorService { * */ protected void initMetric(ItemCollection metric) { - String dbtrNumber = metric.getItemValueString("dbtr.number"); - // Aktuellen Saldo in Map speichern - metricCache.put(dbtrNumber, metric); - // Gauge für jede Währung registrieren - for (String itemName : metric.getItemNames()) { - if (itemName.startsWith(ITEM_SALDO + ".")) { - // currency value found - int pos = (ITEM_SALDO + ".").length(); - String currency = itemName.substring(pos).toUpperCase(); - logger.info("│   │   ├──init metric for debitor " + dbtrNumber + " -> " + currency); - updateGauge(metric, currency); - } - } + String key = metric.getItemValueString("name"); + // Aktuelle metric cachen + metricCache.put(key, metric); + logger.info("│   │   ├──init metric for debitor " + key); + updateGauge(metric); } } \ No newline at end of file