diff --git a/office-alexander-logistics-app/src/main/java/com/alexanderlogistics/metrics/MetricCreditorRestService.java b/office-alexander-logistics-app/src/main/java/com/alexanderlogistics/metrics/MetricCreditorRestService.java index c7ee05e..a18ba93 100644 --- a/office-alexander-logistics-app/src/main/java/com/alexanderlogistics/metrics/MetricCreditorRestService.java +++ b/office-alexander-logistics-app/src/main/java/com/alexanderlogistics/metrics/MetricCreditorRestService.java @@ -55,18 +55,15 @@ public class MetricCreditorRestService { public Response rebuildMetrics() { StringBuffer messageBuffer = new StringBuffer(); long l = System.currentTimeMillis(); - log("├── init cdtr metrics...", messageBuffer); + log("├── rebuild cdtr metrics...", messageBuffer); try { + log("│   ├── delete all metrics", messageBuffer); + metricDataService.deleteAllMetrics(MetricCreditorService.TYPE_METRIC_CREDITOR); // first clear the metric cache metricCreditorService.reset(); log("│   ├── reset metric cache", messageBuffer); - computeMetrics(); - log("│   ├── computing metrics finished in " + (System.currentTimeMillis() - l) + "ms", messageBuffer); - - logger.info("│   ├── save and init metrics..."); - // run in new transaction! - metricCreditorService.refreshGauges(); + computeMetrics(messageBuffer); String message = "├── rebuild cdtr metrics completed in " + (System.currentTimeMillis() - l) @@ -76,8 +73,9 @@ public class MetricCreditorRestService { } catch (Exception e) { e.printStackTrace(); + log("Failed to initialize metrics: " + e.getMessage(), messageBuffer); return Response.serverError() - .entity("Failed to initialize metrics: " + e.getMessage()) + .entity(messageBuffer.toString() + e.getMessage()) .build(); } } @@ -86,35 +84,55 @@ public class MetricCreditorRestService { * Diese Methode berechnet alle Metriken auf basis der existierenden Rechnungen * neu * + * @throws QueryException + * @throws InterruptedException + * */ - public void computeMetrics() { + public void computeMetrics(StringBuffer messageBuffer) throws QueryException, InterruptedException { + long l = System.currentTimeMillis(); + int batchSize = 500; + int totalInvoices = 0; + log("│   │   ├── recalculate metrics...", messageBuffer); + + String query = "($modelversion:rechnungseingang-* OR $modelversion:gutschriftabgleich-*) " + + " AND type:workitem"; + // Gesamtanzahl ermitteln + int totalCount = documentService.count(query); + log("│   │   ├── found " + totalCount + " open invoices", messageBuffer); + + // Berechne Anzahl der benötigten Pages + int totalPages = (int) Math.ceil((double) totalCount / batchSize); + + // Verarbeite Page für Page + for (int pageIndex = 0; pageIndex < totalPages; pageIndex++) { + List invoices = documentService.find(query, batchSize, pageIndex); - logger.info("│   │   ├── group invoices by creditor..."); - try { - int count = 0; - List invoices = documentService.find( - "($modelversion:rechnungseingang-* OR $modelversion:gutschriftabgleich-*) " + - " AND type:workitem", - 9999, 0, - "invoice.number", false); - logger.info("│   │   ├──found " + invoices.size() + " open invoices"); for (ItemCollection invoice : invoices) { try { ItemCollection metricData = metricCreditorService.getMetricByInvoice(invoice); // Jetzt Rechnung addieren metricCreditorService.addInvoice(metricData, invoice); - logger.info("│   │   │   ├──update metric " + InvoiceUtil.getBPId(invoice)); - metricCreditorService.updateMetric(metricData); - count++; + logger.fine("│   │   │   ├── update metric " + InvoiceUtil.getBPId(invoice)); + metricCreditorService.updateMetric(metricData, true); + totalInvoices++; } catch (PluginException e) { - // invalid invoice - e.g. no cdtr. number + // invalid invoice - e.g. no dbtr. number } - } - logger.info("│   │   ├── updated metric for " + count + " invoices."); - } catch (QueryException e) { - e.printStackTrace(); + + // Fortschritt loggen + log("│   │ ├── Processed page " + (pageIndex + 1) + " of " + totalPages + + " (" + totalInvoices + " of " + totalCount + " invoices)", messageBuffer); + // Optional: Kurze Pause nach jedem 5. Batch + Thread.sleep(100); + } + long duration = System.currentTimeMillis() - l; + double invoicesPerSecond = totalInvoices / (duration / 1000.0); + log("│   │   ├── Successfully processed " + totalInvoices + " invoices in " + + duration + "ms (" + String.format("%.1f", invoicesPerSecond) + " invoices/sec)", messageBuffer); + log("│   │   ├── Updated " + metricCreditorService.getMetricCount() + " metrics.", messageBuffer); + } private void log(String message, StringBuffer messageLog) { diff --git a/office-alexander-logistics-app/src/main/java/com/alexanderlogistics/metrics/MetricCreditorService.java b/office-alexander-logistics-app/src/main/java/com/alexanderlogistics/metrics/MetricCreditorService.java index 867bb2a..50de7c5 100644 --- a/office-alexander-logistics-app/src/main/java/com/alexanderlogistics/metrics/MetricCreditorService.java +++ b/office-alexander-logistics-app/src/main/java/com/alexanderlogistics/metrics/MetricCreditorService.java @@ -15,6 +15,7 @@ import org.eclipse.microprofile.metrics.annotation.RegistryScope; import org.imixs.workflow.ItemCollection; import org.imixs.workflow.engine.DocumentService; import org.imixs.workflow.engine.ProcessingEvent; +import org.imixs.workflow.engine.SetupEvent; import org.imixs.workflow.exceptions.PluginException; import com.alexanderlogistics.InvoiceUtil; @@ -79,6 +80,54 @@ public class MetricCreditorService { @ConfigProperty(name = "metrics.enabled", defaultValue = "false") private boolean metricsEnabled; + /** + * Init all metrics during setup. Called by the Imixs SetupService + */ + public void initializeMetrics(@Observes SetupEvent setupEvent) { + if (!metricsEnabled) { + return; + } + + long l = System.currentTimeMillis(); + int batchSize = 500; + int totalMetrics = 0; + + try { + logger.info("├── Initializing creditor metrics from database..."); + String query = "(type:" + TYPE_METRIC_CREDITOR + ")"; + + // Gesamtanzahl ermitteln + int totalCount = documentService.count(query); + + // Berechne Anzahl der benötigten Pages + int totalPages = (int) Math.ceil((double) totalCount / batchSize); + + // Verarbeite Page für Page + for (int pageIndex = 0; pageIndex < totalPages; pageIndex++) { + List metrics = documentService.find(query, batchSize, pageIndex); + + for (ItemCollection metric : metrics) { + updateMetric(metric, false); + totalMetrics++; + } + // Fortschritt loggen + logger.info("│ ├── Processed page " + (pageIndex + 1) + " of " + totalPages + + " (" + totalMetrics + " of " + totalCount + " metrics)"); + // Optional: Kurze Pause nach jedem Batch + Thread.sleep(100); + } + + long duration = System.currentTimeMillis() - l; + double metricsPerSecond = totalMetrics / (duration / 1000.0); + + logger.info("├── Successfully initialized " + totalMetrics + " metrics in " + + duration + "ms (" + String.format("%.1f", metricsPerSecond) + " metrics/sec)"); + + } catch (Exception e) { + logger.warning("Failed to initialize metrics: " + e.getMessage()); + } + } + /** * Reset the internal metricCache and clears all registered Gauges. */ @@ -87,6 +136,10 @@ public class MetricCreditorService { registeredGauges.clear(); } + public long getMetricCount() { + return metricCache.size(); + } + /** * Process Metric only if some data has changed.... * @@ -115,7 +168,7 @@ public class MetricCreditorService { // update last metric only if exists... if (!isNewMetric(lastMetricData)) { subtractInvoice(lastMetricData, lastInvoice); - updateMetric(lastMetricData); + updateMetric(lastMetricData, true); } } catch (PluginException e) { // invalid invoice - e.g. no cdtr. number @@ -127,7 +180,7 @@ public class MetricCreditorService { ItemCollection metricData = getMetricByInvoice(invoice); // Saldo-Berechnung addInvoice(metricData, invoice); - updateMetric(metricData); + updateMetric(metricData, true); logger.info("Metric cdtr update took " + (System.currentTimeMillis() - l) + "ms"); } catch (PluginException e) { // invalid invoice - e.g. no cdtr. number @@ -222,17 +275,23 @@ public class MetricCreditorService { } /** - * Helper method to register a gauge for a creditor + * This method registers and updates the metric meta data objects based on a + * given metricData object. Optional the metric data object is persisted. * - * @param cdtrNumber - the creditor number - * @param cdtrName - the creditor name + * @param metricData - the metricData ItemCollection + * @param persist - if true the metricData entity will be persisted */ - public void updateMetric(ItemCollection metricData) { - + public void updateMetric(ItemCollection metricData, boolean persist) { String metricKey = metricData.getItemValueString("name"); + // Cache aktualisieren metricCache.put(metricKey, metricData); + // In Datenbank persistieren + if (persist) { + documentService.save(metricData); + } + // Prüfen ob Gauge bereits registriert ist if (registeredGauges.add(metricKey)) { // returns true newly added String bpName = metricData.getItemValueString("bp.name"); @@ -267,9 +326,20 @@ public class MetricCreditorService { () -> metricCache.get(metricKey).getItemValueDouble(ITEM_METRIC_SALES), tags.toArray(new Tag[0])); } - } + // /** + // * Helper Method that refreshes all gauges. The method is called by the + // * RestService during a rebuild. + // */ + // public void updateAllMetrics() { + // List keys = getMetricKeys(); + // for (String hashKey : keys) { + // ItemCollection metricData = getMetric(hashKey); + // updateMetric(metricData, false); + // } + // } + /** * Addiert den saldo einer Invoice zu einem metricData object * @@ -316,16 +386,4 @@ public class MetricCreditorService { } - /** - * Helper Method that refreshes all gauges. The method is called by the - * RestService during a rebuild. - */ - public void refreshGauges() { - List keys = getMetricKeys(); - for (String hashKey : keys) { - ItemCollection metricData = getMetric(hashKey); - updateMetric(metricData); - } - } - } \ No newline at end of file diff --git a/office-alexander-logistics-app/src/main/java/com/alexanderlogistics/metrics/MetricDataService.java b/office-alexander-logistics-app/src/main/java/com/alexanderlogistics/metrics/MetricDataService.java index 52fbf0d..f49dd31 100644 --- a/office-alexander-logistics-app/src/main/java/com/alexanderlogistics/metrics/MetricDataService.java +++ b/office-alexander-logistics-app/src/main/java/com/alexanderlogistics/metrics/MetricDataService.java @@ -1,10 +1,13 @@ package com.alexanderlogistics.metrics; +import java.util.List; import java.util.Objects; import org.imixs.workflow.ItemCollection; import org.imixs.workflow.engine.DocumentService; +import org.imixs.workflow.engine.index.SearchService; import org.imixs.workflow.exceptions.PluginException; +import org.imixs.workflow.exceptions.QueryException; import com.alexanderlogistics.InvoiceUtil; @@ -77,4 +80,24 @@ public class MetricDataService { return "HASH" + hash; } + /** + * This method deletes all metrics + * + * @throws PluginException + * + */ + @TransactionAttribute(value = TransactionAttributeType.REQUIRES_NEW) + public void deleteAllMetrics(String metricType) throws PluginException { + try { + String query = "(type:" + metricType + ")"; + List result = documentService.find(query, SearchService.DEFAULT_MAX_SEARCH_RESULT, 0); + for (ItemCollection metric : result) { + documentService.remove(metric); + } + + } catch (IllegalArgumentException | QueryException e) { + throw new PluginException(MetricDataService.class.getName(), + "Failed to delete metrics", e.getMessage(), e); + } + } } \ No newline at end of file diff --git a/office-alexander-logistics-app/src/main/java/com/alexanderlogistics/metrics/MetricDebitorRestService.java b/office-alexander-logistics-app/src/main/java/com/alexanderlogistics/metrics/MetricDebitorRestService.java index f41b97b..4e30fc1 100644 --- a/office-alexander-logistics-app/src/main/java/com/alexanderlogistics/metrics/MetricDebitorRestService.java +++ b/office-alexander-logistics-app/src/main/java/com/alexanderlogistics/metrics/MetricDebitorRestService.java @@ -53,33 +53,30 @@ public class MetricDebitorRestService { @Path("/rebuild") @Produces({ MediaType.TEXT_PLAIN }) public Response rebuildMetrics() { - // Map metricCache = new HashMap(); + + StringBuffer messageBuffer = new StringBuffer(); long l = System.currentTimeMillis(); - logger.info("├── init dbtr metrics..."); + log("├── rebuild dbtr metrics...", messageBuffer); try { + log("│   ├── delete all metrics", messageBuffer); // first clear the metric cache metricDebitorService.reset(); - logger.info("│   ├── reset metric cache"); + log("│   ├── reset metric cache", messageBuffer); - computeMetrics(); - logger.info("│   ├── computing metrics finished in " + (System.currentTimeMillis() - l) + "ms"); - - logger.info("│   ├── save and init metrics ..."); - // run in new transaction! - metricDebitorService.refreshGauges(); + computeMetrics(messageBuffer); String message = "├── rebuild dbtr metrics completed in " + (System.currentTimeMillis() - l) + "ms"; - logger.info(message); - return Response.ok().entity(message).build(); + log(message, messageBuffer); + return Response.ok().entity(messageBuffer.toString()).build(); } catch (Exception e) { e.printStackTrace(); + log("Failed to initialize metrics: " + e.getMessage(), messageBuffer); return Response.serverError() - .entity("Failed to initialize metrics: " + e.getMessage()) + .entity(messageBuffer.toString() + e.getMessage()) .build(); } } @@ -88,37 +85,53 @@ public class MetricDebitorRestService { * Diese Methode berechnet alle Metriken auf basis der existierenden Rechnungen * neu * + * @throws QueryException + * @throws InterruptedException + * */ - public void computeMetrics() { + public void computeMetrics(StringBuffer messageBuffer) throws QueryException, InterruptedException { + long l = System.currentTimeMillis(); + int batchSize = 500; + int totalInvoices = 0; + log("│   │   ├── recalculate metrics...", messageBuffer); - logger.info("│   │   ├── group invoices by debitor..."); - try { - int count = 0; - List invoices = documentService.find( - "$modelversion:rechnungsausgang-* AND type:workitem", - 9999, 0, - "invoice.number", false); + String query = "$modelversion:rechnungsausgang-* AND type:workitem"; + // Gesamtanzahl ermitteln + int totalCount = documentService.count(query); + log("│   │   ├── found " + totalCount + " open invoices", messageBuffer); + + // Berechne Anzahl der benötigten Pages + int totalPages = (int) Math.ceil((double) totalCount / batchSize); + + // Verarbeite Page für Page + for (int pageIndex = 0; pageIndex < totalPages; pageIndex++) { + List invoices = documentService.find(query, batchSize, pageIndex); - logger.info("│   │   ├── found " + invoices.size() + " open invoices"); for (ItemCollection invoice : invoices) { try { - ItemCollection metricData = metricDebitorService.getMetricByInvoice(invoice); - // Jetzt Rechnung addieren metricDebitorService.addInvoice(metricData, invoice); - logger.info("│   │   │   ├──update metric " + InvoiceUtil.getBPId(invoice)); - metricDebitorService.updateMetric(metricData); - count++; + logger.fine("│   │   │   ├── update metric " + InvoiceUtil.getBPId(invoice)); + metricDebitorService.updateMetric(metricData, true); + totalInvoices++; } catch (PluginException e) { // invalid invoice - e.g. no dbtr. number } - } - logger.info("│   │   ├── updated metric for " + count + " invoices."); - } catch (QueryException e) { - e.printStackTrace(); + + // Fortschritt loggen + log("│   │ ├── Processed page " + (pageIndex + 1) + " of " + totalPages + + " (" + totalInvoices + " of " + totalCount + " invoices)", messageBuffer); + // Optional: Kurze Pause nach jedem 5. Batch + Thread.sleep(100); + } + long duration = System.currentTimeMillis() - l; + double invoicesPerSecond = totalInvoices / (duration / 1000.0); + log("│   │   ├── Successfully processed " + totalInvoices + " invoices in " + + duration + "ms (" + String.format("%.1f", invoicesPerSecond) + " invoices/sec)", messageBuffer); + log("│   │   ├── Updated " + metricDebitorService.getMetricCount() + " metrics.", messageBuffer); } diff --git a/office-alexander-logistics-app/src/main/java/com/alexanderlogistics/metrics/MetricDebitorService.java b/office-alexander-logistics-app/src/main/java/com/alexanderlogistics/metrics/MetricDebitorService.java index 5fc057f..01a97b2 100644 --- a/office-alexander-logistics-app/src/main/java/com/alexanderlogistics/metrics/MetricDebitorService.java +++ b/office-alexander-logistics-app/src/main/java/com/alexanderlogistics/metrics/MetricDebitorService.java @@ -15,6 +15,7 @@ import org.eclipse.microprofile.metrics.annotation.RegistryScope; import org.imixs.workflow.ItemCollection; import org.imixs.workflow.engine.DocumentService; import org.imixs.workflow.engine.ProcessingEvent; +import org.imixs.workflow.engine.SetupEvent; import org.imixs.workflow.exceptions.PluginException; import com.alexanderlogistics.InvoiceUtil; @@ -72,6 +73,54 @@ public class MetricDebitorService { @ConfigProperty(name = "metrics.enabled", defaultValue = "false") private boolean metricsEnabled; + /** + * Init all metrics during setup. Called by the Imixs SetupService + */ + public void initializeMetrics(@Observes SetupEvent setupEvent) { + if (!metricsEnabled) { + return; + } + + long l = System.currentTimeMillis(); + int batchSize = 500; + int totalMetrics = 0; + + try { + logger.info("├── Initializing debitor metrics from database..."); + String query = "(type:" + TYPE_METRIC_DEBITOR + ")"; + + // Gesamtanzahl ermitteln + int totalCount = documentService.count(query); + + // Berechne Anzahl der benötigten Pages + int totalPages = (int) Math.ceil((double) totalCount / batchSize); + + // Verarbeite Page für Page + for (int pageIndex = 0; pageIndex < totalPages; pageIndex++) { + List metrics = documentService.find(query, batchSize, pageIndex); + + for (ItemCollection metric : metrics) { + updateMetric(metric, false); + totalMetrics++; + } + // Fortschritt loggen + logger.info("│ ├── Processed page " + (pageIndex + 1) + " of " + totalPages + + " (" + totalMetrics + " of " + totalCount + " metrics)"); + // Optional: Kurze Pause nach jedem Batch + Thread.sleep(100); + } + + long duration = System.currentTimeMillis() - l; + double metricsPerSecond = totalMetrics / (duration / 1000.0); + + logger.info("├── Successfully initialized " + totalMetrics + " metrics in " + + duration + "ms (" + String.format("%.1f", metricsPerSecond) + " metrics/sec)"); + + } catch (Exception e) { + logger.warning("Failed to initialize metrics: " + e.getMessage()); + } + } + /** * Reset internal metricCache and clear registered Gauges. */ @@ -80,6 +129,10 @@ public class MetricDebitorService { registeredGauges.clear(); } + public long getMetricCount() { + return metricCache.size(); + } + /** * Process Metric only if some data has changed.... * @@ -107,7 +160,7 @@ public class MetricDebitorService { ItemCollection lastMetricData = getMetricByInvoice(lastInvoice); if (!isNewMetric(lastMetricData)) { subtractInvoice(lastMetricData, lastInvoice); - updateMetric(lastMetricData); + updateMetric(lastMetricData, true); } } catch (PluginException e) { // invalid invoice - e.g. no cdtr. number @@ -119,7 +172,7 @@ public class MetricDebitorService { ItemCollection metricData = getMetricByInvoice(invoice); // Saldo-Berechnung addInvoice(metricData, invoice); - updateMetric(metricData); + updateMetric(metricData, true); logger.info("Metric dbtr update took " + (System.currentTimeMillis() - l) + "ms"); } catch (PluginException e) { @@ -214,16 +267,23 @@ public class MetricDebitorService { } /** - * Helper method to register a gauge for a debitor + * This method registers and updates the metric meta data objects based on a + * given metricData object. Optional the metric data object is persisted. * * @param metricData - the metricData ItemCollection + * @param persist - if true the metricData entity will be persisted */ - public void updateMetric(ItemCollection metricData) { + public void updateMetric(ItemCollection metricData, boolean persist) { String metricKey = metricData.getItemValueString("name"); // Cache aktualisieren metricCache.put(metricKey, metricData); + // In Datenbank persistieren + if (persist) { + documentService.save(metricData); + } + // Prüfen ob Gauge bereits registriert ist if (registeredGauges.add(metricKey)) { // returns true newly added String bpName = metricData.getItemValueString("bp.name"); @@ -242,6 +302,7 @@ public class MetricDebitorService { logger.fine("register new metric for department: " + department + ", " + metricData.getItemValueString(ITEM_METRIC_BALANCE) + " " + currency); + // Saldo Gauge Metadata balanceMetadata = Metadata.builder() .withName("dbtr.balance") @@ -317,13 +378,12 @@ public class MetricDebitorService { * Helper Method that refreshes all gauges. The method is called by the * RestService during a rebuild. */ - - public void refreshGauges() { - List keys = getMetricKeys(); - for (String hashKey : keys) { - ItemCollection metricData = getMetric(hashKey); - updateMetric(metricData); - } - } + // public void refreshGauges() { + // List keys = getMetricKeys(); + // for (String hashKey : keys) { + // ItemCollection metricData = getMetric(hashKey); + // updateMetric(metricData, false); + // } + // } } \ No newline at end of file