refactoring metric logik

office-alexander-logistics-app/src/main/java/com/alexanderlogistics/metrics/MetricCreditorRestService.java

@@ -55,18 +55,15 @@ public class MetricCreditorRestService {
     public Response rebuildMetrics() {
         StringBuffer messageBuffer = new StringBuffer();
         long l = System.currentTimeMillis();
-        log("├── init cdtr metrics...", messageBuffer);
+        log("├── rebuild cdtr metrics...", messageBuffer);
         try {
+            log("│   ├── delete all metrics", messageBuffer);
+            metricDataService.deleteAllMetrics(MetricCreditorService.TYPE_METRIC_CREDITOR);
             // first clear the metric cache
             metricCreditorService.reset();
             log("│   ├── reset metric cache", messageBuffer);
 
-            computeMetrics();
+            computeMetrics(messageBuffer);
-            log("│   ├── computing metrics finished in " + (System.currentTimeMillis() - l) + "ms", messageBuffer);
-
-            logger.info("│   ├── save and init metrics...");
-            // run in new transaction!
-            metricCreditorService.refreshGauges();
 
             String message = "├── rebuild cdtr metrics completed in "
                     + (System.currentTimeMillis() - l)
@@ -76,8 +73,9 @@ public class MetricCreditorRestService {
 
         } catch (Exception e) {
             e.printStackTrace();
+            log("Failed to initialize metrics: " + e.getMessage(), messageBuffer);
             return Response.serverError()
-                    .entity("Failed to initialize metrics: " + e.getMessage())
+                    .entity(messageBuffer.toString() + e.getMessage())
                     .build();
         }
     }
@@ -86,35 +84,55 @@ public class MetricCreditorRestService {
      * Diese Methode berechnet alle Metriken auf basis der existierenden Rechnungen
      * neu
      * 
+     * @throws QueryException
+     * @throws InterruptedException
+     * 
      */
-    public void computeMetrics() {
+    public void computeMetrics(StringBuffer messageBuffer) throws QueryException, InterruptedException {
+        long l = System.currentTimeMillis();
+        int batchSize = 500;
+        int totalInvoices = 0;
+        log("│   │   ├── recalculate metrics...", messageBuffer);
+
+        String query = "($modelversion:rechnungseingang-* OR $modelversion:gutschriftabgleich-*) " +
+                " AND type:workitem";
+        // Gesamtanzahl ermitteln
+        int totalCount = documentService.count(query);
+        log("│   │   ├── found " + totalCount + " open invoices", messageBuffer);
+
+        // Berechne Anzahl der benötigten Pages
+        int totalPages = (int) Math.ceil((double) totalCount / batchSize);
+
+        // Verarbeite Page für Page
+        for (int pageIndex = 0; pageIndex < totalPages; pageIndex++) {
+            List<ItemCollection> invoices = documentService.find(query, batchSize, pageIndex);
 
-        logger.info("│   │   ├── group invoices by creditor...");
-        try {
-            int count = 0;
-            List<ItemCollection> invoices = documentService.find(
-                    "($modelversion:rechnungseingang-* OR $modelversion:gutschriftabgleich-*) " +
-                            " AND type:workitem",
-                    9999, 0,
-                    "invoice.number", false);
-            logger.info("│   │   ├──found " + invoices.size() + " open invoices");
             for (ItemCollection invoice : invoices) {
                 try {
                     ItemCollection metricData = metricCreditorService.getMetricByInvoice(invoice);
                     // Jetzt Rechnung addieren
                     metricCreditorService.addInvoice(metricData, invoice);
-                    logger.info("│   │   │   ├──update metric " + InvoiceUtil.getBPId(invoice));
+                    logger.fine("│   │   │   ├── update metric " + InvoiceUtil.getBPId(invoice));
-                    metricCreditorService.updateMetric(metricData);
+                    metricCreditorService.updateMetric(metricData, true);
-                    count++;
+                    totalInvoices++;
                 } catch (PluginException e) {
-                    // invalid invoice - e.g. no cdtr. number
+                    // invalid invoice - e.g. no dbtr. number
                 }
-
             }
-            logger.info("│   │   ├── updated metric for " + count + " invoices.");
+
-        } catch (QueryException e) {
+            // Fortschritt loggen
-            e.printStackTrace();
+            log("│   │   ├── Processed page " + (pageIndex + 1) + " of " + totalPages +
+                    " (" + totalInvoices + " of " + totalCount + " invoices)", messageBuffer);
+            // Optional: Kurze Pause nach jedem 5. Batch
+            Thread.sleep(100);
+
         }
+        long duration = System.currentTimeMillis() - l;
+        double invoicesPerSecond = totalInvoices / (duration / 1000.0);
+        log("│   │   ├── Successfully processed " + totalInvoices + " invoices in " +
+                duration + "ms (" + String.format("%.1f", invoicesPerSecond) + " invoices/sec)", messageBuffer);
+        log("│   │   ├── Updated " + metricCreditorService.getMetricCount() + " metrics.", messageBuffer);
+
     }
 
     private void log(String message, StringBuffer messageLog) {

office-alexander-logistics-app/src/main/java/com/alexanderlogistics/metrics/MetricCreditorService.java

@@ -15,6 +15,7 @@ import org.eclipse.microprofile.metrics.annotation.RegistryScope;
 import org.imixs.workflow.ItemCollection;
 import org.imixs.workflow.engine.DocumentService;
 import org.imixs.workflow.engine.ProcessingEvent;
+import org.imixs.workflow.engine.SetupEvent;
 import org.imixs.workflow.exceptions.PluginException;
 
 import com.alexanderlogistics.InvoiceUtil;
@@ -79,6 +80,54 @@ public class MetricCreditorService {
     @ConfigProperty(name = "metrics.enabled", defaultValue = "false")
     private boolean metricsEnabled;
 
+    /**
+     * Init all metrics during setup. Called by the Imixs SetupService
+     */
+    public void initializeMetrics(@Observes SetupEvent setupEvent) {
+        if (!metricsEnabled) {
+            return;
+        }
+
+        long l = System.currentTimeMillis();
+        int batchSize = 500;
+        int totalMetrics = 0;
+
+        try {
+            logger.info("├── Initializing creditor metrics from database...");
+            String query = "(type:" + TYPE_METRIC_CREDITOR + ")";
+
+            // Gesamtanzahl ermitteln
+            int totalCount = documentService.count(query);
+
+            // Berechne Anzahl der benötigten Pages
+            int totalPages = (int) Math.ceil((double) totalCount / batchSize);
+
+            // Verarbeite Page für Page
+            for (int pageIndex = 0; pageIndex < totalPages; pageIndex++) {
+                List<ItemCollection> metrics = documentService.find(query, batchSize, pageIndex);
+
+                for (ItemCollection metric : metrics) {
+                    updateMetric(metric, false);
+                    totalMetrics++;
+                }
+                // Fortschritt loggen
+                logger.info("│   ├── Processed page " + (pageIndex + 1) + " of " + totalPages +
+                        " (" + totalMetrics + " of " + totalCount + " metrics)");
+                // Optional: Kurze Pause nach jedem Batch
+                Thread.sleep(100);
+            }
+
+            long duration = System.currentTimeMillis() - l;
+            double metricsPerSecond = totalMetrics / (duration / 1000.0);
+
+            logger.info("├── Successfully initialized " + totalMetrics + " metrics in " +
+                    duration + "ms (" + String.format("%.1f", metricsPerSecond) + " metrics/sec)");
+
+        } catch (Exception e) {
+            logger.warning("Failed to initialize metrics: " + e.getMessage());
+        }
+    }
+
     /**
      * Reset the internal metricCache and clears all registered Gauges.
      */
@@ -87,6 +136,10 @@ public class MetricCreditorService {
         registeredGauges.clear();
     }
 
+    public long getMetricCount() {
+        return metricCache.size();
+    }
+
     /**
      * Process Metric only if some data has changed....
      * 
@@ -115,7 +168,7 @@ public class MetricCreditorService {
                     // update last metric only if exists...
                     if (!isNewMetric(lastMetricData)) {
                         subtractInvoice(lastMetricData, lastInvoice);
-                        updateMetric(lastMetricData);
+                        updateMetric(lastMetricData, true);
                     }
                 } catch (PluginException e) {
                     // invalid invoice - e.g. no cdtr. number
@@ -127,7 +180,7 @@ public class MetricCreditorService {
                 ItemCollection metricData = getMetricByInvoice(invoice);
                 // Saldo-Berechnung
                 addInvoice(metricData, invoice);
-                updateMetric(metricData);
+                updateMetric(metricData, true);
                 logger.info("Metric cdtr update took " + (System.currentTimeMillis() - l) + "ms");
             } catch (PluginException e) {
                 // invalid invoice - e.g. no cdtr. number
@@ -222,17 +275,23 @@ public class MetricCreditorService {
     }
 
     /**
-     * Helper method to register a gauge for a creditor
+     * This method registers and updates the metric meta data objects based on a
+     * given metricData object. Optional the metric data object is persisted.
      * 
-     * @param cdtrNumber - the creditor number
+     * @param metricData - the metricData ItemCollection
-     * @param cdtrName   - the creditor name
+     * @param persist    - if true the metricData entity will be persisted
      */
-    public void updateMetric(ItemCollection metricData) {
+    public void updateMetric(ItemCollection metricData, boolean persist) {
-
         String metricKey = metricData.getItemValueString("name");
+
         // Cache aktualisieren
         metricCache.put(metricKey, metricData);
 
+        // In Datenbank persistieren
+        if (persist) {
+            documentService.save(metricData);
+        }
+
         // Prüfen ob Gauge bereits registriert ist
         if (registeredGauges.add(metricKey)) { // returns true newly added
             String bpName = metricData.getItemValueString("bp.name");
@@ -267,9 +326,20 @@ public class MetricCreditorService {
                     () -> metricCache.get(metricKey).getItemValueDouble(ITEM_METRIC_SALES),
                     tags.toArray(new Tag[0]));
         }
-
     }
 
+    // /**
+    // * Helper Method that refreshes all gauges. The method is called by the
+    // * RestService during a rebuild.
+    // */
+    // public void updateAllMetrics() {
+    // List<String> keys = getMetricKeys();
+    // for (String hashKey : keys) {
+    // ItemCollection metricData = getMetric(hashKey);
+    // updateMetric(metricData, false);
+    // }
+    // }
+
     /**
      * Addiert den saldo einer Invoice zu einem metricData object
      * 
@@ -316,16 +386,4 @@ public class MetricCreditorService {
 
     }
 
-    /**
-     * Helper Method that refreshes all gauges. The method is called by the
-     * RestService during a rebuild.
-     */
-    public void refreshGauges() {
-        List<String> keys = getMetricKeys();
-        for (String hashKey : keys) {
-            ItemCollection metricData = getMetric(hashKey);
-            updateMetric(metricData);
-        }
-    }
-
 }

office-alexander-logistics-app/src/main/java/com/alexanderlogistics/metrics/MetricDataService.java

@@ -1,10 +1,13 @@
 package com.alexanderlogistics.metrics;
 
+import java.util.List;
 import java.util.Objects;
 
 import org.imixs.workflow.ItemCollection;
 import org.imixs.workflow.engine.DocumentService;
+import org.imixs.workflow.engine.index.SearchService;
 import org.imixs.workflow.exceptions.PluginException;
+import org.imixs.workflow.exceptions.QueryException;
 
 import com.alexanderlogistics.InvoiceUtil;
 
@@ -77,4 +80,24 @@ public class MetricDataService {
         return "HASH" + hash;
     }
 
+    /**
+     * This method deletes all metrics
+     * 
+     * @throws PluginException
+     * 
+     */
+    @TransactionAttribute(value = TransactionAttributeType.REQUIRES_NEW)
+    public void deleteAllMetrics(String metricType) throws PluginException {
+        try {
+            String query = "(type:" + metricType + ")";
+            List<ItemCollection> result = documentService.find(query, SearchService.DEFAULT_MAX_SEARCH_RESULT, 0);
+            for (ItemCollection metric : result) {
+                documentService.remove(metric);
+            }
+
+        } catch (IllegalArgumentException | QueryException e) {
+            throw new PluginException(MetricDataService.class.getName(),
+                    "Failed to delete metrics", e.getMessage(), e);
+        }
+    }
 }

office-alexander-logistics-app/src/main/java/com/alexanderlogistics/metrics/MetricDebitorRestService.java

@@ -53,33 +53,30 @@ public class MetricDebitorRestService {
     @Path("/rebuild")
     @Produces({ MediaType.TEXT_PLAIN })
     public Response rebuildMetrics() {
-        // Map<String, ItemCollection> metricCache = new HashMap<String,
+
-        // ItemCollection>();
+        StringBuffer messageBuffer = new StringBuffer();
         long l = System.currentTimeMillis();
-        logger.info("├── init dbtr metrics...");
+        log("├── rebuild dbtr metrics...", messageBuffer);
 
         try {
+            log("│   ├── delete all metrics", messageBuffer);
             // first clear the metric cache
             metricDebitorService.reset();
-            logger.info("│   ├── reset metric cache");
+            log("│   ├── reset metric cache", messageBuffer);
 
-            computeMetrics();
+            computeMetrics(messageBuffer);
-            logger.info("│   ├── computing metrics finished in " + (System.currentTimeMillis() - l) + "ms");
-
-            logger.info("│   ├── save and init metrics ...");
-            // run in new transaction!
-            metricDebitorService.refreshGauges();
 
             String message = "├── rebuild dbtr metrics completed in "
                     + (System.currentTimeMillis() - l)
                     + "ms";
-            logger.info(message);
+            log(message, messageBuffer);
-            return Response.ok().entity(message).build();
+            return Response.ok().entity(messageBuffer.toString()).build();
 
         } catch (Exception e) {
             e.printStackTrace();
+            log("Failed to initialize metrics: " + e.getMessage(), messageBuffer);
             return Response.serverError()
-                    .entity("Failed to initialize metrics: " + e.getMessage())
+                    .entity(messageBuffer.toString() + e.getMessage())
                     .build();
         }
     }
@@ -88,37 +85,53 @@ public class MetricDebitorRestService {
      * Diese Methode berechnet alle Metriken auf basis der existierenden Rechnungen
      * neu
      * 
+     * @throws QueryException
+     * @throws InterruptedException
+     * 
      */
-    public void computeMetrics() {
+    public void computeMetrics(StringBuffer messageBuffer) throws QueryException, InterruptedException {
+        long l = System.currentTimeMillis();
+        int batchSize = 500;
+        int totalInvoices = 0;
+        log("│   │   ├── recalculate metrics...", messageBuffer);
 
-        logger.info("│   │   ├── group invoices by debitor...");
+        String query = "$modelversion:rechnungsausgang-* AND type:workitem";
-        try {
+        // Gesamtanzahl ermitteln
-            int count = 0;
+        int totalCount = documentService.count(query);
-            List<ItemCollection> invoices = documentService.find(
+        log("│   │   ├── found " + totalCount + " open invoices", messageBuffer);
-                    "$modelversion:rechnungsausgang-* AND type:workitem",
+
-                    9999, 0,
+        // Berechne Anzahl der benötigten Pages
-                    "invoice.number", false);
+        int totalPages = (int) Math.ceil((double) totalCount / batchSize);
+
+        // Verarbeite Page für Page
+        for (int pageIndex = 0; pageIndex < totalPages; pageIndex++) {
+            List<ItemCollection> invoices = documentService.find(query, batchSize, pageIndex);
 
-            logger.info("│   │   ├── found " + invoices.size() + " open invoices");
             for (ItemCollection invoice : invoices) {
                 try {
-
                     ItemCollection metricData = metricDebitorService.getMetricByInvoice(invoice);
-
                     // Jetzt Rechnung addieren
                     metricDebitorService.addInvoice(metricData, invoice);
-                    logger.info("│   │   │   ├──update metric " + InvoiceUtil.getBPId(invoice));
+                    logger.fine("│   │   │   ├── update metric " + InvoiceUtil.getBPId(invoice));
-                    metricDebitorService.updateMetric(metricData);
+                    metricDebitorService.updateMetric(metricData, true);
-                    count++;
+                    totalInvoices++;
                 } catch (PluginException e) {
                     // invalid invoice - e.g. no dbtr. number
                 }
-
             }
-            logger.info("│   │   ├── updated metric for " + count + " invoices.");
+
-        } catch (QueryException e) {
+            // Fortschritt loggen
-            e.printStackTrace();
+            log("│   │   ├── Processed page " + (pageIndex + 1) + " of " + totalPages +
+                    " (" + totalInvoices + " of " + totalCount + " invoices)", messageBuffer);
+            // Optional: Kurze Pause nach jedem 5. Batch
+            Thread.sleep(100);
+
         }
+        long duration = System.currentTimeMillis() - l;
+        double invoicesPerSecond = totalInvoices / (duration / 1000.0);
+        log("│   │   ├── Successfully processed " + totalInvoices + " invoices in " +
+                duration + "ms (" + String.format("%.1f", invoicesPerSecond) + " invoices/sec)", messageBuffer);
+        log("│   │   ├── Updated " + metricDebitorService.getMetricCount() + " metrics.", messageBuffer);
 
     }
 

office-alexander-logistics-app/src/main/java/com/alexanderlogistics/metrics/MetricDebitorService.java

@@ -15,6 +15,7 @@ import org.eclipse.microprofile.metrics.annotation.RegistryScope;
 import org.imixs.workflow.ItemCollection;
 import org.imixs.workflow.engine.DocumentService;
 import org.imixs.workflow.engine.ProcessingEvent;
+import org.imixs.workflow.engine.SetupEvent;
 import org.imixs.workflow.exceptions.PluginException;
 
 import com.alexanderlogistics.InvoiceUtil;
@@ -72,6 +73,54 @@ public class MetricDebitorService {
     @ConfigProperty(name = "metrics.enabled", defaultValue = "false")
     private boolean metricsEnabled;
 
+    /**
+     * Init all metrics during setup. Called by the Imixs SetupService
+     */
+    public void initializeMetrics(@Observes SetupEvent setupEvent) {
+        if (!metricsEnabled) {
+            return;
+        }
+
+        long l = System.currentTimeMillis();
+        int batchSize = 500;
+        int totalMetrics = 0;
+
+        try {
+            logger.info("├── Initializing debitor metrics from database...");
+            String query = "(type:" + TYPE_METRIC_DEBITOR + ")";
+
+            // Gesamtanzahl ermitteln
+            int totalCount = documentService.count(query);
+
+            // Berechne Anzahl der benötigten Pages
+            int totalPages = (int) Math.ceil((double) totalCount / batchSize);
+
+            // Verarbeite Page für Page
+            for (int pageIndex = 0; pageIndex < totalPages; pageIndex++) {
+                List<ItemCollection> metrics = documentService.find(query, batchSize, pageIndex);
+
+                for (ItemCollection metric : metrics) {
+                    updateMetric(metric, false);
+                    totalMetrics++;
+                }
+                // Fortschritt loggen
+                logger.info("│   ├── Processed page " + (pageIndex + 1) + " of " + totalPages +
+                        " (" + totalMetrics + " of " + totalCount + " metrics)");
+                // Optional: Kurze Pause nach jedem Batch
+                Thread.sleep(100);
+            }
+
+            long duration = System.currentTimeMillis() - l;
+            double metricsPerSecond = totalMetrics / (duration / 1000.0);
+
+            logger.info("├── Successfully initialized " + totalMetrics + " metrics in " +
+                    duration + "ms (" + String.format("%.1f", metricsPerSecond) + " metrics/sec)");
+
+        } catch (Exception e) {
+            logger.warning("Failed to initialize metrics: " + e.getMessage());
+        }
+    }
+
     /**
      * Reset internal metricCache and clear registered Gauges.
      */
@@ -80,6 +129,10 @@ public class MetricDebitorService {
         registeredGauges.clear();
     }
 
+    public long getMetricCount() {
+        return metricCache.size();
+    }
+
     /**
      * Process Metric only if some data has changed....
      * 
@@ -107,7 +160,7 @@ public class MetricDebitorService {
                     ItemCollection lastMetricData = getMetricByInvoice(lastInvoice);
                     if (!isNewMetric(lastMetricData)) {
                         subtractInvoice(lastMetricData, lastInvoice);
-                        updateMetric(lastMetricData);
+                        updateMetric(lastMetricData, true);
                     }
                 } catch (PluginException e) {
                     // invalid invoice - e.g. no cdtr. number
@@ -119,7 +172,7 @@ public class MetricDebitorService {
                 ItemCollection metricData = getMetricByInvoice(invoice);
                 // Saldo-Berechnung
                 addInvoice(metricData, invoice);
-                updateMetric(metricData);
+                updateMetric(metricData, true);
 
                 logger.info("Metric dbtr update took " + (System.currentTimeMillis() - l) + "ms");
             } catch (PluginException e) {
@@ -214,16 +267,23 @@ public class MetricDebitorService {
     }
 
     /**
-     * Helper method to register a gauge for a debitor
+     * This method registers and updates the metric meta data objects based on a
+     * given metricData object. Optional the metric data object is persisted.
      * 
      * @param metricData - the metricData ItemCollection
+     * @param persist    - if true the metricData entity will be persisted
      */
-    public void updateMetric(ItemCollection metricData) {
+    public void updateMetric(ItemCollection metricData, boolean persist) {
         String metricKey = metricData.getItemValueString("name");
 
         // Cache aktualisieren
         metricCache.put(metricKey, metricData);
 
+        // In Datenbank persistieren
+        if (persist) {
+            documentService.save(metricData);
+        }
+
         // Prüfen ob Gauge bereits registriert ist
         if (registeredGauges.add(metricKey)) { // returns true newly added
             String bpName = metricData.getItemValueString("bp.name");
@@ -242,6 +302,7 @@ public class MetricDebitorService {
             logger.fine("register new metric for department: " + department +
                     ", " + metricData.getItemValueString(ITEM_METRIC_BALANCE) +
                     " " + currency);
+
             // Saldo Gauge
             Metadata balanceMetadata = Metadata.builder()
                     .withName("dbtr.balance")
@@ -317,13 +378,12 @@ public class MetricDebitorService {
      * Helper Method that refreshes all gauges. The method is called by the
      * RestService during a rebuild.
      */
-
+    // public void refreshGauges() {
-    public void refreshGauges() {
+    // List<String> keys = getMetricKeys();
-        List<String> keys = getMetricKeys();
+    // for (String hashKey : keys) {
-        for (String hashKey : keys) {
+    // ItemCollection metricData = getMetric(hashKey);
-            ItemCollection metricData = getMetric(hashKey);
+    // updateMetric(metricData, false);
-            updateMetric(metricData);
+    // }
-        }
+    // }
-    }
 
 }