Imixs/office-alexander-logistics
3
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactoring metric logik

Browse source
This commit is contained in:
Ralph Soika 2025-02-01 17:21:49 +01:00
parent 51ccd40f18
commit 017ef6a752
5 changed files with 261 additions and 89 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

70
office-alexander-logistics-app/src/main/java/com/alexanderlogistics/metrics/MetricCreditorRestService.java
View file

@ -55,18 +55,15 @@ public class MetricCreditorRestService {
public Response rebuildMetrics() {
StringBuffer messageBuffer = new StringBuffer();
long l = System.currentTimeMillis();
log("├── init cdtr metrics...", messageBuffer);
log("├── rebuild cdtr metrics...", messageBuffer);
try {
log("│   ├── delete all metrics", messageBuffer);
metricDataService.deleteAllMetrics(MetricCreditorService.TYPE_METRIC_CREDITOR);
// first clear the metric cache
metricCreditorService.reset();
log("│   ├── reset metric cache", messageBuffer);
computeMetrics();
log("│   ├── computing metrics finished in " + (System.currentTimeMillis() - l) + "ms", messageBuffer);
logger.info("│   ├── save and init metrics...");
// run in new transaction!
metricCreditorService.refreshGauges();
computeMetrics(messageBuffer);
String message = "├── rebuild cdtr metrics completed in "
+ (System.currentTimeMillis() - l)
@ -76,8 +73,9 @@ public class MetricCreditorRestService {
} catch (Exception e) {
e.printStackTrace();
log("Failed to initialize metrics: " + e.getMessage(), messageBuffer);
return Response.serverError()
.entity("Failed to initialize metrics: " + e.getMessage())
.entity(messageBuffer.toString() + e.getMessage())
.build();
}
}
@ -86,35 +84,55 @@ public class MetricCreditorRestService {
* Diese Methode berechnet alle Metriken auf basis der existierenden Rechnungen
* neu
*
* @throws QueryException
* @throws InterruptedException
*
*/
public void computeMetrics() {
public void computeMetrics(StringBuffer messageBuffer) throws QueryException, InterruptedException {
long l = System.currentTimeMillis();
int batchSize = 500;
int totalInvoices = 0;
log("│   │   ├── recalculate metrics...", messageBuffer);
String query = "($modelversion:rechnungseingang-* OR $modelversion:gutschriftabgleich-*) " +
" AND type:workitem";
// Gesamtanzahl ermitteln
int totalCount = documentService.count(query);
log("│   │   ├── found " + totalCount + " open invoices", messageBuffer);
// Berechne Anzahl der benötigten Pages
int totalPages = (int) Math.ceil((double) totalCount / batchSize);
// Verarbeite Page für Page
for (int pageIndex = 0; pageIndex < totalPages; pageIndex++) {
List<ItemCollection> invoices = documentService.find(query, batchSize, pageIndex);
logger.info("│   │   ├── group invoices by creditor...");
try {
int count = 0;
List<ItemCollection> invoices = documentService.find(
"($modelversion:rechnungseingang-* OR $modelversion:gutschriftabgleich-*) " +
" AND type:workitem",
9999, 0,
"invoice.number", false);
logger.info("│   │   ├──found " + invoices.size() + " open invoices");
for (ItemCollection invoice : invoices) {
try {
ItemCollection metricData = metricCreditorService.getMetricByInvoice(invoice);
// Jetzt Rechnung addieren
metricCreditorService.addInvoice(metricData, invoice);
logger.info("│   │   │   ├──update metric " + InvoiceUtil.getBPId(invoice));
metricCreditorService.updateMetric(metricData);
count++;
logger.fine("│   │   │   ├── update metric " + InvoiceUtil.getBPId(invoice));
metricCreditorService.updateMetric(metricData, true);
totalInvoices++;
} catch (PluginException e) {
// invalid invoice - e.g. no cdtr. number
// invalid invoice - e.g. no dbtr. number
}
}
logger.info("│   │   ├── updated metric for " + count + " invoices.");
} catch (QueryException e) {
e.printStackTrace();
// Fortschritt loggen
log("│   │ ├── Processed page " + (pageIndex + 1) + " of " + totalPages +
" (" + totalInvoices + " of " + totalCount + " invoices)", messageBuffer);
// Optional: Kurze Pause nach jedem 5. Batch
Thread.sleep(100);
}
long duration = System.currentTimeMillis() - l;
double invoicesPerSecond = totalInvoices / (duration / 1000.0);
log("│   │   ├── Successfully processed " + totalInvoices + " invoices in " +
duration + "ms (" + String.format("%.1f", invoicesPerSecond) + " invoices/sec)", messageBuffer);
log("│   │   ├── Updated " + metricCreditorService.getMetricCount() + " metrics.", messageBuffer);
}
private void log(String message, StringBuffer messageLog) {

98
office-alexander-logistics-app/src/main/java/com/alexanderlogistics/metrics/MetricCreditorService.java
View file

@ -15,6 +15,7 @@ import org.eclipse.microprofile.metrics.annotation.RegistryScope;
import org.imixs.workflow.ItemCollection;
import org.imixs.workflow.engine.DocumentService;
import org.imixs.workflow.engine.ProcessingEvent;
import org.imixs.workflow.engine.SetupEvent;
import org.imixs.workflow.exceptions.PluginException;
import com.alexanderlogistics.InvoiceUtil;
@ -79,6 +80,54 @@ public class MetricCreditorService {
@ConfigProperty(name = "metrics.enabled", defaultValue = "false")
private boolean metricsEnabled;
/**
* Init all metrics during setup. Called by the Imixs SetupService
*/
public void initializeMetrics(@Observes SetupEvent setupEvent) {
if (!metricsEnabled) {
return;
}
long l = System.currentTimeMillis();
int batchSize = 500;
int totalMetrics = 0;
try {
logger.info("├── Initializing creditor metrics from database...");
String query = "(type:" + TYPE_METRIC_CREDITOR + ")";
// Gesamtanzahl ermitteln
int totalCount = documentService.count(query);
// Berechne Anzahl der benötigten Pages
int totalPages = (int) Math.ceil((double) totalCount / batchSize);
// Verarbeite Page für Page
for (int pageIndex = 0; pageIndex < totalPages; pageIndex++) {
List<ItemCollection> metrics = documentService.find(query, batchSize, pageIndex);
for (ItemCollection metric : metrics) {
updateMetric(metric, false);
totalMetrics++;
}
// Fortschritt loggen
logger.info("│ ├── Processed page " + (pageIndex + 1) + " of " + totalPages +
" (" + totalMetrics + " of " + totalCount + " metrics)");
// Optional: Kurze Pause nach jedem Batch
Thread.sleep(100);
}
long duration = System.currentTimeMillis() - l;
double metricsPerSecond = totalMetrics / (duration / 1000.0);
logger.info("├── Successfully initialized " + totalMetrics + " metrics in " +
duration + "ms (" + String.format("%.1f", metricsPerSecond) + " metrics/sec)");
} catch (Exception e) {
logger.warning("Failed to initialize metrics: " + e.getMessage());
}
}
/**
* Reset the internal metricCache and clears all registered Gauges.
*/
@ -87,6 +136,10 @@ public class MetricCreditorService {
registeredGauges.clear();
}
public long getMetricCount() {
return metricCache.size();
}
/**
* Process Metric only if some data has changed....
*
@ -115,7 +168,7 @@ public class MetricCreditorService {
// update last metric only if exists...
if (!isNewMetric(lastMetricData)) {
subtractInvoice(lastMetricData, lastInvoice);
updateMetric(lastMetricData);
updateMetric(lastMetricData, true);
}
} catch (PluginException e) {
// invalid invoice - e.g. no cdtr. number
@ -127,7 +180,7 @@ public class MetricCreditorService {
ItemCollection metricData = getMetricByInvoice(invoice);
// Saldo-Berechnung
addInvoice(metricData, invoice);
updateMetric(metricData);
updateMetric(metricData, true);
logger.info("Metric cdtr update took " + (System.currentTimeMillis() - l) + "ms");
} catch (PluginException e) {
// invalid invoice - e.g. no cdtr. number
@ -222,17 +275,23 @@ public class MetricCreditorService {
}
/**
* Helper method to register a gauge for a creditor
* This method registers and updates the metric meta data objects based on a
* given metricData object. Optional the metric data object is persisted.
*
* @param cdtrNumber - the creditor number
* @param cdtrName - the creditor name
* @param metricData - the metricData ItemCollection
* @param persist - if true the metricData entity will be persisted
*/
public void updateMetric(ItemCollection metricData) {
public void updateMetric(ItemCollection metricData, boolean persist) {
String metricKey = metricData.getItemValueString("name");
// Cache aktualisieren
metricCache.put(metricKey, metricData);
// In Datenbank persistieren
if (persist) {
documentService.save(metricData);
}
// Prüfen ob Gauge bereits registriert ist
if (registeredGauges.add(metricKey)) { // returns true newly added
String bpName = metricData.getItemValueString("bp.name");
@ -267,9 +326,20 @@ public class MetricCreditorService {
() -> metricCache.get(metricKey).getItemValueDouble(ITEM_METRIC_SALES),
tags.toArray(new Tag[0]));
}
}
// /**
// * Helper Method that refreshes all gauges. The method is called by the
// * RestService during a rebuild.
// */
// public void updateAllMetrics() {
// List<String> keys = getMetricKeys();
// for (String hashKey : keys) {
// ItemCollection metricData = getMetric(hashKey);
// updateMetric(metricData, false);
// }
// }
/**
* Addiert den saldo einer Invoice zu einem metricData object
*
@ -316,16 +386,4 @@ public class MetricCreditorService {
}
/**
* Helper Method that refreshes all gauges. The method is called by the
* RestService during a rebuild.
*/
public void refreshGauges() {
List<String> keys = getMetricKeys();
for (String hashKey : keys) {
ItemCollection metricData = getMetric(hashKey);
updateMetric(metricData);
}
}
}

23
office-alexander-logistics-app/src/main/java/com/alexanderlogistics/metrics/MetricDataService.java
View file

@ -1,10 +1,13 @@
package com.alexanderlogistics.metrics;
import java.util.List;
import java.util.Objects;
import org.imixs.workflow.ItemCollection;
import org.imixs.workflow.engine.DocumentService;
import org.imixs.workflow.engine.index.SearchService;
import org.imixs.workflow.exceptions.PluginException;
import org.imixs.workflow.exceptions.QueryException;
import com.alexanderlogistics.InvoiceUtil;
@ -77,4 +80,24 @@ public class MetricDataService {
return "HASH" + hash;
}
/**
* This method deletes all metrics
*
* @throws PluginException
*
*/
@TransactionAttribute(value = TransactionAttributeType.REQUIRES_NEW)
public void deleteAllMetrics(String metricType) throws PluginException {
try {
String query = "(type:" + metricType + ")";
List<ItemCollection> result = documentService.find(query, SearchService.DEFAULT_MAX_SEARCH_RESULT, 0);
for (ItemCollection metric : result) {
documentService.remove(metric);
}
} catch (IllegalArgumentException | QueryException e) {
throw new PluginException(MetricDataService.class.getName(),
"Failed to delete metrics", e.getMessage(), e);
}
}
}

75
office-alexander-logistics-app/src/main/java/com/alexanderlogistics/metrics/MetricDebitorRestService.java
View file

@ -53,33 +53,30 @@ public class MetricDebitorRestService {
@Path("/rebuild")
@Produces({ MediaType.TEXT_PLAIN })
public Response rebuildMetrics() {
// Map<String, ItemCollection> metricCache = new HashMap<String,
// ItemCollection>();
StringBuffer messageBuffer = new StringBuffer();
long l = System.currentTimeMillis();
logger.info("├── init dbtr metrics...");
log("├── rebuild dbtr metrics...", messageBuffer);
try {
log("│   ├── delete all metrics", messageBuffer);
// first clear the metric cache
metricDebitorService.reset();
logger.info("│   ├── reset metric cache");
log("│   ├── reset metric cache", messageBuffer);
computeMetrics();
logger.info("│   ├── computing metrics finished in " + (System.currentTimeMillis() - l) + "ms");
logger.info("│   ├── save and init metrics ...");
// run in new transaction!
metricDebitorService.refreshGauges();
computeMetrics(messageBuffer);
String message = "├── rebuild dbtr metrics completed in "
+ (System.currentTimeMillis() - l)
+ "ms";
logger.info(message);
return Response.ok().entity(message).build();
log(message, messageBuffer);
return Response.ok().entity(messageBuffer.toString()).build();
} catch (Exception e) {
e.printStackTrace();
log("Failed to initialize metrics: " + e.getMessage(), messageBuffer);
return Response.serverError()
.entity("Failed to initialize metrics: " + e.getMessage())
.entity(messageBuffer.toString() + e.getMessage())
.build();
}
}
@ -88,37 +85,53 @@ public class MetricDebitorRestService {
* Diese Methode berechnet alle Metriken auf basis der existierenden Rechnungen
* neu
*
* @throws QueryException
* @throws InterruptedException
*
*/
public void computeMetrics() {
public void computeMetrics(StringBuffer messageBuffer) throws QueryException, InterruptedException {
long l = System.currentTimeMillis();
int batchSize = 500;
int totalInvoices = 0;
log("│   │   ├── recalculate metrics...", messageBuffer);
logger.info("│   │   ├── group invoices by debitor...");
try {
int count = 0;
List<ItemCollection> invoices = documentService.find(
"$modelversion:rechnungsausgang-* AND type:workitem",
9999, 0,
"invoice.number", false);
String query = "$modelversion:rechnungsausgang-* AND type:workitem";
// Gesamtanzahl ermitteln
int totalCount = documentService.count(query);
log("│   │   ├── found " + totalCount + " open invoices", messageBuffer);
// Berechne Anzahl der benötigten Pages
int totalPages = (int) Math.ceil((double) totalCount / batchSize);
// Verarbeite Page für Page
for (int pageIndex = 0; pageIndex < totalPages; pageIndex++) {
List<ItemCollection> invoices = documentService.find(query, batchSize, pageIndex);
logger.info("│   │   ├── found " + invoices.size() + " open invoices");
for (ItemCollection invoice : invoices) {
try {
ItemCollection metricData = metricDebitorService.getMetricByInvoice(invoice);
// Jetzt Rechnung addieren
metricDebitorService.addInvoice(metricData, invoice);
logger.info("│   │   │   ├──update metric " + InvoiceUtil.getBPId(invoice));
metricDebitorService.updateMetric(metricData);
count++;
logger.fine("│   │   │   ├── update metric " + InvoiceUtil.getBPId(invoice));
metricDebitorService.updateMetric(metricData, true);
totalInvoices++;
} catch (PluginException e) {
// invalid invoice - e.g. no dbtr. number
}
}
logger.info("│   │   ├── updated metric for " + count + " invoices.");
} catch (QueryException e) {
e.printStackTrace();
// Fortschritt loggen
log("│   │ ├── Processed page " + (pageIndex + 1) + " of " + totalPages +
" (" + totalInvoices + " of " + totalCount + " invoices)", messageBuffer);
// Optional: Kurze Pause nach jedem 5. Batch
Thread.sleep(100);
}
long duration = System.currentTimeMillis() - l;
double invoicesPerSecond = totalInvoices / (duration / 1000.0);
log("│   │   ├── Successfully processed " + totalInvoices + " invoices in " +
duration + "ms (" + String.format("%.1f", invoicesPerSecond) + " invoices/sec)", messageBuffer);
log("│   │   ├── Updated " + metricDebitorService.getMetricCount() + " metrics.", messageBuffer);
}

84
office-alexander-logistics-app/src/main/java/com/alexanderlogistics/metrics/MetricDebitorService.java
View file

@ -15,6 +15,7 @@ import org.eclipse.microprofile.metrics.annotation.RegistryScope;
import org.imixs.workflow.ItemCollection;
import org.imixs.workflow.engine.DocumentService;
import org.imixs.workflow.engine.ProcessingEvent;
import org.imixs.workflow.engine.SetupEvent;
import org.imixs.workflow.exceptions.PluginException;
import com.alexanderlogistics.InvoiceUtil;
@ -72,6 +73,54 @@ public class MetricDebitorService {
@ConfigProperty(name = "metrics.enabled", defaultValue = "false")
private boolean metricsEnabled;
/**
* Init all metrics during setup. Called by the Imixs SetupService
*/
public void initializeMetrics(@Observes SetupEvent setupEvent) {
if (!metricsEnabled) {
return;
}
long l = System.currentTimeMillis();
int batchSize = 500;
int totalMetrics = 0;
try {
logger.info("├── Initializing debitor metrics from database...");
String query = "(type:" + TYPE_METRIC_DEBITOR + ")";
// Gesamtanzahl ermitteln
int totalCount = documentService.count(query);
// Berechne Anzahl der benötigten Pages
int totalPages = (int) Math.ceil((double) totalCount / batchSize);
// Verarbeite Page für Page
for (int pageIndex = 0; pageIndex < totalPages; pageIndex++) {
List<ItemCollection> metrics = documentService.find(query, batchSize, pageIndex);
for (ItemCollection metric : metrics) {
updateMetric(metric, false);
totalMetrics++;
}
// Fortschritt loggen
logger.info("│ ├── Processed page " + (pageIndex + 1) + " of " + totalPages +
" (" + totalMetrics + " of " + totalCount + " metrics)");
// Optional: Kurze Pause nach jedem Batch
Thread.sleep(100);
}
long duration = System.currentTimeMillis() - l;
double metricsPerSecond = totalMetrics / (duration / 1000.0);
logger.info("├── Successfully initialized " + totalMetrics + " metrics in " +
duration + "ms (" + String.format("%.1f", metricsPerSecond) + " metrics/sec)");
} catch (Exception e) {
logger.warning("Failed to initialize metrics: " + e.getMessage());
}
}
/**
* Reset internal metricCache and clear registered Gauges.
*/
@ -80,6 +129,10 @@ public class MetricDebitorService {
registeredGauges.clear();
}
public long getMetricCount() {
return metricCache.size();
}
/**
* Process Metric only if some data has changed....
*
@ -107,7 +160,7 @@ public class MetricDebitorService {
ItemCollection lastMetricData = getMetricByInvoice(lastInvoice);
if (!isNewMetric(lastMetricData)) {
subtractInvoice(lastMetricData, lastInvoice);
updateMetric(lastMetricData);
updateMetric(lastMetricData, true);
}
} catch (PluginException e) {
// invalid invoice - e.g. no cdtr. number
@ -119,7 +172,7 @@ public class MetricDebitorService {
ItemCollection metricData = getMetricByInvoice(invoice);
// Saldo-Berechnung
addInvoice(metricData, invoice);
updateMetric(metricData);
updateMetric(metricData, true);
logger.info("Metric dbtr update took " + (System.currentTimeMillis() - l) + "ms");
} catch (PluginException e) {
@ -214,16 +267,23 @@ public class MetricDebitorService {
}
/**
* Helper method to register a gauge for a debitor
* This method registers and updates the metric meta data objects based on a
* given metricData object. Optional the metric data object is persisted.
*
* @param metricData - the metricData ItemCollection
* @param persist - if true the metricData entity will be persisted
*/
public void updateMetric(ItemCollection metricData) {
public void updateMetric(ItemCollection metricData, boolean persist) {
String metricKey = metricData.getItemValueString("name");
// Cache aktualisieren
metricCache.put(metricKey, metricData);
// In Datenbank persistieren
if (persist) {
documentService.save(metricData);
}
// Prüfen ob Gauge bereits registriert ist
if (registeredGauges.add(metricKey)) { // returns true newly added
String bpName = metricData.getItemValueString("bp.name");
@ -242,6 +302,7 @@ public class MetricDebitorService {
logger.fine("register new metric for department: " + department +
", " + metricData.getItemValueString(ITEM_METRIC_BALANCE) +
" " + currency);
// Saldo Gauge
Metadata balanceMetadata = Metadata.builder()
.withName("dbtr.balance")
@ -317,13 +378,12 @@ public class MetricDebitorService {
* Helper Method that refreshes all gauges. The method is called by the
* RestService during a rebuild.
*/
public void refreshGauges() {
List<String> keys = getMetricKeys();
for (String hashKey : keys) {
ItemCollection metricData = getMetric(hashKey);
updateMetric(metricData);
}
}
// public void refreshGauges() {
// List<String> keys = getMetricKeys();
// for (String hashKey : keys) {
// ItemCollection metricData = getMetric(hashKey);
// updateMetric(metricData, false);
// }
// }
}