From 27698becf549ba0ad0ce16303369ea878aaaa728 Mon Sep 17 00:00:00 2001 From: Ralph Soika Date: Sat, 26 Apr 2025 14:39:03 +0200 Subject: [PATCH] neuester schrei --- RELEASENOTES.md | 8 + doc/BUSINESSPARTNER.md | 5 + .../api/CargosoftMigrationRestService.java | 123 ++++++++++++-- .../xml/BusinessPartnerImportService.java | 151 ++++++++++-------- 4 files changed, 208 insertions(+), 79 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 77ac3e3..69db643 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -10,6 +10,14 @@ - Encoding für CSV Impoort aus cargosoft muss auf `encoding=UTF-8` stehen! - Neues Businesspartner BPMN Modell einspielen +- Erstmal nach Dubletten suchen: + + - https://alexander-logistics-dwc.office-workflow.de/api/cargosoft/remove-dubletten?maxcount=5000 + ACHTUNG kann sehr lange laufen (1 Stunde) + +- Jetzt die Daten synchen +- https://alexander-logistics-dwc.office-workflow.de/api/cargosoft/bp-sync?maxcount=25000 + ACHTUNG kann sehr sehr sehr!! lange laufen (3 Stunde) Eigentlich geht das gar nciht :-((( ### 1.3.3 diff --git a/doc/BUSINESSPARTNER.md b/doc/BUSINESSPARTNER.md index aed38e8..507ad40 100644 --- a/doc/BUSINESSPARTNER.md +++ b/doc/BUSINESSPARTNER.md @@ -63,3 +63,8 @@ Der Service migriert auch die alten zusätzlichen IBAN/BIC felder wenn der Busin ## Anmerkungen zu seltsamen Daten BP4641 existier jetzt 3 mal mit der Debitoren nummer D14641 + +Feldmühle + +- D14169 + K74169 diff --git a/office-alexander-logistics-app/src/main/java/com/alexanderlogistics/api/CargosoftMigrationRestService.java b/office-alexander-logistics-app/src/main/java/com/alexanderlogistics/api/CargosoftMigrationRestService.java index 91863ae..38c757c 100644 --- a/office-alexander-logistics-app/src/main/java/com/alexanderlogistics/api/CargosoftMigrationRestService.java +++ b/office-alexander-logistics-app/src/main/java/com/alexanderlogistics/api/CargosoftMigrationRestService.java @@ -45,6 +45,7 @@ import org.imixs.workflow.exceptions.QueryException; import com.alexanderlogistics.BusinessPartnerService; import com.alexanderlogistics.InvoiceUtil; import com.alexanderlogistics.mahnlauf.MahnlaufService; +import com.alexanderlogistics.xml.BusinessPartnerImportService; import jakarta.ejb.TransactionAttribute; import jakarta.ejb.TransactionAttributeType; @@ -95,6 +96,9 @@ public class CargosoftMigrationRestService implements Serializable { @Inject BusinessPartnerService businessPartnerService; + @Inject + BusinessPartnerImportService businessPartnerImportService; + private static Logger logger = Logger.getLogger(CargosoftMigrationRestService.class.getName()); public CargosoftMigrationRestService() { @@ -102,10 +106,9 @@ public class CargosoftMigrationRestService implements Serializable { } /** - * Dieser agent prüft ob es für Cargosoft Krediotren/Debitoren Daten schon ein - * passendes Businesspartner Workitem gibt. - * Wenn nicht speichert der Service einmal das cargosft entity was dann zur - * automatischen Neuanlage des businesspartner workflows führt. + * Dieser agent speichert einfach alle Cargosoft Krediotren/Debitoren Daten , + * was dann zur automatischen Neuanlage bzw. aktualisierung des businesspartner + * workflows führt. * * * curl -H "Cookie: @@ -155,18 +158,19 @@ public class CargosoftMigrationRestService implements Serializable { for (ItemCollection cargosoftItemCol : cargosoftDataList) { totalObjects++; - if (syncBusinessPartner(cargosoftItemCol)) { - syncs++; - // Explicit flush the lucene search event log + + if (businessPartnerImportService.syncCargosoftCreditorBusinessPartner(cargosoftItemCol) == true) { updateService.updateIndex(); + syncs++; + if (syncs >= maxcount) + break; } - if (syncs >= maxcount) - break; + } // Fortschritt loggen log("│ ├── Processed page " + (pageIndex + 1) + " of " + totalPages + - " (" + syncs + " total syncs)", messageBuffer); + " (" + totalObjects + " objects read, " + syncs + " total syncs)", messageBuffer); // Optional: Kurze Pause nach jedem 5. Batch try { Thread.sleep(100); @@ -293,6 +297,105 @@ public class CargosoftMigrationRestService implements Serializable { return messageBuffer.toString(); } + /** + * Entfernt doppelte Eintrage - sind irgnedwie entstandnen :((( + * + * @return + * @throws QueryException + * @throws AccessDeniedException + * @throws ProcessingErrorException + * @throws PluginException + * @throws ModelException + */ + @GET + @Path("/remove-dubletten") + @Produces({ MediaType.TEXT_PLAIN }) + public String deleteCargosoftDupplicates(@QueryParam("maxcount") int maxcount) + throws QueryException, AccessDeniedException, ProcessingErrorException, PluginException, ModelException { + StringBuffer messageBuffer = new StringBuffer(); + + String query = "(type:cargosoftkreditor)"; + + int deletions = 0; + long l = System.currentTimeMillis(); + int batchSize = 100; + int totalObjects = 0; + log("├── delete cargosoft dupplicaates....", messageBuffer); + + if (isRunning) { + log("├── sync process already running!", messageBuffer); + return messageBuffer.toString(); + } + isRunning = true; + + log("│   ├── maxcount= " + maxcount, messageBuffer); + int totalCount = documentService.count(query); + log("│   ├── found " + totalCount + " cargosoft entries", messageBuffer); + + // Berechne Anzahl der benötigten Pages + int totalPages = (int) Math.ceil((double) totalCount / batchSize); + + // Verarbeite Page für Page + for (int pageIndex = 0; pageIndex < totalPages; pageIndex++) { + + List cargosoftDataList = documentService.find(query, batchSize, pageIndex); + if (cargosoftDataList.size() == 0) { + break; + } + for (ItemCollection cargosoftItemCol : cargosoftDataList) { + totalObjects++; + + String name = cargosoftItemCol.getItemValueString("name"); + String subQuery = "(type:cargosoftkreditor) AND (name:" + name + ")"; + List dupplicateList = documentService.find(subQuery, 100, 0, "$modified", true); + if (dupplicateList.size() > 1) { + // alle bis auf den ersten löschen + log("│   ├── found duplicate: " + name, messageBuffer); + boolean first = true; + for (ItemCollection dubel : dupplicateList) { + + if (first) { + first = false; + continue; + } + // löschen! + documentService.remove(dubel); + deletions++; + } + + } + + if (deletions >= maxcount) + break; + } + // Explicit flush the lucene search event log + updateService.updateIndex(); + + // Fortschritt loggen + // Fortschritt loggen + log("│ ├── Processed page " + (pageIndex + 1) + " of " + totalPages + + " (" + deletions + " total deletions)", messageBuffer); + // Optional: Kurze Pause nach jedem 5. Batch + try { + Thread.sleep(100); + } catch (InterruptedException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + if (deletions >= maxcount) + break; + + } + long duration = System.currentTimeMillis() - l; + double objectsPerSecond = totalObjects / (duration / 1000.0); + log("├── Successfully " + deletions + " duplicated cargosoft objects deleted in " + + duration + "ms (" + String.format("%.1f", objectsPerSecond) + " objects/sec)", messageBuffer); + + isRunning = false; + return messageBuffer.toString(); + } + /** * Hilfsmethode - löscht ein cargoosoft object... * diff --git a/office-alexander-logistics-app/src/main/java/com/alexanderlogistics/xml/BusinessPartnerImportService.java b/office-alexander-logistics-app/src/main/java/com/alexanderlogistics/xml/BusinessPartnerImportService.java index 9988454..b349244 100644 --- a/office-alexander-logistics-app/src/main/java/com/alexanderlogistics/xml/BusinessPartnerImportService.java +++ b/office-alexander-logistics-app/src/main/java/com/alexanderlogistics/xml/BusinessPartnerImportService.java @@ -128,79 +128,92 @@ public class BusinessPartnerImportService { logger.fine("Verify business partner object...."); ItemCollection importDoc = event.getDocument(); - - String name = event.getDocument().getItemValueString("name"); - boolean isCreditor = name.toUpperCase().startsWith("K"); - String partnerID = InvoiceUtil.buildBPID(name); - - // Prüfen ob es diesen partner schon gibt? - ItemCollection businesspartner = lookupBusinessPartner(partnerID); - if (businesspartner == null) { - businesspartner = createBusinessPartner(partnerID); - } - ItemCollection oldBussinessPartnerItemColl = (ItemCollection) businesspartner.clone(); - - // Update Fields... - businesspartner.setItemValue("name", partnerID); - businesspartner.setItemValue("partner.id", partnerID); - if (isCreditor) { - businesspartner.setItemValue("cdtr.number", name); - if (businesspartner.isItemEmpty("cdtr.mail")) { - businesspartner.setItemValue("cdtr.mail", importDoc.getItemValueString("_vendor_email")); - } - } else { - businesspartner.setItemValue("dbtr.number", name); - if (businesspartner.isItemEmpty("dbtr.mail")) { - businesspartner.setItemValue("dbtr.mail", importDoc.getItemValueString("_vendor_email")); - } - } - businesspartner.setItemValue("partner.country", importDoc.getItemValueString("_vendor_country")); - businesspartner.setItemValue("partner.name", importDoc.getItemValueString("_vendor_name")); - businesspartner.setItemValue("partner.phone", importDoc.getItemValueString("_vendor_phone")); - businesspartner.setItemValue("partner.address", importDoc.getItemValueString("_vendor_str")); - businesspartner.setItemValue("partner.zip", importDoc.getItemValueString("_vendor_zip_code")); - businesspartner.setItemValue("partner.city", importDoc.getItemValueString("_vendor_city")); - - // hat sich das Objekt verändert? - if (!businesspartner.equals(oldBussinessPartnerItemColl)) { - - // Prüfe IBAN auf plausibilität - List bankList = BusinessPartnerService.getBanks(businesspartner); - boolean ibanDublette = false; - List dublettenIban = new ArrayList<>(); - for (ItemCollection bank : bankList) { - String iban = bank.getItemValueString("iban"); - // erkenne fehlerhaften kontonummern - if (dublettenIban.contains(iban)) { - // ACHTUNG - uneindeutige Bankverbindung - ibanDublette = true; - } else { - dublettenIban.add(iban); - } - } - - int eventID = 200; // default update - // event errechnen - if (!businesspartner.hasItem(WorkflowKernel.LASTEVENT)) { - // create new! - if (ibanDublette) { - eventID = 300; // Exception - } else { - eventID = 100; // new import - } - } - businesspartner.setEventID(eventID); - try { - workflowService.processWorkItemByNewTransaction(businesspartner); - } catch (AccessDeniedException | ProcessingErrorException | PluginException | ModelException e) { - logger.warning("Failed to update Business Partner Object " + name + " - " + e.getMessage()); - } - } - + syncCargosoftCreditorBusinessPartner(importDoc); } } + /** + * Syncnrhonisert ein drecks Cargosoft Objekt mit dem neuen Busienss partner + * objekt + * + * Gibt true urück wenn sich was verädnert hat + * + * @param creditor + */ + public boolean syncCargosoftCreditorBusinessPartner(ItemCollection importDoc) { + String name = importDoc.getItemValueString("name"); + boolean isCreditor = name.toUpperCase().startsWith("K"); + String partnerID = InvoiceUtil.buildBPID(name); + + // Prüfen ob es diesen partner schon gibt? + ItemCollection businesspartner = lookupBusinessPartner(partnerID); + if (businesspartner == null) { + businesspartner = createBusinessPartner(partnerID); + } + ItemCollection oldBussinessPartnerItemColl = (ItemCollection) businesspartner.clone(); + + // Update Fields... + businesspartner.setItemValue("name", partnerID); + businesspartner.setItemValue("partner.id", partnerID); + if (isCreditor) { + businesspartner.setItemValue("cdtr.number", name); + if (businesspartner.isItemEmpty("cdtr.mail")) { + businesspartner.setItemValue("cdtr.mail", importDoc.getItemValueString("_vendor_email")); + } + } else { + businesspartner.setItemValue("dbtr.number", name); + if (businesspartner.isItemEmpty("dbtr.mail")) { + businesspartner.setItemValue("dbtr.mail", importDoc.getItemValueString("_vendor_email")); + } + } + businesspartner.setItemValue("partner.country", importDoc.getItemValueString("_vendor_country")); + businesspartner.setItemValue("partner.name", importDoc.getItemValueString("_vendor_name")); + businesspartner.setItemValue("partner.phone", importDoc.getItemValueString("_vendor_phone")); + businesspartner.setItemValue("partner.address", importDoc.getItemValueString("_vendor_str")); + businesspartner.setItemValue("partner.zip", importDoc.getItemValueString("_vendor_zip_code")); + businesspartner.setItemValue("partner.city", importDoc.getItemValueString("_vendor_city")); + + // hat sich das Objekt verändert? + if (!businesspartner.equals(oldBussinessPartnerItemColl)) { + + // Prüfe IBAN auf plausibilität + List bankList = BusinessPartnerService.getBanks(businesspartner); + boolean ibanDublette = false; + List dublettenIban = new ArrayList<>(); + for (ItemCollection bank : bankList) { + String iban = bank.getItemValueString("iban"); + // erkenne fehlerhaften kontonummern + if (dublettenIban.contains(iban)) { + // ACHTUNG - uneindeutige Bankverbindung + ibanDublette = true; + } else { + dublettenIban.add(iban); + } + } + + int eventID = 200; // default update + // event errechnen + if (!businesspartner.hasItem(WorkflowKernel.LASTEVENT)) { + // create new! + if (ibanDublette) { + eventID = 300; // Exception + } else { + eventID = 100; // new import + } + } + businesspartner.setEventID(eventID); + try { + workflowService.processWorkItemByNewTransaction(businesspartner); + } catch (AccessDeniedException | ProcessingErrorException | PluginException | ModelException e) { + logger.warning("Failed to update Business Partner Object " + name + " - " + e.getMessage()); + } + return true; + } else { + return false; // no changes + } + } + /** * Erstellt ein neues leeres Business Partner Object *