neuester schrei

2025-04-26 14:39:03 +02:00 · 2025-04-26 14:39:03 +02:00 · 27698becf5
commit 27698becf5
parent 7ba7967a92
4 changed files with 208 additions and 79 deletions
--- a/RELEASENOTES.md
+++ b/RELEASENOTES.md
@ -10,6 +10,14 @@

 - Encoding für CSV Impoort aus cargosoft muss auf `encoding=UTF-8` stehen!
 - Neues Businesspartner BPMN Modell einspielen
+- Erstmal nach Dubletten suchen:
+
+  - https://alexander-logistics-dwc.office-workflow.de/api/cargosoft/remove-dubletten?maxcount=5000
+    ACHTUNG kann sehr lange laufen (1 Stunde)
+
+- Jetzt die Daten synchen
+- https://alexander-logistics-dwc.office-workflow.de/api/cargosoft/bp-sync?maxcount=25000
+  ACHTUNG kann sehr sehr sehr!! lange laufen (3 Stunde) Eigentlich geht das gar nciht :-(((

 ### 1.3.3

--- a/doc/BUSINESSPARTNER.md
+++ b/doc/BUSINESSPARTNER.md
@ -63,3 +63,8 @@ Der Service migriert auch die alten zusätzlichen IBAN/BIC felder wenn der Busin
 ## Anmerkungen zu seltsamen Daten

 BP4641 existier jetzt 3 mal mit der Debitoren nummer D14641
+
+Feldmühle
+
+- D14169
+  K74169
--- a/office-alexander-logistics-app/src/main/java/com/alexanderlogistics/api/CargosoftMigrationRestService.java
+++ b/office-alexander-logistics-app/src/main/java/com/alexanderlogistics/api/CargosoftMigrationRestService.java
@ -45,6 +45,7 @@ import org.imixs.workflow.exceptions.QueryException;
 import com.alexanderlogistics.BusinessPartnerService;
 import com.alexanderlogistics.InvoiceUtil;
 import com.alexanderlogistics.mahnlauf.MahnlaufService;
+import com.alexanderlogistics.xml.BusinessPartnerImportService;

 import jakarta.ejb.TransactionAttribute;
 import jakarta.ejb.TransactionAttributeType;
@ -95,6 +96,9 @@ public class CargosoftMigrationRestService implements Serializable {
    @Inject
    BusinessPartnerService businessPartnerService;

+    @Inject
+    BusinessPartnerImportService businessPartnerImportService;
+
    private static Logger logger = Logger.getLogger(CargosoftMigrationRestService.class.getName());

    public CargosoftMigrationRestService() {
@ -102,10 +106,9 @@ public class CargosoftMigrationRestService implements Serializable {
    }

    /**
-     * Dieser agent prüft ob es für Cargosoft Krediotren/Debitoren Daten schon ein
-     * passendes Businesspartner Workitem gibt.
-     * Wenn nicht speichert der Service einmal das cargosft entity was dann zur
-     * automatischen Neuanlage des businesspartner workflows führt.
+     * Dieser agent speichert einfach alle Cargosoft Krediotren/Debitoren Daten ,
+     * was dann zur automatischen Neuanlage bzw. aktualisierung des businesspartner
+     * workflows führt.
     * 
     * 
     * curl -H "Cookie:
@ -155,18 +158,19 @@ public class CargosoftMigrationRestService implements Serializable {

            for (ItemCollection cargosoftItemCol : cargosoftDataList) {
                totalObjects++;
-                if (syncBusinessPartner(cargosoftItemCol)) {
-                    syncs++;
-                    // Explicit flush the lucene search event log
+
+                if (businessPartnerImportService.syncCargosoftCreditorBusinessPartner(cargosoftItemCol) == true) {
                    updateService.updateIndex();
+                    syncs++;
+                    if (syncs >= maxcount)
+                        break;
                }
-                if (syncs >= maxcount)
-                    break;
+
            }

            // Fortschritt loggen
            log("│   ├── Processed page " + (pageIndex + 1) + " of " + totalPages +
-                    " (" + syncs + " total syncs)", messageBuffer);
+                    " (" + totalObjects + " objects read, " + syncs + " total syncs)", messageBuffer);
            // Optional: Kurze Pause nach jedem 5. Batch
            try {
                Thread.sleep(100);
@ -293,6 +297,105 @@ public class CargosoftMigrationRestService implements Serializable {
        return messageBuffer.toString();
    }

+    /**
+     * Entfernt doppelte Eintrage - sind irgnedwie entstandnen :(((
+     * 
+     * @return
+     * @throws QueryException
+     * @throws AccessDeniedException
+     * @throws ProcessingErrorException
+     * @throws PluginException
+     * @throws ModelException
+     */
+    @GET
+    @Path("/remove-dubletten")
+    @Produces({ MediaType.TEXT_PLAIN })
+    public String deleteCargosoftDupplicates(@QueryParam("maxcount") int maxcount)
+            throws QueryException, AccessDeniedException, ProcessingErrorException, PluginException, ModelException {
+        StringBuffer messageBuffer = new StringBuffer();
+
+        String query = "(type:cargosoftkreditor)";
+
+        int deletions = 0;
+        long l = System.currentTimeMillis();
+        int batchSize = 100;
+        int totalObjects = 0;
+        log("├── delete cargosoft dupplicaates....", messageBuffer);
+
+        if (isRunning) {
+            log("├── sync process already running!", messageBuffer);
+            return messageBuffer.toString();
+        }
+        isRunning = true;
+
+        log("│   ├── maxcount= " + maxcount, messageBuffer);
+        int totalCount = documentService.count(query);
+        log("│   ├── found " + totalCount + " cargosoft entries", messageBuffer);
+
+        // Berechne Anzahl der benötigten Pages
+        int totalPages = (int) Math.ceil((double) totalCount / batchSize);
+
+        // Verarbeite Page für Page
+        for (int pageIndex = 0; pageIndex < totalPages; pageIndex++) {
+
+            List<ItemCollection> cargosoftDataList = documentService.find(query, batchSize, pageIndex);
+            if (cargosoftDataList.size() == 0) {
+                break;
+            }
+            for (ItemCollection cargosoftItemCol : cargosoftDataList) {
+                totalObjects++;
+
+                String name = cargosoftItemCol.getItemValueString("name");
+                String subQuery = "(type:cargosoftkreditor) AND (name:" + name + ")";
+                List<ItemCollection> dupplicateList = documentService.find(subQuery, 100, 0, "$modified", true);
+                if (dupplicateList.size() > 1) {
+                    // alle bis auf den ersten löschen
+                    log("│   ├── found duplicate: " + name, messageBuffer);
+                    boolean first = true;
+                    for (ItemCollection dubel : dupplicateList) {
+
+                        if (first) {
+                            first = false;
+                            continue;
+                        }
+                        // löschen!
+                        documentService.remove(dubel);
+                        deletions++;
+                    }
+
+                }
+
+                if (deletions >= maxcount)
+                    break;
+            }
+            // Explicit flush the lucene search event log
+            updateService.updateIndex();
+
+            // Fortschritt loggen
+            // Fortschritt loggen
+            log("│   ├── Processed page " + (pageIndex + 1) + " of " + totalPages +
+                    " (" + deletions + " total deletions)", messageBuffer);
+            // Optional: Kurze Pause nach jedem 5. Batch
+            try {
+                Thread.sleep(100);
+            } catch (InterruptedException e) {
+                // TODO Auto-generated catch block
+                e.printStackTrace();
+            }
+
+            if (deletions >= maxcount)
+                break;
+
+        }
+        long duration = System.currentTimeMillis() - l;
+        double objectsPerSecond = totalObjects / (duration / 1000.0);
+        log("├── Successfully  " + deletions + " duplicated cargosoft  objects deleted in " +
+                duration + "ms (" + String.format("%.1f", objectsPerSecond) + " objects/sec)", messageBuffer);
+
+        isRunning = false;
+        return messageBuffer.toString();
+    }
+
    /**
     * Hilfsmethode - löscht ein cargoosoft object...
     * 
--- a/office-alexander-logistics-app/src/main/java/com/alexanderlogistics/xml/BusinessPartnerImportService.java
+++ b/office-alexander-logistics-app/src/main/java/com/alexanderlogistics/xml/BusinessPartnerImportService.java
@ -128,79 +128,92 @@ public class BusinessPartnerImportService {

            logger.fine("Verify business partner object....");
            ItemCollection importDoc = event.getDocument();
-
-            String name = event.getDocument().getItemValueString("name");
-            boolean isCreditor = name.toUpperCase().startsWith("K");
-            String partnerID = InvoiceUtil.buildBPID(name);
-
-            // Prüfen ob es diesen partner schon gibt?
-            ItemCollection businesspartner = lookupBusinessPartner(partnerID);
-            if (businesspartner == null) {
-                businesspartner = createBusinessPartner(partnerID);
-            }
-            ItemCollection oldBussinessPartnerItemColl = (ItemCollection) businesspartner.clone();
-
-            // Update Fields...
-            businesspartner.setItemValue("name", partnerID);
-            businesspartner.setItemValue("partner.id", partnerID);
-            if (isCreditor) {
-                businesspartner.setItemValue("cdtr.number", name);
-                if (businesspartner.isItemEmpty("cdtr.mail")) {
-                    businesspartner.setItemValue("cdtr.mail", importDoc.getItemValueString("_vendor_email"));
-                }
-            } else {
-                businesspartner.setItemValue("dbtr.number", name);
-                if (businesspartner.isItemEmpty("dbtr.mail")) {
-                    businesspartner.setItemValue("dbtr.mail", importDoc.getItemValueString("_vendor_email"));
-                }
-            }
-            businesspartner.setItemValue("partner.country", importDoc.getItemValueString("_vendor_country"));
-            businesspartner.setItemValue("partner.name", importDoc.getItemValueString("_vendor_name"));
-            businesspartner.setItemValue("partner.phone", importDoc.getItemValueString("_vendor_phone"));
-            businesspartner.setItemValue("partner.address", importDoc.getItemValueString("_vendor_str"));
-            businesspartner.setItemValue("partner.zip", importDoc.getItemValueString("_vendor_zip_code"));
-            businesspartner.setItemValue("partner.city", importDoc.getItemValueString("_vendor_city"));
-
-            // hat sich das Objekt verändert?
-            if (!businesspartner.equals(oldBussinessPartnerItemColl)) {
-
-                // Prüfe IBAN auf plausibilität
-                List<ItemCollection> bankList = BusinessPartnerService.getBanks(businesspartner);
-                boolean ibanDublette = false;
-                List<String> dublettenIban = new ArrayList<>();
-                for (ItemCollection bank : bankList) {
-                    String iban = bank.getItemValueString("iban");
-                    // erkenne fehlerhaften kontonummern
-                    if (dublettenIban.contains(iban)) {
-                        // ACHTUNG - uneindeutige Bankverbindung
-                        ibanDublette = true;
-                    } else {
-                        dublettenIban.add(iban);
-                    }
-                }
-
-                int eventID = 200; // default update
-                // event errechnen
-                if (!businesspartner.hasItem(WorkflowKernel.LASTEVENT)) {
-                    // create new!
-                    if (ibanDublette) {
-                        eventID = 300; // Exception
-                    } else {
-                        eventID = 100; // new import
-                    }
-                }
-                businesspartner.setEventID(eventID);
-                try {
-                    workflowService.processWorkItemByNewTransaction(businesspartner);
-                } catch (AccessDeniedException | ProcessingErrorException | PluginException | ModelException e) {
-                    logger.warning("Failed to update Business Partner Object " + name + " - " + e.getMessage());
-                }
-            }
-
+            syncCargosoftCreditorBusinessPartner(importDoc);
        }

    }

+    /**
+     * Syncnrhonisert ein drecks Cargosoft Objekt mit dem neuen Busienss partner
+     * objekt
+     * 
+     * Gibt true urück wenn sich was verädnert hat
+     * 
+     * @param creditor
+     */
+    public boolean syncCargosoftCreditorBusinessPartner(ItemCollection importDoc) {
+        String name = importDoc.getItemValueString("name");
+        boolean isCreditor = name.toUpperCase().startsWith("K");
+        String partnerID = InvoiceUtil.buildBPID(name);
+
+        // Prüfen ob es diesen partner schon gibt?
+        ItemCollection businesspartner = lookupBusinessPartner(partnerID);
+        if (businesspartner == null) {
+            businesspartner = createBusinessPartner(partnerID);
+        }
+        ItemCollection oldBussinessPartnerItemColl = (ItemCollection) businesspartner.clone();
+
+        // Update Fields...
+        businesspartner.setItemValue("name", partnerID);
+        businesspartner.setItemValue("partner.id", partnerID);
+        if (isCreditor) {
+            businesspartner.setItemValue("cdtr.number", name);
+            if (businesspartner.isItemEmpty("cdtr.mail")) {
+                businesspartner.setItemValue("cdtr.mail", importDoc.getItemValueString("_vendor_email"));
+            }
+        } else {
+            businesspartner.setItemValue("dbtr.number", name);
+            if (businesspartner.isItemEmpty("dbtr.mail")) {
+                businesspartner.setItemValue("dbtr.mail", importDoc.getItemValueString("_vendor_email"));
+            }
+        }
+        businesspartner.setItemValue("partner.country", importDoc.getItemValueString("_vendor_country"));
+        businesspartner.setItemValue("partner.name", importDoc.getItemValueString("_vendor_name"));
+        businesspartner.setItemValue("partner.phone", importDoc.getItemValueString("_vendor_phone"));
+        businesspartner.setItemValue("partner.address", importDoc.getItemValueString("_vendor_str"));
+        businesspartner.setItemValue("partner.zip", importDoc.getItemValueString("_vendor_zip_code"));
+        businesspartner.setItemValue("partner.city", importDoc.getItemValueString("_vendor_city"));
+
+        // hat sich das Objekt verändert?
+        if (!businesspartner.equals(oldBussinessPartnerItemColl)) {
+
+            // Prüfe IBAN auf plausibilität
+            List<ItemCollection> bankList = BusinessPartnerService.getBanks(businesspartner);
+            boolean ibanDublette = false;
+            List<String> dublettenIban = new ArrayList<>();
+            for (ItemCollection bank : bankList) {
+                String iban = bank.getItemValueString("iban");
+                // erkenne fehlerhaften kontonummern
+                if (dublettenIban.contains(iban)) {
+                    // ACHTUNG - uneindeutige Bankverbindung
+                    ibanDublette = true;
+                } else {
+                    dublettenIban.add(iban);
+                }
+            }
+
+            int eventID = 200; // default update
+            // event errechnen
+            if (!businesspartner.hasItem(WorkflowKernel.LASTEVENT)) {
+                // create new!
+                if (ibanDublette) {
+                    eventID = 300; // Exception
+                } else {
+                    eventID = 100; // new import
+                }
+            }
+            businesspartner.setEventID(eventID);
+            try {
+                workflowService.processWorkItemByNewTransaction(businesspartner);
+            } catch (AccessDeniedException | ProcessingErrorException | PluginException | ModelException e) {
+                logger.warning("Failed to update Business Partner Object " + name + " - " + e.getMessage());
+            }
+            return true;
+        } else {
+            return false; // no changes
+        }
+    }
+
    /**
     * Erstellt ein neues leeres Business Partner Object
     *