/* * Imixs-Workflow * * Copyright (C) 2001-2020 Imixs Software Solutions GmbH, * http://www.imixs.com * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You can receive a copy of the GNU General Public * License at http://www.gnu.org/licenses/gpl.html * * Project: * https://www.imixs.org * https://github.com/imixs/imixs-workflow * * Contributors: * Imixs Software Solutions GmbH - Project Management * Ralph Soika - Software Developer */ package com.alexanderlogistics.xml; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.StringWriter; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.TimeZone; import java.util.logging.Logger; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; import javax.xml.transform.Transformer; import javax.xml.transform.TransformerException; import javax.xml.transform.TransformerFactory; import javax.xml.transform.dom.DOMSource; import javax.xml.transform.stream.StreamResult; import javax.xml.xpath.XPath; import javax.xml.xpath.XPathConstants; import javax.xml.xpath.XPathExpression; import javax.xml.xpath.XPathExpressionException; import javax.xml.xpath.XPathFactory; import org.apache.commons.net.ftp.FTP; import org.apache.commons.net.ftp.FTPClient; import org.apache.commons.net.ftp.FTPFile; import org.apache.commons.net.ftp.FTPSClient; import org.imixs.archive.importer.DocumentImportEvent; import org.imixs.archive.importer.DocumentImportService; import org.imixs.workflow.FileData; import org.imixs.workflow.ItemCollection; import org.imixs.workflow.engine.DocumentService; import org.imixs.workflow.engine.ModelService; import org.imixs.workflow.engine.WorkflowService; import org.imixs.workflow.exceptions.AccessDeniedException; import org.imixs.workflow.exceptions.ModelException; import org.imixs.workflow.exceptions.PluginException; import org.imixs.workflow.exceptions.ProcessingErrorException; import org.imixs.workflow.exceptions.QueryException; import org.w3c.dom.Document; import org.w3c.dom.Node; import org.w3c.dom.NodeList; import org.xml.sax.InputSource; import org.xml.sax.SAXException; import com.alexanderlogistics.InvoiceUtil; import com.alexanderlogistics.KreditorDebitorService; import com.alexanderlogistics.mahnlauf.MahnlaufService; import jakarta.ejb.EJB; import jakarta.ejb.Stateless; import jakarta.enterprise.event.Observes; import jakarta.inject.Inject; /** * Der CargosoftXMLEAkteImportService erweitert den FTPImportService und * kann XML EAkten (Steuerbescheide) von Cargosoft importieren. * Der Service reagiert auf DocumentImportEvents und kann über das Options Feld * konfiguriert werden. *

* Im Gegensatz zum FTPImportService werden die XML Dateien nach einem * speziellen Verfahren geparsed und dann als Steuerbescheid importiert. *

* In den Optionen muss die option "mandant.id" hinterlegt sein. Stimmt * diese beim Import nicht mit der Mandanten Nummer in der XML Datei überein, * wird das XML Dokument gelöscht und nicht importiert. * * * * @author rsoika * */ @Stateless public class CargosoftXMLEAkteImportService { private static Logger logger = Logger.getLogger(CargosoftXMLEAkteImportService.class.getName()); public static final String OPTION_MANDANT_ID = "mandant.id"; public static final String DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss"; public static final String REGEX_IMPORTTEXTPATTERN = "^R[1-8] .*$"; @EJB WorkflowService workflowService; @EJB DocumentService documentService; @EJB ModelService modelService; @EJB KreditorDebitorService kreditorDebitorService; @EJB DocumentImportService documentImportService; @Inject MahnlaufService mahnlaufService; /** * This method reacts on a CDI ImportEvent and reads documents form a ftp * server. * * */ public void onEvent(@Observes DocumentImportEvent event) { // check if source is already completed if (event.getResult() == DocumentImportEvent.PROCESSING_COMPLETED) { logger.finest("...... import source already completed - no processing will be performed."); return; } if (!"CARGOSOFT_EAKTE_XML".equalsIgnoreCase(event.getSource().getItemValueString("type"))) { // ignore data source logger.finest("...... type '" + event.getSource().getItemValueString("type") + "' skipped."); return; } Properties sourceOptions = documentImportService.getOptionsProperties(event.getSource()); // Read Mandand ID form Option list String mandantID = sourceOptions.getProperty(OPTION_MANDANT_ID, ""); if (mandantID == null || mandantID.isEmpty()) { documentImportService.logMessage("...CARGOSOFT_EAKTE_XML Import failed - missing mandant.id in options!", event); event.setResult(DocumentImportEvent.PROCESSING_ERROR); return; } documentImportService.logMessage("...CARGOSOFT_EAKTE_XML Import started: mandant.id=" + mandantID, event); String ftpServer = event.getSource().getItemValueString(DocumentImportService.SOURCE_ITEM_SERVER); String ftpPort = event.getSource().getItemValueString(DocumentImportService.SOURCE_ITEM_PORT); String ftpUser = event.getSource().getItemValueString(DocumentImportService.SOURCE_ITEM_USER); String ftpPassword = event.getSource().getItemValueString(DocumentImportService.SOURCE_ITEM_PASSWORD); String ftpPath = event.getSource().getItemValueString(DocumentImportService.SOURCE_ITEM_SELECTOR); if (!ftpPath.startsWith("/") && !ftpPath.startsWith("./")) { ftpPath = "/" + ftpPath; } if (!ftpPath.endsWith("/")) { ftpPath = ftpPath + "/"; } // if no server is given we exit if (ftpServer.isEmpty()) { logger.warning("...... no server specified!"); return; } if (ftpPort.isEmpty()) { // set default port ftpPort = "21"; } try { documentImportService.logMessage("...connecting to FTP server: " + ftpServer, event); documentImportService.logMessage("...working directory: " + ftpPath, event); FTPClient ftpClient = connectServer(ftpServer, Integer.parseInt(ftpPort), ftpUser, ftpPassword); scannFTPDirectory(ftpClient, ftpPath, mandantID, event); } catch (PluginException pe) { documentImportService.logMessage(pe.getMessage(), event); event.setResult(DocumentImportEvent.PROCESSING_ERROR); return; } // completed event.setResult(DocumentImportEvent.PROCESSING_COMPLETED); } /** * Connects to the FTP Server and returns a ftpClient instance. */ private FTPClient connectServer(String ftpServer, int ftpPort, String ftpUser, String ftpPassword) throws PluginException { FTPClient ftpClient = null; // TLS ftpClient = new FTPSClient("TLS", false); ftpClient.setControlEncoding("UTF-8"); try { ftpClient.connect(ftpServer, ftpPort); if (ftpClient.login(ftpUser, ftpPassword) == false) { throw new PluginException(CargosoftXMLEAkteImportService.class.getName(), "FTP_ERROR", "FTP file transfer failed: login failed!"); } ftpClient.enterLocalPassiveMode(); logger.finest("...... FileType=" + FTP.BINARY_FILE_TYPE); ftpClient.setFileType(FTP.BINARY_FILE_TYPE); ftpClient.setControlEncoding("UTF-8"); } catch (IOException e) { throw new PluginException(CargosoftXMLEAkteImportService.class.getName(), "FTP_ERROR", e.getMessage()); } return ftpClient; } /** * Helper method reads the working FTP directory and starts importing the files. * * @param ftpClient * @param ftpPath * @throws PluginException */ private void scannFTPDirectory(FTPClient ftpClient, String ftpPath, String mandantID, DocumentImportEvent event) throws PluginException { try { // load spaces Pos Expressions Map> spacePosMappings = mahnlaufService.loadSpacePosMappings(); logger.finest("......read directory " + ftpPath); // try to enter the working directory and read all files... boolean bWorkingDir = ftpClient.changeWorkingDirectory(ftpPath); if (bWorkingDir == false) { throw new PluginException(CargosoftXMLEAkteImportService.class.getName(), "FTP_ERROR", "...failed to change into working directory: "); } FTPFile[] allFiles = ftpClient.listFiles(ftpPath); // FTPFile[] allFiles = ftpClient.listFiles(); int count = 0; if (allFiles.length > 0) { documentImportService.logMessage("..." + allFiles.length + " files found ", event); for (FTPFile file : allFiles) { // if this is a directory or symlink then we do ignore this entry if (!file.isFile()) { logger.warning("...'" + file.getName() + "' is not a valid file, object will be ignored!"); continue; } ItemCollection invoice = null; logger.info("import file " + file.getName() + "..."); // String fullFileName = ftpPath + "/" + file.getName(); try (ByteArrayOutputStream is = new ByteArrayOutputStream();) { ftpClient.retrieveFile(file.getName(), is); byte[] rawData = is.toByteArray(); if (rawData != null && rawData.length > 0) { logger.finest("......file '" + file.getName() + "' successful read - bytes size = " + rawData.length); // create new workitem invoice = createWorkitem(event.getSource(), file.getName(), rawData, spacePosMappings); if (invoice != null) { // persist workitem only if not exists and with a matching mandant.id if (!invoice.getItemValueString("mandant.id").equals(mandantID)) { throw new PluginException(CargosoftXMLEAkteImportService.class.getName(), "FTP_ERROR", "Invoice does not match mandantID " + mandantID); } count++; workflowService.processWorkItemByNewTransaction(invoice); // .processWorkItem(invoice); } } // finally delete teh file from the transfer folder ftpClient.deleteFile(ftpPath + file.getName()); // String sourceFilePath = ftpPath + file.getName(); // String destinationFilePath = ftpPath + "processed/" + file.getName(); // boolean success = ftpClient.rename(sourceFilePath, destinationFilePath); // if (!success) { // logger.warning("Failed to move file " + file.getName() + " to " + ftpPath + // "processed/"); // } } catch (AccessDeniedException | ProcessingErrorException | PluginException | ModelException | XPathExpressionException | TransformerException e) { throw new PluginException(CargosoftXMLEAkteImportService.class.getName(), "FTP_ERROR", e.getMessage()); } } documentImportService.logMessage("..." + count + " new files imported.", event); } } catch (IOException e) { logger.severe("FTP I/O Error: " + e.getMessage()); int r = ftpClient.getReplyCode(); logger.severe("FTP ReplyCode=" + r); throw new PluginException(CargosoftXMLEAkteImportService.class.getName(), "FTP_ERROR", e.getMessage()); } finally { // do logout.... try { ftpClient.logout(); ftpClient.disconnect(); } catch (IOException e) { documentImportService.logMessage("...FTP file transfer failed: " + e.getMessage(), event); throw new PluginException(CargosoftXMLEAkteImportService.class.getName(), "FTP_ERROR", e.getMessage()); } } } /** * Creates and processes a new workitem with a given xml information from the * filedata * * * @return * @throws ModelException * @throws PluginException * @throws ProcessingErrorException * @throws AccessDeniedException * @throws TransformerException * @throws XPathExpressionException */ public ItemCollection createWorkitem(ItemCollection source, String fileName, byte[] rawData, Map> spacePosMappings) throws AccessDeniedException, ProcessingErrorException, PluginException, ModelException, XPathExpressionException, TransformerException { ItemCollection workitem = new ItemCollection(); workitem.model(source.getItemValueString(DocumentImportService.SOURCE_ITEM_MODELVERSION)); workitem.task(source.getItemValueInteger(DocumentImportService.SOURCE_ITEM_TASK)); workitem.event(source.getItemValueInteger(DocumentImportService.SOURCE_ITEM_EVENT)); workitem.setWorkflowGroup(source.getItemValueString("workflowgroup")); workitem.setItemValue("cargosoft.import.filename", fileName); // Now we parse the rawData.... InputStream inputStream = new ByteArrayInputStream(rawData); InputSource inputSource = new InputSource(inputStream); DocumentBuilder documentBuilder; try { documentBuilder = DocumentBuilderFactory.newInstance().newDocumentBuilder(); Document doc = documentBuilder.parse(inputSource); // Now we parse the following item values // invoice.number // invoice.currency // invoice.text // invoice.date // dbtr.number // invoice.duedate // invoice.reminder // payment.term // invoice.rate // invoice.base.amount readXMLValue(doc, "/CargoSoftEFile/EFile/References/Reference[@type='client']", workitem, "mandant.id", String.class); readXMLValue(doc, "/CargoSoftEFile/EFile/References/Reference[@type='cs_voucher_number']", workitem, "invoice.number", String.class); readXMLValue(doc, "/CargoSoftEFile/EFile/References/Reference[@type='cs_address_number']", workitem, "cdtr.number", String.class); readXMLValue(doc, "/CargoSoftEFile/EFile/References/Reference[@type='voucherdate']", workitem, "invoice.date", Date.class); readXMLValue(doc, "/CargoSoftEFile/EFile/References/Reference[@type='currency']", workitem, "invoice.currency", String.class); readXMLValue(doc, "/CargoSoftEFile/EFile/References/Reference[@type='currency_rate']", workitem, "invoice.rate", Double.class); readXMLValue(doc, "/CargoSoftEFile/EFile/References/Reference[@type='total_net_amount']", workitem, "invoice.total.net", Double.class); readXMLValue(doc, "/CargoSoftEFile/EFile/References/Reference[@type='total_tax_amount']", workitem, "invoice.total.tax", Double.class); readXMLValue(doc, "/CargoSoftEFile/EFile/References/Reference[@type='reference']", workitem, "invoice.atc.number", String.class); readXMLValue(doc, "/CargoSoftEFile/EFile/References/Reference[@type='booking_period']", workitem, "invoice.booking_period", String.class); readXMLValue(doc, "/CargoSoftEFile/EFile/References/Reference[@type='booking_date']", workitem, "invoice.booking_date", Date.class); readXMLValue(doc, "/CargoSoftEFile/EFile/References/Reference[@type='booking_text']", workitem, "invoice.booking_text", String.class); // verify if invoice is already imported. if (alreadyImported(workitem.getItemValueString("invoice.number"))) { logger.warning("Invoice " + workitem.getItemValueString("invoice.number") + " already imported"); return null; } String cdtrNumber = workitem.getItemValueString("cdtr.number"); if (cdtrNumber.startsWith("K") || cdtrNumber.startsWith("D")) { workitem.setItemValue("cdtr.number", cdtrNumber.substring(1)); } // lookup debitor and set name if (cdtrNumber != null && !cdtrNumber.isEmpty()) try { ItemCollection cdtr = kreditorDebitorService.findCreditor(cdtrNumber); if (cdtr != null) { workitem.setItemValue("cdtr.name", cdtr.getItemValueString("_vendor_name")); // Anhand der Creditoren Stammdaten errrechnen wir die Sprache für diese // Rechnung String country = cdtr.getItemValueString("_VENDOR_COUNTRY"); workitem.setItemValue("invoice.country", country); if (country.equalsIgnoreCase("de") || country.equalsIgnoreCase("ch") || country.equalsIgnoreCase("at")) { workitem.setItemValue("invoice.language", "DE"); } else { workitem.setItemValue("invoice.language", "EN"); } } } catch (PluginException e) { // e.printStackTrace(); } // Row Positions lesen readXMLRows(doc, workitem); // Jetzt noch das Space mapping workitem.setItemValue("invoice.text", workitem.getItemValueString("invoice.positions")); mahnlaufService.mapInvoiceTextToSpace(workitem, spacePosMappings); // Finally we attache the pdf file and the XML content to the workitem. attacheFiles(doc, workitem); } catch (ParserConfigurationException | SAXException | IOException e) { throw new PluginException(CargosoftXMLEAkteImportService.class.getName(), "XML_ERROR", e.getMessage()); } return workitem; } /** * Reads a tag value from the xml tree and set the value into the given * workitem. * * /Invoices/Invoice/InvoiceHeader/Client/Code * * Beispiel Datum: * 2024-05-13T00:00:00+02:00 * * @param doc - xml doc * @param expression - xpath expression * @param workitem * @param itemName * @param itemType */ private void readXMLValue(Document doc, String expression, ItemCollection workitem, String itemName, Class itemType) { // create XPath... XPathFactory xpathFactory = XPathFactory.newInstance(); XPath xpath = xpathFactory.newXPath(); XPathExpression xPathExpression; try { xPathExpression = xpath.compile(expression); // extract node value Node valueNode = (Node) xPathExpression.evaluate(doc, XPathConstants.NODE); if (valueNode != null) { String value = valueNode.getTextContent(); if (itemType == Date.class) { // 2024-05-13T00:00:00+02:00 SimpleDateFormat formatter = new SimpleDateFormat(DATE_FORMAT); formatter.setTimeZone(TimeZone.getTimeZone("CET")); try { workitem.setItemValue(itemName, formatter.parse(value)); } catch (ParseException e) { logger.warning("Invalid Date Format"); } return; } if (itemType == Double.class && value != null && !value.isEmpty()) { try { workitem.setItemValue(itemName, Double.parseDouble(value)); } catch (NumberFormatException e) { // no op } return; } if (itemType == Integer.class && value != null && !value.isEmpty()) { try { workitem.setItemValue(itemName, Integer.parseInt(value)); } catch (NumberFormatException e) { // no op } return; } // Default String format workitem.setItemValue(itemName, value); } } catch (XPathExpressionException e) { logger.warning("Unable to read data field '" + expression + "' : " + e.getMessage()); } } /** * This method attache the pdf file and the XML file to the workitem. * The method removes the file content first form the xml tree and attache the * XML without the pdf data. * * @throws XPathExpressionException * @throws TransformerException */ private void attacheFiles(Document doc, ItemCollection workitem) throws XPathExpressionException, TransformerException { // create XPath... XPathFactory xpathFactory = XPathFactory.newInstance(); XPath xpath = xpathFactory.newXPath(); // read filename and content.... readXMLValue(doc, "/CargoSoftEFile/EFile/Attachments/Attachment/Filename", workitem, "import.filename", String.class); readXMLValue(doc, "/CargoSoftEFile/EFile/Attachments/Attachment/Content", workitem, "import.filedata", String.class); // attache the PDF file content String fileDataString = workitem.getItemValueString("import.filedata"); byte[] decodedPDFData = java.util.Base64.getDecoder().decode(fileDataString); FileData fileData = new FileData(workitem.getItemValueString("import.filename"), decodedPDFData, "application/pdf", null); workitem.addFileData(fileData); // remove the temp items.. workitem.removeItem("import.filename"); workitem.removeItem("import.filedata"); // next remove the Attachments/AttachmentContent from the dom tree XPathExpression attachmentsExpr = xpath .compile("/CargoSoftEFile/EFile/Attachments/Attachment/Content"); NodeList contentNodes = (NodeList) attachmentsExpr.evaluate(doc, XPathConstants.NODESET); // Remove the matching nodes for (int i = 0; i < contentNodes.getLength(); i++) { Node contentNode = contentNodes.item(i); Node parentNode = contentNode.getParentNode(); parentNode.removeChild(contentNode); } // Now attache the XML tree without the file... StringWriter stringWriter = new StringWriter(); TransformerFactory transformerFactory = TransformerFactory.newInstance(); Transformer transformer = transformerFactory.newTransformer(); DOMSource source = new DOMSource(doc); StreamResult result = new StreamResult(stringWriter); transformer.transform(source, result); // byte[] decodedXMLData = // java.util.Base64.getDecoder().decode(stringWriter.toString().); FileData fileDataXML = new FileData(workitem.getItemValueString("cargosoft.import.filename"), stringWriter.toString().getBytes(), "application/xml", null); workitem.addFileData(fileDataXML); } /** * Prüft ob die Belegnummer schon existiert (importiert wurde) * * @param belegNummer * @return */ protected boolean alreadyImported(String belegNummer) { String sQuery = "((type:workitem OR type:workitemarchive) AND $modelversion:steuerbescheid* AND invoice.number:\"" + belegNummer + "\")"; try { // find the textblock... List result = documentService.find(sQuery, 1, 0); if (result.size() > 0) { return true; } } catch (QueryException e) { logger.warning("failed to search invoices by query: " + e.getMessage()); } return false; } /** * This helper method reads the references starting with 'Row_' and creates a * child Workitem for each row. * * The method assumes that the rows are starting with the type 'row_n_' where * 'n' is the row number. * We start with row 1 and read until we found more rows. * * Example: * *

{@code
     *  ... 
        
        3591.340
        0
        IM-GCA-2408-116
     * }
* *

* The last filenumber text will be transferred into the item 'invoice.text' * * @param xml * @return * @throws Exception */ private void readXMLRows(Document doc, ItemCollection workitem) { int row = 1; // Prepare XPath XPathFactory xPathFactory = XPathFactory.newInstance(); XPath xpath = xPathFactory.newXPath(); try { List childItems = new ArrayList<>(); while (true) { // XPath expression to find all Reference nodes starting with "row_" logger.fine("...read row " + row + "..."); XPathExpression expr = xpath .compile("/CargoSoftEFile/EFile/References/Reference[starts-with(@type, 'row_" + row + "_')]"); NodeList nodeList = (NodeList) expr.evaluate(doc, XPathConstants.NODESET); if (nodeList != null && nodeList.getLength() > 0) { ItemCollection childItemCol = new ItemCollection(); for (int i = 0; i < nodeList.getLength(); i++) { Node node = nodeList.item(i); String type = node.getAttributes().getNamedItem("type").getNodeValue(); String value = node.getTextContent(); // Identify which field to set if (type.contains("activity_type")) { childItemCol.setItemValue("activity.type", value); } else if (type.contains("amount")) { try { childItemCol.setItemValue("amount", Double.parseDouble(value)); } catch (NumberFormatException e) { logger.warning("Unable to parse amount: " + e.getMessage()); } } else if (type.contains("tax_code")) { childItemCol.setItemValue("tax.code", value); } else if (type.contains("cs_filenumber")) { childItemCol.setItemValue("filenumber", value); workitem.appendItemValue("invoice.positions", value); } } childItems.add(childItemCol); // continue with next row row++; } else { // no more rows found break; } } // Set child items InvoiceUtil.implodeChildList(workitem, childItems); } catch (XPathExpressionException e) { logger.warning("Unable to read row : " + e.getMessage()); } } }