101 lines
3.3 KiB
Java
101 lines
3.3 KiB
Java
/*
|
|
* Imixs-Workflow
|
|
*
|
|
* Copyright (C) 2001-2020 Imixs Software Solutions GmbH,
|
|
* http://www.imixs.com
|
|
*
|
|
* This program is free software; you can redistribute it and/or
|
|
* modify it under the terms of the GNU General Public License
|
|
* as published by the Free Software Foundation; either version 2
|
|
* of the License, or (at your option) any later version.
|
|
*
|
|
* This program is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
* General Public License for more details.
|
|
*
|
|
* You can receive a copy of the GNU General Public
|
|
* License at http://www.gnu.org/licenses/gpl.html
|
|
*
|
|
* Project:
|
|
* https://www.imixs.org
|
|
* https://github.com/imixs/imixs-workflow
|
|
*
|
|
* Contributors:
|
|
* Imixs Software Solutions GmbH - Project Management
|
|
* Ralph Soika - Software Developer
|
|
*/
|
|
|
|
package ai.dynamixs.rag;
|
|
|
|
import java.util.ArrayList;
|
|
import java.util.List;
|
|
import java.util.logging.Logger;
|
|
import java.util.regex.Matcher;
|
|
import java.util.regex.Pattern;
|
|
|
|
/**
|
|
* The RAGUtil provides methods to chunk markup text
|
|
*
|
|
* @author Ralph Soika
|
|
* @version 1.0
|
|
*
|
|
*/
|
|
public class RAGUtil {
|
|
|
|
private static Logger logger = Logger.getLogger(RAGAdapter.class.getName());
|
|
|
|
/**
|
|
* This helper method creates smaller text chunks from a large markup text.
|
|
*/
|
|
public static List<String> chunkMarkupDocument(String markupText) {
|
|
List<String> chunks = new ArrayList<>();
|
|
StringBuilder currentChunk = new StringBuilder();
|
|
|
|
// Regex für Abschnitts-Header (# oder **)
|
|
Pattern sectionPattern = Pattern.compile("^(# .+|\\*\\*.+\\*\\*)", Pattern.MULTILINE);
|
|
Matcher matcher = sectionPattern.matcher(markupText);
|
|
|
|
int lastEnd = 0;
|
|
while (matcher.find()) {
|
|
// Text zwischen den Headern
|
|
String sectionContent = markupText.substring(lastEnd, matcher.start()).trim();
|
|
|
|
if (!sectionContent.isEmpty()) {
|
|
// Füge Inhalt zum aktuellen Chunk hinzu
|
|
if (currentChunk.length() + sectionContent.length() > 500) {
|
|
flushChunk(chunks, currentChunk);
|
|
}
|
|
currentChunk.append(sectionContent).append("\n");
|
|
}
|
|
|
|
// Header selbst behandeln
|
|
String header = matcher.group(1);
|
|
if (currentChunk.length() + header.length() > 300) {
|
|
flushChunk(chunks, currentChunk);
|
|
}
|
|
currentChunk.append(header).append("\n");
|
|
|
|
lastEnd = matcher.end();
|
|
}
|
|
|
|
// Restlichen Text nach dem letzten Header
|
|
String remainingContent = markupText.substring(lastEnd).trim();
|
|
if (!remainingContent.isEmpty()) {
|
|
currentChunk.append(remainingContent);
|
|
flushChunk(chunks, currentChunk);
|
|
} else if (currentChunk.length() > 0) {
|
|
chunks.add(currentChunk.toString().trim());
|
|
}
|
|
|
|
return chunks;
|
|
}
|
|
|
|
private static void flushChunk(List<String> chunks, StringBuilder currentChunk) {
|
|
if (currentChunk.length() > 0) {
|
|
chunks.add(currentChunk.toString().trim());
|
|
currentChunk.setLength(0);
|
|
}
|
|
}
|
|
|
|
}
|