|
@@ -416,9 +416,14 @@ class IndexingRunner:
|
|
|
if separator:
|
|
|
separator = separator.replace('\\n', '\n')
|
|
|
|
|
|
+ if 'chunk_overlap' in segmentation and segmentation['chunk_overlap']:
|
|
|
+ chunk_overlap = segmentation['chunk_overlap']
|
|
|
+ else:
|
|
|
+ chunk_overlap = 0
|
|
|
+
|
|
|
character_splitter = FixedRecursiveCharacterTextSplitter.from_encoder(
|
|
|
chunk_size=segmentation["max_tokens"],
|
|
|
- chunk_overlap=segmentation.get('chunk_overlap', 0),
|
|
|
+ chunk_overlap=chunk_overlap,
|
|
|
fixed_separator=separator,
|
|
|
separators=["\n\n", "。", ".", " ", ""],
|
|
|
embedding_model_instance=embedding_model_instance
|