Przeglądaj źródła

chunk-overlap None check (#2781)

Co-authored-by: jyong <jyong@dify.ai>
Jyong 1 rok temu
rodzic
commit
6454e1d644
1 zmienionych plików z 6 dodań i 1 usunięć
  1. 6 1
      api/core/indexing_runner.py

+ 6 - 1
api/core/indexing_runner.py

@@ -416,9 +416,14 @@ class IndexingRunner:
             if separator:
                 separator = separator.replace('\\n', '\n')
 
+            if 'chunk_overlap' in segmentation and segmentation['chunk_overlap']:
+                chunk_overlap = segmentation['chunk_overlap']
+            else:
+                chunk_overlap = 0
+
             character_splitter = FixedRecursiveCharacterTextSplitter.from_encoder(
                 chunk_size=segmentation["max_tokens"],
-                chunk_overlap=segmentation.get('chunk_overlap', 0),
+                chunk_overlap=chunk_overlap,
                 fixed_separator=separator,
                 separators=["\n\n", "。", ".", " ", ""],
                 embedding_model_instance=embedding_model_instance