Sync INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH between API and Web (#11230)

2024-12-02 16:29:25 +09:00 · 2024-12-02 16:29:25 +09:00 · 1d8385f7ac
parent f8c966c39c
commit 1d8385f7ac
29 changed files with 51 additions and 40 deletions
--- a/api/.env.example
+++ b/api/.env.example
@ -383,7 +383,7 @@ LOG_DATEFORMAT=%Y-%m-%d %H:%M:%S
 LOG_TZ=UTC

 # Indexing configuration
-INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH=1000
+INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH=4000

 # Workflow runtime configuration
 WORKFLOW_MAX_EXECUTION_STEPS=500
@ -413,4 +413,4 @@ RESET_PASSWORD_TOKEN_EXPIRY_MINUTES=5

 CREATE_TIDB_SERVICE_JOB_ENABLED=false

-RETRIEVAL_TOP_N=0
+RETRIEVAL_TOP_N=0
--- a/api/configs/feature/init.py
+++ b/api/configs/feature/init.py
@ -647,7 +647,7 @@ class IndexingConfig(BaseSettings):

    INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH: PositiveInt = Field(
        description="Maximum token length for text segmentation during indexing",
-        default=1000,
+        default=4000,
    )


--- a/api/controllers/console/datasets/datasets_document.py
+++ b/api/controllers/console/datasets/datasets_document.py
@ -106,6 +106,7 @@ class GetProcessRuleApi(Resource):
        # get default rules
        mode = DocumentService.DEFAULT_RULES["mode"]
        rules = DocumentService.DEFAULT_RULES["rules"]
+        limits = DocumentService.DEFAULT_RULES["limits"]
        if document_id:
            # get the latest process rule
            document = Document.query.get_or_404(document_id)
@ -132,7 +133,7 @@ class GetProcessRuleApi(Resource):
                mode = dataset_process_rule.mode
                rules = dataset_process_rule.rules_dict

-        return {"mode": mode, "rules": rules}
+        return {"mode": mode, "rules": rules, "limits": limits}


 class DatasetDocumentListApi(Resource):
--- a/api/services/dataset_service.py
+++ b/api/services/dataset_service.py
@ -406,6 +406,9 @@ class DocumentService:
            ],
            "segmentation": {"delimiter": "\n", "max_tokens": 500, "chunk_overlap": 50},
        },
+        "limits": {
+            "indexing_max_segmentation_tokens_length": dify_config.INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH,
+        },
    }

    DOCUMENT_METADATA_SCHEMA = {
--- a/docker-legacy/docker-compose.yaml
+++ b/docker-legacy/docker-compose.yaml
@ -210,7 +210,7 @@ services:
      SSRF_PROXY_HTTP_URL: 'http://ssrf_proxy:3128'
      SSRF_PROXY_HTTPS_URL: 'http://ssrf_proxy:3128'
      # Indexing configuration
-      INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH: 1000
+      INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH: 4000
    depends_on:
      - db
      - redis
--- a/docker/.env.example
+++ b/docker/.env.example
@ -683,7 +683,7 @@ SMTP_OPPORTUNISTIC_TLS=false
 # ------------------------------

 # Maximum length of segmentation tokens for indexing
-INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH=1000
+INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH=4000

 # Member invitation link valid time (hours),
 # Default: 72.
--- a/docker/docker-compose.yaml
+++ b/docker/docker-compose.yaml
@ -247,7 +247,7 @@ x-shared-env: &shared-api-worker-env
  SMTP_OPPORTUNISTIC_TLS: ${SMTP_OPPORTUNISTIC_TLS:-false}
  RESEND_API_KEY: ${RESEND_API_KEY:-your-resend-api-key}
  RESEND_API_URL: ${RESEND_API_URL:-https://api.resend.com}
-  INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH: ${INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH:-1000}
+  INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH: ${INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH:-4000}
  INVITE_EXPIRY_HOURS: ${INVITE_EXPIRY_HOURS:-72}
  RESET_PASSWORD_TOKEN_EXPIRY_MINUTES: ${RESET_PASSWORD_TOKEN_EXPIRY_MINUTES:-5}
  CODE_EXECUTION_ENDPOINT: ${CODE_EXECUTION_ENDPOINT:-http://sandbox:8194}
--- a/web/app/components/datasets/create/step-two/index.tsx
+++ b/web/app/components/datasets/create/step-two/index.tsx
@ -122,7 +122,8 @@ const StepTwo = ({
  const setSegmentIdentifier = useCallback((value: string) => {
    doSetSegmentIdentifier(value ? escape(value) : DEFAULT_SEGMENT_IDENTIFIER)
  }, [])
-  const [max, setMax] = useState(4000) // default chunk length
+  const [maxChunkLength, setMaxChunkLength] = useState(4000) // default chunk length
+  const [limitMaxChunkLength, setLimitMaxChunkLength] = useState(4000)
  const [overlap, setOverlap] = useState(50)
  const [rules, setRules] = useState<PreProcessingRule[]>([])
  const [defaultConfig, setDefaultConfig] = useState<Rules>()
@ -196,7 +197,7 @@ const StepTwo = ({
  const resetRules = () => {
    if (defaultConfig) {
      setSegmentIdentifier(defaultConfig.segmentation.separator)
-      setMax(defaultConfig.segmentation.max_tokens)
+      setMaxChunkLength(defaultConfig.segmentation.max_tokens)
      setOverlap(defaultConfig.segmentation.chunk_overlap)
      setRules(defaultConfig.pre_processing_rules)
    }
@ -212,8 +213,8 @@ const StepTwo = ({
  }

  const confirmChangeCustomConfig = () => {
-    if (segmentationType === SegmentType.CUSTOM && max > 4000) {
-      Toast.notify({ type: 'error', message: t('datasetCreation.stepTwo.maxLengthCheck') })
+    if (segmentationType === SegmentType.CUSTOM && maxChunkLength > limitMaxChunkLength) {
+      Toast.notify({ type: 'error', message: t('datasetCreation.stepTwo.maxLengthCheck', { limit: limitMaxChunkLength }) })
      return
    }
    setCustomFileIndexingEstimate(null)
@ -234,7 +235,7 @@ const StepTwo = ({
        pre_processing_rules: rules,
        segmentation: {
          separator: unescape(segmentIdentifier),
-          max_tokens: max,
+          max_tokens: maxChunkLength,
          chunk_overlap: overlap,
        },
      }
@ -339,12 +340,12 @@ const StepTwo = ({
  )
  const getCreationParams = () => {
    let params
-    if (segmentationType === SegmentType.CUSTOM && overlap > max) {
+    if (segmentationType === SegmentType.CUSTOM && overlap > maxChunkLength) {
      Toast.notify({ type: 'error', message: t('datasetCreation.stepTwo.overlapCheck') })
      return
    }
-    if (segmentationType === SegmentType.CUSTOM && max > 4000) {
-      Toast.notify({ type: 'error', message: t('datasetCreation.stepTwo.maxLengthCheck') })
+    if (segmentationType === SegmentType.CUSTOM && maxChunkLength > limitMaxChunkLength) {
+      Toast.notify({ type: 'error', message: t('datasetCreation.stepTwo.maxLengthCheck', { limit: limitMaxChunkLength }) })
      return
    }
    if (isSetting) {
@ -415,7 +416,8 @@ const StepTwo = ({
      const res = await fetchDefaultProcessRule({ url: '/datasets/process-rule' })
      const separator = res.rules.segmentation.separator
      setSegmentIdentifier(separator)
-      setMax(res.rules.segmentation.max_tokens)
+      setMaxChunkLength(res.rules.segmentation.max_tokens)
+      setLimitMaxChunkLength(res.limits.indexing_max_segmentation_tokens_length)
      setOverlap(res.rules.segmentation.chunk_overlap)
      setRules(res.rules.pre_processing_rules)
      setDefaultConfig(res.rules)
@ -432,7 +434,7 @@ const StepTwo = ({
      const max = rules.segmentation.max_tokens
      const overlap = rules.segmentation.chunk_overlap
      setSegmentIdentifier(separator)
-      setMax(max)
+      setMaxChunkLength(max)
      setOverlap(overlap)
      setRules(rules.pre_processing_rules)
      setDefaultConfig(rules)
@ -670,10 +672,10 @@ const StepTwo = ({
                        type="number"
                        className='h-9'
                        placeholder={t('datasetCreation.stepTwo.maxLength') || ''}
-                        value={max}
-                        max={4000}
+                        value={maxChunkLength}
+                        max={limitMaxChunkLength}
                        min={1}
-                        onChange={e => setMax(parseInt(e.target.value.replace(/^0+/, ''), 10))}
+                        onChange={e => setMaxChunkLength(parseInt(e.target.value.replace(/^0+/, ''), 10))}
                      />
                    </div>
                  </div>
--- a/web/i18n/de-DE/dataset-creation.ts
+++ b/web/i18n/de-DE/dataset-creation.ts
@ -142,7 +142,7 @@ const translation = {
    websiteSource: 'Preprocess-Website',
    webpageUnit: 'Seiten',
    separatorTip: 'Ein Trennzeichen ist das Zeichen, das zum Trennen von Text verwendet wird. \\n\\n und \\n sind häufig verwendete Trennzeichen zum Trennen von Absätzen und Zeilen. In Kombination mit Kommas (\\n\\n,\\n) werden Absätze nach Zeilen segmentiert, wenn die maximale Blocklänge überschritten wird. Sie können auch spezielle, von Ihnen selbst definierte Trennzeichen verwenden (z. B. ***).',
-    maxLengthCheck: 'Die maximale Stücklänge sollte weniger als 4000 betragen',
+    maxLengthCheck: 'Die maximale Stücklänge sollte weniger als {{limit}} betragen',
  },
  stepThree: {
    creationTitle: '🎉 Wissen erstellt',
--- a/web/i18n/en-US/dataset-creation.ts
+++ b/web/i18n/en-US/dataset-creation.ts
@ -103,7 +103,7 @@ const translation = {
    separatorTip: 'A delimiter is the character used to separate text. \\n\\n and \\n are commonly used delimiters for separating paragraphs and lines. Combined with commas (\\n\\n,\\n), paragraphs will be segmented by lines when exceeding the maximum chunk length. You can also use special delimiters defined by yourself (e.g. ***).',
    separatorPlaceholder: '\\n\\n for separating paragraphs; \\n for separating lines',
    maxLength: 'Maximum chunk length',
-    maxLengthCheck: 'Maximum chunk length should be less than 4000',
+    maxLengthCheck: 'Maximum chunk length should be less than {{limit}}',
    overlap: 'Chunk overlap',
    overlapTip: 'Setting the chunk overlap can maintain the semantic relevance between them, enhancing the retrieve effect. It is recommended to set 10%-25% of the maximum chunk size.',
    overlapCheck: 'chunk overlap should not bigger than maximum chunk length',
--- a/web/i18n/es-ES/dataset-creation.ts
+++ b/web/i18n/es-ES/dataset-creation.ts
@ -147,7 +147,7 @@ const translation = {
    retrievalSettingTip: 'Para cambiar el método de índice, por favor ve a la ',
    datasetSettingLink: 'configuración del conocimiento.',
    separatorTip: 'Un delimitador es el carácter que se utiliza para separar el texto. \\n\\n y \\n son delimitadores comúnmente utilizados para separar párrafos y líneas. Combinado con comas (\\n\\n,\\n), los párrafos se segmentarán por líneas cuando excedan la longitud máxima del fragmento. También puede utilizar delimitadores especiales definidos por usted mismo (por ejemplo, ***).',
-    maxLengthCheck: 'La longitud máxima del fragmento debe ser inferior a 4000',
+    maxLengthCheck: 'La longitud máxima del fragmento debe ser inferior a {{limit}}',
  },
  stepThree: {
    creationTitle: '🎉 Conocimiento creado',
--- a/web/i18n/fa-IR/dataset-creation.ts
+++ b/web/i18n/fa-IR/dataset-creation.ts
@ -147,7 +147,7 @@ const translation = {
    retrievalSettingTip: 'برای تغییر روش شاخص، لطفاً به',
    datasetSettingLink: 'تنظیمات دانش بروید.',
    separatorTip: 'جداکننده نویسه ای است که برای جداسازی متن استفاده می شود. \\n\\n و \\n معمولا برای جداسازی پاراگراف ها و خطوط استفاده می شوند. همراه با کاما (\\n\\n,\\n)، پاراگراف ها زمانی که از حداکثر طول تکه فراتر می روند، با خطوط تقسیم بندی می شوند. همچنین می توانید از جداکننده های خاصی که توسط خودتان تعریف شده اند استفاده کنید (مثلا ***).',
-    maxLengthCheck: 'حداکثر طول تکه باید کمتر از 4000 باشد',
+    maxLengthCheck: 'حداکثر طول تکه باید کمتر از {{limit}} باشد',
  },
  stepThree: {
    creationTitle: ' دانش ایجاد شد',
--- a/web/i18n/fr-FR/dataset-creation.ts
+++ b/web/i18n/fr-FR/dataset-creation.ts
@ -142,7 +142,7 @@ const translation = {
    webpageUnit: 'Pages',
    websiteSource: 'Site web de prétraitement',
    separatorTip: 'Un délimiteur est le caractère utilisé pour séparer le texte. \\n\\n et \\n sont des délimiteurs couramment utilisés pour séparer les paragraphes et les lignes. Combiné à des virgules (\\n\\n,\\n), les paragraphes seront segmentés par des lignes lorsqu’ils dépasseront la longueur maximale des morceaux. Vous pouvez également utiliser des délimiteurs spéciaux définis par vous-même (par exemple ***).',
-    maxLengthCheck: 'La longueur maximale des morceaux doit être inférieure à 4000',
+    maxLengthCheck: 'La longueur maximale des morceaux doit être inférieure à {{limit}}',
  },
  stepThree: {
    creationTitle: '🎉 Connaissance créée',
--- a/web/i18n/hi-IN/dataset-creation.ts
+++ b/web/i18n/hi-IN/dataset-creation.ts
@ -164,7 +164,7 @@ const translation = {
    retrievalSettingTip: 'इंडेक्स विधि बदलने के लिए, कृपया जाएं ',
    datasetSettingLink: 'ज्ञान सेटिंग्स।',
    separatorTip: 'एक सीमांकक पाठ को अलग करने के लिए उपयोग किया जाने वाला वर्ण है। \\n\\n और \\n आमतौर पर पैराग्राफ और लाइनों को अलग करने के लिए उपयोग किए जाने वाले सीमांकक हैं। अल्पविराम (\\n\\n,\\n) के साथ संयुक्त, अधिकतम खंड लंबाई से अधिक होने पर अनुच्छेदों को पंक्तियों द्वारा खंडित किया जाएगा। आप स्वयं द्वारा परिभाषित विशेष सीमांकक का भी उपयोग कर सकते हैं (उदा. ***).',
-    maxLengthCheck: 'अधिकतम चंक लंबाई 4000 से कम होनी चाहिए',
+    maxLengthCheck: 'अधिकतम चंक लंबाई {{limit}} से कम होनी चाहिए',
  },
  stepThree: {
    creationTitle: '🎉 ज्ञान बनाया गया',
--- a/web/i18n/it-IT/dataset-creation.ts
+++ b/web/i18n/it-IT/dataset-creation.ts
@ -167,7 +167,7 @@ const translation = {
    retrievalSettingTip: 'Per cambiare il metodo di indicizzazione, vai alle ',
    datasetSettingLink: 'impostazioni della Conoscenza.',
    separatorTip: 'Un delimitatore è il carattere utilizzato per separare il testo. \\n\\n e \\n sono delimitatori comunemente usati per separare paragrafi e righe. In combinazione con le virgole (\\n\\n,\\n), i paragrafi verranno segmentati per righe quando superano la lunghezza massima del blocco. È inoltre possibile utilizzare delimitatori speciali definiti dall\'utente (ad es. ***).',
-    maxLengthCheck: 'La lunghezza massima del blocco deve essere inferiore a 4000',
+    maxLengthCheck: 'La lunghezza massima del blocco deve essere inferiore a {{limit}}',
  },
  stepThree: {
    creationTitle: '🎉 Conoscenza creata',
--- a/web/i18n/ja-JP/dataset-creation.ts
+++ b/web/i18n/ja-JP/dataset-creation.ts
@ -147,7 +147,7 @@ const translation = {
    retrievalSettingTip: '検索方法を変更するには、',
    datasetSettingLink: 'ナレッジ設定',
    separatorTip: '区切り文字は、テキストを区切るために使用される文字です。\\n\\n と \\n は、段落と行を区切るために一般的に使用される区切り記号です。カンマ (\\n\\n,\\n) と組み合わせると、最大チャンク長を超えると、段落は行で区切られます。自分で定義した特別な区切り文字を使用することもできます(例:***)。',
-    maxLengthCheck: 'チャンクの最大長は 4000 未満にする必要があります',
+    maxLengthCheck: 'チャンクの最大長は {{limit}} 未満にする必要があります',
  },
  stepThree: {
    creationTitle: '🎉 ナレッジが作成されました',
--- a/web/i18n/ko-KR/dataset-creation.ts
+++ b/web/i18n/ko-KR/dataset-creation.ts
@ -142,7 +142,7 @@ const translation = {
    webpageUnit: '페이지',
    websiteSource: '웹 사이트 전처리',
    separatorTip: '구분 기호는 텍스트를 구분하는 데 사용되는 문자입니다. \\n\\n 및 \\n은 단락과 줄을 구분하는 데 일반적으로 사용되는 구분 기호입니다. 쉼표(\\n\\n,\\n)와 함께 사용하면 최대 청크 길이를 초과할 경우 단락이 줄로 분할됩니다. 직접 정의한 특수 구분 기호(예: ***)를 사용할 수도 있습니다.',
-    maxLengthCheck: '최대 청크 길이는 4000 미만이어야 합니다.',
+    maxLengthCheck: '최대 청크 길이는 {{limit}} 미만이어야 합니다.',
  },
  stepThree: {
    creationTitle: '🎉 지식이 생성되었습니다',
--- a/web/i18n/pl-PL/dataset-creation.ts
+++ b/web/i18n/pl-PL/dataset-creation.ts
@ -155,7 +155,7 @@ const translation = {
    webpageUnit: 'Stron',
    websiteSource: 'Witryna internetowa przetwarzania wstępnego',
    separatorTip: 'Ogranicznik to znak używany do oddzielania tekstu. \\n\\n i \\n są powszechnie używanymi ogranicznikami do oddzielania akapitów i wierszy. W połączeniu z przecinkami (\\n\\n,\\n), akapity będą segmentowane wierszami po przekroczeniu maksymalnej długości fragmentu. Możesz również skorzystać ze zdefiniowanych przez siebie specjalnych ograniczników (np. ***).',
-    maxLengthCheck: 'Maksymalna długość porcji powinna być mniejsza niż 4000',
+    maxLengthCheck: 'Maksymalna długość porcji powinna być mniejsza niż {{limit}}',
  },
  stepThree: {
    creationTitle: '🎉 Utworzono Wiedzę',
--- a/web/i18n/pt-BR/dataset-creation.ts
+++ b/web/i18n/pt-BR/dataset-creation.ts
@ -142,7 +142,7 @@ const translation = {
    websiteSource: 'Site de pré-processamento',
    webpageUnit: 'Páginas',
    separatorTip: 'Um delimitador é o caractere usado para separar o texto. \\n\\n e \\n são delimitadores comumente usados para separar parágrafos e linhas. Combinado com vírgulas (\\n\\n,\\n), os parágrafos serão segmentados por linhas ao exceder o comprimento máximo do bloco. Você também pode usar delimitadores especiais definidos por você (por exemplo, ***).',
-    maxLengthCheck: 'O comprimento máximo do chunk deve ser inferior a 4000',
+    maxLengthCheck: 'O comprimento máximo do chunk deve ser inferior a {{limit}}',
  },
  stepThree: {
    creationTitle: '🎉 Conhecimento criado',
--- a/web/i18n/ro-RO/dataset-creation.ts
+++ b/web/i18n/ro-RO/dataset-creation.ts
@ -142,7 +142,7 @@ const translation = {
    webpageUnit: 'Pagini',
    websiteSource: 'Site-ul web de preprocesare',
    separatorTip: 'Un delimitator este caracterul folosit pentru a separa textul. \\n\\n și \\n sunt delimitatori utilizați în mod obișnuit pentru separarea paragrafelor și liniilor. Combinate cu virgule (\\n\\n,\\n), paragrafele vor fi segmentate pe linii atunci când depășesc lungimea maximă a bucății. De asemenea, puteți utiliza delimitatori speciali definiți de dumneavoastră (de exemplu, ***).',
-    maxLengthCheck: 'Lungimea maximă a bucății trebuie să fie mai mică de 4000',
+    maxLengthCheck: 'Lungimea maximă a bucății trebuie să fie mai mică de {{limit}}',
  },
  stepThree: {
    creationTitle: '🎉 Cunoștință creată',
--- a/web/i18n/ru-RU/dataset-creation.ts
+++ b/web/i18n/ru-RU/dataset-creation.ts
@ -147,7 +147,7 @@ const translation = {
    retrievalSettingTip: 'Чтобы изменить метод индексации, пожалуйста, перейдите в ',
    datasetSettingLink: 'настройки базы знаний.',
    separatorTip: 'Разделитель — это символ, используемый для разделения текста. \\n\\n и \\n — это часто используемые разделители для разделения абзацев и строк. В сочетании с запятыми (\\n\\n,\\n) абзацы будут сегментированы по строкам, если максимальная длина блока превышает их. Вы также можете использовать специальные разделители, определенные вами (например, ***).',
-    maxLengthCheck: 'Максимальная длина блока должна быть меньше 4000',
+    maxLengthCheck: 'Максимальная длина блока должна быть меньше {{limit}}',
  },
  stepThree: {
    creationTitle: '🎉 База знаний создана',
--- a/web/i18n/sl-SI/dataset-creation.ts
+++ b/web/i18n/sl-SI/dataset-creation.ts
@ -152,7 +152,7 @@ const translation = {
    indexSettingTip: 'Če želite spremeniti način indeksiranja in model vdelave, pojdite na ',
    retrievalSettingTip: 'Če želite spremeniti nastavitve iskanja, pojdite na ',
    datasetSettingLink: 'nastavitve Znanja.',
-    maxLengthCheck: 'Največja dolžina kosa mora biti manjša od 4000',
+    maxLengthCheck: 'Največja dolžina kosa mora biti manjša od {{limit}}',
  },
  stepThree: {
    creationTitle: '🎉 Znanje ustvarjeno',
--- a/web/i18n/th-TH/dataset-creation.ts
+++ b/web/i18n/th-TH/dataset-creation.ts
@ -103,7 +103,7 @@ const translation = {
    separatorTip: 'ตัวคั่นคืออักขระที่ใช้ในการแยกข้อความ \\n\\n และ \\n เป็นตัวคั่นที่ใช้กันทั่วไปสําหรับการแยกย่อหน้าและบรรทัด เมื่อรวมกับเครื่องหมายจุลภาค (\\n\\n,\\n) ย่อหน้าจะถูกแบ่งตามบรรทัดเมื่อเกินความยาวของก้อนสูงสุด คุณยังสามารถใช้ตัวคั่นพิเศษที่กําหนดโดยตัวคุณเอง (เช่น ***)',
    separatorPlaceholder: '\\n\\n สําหรับแยกย่อหน้า \\n สําหรับแยกเส้น',
    maxLength: 'ความยาวก้อนสูงสุด',
-    maxLengthCheck: 'ความยาวก้อนสูงสุดควรน้อยกว่า 4000',
+    maxLengthCheck: 'ความยาวก้อนสูงสุดควรน้อยกว่า {{limit}}',
    overlap: 'การทับซ้อนกันของก้อน',
    overlapTip: 'การตั้งค่าการทับซ้อนกันของกลุ่มสามารถรักษาความเกี่ยวข้องทางความหมายระหว่างกันได้ ขอแนะนําให้ตั้งค่า 10%-25% ของขนาดก้อนสูงสุด',
    overlapCheck: 'การทับซ้อนกันของก้อนไม่ควรใหญ่กว่าความยาวของก้อนสูงสุด',
--- a/web/i18n/tr-TR/dataset-creation.ts
+++ b/web/i18n/tr-TR/dataset-creation.ts
@ -147,7 +147,7 @@ const translation = {
    retrievalSettingTip: 'Dizin yöntemini değiştirmek için, lütfen',
    datasetSettingLink: 'Bilgi ayarlarına gidin.',
    separatorTip: 'Sınırlayıcı, metni ayırmak için kullanılan karakterdir. \\n\\n ve \\n, paragrafları ve satırları ayırmak için yaygın olarak kullanılan sınırlayıcılardır. Virgüllerle (\\n\\n,\\n) birleştirildiğinde, paragraflar maksimum öbek uzunluğunu aştığında satırlarla bölünür. Kendiniz tarafından tanımlanan özel sınırlayıcıları da kullanabilirsiniz (örn.',
-    maxLengthCheck: 'Maksimum yığın uzunluğu 4000\'den az olmalıdır',
+    maxLengthCheck: 'Maksimum yığın uzunluğu {{limit}}\'den az olmalıdır',
  },
  stepThree: {
    creationTitle: '🎉 Bilgi oluşturuldu',
--- a/web/i18n/uk-UA/dataset-creation.ts
+++ b/web/i18n/uk-UA/dataset-creation.ts
@ -142,7 +142,7 @@ const translation = {
    webpageUnit: 'Сторінок',
    websiteSource: 'Веб-сайт попередньої обробки',
    separatorTip: 'Роздільник – це символ, який використовується для поділу тексту. \\n\\n та \\n є часто використовуваними роздільниками для відокремлення абзаців та рядків. У поєднанні з комами (\\n\\n,\\n) абзаци будуть розділені лініями, якщо вони перевищують максимальну довжину фрагмента. Ви також можете використовувати спеціальні роздільники, визначені вами (наприклад, ***).',
-    maxLengthCheck: 'Максимальна довжина шматка має бути меншою за 4000',
+    maxLengthCheck: 'Максимальна довжина шматка має бути меншою за {{limit}}',
  },
  stepThree: {
    creationTitle: '🎉 Знання створено',
--- a/web/i18n/vi-VN/dataset-creation.ts
+++ b/web/i18n/vi-VN/dataset-creation.ts
@ -142,7 +142,7 @@ const translation = {
    websiteSource: 'Trang web tiền xử lý',
    webpageUnit: 'Trang',
    separatorTip: 'Dấu phân cách là ký tự được sử dụng để phân tách văn bản. \\n\\n và \\n là dấu phân cách thường được sử dụng để tách các đoạn văn và dòng. Kết hợp với dấu phẩy (\\n\\n,\\n), các đoạn văn sẽ được phân đoạn theo các dòng khi vượt quá độ dài đoạn tối đa. Bạn cũng có thể sử dụng dấu phân cách đặc biệt do chính bạn xác định (ví dụ: ***).',
-    maxLengthCheck: 'Chiều dài đoạn tối đa phải nhỏ hơn 4000',
+    maxLengthCheck: 'Chiều dài đoạn tối đa phải nhỏ hơn {{limit}}',
  },
  stepThree: {
    creationTitle: '🎉 Kiến thức đã được tạo',
--- a/web/i18n/zh-Hans/dataset-creation.ts
+++ b/web/i18n/zh-Hans/dataset-creation.ts
@ -103,7 +103,7 @@ const translation = {
    separatorTip: '分隔符是用于分隔文本的字符。\\n\\n 和 \\n 是常用于分隔段落和行的分隔符。用逗号连接分隔符（\\n\\n,\\n），当段落超过最大块长度时，会按行进行分割。你也可以使用自定义的特殊分隔符（例如 ***）。',
    separatorPlaceholder: '\\n\\n 用于分段；\\n 用于分行',
    maxLength: '分段最大长度',
-    maxLengthCheck: '分段最大长度不能大于 4000',
+    maxLengthCheck: '分段最大长度不能大于 {{limit}}',
    overlap: '分段重叠长度',
    overlapTip: '设置分段之间的重叠长度可以保留分段之间的语义关系，提升召回效果。建议设置为最大分段长度的10%-25%',
    overlapCheck: '分段重叠长度不能大于分段最大长度',
--- a/web/i18n/zh-Hant/dataset-creation.ts
+++ b/web/i18n/zh-Hant/dataset-creation.ts
@ -142,7 +142,7 @@ const translation = {
    websiteSource: '預處理網站',
    webpageUnit: '頁面',
    separatorTip: '分隔符是用於分隔文字的字元。\\n\\n 和 \\n 是分隔段落和行的常用分隔符。與逗號 （\\n\\n，\\n） 組合使用時，當超過最大區塊長度時，段落將按行分段。您也可以使用自定義的特殊分隔符（例如 ***）。',
-    maxLengthCheck: '塊最大長度應小於 4000',
+    maxLengthCheck: '塊最大長度應小於 {{limit}}',
  },
  stepThree: {
    creationTitle: '🎉 知識庫已建立',
--- a/web/models/datasets.ts
+++ b/web/models/datasets.ts
@ -175,6 +175,7 @@ export type ProcessMode = 'automatic' | 'custom'
 export type ProcessRuleResponse = {
  mode: ProcessMode
  rules: Rules
+  limits: Limits
 }

 export type Rules = {
@ -182,6 +183,10 @@ export type Rules = {
  segmentation: Segmentation
 }

+export type Limits = {
+  indexing_max_segmentation_tokens_length: number
+}
+
 export type PreProcessingRule = {
  id: string
  enabled: boolean