feat: support function tokenSeparators for custom tokenization

docs/demo/custom-tokenize.md

@@ -0,0 +1,8 @@
+---
+title: custom-tokenize
+nav:
+  title: Demo
+  path: /demo
+---
+
+<code src="../examples/custom-tokenize.tsx"></code>

docs/examples/custom-tokenize.tsx

@@ -0,0 +1,28 @@
+import React from 'react';
+import Select from '@rc-component/select';
+import '../../assets/index.less';
+
+const tokenize = (input: string): string[] => {
+  const tokens: string[] = [];
+  const regex = /"([^"]*)"|([^,\n]+)/g;
+  let m: RegExpExecArray | null = regex.exec(input);
+  while (m !== null) {
+    tokens.push((m[1] ?? m[2]).trim());
+    m = regex.exec(input);
+  }
+  return tokens.filter(Boolean);
+};
+
+const Demo: React.FC = () => (
+  <>
+    <h2>自定义分词（引号感知）</h2>
+    <Select
+      mode="tags"
+      style={{ width: '100%' }}
+      tokenize={tokenize}
+      placeholder='Try paste: "San Francisco, CA", New York'
+    />
+  </>
+);
+
+export default Demo;

src/BaseSelect/index.tsx

@@ -183,6 +183,12 @@ export interface BaseSelectProps
 
   // >>> Search
   tokenSeparators?: string[];
+  /**
+   * Custom tokenization. When provided, takes precedence over `tokenSeparators`.
+   * Receives the current input text and returns an array of tags. Return `[input]`
+   * (or any single-element array equal to input) to indicate "no split, keep typing".
+   */
+  tokenize?: (input: string) => string[];
 
   // >>> Icons
   allowClear?: boolean | { clearIcon?: React.ReactNode };
@@ -282,6 +288,7 @@ const BaseSelect = React.forwardRef<BaseSelectRef, BaseSelectProps>((props, ref)
     onSearch,
     onSearchSplit,
     tokenSeparators,
+    tokenize,
 
     // Icons
     allowClear,
@@ -371,8 +378,10 @@ const BaseSelect = React.forwardRef<BaseSelectRef, BaseSelectProps>((props, ref)
 
   // ============================= Search =============================
   const tokenWithEnter = React.useMemo<boolean>(
-    () => (tokenSeparators || []).some((tokenSeparator) => ['\n', '\r\n'].includes(tokenSeparator)),
-    [tokenSeparators],
+    () =>
+      !!tokenize ||
+      (tokenSeparators || []).some((tokenSeparator) => ['\n', '\r\n'].includes(tokenSeparator)),
+    [tokenize, tokenSeparators],
   );
 
   const onInternalSearch = (searchText: string, fromTyping: boolean, isCompositing: boolean) => {
@@ -383,13 +392,22 @@ const BaseSelect = React.forwardRef<BaseSelectRef, BaseSelectProps>((props, ref)
     let newSearchText = searchText;
     onActiveValueChange?.(null);
 
-    const separatedList = getSeparatedContent(
-      searchText,
-      tokenSeparators,
-      isValidCount(maxCount) ? maxCount - displayValues.length : undefined,
-    );
+    const cap = isValidCount(maxCount) ? maxCount - displayValues.length : undefined;
+
+    let separatedList: string[] | null;
+    if (tokenize) {
+      const tokens = tokenize(searchText);
+      const isUnchanged = Array.isArray(tokens) && tokens.length === 1 && tokens[0] === searchText;
+      if (Array.isArray(tokens) && tokens.length > 0 && !isUnchanged) {
+        separatedList = typeof cap !== 'undefined' ? tokens.slice(0, cap) : tokens;
+      } else {
+        separatedList = null;
+      }
+    } else {
+      separatedList = getSeparatedContent(searchText, tokenSeparators, cap);
+    }
 
-    // Check if match the `tokenSeparators`
+    // Check if match the `tokenSeparators` or custom `tokenize`
     const patchLabels: string[] = isCompositing ? null : separatedList;
 
     // Ignore combobox since it's not split-able

tests/Multiple.test.tsx

@@ -80,6 +80,34 @@ describe('Select.Multiple', () => {
     expectOpen(container, false);
   });
 
+  it('tokenize prop on multiple only adds values that match options', () => {
+    const handleChange = jest.fn();
+    const tokenize = (input: string) => input.split(',').map((s) => s.trim());
+    const { container } = render(
+      <Select
+        mode="multiple"
+        optionLabelProp="children"
+        tokenize={tokenize}
+        onChange={handleChange}
+      >
+        <OptGroup key="group1">
+          <Option value="1">One</Option>
+        </OptGroup>
+        <OptGroup key="group2">
+          <Option value="2">Two</Option>
+        </OptGroup>
+      </Select>,
+    );
+    fireEvent.paste(container.querySelector('input'), {
+      clipboardData: { getData: () => 'One,Two,Unknown' },
+    });
+    fireEvent.change(container.querySelector('input'), {
+      target: { value: 'One,Two,Unknown' },
+    });
+    expect(handleChange).toHaveBeenCalledWith(['1', '2'], expect.anything());
+    expect(container.querySelector('input').value).toBe('');
+  });
+
   it('tokenize input when mode=tags and open=false', () => {
     const handleChange = jest.fn();
     const handleSelect = jest.fn();

tests/Tags.test.tsx

@@ -82,6 +82,71 @@ describe('Select.Tags', () => {
     expectOpen(container, false);
   });
 
+  it('tokenize prop overrides tokenSeparators with quote-aware splitting', () => {
+    const handleChange = jest.fn();
+    const tokenize = jest.fn((input: string) => {
+      const tokens: string[] = [];
+      const regex = /"([^"]*)"|([^,\n]+)/g;
+      let m: RegExpExecArray | null = regex.exec(input);
+      while (m !== null) {
+        tokens.push((m[1] ?? m[2]).trim());
+        m = regex.exec(input);
+      }
+      return tokens.filter(Boolean);
+    });
+    const { container } = render(
+      <Select mode="tags" tokenSeparators={[',']} tokenize={tokenize} onChange={handleChange}>
+        <Option value="1">1</Option>
+      </Select>,
+    );
+
+    fireEvent.change(container.querySelector('input'), { target: { value: '"a, b", c' } });
+
+    expect(tokenize).toHaveBeenCalled();
+    expect(handleChange).toHaveBeenCalledWith(['a, b', 'c'], expect.anything());
+    expect(container.querySelector('input').value).toBe('');
+  });
+
+  it('tokenize prop respects maxCount', () => {
+    const handleChange = jest.fn();
+    const tokenize = () => ['a', 'b', 'c', 'd', 'e'];
+    const { container } = render(
+      <Select mode="tags" maxCount={3} tokenize={tokenize} onChange={handleChange} />,
+    );
+    fireEvent.change(container.querySelector('input'), { target: { value: 'x' } });
+    expect(handleChange).toHaveBeenCalledWith(['a', 'b', 'c'], expect.anything());
+  });
+
+  it('tokenize prop ignored during composition', () => {
+    const handleChange = jest.fn();
+    const tokenize = (input: string) => input.split(',').map((s) => s.trim());
+    const { container } = render(
+      <Select mode="tags" tokenize={tokenize} onChange={handleChange}>
+        <Option value="1">1</Option>
+      </Select>,
+    );
+    fireEvent.compositionStart(container.querySelector('input'));
+    fireEvent.change(container.querySelector('input'), { target: { value: '2,3,4' } });
+    expect(handleChange).not.toHaveBeenCalled();
+    handleChange.mockReset();
+    fireEvent.compositionEnd(container.querySelector('input'));
+    fireEvent.change(container.querySelector('input'), { target: { value: '2,3,4' } });
+    expect(handleChange).toHaveBeenCalledWith(['2', '3', '4'], expect.anything());
+  });
+
+  it('tokenize prop returning [input] keeps typing', () => {
+    const handleChange = jest.fn();
+    const tokenize = (input: string) => [input];
+    const { container } = render(
+      <Select mode="tags" tokenize={tokenize} onChange={handleChange}>
+        <Option value="1">1</Option>
+      </Select>,
+    );
+    fireEvent.change(container.querySelector('input'), { target: { value: 'hello' } });
+    expect(handleChange).not.toHaveBeenCalled();
+    expect(container.querySelector<HTMLInputElement>('input').value).toBe('hello');
+  });
+
   it('should not separate words when compositing but trigger after composition end', () => {
     const handleChange = jest.fn();
     const handleSelect = jest.fn();