import { CodeGroup } from '@/app/components/develop/code.tsx' import { Row, Col, Properties, Property, Heading, SubProperty, Paragraph } from '@/app/components/develop/md.tsx' # 数据集 API

### Request Body 数据集名称 ```bash {{ title: 'cURL' }} curl --location --request POST 'https://api.dify.ai/v1/datasets' \ --header 'Authorization: Bearer {api_key}' \ --header 'Content-Type: application/json' \ --data-raw '{ "name": "name" }' ``` ```json {{ title: 'Response' }} { "id": "", "name": "name", "description": null, "provider": "vendor", "permission": "only_me", "data_source_type": null, "indexing_technique": null, "app_count": 0, "document_count": 0, "word_count": 0, "created_by": "", "created_at": 1695636173, "updated_by": "", "updated_at": 1695636173, "embedding_model": null, "embedding_model_provider": null, "embedding_available": null } ``` --- ### Path Query 页码 返回条数,默认 20,范围 1-100 ```bash {{ title: 'cURL' }} curl --location --request GET 'https://api.dify.ai/v1/datasets?page=1&limit=20' \ --header 'Authorization: Bearer {api_key}' ``` ```json {{ title: 'Response' }} { "data": [ { "id": "", "name": "数据集名称", "description": "描述信息", "permission": "only_me", "data_source_type": "upload_file", "indexing_technique": "", "app_count": 2, "document_count": 10, "word_count": 1200, "created_by": "", "created_at": "", "updated_by": "", "updated_at": "" }, ... ], "has_more": true, "limit": 20, "total": 50, "page": 1 } ``` --- 此接口基于已存在数据集,在此数据集的基础上通过文本创建新的文档 ### Path Params 数据集 ID ### Request Body 文档名称 文档内容 索引方式 - high_quality 高质量:使用 embedding 模型进行嵌入,构建为向量数据库索引 - economy 经济:使用 Keyword Table Index 的倒排索引进行构建 处理规则 - mode (string) 清洗、分段模式 ,automatic 自动 / custom 自定义 - rules (text) 自定义规则(自动模式下,该字段为空) - pre_processing_rules (array[object]) 预处理规则 - id (string) 预处理规则的唯一标识符 - 枚举: - remove_extra_spaces 替换连续空格、换行符、制表符 - remove_urls_emails 删除 URL、电子邮件地址 - enabled (bool) 是否选中该规则,不传入文档 ID 时代表默认值 - segmentation (object) 分段规则 - separator 自定义分段标识符,目前仅允许设置一个分隔符。默认为 \n - max_tokens 最大长度 (token) 默认为 1000 ```bash {{ title: 'cURL' }} curl --location --request POST 'https://api.dify.ai/v1/datasets/{dataset_id}/document/create_by_text' \ --header 'Authorization: Bearer {api_key}' \ --header 'Content-Type: application/json' \ --data-raw '{ "name": "text", "text": "text", "indexing_technique": "high_quality", "process_rule": { "mode": "automatic" } }' ``` ```json {{ title: 'Response' }} { "document": { "id": "", "position": 1, "data_source_type": "upload_file", "data_source_info": { "upload_file_id": "" }, "dataset_process_rule_id": "", "name": "text.txt", "created_from": "api", "created_by": "", "created_at": 1695690280, "tokens": 0, "indexing_status": "waiting", "error": null, "enabled": true, "disabled_at": null, "disabled_by": null, "archived": false, "display_status": "queuing", "word_count": 0, "hit_count": 0, "doc_form": "text_model" }, "batch": "" } ``` --- 此接口基于已存在数据集,在此数据集的基础上通过文件创建新的文档 ### Path Params 数据集 ID ### Request Body 源文档 ID (选填) - 用于重新上传文档或修改文档清洗、分段配置,缺失的信息从源文档复制 - 源文档不可为归档的文档 - 当传入 original_document_id 时,代表文档进行更新操作,process_rule 为可填项目,不填默认使用源文档的分段方式 - 未传入 original_document_id 时,代表文档进行新增操作,process_rule 为必填 需要上传的文件。 索引方式 - high_quality 高质量:使用 embedding 模型进行嵌入,构建为向量数据库索引 - economy 经济:使用 Keyword Table Index 的倒排索引进行构建 处理规则 - mode (string) 清洗、分段模式 ,automatic 自动 / custom 自定义。 - rules (text) 自定义规则(自动模式下,该字段为空) - pre_processing_rules (array[object]) 预处理规则 - id (string) 预处理规则的唯一标识符 - 枚举: - remove_extra_spaces 替换连续空格、换行符、制表符 - remove_urls_emails 删除 URL、电子邮件地址 - enabled (bool) 是否选中该规则,不传入文档 ID 时代表默认值。 - segmentation (object) 分段规则 - separator 自定义分段标识符,目前仅允许设置一个分隔符,默认为 \n - max_tokens 最大长度 (token) 默认为 1000 ```bash {{ title: 'cURL' }} curl --location POST 'https://api.dify.ai/v1/datasets/{dataset_id}/document/create_by_file' \ --header 'Authorization: Bearer {api_key}' \ --form 'data="{\"name\":\"Dify\",\"indexing_technique\":\"high_quality\",\"process_rule\":{\"rules\":{\"pre_processing_rules\":[{\"id\":\"remove_extra_spaces\",\"enabled\":true},{\"id\":\"remove_urls_emails\",\"enabled\":true}],\"segmentation\":{\"separator\":\"###\",\"max_tokens\":500}},\"mode\":\"custom\"}}";type=text/plain' \ --form 'file=@"/path/to/file"' ``` ```json {{ title: 'Response' }} { "document": { "id": "", "position": 1, "data_source_type": "upload_file", "data_source_info": { "upload_file_id": "" }, "dataset_process_rule_id": "", "name": "Dify.txt", "created_from": "api", "created_by": "", "created_at": 1695308667, "tokens": 0, "indexing_status": "waiting", "error": null, "enabled": true, "disabled_at": null, "disabled_by": null, "archived": false, "display_status": "queuing", "word_count": 0, "hit_count": 0, "doc_form": "text_model" }, "batch": "" } ``` --- 此接口基于已存在数据集,在此数据集的基础上通过文本更新文档 ### Path Params 数据集 ID 文档 ID ### Request Body 文档名称 (选填) 文档内容(选填) 处理规则(选填) - mode (string) 清洗、分段模式 ,automatic 自动 / custom 自定义。 - rules (text) 自定义规则(自动模式下,该字段为空) - pre_processing_rules (array[object]) 预处理规则 - id (string) 预处理规则的唯一标识符 - 枚举: - remove_extra_spaces 替换连续空格、换行符、制表符 - remove_urls_emails 删除 URL、电子邮件地址 - enabled (bool) 是否选中该规则,不传入文档 ID 时代表默认值。 - segmentation (object) 分段规则 - separator 自定义分段标识符,目前仅允许设置一个分隔符。默认为 \n - max_tokens 最大长度 (token) 默认为 1000 ```bash {{ title: 'cURL' }} curl --location --request POST 'https://api.dify.ai/v1/datasets/{dataset_id}/documents/{document_id}/update_by_text' \ --header 'Authorization: Bearer {api_key}' \ --header 'Content-Type: application/json' \ --data-raw '{ "name": "name", "text": "text" }' ``` ```json {{ title: 'Response' }} { "document": { "id": "", "position": 1, "data_source_type": "upload_file", "data_source_info": { "upload_file_id": "" }, "dataset_process_rule_id": "", "name": "name.txt", "created_from": "api", "created_by": "", "created_at": 1695308667, "tokens": 0, "indexing_status": "waiting", "error": null, "enabled": true, "disabled_at": null, "disabled_by": null, "archived": false, "display_status": "queuing", "word_count": 0, "hit_count": 0, "doc_form": "text_model" }, "batch": "" } ``` --- 此接口基于已存在数据集,在此数据集的基础上通过文件更新文档的操作。 ### Path Params 数据集 ID 文档 ID ### Request Body 文档名称 (选填) 需要上传的文件 处理规则(选填) - mode (string) 清洗、分段模式 ,automatic 自动 / custom 自定义。 - rules (text) 自定义规则(自动模式下,该字段为空) - pre_processing_rules (array[object]) 预处理规则 - id (string) 预处理规则的唯一标识符 - 枚举: - remove_extra_spaces 替换连续空格、换行符、制表符 - remove_urls_emails 删除 URL、电子邮件地址 - enabled (bool) 是否选中该规则,不传入文档 ID 时代表默认值 - segmentation (object) 分段规则 - separator 自定义分段标识符,目前仅允许设置一个分隔符,默认为 \n - max_tokens 最大长度 (token) 默认为 1000 ```bash {{ title: 'cURL' }} curl --location POST 'https://api.dify.ai/v1/datasets/{dataset_id}/document/{document_id}/create_by_file' \ --header 'Authorization: Bearer {api_key}' \ --form 'data="{\"name\":\"Dify\",\"indexing_technique\":\"high_quality\",\"process_rule\":{\"rules\":{\"pre_processing_rules\":[{\"id\":\"remove_extra_spaces\",\"enabled\":true},{\"id\":\"remove_urls_emails\",\"enabled\":true}],\"segmentation\":{\"separator\":\"###\",\"max_tokens\":500}},\"mode\":\"custom\"}}";type=text/plain' \ --form 'file=@"/path/to/file"' ``` ```json {{ title: 'Response' }} { "document": { "id": "", "position": 1, "data_source_type": "upload_file", "data_source_info": { "upload_file_id": "" }, "dataset_process_rule_id": "", "name": "Dify.txt", "created_from": "api", "created_by": "", "created_at": 1695308667, "tokens": 0, "indexing_status": "waiting", "error": null, "enabled": true, "disabled_at": null, "disabled_by": null, "archived": false, "display_status": "queuing", "word_count": 0, "hit_count": 0, "doc_form": "text_model" }, "batch": "20230921150427533684" } ``` --- ### Path Params 数据集 ID 上传文档的批次号 ```bash {{ title: 'cURL' }} curl --location --request GET 'https://api.dify.ai/v1/datasets/{dataset_id}/documents/{batch}/indexing-status' \ --header 'Authorization: Bearer {api_key}' \ ``` ```json {{ title: 'Response' }} { "data":[{ "id": "", "indexing_status": "indexing", "processing_started_at": 1681623462.0, "parsing_completed_at": 1681623462.0, "cleaning_completed_at": 1681623462.0, "splitting_completed_at": 1681623462.0, "completed_at": null, "paused_at": null, "error": null, "stopped_at": null, "completed_segments": 24, "total_segments": 100 }] } ``` --- ### Path Params 数据集 ID 文档 ID ```bash {{ title: 'cURL' }} curl --location --request DELETE 'https://api.dify.ai/v1/datasets/{dataset_id}/documents/{document_id}' \ --header 'Authorization: Bearer {api_key}' \ ``` ```json {{ title: 'Response' }} { "result": "success" } ``` --- ### Path Params 数据集 ID ### Path Query 搜索关键词,可选,目前仅搜索文档名称 页码,可选 返回条数,可选,默认 20,范围 1-100 ```bash {{ title: 'cURL' }} curl --location --request GET 'https://api.dify.ai/v1/datasets/{dataset_id}/documents' \ --header 'Authorization: Bearer {api_key}' \ ``` ```json {{ title: 'Response' }} { "data": [ { "id": "", "position": 1, "data_source_type": "file_upload", "data_source_info": null, "dataset_process_rule_id": null, "name": "dify", "created_from": "", "created_by": "", "created_at": 1681623639, "tokens": 0, "indexing_status": "waiting", "error": null, "enabled": true, "disabled_at": null, "disabled_by": null, "archived": false }, ], "has_more": false, "limit": 20, "total": 9, "page": 1 } ``` --- ### Path Params 数据集 ID 文档 ID ### Request Body segments (object list) 分段内容 - content (text) 文本内容/问题内容,必填 - answer(text) 答案内容,非必填,如果数据集的模式为qa模式则传值 - keywords(list) 关键字,非必填 ```bash {{ title: 'cURL' }} curl --location --request POST 'https://api.dify.ai/v1/datasets/{dataset_id}/documents/{document_id}/segments' \ --header 'Authorization: Bearer {api_key}' \ --header 'Content-Type: application/json' \ --data-raw '{ "segments": [ { "content": "1", "answer": "1", "keywords": ["a"] } ] }' ``` ```json {{ title: 'Response' }} { "data": [{ "id": "", "position": 1, "document_id": "", "content": "1", "answer": "1", "word_count": 25, "tokens": 0, "keywords": [ "a" ], "index_node_id": "", "index_node_hash": "", "hit_count": 0, "enabled": true, "disabled_at": null, "disabled_by": null, "status": "completed", "created_by": "", "created_at": 1695312007, "indexing_at": 1695312007, "completed_at": 1695312007, "error": null, "stopped_at": null }], "doc_form": "text_model" } ``` --- 错误信息 - **document_indexing**: 文档索引失败 - **provider_not_initialize**: Embedding 模型未配置 - **not_found**,文档不存在 - **dataset_name_duplicate**: 数据集名称重复 - **provider_quota_exceeded**: 模型额度超过限制 - **dataset_not_initialized**: 数据集还未初始化 - **unsupported_file_type**: 不支持的文件类型 - 目前只支持:txt, markdown, md, pdf, html, htm, xlsx, docx, csv - **too_many_files**: 文件数量过多,暂时只支持单一文件上传 - **file_too_large*: 文件太大,默认支持15M以下, 具体需要参考环境变量配置