diff --git a/app-builder/plugins/data-mate-knowledge/pom.xml b/app-builder/plugins/data-mate-knowledge/pom.xml new file mode 100644 index 000000000..1a637780c --- /dev/null +++ b/app-builder/plugins/data-mate-knowledge/pom.xml @@ -0,0 +1,142 @@ + + + 4.0.0 + + modelengine.fit.jade + app-builder-plugin-parent + 1.0.0-SNAPSHOT + + + modelengine.jade.plugin + data-mate-knowledge + + + 17 + 17 + UTF-8 + + + + + + org.fitframework + fit-api + + + + + modelengine.fit.jade.service + knowledge-service + + + + + org.projectlombok + lombok + + + + + org.mapstruct + mapstruct + + + org.mapstruct + mapstruct-processor + + + + + com.fasterxml.jackson.core + jackson-databind + + + + + org.fitframework + fit-test-framework + + + org.junit.jupiter + junit-jupiter + + + org.mockito + mockito-core + + + org.assertj + assertj-core + + + + + + + org.fitframework + fit-build-maven-plugin + ${fit.version} + + + build-plugin + + build-plugin + + + + package-plugin + + package-plugin + + + + + + org.fitframework + fit-dependency-maven-plugin + ${fit.version} + + + dependency + compile + + dependency + + + + + + org.apache.maven.plugins + maven-jar-plugin + ${maven.jar.version} + + + + FIT Lab + + + + + + org.apache.maven.plugins + maven-antrun-plugin + ${maven.antrun.version} + + + install + + + + + + + run + + + + + + + \ No newline at end of file diff --git a/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/convertor/ParamConvertor.java b/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/convertor/ParamConvertor.java new file mode 100644 index 000000000..765223b52 --- /dev/null +++ b/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/convertor/ParamConvertor.java @@ -0,0 +1,142 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. + * This file is a part of the ModelEngine Project. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package modelengine.fit.jade.datamate.knowledge.convertor; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import modelengine.fitframework.util.StringUtils; +import modelengine.jade.knowledge.KnowledgeRepo; +import modelengine.jade.knowledge.ReferenceLimit; +import modelengine.jade.knowledge.document.KnowledgeDocument; +import modelengine.fit.jade.datamate.knowledge.dto.DataMateRetrievalParam; +import modelengine.fit.jade.datamate.knowledge.entity.DataMateKnowledgeEntity; +import modelengine.fit.jade.datamate.knowledge.entity.DataMateRetrievalChunksEntity; +import modelengine.jade.knowledge.support.FlatKnowledgeOption; + +import org.mapstruct.Mapper; +import org.mapstruct.Mapping; +import org.mapstruct.Named; +import org.mapstruct.factory.Mappers; + +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.HashMap; +import java.util.Map; + +/** + * DataMate 内部数据的转换器接口。 + * + * @author 陈镕希 + * @since 2025-12-15 + */ +@Mapper +public interface ParamConvertor { + ParamConvertor INSTANCE = Mappers.getMapper(ParamConvertor.class); + int TOP = 400; + + /** + * 将 {@link DataMateKnowledgeEntity} 转换为 {@link KnowledgeRepo}。 + * + * @param entity 表示待转换的 {@link DataMateKnowledgeEntity}。 + * @return 转换完成的 {@link KnowledgeRepo}。 + */ + @Mapping(target = "type", source = "entity", qualifiedByName = "mapIndexTypeToType") + @Mapping(target = "createdAt", source = "entity", qualifiedByName = "stringToLocalDateTime") + KnowledgeRepo convertToKnowledgeRepo(DataMateKnowledgeEntity entity); + + /** + * 将 DataMate 知识库的检索 type 映射为 平台知识库元数据 type。 + * + * @param entity 表示待转换的 {@link DataMateKnowledgeEntity}。 + * @return 表示转换完成的 {@link String}。 + */ + @Named("mapIndexTypeToType") + default String mapIndexTypeToType(DataMateKnowledgeEntity entity) { + return entity == null ? null : entity.getEmbeddingModel(); + } + + /** + * 将 DataMate 知识库的 createdAt 映射为 平台知识库元数据 createdAt。 + * + * @param entity 表示待转换的 {@link DataMateKnowledgeEntity}。 + * @return 表示转换完成的 {@link LocalDateTime}。 + */ + @Named("stringToLocalDateTime") + default LocalDateTime stringToLocalDateTime(DataMateKnowledgeEntity entity) { + String dateStr = entity.getCreatedAt(); + if (dateStr == null || StringUtils.isEmpty(dateStr)) { + return null; + } + return LocalDateTime.parse(dateStr, DateTimeFormatter.ISO_DATE_TIME); + } + + /** + * 将 {@link FlatKnowledgeOption} 转换为 {@link DataMateRetrievalParam}。 + * + * @param option 表示待转换的 {@link FlatKnowledgeOption}。 + * @return 转换完成的 {@link DataMateRetrievalParam}。 + */ + @Mapping(target = "knowledgeBaseIds", source = "repoIds") + @Mapping(target = "query", source = "query") + @Mapping(target = "topK", source = "referenceLimit", qualifiedByName = "mapReferenceLimitToTop") + @Mapping(target = "threshold", source = "similarityThreshold") + DataMateRetrievalParam convertToRetrievalParam(FlatKnowledgeOption option); + + /** + * 将平台检索请求 ReferenceLimit 映射为 DataMate 检索请求 top。 + * + * @param limit 表示待转换的 {@link ReferenceLimit}。 + * @return 转换完成的 {@link int}。 + */ + @Named("mapReferenceLimitToTop") + default int mapReferenceLimitToTop(ReferenceLimit limit) { + if (limit == null) { + return TOP; + } + return limit.getValue(); + } + + /** + * 将 {@link DataMateRetrievalChunksEntity} 转换为 {@link KnowledgeDocument}。 + * + * @param entity 表示待转换的 {@link DataMateRetrievalChunksEntity}。 + * @return 转换完成的 {@link KnowledgeDocument}。 + */ + @Mapping(target = "id", expression = "java(entity.chunkId())") + @Mapping(target = "text", expression = "java(entity.content())") + @Mapping(target = "score", expression = "java(entity.retrievalScore())") + @Mapping(target = "metadata", source = ".", qualifiedByName = "mapChunksEntityToMetadata") + KnowledgeDocument convertToKnowledgeDocument(DataMateRetrievalChunksEntity entity); + + /** + * 将 DataMate 检索结果 entity 映射为 平台检索结果 metadata。 + * + * @param entity 表示待转换的 {@link DataMateRetrievalChunksEntity}。 + * @return 转换完成的 {@link Map}{@code <}{@link String}{@code , }{@link Object}{@code >}。 + */ + @Named("mapChunksEntityToMetadata") + default Map mapChunksEntityToMetadata(DataMateRetrievalChunksEntity entity) { + Map metadata = new HashMap<>(); + if (entity == null || entity.getEntity() == null) { + return metadata; + } + metadata.put("primaryKey", entity.getPrimaryKey()); + String rawMetadata = entity.getEntity().getMetadata(); + if (!StringUtils.isEmpty(rawMetadata)) { + try { + Map parsed = new ObjectMapper().readValue(rawMetadata, Map.class); + metadata.put("fileId", parsed.get("original_file_id")); + metadata.put("fileName", parsed.get("file_name")); + metadata.putAll(parsed); + } catch (JsonProcessingException ex) { + metadata.put("metadata", rawMetadata); + } + } + return metadata; + } +} + diff --git a/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/dto/DataMateKnowledgeListQueryParam.java b/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/dto/DataMateKnowledgeListQueryParam.java new file mode 100644 index 000000000..4574efd33 --- /dev/null +++ b/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/dto/DataMateKnowledgeListQueryParam.java @@ -0,0 +1,43 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. + * This file is a part of the ModelEngine Project. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package modelengine.fit.jade.datamate.knowledge.dto; + +import lombok.Builder; +import lombok.Data; +import modelengine.fitframework.serialization.annotation.SerializeStrategy; + +/** + * DataMate 知识库列表查询参数。 + * + * @author 陈镕希 + * @since 2025-12-15 + */ +@Data +@Builder +@SerializeStrategy(include = SerializeStrategy.Include.NON_NULL) +public class DataMateKnowledgeListQueryParam { + /** + * 页码，从0开始。 + */ + private Integer page; + + /** + * 每页大小。 + */ + private Integer size; + + /** + * 知识库名称过滤。 + */ + private String name; + + /** + * 知识库描述过滤。 + */ + private String description; +} + diff --git a/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/dto/DataMateRetrievalParam.java b/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/dto/DataMateRetrievalParam.java new file mode 100644 index 000000000..8b1277277 --- /dev/null +++ b/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/dto/DataMateRetrievalParam.java @@ -0,0 +1,44 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. + * This file is a part of the ModelEngine Project. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package modelengine.fit.jade.datamate.knowledge.dto; + +import lombok.Builder; +import lombok.Data; +import modelengine.fitframework.annotation.Property; + +import java.util.List; + +/** + * DataMate 知识库检索查询参数。 + * + * @author 陈镕希 + * @since 2025-12-15 + */ +@Data +@Builder +public class DataMateRetrievalParam { + /** + * 检索 query。 + */ + private String query; + /** + * 返回前多少的条目。 + */ + @Property(description = "topK", name = "topK") + private Integer topK; + /** + * 相关性阈值。 + */ + @Property(description = "threshold", name = "threshold") + private Double threshold; + /** + * 指定知识库的id集合。 + */ + @Property(description = "knowledgeBaseIds", name = "knowledgeBaseIds") + private List knowledgeBaseIds; +} + diff --git a/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/entity/DataMateKnowledgeEntity.java b/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/entity/DataMateKnowledgeEntity.java new file mode 100644 index 000000000..c6138923c --- /dev/null +++ b/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/entity/DataMateKnowledgeEntity.java @@ -0,0 +1,110 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. + * This file is a part of the ModelEngine Project. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package modelengine.fit.jade.datamate.knowledge.entity; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * DataMate 知识库 Entity。 + * + * @author 陈镕希 + * @since 2025-12-15 + */ +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +@JsonIgnoreProperties(ignoreUnknown = true) +public class DataMateKnowledgeEntity { + /** + * 知识库id。 + */ + private String id; + + /** + * 知识库名称。 + */ + private String name; + + /** + * 知识库描述。 + */ + private String description; + + /** + * 知识库创建时间。 + */ + private String createdAt; + + /** + * 知识库更新时间。 + */ + private String updatedAt; + + /** + * 创建人。 + */ + private String createdBy; + + /** + * 更新人。 + */ + private String updatedBy; + + /** + * 嵌入模型名称。 + */ + private String embeddingModel; + + /** + * 聊天模型名称。 + */ + private String chatModel; + + /** + * 文件数量。 + */ + private Long fileCount; + + /** + * chunk 数量。 + */ + private Long chunkCount; + + /** + * 嵌入模型配置。 + */ + private ModelConfig embedding; + + /** + * 聊天模型配置。 + */ + private ModelConfig chat; + + @Data + @JsonIgnoreProperties(ignoreUnknown = true) + public static class ModelConfig { + private String id; + private String createdAt; + private String updatedAt; + private String createdBy; + private String updatedBy; + private String modelName; + private String provider; + private String baseUrl; + private String apiKey; + private String type; + private Boolean isEnabled; + private Boolean isDefault; + } +} + diff --git a/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/entity/DataMateKnowledgeListEntity.java b/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/entity/DataMateKnowledgeListEntity.java new file mode 100644 index 000000000..c54247b56 --- /dev/null +++ b/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/entity/DataMateKnowledgeListEntity.java @@ -0,0 +1,57 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. + * This file is a part of the ModelEngine Project. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package modelengine.fit.jade.datamate.knowledge.entity; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +/** + * DataMate 知识库列表 Entity。 + * + * @author 陈镕希 + * @since 2025-12-15 + */ +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +@JsonIgnoreProperties(ignoreUnknown = true) +public class DataMateKnowledgeListEntity { + /** + * 当前页码（从 0 开始）。 + */ + private Integer page; + + /** + * 每页数量。 + */ + private Integer size; + + /** + * 总条目数。 + */ + private Integer totalElements; + + /** + * 总页数。 + */ + private Integer totalPages; + + /** + * 知识库列表查询数据。 + */ + @JsonProperty("content") + private List content; +} + diff --git a/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/entity/DataMateResponse.java b/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/entity/DataMateResponse.java new file mode 100644 index 000000000..9034f7721 --- /dev/null +++ b/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/entity/DataMateResponse.java @@ -0,0 +1,68 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. + * This file is a part of the ModelEngine Project. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package modelengine.fit.jade.datamate.knowledge.entity; + +import com.fasterxml.jackson.databind.JavaType; +import com.fasterxml.jackson.databind.ObjectMapper; + +import lombok.Data; +import modelengine.fitframework.util.ObjectUtils; + +import java.util.List; +import java.util.Map; + +/** + * DataMate 接口返回值结构。 + * + * @author 陈镕希 + * @since 2025-12-15 + */ +@Data +public class DataMateResponse { + private T data; + private String code; + private String message; + + /** + * 从响应上下文 Map 中解析并构建 {@link DataMateResponse} 对象。 + *

+ * 针对 DataMateRetrievalResult 的特殊性，如果 context 中的 data 是一个 List， + * 会自动将其封装进 DataMateRetrievalResult 对象的 data 属性中。 + *

+ * + * @param 响应数据的泛型类型。 + * @param context 包含响应信息的上下文 {@link Map}。 + * @param type 目标类型的 {@link Class}。 + * @return 返回构建好的 {@link DataMateResponse} 实例。 + */ + public static DataMateResponse from(Map context, Class type) { + DataMateResponse response = new DataMateResponse<>(); + ObjectMapper objectMapper = new ObjectMapper(); + + Object rawData = context.get("data"); + + // 核心逻辑：如果目标类型是 DataMateRetrievalResult，且 rawData 是个 List + if (type.equals(DataMateRetrievalResult.class) && rawData instanceof List) { + DataMateRetrievalResult resultObject = new DataMateRetrievalResult(); + // 将 List 里的 Map 转换为 List + JavaType listType = objectMapper.getTypeFactory() + .constructCollectionType(List.class, DataMateRetrievalChunksEntity.class); + List chunks = objectMapper.convertValue(rawData, listType); + + resultObject.setData(chunks); + response.data = (T) resultObject; // 强制转换回泛型 T + } else { + // 其他常规情况按原逻辑处理 + response.data = objectMapper.convertValue(rawData, type); + } + + response.code = ObjectUtils.cast(context.get("code")); + response.message = ObjectUtils.cast(context.get("message")); + return response; + } +} + diff --git a/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/entity/DataMateRetrievalChunksEntity.java b/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/entity/DataMateRetrievalChunksEntity.java new file mode 100644 index 000000000..5675cbb83 --- /dev/null +++ b/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/entity/DataMateRetrievalChunksEntity.java @@ -0,0 +1,57 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. + * This file is a part of the ModelEngine Project. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package modelengine.fit.jade.datamate.knowledge.entity; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +import lombok.Data; + +/** + * DataMate 知识库检索 chunk 结果。 + * + * @author 陈镕希 + * @since 2025-12-15 + */ +@Data +@JsonIgnoreProperties(ignoreUnknown = true) +public class DataMateRetrievalChunksEntity { + /** + * 查询结果实体。 + */ + private RagChunk entity; + + /** + * 匹配分值。 + */ + private Double score; + + /** + * 主键字段名。 + */ + private String primaryKey; + + public String chunkId() { + return this.entity != null ? this.entity.getId() : null; + } + + public String content() { + return this.entity != null ? this.entity.getText() : null; + } + + public double retrievalScore() { + return this.score == null ? 0 : this.score; + } + + @Data + @JsonIgnoreProperties(ignoreUnknown = true) + public static class RagChunk { + private String id; + private String text; + private String metadata; + } +} + diff --git a/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/entity/DataMateRetrievalResult.java b/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/entity/DataMateRetrievalResult.java new file mode 100644 index 000000000..e9db42827 --- /dev/null +++ b/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/entity/DataMateRetrievalResult.java @@ -0,0 +1,29 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. + * This file is a part of the ModelEngine Project. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package modelengine.fit.jade.datamate.knowledge.entity; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +import lombok.Data; + +import java.util.List; + +/** + * DataMate 知识库检索结果。 + * + * @author 陈镕希 + * @since 2025-12-15 + */ +@Data +@JsonIgnoreProperties(ignoreUnknown = true) +public class DataMateRetrievalResult { + /** + * 检索结果集合。 + */ + private List data; +} + diff --git a/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/entity/PageVoKnowledgeList.java b/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/entity/PageVoKnowledgeList.java new file mode 100644 index 000000000..2163e0acd --- /dev/null +++ b/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/entity/PageVoKnowledgeList.java @@ -0,0 +1,33 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. + * This file is a part of the ModelEngine Project. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package modelengine.fit.jade.datamate.knowledge.entity; + +import lombok.Builder; +import lombok.Data; + +import java.util.List; + +/** + * 表示知识库列表分页数据对象。 + * + * @author 陈镕希 + * @since 2025-12-15 + */ +@Data +@Builder +public class PageVoKnowledgeList { + /** + * 知识库列表查询数据。 + */ + private List knowledgeEntityList; + + /** + * 知识库总数。 + */ + private int total; +} + diff --git a/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/external/DataMateKnowledgeBaseManager.java b/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/external/DataMateKnowledgeBaseManager.java new file mode 100644 index 000000000..0341270a1 --- /dev/null +++ b/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/external/DataMateKnowledgeBaseManager.java @@ -0,0 +1,157 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. + * This file is a part of the ModelEngine Project. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package modelengine.fit.jade.datamate.knowledge.external; + +import static modelengine.fit.http.protocol.MessageHeaderNames.AUTHORIZATION; +import static modelengine.fit.http.protocol.MessageHeaderNames.CONTENT_TYPE; +import static modelengine.jade.knowledge.code.KnowledgeManagerRetCode.AUTHENTICATION_ERROR; +import static modelengine.jade.knowledge.code.KnowledgeManagerRetCode.CLIENT_REQUEST_ERROR; +import static modelengine.jade.knowledge.code.KnowledgeManagerRetCode.INTERNAL_SERVICE_ERROR; +import static modelengine.jade.knowledge.code.KnowledgeManagerRetCode.NOT_FOUND; +import static modelengine.jade.knowledge.code.KnowledgeManagerRetCode.QUERY_KNOWLEDGE_ERROR; +import static modelengine.jade.knowledge.code.KnowledgeManagerRetCode.QUERY_KNOWLEDGE_LIST_ERROR; + +import modelengine.fit.http.client.HttpClassicClient; +import modelengine.fit.http.client.HttpClassicClientFactory; +import modelengine.fit.http.client.HttpClassicClientRequest; +import modelengine.fit.http.client.HttpClientResponseException; +import modelengine.fit.http.entity.Entity; +import modelengine.fit.http.protocol.HttpRequestMethod; +import modelengine.fit.jade.datamate.knowledge.dto.DataMateKnowledgeListQueryParam; +import modelengine.fit.jade.datamate.knowledge.dto.DataMateRetrievalParam; +import modelengine.fit.jade.datamate.knowledge.entity.DataMateKnowledgeListEntity; +import modelengine.fit.jade.datamate.knowledge.entity.DataMateResponse; +import modelengine.fit.jade.datamate.knowledge.entity.DataMateRetrievalResult; +import modelengine.fitframework.annotation.Component; +import modelengine.fitframework.annotation.Value; +import modelengine.fitframework.exception.ClientException; +import modelengine.fitframework.inspection.Validation; +import modelengine.fitframework.log.Logger; +import modelengine.fitframework.util.LazyLoader; +import modelengine.fitframework.util.MapBuilder; +import modelengine.fitframework.util.ObjectUtils; +import modelengine.fitframework.util.StringUtils; +import modelengine.jade.knowledge.code.KnowledgeManagerRetCode; +import modelengine.jade.knowledge.exception.KnowledgeException; + +import java.util.HashMap; +import java.util.Map; + +/** + * 表示 DataMate 知识库的调用工具。 + * + * @author 陈镕希 + * @since 2025-12-15 + */ +@Component +public class DataMateKnowledgeBaseManager { + private static final Logger log = Logger.get(DataMateKnowledgeBaseManager.class); + private static final String BEARER = "Bearer "; + private static final String CONTENT_TYPE_JSON = "application/json"; + /** 默认访问超时时间（秒）。 */ + private static final int DEFAULT_TIMEOUT_SECONDS = 30; + + private final Map dataMateUrls; + private final HttpClassicClientFactory httpClientFactory; + private final LazyLoader httpClient; + private final Map exceptionMap = new HashMap<>(); + /** 访问超时时间（秒），用于连接、读、请求超时。 */ + private final int timeoutSeconds; + + public DataMateKnowledgeBaseManager(@Value("${datamate.url}") Map dataMateUrls, + HttpClassicClientFactory httpClientFactory, + @Value("${datamate.timeout:30}") int timeoutSeconds) { + this.dataMateUrls = dataMateUrls; + this.httpClientFactory = httpClientFactory; + this.timeoutSeconds = timeoutSeconds > 0 ? timeoutSeconds : DEFAULT_TIMEOUT_SECONDS; + this.httpClient = new LazyLoader<>(this::getHttpClient); + this.exceptionMap.put(500, INTERNAL_SERVICE_ERROR); + this.exceptionMap.put(401, AUTHENTICATION_ERROR); + this.exceptionMap.put(404, NOT_FOUND); + this.exceptionMap.put(400, CLIENT_REQUEST_ERROR); + } + + /** + * 获取 DataMate 知识库列表。 + * + * @param apiKey 表示知识库接口鉴权 api key 的 {@link String}。 + * @param param 表示知识库列表查询参数的 {@link DataMateKnowledgeListQueryParam}。 + * @return 表示知识库列表的 {@link DataMateKnowledgeListEntity}。 + */ + public DataMateKnowledgeListEntity listRepos(String apiKey, DataMateKnowledgeListQueryParam param) { + HttpClassicClientRequest request = + this.httpClient.get().createRequest(HttpRequestMethod.POST, this.dataMateUrls.get("list")); + request.entity(Entity.createObject(request, param)); + if (StringUtils.isNotEmpty(apiKey)) { + request.headers().set(AUTHORIZATION, BEARER + apiKey); + } + try { + Object object = this.httpClient.get().exchangeForEntity(request, Object.class); + Map response = + ObjectUtils.toCustomObject(Validation.notNull(object, "The response body is abnormal."), Map.class); + DataMateResponse resp = + DataMateResponse.from(response, DataMateKnowledgeListEntity.class); + return Validation.notNull(resp.getData(), "The response body is abnormal."); + } catch (ClientException ex) { + log.error(QUERY_KNOWLEDGE_LIST_ERROR.getMsg(), ex.getMessage()); + throw new KnowledgeException(QUERY_KNOWLEDGE_LIST_ERROR, ex, ex.getMessage()); + } catch (HttpClientResponseException ex) { + throw this.handleException(ex); + } + } + + /** + * DataMate 知识库检索。 + * + * @param apiKey 表示知识库接口鉴权 api key 的 {@link String}。 + * @param param 表示知识库检索查询参数的 {@link DataMateRetrievalParam}。 + * @return 表示知识库检索结果的 {@link DataMateRetrievalResult}。 + */ + public DataMateRetrievalResult retrieve(String apiKey, DataMateRetrievalParam param) { + HttpClassicClientRequest request = + this.httpClient.get().createRequest(HttpRequestMethod.POST, this.dataMateUrls.get("retrieve")); + request.entity(Entity.createObject(request, param)); + if (StringUtils.isNotEmpty(apiKey)) { + request.headers().set(AUTHORIZATION, BEARER + apiKey); + } + request.headers().set(CONTENT_TYPE, CONTENT_TYPE_JSON); + try { + Object object = this.httpClient.get().exchangeForEntity(request, Object.class); + Map response = + ObjectUtils.toCustomObject(Validation.notNull(object, "The response body is abnormal."), Map.class); + DataMateResponse resp = + DataMateResponse.from(response, DataMateRetrievalResult.class); + return Validation.notNull(resp.getData(), "The response body is abnormal."); + } catch (ClientException ex) { + log.error(QUERY_KNOWLEDGE_ERROR.getMsg(), ex.getMessage()); + throw new KnowledgeException(QUERY_KNOWLEDGE_ERROR, ex, ex.getMessage()); + } catch (HttpClientResponseException ex) { + throw this.handleException(ex); + } + } + + private KnowledgeException handleException(HttpClientResponseException ex) { + int statusCode = ex.statusCode(); + log.error(this.exceptionMap.get(statusCode).getMsg(), ex); + return new KnowledgeException(this.exceptionMap.get(statusCode), ex, ex.getSimpleMessage()); + } + + private HttpClassicClient getHttpClient() { + int timeoutMs = this.timeoutSeconds * 1000; + Map custom = MapBuilder.get() + .put("client.http.secure.ignore-trust", true) + .put("client.http.secure.ignore-hostname", true) + .build(); + return this.httpClientFactory.create(HttpClassicClientFactory.Config.builder() + .custom(custom) + .connectTimeout(timeoutMs) + .socketTimeout(timeoutMs) + .connectionRequestTimeout(timeoutMs) + .build()); + } +} + diff --git a/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/service/DataMateKnowledgeRepoServiceImpl.java b/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/service/DataMateKnowledgeRepoServiceImpl.java new file mode 100644 index 000000000..bfbe02d53 --- /dev/null +++ b/app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/service/DataMateKnowledgeRepoServiceImpl.java @@ -0,0 +1,155 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. + * This file is a part of the ModelEngine Project. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package modelengine.fit.jade.datamate.knowledge.service; + +import modelengine.fit.jade.datamate.knowledge.convertor.ParamConvertor; +import modelengine.fit.jade.datamate.knowledge.dto.DataMateKnowledgeListQueryParam; +import modelengine.fit.jade.datamate.knowledge.dto.DataMateRetrievalParam; +import modelengine.fit.jade.datamate.knowledge.entity.DataMateKnowledgeEntity; +import modelengine.fit.jade.datamate.knowledge.entity.DataMateKnowledgeListEntity; +import modelengine.fit.jade.datamate.knowledge.entity.DataMateRetrievalChunksEntity; +import modelengine.fit.jade.datamate.knowledge.entity.DataMateRetrievalResult; +import modelengine.fit.jade.datamate.knowledge.entity.PageVoKnowledgeList; +import modelengine.fit.jade.datamate.knowledge.external.DataMateKnowledgeBaseManager; +import modelengine.fitframework.annotation.Component; +import modelengine.fitframework.annotation.Fitable; +import modelengine.fitframework.inspection.Validation; +import modelengine.jade.common.vo.PageVo; +import modelengine.jade.knowledge.FilterConfig; +import modelengine.jade.knowledge.KnowledgeI18nInfo; +import modelengine.jade.knowledge.KnowledgeI18nService; +import modelengine.jade.knowledge.KnowledgeProperty; +import modelengine.jade.knowledge.KnowledgeRepo; +import modelengine.jade.knowledge.KnowledgeRepoService; +import modelengine.jade.knowledge.ListRepoQueryParam; +import modelengine.jade.knowledge.document.KnowledgeDocument; +import modelengine.jade.knowledge.enums.FilterType; +import modelengine.jade.knowledge.enums.IndexType; +import modelengine.jade.knowledge.support.FlatFilterConfig; +import modelengine.jade.knowledge.support.FlatKnowledgeOption; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +/** + * 表示知识库服务在 DataMate 中的实现。 + * + * @author 陈镕希 + * @since 2025-12-15 + */ +@Component +public class DataMateKnowledgeRepoServiceImpl implements KnowledgeRepoService { + /** + * DataMate 知识库的服务唯一标识。 + */ + public static final String FITABLE_ID_DEFAULT = "dataMateKnowledge"; + + private static final int DEFAULT_TOP_K = 3; + private static final int MAX_TOP_K = 10; + private static final float DEFAULT_THRESHOLD = 0.1f; + + private final DataMateKnowledgeBaseManager knowledgeBaseManager; + private final KnowledgeI18nService knowledgeI18nService; + + public DataMateKnowledgeRepoServiceImpl(DataMateKnowledgeBaseManager knowledgeBaseManager, + KnowledgeI18nService knowledgeI18nService) { + this.knowledgeBaseManager = knowledgeBaseManager; + this.knowledgeI18nService = knowledgeI18nService; + } + + @Override + @Fitable(FITABLE_ID_DEFAULT) + public PageVo listRepos(String apiKey, ListRepoQueryParam param) { + Validation.notNull(param, "The query param cannot be null."); + PageVoKnowledgeList pageVoKnowledgeList = this.queryKnowledgeList(apiKey, param); + List repos = pageVoKnowledgeList.getKnowledgeEntityList() + .stream() + .map(ParamConvertor.INSTANCE::convertToKnowledgeRepo) + .toList(); + return PageVo.of(pageVoKnowledgeList.getTotal(), repos); + } + + @Override + @Fitable(FITABLE_ID_DEFAULT) + public KnowledgeProperty getProperty(String apiKey) { + KnowledgeI18nInfo semanticInfo = this.knowledgeI18nService.localizeText(IndexType.SEMANTIC); + KnowledgeProperty.IndexInfo semanticIndex = new KnowledgeProperty.IndexInfo(IndexType.SEMANTIC, + semanticInfo.getName(), + semanticInfo.getDescription()); + KnowledgeI18nInfo fullTextInfo = this.knowledgeI18nService.localizeText(IndexType.FULL_TEXT); + KnowledgeProperty.IndexInfo fullTextIndex = new KnowledgeProperty.IndexInfo(IndexType.FULL_TEXT, + fullTextInfo.getName(), + fullTextInfo.getDescription()); + KnowledgeI18nInfo hybridInfo = this.knowledgeI18nService.localizeText(IndexType.HYBRID); + KnowledgeProperty.IndexInfo hybridIndex = + new KnowledgeProperty.IndexInfo(IndexType.HYBRID, hybridInfo.getName(), hybridInfo.getDescription()); + KnowledgeI18nInfo referenceInfo = this.knowledgeI18nService.localizeText(FilterType.REFERENCE_TOP_K); + FlatFilterConfig topKFilter = new FlatFilterConfig(FilterConfig.custom() + .name(referenceInfo.getName()) + .description(referenceInfo.getDescription()) + .type(FilterType.REFERENCE_TOP_K) + .minimum(1) + .maximum(MAX_TOP_K) + .defaultValue(DEFAULT_TOP_K) + .build()); + KnowledgeI18nInfo relevancyInfo = this.knowledgeI18nService.localizeText(FilterType.SIMILARITY_THRESHOLD); + FlatFilterConfig similarityFilter = new FlatFilterConfig(FilterConfig.custom() + .name(relevancyInfo.getName()) + .description(relevancyInfo.getDescription()) + .type(FilterType.SIMILARITY_THRESHOLD) + .minimum(0) + .maximum(1) + .defaultValue(DEFAULT_THRESHOLD) + .build()); + KnowledgeI18nInfo rerankInfo = new KnowledgeI18nInfo(this.knowledgeI18nService.localizeText("rerankParam"), + this.knowledgeI18nService.localizeText("rerankParam.description")); + KnowledgeProperty.RerankConfig rerankConfig = + new KnowledgeProperty.RerankConfig("boolean", rerankInfo.getName(), rerankInfo.getDescription(), false); + return new KnowledgeProperty(Arrays.asList(semanticIndex, fullTextIndex, hybridIndex), + Arrays.asList(topKFilter, similarityFilter), + Collections.singletonList(rerankConfig)); + } + + @Override + @Fitable(FITABLE_ID_DEFAULT) + public List retrieve(String apiKey, FlatKnowledgeOption option) { + Validation.notNull(option, "The knowledge option cannot be null."); + DataMateRetrievalParam param = ParamConvertor.INSTANCE.convertToRetrievalParam(option); + DataMateRetrievalResult result = this.knowledgeBaseManager.retrieve(apiKey, param); + List chunks = result.getData() == null + ? Collections.emptyList() + : result.getData(); + return chunks + .stream() + .map(ParamConvertor.INSTANCE::convertToKnowledgeDocument) + .collect(Collectors.toList()); + } + + private PageVoKnowledgeList queryKnowledgeList(String apiKey, ListRepoQueryParam param) { + int page = Math.max(param.getPageIndex() - 1, 0); + int size = param.getPageSize(); + DataMateKnowledgeListEntity listEntity = this.executeQuery(apiKey, param.getRepoName(), page, size); + List content = listEntity.getContent() == null + ? Collections.emptyList() + : listEntity.getContent(); + return PageVoKnowledgeList.builder() + .knowledgeEntityList(content) + .total(listEntity.getTotalElements() == null ? 0 : listEntity.getTotalElements()) + .build(); + } + + private DataMateKnowledgeListEntity executeQuery(String apiKey, String repoName, int page, int size) { + DataMateKnowledgeListQueryParam queryParam = DataMateKnowledgeListQueryParam.builder() + .name(repoName) + .page(page) + .size(size) + .build(); + return this.knowledgeBaseManager.listRepos(apiKey, queryParam); + } +} diff --git a/app-builder/plugins/data-mate-knowledge/src/main/resources/application.yml b/app-builder/plugins/data-mate-knowledge/src/main/resources/application.yml new file mode 100644 index 000000000..951590cf2 --- /dev/null +++ b/app-builder/plugins/data-mate-knowledge/src/main/resources/application.yml @@ -0,0 +1,12 @@ +fit: + beans: + packages: + - 'modelengine.fit.jade.datamate.knowledge' +# 默认使用 ME 集群内 DataMate 服务名，可通过环境变量或 profile 覆盖 +# timeout: 访问超时时间（秒），默认 30，可按场景自定义 +datamate: + url: + list: 'https://datamate-gateway:8080/api/knowledge-base/list' + retrieve: 'https://datamate-gateway:8080/api/knowledge-base/retrieve' + # timeout: 30 + diff --git a/app-builder/plugins/data-mate-knowledge/src/test/java/modelengine/fit/jade/datamate/knowledge/service/DataMateKnowledgeRepoServiceImplTest.java b/app-builder/plugins/data-mate-knowledge/src/test/java/modelengine/fit/jade/datamate/knowledge/service/DataMateKnowledgeRepoServiceImplTest.java new file mode 100644 index 000000000..ac6a83a7b --- /dev/null +++ b/app-builder/plugins/data-mate-knowledge/src/test/java/modelengine/fit/jade/datamate/knowledge/service/DataMateKnowledgeRepoServiceImplTest.java @@ -0,0 +1,267 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. + * This file is a part of the ModelEngine Project. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package modelengine.fit.jade.datamate.knowledge.service; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.when; + +import modelengine.fit.jade.datamate.knowledge.entity.DataMateKnowledgeEntity; +import modelengine.fit.jade.datamate.knowledge.entity.DataMateKnowledgeListEntity; +import modelengine.fit.jade.datamate.knowledge.entity.DataMateRetrievalChunksEntity; +import modelengine.fit.jade.datamate.knowledge.entity.DataMateRetrievalResult; +import modelengine.fit.jade.datamate.knowledge.external.DataMateKnowledgeBaseManager; +import modelengine.fitframework.annotation.Fit; +import modelengine.fitframework.test.annotation.FitTestWithJunit; +import modelengine.fitframework.test.annotation.Mock; +import modelengine.jade.common.vo.PageVo; +import modelengine.jade.knowledge.KnowledgeProperty; +import modelengine.jade.knowledge.document.KnowledgeDocument; +import modelengine.jade.knowledge.KnowledgeRepo; +import modelengine.jade.knowledge.KnowledgeI18nInfo; +import modelengine.jade.knowledge.KnowledgeI18nService; +import modelengine.jade.knowledge.KnowledgeOption; +import modelengine.jade.knowledge.ListRepoQueryParam; +import modelengine.jade.knowledge.ReferenceLimit; +import modelengine.jade.knowledge.enums.FilterType; +import modelengine.jade.knowledge.enums.IndexType; +import modelengine.jade.knowledge.support.FlatKnowledgeOption; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.List; + +/** + * {@link DataMateKnowledgeRepoServiceImpl} 的测试类。 + * 不使用 @Nested，确保 FitTestWithJunit 能正确注入 @Mock 与 @Fit。 + */ +@FitTestWithJunit(includeClasses = {DataMateKnowledgeRepoServiceImpl.class}) +class DataMateKnowledgeRepoServiceImplTest { + + private static final String API_KEY = "test-api-key"; + + @Mock + private DataMateKnowledgeBaseManager knowledgeBaseManager; + + @Mock + private KnowledgeI18nService knowledgeI18nService; + + @Fit + private DataMateKnowledgeRepoServiceImpl knowledgeRepoService; + + @Test + @DisplayName("listRepos - 查询知识库列表成功") + void shouldReturnPageWhenListRepos() { + DataMateKnowledgeEntity entity = DataMateKnowledgeEntity.builder() + .id("kb-1") + .name("测试知识库") + .description("描述") + .createdAt("2025-12-15T10:00:00") + .embeddingModel("embedding-v1") + .build(); + + DataMateKnowledgeListEntity listEntity = DataMateKnowledgeListEntity.builder() + .page(0) + .size(10) + .totalElements(1) + .totalPages(1) + .content(Collections.singletonList(entity)) + .build(); + + ListRepoQueryParam param = new ListRepoQueryParam(); + param.setPageIndex(1); + param.setPageSize(10); + param.setRepoName("test"); + + when(knowledgeBaseManager.listRepos(anyString(), any())).thenReturn(listEntity); + + PageVo result = knowledgeRepoService.listRepos(API_KEY, param); + + assertThat(result.getTotal()).isEqualTo(1); + assertThat(result.getItems()).hasSize(1); + assertThat(result.getItems().get(0)) + .extracting(KnowledgeRepo::id, KnowledgeRepo::name, KnowledgeRepo::description, KnowledgeRepo::type) + .containsExactly("kb-1", "测试知识库", "描述", "embedding-v1"); + } + + @Test + @DisplayName("listRepos - 列表为空时返回空分页") + void shouldReturnEmptyPageWhenContentEmpty() { + DataMateKnowledgeListEntity listEntity = DataMateKnowledgeListEntity.builder() + .page(0) + .size(10) + .totalElements(0) + .totalPages(0) + .content(Collections.emptyList()) + .build(); + + ListRepoQueryParam param = new ListRepoQueryParam(); + param.setPageIndex(1); + param.setPageSize(10); + + when(knowledgeBaseManager.listRepos(anyString(), any())).thenReturn(listEntity); + + PageVo result = knowledgeRepoService.listRepos(API_KEY, param); + + assertThat(result.getTotal()).isEqualTo(0); + assertThat(result.getItems()).isEmpty(); + } + + @Test + @DisplayName("listRepos - content 为 null 时按空列表处理") + void shouldReturnEmptyPageWhenContentNull() { + DataMateKnowledgeListEntity listEntity = DataMateKnowledgeListEntity.builder() + .page(0) + .size(10) + .totalElements(0) + .totalPages(0) + .content(null) + .build(); + + ListRepoQueryParam param = new ListRepoQueryParam(); + param.setPageIndex(1); + param.setPageSize(10); + + when(knowledgeBaseManager.listRepos(anyString(), any())).thenReturn(listEntity); + + PageVo result = knowledgeRepoService.listRepos(API_KEY, param); + + assertThat(result.getTotal()).isEqualTo(0); + assertThat(result.getItems()).isEmpty(); + } + + @Test + @DisplayName("listRepos - totalElements 为 null 时 total 为 0") + void shouldUseZeroTotalWhenTotalElementsNull() { + DataMateKnowledgeListEntity listEntity = DataMateKnowledgeListEntity.builder() + .page(0) + .size(10) + .totalElements(null) + .totalPages(null) + .content(Collections.emptyList()) + .build(); + + ListRepoQueryParam param = new ListRepoQueryParam(); + param.setPageIndex(1); + param.setPageSize(10); + + when(knowledgeBaseManager.listRepos(anyString(), any())).thenReturn(listEntity); + + PageVo result = knowledgeRepoService.listRepos(API_KEY, param); + + assertThat(result.getTotal()).isEqualTo(0); + } + + @Test + @DisplayName("listRepos - param 为 null 时抛出异常") + void shouldThrowWhenListReposParamNull() { + assertThatThrownBy(() -> knowledgeRepoService.listRepos(API_KEY, null)) + .hasMessageContaining("query param"); + } + + @Test + @DisplayName("getProperty - 获取检索配置成功") + void shouldReturnPropertyWhenGetProperty() { + when(knowledgeI18nService.localizeText(any(IndexType.class))) + .thenReturn(new KnowledgeI18nInfo("语义检索", "描述")); + doAnswer(inv -> { + Object arg = inv.getArgument(0); + if (arg == FilterType.REFERENCE_TOP_K) { + return new KnowledgeI18nInfo("引用上限", "最大召回条数"); + } + if (arg == FilterType.SIMILARITY_THRESHOLD) { + return new KnowledgeI18nInfo("最低相关度", "相似度阈值"); + } + return new KnowledgeI18nInfo("", ""); + }).when(knowledgeI18nService).localizeText(any(FilterType.class)); + when(knowledgeI18nService.localizeText("rerankParam")).thenReturn("结果重排"); + when(knowledgeI18nService.localizeText("rerankParam.description")).thenReturn("重排描述"); + + KnowledgeProperty property = knowledgeRepoService.getProperty(API_KEY); + + assertThat(property.getIndexType()).hasSize(3); + assertThat(property.getIndexType()) + .extracting(KnowledgeProperty.IndexInfo::type) + .containsExactly(IndexType.SEMANTIC.value(), IndexType.FULL_TEXT.value(), IndexType.HYBRID.value()); + assertThat(property.getFilterConfig()).hasSize(2); + assertThat(property.getRerankConfig()).hasSize(1); + } + + @Test + @DisplayName("retrieve - 检索知识库成功") + void shouldReturnDocumentsWhenRetrieve() { + DataMateRetrievalChunksEntity.RagChunk ragChunk = new DataMateRetrievalChunksEntity.RagChunk(); + ragChunk.setId("chunk-1"); + ragChunk.setText("检索到的文本内容"); + ragChunk.setMetadata("{\"original_file_id\":\"file-1\",\"file_name\":\"doc.pdf\"}"); + + DataMateRetrievalChunksEntity chunk = new DataMateRetrievalChunksEntity(); + chunk.setEntity(ragChunk); + chunk.setScore(0.85); + chunk.setPrimaryKey("pk-1"); + + DataMateRetrievalResult result = new DataMateRetrievalResult(); + result.setData(Collections.singletonList(chunk)); + + ReferenceLimit referenceLimit = new ReferenceLimit(); + referenceLimit.setValue(3); + referenceLimit.setType(FilterType.REFERENCE_TOP_K.value()); + + FlatKnowledgeOption option = new FlatKnowledgeOption(KnowledgeOption.custom() + .query("用户问题") + .repoIds(Collections.singletonList("kb-1")) + .indexType(IndexType.SEMANTIC) + .similarityThreshold(0.5f) + .referenceLimit(referenceLimit) + .build()); + + when(knowledgeBaseManager.retrieve(anyString(), any())).thenReturn(result); + + List documents = knowledgeRepoService.retrieve(API_KEY, option); + + assertThat(documents).hasSize(1); + assertThat(documents.get(0).getId()).isEqualTo("chunk-1"); + assertThat(documents.get(0).getText()).isEqualTo("检索到的文本内容"); + assertThat(documents.get(0).getScore()).isEqualTo(0.85); + assertThat(documents.get(0).getMetadata()) + .containsEntry("fileId", "file-1") + .containsEntry("fileName", "doc.pdf") + .containsEntry("primaryKey", "pk-1"); + } + + @Test + @DisplayName("retrieve - 检索结果 data 为 null 时返回空列表") + void shouldReturnEmptyListWhenDataNull() { + DataMateRetrievalResult result = new DataMateRetrievalResult(); + result.setData(null); + + FlatKnowledgeOption option = new FlatKnowledgeOption(KnowledgeOption.custom() + .query("query") + .repoIds(Collections.emptyList()) + .indexType(IndexType.SEMANTIC) + .referenceLimit(new ReferenceLimit()) + .build()); + + when(knowledgeBaseManager.retrieve(anyString(), any())).thenReturn(result); + + List documents = knowledgeRepoService.retrieve(API_KEY, option); + + assertThat(documents).isEmpty(); + } + + @Test + @DisplayName("retrieve - option 为 null 时抛出异常") + void shouldThrowWhenRetrieveOptionNull() { + assertThatThrownBy(() -> knowledgeRepoService.retrieve(API_KEY, null)) + .hasMessageContaining("knowledge option"); + } +} diff --git a/app-builder/plugins/knowledge-manager/src/main/resources/application.yml b/app-builder/plugins/knowledge-manager/src/main/resources/application.yml index 804c451b0..f81dd2fc8 100644 --- a/app-builder/plugins/knowledge-manager/src/main/resources/application.yml +++ b/app-builder/plugins/knowledge-manager/src/main/resources/application.yml @@ -32,4 +32,7 @@ knowledge: support-list: - groupId: qianfanKnowledge name: 百度千帆知识库 - description: 百度千帆知识库 \ No newline at end of file + description: 百度千帆知识库 + - groupId: dataMateKnowledge + name: DataMate知识库 + description: DataMate知识库 \ No newline at end of file diff --git a/app-builder/plugins/pom.xml b/app-builder/plugins/pom.xml index d89361406..e16f10165 100644 --- a/app-builder/plugins/pom.xml +++ b/app-builder/plugins/pom.xml @@ -39,6 +39,7 @@ app-base app-metrics app-metrics-influxdb + data-mate-knowledge eval-dataset eval-task flow-graph-db-driver