!16 文档向量化demo

Merge pull request !16 from 小新/master-jdk21-ai

yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/DocService.java

@@ -0,0 +1,16 @@
+package cn.iocoder.yudao.module.ai.service.knowledge;
+
+/**
+ * AI 知识库 Service 接口
+ *
+ * @author xiaoxin
+ */
+public interface DocService {
+
+
+    /**
+     * 向量化文档
+     */
+    void embeddingDoc();
+
+}

yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/DocServiceImpl.java

@@ -0,0 +1,44 @@
+package cn.iocoder.yudao.module.ai.service.knowledge;
+
+import jakarta.annotation.Resource;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.ai.document.Document;
+import org.springframework.ai.reader.tika.TikaDocumentReader;
+import org.springframework.ai.transformer.splitter.TokenTextSplitter;
+import org.springframework.ai.vectorstore.RedisVectorStore;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+
+import java.util.List;
+
+/**
+ * AI 知识库 Service 实现类
+ *
+ * @author xiaoxin
+ */
+@Service
+@Slf4j
+public class DocServiceImpl implements DocService {
+
+    @Resource
+    RedisVectorStore vectorStore;
+    @Resource
+    TokenTextSplitter tokenTextSplitter;
+
+    // TODO @xin 临时测试用，后续删
+    @Value("classpath:/webapp/test/Fel.pdf")
+    private org.springframework.core.io.Resource data;
+
+
+    @Override
+    public void embeddingDoc() {
+        // 读取文件
+        org.springframework.core.io.Resource file = data;
+        TikaDocumentReader loader = new TikaDocumentReader(file);
+        List<Document> documents = loader.get();
+        // 文档分段
+        List<Document> segments = tokenTextSplitter.apply(documents);
+        // 向量化并存储
+        vectorStore.add(segments);
+    }
+}

yudao-module-ai/yudao-spring-boot-starter-ai/pom.xml

@@ -40,6 +40,28 @@
             <version>${spring-ai.version}</version>
         </dependency>
 
+        <dependency>
+            <groupId>org.springframework.ai</groupId>
+            <artifactId>spring-ai-transformers-spring-boot-starter</artifactId>
+            <version>${spring-ai.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.ai</groupId>
+            <artifactId>spring-ai-tika-document-reader</artifactId>
+            <version>${spring-ai.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.ai</groupId>
+            <artifactId>spring-ai-redis-store</artifactId>
+            <version>${spring-ai.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.data</groupId>
+            <artifactId>spring-data-redis</artifactId>
+            <optional>true</optional>
+        </dependency>
+
+
         <dependency>
             <groupId>cn.iocoder.boot</groupId>
             <artifactId>yudao-common</artifactId>

yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/cn/iocoder/yudao/framework/ai/config/YudaoAiAutoConfiguration.java

@@ -10,11 +10,18 @@ import cn.iocoder.yudao.framework.ai.core.model.xinghuo.XingHuoChatModel;
 import cn.iocoder.yudao.framework.ai.core.model.xinghuo.XingHuoChatOptions;
 import com.alibaba.cloud.ai.tongyi.TongYiAutoConfiguration;
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.ai.autoconfigure.vectorstore.redis.RedisVectorStoreProperties;
+import org.springframework.ai.document.MetadataMode;
+import org.springframework.ai.transformer.splitter.TokenTextSplitter;
+import org.springframework.ai.transformers.TransformersEmbeddingModel;
+import org.springframework.ai.vectorstore.RedisVectorStore;
 import org.springframework.boot.autoconfigure.AutoConfiguration;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.boot.autoconfigure.data.redis.RedisProperties;
 import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Import;
+import redis.clients.jedis.JedisPooled;
 
 /**
  * 芋道 AI 自动配置
@@ -73,4 +80,33 @@ public class YudaoAiAutoConfiguration {
         return new SunoApi(yudaoAiProperties.getSuno().getBaseUrl());
     }
 
+    // ========== rag 相关 ==========
+    @Bean
+    public TransformersEmbeddingModel transformersEmbeddingClient() {
+        return new TransformersEmbeddingModel(MetadataMode.EMBED);
+    }
+
+    /**
+     * 我们启动有加载很多 Embedding 模型，不晓得取哪个好，先 new 个 TransformersEmbeddingModel 跑
+     */
+    @Bean
+    public RedisVectorStore vectorStore(TransformersEmbeddingModel transformersEmbeddingModel, RedisVectorStoreProperties properties,
+                                        RedisProperties redisProperties) {
+        var config = RedisVectorStore.RedisVectorStoreConfig.builder()
+                .withIndexName(properties.getIndex())
+                .withPrefix(properties.getPrefix())
+                .build();
+
+        RedisVectorStore redisVectorStore = new RedisVectorStore(config, transformersEmbeddingModel,
+                new JedisPooled(redisProperties.getHost(), redisProperties.getPort()),
+                properties.isInitializeSchema());
+        redisVectorStore.afterPropertiesSet();
+        return redisVectorStore;
+    }
+
+    @Bean
+    public TokenTextSplitter tokenTextSplitter() {
+        return new TokenTextSplitter(500, 100, 5, 10000, true);
+    }
+
 }

yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/org/springframework/ai/autoconfigure/vectorstore/redis/RedisVectorStoreAutoConfiguration.java

@@ -0,0 +1,59 @@
+/*
+ * Copyright 2023 - 2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.ai.autoconfigure.vectorstore.redis;
+
+import org.springframework.ai.embedding.EmbeddingModel;
+import org.springframework.ai.vectorstore.RedisVectorStore;
+import org.springframework.ai.vectorstore.RedisVectorStore.RedisVectorStoreConfig;
+import org.springframework.boot.autoconfigure.AutoConfiguration;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
+import org.springframework.boot.autoconfigure.data.redis.RedisAutoConfiguration;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
+import org.springframework.context.annotation.Bean;
+import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
+import redis.clients.jedis.JedisPooled;
+
+/**
+ * TODO @xin 先拿 spring-ai 最新代码覆盖，1.0.0-M1 跟 redis 自动配置会冲突
+ *
+ * @author Christian Tzolov
+ * @author Eddú Meléndez
+ */
+@AutoConfiguration(after = RedisAutoConfiguration.class)
+@ConditionalOnClass({JedisPooled.class, JedisConnectionFactory.class, RedisVectorStore.class, EmbeddingModel.class})
+//@ConditionalOnBean(JedisConnectionFactory.class)
+@EnableConfigurationProperties(RedisVectorStoreProperties.class)
+public class RedisVectorStoreAutoConfiguration {
+
+
+
+    @Bean
+    @ConditionalOnMissingBean
+    public RedisVectorStore vectorStore(EmbeddingModel embeddingModel, RedisVectorStoreProperties properties,
+                                        JedisConnectionFactory jedisConnectionFactory) {
+
+        var config = RedisVectorStoreConfig.builder()
+                .withIndexName(properties.getIndex())
+                .withPrefix(properties.getPrefix())
+                .build();
+
+        return new RedisVectorStore(config, embeddingModel,
+                new JedisPooled(jedisConnectionFactory.getHostName(), jedisConnectionFactory.getPort()),
+                properties.isInitializeSchema());
+    }
+
+}

yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/org/springframework/ai/vectorstore/RedisVectorStore.java

@@ -0,0 +1,456 @@
+/*
+ * Copyright 2023 - 2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.ai.vectorstore;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.ai.document.Document;
+import org.springframework.ai.embedding.EmbeddingModel;
+import org.springframework.ai.vectorstore.filter.FilterExpressionConverter;
+import org.springframework.beans.factory.InitializingBean;
+import org.springframework.util.Assert;
+import org.springframework.util.CollectionUtils;
+import redis.clients.jedis.JedisPooled;
+import redis.clients.jedis.Pipeline;
+import redis.clients.jedis.json.Path2;
+import redis.clients.jedis.search.*;
+import redis.clients.jedis.search.Schema.FieldType;
+import redis.clients.jedis.search.schemafields.*;
+import redis.clients.jedis.search.schemafields.VectorField.VectorAlgorithm;
+
+import java.text.MessageFormat;
+import java.util.*;
+import java.util.function.Function;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+
+/**
+ * The RedisVectorStore is for managing and querying vector data in a Redis database. It
+ * offers functionalities like adding, deleting, and performing similarity searches on
+ * documents.
+ *
+ * The store utilizes RedisJSON and RedisSearch to handle JSON documents and to index and
+ * search vector data. It supports various vector algorithms (e.g., FLAT, HSNW) for
+ * efficient similarity searches. Additionally, it allows for custom metadata fields in
+ * the documents to be stored alongside the vector and content data.
+ *
+ * This class requires a RedisVectorStoreConfig configuration object for initialization,
+ * which includes settings like Redis URI, index name, field names, and vector algorithms.
+ * It also requires an EmbeddingModel to convert documents into embeddings before storing
+ * them.
+ *
+ * @author Julien Ruaux
+ * @author Christian Tzolov
+ * @author Eddú Meléndez
+ * @see VectorStore
+ * @see RedisVectorStoreConfig
+ * @see EmbeddingModel
+ */
+public class RedisVectorStore implements VectorStore, InitializingBean {
+
+    public enum Algorithm {
+
+        FLAT, HSNW
+
+    }
+
+    public record MetadataField(String name, FieldType fieldType) {
+
+        public static MetadataField text(String name) {
+            return new MetadataField(name, FieldType.TEXT);
+        }
+
+        public static MetadataField numeric(String name) {
+            return new MetadataField(name, FieldType.NUMERIC);
+        }
+
+        public static MetadataField tag(String name) {
+            return new MetadataField(name, FieldType.TAG);
+        }
+
+    }
+
+    /**
+     * Configuration for the Redis vector store.
+     */
+    public static final class RedisVectorStoreConfig {
+
+        private final String indexName;
+
+        private final String prefix;
+
+        private final String contentFieldName;
+
+        private final String embeddingFieldName;
+
+        private final Algorithm vectorAlgorithm;
+
+        private final List<MetadataField> metadataFields;
+
+        private RedisVectorStoreConfig() {
+            this(builder());
+        }
+
+        private RedisVectorStoreConfig(Builder builder) {
+            this.indexName = builder.indexName;
+            this.prefix = builder.prefix;
+            this.contentFieldName = builder.contentFieldName;
+            this.embeddingFieldName = builder.embeddingFieldName;
+            this.vectorAlgorithm = builder.vectorAlgorithm;
+            this.metadataFields = builder.metadataFields;
+        }
+
+        /**
+         * Start building a new configuration.
+         * @return The entry point for creating a new configuration.
+         */
+        public static Builder builder() {
+
+            return new Builder();
+        }
+
+        /**
+         * {@return the default config}
+         */
+        public static RedisVectorStoreConfig defaultConfig() {
+
+            return builder().build();
+        }
+
+        public static class Builder {
+
+            private String indexName = DEFAULT_INDEX_NAME;
+
+            private String prefix = DEFAULT_PREFIX;
+
+            private String contentFieldName = DEFAULT_CONTENT_FIELD_NAME;
+
+            private String embeddingFieldName = DEFAULT_EMBEDDING_FIELD_NAME;
+
+            private Algorithm vectorAlgorithm = DEFAULT_VECTOR_ALGORITHM;
+
+            private List<MetadataField> metadataFields = new ArrayList<>();
+
+            private Builder() {
+            }
+
+            /**
+             * Configures the Redis index name to use.
+             * @param name the index name to use
+             * @return this builder
+             */
+            public Builder withIndexName(String name) {
+                this.indexName = name;
+                return this;
+            }
+
+            /**
+             * Configures the Redis key prefix to use (default: "embedding:").
+             * @param prefix the prefix to use
+             * @return this builder
+             */
+            public Builder withPrefix(String prefix) {
+                this.prefix = prefix;
+                return this;
+            }
+
+            /**
+             * Configures the Redis content field name to use.
+             * @param name the content field name to use
+             * @return this builder
+             */
+            public Builder withContentFieldName(String name) {
+                this.contentFieldName = name;
+                return this;
+            }
+
+            /**
+             * Configures the Redis embedding field name to use.
+             * @param name the embedding field name to use
+             * @return this builder
+             */
+            public Builder withEmbeddingFieldName(String name) {
+                this.embeddingFieldName = name;
+                return this;
+            }
+
+            /**
+             * Configures the Redis vector algorithmto use.
+             * @param algorithm the vector algorithm to use
+             * @return this builder
+             */
+            public Builder withVectorAlgorithm(Algorithm algorithm) {
+                this.vectorAlgorithm = algorithm;
+                return this;
+            }
+
+            public Builder withMetadataFields(MetadataField... fields) {
+                return withMetadataFields(Arrays.asList(fields));
+            }
+
+            public Builder withMetadataFields(List<MetadataField> fields) {
+                this.metadataFields = fields;
+                return this;
+            }
+
+            /**
+             * {@return the immutable configuration}
+             */
+            public RedisVectorStoreConfig build() {
+
+                return new RedisVectorStoreConfig(this);
+            }
+
+        }
+
+    }
+
+    private final boolean initializeSchema;
+
+    public static final String DEFAULT_INDEX_NAME = "spring-ai-index";
+
+    public static final String DEFAULT_CONTENT_FIELD_NAME = "content";
+
+    public static final String DEFAULT_EMBEDDING_FIELD_NAME = "embedding";
+
+    public static final String DEFAULT_PREFIX = "embedding:";
+
+    public static final Algorithm DEFAULT_VECTOR_ALGORITHM = Algorithm.HSNW;
+
+    private static final String QUERY_FORMAT = "%s=>[KNN %s @%s $%s AS %s]";
+
+    private static final Path2 JSON_SET_PATH = Path2.of("$");
+
+    private static final String JSON_PATH_PREFIX = "$.";
+
+    private static final Logger logger = LoggerFactory.getLogger(RedisVectorStore.class);
+
+    private static final Predicate<Object> RESPONSE_OK = Predicate.isEqual("OK");
+
+    private static final Predicate<Object> RESPONSE_DEL_OK = Predicate.isEqual(1l);
+
+    private static final String VECTOR_TYPE_FLOAT32 = "FLOAT32";
+
+    private static final String EMBEDDING_PARAM_NAME = "BLOB";
+
+    public static final String DISTANCE_FIELD_NAME = "vector_score";
+
+    private static final String DEFAULT_DISTANCE_METRIC = "COSINE";
+
+    private final JedisPooled jedis;
+
+    private final EmbeddingModel embeddingModel;
+
+    private final RedisVectorStoreConfig config;
+
+    private FilterExpressionConverter filterExpressionConverter;
+
+    public RedisVectorStore(RedisVectorStoreConfig config, EmbeddingModel embeddingModel, JedisPooled jedis,
+                            boolean initializeSchema) {
+
+        Assert.notNull(config, "Config must not be null");
+        Assert.notNull(embeddingModel, "Embedding model must not be null");
+        this.initializeSchema = initializeSchema;
+
+        this.jedis = jedis;
+        this.embeddingModel = embeddingModel;
+        this.config = config;
+        this.filterExpressionConverter = new RedisFilterExpressionConverter(this.config.metadataFields);
+    }
+
+    public JedisPooled getJedis() {
+        return this.jedis;
+    }
+
+    @Override
+    public void add(List<Document> documents) {
+        try (Pipeline pipeline = this.jedis.pipelined()) {
+            for (Document document : documents) {
+                var embedding = this.embeddingModel.embed(document);
+                document.setEmbedding(embedding);
+
+                var fields = new HashMap<String, Object>();
+                fields.put(this.config.embeddingFieldName, embedding);
+                fields.put(this.config.contentFieldName, document.getContent());
+                fields.putAll(document.getMetadata());
+                pipeline.jsonSetWithEscape(key(document.getId()), JSON_SET_PATH, fields);
+            }
+            List<Object> responses = pipeline.syncAndReturnAll();
+            Optional<Object> errResponse = responses.stream().filter(Predicate.not(RESPONSE_OK)).findAny();
+            if (errResponse.isPresent()) {
+                String message = MessageFormat.format("Could not add document: {0}", errResponse.get());
+                if (logger.isErrorEnabled()) {
+                    logger.error(message);
+                }
+                throw new RuntimeException(message);
+            }
+        }
+    }
+
+    private String key(String id) {
+        return this.config.prefix + id;
+    }
+
+    @Override
+    public Optional<Boolean> delete(List<String> idList) {
+        try (Pipeline pipeline = this.jedis.pipelined()) {
+            for (String id : idList) {
+                pipeline.jsonDel(key(id));
+            }
+            List<Object> responses = pipeline.syncAndReturnAll();
+            Optional<Object> errResponse = responses.stream().filter(Predicate.not(RESPONSE_DEL_OK)).findAny();
+            if (errResponse.isPresent()) {
+                if (logger.isErrorEnabled()) {
+                    logger.error("Could not delete document: {}", errResponse.get());
+                }
+                return Optional.of(false);
+            }
+            return Optional.of(true);
+        }
+    }
+
+    @Override
+    public List<Document> similaritySearch(SearchRequest request) {
+
+        Assert.isTrue(request.getTopK() > 0, "The number of documents to returned must be greater than zero");
+        Assert.isTrue(request.getSimilarityThreshold() >= 0 && request.getSimilarityThreshold() <= 1,
+                "The similarity score is bounded between 0 and 1; least to most similar respectively.");
+
+        String filter = nativeExpressionFilter(request);
+
+        String queryString = String.format(QUERY_FORMAT, filter, request.getTopK(), this.config.embeddingFieldName,
+                EMBEDDING_PARAM_NAME, DISTANCE_FIELD_NAME);
+
+        List<String> returnFields = new ArrayList<>();
+        this.config.metadataFields.stream().map(MetadataField::name).forEach(returnFields::add);
+        returnFields.add(this.config.embeddingFieldName);
+        returnFields.add(this.config.contentFieldName);
+        returnFields.add(DISTANCE_FIELD_NAME);
+        var embedding = toFloatArray(this.embeddingModel.embed(request.getQuery()));
+        Query query = new Query(queryString).addParam(EMBEDDING_PARAM_NAME, RediSearchUtil.toByteArray(embedding))
+                .returnFields(returnFields.toArray(new String[0]))
+                .setSortBy(DISTANCE_FIELD_NAME, true)
+                .dialect(2);
+
+        SearchResult result = this.jedis.ftSearch(this.config.indexName, query);
+        return result.getDocuments()
+                .stream()
+                .filter(d -> similarityScore(d) >= request.getSimilarityThreshold())
+                .map(this::toDocument)
+                .toList();
+    }
+
+    private Document toDocument(redis.clients.jedis.search.Document doc) {
+        var id = doc.getId().substring(this.config.prefix.length());
+        var content = doc.hasProperty(this.config.contentFieldName) ? doc.getString(this.config.contentFieldName)
+                : null;
+        Map<String, Object> metadata = this.config.metadataFields.stream()
+                .map(MetadataField::name)
+                .filter(doc::hasProperty)
+                .collect(Collectors.toMap(Function.identity(), doc::getString));
+        metadata.put(DISTANCE_FIELD_NAME, 1 - similarityScore(doc));
+        return new Document(id, content, metadata);
+    }
+
+    private float similarityScore(redis.clients.jedis.search.Document doc) {
+        return (2 - Float.parseFloat(doc.getString(DISTANCE_FIELD_NAME))) / 2;
+    }
+
+    private String nativeExpressionFilter(SearchRequest request) {
+        if (request.getFilterExpression() == null) {
+            return "*";
+        }
+        return "(" + this.filterExpressionConverter.convertExpression(request.getFilterExpression()) + ")";
+    }
+
+    @Override
+    public void afterPropertiesSet() {
+
+        if (!this.initializeSchema) {
+            return;
+        }
+
+        // If index already exists don't do anything
+        if (this.jedis.ftList().contains(this.config.indexName)) {
+            return;
+        }
+
+        String response = this.jedis.ftCreate(this.config.indexName,
+                FTCreateParams.createParams().on(IndexDataType.JSON).addPrefix(this.config.prefix), schemaFields());
+        if (!RESPONSE_OK.test(response)) {
+            String message = MessageFormat.format("Could not create index: {0}", response);
+            throw new RuntimeException(message);
+        }
+    }
+
+    private Iterable<SchemaField> schemaFields() {
+        Map<String, Object> vectorAttrs = new HashMap<>();
+        vectorAttrs.put("DIM", this.embeddingModel.dimensions());
+        vectorAttrs.put("DISTANCE_METRIC", DEFAULT_DISTANCE_METRIC);
+        vectorAttrs.put("TYPE", VECTOR_TYPE_FLOAT32);
+        List<SchemaField> fields = new ArrayList<>();
+        fields.add(TextField.of(jsonPath(this.config.contentFieldName)).as(this.config.contentFieldName).weight(1.0));
+        fields.add(VectorField.builder()
+                .fieldName(jsonPath(this.config.embeddingFieldName))
+                .algorithm(vectorAlgorithm())
+                .attributes(vectorAttrs)
+                .as(this.config.embeddingFieldName)
+                .build());
+
+        if (!CollectionUtils.isEmpty(this.config.metadataFields)) {
+            for (MetadataField field : this.config.metadataFields) {
+                fields.add(schemaField(field));
+            }
+        }
+        return fields;
+    }
+
+    private SchemaField schemaField(MetadataField field) {
+        String fieldName = jsonPath(field.name);
+        switch (field.fieldType) {
+            case NUMERIC:
+                return NumericField.of(fieldName).as(field.name);
+            case TAG:
+                return TagField.of(fieldName).as(field.name);
+            case TEXT:
+                return TextField.of(fieldName).as(field.name);
+            default:
+                throw new IllegalArgumentException(
+                        MessageFormat.format("Field {0} has unsupported type {1}", field.name, field.fieldType));
+        }
+    }
+
+    private VectorAlgorithm vectorAlgorithm() {
+        if (config.vectorAlgorithm == Algorithm.HSNW) {
+            return VectorAlgorithm.HNSW;
+        }
+        return VectorAlgorithm.FLAT;
+    }
+
+    private String jsonPath(String field) {
+        return JSON_PATH_PREFIX + field;
+    }
+
+    private static float[] toFloatArray(List<Double> embeddingDouble) {
+        float[] embeddingFloat = new float[embeddingDouble.size()];
+        int i = 0;
+        for (Double d : embeddingDouble) {
+            embeddingFloat[i++] = d.floatValue();
+        }
+        return embeddingFloat;
+    }
+
+}

yudao-module-ai/yudao-spring-boot-starter-ai/src/main/resources/webapp/test/Fel.pdf

@@ -0,0 +1,311 @@
+                   Fel 表达式引擎
+
+一、名词解释
+
+      Fel：全称是 Fast Expression Language，一种开源表达式引擎。
+      EL：表达式语言，用于求表达式的值。
+      Ast：抽象语法树，一般由语法分析工具生成。1+2*3 会解析结果如下所示：
+
+二、简介
+
+      Fel（Fast Expression Language)在源自于企业项目，设计目标是为了满足不断变化的功能需
+求和性能需求。
+
+      Fel 是开放的，引擎执行中的多个模块都可以扩展或替换。Fel 的执行主要是通过函数实现,
+运算符(+、-等都是 Fel 函数），所有这些函数都是可以替换的，扩展函数也非常简单。
+
+      Fel 有双引擎，同时支持解释执行和编译执行。可以根据性能要求选择执行方式。编译执行
+就是将表达式编译成字节码（生成 java 代码和编译模块都是可以扩展和替换的）
+
+      Fel 基于 Java1.5开发，适用于 Java1.5及以上版本。
+
+1. 特点
+
+      易用性：API 使用简单，语法简洁，和 java 语法很相似。
+      轻量级：整个包只有200多 KB。
+      高 效：目前没有发现有开源的表达式引擎比 Fel 快。
+      扩展性：采用模块化设计，可灵活控制表达式的执行过程。
+      根函数：Fel 支持根函数，“$('Math')”在 Fel 中是常用的使用函数的方式。
+      $函数：通过$函数，Fel 可以方便的调用工具类或对象的方法（并不需要任何附加代码），
+2. 不足
+
+      支持脚本：否。
+
+3. 适应场景
+
+      Fel 适合处理海量数据，Fel 良好的扩展性可以更好的帮助用户处理数据。
+      Fel 同样适用于其他需要使用表达式引擎的地方（如果工作流、公式计算、数据有效性校验
+等等）
+
+三、安装
+
+1. 获取 Fel
+
+      项目主页：http://code.google.com/p/fast-el/
+      下载地址：http://code.google.com/p/fast-el/downloads/list
+
+2. Jdk1.6 环境
+
+      使用：将 fel.jar 加入 classpath 即可。
+      构建 Fel：下载 fel-all.tar.gz，解压后将 src 作为源码文件夹，并且将 lib/antlr-min.jar 加入
+classpath 即可。
+
+3. Jdk1.5 环境：
+
+      与 jdk1.6 环境下的区别在于，需要添加 jdk 内置的 tools.jar 到 classpath。
+
+四、功能
+
+1. 算术表达式：
+
+FelEngine fel= new FelEngineImpl();
+Object result= fel.eval("5000*12+7500");
+System.out.println(result);
+
+输出结果：67500
+2. 变量
+
+使用变量，其代码如下所示：
+
+FelContext ctx= fel.getContext();
+ctx.set("单价", 5000);
+ctx.set("数量", 12);
+ctx.set("运费", 7500);
+Object result= fel.eval("单价*数量+运费");
+System.out.println(result);
+
+输出结果：67500
+
+3. 调用 JAVA 方法
+
+FelEngine fel= new FelEngineImpl();
+FelContext ctx= fel.getContext();
+ctx.set("out", System.out);
+fel.eval("out.println('Hello Everybody'.substring(6))");
+
+输出结果：Everybody
+
+4. 自定义上下文环境
+
+//负责提供气象服务的上下文环境
+FelContext ctx= new AbstractConetxt() {
+
+      public Object get(Object name) {
+            if("天气".equals(name)){
+                  return "晴";
+            }
+            if("温度".equals(name)){
+                  return 25;
+            }
+            return null;
+
+      }
+};
+FelEngine fel= new FelEngineImpl(ctx);
+Object eval = fel.eval("'天气:'+天气+';温度:'+温度");
+System.out.println(eval);
+
+输出结果：天气:晴;温度:25
+
+5. 多层上下文环境（命名空间）
+
+FelEngine fel= new FelEngineImpl();
+String costStr= "成本";
+String priceStr="价格";
+FelContext baseCtx= fel.getContext();
+//父级上下文中设置成本和价格
+baseCtx.set(costStr, 50);
+baseCtx.set(priceStr,100);
+
+String exp= priceStr+"-"+costStr;
+Object baseCost= fel.eval(exp);
+System.out.println("期望利润：" + baseCost);
+
+FelContext ctx= new ContextChain(baseCtx, new MapContext());
+//通货膨胀导致成本增加（子级上下文 中设置成本，会覆盖父级上下文中的成本）
+ctx.set(costStr,50+20 );
+Object allCost= fel.eval(exp, ctx);
+System.out.println("实际利润：" + allCost);
+
+输出结果：
+期望利润：50
+实际利润：30
+
+6. 编译执行
+
+FelEngine fel= new FelEngineImpl();
+FelContext ctx= fel.getContext();
+ctx.set("单价", 5000);
+ctx.set("数量", 12);
+ctx.set("运费", 7500);
+Expression exp= fel.compile("单价*数量+运费",ctx);
+Object result= exp.eval(ctx);
+System.out.println(result);
+
+执行结果：67500
+备注：适合处理海量数据，编译执行的速度基本与 Java 字节码执行速度一样快。
+
+7. 自定义函数
+
+          //定义 hello 函数
+    Function fun= new CommonFunction() {
+
+            public String getName() {
+                  return "hello";
+
+            }
+
+          /*
+               * 调用 hello("xxx")时执行的代码
+             */
+            @Override
+            public Object call(Object[] arguments) {
+
+                  Object msg= null;
+                  if(arguments!= null && arguments.length>0){
+
+                         msg= arguments[0];
+                  }
+                  return ObjectUtils.toString(msg);
+            }
+
+      };
+      FelEngine e= new FelEngineImpl();
+      //添加函数到引擎中。
+      e.addFun(fun);
+      String exp= "hello('fel')";
+      //解释执行
+    Object eval = e.eval(exp);
+      System.out.println("hello "+eval);
+      //编译执行
+    Expression compile= e.compile(exp, null);
+      eval = compile.eval(null);
+      System.out.println("hello "+eval);
+
+执行结果：
+hello fel hello fel
+
+8. 调用静态方法
+
+      如果你觉得上面的自定义函数也麻烦，Fel 提供的$函数可以方便的调用工具类的方法 熟悉
+jQuery 的朋友肯定知道"$"函数的威力。Fel 东施效颦，也实现了一个"$"函数,其作用是获取 class
+和创建对象。结合点操作符，可以轻易的调用工具类或对象的方法。
+
+       //调用 Math.min(1,2)
+ FelEngine.instance.eval("$('Math').min(1,2)");
+ //调用 new Foo().toString();
+ FelEngine.instance.eval("$('com.greenpineyu.test.Foo.new').toString()");
+
+      通过"$('class').method"形式的语法，就可以调用任何等三方类包（commons lang 等）及自
+定义工具类的方法，也可以创建对象，调用对象的方法。如果有需要，还可以直接注册 Java
+Method 到函数管理器中。
+五、安全（始于 0.8 版本）
+
+      为了防止出现“${'System'}.exit(1)”这样的表达式导致系统崩溃。Fel 加入了安全管理器，主
+要是对方法访问进行控制。安全管理器中通过允许访问的方法列表（黑名单）和禁止访问的方
+法列表（白名单）来控制方法访问。将 "java.lang.System. *"加入到黑名单，表示 System 类的
+所以方法都不能访问。将"java.lang.Math. *"加入白名单，表示只能访问 Math 类中的方法。如
+果你不喜欢这个安全管理器，可以自己开发一个，非常简单，只需要实现一个方法就可以了。
+
+六、语法
+
+1. 简介
+
+      Fel 的语法非常简单，基本和 Java 语法没没有区别。Fel 语法是 Java 语法的一个子集，支
+持的运算符有限。如果熟悉语法的话，只需要关注一下 Fel 支持的运算即可。
+
+      Fel 表达式由常量、变量、函数、运算符组成。详见下文。
+
+2. 常量
+
+常量         说明
+Number     1,1.0 等
+Boolean    true,false
+String     'abc',"abc"
+
+3. 变量
+
+变量命名规则与 java 变量相同。变量名由字母、数字、下划线、$组成。
+
+变量         说明
+
+abc 以字母开头
+
+abc 以下划线开头
+
+$abc 以$开头
+
+变量         中文变量名也可以
+
+4. 函数
+
+函数名的语法规则与变量相同
+
+函数         说明
+
+$('Math')  $是变量名，'Math'是参数
+5. 运算符
+
+运算符        类型  优先级     说明
+？：         条件          三元操作符
+           逻辑       优  或操作
+||                  先  与操作
+&&         关系       级  等于、不等于
+==、!=               从  大于、小于、大于等于、小于等于
+>、<、<=、>=  算术       低  加减
++、-        逻辑       到  乘、除、取模
+*、/、%               高  取反操作
+!
+
+七、结构
+
+1. 主要组成部分
+
+      引擎由四部分组成，每个组成部分都可以被替换。组成部分如下所示：
+
+组件         说明
+解析器        将表达式解析成 Ast 节点
+解释执行       负责执行节点
+执行上下文      负责提供变量
+编译器        负责生成代码并编译
+
+2. 主要流程
+
+执行表达式的流程如下所示：
+组件            说明
+解析表达式         默认 Antlr 解析表达式，生成 Ast 节点。
+Ast 节点        由解析器生成的抽象语法树节点
+节点解释器         每一个节点都有一个解释器，负责解释节点。
+编译器           负责生成表达式对应 Java 类并编译，生成 Expression 对象
+
+3. 引擎上下文
+
+      引擎上下文负责存取变量，它是脚本引擎与 Java 对象之间的桥梁。其结构如下所示：
+
+组件            说明
+FelContext    负责存储和保存变量
+MapContext    使用 Map 来保存变量
+ContextChain  上下文链，保存两个上下文的引用。存取变量委托引用的上下文处理。
+
+4. 表达式解析
+
+      解析表达式的过程就是将表达式解析成 Ast 节点的过程，目前 Fel 由 Antlr 负责解析表达式，
+生成 Ast 节点（FelNode),流程如下所示：
+5. 解释执行
+
+      FelNode(Ast 节点）中都包含一个解释器，不同的节点有不同的解释器，解释器负责求节点
+值，流程如下所示：
+
+6. 编译执行
+
+      FelNode(Ast 节点）中都包含一个代码生成器，负责将 Fel 表达式转换成 java 表达式。编译
+模块负责将 java 表达式封装成 Java 类、编译、创建 Expression。再通过调用 Expression.eval 求
+表达式的值。其过程如下所示：
+
+7. 小结
+
+      在上文介绍的 Fel 组件中，绝大多数组件都是可能被替换的。通过在运行时替换一些小粒
+度的组件，就可以灵活控制 Fel 的执行过程，很有实用价值。有些变化较少的组件(Antlr 解析器)
+的替换过程还不是挺方便，随着 Fel 的发展，会慢慢重构，努力使 Fel 转变成让人满意的模块
+化结构。
+

yudao-server/src/main/resources/application.yaml

@@ -153,6 +153,10 @@ spring:
 
 spring:
   ai:
+    vectorstore:
+      redis:
+        index: default-index
+        prefix: "default:"
     qianfan: # 文心一言
       api-key: x0cuLZ7XsaTCU08vuJWO87Lg
       secret-key: R9mYF9dl9KASgi5RUq0FQt3wRisSnOcK