智能题库系统的数据结构：知识点的向量化表示与相似题检索

智能题库系统的数据结构知识点的向量化表示与相似题检索一、这题我做过三遍了重复刷题如何拖垮学习效率一个AI题库有50万道数学题。当学生某题做错后系统需要推荐3道知识点相同但不会一模一样的相似题。如果只用知识点标签匹配WHERE knowledge_point 二次函数求最值返回的30道题中有28道上个月刚做过——学生看到题目的瞬间就想起了答案完全失去了练习价值。相似但不同的检索需求需要两个层面的匹配知识点层面考察的知识点必须相同或高度相关精确匹配题目内容层面题目的数值、情境、表述方式要有差异语义相似但非相同这才是向量检索真正的用武之地。将题目编码成768维向量余弦相似度在0.85-0.95之间的题目就是同类不同题。二、题库的向量化存储架构三、实现方案MySQL题目基础表CREATE TABLE questions ( id BIGINT AUTO_INCREMENT PRIMARY KEY, question_uuid CHAR(32) NOT NULL UNIQUE, subject ENUM(MATH,PHYSICS,CHEM,BIOLOGY,ENGLISH), knowledge_points JSON, -- [方程,二次函数,求根公式] difficulty DECIMAL(3,2), -- 0.00-1.00 question_type ENUM(CHOICE,FILL,PROOF,CALCULATION), question_text TEXT NOT NULL, -- 题目文本 answer TEXT, explanation TEXT, options JSON, -- 选择题选项 embedding_id VARCHAR(64), -- 关联到Milvus的向量ID usage_count INT DEFAULT 0, correct_rate DECIMAL(4,3), avg_time_sec INT, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, INDEX idx_knowledge ((CAST(knowledge_points-$[0] AS CHAR(64)))), INDEX idx_difficulty (difficulty), INDEX idx_subject_type (subject, question_type) ) ENGINEInnoDB;向量化与检索实现from sentence_transformers import SentenceTransformer from pymilvus import Collection, connections, FieldSchema, CollectionSchema, DataType class QuestionVectorStore: def __init__(self): self.encoder SentenceTransformer( paraphrase-multilingual-MiniLM-L12-v2 ) connections.connect(hostlocalhost, port19530) self._init_collection() def _init_collection(self): 初始化Milvus集合 fields [ FieldSchema(nameid, dtypeDataType.INT64, is_primaryTrue, auto_idTrue), FieldSchema(namequestion_uuid, dtypeDataType.VARCHAR, max_length32), FieldSchema(nameembedding, dtypeDataType.FLOAT_VECTOR, dim384), FieldSchema(namesubject, dtypeDataType.VARCHAR, max_length16), FieldSchema(namedifficulty, dtypeDataType.FLOAT), ] schema CollectionSchema(fields, 题库向量集合) self.collection Collection(questions, schema) # 创建IVF_FLAT索引 index_params { metric_type: IP, index_type: IVF_FLAT, params: {nlist: 1024} } self.collection.create_index(embedding, index_params) self.collection.load() def embed_question(self, question_text: str, knowledge_points: list) - list: 将题目编码为向量 # 拼接题目文本和知识点增强语义 enhanced_text ( f知识点: {, .join(knowledge_points)}. f题目: {question_text} ) embedding self.encoder.encode(enhanced_text).tolist() return embedding def insert_question(self, question_uuid: str, question_text: str, knowledge_points: list, subject: str, difficulty: float): 插入单条题目的向量 embedding self.embed_question(question_text, knowledge_points) try: self.collection.insert([{ question_uuid: question_uuid, embedding: embedding, subject: subject, difficulty: difficulty }]) self.collection.flush() except Exception as e: raise VectorInsertException( f向量插入失败: {question_uuid}, e ) def search_similar(self, question_uuid: str, knowledge_point: str, subject: str, difficulty: float, top_k: int 20) - list: 检索相似题目排除原题 # 获取原题的向量作为查询向量 try: results self.collection.query( exprfquestion_uuid {question_uuid}, output_fields[embedding, question_uuid] ) if not results: return [] query_vector results[0][embedding] except Exception as e: raise VectorSearchException( f获取查询向量失败: {question_uuid}, e ) # 检索 search_params { metric_type: IP, params: {nprobe: 32} } try: results self.collection.search( data[query_vector], anns_fieldembedding, paramsearch_params, limittop_k * 3, # 多获取一些用于后过滤 exprfsubject {subject}, # 同学科过滤 output_fields[question_uuid, difficulty] ) # 后处理过滤 filtered [] for hit in results[0]: qid hit.entity.get(question_uuid) # 排除原题 if qid question_uuid: continue # 排除过于相似0.98可能是重复题 if hit.score 0.98: continue # 难度过滤难度差在±0.2范围内 q_difficulty hit.entity.get(difficulty, 0.5) if abs(q_difficulty - difficulty) 0.2: continue filtered.append({ question_uuid: qid, similarity: float(hit.score), difficulty: q_difficulty }) if len(filtered) top_k: break return filtered except Exception as e: raise VectorSearchException( f相似题检索失败: {question_uuid}, e )四、题库向量化的三个边界边界一多步骤题的语义丢失。先化简再求值这类题目由两个子任务构成单一的768维向量可能丢失子任务之间的逻辑关系。解决方案是将题目拆分为子步骤分别编码检索时对子步骤向量做加权平均。边界二数学公式的文本表示。LaTeX公式$$x\frac{-b\pm\sqrt{b^2-4ac}}{2a}$$如果直接输入BERT模型只看到原始字符串而非数学语义。需要使用专门的数学公式编码器如MathBERT或在预处理时将公式转换为自然语言描述。边界三题型差异导致的不公平比较。选择题和证明题的题目长度差异巨大。选择题可能只有30个字证明题可能有200个字。在向量空间中长文本的向量模长天然更大会压制短文本的相似度计算。必须做L2归一化。五、总结智能题库的相似题检索是典型的标量过滤向量检索场景知识点标签提供精确过滤同知识点向量相似度提供语义泛化不同表述的同类题目。Milvus原生支持标量过滤ANN检索的组合查询非常适合这个场景。一句话总结「知识点保证精准度向量保证多样性」。本文属于「行业场景与项目复盘」系列探讨智能题库中知识点的向量化表示与相似题检索方案。