向量数据库配置

向量数据库是 AnythingLLM 实现语义搜索和检索增强生成（RAG）的核心组件。本指南详细介绍如何配置和优化各种向量数据库解决方案。

支持的向量数据库

云端解决方案

Pinecone: 完全托管的向量数据库服务
Weaviate Cloud: 托管的 Weaviate 实例
Qdrant Cloud: 托管的 Qdrant 服务

自托管解决方案

Chroma: 轻量级的开源向量数据库
Weaviate: 开源的向量搜索引擎
Qdrant: 高性能的向量相似性搜索引擎
Milvus: 大规模向量数据库
LanceDB: 嵌入式向量数据库

传统数据库扩展

pgvector: PostgreSQL 向量扩展
Redis Search: Redis 向量搜索模块

Pinecone 配置

基础配置

javascript

// Pinecone 环境变量
PINECONE_API_KEY=your-pinecone-api-key
PINECONE_ENVIRONMENT=us-west1-gcp
PINECONE_INDEX=anythingllm-index

// 高级配置
PINECONE_NAMESPACE=default
PINECONE_TOP_K=10
PINECONE_INCLUDE_METADATA=true

索引创建和管理

javascript

// Pinecone 索引配置
const pineconeConfig = {
  apiKey: process.env.PINECONE_API_KEY,
  environment: process.env.PINECONE_ENVIRONMENT,
  
  // 索引配置
  index: {
    name: "anythingllm-index",
    dimension: 1536,  // OpenAI ada-002 嵌入维度
    metric: "cosine", // cosine, euclidean, dotproduct
    pods: 1,
    replicas: 1,
    podType: "p1.x1"  // p1.x1, p1.x2, p1.x4, p1.x8
  },
  
  // 命名空间配置
  namespaces: {
    default: "default",
    workspace: (workspaceId) => `workspace-${workspaceId}`,
    user: (userId) => `user-${userId}`
  },
  
  // 查询配置
  query: {
    topK: 10,
    includeMetadata: true,
    includeValues: false,
    filter: {}
  }
};

// 索引创建示例
async function createPineconeIndex() {
  const pinecone = new PineconeClient();
  await pinecone.init({
    apiKey: process.env.PINECONE_API_KEY,
    environment: process.env.PINECONE_ENVIRONMENT
  });
  
  await pinecone.createIndex({
    createRequest: {
      name: "anythingllm-index",
      dimension: 1536,
      metric: "cosine",
      pods: 1,
      replicas: 1,
      pod_type: "p1.x1"
    }
  });
}

性能优化

javascript

// Pinecone 性能优化
const pineconeOptimization = {
  // 批量操作
  batching: {
    upsertBatchSize: 100,
    queryBatchSize: 10,
    deleteBatchSize: 1000
  },
  
  // 连接池
  connectionPool: {
    maxConnections: 10,
    keepAlive: true,
    timeout: 30000
  },
  
  // 缓存策略
  caching: {
    enabled: true,
    ttl: 3600,  // 1小时
    maxSize: 1000
  },
  
  // 索引优化
  indexOptimization: {
    podType: "p1.x2",  // 更高性能的 pod
    replicas: 2,       // 增加副本提高可用性
    shards: 1          // 分片配置
  }
};

Chroma 配置

基础配置

javascript

// Chroma 环境变量
CHROMA_ENDPOINT=http://localhost:8000
CHROMA_COLLECTION=anythingllm_collection
CHROMA_AUTH_TOKEN=your-chroma-token  // 可选

// 持久化配置
CHROMA_PERSIST_DIRECTORY=./chroma_db
CHROMA_ALLOW_RESET=false

集合配置

javascript

// Chroma 集合配置
const chromaConfig = {
  endpoint: process.env.CHROMA_ENDPOINT || "http://localhost:8000",
  
  // 集合配置
  collection: {
    name: "anythingllm_collection",
    metadata: {
      "hnsw:space": "cosine",
      "hnsw:construction_ef": 200,
      "hnsw:M": 16
    },
    embeddingFunction: "openai"  // openai, sentence-transformers, custom
  },
  
  // 查询配置
  query: {
    nResults: 10,
    includeMetadata: true,
    includeDocuments: true,
    includeDistances: true
  },
  
  // 过滤配置
  filtering: {
    enabled: true,
    operators: ["$eq", "$ne", "$gt", "$gte", "$lt", "$lte", "$in", "$nin"]
  }
};

// 集合创建示例
async function createChromaCollection() {
  const { ChromaClient } = require('chromadb');
  const client = new ChromaClient({
    path: process.env.CHROMA_ENDPOINT
  });
  
  const collection = await client.createCollection({
    name: "anythingllm_collection",
    metadata: {
      "hnsw:space": "cosine",
      "hnsw:construction_ef": 200,
      "hnsw:M": 16
    }
  });
  
  return collection;
}

Docker 部署

yaml

# docker-compose.yml for Chroma
version: '3.8'
services:
  chroma:
    image: ghcr.io/chroma-core/chroma:latest
    ports:
      - "8000:8000"
    volumes:
      - chroma_data:/chroma/chroma
    environment:
      - CHROMA_SERVER_HOST=0.0.0.0
      - CHROMA_SERVER_HTTP_PORT=8000
      - CHROMA_SERVER_CORS_ALLOW_ORIGINS=["*"]
    command: ["uvicorn", "chromadb.app:app", "--host", "0.0.0.0", "--port", "8000"]

volumes:
  chroma_data:
    driver: local

Weaviate 配置

基础配置

javascript

// Weaviate 环境变量
WEAVIATE_ENDPOINT=http://localhost:8080
WEAVIATE_API_KEY=your-weaviate-api-key  // 可选
WEAVIATE_CLASS_NAME=AnythingLLMDocument
WEAVIATE_SCHEME=http  // http 或 https

模式定义

javascript

// Weaviate 模式配置
const weaviateSchema = {
  class: "AnythingLLMDocument",
  description: "AnythingLLM document storage",
  vectorizer: "text2vec-openai",
  moduleConfig: {
    "text2vec-openai": {
      model: "ada",
      modelVersion: "002",
      type: "text"
    }
  },
  properties: [
    {
      name: "content",
      dataType: ["text"],
      description: "Document content",
      moduleConfig: {
        "text2vec-openai": {
          skip: false,
          vectorizePropertyName: false
        }
      }
    },
    {
      name: "title",
      dataType: ["string"],
      description: "Document title"
    },
    {
      name: "source",
      dataType: ["string"],
      description: "Document source"
    },
    {
      name: "workspaceId",
      dataType: ["string"],
      description: "Workspace identifier"
    },
    {
      name: "metadata",
      dataType: ["object"],
      description: "Additional metadata"
    }
  ]
};

// 创建 Weaviate 类
async function createWeaviateClass() {
  const weaviate = require('weaviate-ts-client').default;
  
  const client = weaviate.client({
    scheme: 'http',
    host: 'localhost:8080',
    apiKey: new weaviate.ApiKey(process.env.WEAVIATE_API_KEY)
  });
  
  await client.schema
    .classCreator()
    .withClass(weaviateSchema)
    .do();
}

查询配置

javascript

// Weaviate 查询配置
const weaviateQuery = {
  // 向量搜索
  nearVector: {
    vector: [], // 查询向量
    certainty: 0.7,
    distance: 0.3
  },
  
  // 文本搜索
  nearText: {
    concepts: ["search query"],
    certainty: 0.7,
    distance: 0.3
  },
  
  // 混合搜索
  hybrid: {
    query: "search query",
    alpha: 0.5,  // 0 = 纯关键词搜索, 1 = 纯向量搜索
    vector: []
  },
  
  // 过滤器
  where: {
    path: ["workspaceId"],
    operator: "Equal",
    valueString: "workspace-123"
  },
  
  // 返回字段
  fields: "content title source metadata _additional { id certainty distance }"
};

Qdrant 配置

基础配置

javascript

// Qdrant 环境变量
QDRANT_ENDPOINT=http://localhost:6333
QDRANT_API_KEY=your-qdrant-api-key  // 可选
QDRANT_COLLECTION=anythingllm_collection
QDRANT_VECTOR_SIZE=1536

集合配置

javascript

// Qdrant 集合配置
const qdrantConfig = {
  endpoint: process.env.QDRANT_ENDPOINT,
  apiKey: process.env.QDRANT_API_KEY,
  
  // 集合配置
  collection: {
    name: "anythingllm_collection",
    vectors: {
      size: 1536,
      distance: "Cosine"  // Cosine, Euclid, Dot
    },
    optimizers_config: {
      deleted_threshold: 0.2,
      vacuum_min_vector_number: 1000,
      default_segment_number: 0,
      max_segment_size: null,
      memmap_threshold: null,
      indexing_threshold: 20000,
      flush_interval_sec: 5,
      max_optimization_threads: 1
    },
    wal_config: {
      wal_capacity_mb: 32,
      wal_segments_ahead: 0
    },
    hnsw_config: {
      m: 16,
      ef_construct: 100,
      full_scan_threshold: 10000,
      max_indexing_threads: 0,
      on_disk: false,
      payload_m: null
    }
  }
};

// 创建 Qdrant 集合
async function createQdrantCollection() {
  const { QdrantClient } = require('@qdrant/js-client-rest');
  
  const client = new QdrantClient({
    url: process.env.QDRANT_ENDPOINT,
    apiKey: process.env.QDRANT_API_KEY
  });
  
  await client.createCollection("anythingllm_collection", {
    vectors: {
      size: 1536,
      distance: "Cosine"
    },
    optimizers_config: {
      default_segment_number: 2
    },
    replication_factor: 2
  });
}

搜索配置

javascript

// Qdrant 搜索配置
const qdrantSearch = {
  // 基础搜索
  search: {
    collection_name: "anythingllm_collection",
    vector: [], // 查询向量
    limit: 10,
    score_threshold: 0.7,
    with_payload: true,
    with_vectors: false
  },
  
  // 过滤搜索
  searchWithFilter: {
    collection_name: "anythingllm_collection",
    vector: [],
    filter: {
      must: [
        {
          key: "workspaceId",
          match: {
            value: "workspace-123"
          }
        }
      ]
    },
    limit: 10,
    with_payload: true
  },
  
  // 批量搜索
  searchBatch: {
    collection_name: "anythingllm_collection",
    searches: [
      {
        vector: [],
        limit: 5,
        filter: {}
      }
    ]
  }
};

pgvector 配置

基础配置

javascript

// PostgreSQL + pgvector 配置
POSTGRES_HOST=localhost
POSTGRES_PORT=5432
POSTGRES_DB=anythingllm
POSTGRES_USER=anythingllm
POSTGRES_PASSWORD=your-password
PGVECTOR_DIMENSION=1536

数据库设置

sql

-- 启用 pgvector 扩展
CREATE EXTENSION IF NOT EXISTS vector;

-- 创建向量表
CREATE TABLE IF NOT EXISTS document_embeddings (
    id SERIAL PRIMARY KEY,
    document_id VARCHAR(255) NOT NULL,
    workspace_id VARCHAR(255) NOT NULL,
    content TEXT NOT NULL,
    metadata JSONB,
    embedding vector(1536),
    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
    updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);

-- 创建向量索引
CREATE INDEX IF NOT EXISTS idx_document_embeddings_vector 
ON document_embeddings USING ivfflat (embedding vector_cosine_ops) 
WITH (lists = 100);

-- 创建其他索引
CREATE INDEX IF NOT EXISTS idx_document_embeddings_workspace 
ON document_embeddings (workspace_id);

CREATE INDEX IF NOT EXISTS idx_document_embeddings_document 
ON document_embeddings (document_id);

查询优化

javascript

// pgvector 查询配置
const pgvectorConfig = {
  // 连接配置
  connection: {
    host: process.env.POSTGRES_HOST,
    port: process.env.POSTGRES_PORT,
    database: process.env.POSTGRES_DB,
    user: process.env.POSTGRES_USER,
    password: process.env.POSTGRES_PASSWORD,
    ssl: process.env.NODE_ENV === 'production'
  },
  
  // 向量搜索配置
  search: {
    // 设置搜索精度
    setSearchPath: "SET ivfflat.probes = 10;",
    
    // 相似性搜索查询
    similarityQuery: `
      SELECT 
        document_id,
        content,
        metadata,
        1 - (embedding <=> $1) as similarity
      FROM document_embeddings
      WHERE workspace_id = $2
      ORDER BY embedding <=> $1
      LIMIT $3;
    `,
    
    // 阈值过滤查询
    thresholdQuery: `
      SELECT 
        document_id,
        content,
        metadata,
        1 - (embedding <=> $1) as similarity
      FROM document_embeddings
      WHERE workspace_id = $2
        AND 1 - (embedding <=> $1) > $3
      ORDER BY embedding <=> $1
      LIMIT $4;
    `
  }
};

向量数据库选择指南

性能对比

javascript

// 向量数据库性能对比
const performanceComparison = {
  pinecone: {
    latency: "低",
    throughput: "高",
    scalability: "优秀",
    cost: "中等",
    maintenance: "无需维护",
    useCase: "生产环境，高并发"
  },
  
  chroma: {
    latency: "中等",
    throughput: "中等",
    scalability: "良好",
    cost: "低",
    maintenance: "简单",
    useCase: "开发测试，中小规模"
  },
  
  weaviate: {
    latency: "低",
    throughput: "高",
    scalability: "优秀",
    cost: "中等",
    maintenance: "中等",
    useCase: "企业级，复杂查询"
  },
  
  qdrant: {
    latency: "低",
    throughput: "高",
    scalability: "优秀",
    cost: "低",
    maintenance: "中等",
    useCase: "高性能，自托管"
  },
  
  pgvector: {
    latency: "中等",
    throughput: "中等",
    scalability: "良好",
    cost: "低",
    maintenance: "中等",
    useCase: "现有 PostgreSQL 环境"
  }
};

选择决策树

javascript

// 向量数据库选择决策
function selectVectorDatabase(requirements) {
  const {
    scale,           // small, medium, large
    budget,          // low, medium, high
    maintenance,     // minimal, moderate, high
    latency,         // low, medium, high
    features         // basic, advanced, enterprise
  } = requirements;
  
  // 大规模，高预算，低维护
  if (scale === 'large' && budget === 'high' && maintenance === 'minimal') {
    return 'pinecone';
  }
  
  // 中小规模，低预算，简单维护
  if (scale === 'small' && budget === 'low') {
    return 'chroma';
  }
  
  // 企业级功能需求
  if (features === 'enterprise') {
    return 'weaviate';
  }
  
  // 高性能，自托管
  if (latency === 'low' && maintenance === 'moderate') {
    return 'qdrant';
  }
  
  // 现有 PostgreSQL 环境
  if (requirements.existingPostgres) {
    return 'pgvector';
  }
  
  // 默认推荐
  return 'chroma';
}

数据迁移

跨数据库迁移

javascript

// 向量数据库迁移工具
class VectorDatabaseMigrator {
  constructor(source, target) {
    this.source = source;
    this.target = target;
  }
  
  async migrate() {
    console.log('开始迁移向量数据...');
    
    // 1. 导出源数据
    const data = await this.exportData();
    
    // 2. 转换数据格式
    const transformedData = await this.transformData(data);
    
    // 3. 导入目标数据库
    await this.importData(transformedData);
    
    // 4. 验证迁移结果
    await this.validateMigration();
    
    console.log('迁移完成');
  }
  
  async exportData() {
    const batchSize = 1000;
    let offset = 0;
    const allData = [];
    
    while (true) {
      const batch = await this.source.query({
        limit: batchSize,
        offset: offset
      });
      
      if (batch.length === 0) break;
      
      allData.push(...batch);
      offset += batchSize;
      
      console.log(`已导出 ${allData.length} 条记录`);
    }
    
    return allData;
  }
  
  async transformData(data) {
    return data.map(item => ({
      id: item.id,
      vector: item.embedding || item.vector,
      metadata: {
        content: item.content,
        source: item.source,
        workspaceId: item.workspaceId,
        ...item.metadata
      }
    }));
  }
  
  async importData(data) {
    const batchSize = 100;
    
    for (let i = 0; i < data.length; i += batchSize) {
      const batch = data.slice(i, i + batchSize);
      await this.target.upsert(batch);
      
      console.log(`已导入 ${Math.min(i + batchSize, data.length)} / ${data.length} 条记录`);
    }
  }
  
  async validateMigration() {
    const sourceCount = await this.source.count();
    const targetCount = await this.target.count();
    
    if (sourceCount !== targetCount) {
      throw new Error(`迁移验证失败: 源数据库 ${sourceCount} 条，目标数据库 ${targetCount} 条`);
    }
    
    console.log(`迁移验证成功: ${targetCount} 条记录`);
  }
}

// 使用示例
async function migratePineconeToQdrant() {
  const pinecone = new PineconeClient(/* config */);
  const qdrant = new QdrantClient(/* config */);
  
  const migrator = new VectorDatabaseMigrator(pinecone, qdrant);
  await migrator.migrate();
}

监控和维护

性能监控

javascript

// 向量数据库监控
const vectorDBMonitoring = {
  // 性能指标
  metrics: {
    queryLatency: {
      p50: 0,
      p95: 0,
      p99: 0
    },
    throughput: {
      queriesPerSecond: 0,
      insertsPerSecond: 0
    },
    storage: {
      totalVectors: 0,
      storageSize: 0,
      indexSize: 0
    },
    errors: {
      queryErrors: 0,
      insertErrors: 0,
      connectionErrors: 0
    }
  },
  
  // 健康检查
  healthCheck: async () => {
    try {
      const startTime = Date.now();
      await vectorDB.ping();
      const latency = Date.now() - startTime;
      
      return {
        status: 'healthy',
        latency: latency,
        timestamp: new Date().toISOString()
      };
    } catch (error) {
      return {
        status: 'unhealthy',
        error: error.message,
        timestamp: new Date().toISOString()
      };
    }
  },
  
  // 告警配置
  alerts: {
    highLatency: {
      threshold: 1000,  // 1秒
      action: 'email'
    },
    lowThroughput: {
      threshold: 10,    // 10 QPS
      action: 'slack'
    },
    storageLimit: {
      threshold: 0.8,   // 80%
      action: 'email'
    }
  }
};

维护任务

javascript

// 向量数据库维护
class VectorDBMaintenance {
  constructor(vectorDB) {
    this.vectorDB = vectorDB;
  }
  
  // 索引优化
  async optimizeIndex() {
    console.log('开始索引优化...');
    
    // 重建索引
    await this.vectorDB.rebuildIndex();
    
    // 压缩存储
    await this.vectorDB.compact();
    
    console.log('索引优化完成');
  }
  
  // 数据清理
  async cleanupData() {
    console.log('开始数据清理...');
    
    // 删除过期数据
    const expiredData = await this.vectorDB.findExpired();
    await this.vectorDB.delete(expiredData);
    
    // 删除重复数据
    const duplicates = await this.vectorDB.findDuplicates();
    await this.vectorDB.deduplicate(duplicates);
    
    console.log('数据清理完成');
  }
  
  // 备份数据
  async backupData() {
    console.log('开始数据备份...');
    
    const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
    const backupPath = `./backups/vectordb-${timestamp}`;
    
    await this.vectorDB.backup(backupPath);
    
    console.log(`数据备份完成: ${backupPath}`);
  }
  
  // 定期维护
  scheduleMaintenanceTasks() {
    // 每日索引优化
    cron.schedule('0 2 * * *', () => {
      this.optimizeIndex();
    });
    
    // 每周数据清理
    cron.schedule('0 3 * * 0', () => {
      this.cleanupData();
    });
    
    // 每日备份
    cron.schedule('0 1 * * *', () => {
      this.backupData();
    });
  }
}

选择合适的向量数据库并正确配置是实现高效语义搜索的关键。根据您的具体需求、预算和技术栈选择最适合的解决方案，并定期进行监控和维护以确保最佳性能。

向量数据库配置 ​

支持的向量数据库 ​

云端解决方案 ​

自托管解决方案 ​

传统数据库扩展 ​

Pinecone 配置 ​

基础配置 ​

索引创建和管理 ​

性能优化 ​

Chroma 配置 ​

基础配置 ​

集合配置 ​

Docker 部署 ​

Weaviate 配置 ​

基础配置 ​

模式定义 ​

查询配置 ​

Qdrant 配置 ​

基础配置 ​

集合配置 ​

搜索配置 ​

pgvector 配置 ​

基础配置 ​

数据库设置 ​

查询优化 ​

向量数据库选择指南 ​

性能对比 ​

选择决策树 ​

数据迁移 ​

跨数据库迁移 ​

监控和维护 ​

性能监控 ​

维护任务 ​

向量数据库配置

支持的向量数据库

云端解决方案

自托管解决方案

传统数据库扩展

Pinecone 配置

基础配置

索引创建和管理

性能优化

Chroma 配置

基础配置

集合配置

Docker 部署

Weaviate 配置

基础配置

模式定义

查询配置

Qdrant 配置

基础配置

集合配置

搜索配置

pgvector 配置

基础配置

数据库设置

查询优化

向量数据库选择指南

性能对比

选择决策树

数据迁移

跨数据库迁移

监控和维护

性能监控

维护任务