1. Indexing (offline):
   Documents → Chunking → Embedding (OpenAI/Cohere) → Vector DB

2. Query (realtime):
   User question → Embedding → Vector search → Top K results
   → Combine with question → LLM → Answer

from qdrant_client import QdrantClient
from qdrant_client.models import Distance, VectorParams, PointStruct
import openai

# Initialize client
client = QdrantClient(url="http://localhost:6333")

# Create collection
client.create_collection(
    collection_name="knowledge_base",
    vectors_config=VectorParams(size=1536, distance=Distance.COSINE),
)

# Index documents
def index_documents(docs):
    openai.api_key = "sk-..."

    points = []
    for i, doc in enumerate(docs):
        # Generate embedding
        response = openai.Embedding.create(
            input=doc["text"],
            model="text-embedding-3-small"
        )
        embedding = response['data'][0]['embedding']

        # Create point with metadata
        points.append(PointStruct(
            id=i,
            vector=embedding,
            payload={
                "text": doc["text"],
                "source": doc["source"],
                "created_at": doc["created_at"]
            }
        ))

    # Batch upsert
    client.upsert(
        collection_name="knowledge_base",
        points=points,
        wait=True
    )

# Query with filters
def search(query, source_filter=None):
    # Generate query embedding
    response = openai.Embedding.create(
        input=query,
        model="text-embedding-3-small"
    )
    query_vector = response['data'][0]['embedding']

    # Search with optional filter
    filter_clause = None
    if source_filter:
        filter_clause = {
            "must": [
                {"key": "source", "match": {"value": source_filter}}
            ]
        }

    results = client.search(
        collection_name="knowledge_base",
        query_vector=query_vector,
        query_filter=filter_clause,
        limit=5
    )

    return [{"text": hit.payload["text"], "score": hit.score} for hit in results]

# Usage
docs = [
    {"text": "PostgreSQL is a relational database", "source": "docs", "created_at": "2024-01-01"},
    {"text": "Redis is an in-memory data store", "source": "blog", "created_at": "2024-01-02"},
]
index_documents(docs)

results = search("What is PostgreSQL?", source_filter="docs")
print(results[0]["text"])  # "PostgreSQL is a relational database"

from pinecone import Pinecone, ServerlessSpec
import openai

# Initialize
pc = Pinecone(api_key="pcsk-...")
openai.api_key = "sk-..."

# Create index (serverless auto-scales)
pc.create_index(
    name="knowledge-base",
    dimension=1536,
    metric="cosine",
    spec=ServerlessSpec(cloud="aws", region="us-east-1")
)

index = pc.Index("knowledge-base")

# Index documents
def index_documents(docs):
    vectors = []
    for i, doc in enumerate(docs):
        response = openai.Embedding.create(
            input=doc["text"],
            model="text-embedding-3-small"
        )
        embedding = response['data'][0]['embedding']

        vectors.append({
            "id": str(i),
            "values": embedding,
            "metadata": {
                "text": doc["text"],
                "source": doc["source"]
            }
        })

    # Upsert in batches of 100
    for i in range(0, len(vectors), 100):
        index.upsert(vectors=vectors[i:i+100])

# Query
def search(query, source_filter=None):
    response = openai.Embedding.create(
        input=query,
        model="text-embedding-3-small"
    )
    query_vector = response['data'][0]['embedding']

    filter_dict = {"source": source_filter} if source_filter else None

    results = index.query(
        vector=query_vector,
        top_k=5,
        include_metadata=True,
        filter=filter_dict
    )

    return [{"text": match.metadata["text"], "score": match.score} 
            for match in results.matches]

import weaviate
import openai

# Initialize
client = weaviate.Client("http://localhost:8080")
openai.api_key = "sk-..."

# Create schema
schema = {
    "class": "Document",
    "vectorizer": "none",  # We provide embeddings
    "properties": [
        {"name": "text", "dataType": ["text"]},
        {"name": "source", "dataType": ["string"]},
    ]
}
client.schema.create_class(schema)

# Index documents
def index_documents(docs):
    for doc in docs:
        response = openai.Embedding.create(
            input=doc["text"],
            model="text-embedding-3-small"
        )
        embedding = response['data'][0]['embedding']

        client.data_object.create(
            data_object={
                "text": doc["text"],
                "source": doc["source"]
            },
            class_name="Document",
            vector=embedding
        )

# Hybrid search (BM25 + vector)
def hybrid_search(query, alpha=0.5):
    """
    alpha=0: pure BM25 keyword search
    alpha=1: pure vector search
    alpha=0.5: balanced hybrid
    """
    response = openai.Embedding.create(
        input=query,
        model="text-embedding-3-small"
    )
    query_vector = response['data'][0]['embedding']

    results = (
        client.query
        .get("Document", ["text", "source"])
        .with_hybrid(
            query=query,
            vector=query_vector,
            alpha=alpha
        )
        .with_limit(5)
        .do()
    )

    return results["data"]["Get"]["Document"]

# Pure vector search
def vector_search(query):
    response = openai.Embedding.create(
        input=query,
        model="text-embedding-3-small"
    )
    query_vector = response['data'][0]['embedding']

    results = (
        client.query
        .get("Document", ["text", "source"])
        .with_near_vector({"vector": query_vector})
        .with_limit(5)
        .do()
    )

    return results["data"]["Get"]["Document"]

# Keyword-heavy query benefits from hybrid
results = hybrid_search("PostgreSQL version 16 features")

from sentence_transformers import CrossEncoder

reranker = CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2')

def search_with_reranking(query, top_k=5, rerank_top_k=20):
    # Step 1: Vector search (retrieve 20 candidates)
    candidates = search(query, limit=rerank_top_k)

    # Step 2: Rerank with cross-encoder
    pairs = [(query, doc["text"]) for doc in candidates]
    scores = reranker.predict(pairs)

    # Step 3: Sort by reranker scores
    reranked = sorted(zip(candidates, scores), key=lambda x: x[1], reverse=True)

    return [doc for doc, score in reranked[:top_k]]

# Qdrant: Filter by date range + user_id
from qdrant_client.models import Filter, FieldCondition, Range

results = client.search(
    collection_name="knowledge_base",
    query_vector=query_vector,
    query_filter=Filter(
        must=[
            FieldCondition(
                key="user_id",
                match={"value": "user-123"}
            ),
            FieldCondition(
                key="created_at",
                range=Range(
                    gte="2024-01-01",
                    lte="2024-12-31"
                )
            )
        ]
    ),
    limit=5
)

from langchain.text_splitter import RecursiveCharacterTextSplitter

# Strategy 1: Fixed-size chunks (naive)
def chunk_fixed(text, size=512):
    return [text[i:i+size] for i in range(0, len(text), size)]

# Strategy 2: Recursive splitting (preserves structure)
splitter = RecursiveCharacterTextSplitter(
    chunk_size=512,
    chunk_overlap=50,
    separators=["\n\n", "\n", ". ", " ", ""]
)
chunks = splitter.split_text(long_document)

# Strategy 3: Semantic chunking (best quality)
from langchain_experimental.text_splitter import SemanticChunker

semantic_splitter = SemanticChunker(
    embeddings=OpenAIEmbeddings(),
    breakpoint_threshold_type="percentile",
    breakpoint_threshold_amount=95
)
semantic_chunks = semantic_splitter.split_text(long_document)

# Qdrant: Use collections per tenant (small tenant count)
client.create_collection(
    collection_name=f"tenant_{tenant_id}",
    vectors_config=VectorParams(size=1536, distance=Distance.COSINE)
)

# Or use payload filtering (many tenants)
client.upsert(
    collection_name="shared_collection",
    points=[
        PointStruct(
            id=doc_id,
            vector=embedding,
            payload={"tenant_id": tenant_id, "text": text}
        )
    ]
)

# Query with tenant filter
results = client.search(
    collection_name="shared_collection",
    query_vector=query_vector,
    query_filter=Filter(
        must=[{"key": "tenant_id", "match": {"value": tenant_id}}]
    )
)

# Pinecone: Use namespaces
index.upsert(vectors=[...], namespace=f"tenant_{tenant_id}")
results = index.query(vector=query_vector, namespace=f"tenant_{tenant_id}")

# Prometheus metrics
from prometheus_client import Histogram, Counter

query_latency = Histogram(
    'vector_db_query_latency_seconds',
    'Query latency in seconds',
    buckets=[0.01, 0.05, 0.1, 0.5, 1.0]
)

query_count = Counter(
    'vector_db_queries_total',
    'Total queries',
    ['status']  # success, error
)

@query_latency.time()
def search_wrapper(query):
    try:
        results = client.search(...)
        query_count.labels(status='success').inc()
        return results
    except Exception as e:
        query_count.labels(status='error').inc()
        raise

# Alert rules (Prometheus)
# Alert if p95 latency > 100ms for 5 minutes
# Alert if error rate > 1% for 5 minutes

# Export from Pinecone
pinecone_index = pc.Index("old-index")
all_ids = pinecone_index.list()  # Paginated

vectors = []
for id_batch in chunk_ids(all_ids, 100):
    fetch_result = pinecone_index.fetch(ids=id_batch)
    vectors.extend(fetch_result.vectors.values())

# Import to Qdrant
qdrant_client = QdrantClient(url="http://localhost:6333")

qdrant_client.create_collection(
    collection_name="new-collection",
    vectors_config=VectorParams(size=1536, distance=Distance.COSINE)
)

points = [
    PointStruct(
        id=int(v.id),
        vector=v.values,
        payload=v.metadata
    )
    for v in vectors
]

qdrant_client.upsert(collection_name="new-collection", points=points, wait=True)

# Qdrant: Higher ef = more accurate but slower
results = client.search(
    collection_name="docs",
    query_vector=query_vector,
    search_params={"hnsw_ef": 128}  # Default: 64
)

client.upsert(
    collection_name="docs",
    points=[PointStruct(id=existing_id, vector=new_embedding, payload=new_metadata)]
)

client.set_payload(
    collection_name="docs",
    payload={"updated_at": "2024-03-16"},
    points=[existing_id]
)

# Qdrant: Store both OpenAI and Cohere embeddings
client.create_collection(
    collection_name="multi_vector",
    vectors_config={
        "openai": VectorParams(size=1536, distance=Distance.COSINE),
        "cohere": VectorParams(size=1024, distance=Distance.COSINE)
    }
)

client.upsert(
    collection_name="multi_vector",
    points=[
        PointStruct(
            id=1,
            vector={
                "openai": openai_embedding,
                "cohere": cohere_embedding
            },
            payload={"text": "..."}
        )
    ]
)

# Search using specific vector
results = client.search(
    collection_name="multi_vector",
    query_vector=("openai", query_embedding),
    limit=5
)

# Index parent summaries
summary_embedding = embed(document_summary)
client.upsert(points=[
    PointStruct(
        id=doc_id,
        vector=summary_embedding,
        payload={"type": "summary", "doc_id": doc_id}
    )
])

# Index child chunks
for i, chunk in enumerate(chunks):
    chunk_embedding = embed(chunk)
    client.upsert(points=[
        PointStruct(
            id=f"{doc_id}_chunk_{i}",
            vector=chunk_embedding,
            payload={"type": "chunk", "doc_id": doc_id, "text": chunk}
        )
    ])

# Two-stage retrieval
summary_results = client.search(query_vector=query_embedding, limit=3)
relevant_doc_ids = [r.payload["doc_id"] for r in summary_results]

chunk_results = client.search(
    query_vector=query_embedding,
    query_filter=Filter(must=[{"key": "doc_id", "match": {"any": relevant_doc_ids}}]),
    limit=10
)