Cloud Vector Databases

1# pip install pinecone-client
2from pinecone import Pinecone, ServerlessSpec
3
4pc = Pinecone(api_key="your-api-key")
5
6# Create index
7pc.create_index(
8    name="rag-knowledge-base",
9    dimension=1536,          # OpenAI text-embedding-3-small
10    metric="cosine",
11    spec=ServerlessSpec(
12        cloud="aws",
13        region="us-east-1"
14    )
15)
16
17# Connect to index
18index = pc.Index("rag-knowledge-base")
19print(index.describe_index_stats())

1from openai import OpenAI
2
3openai_client = OpenAI()
4
5def get_embedding(text, model="text-embedding-3-small"):
6    response = openai_client.embeddings.create(input=text, model=model)
7    return response.data[0].embedding
8
9# Prepare documents
10documents = [
11    {"id": "doc1", "text": "RAG architecture combines retrieval and generation.", "source": "rag-guide"},
12    {"id": "doc2", "text": "Pinecone is a managed vector database service.", "source": "pinecone-docs"},
13    {"id": "doc3", "text": "Embeddings represent text as numerical vectors.", "source": "ml-basics"},
14]
15
16# Upsert with embeddings + metadata
17vectors = []
18for doc in documents:
19    embedding = get_embedding(doc["text"])
20    vectors.append({
21        "id": doc["id"],
22        "values": embedding,
23        "metadata": {
24            "text": doc["text"],
25            "source": doc["source"]
26        }
27    })
28
29index.upsert(vectors=vectors)
30print(f"Upserted {len(vectors)} vectors")

1# Semantic search
2query = "How does RAG work?"
3query_embedding = get_embedding(query)
4
5results = index.query(
6    vector=query_embedding,
7    top_k=3,
8    include_metadata=True
9)
10
11for match in results['matches']:
12    print(f"Score: {match['score']:.4f}")
13    print(f"Text: {match['metadata']['text']}")
14    print(f"Source: {match['metadata']['source']}")
15    print()

1# Namespaces = logical partitions (multi-tenant)
2# Upsert to specific namespace
3index.upsert(vectors=vectors, namespace="company-a")
4index.upsert(vectors=vectors, namespace="company-b")
5
6# Query within namespace
7results = index.query(
8    vector=query_embedding,
9    top_k=5,
10    namespace="company-a",
11    include_metadata=True
12)

1# pip install weaviate-client
2import weaviate
3from weaviate.classes.config import Configure, Property, DataType
4
5# Connect to Weaviate Cloud
6client = weaviate.connect_to_weaviate_cloud(
7    cluster_url="your-cluster-url",
8    auth_credentials=weaviate.auth.AuthApiKey("your-api-key")
9)
10
11# Or local Docker instance
12# client = weaviate.connect_to_local()

1# Create collection with vectorizer
2client.collections.create(
3    name="Document",
4    vectorizer_config=Configure.Vectorizer.text2vec_openai(
5        model="text-embedding-3-small"
6    ),
7    properties=[
8        Property(name="content", data_type=DataType.TEXT),
9        Property(name="source", data_type=DataType.TEXT),
10        Property(name="category", data_type=DataType.TEXT),
11        Property(name="page_number", data_type=DataType.INT),
12    ]
13)
14
15collection = client.collections.get("Document")

1# Add objects (auto-vectorized)
2collection.data.insert({
3    "content": "RAG pipelines use vector search to find relevant documents.",
4    "source": "rag-handbook",
5    "category": "architecture",
6    "page_number": 15
7})
8
9# Batch insert
10with collection.batch.dynamic() as batch:
11    for doc in documents:
12        batch.add_object(properties=doc)
13
14# Semantic search
15response = collection.query.near_text(
16    query="vector database comparison",
17    limit=5,
18    return_metadata=["distance"]
19)
20
21for obj in response.objects:
22    print(f"Distance: {obj.metadata.distance:.4f}")
23    print(f"Content: {obj.properties['content'][:100]}")

1# Weaviate's killer feature: hybrid search
2response = collection.query.hybrid(
3    query="RAG chunking strategy",
4    alpha=0.5,  # 0 = pure keyword (BM25), 1 = pure vector
5    limit=5,
6    return_metadata=["score"]
7)
8
9for obj in response.objects:
10    print(f"Score: {obj.metadata.score:.4f}")
11    print(f"Content: {obj.properties['content'][:100]}")
12    print()

1from langchain_community.vectorstores import Chroma
2from langchain_openai import OpenAIEmbeddings
3
4embeddings = OpenAIEmbeddings(model="text-embedding-3-small")
5
6# Create vector store
7vectorstore = Chroma.from_texts(
8    texts=["doc1 content", "doc2 content", "doc3 content"],
9    embedding=embeddings,
10    metadatas=[{"source": "a"}, {"source": "b"}, {"source": "c"}],
11    persist_directory="./chroma_langchain"
12)
13
14# Search
15docs = vectorstore.similarity_search("query text", k=3)
16for doc in docs:
17    print(doc.page_content[:100])

1from langchain_pinecone import PineconeVectorStore
2
3vectorstore = PineconeVectorStore(
4    index_name="rag-knowledge-base",
5    embedding=embeddings,
6    pinecone_api_key="your-key"
7)
8
9# Add documents
10vectorstore.add_texts(
11    texts=["content1", "content2"],
12    metadatas=[{"source": "a"}, {"source": "b"}]
13)
14
15# Search
16docs = vectorstore.similarity_search("query", k=5)

1# Convert to retriever
2retriever = vectorstore.as_retriever(
3    search_type="similarity",
4    search_kwargs={"k": 5}
5)
6
7# Or MMR (Maximal Marginal Relevance) for diversity
8retriever = vectorstore.as_retriever(
9    search_type="mmr",
10    search_kwargs={"k": 5, "fetch_k": 20, "lambda_mult": 0.7}
11)
12
13# Use in RAG chain
14relevant_docs = retriever.invoke("How to implement RAG?")