Embeddings Explained — How Semantic Search Works

# Naive example: one-hot encoding (outdated)
vocabulary = ["cat", "dog", "bird"]

# "cat" -> [1, 0, 0]
# "dog" -> [0, 1, 0]
# "bird" -> [0, 0, 1]

# Problem: loses semantic relationships, high dimensionality for large vocabularies

# Conceptual example (not actual implementation)
# If "king - man + woman ≈ queen" in the embedding space
# The model learned semantic relationships

word2vec_embedding = {
    "cat": [0.2, -0.5, 0.1, 0.8, ...],
    "dog": [0.3, -0.4, 0.15, 0.75, ...],  # Similar to cat
    "quantum": [-0.9, 0.1, 0.2, -0.5, ...],  # Different from cat/dog
}

from sentence_transformers import SentenceTransformer

# Load pre-trained model
model = SentenceTransformer('all-MiniLM-L6-v2')

# Generate embeddings (384 dimensions)
sentences = [
    "A cat is a fluffy pet",
    "Dogs are loyal animals",
    "Quantum computing uses qubits"
]

embeddings = model.encode(sentences)
print(embeddings.shape)  # (3, 384)

# Compute similarity
from scipy.spatial.distance import cosine

similarity_cat_dog = 1 - cosine(embeddings[0], embeddings[1])
similarity_cat_quantum = 1 - cosine(embeddings[0], embeddings[2])

print(f"Cat-Dog similarity: {similarity_cat_dog:.2f}")  # High (~0.7)
print(f"Cat-Quantum similarity: {similarity_cat_quantum:.2f}")  # Low (~0.1)

from sentence_transformers import SentenceTransformer
import numpy as np

model = SentenceTransformer('all-MiniLM-L6-v2')

# Your documents
documents = [
    "Python is a programming language",
    "Machine learning is a subset of AI",
    "Data science involves statistics",
    "Neural networks process information",
    "A cat is a furry pet"
]

# Generate embeddings
embeddings = model.encode(documents, convert_to_tensor=True)

from sklearn.metrics.pairwise import cosine_similarity

# Store embeddings and documents
index = {
    'documents': documents,
    'embeddings': embeddings.numpy()
}

def semantic_search(query: str, top_k: int = 3) -> list:
    """Find most similar documents."""
    # Embed query
    query_embedding = model.encode([query], convert_to_tensor=True)

    # Compute similarities
    similarities = cosine_similarity(
        query_embedding.numpy(),
        index['embeddings']
    )[0]

    # Get top-k
    top_indices = np.argsort(similarities)[-top_k:][::-1]

    results = [
        {
            'document': index['documents'][i],
            'similarity': similarities[i]
        }
        for i in top_indices
    ]

    return results

# Test
results = semantic_search("What is machine learning?")
for result in results:
    print(f"Score: {result['similarity']:.3f} - {result['document']}")

Score: 0.823 - Machine learning is a subset of AI
Score: 0.654 - Python is a programming language
Score: 0.521 - Neural networks process information

from openai import OpenAI

client = OpenAI(api_key="your-api-key")

def embed_with_openai(texts: list) -> np.ndarray:
    """Generate embeddings using OpenAI."""
    response = client.embeddings.create(
        input=texts,
        model="text-embedding-3-small"
    )

    embeddings = [item.embedding for item in response.data]
    return np.array(embeddings)

# Embed documents
doc_embeddings = embed_with_openai(documents)

# Embed query
query = "What is AI?"
query_embedding = embed_with_openai([query])[0]

# Find similar
similarities = np.dot(doc_embeddings, query_embedding) / (
    np.linalg.norm(doc_embeddings, axis=1) *
    np.linalg.norm(query_embedding)
)

top_idx = np.argmax(similarities)
print(f"Most similar: {documents[top_idx]}")

# Add metadata to documents
indexed_docs = [
    {"text": "Python programming", "category": "programming", "year": 2023},
    {"text": "Machine learning basics", "category": "ai", "year": 2024},
    {"text": "Web development", "category": "programming", "year": 2023},
]

# Embed texts
embeddings = model.encode([doc['text'] for doc in indexed_docs])

def semantic_search_with_filter(
    query: str,
    category_filter: str = None,
    top_k: int = 3
) -> list:
    """Search with optional metadata filtering."""
    query_emb = model.encode([query])

    # Compute similarities
    similarities = cosine_similarity(query_emb, embeddings)[0]

    # Filter by category if specified
    filtered_results = []
    for i, (sim, doc) in enumerate(zip(similarities, indexed_docs)):
        if category_filter is None or doc['category'] == category_filter:
            filtered_results.append((sim, doc, i))

    # Sort and return top-k
    filtered_results.sort(key=lambda x: x[0], reverse=True)
    return filtered_results[:top_k]

# Search filtered
results = semantic_search_with_filter(
    "Python tutorial",
    category_filter="programming",
    top_k=2
)

from sklearn.decomposition import PCA

# Reduce dimensionality for efficiency
pca = PCA(n_components=128)
reduced_embeddings = pca.fit_transform(embeddings)

# Trade-off: less storage and faster search, but slight accuracy loss
# Original: 384 dimensions
# Reduced: 128 dimensions (67% smaller)

# Use reduced embeddings for large-scale search

# Cosine similarity (most common for embeddings)
from scipy.spatial.distance import cosine
cosine_dist = cosine(emb1, emb2)

# Euclidean distance
euclidean_dist = np.linalg.norm(emb1 - emb2)

# Dot product (fast, used in vector databases)
dot_product = np.dot(emb1, emb2)

# For normalized embeddings, cosine similarity = dot product