Multimodal AI Guide 2026 — Text, Images, Audio and Video in One API

import base64
from openai import OpenAI
 
client = OpenAI()
 
def analyze_image(image_path: str, question: str) -> str:
    with open(image_path, "rb") as f:
        image_data = base64.b64encode(f.read()).decode("utf-8")
 
    ext = image_path.split(".")[-1].lower()
    mime_map = {"jpg": "jpeg", "jpeg": "jpeg", "png": "png", "gif": "gif", "webp": "webp"}
 
    response = client.chat.completions.create(
        model="gpt-4o",
        messages=[{
            "role": "user",
            "content": [
                {"type": "text", "text": question},
                {"type": "image_url", "image_url": {"url": f"data:image/{mime_map.get(ext, ext)};base64,{image_data}"}},
            ],
        }],
        max_tokens=500,
    )
    return response.choices[0].message.content
 
# Real-world use cases
print(analyze_image("chart.png", "Extract all data points from this chart as a table"))
print(analyze_image("receipt.jpg", "Extract store name, date, items, and total cost as JSON"))
print(analyze_image("code_screenshot.png", "What bugs or issues do you see in this code?"))
print(analyze_image("whiteboard.jpg", "Transcribe all text and equations from this whiteboard"))

import fitz  # PyMuPDF
import json
 
def extract_invoice_data(pdf_path: str) -> dict:
    doc = fitz.open(pdf_path)
    page = doc[0]
    mat = fitz.Matrix(2, 2)  # 2x zoom improves OCR accuracy
    pix = page.get_pixmap(matrix=mat)
    img_bytes = pix.tobytes("png")
    img_base64 = base64.b64encode(img_bytes).decode()
 
    prompt = """Extract invoice data as JSON with these exact keys:
    invoice_number, date, vendor_name, vendor_address,
    line_items (array of {description, quantity, unit_price, total}),
    subtotal, tax, total_amount, payment_due_date"""
 
    response = client.chat.completions.create(
        model="gpt-4o",
        response_format={"type": "json_object"},
        messages=[{
            "role": "user",
            "content": [
                {"type": "text", "text": prompt},
                {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{img_base64}"}},
            ],
        }],
    )
    return json.loads(response.choices[0].message.content)

def transcribe_audio(audio_path: str, language: str = "en") -> dict:
    with open(audio_path, "rb") as f:
        transcript = client.audio.transcriptions.create(
            model="whisper-1",
            file=f,
            language=language,
            response_format="verbose_json",
            timestamp_granularities=["word"],
        )
    return {
        "text": transcript.text,
        "words": [(w.word, w.start, w.end) for w in transcript.words],
        "duration": transcript.duration,
    }

from pathlib import Path
 
def text_to_speech(text: str, output_path: str = "output.mp3", voice: str = "alloy"):
    """
    Voices: alloy, echo, fable, onyx, nova, shimmer
    Models: tts-1 (fast) or tts-1-hd (high quality)
    """
    response = client.audio.speech.create(
        model="tts-1-hd",
        voice=voice,
        input=text,
        speed=1.0,
    )
    Path(output_path).write_bytes(response.content)
 
text_to_speech("Welcome to webcoderspeed.com. Let's learn AI together.", voice="nova")

import google.generativeai as genai
import time
 
genai.configure(api_key="your-key")
model = genai.GenerativeModel("gemini-2.0-flash")
 
def analyze_video(video_path: str, questions: list[str]) -> dict:
    print(f"Uploading {video_path}...")
    video_file = genai.upload_file(path=video_path)
 
    while video_file.state.name == "PROCESSING":
        time.sleep(2)
        video_file = genai.get_file(video_file.name)
 
    if video_file.state.name == "FAILED":
        raise ValueError("Video processing failed")
 
    answers = {}
    for question in questions:
        response = model.generate_content([video_file, question])
        answers[question] = response.text
 
    return answers
 
results = analyze_video("lecture.mp4", [
    "Create a detailed transcript of the lecture",
    "List the main topics covered in order",
    "Identify any code shown on screen",
    "Generate 5 quiz questions based on the content",
])

from fastapi import FastAPI, UploadFile, File, Form
from PIL import Image
import io
 
app = FastAPI()
 
@app.post("/visual-qa")
async def visual_qa(image: UploadFile = File(...), question: str = Form(...)) -> dict:
    img_bytes = await image.read()
    img = Image.open(io.BytesIO(img_bytes))
 
    # Resize if too large to control costs
    img.thumbnail((1024, 1024), Image.LANCZOS)
 
    output = io.BytesIO()
    img.save(output, format="PNG")
    img_base64 = base64.b64encode(output.getvalue()).decode()
 
    response = client.chat.completions.create(
        model="gpt-4o",
        messages=[{
            "role": "user",
            "content": [
                {"type": "text", "text": question},
                {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{img_base64}"}},
            ],
        }],
        max_tokens=500,
    )
 
    return {
        "question": question,
        "answer": response.choices[0].message.content,
        "tokens_used": response.usage.total_tokens,
    }