Overview

class KnowledgeBase(BaseModel, WorkspaceAwareModel):
    id = models.UUIDField(primary_key=True)
    name = models.CharField(max_length=200)
    description = models.TextField(blank=True)
    created_by = models.ForeignKey(User, on_delete=models.SET_NULL)
    is_shared_with_workspace = models.BooleanField(default=False)
    documents = models.ManyToManyField(Document, through="KnowledgeBaseDocument")

class Document(BaseModel, WorkspaceAwareModel):
    id = models.UUIDField(primary_key=True)
    name = models.CharField(max_length=500)
    uploaded_by = models.ForeignKey(User, on_delete=models.SET_NULL)

    # Source
    source_type = models.CharField(choices=["file", "web_url", "youtube"])
    source_url = models.URLField(max_length=2048, blank=True)

    # S3 Storage
    s3_key = models.CharField(max_length=1024, blank=True)
    original_filename = models.CharField(max_length=500, blank=True)
    file_type = models.CharField(max_length=20, blank=True)   # "pdf", "docx", etc.
    mime_type = models.CharField(max_length=100, blank=True)
    file_size = models.PositiveBigIntegerField(default=0)

    # Processing
    processing_status = models.CharField(
        choices=["pending", "processing", "ready", "failed"]
    )
    processing_error = models.TextField(blank=True)
    embedding_model = models.CharField(max_length=100, blank=True)
    chunk_count = models.PositiveIntegerField(default=0)

class DocumentChunk(BaseModel):
    id = models.UUIDField(primary_key=True)
    document = models.ForeignKey(Document, on_delete=models.CASCADE)
    chunk_index = models.PositiveIntegerField()
    content = models.TextField()

    # Vector embedding (pgvector)
    embedding = VectorField(dimensions=1536)
    embedding_model = models.CharField(max_length=100)

    # Citation metadata
    page_number = models.PositiveIntegerField(null=True)
    section_heading = models.CharField(max_length=500, blank=True)
    token_count = models.PositiveIntegerField(default=0)
    chunk_metadata = models.JSONField(default=dict)

    class Meta:
        indexes = [
            HnswIndex(fields=["embedding"], m=16, ef_construction=64, opclasses=["vector_cosine_ops"]),
            models.Index(fields=["document", "chunk_index"]),
        ]

Document created (pending)
    │
    ▼
Celery task dispatched
    │
    ├─ File → Download from S3 → Parse (Docling, or the fallback parser) → Chunk → Embed → Save
    ├─ Web URL → Extract (Trafilatura) → Chunk → Embed → Save
    └─ YouTube → Extract transcript → Chunk → Embed → Save
    │
    ▼
Document status → "ready" (or "failed", with a plain message for the user)

User question in chat
    │
    ▼
Rewrite into a standalone query + alternative phrasings
    │
    ▼
Ready documents in the chat's knowledge bases that the asker can read
    │
    ├─ Semantic search (pgvector cosine similarity) → top 20 per phrasing
    ├─ Keyword search (PostgreSQL full-text) → top 20 per phrasing
    │
    ▼
Reciprocal Rank Fusion → 20-candidate pool
    │
    ▼
LLM rerank → top 5
    │
    ▼
Return to agent as tool result, to cite