Update and fix module docs

microsoft · ekzhu · Jul 7, 2025 · Jul 6, 2025 · Jul 6, 2025 · Jul 6, 2025
commit e985880e66dc28ab0feb7bc445656ef97c169097
diff --git a/python/packages/autogen-core/.gitignore b/python/packages/autogen-core/.gitignore
@@ -171,3 +171,6 @@ docs/**/jupyter_execute
 
 # Temporary files
 tmp_code_*.py
+
+# Reference files
+docs/src/reference
diff --git a/python/packages/autogen-core/docs/src/generate_api_toc.py b/python/packages/autogen-core/docs/src/generate_api_toc.py
@@ -7,30 +7,58 @@
 """
 
 import os
-import sys
 from pathlib import Path
 from typing import List, Dict, Set
-import importlib.util
-import ast
 import re
 
 
+# Constants for package filtering and organization
+DOCUMENTED_PACKAGES = ["autogen_core", "autogen_agentchat", "autogen_ext"]
+
+PACKAGE_SECTIONS = {
+    "autogen_agentchat": "AutoGen AgentChat",
+    "autogen_core": "AutoGen Core", 
+    "autogen_ext": "AutoGen Extensions"
+}
+
+# Exclusion patterns for submodules that are re-exported by parent modules
+EXCLUSION_PATTERNS = [
+    # task_centric_memory re-exports from memory_controller and utils
+    (r'^autogen_ext\.experimental\.task_centric_memory\.memory_controller$', 
+     'autogen_ext.experimental.task_centric_memory'),
+    # utils package re-exports from utils.apprentice and other utils submodules
+    (r'^autogen_ext\.experimental\.task_centric_memory\.utils\.apprentice$', 
+     'autogen_ext.experimental.task_centric_memory.utils'),
+    (r'^autogen_ext\.experimental\.task_centric_memory\.utils\.chat_completion_client_recorder$', 
+     'autogen_ext.experimental.task_centric_memory.utils'),
+    (r'^autogen_ext\.experimental\.task_centric_memory\.utils\.grader$', 
+     'autogen_ext.experimental.task_centric_memory.utils'),
+    (r'^autogen_ext\.experimental\.task_centric_memory\.utils\.page_logger$', 
+     'autogen_ext.experimental.task_centric_memory.utils'),
+    (r'^autogen_ext\.experimental\.task_centric_memory\.utils\.teachability$', 
+     'autogen_ext.experimental.task_centric_memory.utils'),
+]
+
+
 def find_python_packages() -> List[Path]:
-    """Find all Python packages in the workspace."""
+    """Find documented Python packages in the workspace."""
     packages_dir = Path(__file__).parent.parent.parent.parent.parent / "packages"
     python_packages = []
 
     for package_dir in packages_dir.iterdir():
-        if package_dir.is_dir() and package_dir.name.startswith("autogen"):
-            src_dir = package_dir / "src"
-            if src_dir.exists():
-                python_packages.append(src_dir)
+        if package_dir.is_dir():
+            # Check if this package is in our documented packages list
+            package_name = package_dir.name.replace("-", "_")
+            if package_name in DOCUMENTED_PACKAGES:
+                src_dir = package_dir / "src"
+                if src_dir.exists():
+                    python_packages.append(src_dir)
 
     return python_packages
 
 
 def get_module_hierarchy(package_root: Path) -> Dict[str, Set[str]]:
-    """Get the module hierarchy for a package."""
+    """Get the module hierarchy for a package, filtering only documented packages."""
     modules: Dict[str, Set[str]] = {}
 
     for root, dirs, files in os.walk(package_root):
@@ -52,10 +80,12 @@ def get_module_hierarchy(package_root: Path) -> Dict[str, Set[str]]:
                     module_name = '.'.join(module_parts)
                     package_name = module_parts[0]
 
-                    if package_name not in modules:
-                        modules[package_name] = set()
-
-                    modules[package_name].add(module_name)
+                    # Only include modules from documented packages
+                    if package_name in DOCUMENTED_PACKAGES:
+                        if package_name not in modules:
+                            modules[package_name] = set()
+
+                        modules[package_name].add(module_name)
 
         # Also check for directories with __init__.py (packages, excluding private)
         for dir_name in dirs:
@@ -69,48 +99,44 @@ def get_module_hierarchy(package_root: Path) -> Dict[str, Set[str]]:
                         module_name = '.'.join(module_parts)
                         package_name = module_parts[0]
 
-                        if package_name not in modules:
-                            modules[package_name] = set()
-
-                        modules[package_name].add(module_name)
+                        # Only include modules from documented packages
+                        if package_name in DOCUMENTED_PACKAGES:
+                            if package_name not in modules:
+                                modules[package_name] = set()
+
+                            modules[package_name].add(module_name)
 
     return modules
 
 
 def should_exclude_submodule(module_name: str, all_modules: Set[str]) -> bool:
     """Check if a submodule should be excluded to avoid duplicate documentation."""
-    # Define patterns where parent modules re-export submodule contents
-    exclusion_patterns = [
-        # task_centric_memory re-exports from memory_controller and utils
-        (r'^autogen_ext\.experimental\.task_centric_memory\.memory_controller$', 
-         'autogen_ext.experimental.task_centric_memory'),
-        # utils package re-exports from utils.apprentice and other utils submodules
-        (r'^autogen_ext\.experimental\.task_centric_memory\.utils\.apprentice$', 
-         'autogen_ext.experimental.task_centric_memory.utils'),
-        (r'^autogen_ext\.experimental\.task_centric_memory\.utils\.chat_completion_client_recorder$', 
-         'autogen_ext.experimental.task_centric_memory.utils'),
-        (r'^autogen_ext\.experimental\.task_centric_memory\.utils\.grader$', 
-         'autogen_ext.experimental.task_centric_memory.utils'),
-        (r'^autogen_ext\.experimental\.task_centric_memory\.utils\.page_logger$', 
-         'autogen_ext.experimental.task_centric_memory.utils'),
-        (r'^autogen_ext\.experimental\.task_centric_memory\.utils\.teachability$', 
-         'autogen_ext.experimental.task_centric_memory.utils'),
-        # Add more patterns as needed for other modules that re-export
-    ]
-
-    import re
-    for pattern, parent_module in exclusion_patterns:
+    for pattern, parent_module in EXCLUSION_PATTERNS:
         if re.match(pattern, module_name) and parent_module in all_modules:
             return True
 
     return False
 
 
+def clean_rst_files(reference_dir: Path) -> None:
+    """Clean existing RST files to ensure fresh generation."""
+    python_ref_dir = reference_dir / "python"
+    if python_ref_dir.exists():
+        print("🧹 Cleaning existing .rst files...")
+        rst_files = list(python_ref_dir.glob("*.rst"))
+        for rst_file in rst_files:
+            rst_file.unlink()
+        print(f"   Removed {len(rst_files)} existing .rst files")
+
+
 def generate_rst_files(package_roots: List[Path], reference_dir: Path) -> Set[str]:
     """Generate .rst files for all modules found in the packages."""
     python_ref_dir = reference_dir / "python"
     python_ref_dir.mkdir(exist_ok=True)
 
+    # Clean existing RST files first
+    clean_rst_files(reference_dir)
+
     generated_files = set()
     all_module_names = set()
 
@@ -161,42 +187,34 @@ def generate_rst_files(package_roots: List[Path], reference_dir: Path) -> Set[st
 
 def generate_toctree_from_rst_files(reference_dir: Path) -> Dict[str, List[str]]:
     """Generate toctree entries directly from existing .rst files."""
-    toctree_sections: Dict[str, List[str]] = {
-        "AutoGen AgentChat": [],
-        "AutoGen Core": [],
-        "AutoGen Extensions": []
-    }
+    # Initialize sections using constants
+    toctree_sections: Dict[str, List[str]] = {section: [] for section in PACKAGE_SECTIONS.values()}
 
     python_ref_dir = reference_dir / "python"
     if not python_ref_dir.exists():
         return toctree_sections
 
-    # Collect modules by package
-    agentchat_modules = []
-    core_modules = []
-    ext_modules = []
+    # Collect modules by package using constants
+    modules_by_section: Dict[str, List[str]] = {section: [] for section in PACKAGE_SECTIONS.values()}
 
     # Get all .rst files and organize them by package
     for rst_file in python_ref_dir.glob("*.rst"):
         module_name = rst_file.stem  # filename without .rst extension
-        rst_path = f"python/{module_name}"
 
-        if module_name.startswith("autogen_agentchat"):
-            agentchat_modules.append(module_name)
-        elif module_name.startswith("autogen_core"):
-            core_modules.append(module_name)
-        elif module_name.startswith("autogen_ext"):
-            ext_modules.append(module_name)
+        # Find which documented package this module belongs to
+        for package_prefix, section_name in PACKAGE_SECTIONS.items():
+            if module_name.startswith(package_prefix):
+                modules_by_section[section_name].append(module_name)
+                break
 
     # Sort modules so parent modules come before child modules
     def sort_modules_hierarchically(modules):
         """Sort modules so that parent modules come before child modules."""
         return sorted(modules, key=lambda x: (x.count('.'), x))
 
     # Apply hierarchical sorting and convert to rst paths
-    toctree_sections["AutoGen AgentChat"] = [f"python/{m}" for m in sort_modules_hierarchically(agentchat_modules)]
-    toctree_sections["AutoGen Core"] = [f"python/{m}" for m in sort_modules_hierarchically(core_modules)]
-    toctree_sections["AutoGen Extensions"] = [f"python/{m}" for m in sort_modules_hierarchically(ext_modules)]
+    for section_name, modules in modules_by_section.items():
+        toctree_sections[section_name] = [f"python/{m}" for m in sort_modules_hierarchically(modules)]
 
     return toctree_sections
 

diff --git a/python/packages/autogen-ext/src/autogen_ext/memory/chromadb/_chromadb.py b/python/packages/autogen-ext/src/autogen_ext/memory/chromadb/_chromadb.py
@@ -325,7 +325,7 @@ async def update_context(
                                     memory_strings.append(f"Memory: {memory.content}")
 
                             # Add to context with custom header
-                            memory_context = "IMPORTANT USER INFORMATION:\n" + "\n".join(memory_strings)
+                            memory_context = "IMPORTANT USER INFORMATION:\\n" + "\\n".join(memory_strings)
                             await model_context.add_message(SystemMessage(content=memory_context))
 
                         return UpdateContextResult(memories=query_results)

diff --git a/python/packages/autogen-ext/src/autogen_ext/memory/mem0.py b/python/packages/autogen-ext/src/autogen_ext/memory/mem0.py
@@ -23,15 +23,7 @@
 
 
 class Mem0MemoryConfig(BaseModel):
-    """Configuration for Mem0Memory component.
-
-    Attributes:
-        user_id: Optional user ID for memory operations. If not provided, a UUID will be generated.
-        limit: Maximum number of results to return in memory queries.
-        is_cloud: Whether to use cloud Mem0 client (True) or local client (False).
-        api_key: API key for cloud Mem0 client. Required if is_cloud=True.
-        config: Configuration dictionary for local Mem0 client. Required if is_cloud=False.
-    """
+    """Configuration for Mem0Memory component."""
 
     user_id: Optional[str] = Field(
         default=None, description="User ID for memory operations. If not provided, a UUID will be generated."
@@ -68,16 +60,17 @@ class Mem0Memory(Memory, Component[Mem0MemoryConfig], ComponentBase[Mem0MemoryCo
     relevant memories.
 
     Examples:
-        ```python
-        # Create a cloud Mem0Memory
-        memory = Mem0Memory(is_cloud=True)
 
-        # Add something to memory
-        await memory.add(MemoryContent(content="Important information to remember"))
+        .. code-block:: python
+
+            # Create a cloud Mem0Memory
+            memory = Mem0Memory(is_cloud=True)
+
+            # Add something to memory
+            await memory.add(MemoryContent(content="Important information to remember"))
 
-        # Retrieve memories with a search query
-        results = await memory.query("relevant information")
-        ```
+            # Retrieve memories with a search query
+            results = await memory.query("relevant information")
     """
 
     component_type = "memory"

diff --git a/python/packages/autogen-ext/src/autogen_ext/models/openai/_transformation/registry.py b/python/packages/autogen-ext/src/autogen_ext/models/openai/_transformation/registry.py
@@ -74,10 +74,16 @@ def register_transformer(api: str, model_family: str, transformer_map: Transform
     Registers a transformer map for a given model family.
 
     Example:
-        register_transformer("gpt-4o", {
-            UserMessage: user_message_to_oai,
-            SystemMessage: system_message_to_oai,
-        })
+
+        .. code-block:: python
+
+            register_transformer(
+                "gpt-4o",
+                {
+                    UserMessage: user_message_to_oai,
+                    SystemMessage: system_message_to_oai,
+                },
+            )
     """
     MESSAGE_TRANSFORMERS[api][model_family] = transformer_map