Merge pull request #39 from crestalnetwork/feat/skill-store

Refactor: skills architecture

CHANGELOG.md

@@ -1,5 +1,22 @@
 # Changelog
 
+## 2025-01-09
+
+### New Features
+- Add IntentKitSkill abstract class, for now, it has a skill store interface out of the box
+- Use skill store in Twitter skills, fetch skills will store the last processed tweet ID, prevent duplicate processing
+- CDP Skills Filter in Agent, choose the skills you want only, the less skills, the better performance
+
+### Improvements
+- Add a document for skill contributors: [How to add a new skill](docs/contributing/skills.md)
+
+## 2025-01-08
+
+### New Features
+- Add `prompt_append` to Agent, it will be appended to the entire prompt as system role, it has stronger priority
+- When you use web debug mode, you can see the entire prompt sent to the AI model
+- You can use new query param `thread` to debug any conversation thread
+
 ## 2025-01-07
 
 ### New Features

README.md

@@ -10,10 +10,10 @@ This project is currently in alpha stage and is not recommended for production u
 
 - 🤖 Multiple Agent Support
 - 🔄 Autonomous Agent Management
-- 🔗 Blockchain Integration (CDP for now, will add more)
+- 🔗 Blockchain Integration (EVM for now, will add more)
 - 🐦 Social Media Integration (Twitter,Telegram for now, will add more)
 - 🛠️ Extensible Skill System
-- 🔌 Extensible Plugin System
+- 🔌 Extensible Plugin System (WIP)
 
 ## Quick Start
 
@@ -142,6 +142,21 @@ See `example.env` for all available options.
 
 Contributions are welcome! Please read our [Contributing Guidelines](CONTRIBUTING.md) before submitting a pull request.
 
+### Contribute Skills
+
+If you want to add a skill collection, follow these steps:
+
+1. Create a new skill collection in the `skills/` directory
+2. Implement the skill interface
+3. Register the skill in `skill/YOUR_SKILL_COLLECTION/__init__.py`
+
+If you want to add a new skill, follow these steps:
+
+1. Create a new skill in the `skills/common/` directory
+2. Register the skill in `skills/common/__init__.py`
+
+See the [Skill Development Guide](docs/contributing/skills.md) for more information.
+
 ## License
 
 This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.

abstracts/skill.py

@@ -0,0 +1,87 @@
+from abc import ABC, abstractmethod
+from typing import Any, Dict, Optional
+
+from langchain_core.tools import BaseTool
+
+
+class IntentKitSkill(BaseTool):
+    """Abstract base class for IntentKit skills.
+    Will have predefined abilities.
+    """
+
+    agent_id: str
+    store: "SkillStoreABC"
+
+
+class SkillStoreABC(ABC):
+    """Abstract base class for skill data storage operations.
+
+    This class defines the interface for interacting with skill-related data
+    for both agents and threads.
+    """
+
+    @abstractmethod
+    def get_agent_skill_data(
+        self, agent_id: str, skill: str, key: str
+    ) -> Optional[Dict[str, Any]]:
+        """Get skill data for an agent.
+
+        Args:
+            agent_id: ID of the agent
+            skill: Name of the skill
+            key: Data key
+
+        Returns:
+            Dictionary containing the skill data if found, None otherwise
+        """
+        pass
+
+    @abstractmethod
+    def save_agent_skill_data(
+        self, agent_id: str, skill: str, key: str, data: Dict[str, Any]
+    ) -> None:
+        """Save or update skill data for an agent.
+
+        Args:
+            agent_id: ID of the agent
+            skill: Name of the skill
+            key: Data key
+            data: JSON data to store
+        """
+        pass
+
+    @abstractmethod
+    def get_thread_skill_data(
+        self, thread_id: str, skill: str, key: str
+    ) -> Optional[Dict[str, Any]]:
+        """Get skill data for a thread.
+
+        Args:
+            thread_id: ID of the thread
+            skill: Name of the skill
+            key: Data key
+
+        Returns:
+            Dictionary containing the skill data if found, None otherwise
+        """
+        pass
+
+    @abstractmethod
+    def save_thread_skill_data(
+        self,
+        thread_id: str,
+        agent_id: str,
+        skill: str,
+        key: str,
+        data: Dict[str, Any],
+    ) -> None:
+        """Save or update skill data for a thread.
+
+        Args:
+            thread_id: ID of the thread
+            agent_id: ID of the agent that owns this thread
+            skill: Name of the skill
+            key: Data key
+            data: JSON data to store
+        """
+        pass

app/config/config.py

@@ -63,6 +63,9 @@ def __init__(self):
         self.debug_resp = (
             self.load("DEBUG_RESP", "false") == "true"
         )  # Agent response with thought steps and time cost
+        self.debug_checkpoint = (
+            self.load("DEBUG_CHECKPOINT", "false") == "true"
+        )  # log with checkpoint
         # Internal
         self.internal_base_url = self.load("INTERNAL_BASE_URL", "http://intent-api")
         # Admin

app/core/engine.py

@@ -33,8 +33,9 @@
 from abstracts.graph import AgentState
 from app.config.config import config
 from app.core.graph import create_agent
+from app.core.skill import SkillStore
 from app.models.agent import Agent
-from app.models.db import get_coon, get_engine
+from app.models.db import get_coon, get_engine, get_session
 from skill_sets import get_skill_set
 from skills.common import get_common_skill
 from skills.crestal import get_crestal_skill
@@ -115,6 +116,9 @@ def initialize_agent(aid):
         # ==== Load skills
         tools: list[BaseTool] = []
 
+        # init skill store first
+        skill_store = SkillStore(get_session)
+
         # Configure CDP Agentkit Langchain Extension.
         if agent.cdp_enabled:
             values = {
@@ -133,7 +137,13 @@ def initialize_agent(aid):
                 db.commit()
             # Initialize CDP Agentkit Toolkit and get tools.
             cdp_toolkit = CdpToolkit.from_cdp_agentkit_wrapper(agentkit)
-            tools.extend(cdp_toolkit.get_tools())
+            cdp_tools = cdp_toolkit.get_tools()
+            # Filter the tools to only include the ones that in agent.cdp_skills.
+            if agent.cdp_skills and len(agent.cdp_skills) > 0:
+                cdp_tools = [
+                    tool for tool in cdp_tools if tool.name in agent.cdp_skills
+                ]
+            tools.extend(cdp_tools)
 
         # Twitter skills
         if (
@@ -143,7 +153,11 @@ def initialize_agent(aid):
         ):
             twitter_client = tweepy.Client(**agent.twitter_config)
             for skill in agent.twitter_skills:
-                tools.append(get_twitter_skill(skill, twitter_client))
+                try:
+                    s = get_twitter_skill(skill, twitter_client, skill_store, aid)
+                    tools.append(s)
+                except Exception as e:
+                    logger.warning(e)
 
         # Crestal skills
         if agent.crestal_skills:
@@ -186,7 +200,7 @@ def formatted_prompt(state: AgentState):
             tools=tools,
             checkpointer=memory,
             state_modifier=formatted_prompt,
-            debug=config.debug,
+            debug=config.debug_checkpoint,
         )
 
 

app/core/skill.py

@@ -0,0 +1,99 @@
+from typing import Any, Callable, Dict, Optional
+
+from abstracts.skill import SkillStoreABC
+from app.models.skill import AgentSkillData, ThreadSkillData
+
+
+class SkillStore(SkillStoreABC):
+    """Implementation of skill data storage operations.
+
+    This class provides concrete implementations for storing and retrieving
+    skill-related data for both agents and threads using SQLModel-based storage.
+
+    Args:
+        get_session: A callable that returns a database session
+    """
+
+    def __init__(self, get_session: Callable[[], Any]) -> None:
+        self._get_session = get_session
+
+    def get_agent_skill_data(
+        self, agent_id: str, skill: str, key: str
+    ) -> Optional[Dict[str, Any]]:
+        """Get skill data for an agent.
+
+        Args:
+            agent_id: ID of the agent
+            skill: Name of the skill
+            key: Data key
+
+        Returns:
+            Dictionary containing the skill data if found, None otherwise
+        """
+        with self._get_session() as session:
+            return AgentSkillData.get(agent_id, skill, key, session)
+
+    def save_agent_skill_data(
+        self, agent_id: str, skill: str, key: str, data: Dict[str, Any]
+    ) -> None:
+        """Save or update skill data for an agent.
+
+        Args:
+            agent_id: ID of the agent
+            skill: Name of the skill
+            key: Data key
+            data: JSON data to store
+        """
+        with self._get_session() as session:
+            skill_data = AgentSkillData(
+                agent_id=agent_id,
+                skill=skill,
+                key=key,
+                data=data,
+            )
+            skill_data.save(session)
+            session.commit()
+
+    def get_thread_skill_data(
+        self, thread_id: str, skill: str, key: str
+    ) -> Optional[Dict[str, Any]]:
+        """Get skill data for a thread.
+
+        Args:
+            thread_id: ID of the thread
+            skill: Name of the skill
+            key: Data key
+
+        Returns:
+            Dictionary containing the skill data if found, None otherwise
+        """
+        with self._get_session() as session:
+            return ThreadSkillData.get(thread_id, skill, key, session)
+
+    def save_thread_skill_data(
+        self,
+        thread_id: str,
+        agent_id: str,
+        skill: str,
+        key: str,
+        data: Dict[str, Any],
+    ) -> None:
+        """Save or update skill data for a thread.
+
+        Args:
+            thread_id: ID of the thread
+            agent_id: ID of the agent that owns this thread
+            skill: Name of the skill
+            key: Data key
+            data: JSON data to store
+        """
+        with self._get_session() as session:
+            skill_data = ThreadSkillData(
+                thread_id=thread_id,
+                agent_id=agent_id,
+                skill=skill,
+                key=key,
+                data=data,
+            )
+            skill_data.save(session)
+            session.commit()

app/models/agent.py

@@ -30,19 +30,21 @@ class Agent(SQLModel, table=True):
     # if cdp_enabled, will load cdp skills
     # if the cdp_skills is empty, will load all
     cdp_enabled: bool = Field(default=False)
-    cdp_skills: Optional[List[str]] = Field(sa_column=Column(JSONB, nullable=True))
+    cdp_skills: Optional[List[str]] = Field(sa_column=Column(ARRAY(String)))
     cdp_wallet_data: Optional[str]
     cdp_network_id: Optional[str]
     # if twitter_enabled, the twitter_entrypoint will be enabled, twitter_config will be checked
-    twitter_enabled: bool = Field(default=False)
+    twitter_enabled: bool = Field(default=False)  # TODO: to be deprecated
+    twitter_entrypoint_enabled: bool = Field(default=False)  # TODO: add for future use
     twitter_config: Optional[dict] = Field(sa_column=Column(JSONB, nullable=True))
     # twitter skills require config, but not require twitter_enabled flag.
     # As long as twitter_skills is not empty, the corresponding skills will be loaded.
     twitter_skills: Optional[List[str]] = Field(sa_column=Column(ARRAY(String)))
-    telegram_enabled: bool = Field(default=False)
+    # if telegram_enabled, the telegram_entrypoint will be enabled, telegram_config will be checked
+    telegram_enabled: bool = Field(default=False)  # TODO: to be deprecated
+    telegram_entrypoint_enabled: bool = Field(default=False)  # TODO: add for future use
     telegram_config: Optional[dict] = Field(sa_column=Column(JSONB, nullable=True))
-    # twitter skills require config, but not require twitter_enabled flag.
-    # As long as twitter_skills is not empty, the corresponding skills will be loaded.
+    # telegram skills not used for now
     telegram_skills: Optional[List[str]] = Field(sa_column=Column(ARRAY(String)))
     # crestal skills
     crestal_skills: Optional[List[str]] = Field(sa_column=Column(ARRAY(String)))

app/models/db.py

@@ -1,3 +1,4 @@
+from contextlib import contextmanager
 from typing import Generator
 from urllib.parse import quote_plus
 
@@ -69,6 +70,31 @@ def get_db() -> Generator[Session, None, None]:
         yield session
 
 
+@contextmanager
+def get_session() -> Session:
+    """Get a database session using a context manager.
+
+    This function is designed to be used with the 'with' statement,
+    ensuring proper session cleanup.
+
+    Returns:
+        Session: A SQLModel session that will be automatically closed
+
+    Example:
+        ```python
+        with get_session() as session:
+            # use session here
+            session.query(...)
+        # session is automatically closed
+        ```
+    """
+    session = Session(engine)
+    try:
+        yield session
+    finally:
+        session.close()
+
+
 def get_coon_str() -> str:
     return conn_str