Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feat search conversations #5775

Draft
wants to merge 47 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
47 commits
Select commit Hold shift + click to select a range
74682ef
Conversation endpoint
tofarr Dec 23, 2024
fbae222
Merge branch 'main' into feat-search-conversations
tofarr Dec 23, 2024
dc82094
Added endpoint for conversation search
tofarr Dec 23, 2024
e3f9b7e
WIP
tofarr Dec 23, 2024
3ac818b
WIP
tofarr Dec 23, 2024
76f359c
Added specific types
tofarr Dec 23, 2024
251b47f
Merge branch 'main' into feat-search-conversations
tofarr Dec 26, 2024
413e1b7
Merge branch 'main' into feat-search-conversations
tofarr Dec 27, 2024
fe0acad
Metadata
tofarr Dec 27, 2024
5edc93d
Search conversations endpoint
tofarr Dec 27, 2024
8781657
Add unit tests for search_utils and implement page_id_to_offset
openhands-agent Dec 27, 2024
bdbce31
Made tests more generic
tofarr Dec 27, 2024
b182ba0
Removed something I don't know how it got in there
tofarr Dec 27, 2024
ae9ab6d
Now returning selected repository
tofarr Dec 27, 2024
74a2068
Added get conversation endpoint
tofarr Dec 27, 2024
9d43f1f
Merge branch 'main' into feat-search-conversations
tofarr Dec 27, 2024
8d0896b
Lint fix
tofarr Dec 27, 2024
ede28d5
dded tests
tofarr Dec 27, 2024
6b30f2a
Lint fixes
tofarr Dec 27, 2024
4aeeb3f
Merge branch 'main' into feat-search-conversations
tofarr Dec 27, 2024
7bd01d2
WIP
tofarr Dec 27, 2024
0220b69
Ruff
tofarr Dec 27, 2024
4172365
Lint fix
tofarr Dec 27, 2024
4305335
WIP
tofarr Dec 27, 2024
a90dd21
Fix tests
tofarr Dec 27, 2024
a8379d7
WIP
tofarr Dec 27, 2024
e9d3f4b
WIP
tofarr Dec 27, 2024
02cd435
WIP
tofarr Dec 27, 2024
c54a050
Merge branch 'main' into feat-search-conversations
tofarr Dec 27, 2024
d7ccd78
WIP
tofarr Dec 27, 2024
64d8dc8
WIP
tofarr Dec 27, 2024
392e1cc
Merge branch 'main' into feat-search-conversations
tofarr Dec 28, 2024
77f0b57
Merge branch 'main' into feat-search-conversations
tofarr Dec 30, 2024
9f6b63c
Merge branch 'feat-search-conversations' of github.com:All-Hands-AI/O…
tofarr Dec 30, 2024
858d4cb
Ruff
tofarr Dec 30, 2024
5b197b4
Merge branch 'main' into feat-search-conversations
tofarr Dec 31, 2024
4181d09
WIP
tofarr Dec 31, 2024
c204108
Ruff
tofarr Dec 31, 2024
b1e161b
Merge branch 'main' into feat-search-conversations
tofarr Dec 31, 2024
9d171b5
WIP
tofarr Dec 31, 2024
2312dba
Merge branch 'main' into feat-search-conversations
tofarr Dec 31, 2024
3707971
WIP
tofarr Dec 31, 2024
8f70910
Merge branch 'main' into feat-search-conversations
tofarr Dec 31, 2024
bdd23da
WIP
tofarr Dec 31, 2024
0e03376
Test fixes
tofarr Dec 31, 2024
d3d1b28
Ruff
tofarr Dec 31, 2024
c5f8850
Lint fix
tofarr Dec 31, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions openhands/server/data_models/conversation_info.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
from dataclasses import dataclass
from datetime import datetime

from openhands.server.data_models.conversation_status import ConversationStatus


@dataclass
class ConversationInfo:
"""Information about a conversation"""

id: str
title: str | None = None
last_updated_at: datetime | None = None
status: ConversationStatus = ConversationStatus.STOPPED
selected_repository: str | None = None
9 changes: 9 additions & 0 deletions openhands/server/data_models/conversation_info_result_set.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
from dataclasses import dataclass, field

from openhands.server.data_models.conversation_info import ConversationInfo


@dataclass
class ConversationInfoResultSet:
results: list[ConversationInfo] = field(default_factory=list)
next_page_id: str | None = None
1 change: 1 addition & 0 deletions openhands/server/data_models/conversation_metadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,3 +6,4 @@ class ConversationMetadata:
conversation_id: str
github_user_id: str
selected_repository: str | None
title: str | None = None
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
from dataclasses import dataclass, field

from openhands.server.data_models.conversation_metadata import ConversationMetadata


@dataclass
class ConversationMetadataResultSet:
results: list[ConversationMetadata] = field(default_factory=list)
next_page_id: str | None = None
6 changes: 6 additions & 0 deletions openhands/server/data_models/conversation_status.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
from enum import Enum


class ConversationStatus(Enum):
RUNNING = 'RUNNING'
STOPPED = 'STOPPED'
2 changes: 1 addition & 1 deletion openhands/server/middleware.py
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ def _should_attach(self, request) -> bool:
if request.url.path.startswith('/api/conversation'):
# FIXME: we should be able to use path_params
path_parts = request.url.path.split('/')
if len(path_parts) > 3:
if len(path_parts) > 4:
conversation_id = request.url.path.split('/')[3]
if not conversation_id:
return False
Expand Down
122 changes: 117 additions & 5 deletions openhands/server/routes/new_conversation.py
Original file line number Diff line number Diff line change
@@ -1,16 +1,26 @@
import json
import uuid
from datetime import datetime

from fastapi import APIRouter, Request
from fastapi.responses import JSONResponse
from github import Github
from pydantic import BaseModel

from openhands.core.logger import openhands_logger as logger
from openhands.server.data_models.conversation_info import ConversationInfo
from openhands.server.data_models.conversation_metadata import ConversationMetadata
from openhands.server.data_models.conversation_info_result_set import (
ConversationInfoResultSet,
)
from openhands.server.data_models.conversation_status import ConversationStatus
from openhands.server.routes.settings import ConversationStoreImpl, SettingsStoreImpl
from openhands.server.session.conversation_init_data import ConversationInitData
from openhands.server.shared import config, session_manager
from openhands.utils.async_utils import call_sync_from_async
from openhands.storage.files import FileStore
from openhands.storage.locations import get_conversation_events_dir
from openhands.utils.async_utils import call_sync_from_async, wait_all
from openhands.utils.search_utils import offset_to_page_id, page_id_to_offset

app = APIRouter(prefix='/api')

Expand All @@ -29,9 +39,7 @@ async def new_conversation(request: Request, data: InitSessionRequest):
using the returned conversation ID
"""
logger.info('Initializing new conversation')
github_token = ''
if data.github_token:
github_token = data.github_token
github_token = data.github_token or ''

logger.info('Loading settings')
settings_store = await SettingsStoreImpl.get_instance(config, github_token)
Expand All @@ -45,7 +53,6 @@ async def new_conversation(request: Request, data: InitSessionRequest):
session_init_args['github_token'] = github_token
session_init_args['selected_repository'] = data.selected_repository
conversation_init_data = ConversationInitData(**session_init_args)

logger.info('Loading conversation store')
conversation_store = await ConversationStoreImpl.get_instance(config, github_token)
logger.info('Conversation store loaded')
Expand Down Expand Up @@ -78,3 +85,108 @@ async def new_conversation(request: Request, data: InitSessionRequest):
)
logger.info(f'Finished initializing conversation {conversation_id}')
return JSONResponse(content={'status': 'ok', 'conversation_id': conversation_id})


@app.get('/conversations')
async def search_conversations(
request: Request,
page_id: str | None = None,
limit: int = 20,
) -> ConversationInfoResultSet:
github_token = getattr(request.state, 'github_token', '') or ''
conversation_store = await ConversationStoreImpl.get_instance(config, github_token)
conversation_metadata_result_set = await conversation_store.search(page_id, limit)
conversation_ids = set(
conversation.conversation_id
for conversation in conversation_metadata_result_set.results
)
running_conversations = await session_manager.get_agent_loop_running(
set(conversation_ids)
)
result = ConversationInfoResultSet(
results=await wait_all(
_get_conversation_info(
conversation=conversation,
is_running=conversation.conversation_id in running_conversations,
)
for conversation in conversation_metadata_result_set.results
),
next_page_id=conversation_metadata_result_set.next_page_id,
)
return result


async def _get_conversation_info(
conversation: ConversationMetadata,
is_running: bool,
) -> ConversationInfo | None:
try:
file_store = session_manager.file_store
events_dir = get_conversation_events_dir(conversation.conversation_id)
events = file_store.list(events_dir)
events = sorted(events)
event_path = events[-1]
event = json.loads(file_store.read(event_path))
return ConversationInfo(
id=conversation.conversation_id,
title=conversation.title,
last_updated_at=datetime.fromisoformat(event.get('timestamp')),
selected_repository=conversation.selected_repository,
status=ConversationStatus.RUNNING
if is_running
else ConversationStatus.STOPPED,
)
except Exception: # type: ignore
logger.warning(
f'Error loading conversation: {conversation.conversation_id}',
exc_info=True,
stack_info=True,
)
return None


@app.get('/conversations/{conversation_id}')
async def get_conversation(
conversation_id: str, request: Request
) -> ConversationInfo | None:
github_token = getattr(request.state, 'github_token', '') or ''
conversation_store = await ConversationStoreImpl.get_instance(config, github_token)
try:
metadata = await conversation_store.get_metadata(conversation_id)
is_running = await session_manager.is_agent_loop_running(conversation_id)
conversation_info = await _get_conversation_info(metadata, is_running)
return conversation_info
except FileNotFoundError:
return None


@app.post('/conversations/{conversation_id}')
async def update_conversation(
conversation_id: str, title: str, request: Request
) -> bool:
github_token = getattr(request.state, 'github_token', '') or ''
conversation_store = await ConversationStoreImpl.get_instance(config, github_token)
metadata = await conversation_store.get_metadata(conversation_id)
if not metadata:
return False
metadata.title = title
await conversation_store.save_metadata(metadata)
return True


@app.delete('/conversations/{conversation_id}')
async def delete_conversation(
conversation_id: str,
request: Request,
) -> bool:
github_token = getattr(request.state, 'github_token', '') or ''
conversation_store = await ConversationStoreImpl.get_instance(config, github_token)
try:
await conversation_store.get_metadata(conversation_id)
except FileNotFoundError:
return False
is_running = await session_manager.is_agent_loop_running(conversation_id)
if is_running:
return False
await conversation_store.delete_metadata(conversation_id)
return True
8 changes: 2 additions & 6 deletions openhands/server/routes/settings.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,8 @@


@app.get('/settings')
async def load_settings(
request: Request,
) -> Settings | None:
github_token = ''
if hasattr(request.state, 'github_token'):
github_token = request.state.github_token
async def load_settings(request: Request) -> Settings | None:
github_token = getattr(request.state, 'github_token', '') or ''
try:
settings_store = await SettingsStoreImpl.get_instance(config, github_token)
settings = await settings_store.load()
Expand Down
15 changes: 15 additions & 0 deletions openhands/storage/conversation/conversation_store.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@

from openhands.core.config.app_config import AppConfig
from openhands.server.data_models.conversation_metadata import ConversationMetadata
from openhands.server.data_models.conversation_metadata_result_set import (
ConversationMetadataResultSet,
)


class ConversationStore(ABC):
Expand All @@ -19,10 +22,22 @@ async def save_metadata(self, metadata: ConversationMetadata):
async def get_metadata(self, conversation_id: str) -> ConversationMetadata:
"""Load conversation metadata"""

@abstractmethod
async def delete_metadata(self, conversation_id: str) -> None:
"""delete conversation metadata"""

@abstractmethod
async def exists(self, conversation_id: str) -> bool:
"""Check if conversation exists"""

@abstractmethod
async def search(
self,
page_id: str | None = None,
limit: int = 20,
) -> ConversationMetadataResultSet:
"""Search conversations"""

@classmethod
@abstractmethod
async def get_instance(
Expand Down
46 changes: 45 additions & 1 deletion openhands/storage/conversation/file_conversation_store.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,20 @@
from dataclasses import dataclass

from openhands.core.config.app_config import AppConfig
from openhands.core.logger import openhands_logger as logger
from openhands.server.data_models.conversation_metadata_result_set import (
ConversationMetadataResultSet,
)
from openhands.storage import get_file_store
from openhands.storage.conversation.conversation_store import ConversationStore
from openhands.server.data_models.conversation_metadata import ConversationMetadata
from openhands.storage.files import FileStore
from openhands.storage.locations import get_conversation_metadata_filename
from openhands.storage.locations import (
CONVERSATION_BASE_DIR,
get_conversation_metadata_filename,
)
from openhands.utils.async_utils import call_sync_from_async
from openhands.utils.search_utils import offset_to_page_id, page_id_to_offset


@dataclass
Expand All @@ -26,6 +34,10 @@ async def get_metadata(self, conversation_id: str) -> ConversationMetadata:
json_str = await call_sync_from_async(self.file_store.read, path)
return ConversationMetadata(**json.loads(json_str))

async def delete_metadata(self, conversation_id: str) -> None:
path = self.get_conversation_metadata_filename(conversation_id)
await call_sync_from_async(self.file_store.delete, path)

async def exists(self, conversation_id: str) -> bool:
path = self.get_conversation_metadata_filename(conversation_id)
try:
Expand All @@ -34,6 +46,38 @@ async def exists(self, conversation_id: str) -> bool:
except FileNotFoundError:
return False

async def search(
self,
page_id: str | None = None,
limit: int = 20,
) -> ConversationMetadataResultSet:
conversations: list[ConversationMetadata] = []
metadata_dir = self.get_conversation_metadata_dir()
conversation_ids = [
path.split('/')[-2]
for path in self.file_store.list(metadata_dir)
if not path.startswith(f'{metadata_dir}/.')
]
num_conversations = len(conversation_ids)
start = page_id_to_offset(page_id)
end = min(limit + start, num_conversations)
conversation_ids = conversation_ids[start:end]
conversations = []
for conversation_id in conversation_ids:
try:
conversations.append(await self.get_metadata(conversation_id))
except Exception:
logger.warning(
f'Error loading conversation: {conversation_id}',
exc_info=True,
stack_info=True,
)
next_page_id = offset_to_page_id(end, end < num_conversations)
return ConversationMetadataResultSet(conversations, next_page_id)

def get_conversation_metadata_dir(self) -> str:
return CONVERSATION_BASE_DIR

def get_conversation_metadata_filename(self, conversation_id: str) -> str:
return get_conversation_metadata_filename(conversation_id)

Expand Down
15 changes: 15 additions & 0 deletions openhands/utils/search_utils.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
import base64


def offset_to_page_id(offset: int, has_next: bool) -> str | None:
if not has_next:
return None
next_page_id = base64.b64encode(str(offset).encode()).decode()
return next_page_id


def page_id_to_offset(page_id: str | None) -> int:
if not page_id:
return 0
offset = int(base64.b64decode(page_id).decode())
return offset
Loading
Loading