From 45893ce89767e2a879a361da5db73010f2cb867e Mon Sep 17 00:00:00 2001 From: Din Date: Thu, 3 Apr 2025 01:13:54 +0500 Subject: [PATCH 1/8] add listener on 'page' in context --- README.md | 2 +- src/lmnr/sdk/browser/playwright_otel.py | 94 +++++++++++++++++++++++-- 2 files changed, 91 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index f57f8a3..b4688fe 100644 --- a/README.md +++ b/README.md @@ -264,6 +264,6 @@ async for chunk in client.agent.run( ): if chunk.chunkType == 'step': print(chunk.summary) - else if chunk.chunkType == 'finalOutput': + elif chunk.chunkType == 'finalOutput': print(chunk.content.result.content) ``` diff --git a/src/lmnr/sdk/browser/playwright_otel.py b/src/lmnr/sdk/browser/playwright_otel.py index d9c3212..b7e88ee 100644 --- a/src/lmnr/sdk/browser/playwright_otel.py +++ b/src/lmnr/sdk/browser/playwright_otel.py @@ -22,8 +22,11 @@ from wrapt import wrap_function_wrapper try: - from playwright.async_api import Browser - from playwright.sync_api import Browser as SyncBrowser + from playwright.async_api import Browser, BrowserContext + from playwright.sync_api import ( + Browser as SyncBrowser, + BrowserContext as SyncBrowserContext, + ) except ImportError as e: raise ImportError( f"Attempted to import {__file__}, but it is designed " @@ -84,8 +87,12 @@ def _wrap_new_browser_sync( set_span_in_context(span, get_current()) _context_spans[id(context)] = span span.set_attribute("lmnr.internal.has_browser_session", True) + trace_id = format(span.get_span_context().trace_id, "032x") + context.on( + "page", + lambda page: handle_navigation_sync(page, session_id, trace_id, client), + ) for page in context.pages: - trace_id = format(span.get_span_context().trace_id, "032x") handle_navigation_sync(page, session_id, trace_id, client) return browser @@ -106,12 +113,67 @@ async def _wrap_new_browser_async( set_span_in_context(span, get_current()) _context_spans[id(context)] = span span.set_attribute("lmnr.internal.has_browser_session", True) + trace_id = format(span.get_span_context().trace_id, "032x") + + async def handle_page_navigation(page): + return await handle_navigation_async(page, session_id, trace_id, client) + + context.on("page", handle_page_navigation) for page in context.pages: - trace_id = format(span.get_span_context().trace_id, "032x") await handle_navigation_async(page, session_id, trace_id, client) return browser +@with_tracer_and_client_wrapper +def _wrap_new_context_sync( + tracer: Tracer, client: LaminarClient, to_wrap, wrapped, instance, args, kwargs +): + context: SyncBrowserContext = wrapped(*args, **kwargs) + session_id = str(uuid.uuid4().hex) + span = get_current_span() + if span == INVALID_SPAN: + span = tracer.start_span( + name=f"{to_wrap.get('object')}.{to_wrap.get('method')}" + ) + set_span_in_context(span, get_current()) + _context_spans[id(context)] = span + span.set_attribute("lmnr.internal.has_browser_session", True) + trace_id = format(span.get_span_context().trace_id, "032x") + + context.on( + "page", + lambda page: handle_navigation_sync(page, session_id, trace_id, client), + ) + for page in context.pages: + handle_navigation_sync(page, session_id, trace_id, client) + return context + + +@with_tracer_and_client_wrapper +async def _wrap_new_context_async( + tracer: Tracer, client: AsyncLaminarClient, to_wrap, wrapped, instance, args, kwargs +): + context: SyncBrowserContext = await wrapped(*args, **kwargs) + session_id = str(uuid.uuid4().hex) + span = get_current_span() + if span == INVALID_SPAN: + span = tracer.start_span( + name=f"{to_wrap.get('object')}.{to_wrap.get('method')}" + ) + set_span_in_context(span, get_current()) + _context_spans[id(context)] = span + span.set_attribute("lmnr.internal.has_browser_session", True) + trace_id = format(span.get_span_context().trace_id, "032x") + + async def handle_page_navigation(page): + return await handle_navigation_async(page, session_id, trace_id, client) + + context.on("page", handle_page_navigation) + for page in context.pages: + await handle_navigation_async(page, session_id, trace_id, client) + return context + + @with_tracer_and_client_wrapper def _wrap_close_browser_sync( tracer: Tracer, @@ -191,6 +253,18 @@ async def _wrap_close_browser_async( "method": "close", "wrapper": _wrap_close_browser_sync, }, + { + "package": "playwright.sync_api", + "object": "Browser", + "method": "new_context", + "wrapper": _wrap_new_context_sync, + }, + { + "package": "playwright.sync_api", + "object": "BrowserType", + "method": "launch_persistent_context", + "wrapper": _wrap_new_context_sync, + }, ] WRAPPED_METHODS_ASYNC = [ @@ -230,6 +304,18 @@ async def _wrap_close_browser_async( "method": "close", "wrapper": _wrap_close_browser_async, }, + { + "package": "playwright.async_api", + "object": "Browser", + "method": "new_context", + "wrapper": _wrap_new_context_async, + }, + { + "package": "playwright.sync_api", + "object": "BrowserType", + "method": "launch_persistent_context", + "wrapper": _wrap_new_context_sync, + }, ] From 69a211dd3471acb79f396a561ce8027702f1cfda Mon Sep 17 00:00:00 2001 From: Din Date: Thu, 3 Apr 2025 12:22:45 +0500 Subject: [PATCH 2/8] quick fix: keep events as set, not array --- src/lmnr/sdk/browser/pw_utils.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/lmnr/sdk/browser/pw_utils.py b/src/lmnr/sdk/browser/pw_utils.py index 44a438a..57dfaa4 100644 --- a/src/lmnr/sdk/browser/pw_utils.py +++ b/src/lmnr/sdk/browser/pw_utils.py @@ -36,7 +36,7 @@ () => { const BATCH_SIZE = 1000; // Maximum events to store in memory - window.lmnrRrwebEventsBatch = []; + window.lmnrRrwebEventsBatch = new Set(); // Utility function to compress individual event data async function compressEventData(data) { @@ -50,8 +50,8 @@ window.lmnrGetAndClearEvents = () => { const events = window.lmnrRrwebEventsBatch; - window.lmnrRrwebEventsBatch = []; - return events; + window.lmnrRrwebEventsBatch = new Set(); + return Array.from(events); }; // Add heartbeat events @@ -62,11 +62,11 @@ timestamp: Date.now() }; - window.lmnrRrwebEventsBatch.push(heartbeat); + window.lmnrRrwebEventsBatch.add(heartbeat); // Prevent memory issues by limiting batch size - if (window.lmnrRrwebEventsBatch.length > BATCH_SIZE) { - window.lmnrRrwebEventsBatch = window.lmnrRrwebEventsBatch.slice(-BATCH_SIZE); + if (window.lmnrRrwebEventsBatch.size > BATCH_SIZE) { + window.lmnrRrwebEventsBatch = new Set(Array.from(window.lmnrRrwebEventsBatch).slice(-BATCH_SIZE)); } }, 1000); @@ -81,7 +81,7 @@ ...event, data: await compressEventData(event.data) }; - window.lmnrRrwebEventsBatch.push(compressedEvent); + window.lmnrRrwebEventsBatch.add(compressedEvent); } }); } From d2508aef5c1217bf4108b15fb8c30ffffd65beeb Mon Sep 17 00:00:00 2001 From: Din Date: Thu, 3 Apr 2025 17:43:40 +0500 Subject: [PATCH 3/8] rename flush + wip: add screenshots --- src/lmnr/sdk/laminar.py | 7 ++++++- src/lmnr/sdk/types.py | 17 ++++++++++------- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/src/lmnr/sdk/laminar.py b/src/lmnr/sdk/laminar.py index ac3e662..d7d5ed5 100644 --- a/src/lmnr/sdk/laminar.py +++ b/src/lmnr/sdk/laminar.py @@ -639,9 +639,14 @@ def deserialize_span_context( return LaminarSpanContext.deserialize(span_context) @classmethod - def shutdown(cls): + def flush(cls): TracerManager.flush() + @classmethod + def shutdown(cls): + # other shutdown logic could be added here + cls.flush() + @classmethod def set_session( cls, diff --git a/src/lmnr/sdk/types.py b/src/lmnr/sdk/types.py index 123517a..025cff0 100644 --- a/src/lmnr/sdk/types.py +++ b/src/lmnr/sdk/types.py @@ -367,19 +367,21 @@ class ModelProvider(str, Enum): class RunAgentRequest(pydantic.BaseModel): prompt: str - state: Optional[str] = None - parent_span_context: Optional[str] = None - model_provider: Optional[ModelProvider] = None - model: Optional[str] = None - stream: bool = False - enable_thinking: bool = True - cdp_url: Optional[str] = None + state: Optional[str] = pydantic.Field(default=None) + parent_span_context: Optional[str] = pydantic.Field(default=None) + model_provider: Optional[ModelProvider] = pydantic.Field(default=None) + model: Optional[str] = pydantic.Field(default=None) + stream: bool = pydantic.Field(default=False) + enable_thinking: bool = pydantic.Field(default=True) + cdp_url: Optional[str] = pydantic.Field(default=None) + return_screenshots: bool = pydantic.Field(default=False) def to_dict(self): result = { "prompt": self.prompt, "stream": self.stream, "enableThinking": self.enable_thinking, + "returnScreenshots": self.return_screenshots, } if self.state: result["state"] = self.state @@ -409,6 +411,7 @@ class StepChunkContent(pydantic.BaseModel): messageId: uuid.UUID actionResult: ActionResult summary: str + screenshot: Optional[str] = pydantic.Field(default=None) class FinalOutputChunkContent(pydantic.BaseModel): From fe10ec8d2f0b0e0480ed509d684ef420db855c70 Mon Sep 17 00:00:00 2001 From: Din Date: Thu, 3 Apr 2025 22:44:37 +0500 Subject: [PATCH 4/8] small updates for screenshots --- .../sdk/client/asynchronous/resources/agent.py | 14 ++++++++++---- src/lmnr/sdk/client/synchronous/resources/agent.py | 12 +++++++++--- src/lmnr/sdk/evaluations.py | 8 +++----- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/src/lmnr/sdk/client/asynchronous/resources/agent.py b/src/lmnr/sdk/client/asynchronous/resources/agent.py index 902ca3a..f068b3d 100644 --- a/src/lmnr/sdk/client/asynchronous/resources/agent.py +++ b/src/lmnr/sdk/client/asynchronous/resources/agent.py @@ -35,6 +35,7 @@ async def run( model_provider: Optional[ModelProvider] = None, model: Optional[str] = None, enable_thinking: bool = True, + return_screenshots: bool = False, ) -> AsyncIterator[RunAgentResponseChunk]: """Run Laminar index agent in streaming mode. @@ -45,7 +46,7 @@ async def run( model_provider (Optional[ModelProvider], optional): LLM model provider model (Optional[str], optional): LLM model name enable_thinking (bool, optional): whether to enable thinking on the underlying LLM. Default to True. - + return_screenshots (bool, optional): whether to return screenshots of the agent's states at every step. Default to False. Returns: AsyncIterator[RunAgentResponseChunk]: a generator of response chunks """ @@ -59,6 +60,7 @@ async def run( model_provider: Optional[ModelProvider] = None, model: Optional[str] = None, enable_thinking: bool = True, + return_screenshots: bool = False, ) -> AgentOutput: """Run Laminar index agent. @@ -68,7 +70,7 @@ async def run( model_provider (Optional[ModelProvider], optional): LLM model provider model (Optional[str], optional): LLM model name enable_thinking (bool, optional): whether to enable thinking on the underlying LLM. Default to True. - + return_screenshots (bool, optional): whether to return screenshots of the agent's states at every step. Default to False. Returns: AgentOutput: agent output """ @@ -83,6 +85,7 @@ async def run( model: Optional[str] = None, stream: Literal[False] = False, enable_thinking: bool = True, + return_screenshots: bool = False, ) -> AgentOutput: """Run Laminar index agent. @@ -93,7 +96,7 @@ async def run( model (Optional[str], optional): LLM model name stream (Literal[False], optional): whether to stream the agent's response enable_thinking (bool, optional): whether to enable thinking on the underlying LLM. Default to True. - + return_screenshots (bool, optional): whether to return screenshots of the agent's states at every step. Default to False. Returns: AgentOutput: agent output """ @@ -107,6 +110,7 @@ async def run( model: Optional[str] = None, stream: bool = False, enable_thinking: bool = True, + return_screenshots: bool = False, ) -> Union[AgentOutput, Awaitable[AsyncIterator[RunAgentResponseChunk]]]: """Run Laminar index agent. @@ -117,7 +121,7 @@ async def run( model (Optional[str], optional): LLM model name stream (bool, optional): whether to stream the agent's response enable_thinking (bool, optional): whether to enable thinking on the underlying LLM. Default to True. - + return_screenshots (bool, optional): whether to return screenshots of the agent's states at every step. Default to False. Returns: Union[AgentOutput, AsyncIterator[RunAgentResponseChunk]]: agent output or a generator of response chunks """ @@ -146,6 +150,7 @@ async def run( # https://aws.amazon.com/blogs/networking-and-content-delivery/introducing-nlb-tcp-configurable-idle-timeout/ stream=True, enable_thinking=enable_thinking, + return_screenshots=return_screenshots, ) # For streaming case, use a generator function @@ -181,6 +186,7 @@ async def __run_streaming( line = line[6:] if line: chunk = RunAgentResponseChunk.model_validate_json(line) + print(line) yield chunk.root async def __run_non_streaming(self, request: RunAgentRequest) -> AgentOutput: diff --git a/src/lmnr/sdk/client/synchronous/resources/agent.py b/src/lmnr/sdk/client/synchronous/resources/agent.py index 76fc87e..af68eb6 100644 --- a/src/lmnr/sdk/client/synchronous/resources/agent.py +++ b/src/lmnr/sdk/client/synchronous/resources/agent.py @@ -27,6 +27,7 @@ def run( model_provider: Optional[ModelProvider] = None, model: Optional[str] = None, enable_thinking: bool = True, + return_screenshots: bool = False, ) -> Generator[RunAgentResponseChunk, None, None]: """Run Laminar index agent in streaming mode. @@ -37,7 +38,7 @@ def run( model_provider (Optional[ModelProvider], optional): LLM model provider model (Optional[str], optional): LLM model name enable_thinking (bool, optional): whether to enable thinking on the underlying LLM. Default to True. - + return_screenshots (bool, optional): whether to return screenshots of the agent's states at every step. Default to False. Returns: Generator[RunAgentResponseChunk, None, None]: a generator of response chunks """ @@ -51,6 +52,7 @@ def run( model_provider: Optional[ModelProvider] = None, model: Optional[str] = None, enable_thinking: bool = True, + return_screenshots: bool = False, ) -> AgentOutput: """Run Laminar index agent. @@ -60,6 +62,7 @@ def run( model_provider (Optional[ModelProvider], optional): LLM model provider model (Optional[str], optional): LLM model name enable_thinking (bool, optional): whether to enable thinking on the underlying LLM. Default to True. + return_screenshots (bool, optional): whether to return screenshots of the agent's states at every step. Default to False. Returns: AgentOutput: agent output @@ -75,6 +78,7 @@ def run( model: Optional[str] = None, stream: Literal[False] = False, enable_thinking: bool = True, + return_screenshots: bool = False, ) -> AgentOutput: """Run Laminar index agent. @@ -85,7 +89,7 @@ def run( model (Optional[str], optional): LLM model name stream (Literal[False], optional): whether to stream the agent's response enable_thinking (bool, optional): whether to enable thinking on the underlying LLM. Default to True. - cdp_url (Optional[str], optional): CDP URL to connect to an existing browser session. + return_screenshots (bool, optional): whether to return screenshots of the agent's states at every step. Default to False. Returns: AgentOutput: agent output @@ -100,6 +104,7 @@ def run( model: Optional[str] = None, stream: bool = False, enable_thinking: bool = True, + return_screenshots: bool = False, ) -> Union[AgentOutput, Generator[RunAgentResponseChunk, None, None]]: """Run Laminar index agent. @@ -110,7 +115,7 @@ def run( model (Optional[str], optional): LLM model name stream (bool, optional): whether to stream the agent's response enable_thinking (bool, optional): whether to enable thinking on the underlying LLM. Default to True. - cdp_url (Optional[str], optional): CDP URL to connect to an existing browser session. + return_screenshots (bool, optional): whether to return screenshots of the agent's states at every step. Default to False. Returns: Union[AgentOutput, Generator[RunAgentResponseChunk, None, None]]: agent output or a generator of response chunks @@ -140,6 +145,7 @@ def run( # https://aws.amazon.com/blogs/networking-and-content-delivery/introducing-nlb-tcp-configurable-idle-timeout/ stream=True, enable_thinking=enable_thinking, + return_screenshots=return_screenshots, ) # For streaming case, use a generator function diff --git a/src/lmnr/sdk/evaluations.py b/src/lmnr/sdk/evaluations.py index 94144ff..4162ece 100644 --- a/src/lmnr/sdk/evaluations.py +++ b/src/lmnr/sdk/evaluations.py @@ -35,15 +35,13 @@ def get_evaluation_url( project_id: str, evaluation_id: str, base_url: Optional[str] = None ): - if not base_url: + if not base_url or base_url == "https://api.lmnr.ai": base_url = "https://www.lmnr.ai" url = base_url - if url.endswith("/"): - url = url[:-1] + url = re.sub(r"\/$", "", url) if url.endswith("localhost") or url.endswith("127.0.0.1"): - # We best effort assume that the frontend is running on port 3000 - # TODO: expose the frontend port? + # We best effort assume that the frontend is running on port 5667 url = url + ":5667" return f"{url}/project/{project_id}/evaluations/{evaluation_id}" From 97e03c64d696545e4c61043f034384e7da8f25b5 Mon Sep 17 00:00:00 2001 From: Din Date: Fri, 4 Apr 2025 13:21:25 +0500 Subject: [PATCH 5/8] v0.5.1 new tab --- pyproject.toml | 2 +- src/lmnr/version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9f2aac3..68b721e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ [project] name = "lmnr" -version = "0.5.0" +version = "0.5.1" description = "Python SDK for Laminar" authors = [ { name = "lmnr.ai", email = "founders@lmnr.ai" } diff --git a/src/lmnr/version.py b/src/lmnr/version.py index a984841..78cfb7d 100644 --- a/src/lmnr/version.py +++ b/src/lmnr/version.py @@ -3,7 +3,7 @@ from packaging import version -__version__ = "0.5.0" +__version__ = "0.5.1" PYTHON_VERSION = f"{sys.version_info.major}.{sys.version_info.minor}" From 945ccfb4bf5f3a0387312beb219f8ed66e279463 Mon Sep 17 00:00:00 2001 From: Din Date: Fri, 4 Apr 2025 14:32:46 +0500 Subject: [PATCH 6/8] fix flush --- src/lmnr/openllmetry_sdk/__init__.py | 5 ++--- src/lmnr/openllmetry_sdk/tracing/tracing.py | 7 ++++++- src/lmnr/sdk/laminar.py | 12 ++++++++++-- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/src/lmnr/openllmetry_sdk/__init__.py b/src/lmnr/openllmetry_sdk/__init__.py index e96406e..112f162 100644 --- a/src/lmnr/openllmetry_sdk/__init__.py +++ b/src/lmnr/openllmetry_sdk/__init__.py @@ -67,6 +67,5 @@ def init( ) @staticmethod - def flush(): - if getattr(TracerManager, "__tracer_wrapper", None): - TracerManager.__tracer_wrapper.flush() + def flush() -> bool: + return TracerManager.__tracer_wrapper.flush() diff --git a/src/lmnr/openllmetry_sdk/tracing/tracing.py b/src/lmnr/openllmetry_sdk/tracing/tracing.py index 0e7ce28..a923744 100644 --- a/src/lmnr/openllmetry_sdk/tracing/tracing.py +++ b/src/lmnr/openllmetry_sdk/tracing/tracing.py @@ -236,8 +236,13 @@ def clear(cls): cls.__span_id_to_path = {} cls.__span_id_lists = {} - def flush(self): + def shutdown(self): self.__spans_processor.force_flush() + self.__spans_processor.shutdown() + self.__tracer_provider.shutdown() + + def flush(self): + return self.__spans_processor.force_flush() def get_tracer(self): return self.__tracer_provider.get_tracer(TRACER_NAME) diff --git a/src/lmnr/sdk/laminar.py b/src/lmnr/sdk/laminar.py index d7d5ed5..4e73f83 100644 --- a/src/lmnr/sdk/laminar.py +++ b/src/lmnr/sdk/laminar.py @@ -639,8 +639,16 @@ def deserialize_span_context( return LaminarSpanContext.deserialize(span_context) @classmethod - def flush(cls): - TracerManager.flush() + def flush(cls) -> bool: + """Flush the internal tracer. + + Returns: + bool: True if the tracer was flushed, False otherwise + (e.g. no tracer or timeout). + """ + if not cls.is_initialized(): + return False + return TracerManager.flush() @classmethod def shutdown(cls): From b532f8349572816711064b1aee225b44a170fbfd Mon Sep 17 00:00:00 2001 From: Din Date: Fri, 4 Apr 2025 14:44:22 +0500 Subject: [PATCH 7/8] rebase and update docstring --- src/lmnr/sdk/evaluations.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lmnr/sdk/evaluations.py b/src/lmnr/sdk/evaluations.py index 4162ece..c1261b9 100644 --- a/src/lmnr/sdk/evaluations.py +++ b/src/lmnr/sdk/evaluations.py @@ -406,8 +406,8 @@ def evaluate( If there is no event loop, creates it and runs the evaluation until completion. - If there is an event loop, schedules the evaluation as a task in the - event loop and returns an awaitable handle. + If there is an event loop, returns an awaitable handle immediately. IMPORTANT: + You must await the call to `evaluate`. Parameters: data (Union[list[EvaluationDatapoint|dict]], EvaluationDataset]):\ From 404ad3ca5bb9c7992f8fed7b0c24eab4ecc6f395 Mon Sep 17 00:00:00 2001 From: Din Date: Fri, 4 Apr 2025 14:49:33 +0500 Subject: [PATCH 8/8] fix ellipsis comment: remove debug print --- src/lmnr/sdk/client/asynchronous/resources/agent.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/lmnr/sdk/client/asynchronous/resources/agent.py b/src/lmnr/sdk/client/asynchronous/resources/agent.py index f068b3d..4ad2ab1 100644 --- a/src/lmnr/sdk/client/asynchronous/resources/agent.py +++ b/src/lmnr/sdk/client/asynchronous/resources/agent.py @@ -186,7 +186,6 @@ async def __run_streaming( line = line[6:] if line: chunk = RunAgentResponseChunk.model_validate_json(line) - print(line) yield chunk.root async def __run_non_streaming(self, request: RunAgentRequest) -> AgentOutput: