add framework for adding new model evaluators

Author: Divij97

prompttools/utils/autoeval.py

@@ -5,12 +5,15 @@
 # LICENSE file in the root directory of this source tree.
 
 
-import os
 from typing import Dict
+<<<<<<< HEAD
 import openai
 import pandas.core.series
+=======
+>>>>>>> 862ca6e (add framework for adding new model evaluators)
 import jinja2
-from .error import PromptToolsUtilityError
+
+from .model_evaluators.EvaluatorUtils import get_evaluator_for_model
 
 EVALUATION_SYSTEM_PROMPT = """
 Determine whether or not the response is following directions.
@@ -21,18 +24,14 @@
 EVALUATION_USER_TEMPLATE = """
 PROMPT: {{prompt}}
 RESPONSE: {{response}}
-ANSWER:
 """
 
 
-def _get_messages(prompt: str, response: str):
+def _get_user_prompt(prompt: str, response: str):
     environment = jinja2.Environment()
     template = environment.from_string(EVALUATION_USER_TEMPLATE)
-    user_message = template.render({"prompt": prompt, "response": response})
-    return [
-        {"role": "system", "content": EVALUATION_SYSTEM_PROMPT},
-        {"role": "user", "content": user_message},
-    ]
+    user_prompt = template.render({"prompt": prompt, "response": response})
+    return user_prompt
 
 
 def compute(prompt: str, response: str, model: str = "gpt-4") -> float:
@@ -46,10 +45,8 @@ def compute(prompt: str, response: str, model: str = "gpt-4") -> float:
         model (str): The OpenAI chat model to use for generating an expected response.
             Defaults to GPT-4.
     """
-    if not os.environ["OPENAI_API_KEY"]:
-        raise PromptToolsUtilityError
-    evaluation = openai.ChatCompletion.create(model=model, messages=_get_messages(prompt, response))
-    return 1.0 if "RIGHT" in evaluation["choices"][0]["message"]["content"] else 0.0
+    response = get_evaluator_for_model(model).evaluate(model, EVALUATION_SYSTEM_PROMPT, _get_user_prompt())
+    return 1.0 if "RIGHT" in response else 0.0
 
 
 def evaluate(prompt: str, response: str, _metadata: Dict) -> float:

prompttools/utils/autoeval_from_expected.py

@@ -5,11 +5,13 @@
 # LICENSE file in the root directory of this source tree.
 
 
-import os
-import openai
 import jinja2
+<<<<<<< HEAD
 import pandas
 from .error import PromptToolsUtilityError
+=======
+from .model_evaluators.EvaluatorUtils import get_evaluator_for_model
+>>>>>>> 862ca6e (add framework for adding new model evaluators)
 
 EVALUATION_SYSTEM_PROMPT = """
 You are a grader evaluating responses to math questions.
@@ -25,14 +27,11 @@
 """
 
 
-def _get_messages(prompt: str, expected: str, response: str):
+def _get_user_prompt(prompt: str, expected: str, response: str):
     environment = jinja2.Environment()
     template = environment.from_string(EVALUATION_USER_TEMPLATE)
-    user_message = template.render({"prompt": prompt, "expected": expected, "actual": response})
-    return [
-        {"role": "system", "content": EVALUATION_SYSTEM_PROMPT},
-        {"role": "user", "content": user_message},
-    ]
+    user_prompt = template.render({"prompt": prompt, "expected": expected, "actual": response})
+    return user_prompt
 
 
 # TODO: Should this be removed since no one is using it?
@@ -47,9 +46,9 @@ def compute(prompt: str, expected: str, response: str, model: str = "gpt-4") ->
         model (str): The OpenAI chat model to use for generating an expected response.
             Defaults to GPT-4.
     """
-    if not os.environ["OPENAI_API_KEY"]:
-        raise PromptToolsUtilityError("Missing API key for evaluation.")
-    evaluation = openai.ChatCompletion.create(model=model, messages=_get_messages(prompt, expected, response))
+    evaluation = get_evaluator_for_model(model).evaluate(
+        model, EVALUATION_SYSTEM_PROMPT, _get_user_prompt(prompt, expected, response)
+    )
     return 1.0 if "RIGHT" in evaluation["choices"][0]["message"]["content"] else 0.0
 
 

prompttools/utils/autoeval_scoring.py

@@ -5,38 +5,38 @@
 # LICENSE file in the root directory of this source tree.
 
 
+<<<<<<< HEAD
 import os
 import pandas.core.series
+=======
+>>>>>>> 862ca6e (add framework for adding new model evaluators)
 import jinja2
 
+from prompttools.utils.model_evaluators.EvaluatorUtils import get_evaluator_for_model
+
 try:
     import anthropic
 except ImportError:
     anthropic = None
 
-
-AUTO_EVAL_PROMPT_TEMPLATE = """
-{{HUMAN_PROMPT}} Given the fact {{fact}}
-
-Evaluate the following Answer on a scale from 1 - 7. Please only respond with an integer from 1 - 7 with no other text.
+EVALUATION_SYSTEM_PROMPT = """
+Given the Fact and Statement, Evaluate the statement on a scale from 1 - 7.
+Please only respond with an integer from 1 - 7 with no other text.
 Lower score means the answer is factually wrong, higher score means the answer is correct. A medium score for
-uncertain but not wrong.
-
-Answer: {{model_answer}}
+uncertain but not wrong"""
 
-{{AI_PROMPT}}
-"""
+USER_PROMPT = """
+Fact: {{fact}}
+Statement: {{model_answer}}"""
 
 
-def _generate_auto_eval_prompt(fact: str, model_answer: str):
+def _generate_user_prompt(fact: str, model_answer: str):
     environment = jinja2.Environment()
-    template = environment.from_string(AUTO_EVAL_PROMPT_TEMPLATE)
+    template = environment.from_string(USER_PROMPT)
     auto_eval_prompt = template.render(
         {
-            "HUMAN_PROMPT": anthropic.HUMAN_PROMPT,
-            "AI_PROMPT": anthropic.AI_PROMPT,
             "fact": fact,
-            "model_answer": model_answer,
+            "statement": model_answer,
         }
     )
     return auto_eval_prompt
@@ -54,13 +54,10 @@ def compute(fact: str, model_answer: str, model: str = "claude-2") -> float:
         model (str): The model that will be judging how close is the response from the truth.
             Defaults to Claude 2.
     """
-    if not os.environ["ANTHROPIC_API_KEY"]:
-        raise RuntimeError("Missing API key for evaluation.")
-    client = anthropic.Anthropic(api_key=os.environ["ANTHROPIC_API_KEY"])
-    completion_response = client.completions.create(
-        max_tokens_to_sample=100, model=model, prompt=_generate_auto_eval_prompt(fact, model_answer)
+    response = get_evaluator_for_model(model).evaluate(
+        model, EVALUATION_SYSTEM_PROMPT, _generate_user_prompt(fact, model_answer)
     )
-    return int(completion_response.completion)
+    return int(response)
 
 
 def autoeval_scoring(row: pandas.core.series.Series, expected: str, response_column_name: str = "response") -> float:
@@ -73,9 +70,13 @@ def autoeval_scoring(row: pandas.core.series.Series, expected: str, response_col
         expected (str): the expected response
         response_column_name (str): name of the column that contains the model's response, defaults to ``"response"``
     """
+<<<<<<< HEAD
     if anthropic is None:
         raise ModuleNotFoundError(
             "Package `anthropic` is required to be installed to use this experiment."
             "Please use `pip install anthropic` to install the package"
         )
     return compute(fact=expected, model_answer=row[response_column_name])
+=======
+    return compute(fact=expected, model_answer=response)
+>>>>>>> 862ca6e (add framework for adding new model evaluators)

prompttools/utils/model_evaluators/AnthropicEvaluator.py

@@ -0,0 +1,55 @@
+# Copyright (c) Hegel AI, Inc.
+# All rights reserved.
+#
+# This source code's license can be found in the
+# LICENSE file in the root directory of this source tree.
+
+
+from overrides import override
+from .ModelEvaluator import ModelEvaluator
+import jinja2
+import anthropic
+import os
+
+ANTHROPIC_API_AUTOEVAL_TEMPLATE = """
+{{HUMAN_PROMPT}} {{EVALUATION_SYSTEM_PROMPT}}
+{{USER_MESSAGE}} {{AI_PROMPT}}
+"""
+
+
+class AnthropicEvaluator(ModelEvaluator):
+    def __init__(self) -> None:
+        self.client = None
+        self.supported_models = ["claude-1", "claude-2"]
+
+    def supports_model(self, model: str):
+        return model in self.supports_model(model)
+
+    @override
+    def evaluate(self, model: str, evaluation_system_prompt: str, user_message: str):
+        if anthropic is None:
+            raise ModuleNotFoundError(
+                "Package `anthropic` is required to be installed to use this experiment."
+                "   Please use `pip install anthropic` to install the package"
+            )
+
+        if not os.environ["ANTHROPIC_API_KEY"]:
+            raise RuntimeError("Missing API key for evaluation.")
+
+        if not self.client:
+            self.client = anthropic.Anthropic(api_key=os.environ["ANTHROPIC_API_KEY"])
+
+        environment = jinja2.Environment()
+        template = environment.from_string(ANTHROPIC_API_AUTOEVAL_TEMPLATE)
+        eval_prompt = template.render(
+            {
+                "HUMAN_PROMPT": anthropic.HUMAN_PROMPT,
+                "EVALUATION_SYSTEM_PROMPT": evaluation_system_prompt,
+                "USER_MESSAGE": user_message,
+                "AI_PROMPT": anthropic.AI_PROMPT,
+            }
+        )
+
+        response = self.client.completions.create(max_tokens_to_sample=100, model=model, prompt=eval_prompt)
+
+        return response.completion

prompttools/utils/model_evaluators/EvaluatorUtils.py

@@ -0,0 +1,17 @@
+# Copyright (c) Hegel AI, Inc.
+# All rights reserved.
+#
+# This source code's license can be found in the
+# LICENSE file in the root directory of this source tree.
+
+from .ModelEvaluator import ModelEvaluator
+from .GptEvaluator import GptEvaluator
+from .AnthropicEvaluator import AnthropicEvaluator
+
+Evaluators = [GptEvaluator(), AnthropicEvaluator()]
+
+
+def get_evaluator_for_model(model: str) -> ModelEvaluator:
+    for evaluator in Evaluators:
+        if evaluator.supports_model(model):
+            return evaluator

prompttools/utils/model_evaluators/GptEvaluator.py

@@ -0,0 +1,63 @@
+# Copyright (c) Hegel AI, Inc.
+# All rights reserved.
+#
+# This source code's license can be found in the
+# LICENSE file in the root directory of this source tree.
+
+import os
+import jinja2
+from overrides import override
+
+from prompttools.utils.error import PromptToolsUtilityError
+from .ModelEvaluator import ModelEvaluator
+import openai
+
+OPENAI_EVAL_PROMPT = """
+{{USER_MESSAGE}}
+ANSWER:
+"""
+
+
+class GptEvaluator(ModelEvaluator):
+    def __init__(self) -> None:
+        # source: https://platform.openai.com/docs/models/model-endpoint-compatibility
+        self.supported_models = [
+            "gpt-4",
+            "gpt-4-0613",
+            "gpt-4-32k",
+            "gpt-4-32k-0613",
+            "gpt-3.5-turbo",
+            "gpt-3.5-turbo-0613",
+            "gpt-3.5-turbo-16k",
+            "gpt-3.5-turbo-16k-0613",
+        ]
+
+    @override
+    def supports_model(self, model) -> bool:
+        return model in self.supported_models
+
+    @override
+    def evaluate(self, model: str, evaluation_system_prompt: str, user_message: str):
+        if not os.environ["OPENAI_API_KEY"]:
+            raise PromptToolsUtilityError
+
+        response = openai.ChatCompletion.create(
+            model=model, messages=self.get_messages(evaluation_system_prompt, user_message)
+        )
+        return response["choices"][0]["message"]["content"]
+
+    def get_messages(self, evaluation_system_prompt, user_message) -> list:
+        environment = jinja2.Environment()
+        template = environment.from_string(OPENAI_EVAL_PROMPT)
+        eval_prompt = template.render(
+            {
+                "USER_MESSAGE": user_message,
+            }
+        )
+
+        messages = [
+            {"role": "system", "content": evaluation_system_prompt},
+            {"role": "user", "content": eval_prompt},
+        ]
+
+        return messages

prompttools/utils/model_evaluators/ModelEvaluator.py

@@ -0,0 +1,18 @@
+# Copyright (c) Hegel AI, Inc.
+# All rights reserved.
+#
+# This source code's license can be found in the
+# LICENSE file in the root directory of this source tree.
+
+
+from abc import ABC, abstractmethod
+
+
+class ModelEvaluator(ABC):
+    @abstractmethod
+    def evaluate(self, model: str, evaluation_system_prompt: str, user_message: str):
+        pass
+
+    @abstractmethod
+    def supports_model(self, model: str):
+        pass