Run from yaml, details preview

Author: gmorgachev

.gitignore

@@ -3,6 +3,7 @@
 *.py[cod]
 *$py.class
 *.sqlite
+*.db
 
 # C extensions
 *.so

resources/experiments.yml

@@ -0,0 +1,23 @@
+- openai_url: https://api.qdrant.mil-team.ru/chat-1/v1/
+  model_name: Compressa-Qwen2-72B-Instruct
+  experiment_name: "File Prompts Run 1"
+  description: "Experiment using prompts from a file with 10 tasks and 5 runners"
+  prompts_file: resources/prompts.csv
+  num_tasks: 10
+  num_runners: 5
+  generate_prompts: false
+  num_prompts: 0
+  prompt_length: 0
+  max_tokens: 1000
+
+- openai_url: https://api.qdrant.mil-team.ru/chat-1/v1/
+  model_name: Compressa-Qwen2-72B-Instruct
+  experiment_name: "Qwen2-72B Long Input / Short Output"
+  description: "Experiment using prompts from a file with 20 tasks and 10 runners"
+  prompts_file: resources/prompts.csv
+  num_tasks: 20
+  num_runners: 10
+  generate_prompts: true
+  num_prompts: 10
+  prompt_length: 10000
+  max_tokens: 100

src/compressa/perf/cli/__main__.py

@@ -3,6 +3,7 @@
     run_experiment,
     report_experiment,
     list_experiments,
+    run_experiments_from_yaml,
     DEFAULT_DB_PATH,
 )
 
@@ -34,7 +35,18 @@ def report_experiment_args(args):
 
 
 def list_experiments_args(args):
-    list_experiments(db=args.db)
+    list_experiments(
+        db=args.db,
+        show_parameters=args.show_parameters
+    )
+
+
+def run_experiments_from_yaml_args(args):
+    run_experiments_from_yaml(
+        yaml_file=args.yaml_file,
+        db=args.db,
+        openai_api_key=args.openai_api_key
+    )
 
 
 def main():
@@ -159,10 +171,38 @@ def main():
         default=DEFAULT_DB_PATH,
         help="Path to the SQLite database",
     )
+    parser_list.add_argument(
+        "--show-parameters",
+        action="store_true",
+        help="Show all parameters for each experiment"
+    )
     parser_list.set_defaults(func=list_experiments_args)
 
+    parser_yaml = subparsers.add_parser(
+        "measure-from-yaml",
+        help="Run experiments from a YAML configuration file",
+    )
+    parser_yaml.add_argument(
+        "yaml_file",
+        help="YAML configuration file for experiments",
+    )
+    parser_yaml.add_argument(
+        "--db",
+        type=str,
+        default=DEFAULT_DB_PATH,
+        help="Path to the SQLite database",
+    )
+    parser_yaml.add_argument(
+        "--openai_api_key",
+        type=str,
+        required=True,
+        help="OpenAI API key",
+    )
+    parser_yaml.set_defaults(func=run_experiments_from_yaml_args)
+
     args = parser.parse_args()
     args.func(args)
 
+
 if __name__ == "__main__":
     main()

src/compressa/perf/cli/tools.py

@@ -1,5 +1,6 @@
 import sqlite3
 from tabulate import tabulate
+from typing import List
 from compressa.perf.experiment.inference import ExperimentRunner
 from compressa.perf.experiment.analysis import Analyzer
 from compressa.perf.data.models import Experiment
@@ -15,6 +16,7 @@
 import sys
 import random
 import string
+from compressa.perf.experiment.config import load_yaml_configs, ExperimentConfig
 
 DEFAULT_DB_PATH = "compressa-perf-db.sqlite"
 
@@ -175,6 +177,7 @@ def report_experiment(
 
 def list_experiments(
     db: str = DEFAULT_DB_PATH,
+    show_parameters: bool = False,
 ):
     with sqlite3.connect(db) as conn:
         ensure_db_initialized(conn)
@@ -185,17 +188,59 @@ def list_experiments(
             print("No experiments found in the database.")
             return
 
-        table_data = [
-            [
+        table_data = []
+        headers = ["ID", "Name", "Date", "Description"]
+
+        if show_parameters:
+            headers.extend(["Parameters"])
+
+        desciptiont_length = 20 if show_parameters else 50
+        for exp in experiments:
+            row = [
                 exp.id,
                 exp.experiment_name,
                 exp.experiment_date.strftime("%Y-%m-%d %H:%M:%S") if exp.experiment_date else "N/A",
-                exp.description[:50] + "..." if exp.description and len(exp.description) > 50 else exp.description
+                exp.description[:desciptiont_length] + "..." if exp.description and len(exp.description) > desciptiont_length else exp.description
             ]
-            for exp in experiments
-        ]
+
+            if show_parameters:
+                parameters = fetch_parameters_by_experiment(conn, exp.id)
+                param_str = "\n".join([f"{p.key}: {format_value(p.value, precision=2)}" for p in parameters])
+                row.append(param_str)
+
+            table_data.append(row)
 
         print("\nList of Experiments:")
-        print(tabulate(table_data, headers=["ID", "Name", "Date", "Description"], tablefmt="grid"))
+        print(tabulate(table_data, headers=headers, tablefmt="grid"))
 
 
+
+def run_experiments_from_yaml(
+    yaml_file: str,
+    db: str = DEFAULT_DB_PATH,
+    openai_api_key: str = None,
+):
+    if not openai_api_key:
+        raise ValueError("OPENAI_API_KEY is not set")
+
+    configs = load_yaml_configs(yaml_file)
+    
+    for config in configs:
+        run_experiment(
+            db=db,
+            openai_api_key=openai_api_key,
+            openai_url=config.openai_url,
+            model_name=config.model_name,
+            experiment_name=config.experiment_name,
+            description=config.description,
+            prompts_file=config.prompts_file,
+            num_tasks=config.num_tasks,
+            num_runners=config.num_runners,
+            generate_prompts=config.generate_prompts,
+            num_prompts=config.num_prompts,
+            prompt_length=config.prompt_length,
+            max_tokens=config.max_tokens,
+        )
+    
+    # List experiments after running them
+    list_experiments(db=db)

src/compressa/perf/experiment/config.py

@@ -0,0 +1,30 @@
+import yaml
+from typing import (
+    List,
+    Dict,
+)
+from dataclasses import dataclass
+
+@dataclass
+class ExperimentConfig:
+    openai_url: str
+    model_name: str
+    experiment_name: str
+    description: str
+    num_tasks: int
+    num_runners: int
+    generate_prompts: bool
+    num_prompts: int
+    prompt_length: int
+    max_tokens: int
+    prompts_file: str = None
+
+def load_yaml_configs(file_path: str) -> List[ExperimentConfig]:
+    with open(file_path, 'r') as file:
+        config_data = yaml.safe_load(file)
+    
+    if not isinstance(config_data, list):
+        config_data = [config_data]
+    
+    return [ExperimentConfig(**experiment) for experiment in config_data]
+

tests/test_inference.py

@@ -46,6 +46,7 @@ def test_inference(self):
             measurement = runner.run_inference(
                 experiment_id=experiment.id,
                 prompt="Hello, world!",
+                max_tokens=100,
             )
 
             if measurement: