produce_tables.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create GSM-8K Pass@K and Accuracy table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nvidia/OpenMath-Mistral-7B-v0.1-hf\t& 0.1 & 0.679 & 0.715 & 0.747 & 0.770 & 0.6687\\\\\n",
      "nvidia/OpenMath-Mistral-7B-v0.1-hf\t& 0.3 & 0.681 & 0.752 & 0.813 & 0.848 & 0.6679\\\\\n",
      "nvidia/OpenMath-Mistral-7B-v0.1-hf\t& 0.5 & 0.669 & 0.763 & 0.840 & 0.875 & 0.6497\\\\\n",
      "meta-llama/Meta-Llama-3-8B\t& 0.1 & 0.448 & 0.498 & 0.558 & 0.599 & 0.4466\\\\\n",
      "meta-llama/Meta-Llama-3-8B\t& 0.3 & 0.448 & 0.545 & 0.650 & 0.715 & 0.4405\\\\\n",
      "meta-llama/Meta-Llama-3-8B\t& 0.5 & 0.427 & 0.557 & 0.692 & 0.763 & 0.4139\\\\\n",
      "codellama/CodeLlama-34b-Python-hf\t& 0.1 & 0.447 & 0.498 & 0.557 & 0.592 & 0.4367\\\\\n",
      "codellama/CodeLlama-34b-Python-hf\t& 0.3 & 0.450 & 0.545 & 0.650 & 0.716 & 0.4359\\\\\n",
      "codellama/CodeLlama-34b-Python-hf\t& 0.5 & 0.433 & 0.550 & 0.676 & 0.749 & 0.4185\\\\\n",
      "mistralai/Mistral-7B-v0.1\t& 0.1 & 0.360 & 0.409 & 0.462 & 0.496 & 0.3571\\\\\n",
      "mistralai/Mistral-7B-v0.1\t& 0.3 & 0.368 & 0.458 & 0.564 & 0.640 & 0.3654\\\\\n",
      "mistralai/Mistral-7B-v0.1\t& 0.5 & 0.352 & 0.468 & 0.605 & 0.691 & 0.3419\\\\\n",
      "codellama/CodeLlama-13b-Python-hf\t& 0.1 & 0.329 & 0.376 & 0.432 & 0.475 & 0.3252\\\\\n",
      "codellama/CodeLlama-13b-Python-hf\t& 0.3 & 0.324 & 0.412 & 0.518 & 0.594 & 0.3048\\\\\n",
      "codellama/CodeLlama-13b-Python-hf\t& 0.5 & 0.313 & 0.419 & 0.551 & 0.639 & 0.3124\\\\\n",
      "bigcode/starcoder2-7b\t& 0.1 & 0.329 & 0.376 & 0.428 & 0.463 & 0.3351\\\\\n",
      "bigcode/starcoder2-7b\t& 0.3 & 0.330 & 0.419 & 0.530 & 0.601 & 0.3283\\\\\n",
      "bigcode/starcoder2-7b\t& 0.5 & 0.320 & 0.426 & 0.559 & 0.647 & 0.2995\\\\\n",
      "codellama/CodeLlama-7b-Python-hf\t& 0.1 & 0.239 & 0.279 & 0.325 & 0.356 & 0.2396\\\\\n",
      "codellama/CodeLlama-7b-Python-hf\t& 0.3 & 0.238 & 0.316 & 0.420 & 0.494 & 0.2320\\\\\n",
      "codellama/CodeLlama-7b-Python-hf\t& 0.5 & 0.225 & 0.321 & 0.454 & 0.553 & 0.2267\\\\\n",
      "bigcode/starcoder2-3b\t& 0.1 & 0.247 & 0.286 & 0.332 & 0.362 & 0.2449\\\\\n",
      "bigcode/starcoder2-3b\t& 0.3 & 0.247 & 0.321 & 0.413 & 0.479 & 0.2456\\\\\n",
      "bigcode/starcoder2-3b\t& 0.5 & 0.239 & 0.330 & 0.454 & 0.547 & 0.2320\\\\\n",
      "THUDM/codegeex2-6b\t& 0.1 & 0.161 & 0.195 & 0.239 & 0.271 & 0.1622\\\\\n",
      "THUDM/codegeex2-6b\t& 0.3 & 0.154 & 0.215 & 0.305 & 0.380 & 0.1592\\\\\n",
      "THUDM/codegeex2-6b\t& 0.5 & 0.151 & 0.224 & 0.334 & 0.428 & 0.1607\\\\\n",
      "Salesforce/codegen25-7b-mono\t& 0.1 & 0.125 & 0.153 & 0.187 & 0.213 & 0.1228\\\\\n",
      "Salesforce/codegen25-7b-mono\t& 0.3 & 0.131 & 0.183 & 0.256 & 0.313 & 0.1130\\\\\n",
      "Salesforce/codegen25-7b-mono\t& 0.5 & 0.119 & 0.179 & 0.270 & 0.343 & 0.1016\\\\\n",
      "Salesforce/codegen-16B-mono\t& 0.1 & 0.119 & 0.146 & 0.186 & 0.216 & 0.1145\\\\\n",
      "Salesforce/codegen-16B-mono\t& 0.3 & 0.115 & 0.164 & 0.239 & 0.306 & 0.1160\\\\\n",
      "Salesforce/codegen-16B-mono\t& 0.5 & 0.109 & 0.168 & 0.267 & 0.355 & 0.1061\\\\\n",
      "Salesforce/codegen-6B-mono\t& 0.1 & 0.086 & 0.108 & 0.134 & 0.153 & 0.0895\\\\\n",
      "Salesforce/codegen-6B-mono\t& 0.3 & 0.082 & 0.122 & 0.187 & 0.244 & 0.0720\\\\\n",
      "Salesforce/codegen-6B-mono\t& 0.5 & 0.079 & 0.124 & 0.202 & 0.274 & 0.0804\\\\\n",
      "Salesforce/codegen-2B-mono\t& 0.1 & 0.047 & 0.068 & 0.102 & 0.130 & 0.0470\\\\\n",
      "Salesforce/codegen-2B-mono\t& 0.3 & 0.043 & 0.072 & 0.125 & 0.177 & 0.0455\\\\\n",
      "Salesforce/codegen-2B-mono\t& 0.5 & 0.045 & 0.076 & 0.139 & 0.202 & 0.0394\\\\\n",
      "EleutherAI/gpt-j-6b\t& 0.1 & 0.029 & 0.040 & 0.055 & 0.064 & 0.0220\\\\\n",
      "EleutherAI/gpt-j-6b\t& 0.3 & 0.027 & 0.046 & 0.085 & 0.123 & 0.0273\\\\\n",
      "EleutherAI/gpt-j-6b\t& 0.5 & 0.026 & 0.046 & 0.088 & 0.133 & 0.0227\\\\\n",
      "facebook/incoder-6B\t& 0.1 & 0.027 & 0.035 & 0.050 & 0.065 & 0.0243\\\\\n",
      "facebook/incoder-6B\t& 0.3 & 0.030 & 0.048 & 0.081 & 0.118 & 0.0288\\\\\n",
      "facebook/incoder-6B\t& 0.5 & 0.028 & 0.050 & 0.096 & 0.145 & 0.0303\\\\\n",
      "Salesforce/codegen-350M-mono\t& 0.1 & 0.013 & 0.019 & 0.027 & 0.033 & 0.0129\\\\\n",
      "Salesforce/codegen-350M-mono\t& 0.3 & 0.013 & 0.023 & 0.043 & 0.063 & 0.0099\\\\\n",
      "Salesforce/codegen-350M-mono\t& 0.5 & 0.013 & 0.024 & 0.049 & 0.079 & 0.0106\\\\\n",
      "facebook/incoder-1B\t& 0.1 & 0.003 & 0.004 & 0.006 & 0.008 & 0.0038\\\\\n",
      "facebook/incoder-1B\t& 0.3 & 0.005 & 0.008 & 0.012 & 0.017 & 0.0045\\\\\n",
      "facebook/incoder-1B\t& 0.5 & 0.005 & 0.009 & 0.019 & 0.030 & 0.0053\\\\\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv(\"GSM-res.csv\")\n",
    "\n",
    "df = df.drop(columns=[\"Unnamed: 0\"])\n",
    "df[\"pass@1\"] = df[\"pass@1\"] / 1319\n",
    "df[\"pass@2\"] = df[\"pass@2\"] / 1319\n",
    "df[\"pass@5\"] = df[\"pass@5\"] / 1319\n",
    "df[\"pass@10\"] = df[\"pass@10\"] / 1319\n",
    "# df = df.sort_values(by=[\"model\", \"temperature\", \"accuracy\"], ascending=[False, True, False])\n",
    "for i, row in df.iterrows():\n",
    "    print(f\"{row[\"model\"]}\\t& {row[\"temperature\"]} & {row[\"pass@1\"]:.3f} & {row[\"pass@2\"]:.3f} & {row[\"pass@5\"]:.3f} & {row[\"pass@10\"]:.3f} & {row[\"accuracy\"]:.4f}\\\\\\\\\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create pass@k and accuracy table for SVAMP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nvidia/OpenMath-Mistral-7B-v0.1-hf\t& 0.1 & 0.832 & 0.853 & 0.871 & 0.880 & 0.827\\\\\n",
      "nvidia/OpenMath-Mistral-7B-v0.1-hf\t& 0.3 & 0.836 & 0.879 & 0.910 & 0.923 & 0.833\\\\\n",
      "nvidia/OpenMath-Mistral-7B-v0.1-hf\t& 0.5 & 0.824 & 0.883 & 0.923 & 0.941 & 0.821\\\\\n",
      "meta-llama/Meta-Llama-3-8B\t& 0.1 & 0.758 & 0.787 & 0.815 & 0.832 & 0.753\\\\\n",
      "meta-llama/Meta-Llama-3-8B\t& 0.3 & 0.752 & 0.819 & 0.875 & 0.900 & 0.734\\\\\n",
      "meta-llama/Meta-Llama-3-8B\t& 0.5 & 0.718 & 0.823 & 0.900 & 0.935 & 0.716\\\\\n",
      "codellama/CodeLlama-34b-Python-hf\t& 0.1 & 0.720 & 0.750 & 0.778 & 0.794 & 0.725\\\\\n",
      "codellama/CodeLlama-34b-Python-hf\t& 0.3 & 0.721 & 0.783 & 0.842 & 0.878 & 0.728\\\\\n",
      "codellama/CodeLlama-34b-Python-hf\t& 0.5 & 0.711 & 0.792 & 0.864 & 0.896 & 0.714\\\\\n",
      "mistralai/Mistral-7B-v0.1\t& 0.1 & 0.711 & 0.743 & 0.773 & 0.791 & 0.708\\\\\n",
      "mistralai/Mistral-7B-v0.1\t& 0.3 & 0.706 & 0.780 & 0.844 & 0.877 & 0.700\\\\\n",
      "mistralai/Mistral-7B-v0.1\t& 0.5 & 0.685 & 0.787 & 0.872 & 0.913 & 0.678\\\\\n",
      "codellama/CodeLlama-13b-Python-hf\t& 0.1 & 0.663 & 0.701 & 0.738 & 0.762 & 0.664\\\\\n",
      "codellama/CodeLlama-13b-Python-hf\t& 0.3 & 0.643 & 0.724 & 0.801 & 0.843 & 0.627\\\\\n",
      "codellama/CodeLlama-13b-Python-hf\t& 0.5 & 0.627 & 0.736 & 0.831 & 0.882 & 0.599\\\\\n",
      "bigcode/starcoder2-7b\t& 0.1 & 0.649 & 0.685 & 0.724 & 0.751 & 0.652\\\\\n",
      "bigcode/starcoder2-7b\t& 0.3 & 0.646 & 0.720 & 0.799 & 0.845 & 0.653\\\\\n",
      "bigcode/starcoder2-7b\t& 0.5 & 0.627 & 0.730 & 0.829 & 0.883 & 0.645\\\\\n",
      "codellama/CodeLlama-7b-Python-hf\t& 0.1 & 0.603 & 0.645 & 0.688 & 0.715 & 0.606\\\\\n",
      "codellama/CodeLlama-7b-Python-hf\t& 0.3 & 0.595 & 0.681 & 0.767 & 0.814 & 0.600\\\\\n",
      "codellama/CodeLlama-7b-Python-hf\t& 0.5 & 0.573 & 0.685 & 0.793 & 0.853 & 0.566\\\\\n",
      "GPT-3/davinci-codex\t& 0.1 & 0.590 & 0.637 & 0.684 & 0.713 & 0.589\\\\\n",
      "GPT-3/davinci-codex\t& 0.3 & 0.574 & 0.658 & 0.745 & 0.793 & 0.577\\\\\n",
      "GPT-3/davinci-codex\t& 0.5 & 0.559 & 0.667 & 0.776 & 0.839 & 0.553\\\\\n",
      "bigcode/starcoder2-3b\t& 0.1 & 0.587 & 0.621 & 0.657 & 0.680 & 0.583\\\\\n",
      "bigcode/starcoder2-3b\t& 0.3 & 0.577 & 0.655 & 0.732 & 0.778 & 0.573\\\\\n",
      "bigcode/starcoder2-3b\t& 0.5 & 0.558 & 0.664 & 0.767 & 0.824 & 0.543\\\\\n",
      "GPT-3/cushman-codex\t& 0.1 & 0.510 & 0.556 & 0.605 & 0.635 & 0.516\\\\\n",
      "GPT-3/cushman-codex\t& 0.3 & 0.501 & 0.589 & 0.682 & 0.737 & 0.490\\\\\n",
      "GPT-3/cushman-codex\t& 0.5 & 0.474 & 0.590 & 0.711 & 0.789 & 0.476\\\\\n",
      "THUDM/codegeex2-6b\t& 0.1 & 0.470 & 0.513 & 0.559 & 0.590 & 0.464\\\\\n",
      "THUDM/codegeex2-6b\t& 0.3 & 0.462 & 0.554 & 0.657 & 0.723 & 0.470\\\\\n",
      "THUDM/codegeex2-6b\t& 0.5 & 0.441 & 0.562 & 0.700 & 0.777 & 0.440\\\\\n",
      "Salesforce/codegen25-7b-mono\t& 0.1 & 0.495 & 0.536 & 0.582 & 0.609 & 0.497\\\\\n",
      "Salesforce/codegen25-7b-mono\t& 0.3 & 0.493 & 0.577 & 0.667 & 0.717 & 0.494\\\\\n",
      "Salesforce/codegen25-7b-mono\t& 0.5 & 0.476 & 0.582 & 0.691 & 0.757 & 0.461\\\\\n",
      "Salesforce/codegen-16B-mono\t& 0.1 & 0.387 & 0.433 & 0.488 & 0.523 & 0.387\\\\\n",
      "Salesforce/codegen-16B-mono\t& 0.3 & 0.392 & 0.483 & 0.587 & 0.654 & 0.391\\\\\n",
      "Salesforce/codegen-16B-mono\t& 0.5 & 0.376 & 0.487 & 0.612 & 0.691 & 0.367\\\\\n",
      "Salesforce/codegen-6B-mono\t& 0.1 & 0.335 & 0.380 & 0.432 & 0.467 & 0.334\\\\\n",
      "Salesforce/codegen-6B-mono\t& 0.3 & 0.336 & 0.431 & 0.546 & 0.623 & 0.335\\\\\n",
      "Salesforce/codegen-6B-mono\t& 0.5 & 0.322 & 0.442 & 0.592 & 0.692 & 0.320\\\\\n",
      "Salesforce/codegen-2B-mono\t& 0.1 & 0.266 & 0.315 & 0.373 & 0.409 & 0.270\\\\\n",
      "Salesforce/codegen-2B-mono\t& 0.3 & 0.264 & 0.359 & 0.484 & 0.571 & 0.255\\\\\n",
      "Salesforce/codegen-2B-mono\t& 0.5 & 0.258 & 0.375 & 0.538 & 0.648 & 0.268\\\\\n",
      "EleutherAI/gpt-j-6B\t& 0.1 & 0.213 & 0.252 & 0.296 & 0.324 & 0.206\\\\\n",
      "EleutherAI/gpt-j-6B\t& 0.3 & 0.228 & 0.310 & 0.420 & 0.500 & 0.229\\\\\n",
      "EleutherAI/gpt-j-6B\t& 0.5 & 0.194 & 0.293 & 0.434 & 0.532 & 0.189\\\\\n",
      "facebook/incoder-6B\t& 0.1 & 0.171 & 0.208 & 0.253 & 0.284 & 0.163\\\\\n",
      "facebook/incoder-6B\t& 0.3 & 0.174 & 0.249 & 0.354 & 0.433 & 0.174\\\\\n",
      "facebook/incoder-6B\t& 0.5 & 0.165 & 0.256 & 0.396 & 0.502 & 0.158\\\\\n",
      "Salesforce/codegen-350M-mono\t& 0.1 & 0.085 & 0.107 & 0.135 & 0.154 & 0.088\\\\\n",
      "Salesforce/codegen-350M-mono\t& 0.3 & 0.074 & 0.114 & 0.183 & 0.244 & 0.069\\\\\n",
      "Salesforce/codegen-350M-mono\t& 0.5 & 0.066 & 0.112 & 0.201 & 0.283 & 0.070\\\\\n",
      "facebook/incoder-1B\t& 0.1 & 0.059 & 0.070 & 0.087 & 0.100 & 0.061\\\\\n",
      "facebook/incoder-1B\t& 0.3 & 0.057 & 0.082 & 0.121 & 0.152 & 0.058\\\\\n",
      "facebook/incoder-1B\t& 0.5 & 0.057 & 0.088 & 0.139 & 0.183 & 0.059\\\\\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv(\"SVAMP-res.csv\")\n",
    "\n",
    "df = df.drop(columns=[\"Unnamed: 0\"])\n",
    "df[\"pass@1\"] = df[\"pass@1\"] / 1000\n",
    "df[\"pass@2\"] = df[\"pass@2\"] / 1000\n",
    "df[\"pass@5\"] = df[\"pass@5\"] / 1000\n",
    "df[\"pass@10\"] = df[\"pass@10\"] / 1000\n",
    "# df = df.sort_values(by=[\"model\", \"temperature\", \"accuracy\"], ascending=[False, True, False])\n",
    "\n",
    "for i, row in df.iterrows():\n",
    "    print(f\"{row[\"model\"]}\\t& {row[\"temperature\"]} & {row[\"pass@1\"]:.3f} & {row[\"pass@2\"]:.3f} & {row[\"pass@5\"]:.3f} & {row[\"pass@10\"]:.3f} & {row[\"accuracy\"]:.3f}\\\\\\\\\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create voting table for SVAMP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nvidia/OpenMath-Mistral-7B-v0.1-hf\t& 0.1 & 0.827 & 0.838 & 0.838 & 0.838 & 0.839 & 0.839 & 0.837 & 0.842 & 0.841\\\\\n",
      "nvidia/OpenMath-Mistral-7B-v0.1-hf\t& 0.3 & 0.833 & 0.848 & 0.849 & 0.859 & 0.856 & 0.861 & 0.863 & 0.864 & 0.862\\\\\n",
      "nvidia/OpenMath-Mistral-7B-v0.1-hf\t& 0.5 & 0.821 & 0.850 & 0.853 & 0.864 & 0.872 & 0.871 & 0.872 & 0.878 & 0.877\\\\\n",
      "meta-llama/Meta-Llama-3-8B\t& 0.1 & 0.753 & 0.755 & 0.759 & 0.762 & 0.766 & 0.764 & 0.763 & 0.769 & 0.766\\\\\n",
      "meta-llama/Meta-Llama-3-8B\t& 0.3 & 0.734 & 0.763 & 0.770 & 0.783 & 0.784 & 0.790 & 0.795 & 0.792 & 0.797\\\\\n",
      "meta-llama/Meta-Llama-3-8B\t& 0.5 & 0.716 & 0.765 & 0.789 & 0.795 & 0.806 & 0.804 & 0.808 & 0.810 & 0.804\\\\\n",
      "codellama/CodeLlama-34b-Python-hf\t& 0.1 & 0.725 & 0.730 & 0.729 & 0.728 & 0.729 & 0.734 & 0.733 & 0.736 & 0.733\\\\\n",
      "codellama/CodeLlama-34b-Python-hf\t& 0.3 & 0.728 & 0.739 & 0.745 & 0.744 & 0.745 & 0.746 & 0.748 & 0.749 & 0.744\\\\\n",
      "codellama/CodeLlama-34b-Python-hf\t& 0.5 & 0.714 & 0.750 & 0.758 & 0.767 & 0.766 & 0.768 & 0.779 & 0.774 & 0.773\\\\\n",
      "mistralai/Mistral-7B-v0.1\t& 0.1 & 0.708 & 0.713 & 0.722 & 0.725 & 0.725 & 0.725 & 0.727 & 0.723 & 0.730\\\\\n",
      "mistralai/Mistral-7B-v0.1\t& 0.3 & 0.700 & 0.730 & 0.736 & 0.736 & 0.749 & 0.751 & 0.752 & 0.756 & 0.752\\\\\n",
      "mistralai/Mistral-7B-v0.1\t& 0.5 & 0.678 & 0.721 & 0.741 & 0.753 & 0.759 & 0.764 & 0.768 & 0.764 & 0.769\\\\\n",
      "codellama/CodeLlama-13b-Python-hf\t& 0.1 & 0.664 & 0.671 & 0.676 & 0.672 & 0.678 & 0.682 & 0.684 & 0.680 & 0.684\\\\\n",
      "codellama/CodeLlama-13b-Python-hf\t& 0.3 & 0.627 & 0.667 & 0.671 & 0.677 & 0.679 & 0.682 & 0.690 & 0.691 & 0.693\\\\\n",
      "codellama/CodeLlama-13b-Python-hf\t& 0.5 & 0.599 & 0.650 & 0.681 & 0.706 & 0.709 & 0.702 & 0.713 & 0.708 & 0.708\\\\\n",
      "bigcode/starcoder2-7b\t& 0.1 & 0.652 & 0.657 & 0.656 & 0.656 & 0.658 & 0.657 & 0.657 & 0.658 & 0.656\\\\\n",
      "bigcode/starcoder2-7b\t& 0.3 & 0.653 & 0.677 & 0.677 & 0.681 & 0.680 & 0.675 & 0.682 & 0.681 & 0.679\\\\\n",
      "bigcode/starcoder2-7b\t& 0.5 & 0.645 & 0.677 & 0.686 & 0.690 & 0.692 & 0.698 & 0.699 & 0.700 & 0.701\\\\\n",
      "codellama/CodeLlama-7b-Python-hf\t& 0.1 & 0.606 & 0.605 & 0.607 & 0.608 & 0.610 & 0.612 & 0.613 & 0.613 & 0.614\\\\\n",
      "codellama/CodeLlama-7b-Python-hf\t& 0.3 & 0.600 & 0.618 & 0.628 & 0.639 & 0.642 & 0.640 & 0.646 & 0.642 & 0.639\\\\\n",
      "codellama/CodeLlama-7b-Python-hf\t& 0.5 & 0.566 & 0.610 & 0.628 & 0.639 & 0.649 & 0.656 & 0.660 & 0.661 & 0.665\\\\\n",
      "GPT-3/davinci-codex\t& 0.1 & 0.589 & 0.599 & 0.603 & 0.604 & 0.607 & 0.601 & 0.608 & 0.610 & 0.610\\\\\n",
      "GPT-3/davinci-codex\t& 0.3 & 0.577 & 0.595 & 0.599 & 0.603 & 0.610 & 0.606 & 0.619 & 0.619 & 0.620\\\\\n",
      "GPT-3/davinci-codex\t& 0.5 & 0.553 & 0.601 & 0.602 & 0.611 & 0.623 & 0.632 & 0.629 & 0.632 & 0.638\\\\\n",
      "bigcode/starcoder2-3b\t& 0.1 & 0.583 & 0.590 & 0.588 & 0.592 & 0.593 & 0.594 & 0.596 & 0.596 & 0.596\\\\\n",
      "bigcode/starcoder2-3b\t& 0.3 & 0.573 & 0.598 & 0.609 & 0.612 & 0.620 & 0.619 & 0.621 & 0.616 & 0.622\\\\\n",
      "bigcode/starcoder2-3b\t& 0.5 & 0.543 & 0.589 & 0.610 & 0.616 & 0.627 & 0.619 & 0.625 & 0.629 & 0.629\\\\\n",
      "GPT-3/cushman-codex\t& 0.1 & 0.516 & 0.513 & 0.516 & 0.518 & 0.520 & 0.519 & 0.517 & 0.522 & 0.516\\\\\n",
      "GPT-3/cushman-codex\t& 0.3 & 0.490 & 0.520 & 0.525 & 0.539 & 0.530 & 0.539 & 0.547 & 0.547 & 0.549\\\\\n",
      "GPT-3/cushman-codex\t& 0.5 & 0.476 & 0.503 & 0.525 & 0.537 & 0.538 & 0.561 & 0.563 & 0.567 & 0.571\\\\\n",
      "THUDM/codegeex2-6b\t& 0.1 & 0.464 & 0.474 & 0.474 & 0.479 & 0.485 & 0.483 & 0.485 & 0.481 & 0.488\\\\\n",
      "THUDM/codegeex2-6b\t& 0.3 & 0.470 & 0.488 & 0.508 & 0.508 & 0.519 & 0.518 & 0.527 & 0.528 & 0.527\\\\\n",
      "THUDM/codegeex2-6b\t& 0.5 & 0.440 & 0.464 & 0.484 & 0.509 & 0.510 & 0.523 & 0.531 & 0.531 & 0.544\\\\\n",
      "Salesforce/codegen25-7b-mono\t& 0.1 & 0.497 & 0.505 & 0.502 & 0.505 & 0.502 & 0.502 & 0.501 & 0.496 & 0.502\\\\\n",
      "Salesforce/codegen25-7b-mono\t& 0.3 & 0.494 & 0.510 & 0.531 & 0.529 & 0.530 & 0.528 & 0.529 & 0.528 & 0.535\\\\\n",
      "Salesforce/codegen25-7b-mono\t& 0.5 & 0.461 & 0.496 & 0.514 & 0.526 & 0.541 & 0.554 & 0.555 & 0.556 & 0.558\\\\\n",
      "Salesforce/codegen-16B-mono\t& 0.1 & 0.387 & 0.385 & 0.387 & 0.389 & 0.392 & 0.383 & 0.385 & 0.386 & 0.387\\\\\n",
      "Salesforce/codegen-16B-mono\t& 0.3 & 0.391 & 0.411 & 0.420 & 0.429 & 0.432 & 0.433 & 0.437 & 0.439 & 0.439\\\\\n",
      "Salesforce/codegen-16B-mono\t& 0.5 & 0.367 & 0.400 & 0.428 & 0.433 & 0.448 & 0.457 & 0.464 & 0.454 & 0.460\\\\\n",
      "Salesforce/codegen-6B-mono\t& 0.1 & 0.334 & 0.329 & 0.335 & 0.334 & 0.337 & 0.342 & 0.341 & 0.341 & 0.343\\\\\n",
      "Salesforce/codegen-6B-mono\t& 0.3 & 0.335 & 0.353 & 0.358 & 0.371 & 0.372 & 0.380 & 0.380 & 0.380 & 0.389\\\\\n",
      "Salesforce/codegen-6B-mono\t& 0.5 & 0.320 & 0.340 & 0.351 & 0.362 & 0.381 & 0.378 & 0.385 & 0.389 & 0.393\\\\\n",
      "Salesforce/codegen-2B-mono\t& 0.1 & 0.270 & 0.269 & 0.279 & 0.277 & 0.276 & 0.277 & 0.279 & 0.279 & 0.276\\\\\n",
      "Salesforce/codegen-2B-mono\t& 0.3 & 0.255 & 0.264 & 0.282 & 0.285 & 0.296 & 0.304 & 0.306 & 0.300 & 0.305\\\\\n",
      "Salesforce/codegen-2B-mono\t& 0.5 & 0.268 & 0.287 & 0.305 & 0.311 & 0.323 & 0.331 & 0.335 & 0.336 & 0.343\\\\\n",
      "EleutherAI/gpt-j-6B\t& 0.1 & 0.206 & 0.216 & 0.213 & 0.216 & 0.215 & 0.216 & 0.218 & 0.219 & 0.222\\\\\n",
      "EleutherAI/gpt-j-6B\t& 0.3 & 0.229 & 0.236 & 0.254 & 0.243 & 0.251 & 0.250 & 0.255 & 0.257 & 0.259\\\\\n",
      "EleutherAI/gpt-j-6B\t& 0.5 & 0.189 & 0.207 & 0.218 & 0.228 & 0.231 & 0.237 & 0.241 & 0.245 & 0.249\\\\\n",
      "facebook/incoder-6B\t& 0.1 & 0.163 & 0.162 & 0.166 & 0.173 & 0.175 & 0.174 & 0.173 & 0.170 & 0.173\\\\\n",
      "facebook/incoder-6B\t& 0.3 & 0.174 & 0.178 & 0.182 & 0.180 & 0.186 & 0.185 & 0.191 & 0.194 & 0.194\\\\\n",
      "facebook/incoder-6B\t& 0.5 & 0.158 & 0.180 & 0.189 & 0.202 & 0.210 & 0.203 & 0.207 & 0.208 & 0.207\\\\\n",
      "Salesforce/codegen-350M-mono\t& 0.1 & 0.088 & 0.085 & 0.086 & 0.088 & 0.085 & 0.086 & 0.088 & 0.087 & 0.089\\\\\n",
      "Salesforce/codegen-350M-mono\t& 0.3 & 0.069 & 0.070 & 0.072 & 0.082 & 0.082 & 0.081 & 0.083 & 0.085 & 0.085\\\\\n",
      "Salesforce/codegen-350M-mono\t& 0.5 & 0.070 & 0.071 & 0.068 & 0.069 & 0.067 & 0.068 & 0.071 & 0.077 & 0.079\\\\\n",
      "facebook/incoder-1B\t& 0.1 & 0.061 & 0.064 & 0.061 & 0.058 & 0.059 & 0.059 & 0.057 & 0.057 & 0.057\\\\\n",
      "facebook/incoder-1B\t& 0.3 & 0.058 & 0.057 & 0.054 & 0.057 & 0.061 & 0.063 & 0.064 & 0.065 & 0.065\\\\\n",
      "facebook/incoder-1B\t& 0.5 & 0.059 & 0.068 & 0.071 & 0.071 & 0.064 & 0.066 & 0.063 & 0.063 & 0.068\\\\\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv(\"SVAMP-res-voteing.csv\")\n",
    "df_r = pd.read_csv(\"SVAMP-res.csv\")\n",
    "df_r = df_r[[\"model\", \"temperature\", \"nshots\", \"accuracy\"]]\n",
    "df = df.merge(df_r, on=[\"model\", \"temperature\", \"nshots\"], how=\"inner\", validate=\"1:1\")\n",
    "\n",
    "df = df.drop(columns=[\"Unnamed: 0\"])\n",
    "# df = df.sort_values(by=[\"model\", \"temperature\", \"accuracy_10\"], ascending=[False, True, False])\n",
    "\n",
    "for i, row in df.iterrows():\n",
    "    #print(row)\n",
    "    line = f\"{row[\"model\"]}\\t& {row[\"temperature\"]} & {row[\"accuracy\"]:.3f} & \" + \" & \".join([f\"{row[f\"accuracy_{n}\"]:.3f}\" for n in range(3, 11)]) + \"\\\\\\\\\"\n",
    "    print(line)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create Voring table from GSM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nvidia/OpenMath-Mistral-7B-v0.1-hf\t& 0.1 & 0.669 & 0.678 & 0.681 & 0.686 & 0.687 & 0.694 & 0.687 & 0.693 & 0.691\\\\\n",
      "nvidia/OpenMath-Mistral-7B-v0.1-hf\t& 0.3 & 0.668 & 0.700 & 0.703 & 0.712 & 0.716 & 0.722 & 0.725 & 0.720 & 0.728\\\\\n",
      "nvidia/OpenMath-Mistral-7B-v0.1-hf\t& 0.5 & 0.650 & 0.703 & 0.712 & 0.725 & 0.734 & 0.736 & 0.743 & 0.744 & 0.744\\\\\n",
      "meta-llama/Meta-Llama-3-8B\t& 0.1 & 0.447 & 0.454 & 0.455 & 0.449 & 0.450 & 0.452 & 0.453 & 0.455 & 0.457\\\\\n",
      "meta-llama/Meta-Llama-3-8B\t& 0.3 & 0.440 & 0.467 & 0.477 & 0.487 & 0.493 & 0.504 & 0.506 & 0.505 & 0.502\\\\\n",
      "meta-llama/Meta-Llama-3-8B\t& 0.5 & 0.414 & 0.463 & 0.486 & 0.496 & 0.509 & 0.525 & 0.530 & 0.536 & 0.537\\\\\n",
      "codellama/CodeLlama-34b-Python-hf\t& 0.1 & 0.437 & 0.452 & 0.448 & 0.451 & 0.452 & 0.458 & 0.454 & 0.455 & 0.454\\\\\n",
      "codellama/CodeLlama-34b-Python-hf\t& 0.3 & 0.436 & 0.483 & 0.492 & 0.497 & 0.501 & 0.508 & 0.511 & 0.517 & 0.521\\\\\n",
      "codellama/CodeLlama-34b-Python-hf\t& 0.5 & 0.418 & 0.462 & 0.495 & 0.501 & 0.520 & 0.522 & 0.525 & 0.538 & 0.541\\\\\n",
      "mistralai/Mistral-7B-v0.1\t& 0.1 & 0.357 & 0.360 & 0.363 & 0.366 & 0.366 & 0.375 & 0.370 & 0.369 & 0.369\\\\\n",
      "mistralai/Mistral-7B-v0.1\t& 0.3 & 0.365 & 0.393 & 0.401 & 0.404 & 0.409 & 0.414 & 0.412 & 0.423 & 0.419\\\\\n",
      "mistralai/Mistral-7B-v0.1\t& 0.5 & 0.342 & 0.368 & 0.391 & 0.423 & 0.427 & 0.437 & 0.436 & 0.438 & 0.436\\\\\n",
      "codellama/CodeLlama-13b-Python-hf\t& 0.1 & 0.325 & 0.330 & 0.331 & 0.338 & 0.345 & 0.342 & 0.339 & 0.342 & 0.338\\\\\n",
      "codellama/CodeLlama-13b-Python-hf\t& 0.3 & 0.305 & 0.337 & 0.344 & 0.360 & 0.360 & 0.366 & 0.369 & 0.376 & 0.377\\\\\n",
      "codellama/CodeLlama-13b-Python-hf\t& 0.5 & 0.312 & 0.347 & 0.365 & 0.383 & 0.395 & 0.397 & 0.395 & 0.404 & 0.398\\\\\n",
      "bigcode/starcoder2-7b\t& 0.1 & 0.335 & 0.332 & 0.335 & 0.338 & 0.337 & 0.335 & 0.339 & 0.334 & 0.338\\\\\n",
      "bigcode/starcoder2-7b\t& 0.3 & 0.328 & 0.354 & 0.360 & 0.362 & 0.368 & 0.369 & 0.369 & 0.368 & 0.378\\\\\n",
      "bigcode/starcoder2-7b\t& 0.5 & 0.299 & 0.336 & 0.346 & 0.365 & 0.380 & 0.388 & 0.396 & 0.404 & 0.407\\\\\n",
      "codellama/CodeLlama-7b-Python-hf\t& 0.1 & 0.240 & 0.246 & 0.247 & 0.246 & 0.245 & 0.247 & 0.253 & 0.255 & 0.257\\\\\n",
      "codellama/CodeLlama-7b-Python-hf\t& 0.3 & 0.232 & 0.246 & 0.259 & 0.263 & 0.269 & 0.280 & 0.272 & 0.279 & 0.278\\\\\n",
      "codellama/CodeLlama-7b-Python-hf\t& 0.5 & 0.227 & 0.245 & 0.267 & 0.270 & 0.286 & 0.288 & 0.291 & 0.293 & 0.296\\\\\n",
      "bigcode/starcoder2-3b\t& 0.1 & 0.245 & 0.253 & 0.249 & 0.253 & 0.252 & 0.250 & 0.253 & 0.251 & 0.253\\\\\n",
      "bigcode/starcoder2-3b\t& 0.3 & 0.246 & 0.261 & 0.273 & 0.282 & 0.284 & 0.286 & 0.289 & 0.285 & 0.291\\\\\n",
      "bigcode/starcoder2-3b\t& 0.5 & 0.232 & 0.252 & 0.268 & 0.284 & 0.292 & 0.291 & 0.297 & 0.300 & 0.299\\\\\n",
      "THUDM/codegeex2-6b\t& 0.1 & 0.162 & 0.162 & 0.164 & 0.166 & 0.168 & 0.168 & 0.170 & 0.168 & 0.169\\\\\n",
      "THUDM/codegeex2-6b\t& 0.3 & 0.159 & 0.166 & 0.174 & 0.183 & 0.182 & 0.185 & 0.183 & 0.187 & 0.186\\\\\n",
      "THUDM/codegeex2-6b\t& 0.5 & 0.161 & 0.166 & 0.178 & 0.184 & 0.193 & 0.198 & 0.202 & 0.209 & 0.215\\\\\n",
      "Salesforce/codegen25-7b-mono\t& 0.1 & 0.123 & 0.125 & 0.133 & 0.133 & 0.133 & 0.129 & 0.134 & 0.134 & 0.136\\\\\n",
      "Salesforce/codegen25-7b-mono\t& 0.3 & 0.113 & 0.134 & 0.143 & 0.150 & 0.154 & 0.157 & 0.157 & 0.156 & 0.157\\\\\n",
      "Salesforce/codegen25-7b-mono\t& 0.5 & 0.102 & 0.116 & 0.133 & 0.135 & 0.140 & 0.148 & 0.159 & 0.152 & 0.156\\\\\n",
      "Salesforce/codegen-16B-mono\t& 0.1 & 0.114 & 0.118 & 0.121 & 0.125 & 0.118 & 0.123 & 0.122 & 0.121 & 0.124\\\\\n",
      "Salesforce/codegen-16B-mono\t& 0.3 & 0.116 & 0.126 & 0.134 & 0.137 & 0.140 & 0.136 & 0.140 & 0.140 & 0.138\\\\\n",
      "Salesforce/codegen-16B-mono\t& 0.5 & 0.106 & 0.114 & 0.127 & 0.133 & 0.137 & 0.138 & 0.140 & 0.146 & 0.143\\\\\n",
      "Salesforce/codegen-6B-mono\t& 0.1 & 0.089 & 0.091 & 0.096 & 0.096 & 0.093 & 0.092 & 0.093 & 0.092 & 0.093\\\\\n",
      "Salesforce/codegen-6B-mono\t& 0.3 & 0.072 & 0.080 & 0.087 & 0.098 & 0.100 & 0.098 & 0.098 & 0.094 & 0.095\\\\\n",
      "Salesforce/codegen-6B-mono\t& 0.5 & 0.080 & 0.083 & 0.084 & 0.090 & 0.098 & 0.099 & 0.098 & 0.098 & 0.092\\\\\n",
      "Salesforce/codegen-2B-mono\t& 0.1 & 0.047 & 0.046 & 0.044 & 0.049 & 0.046 & 0.043 & 0.046 & 0.043 & 0.045\\\\\n",
      "Salesforce/codegen-2B-mono\t& 0.3 & 0.045 & 0.052 & 0.047 & 0.052 & 0.051 & 0.049 & 0.048 & 0.046 & 0.047\\\\\n",
      "Salesforce/codegen-2B-mono\t& 0.5 & 0.039 & 0.040 & 0.043 & 0.045 & 0.049 & 0.048 & 0.044 & 0.044 & 0.042\\\\\n",
      "EleutherAI/gpt-j-6b\t& 0.1 & 0.022 & 0.027 & 0.030 & 0.027 & 0.027 & 0.030 & 0.030 & 0.029 & 0.030\\\\\n",
      "EleutherAI/gpt-j-6b\t& 0.3 & 0.027 & 0.024 & 0.026 & 0.025 & 0.029 & 0.028 & 0.027 & 0.027 & 0.027\\\\\n",
      "EleutherAI/gpt-j-6b\t& 0.5 & 0.023 & 0.028 & 0.030 & 0.031 & 0.033 & 0.033 & 0.031 & 0.032 & 0.030\\\\\n",
      "facebook/incoder-6B\t& 0.1 & 0.024 & 0.024 & 0.025 & 0.023 & 0.025 & 0.028 & 0.027 & 0.027 & 0.027\\\\\n",
      "facebook/incoder-6B\t& 0.3 & 0.029 & 0.030 & 0.027 & 0.030 & 0.030 & 0.033 & 0.034 & 0.033 & 0.033\\\\\n",
      "facebook/incoder-6B\t& 0.5 & 0.030 & 0.028 & 0.029 & 0.033 & 0.030 & 0.035 & 0.034 & 0.034 & 0.037\\\\\n",
      "Salesforce/codegen-350M-mono\t& 0.1 & 0.013 & 0.014 & 0.012 & 0.013 & 0.013 & 0.012 & 0.014 & 0.013 & 0.014\\\\\n",
      "Salesforce/codegen-350M-mono\t& 0.3 & 0.010 & 0.010 & 0.013 & 0.011 & 0.009 & 0.011 & 0.012 & 0.013 & 0.013\\\\\n",
      "Salesforce/codegen-350M-mono\t& 0.5 & 0.011 & 0.010 & 0.008 & 0.008 & 0.008 & 0.009 & 0.008 & 0.006 & 0.008\\\\\n",
      "facebook/incoder-1B\t& 0.1 & 0.004 & 0.003 & 0.004 & 0.003 & 0.003 & 0.003 & 0.003 & 0.003 & 0.003\\\\\n",
      "facebook/incoder-1B\t& 0.3 & 0.005 & 0.005 & 0.005 & 0.005 & 0.005 & 0.005 & 0.005 & 0.005 & 0.006\\\\\n",
      "facebook/incoder-1B\t& 0.5 & 0.005 & 0.004 & 0.004 & 0.005 & 0.005 & 0.005 & 0.005 & 0.005 & 0.005\\\\\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv(\"GSM-res-voting.csv\")\n",
    "df_r = pd.read_csv(\"GSM-res.csv\")\n",
    "\n",
    "\n",
    "df_r = df_r[[\"model\", \"temperature\", \"nshots\", \"accuracy\"]]\n",
    "\n",
    "\n",
    "df = df.merge(df_r, on=[\"model\", \"temperature\", \"nshots\"], how=\"inner\", validate=\"1:1\")\n",
    "\n",
    "\n",
    "df = df.drop(columns=[\"Unnamed: 0\"])\n",
    "# df = df.sort_values(by=[\"model\", \"temperature\", \"accuracy_10\"], ascending=[False, True, False])\n",
    "\n",
    "for i, row in df.iterrows():\n",
    "    #print(row)\n",
    "    line = f\"{row[\"model\"]}\\t& {row[\"temperature\"]} & {row[\"accuracy\"]:.3f} & \" + \" & \".join([f\"{row[f\"accuracy_{n}\"]:.3f}\" for n in range(3, 11)]) + \"\\\\\\\\\"\n",
    "    print(line)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}