GPT_Auto_Review.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "7b72f689",
   "metadata": {},
   "source": [
    "Simple proof of concept that we can extract key information from PDF of a research paper. We demonstrate extraction of title, abstract, and first section. \n",
    "\n",
    "It appears getting all the sections is tricky with OCR. Also, figures and other visual components are not recoverable. Furthermore, math is not rendered properly or even at all depending on formatting. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "e2edd6e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "import PyPDF2\n",
    "import pdfplumber\n",
    "\n",
    "import re\n",
    "\n",
    "import pytesseract\n",
    "from pdf2image import convert_from_path\n",
    "\n",
    "def extract_text_from_pdf(pdf_path):\n",
    "    images = convert_from_path(pdf_path)\n",
    "    text = ''\n",
    "    for i in range(min(10,len(images))):\n",
    "        text += pytesseract.image_to_string(images[i])\n",
    "    return text\n",
    "\n",
    "def extract_title(text):\n",
    "    lines = text.split('\\n')\n",
    "    for line in lines:\n",
    "        if line.strip():\n",
    "            return line.strip()\n",
    "    return None\n",
    "\n",
    "def extract_abstract(text):\n",
    "    abstract_start = 'Abstract\\n\\n'\n",
    "    abstract_end = '\\n\\n1 Introduction'\n",
    "\n",
    "    abstract_regex = rf\"{abstract_start}(.+?){abstract_end}\"\n",
    "    match = re.search(abstract_regex, text, re.S)\n",
    "    \n",
    "    if match:\n",
    "        abstract = match.group(1).strip()\n",
    "        abstract = re.sub(r'\\s+', ' ', abstract)  # Replace multiple consecutive spaces with a single space\n",
    "        return abstract\n",
    "    else:\n",
    "        return None\n",
    "\n",
    "def extract_section(text, section_number):\n",
    "    lines = text.split('\\n')\n",
    "    section_start_pattern = f\"^{section_number} \"\n",
    "    section_end_pattern = f\"^{section_number + 1} \"\n",
    "\n",
    "    section_started = False\n",
    "    section_content_lines = []\n",
    "\n",
    "    for line in lines:\n",
    "        if re.match(section_end_pattern, line):\n",
    "            break\n",
    "\n",
    "        if section_started:\n",
    "            section_content_lines.append(line)\n",
    "\n",
    "        if re.match(section_start_pattern, line):\n",
    "            section_started = True\n",
    "\n",
    "    section_content = \" \".join(section_content_lines)\n",
    "    section_content = re.sub(r'\\s+', ' ', section_content)  # Replace multiple consecutive spaces with a single space\n",
    "\n",
    "    return section_content if section_content else None\n",
    "\n",
    "def read_text_file(file_path):\n",
    "    with open(file_path, 'r', encoding='utf-8') as file:\n",
    "        content = file.read()\n",
    "    return content"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3d88a1b2",
   "metadata": {},
   "source": [
    "This just extracts text without consideration of formatting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b77a3f84",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "48381\n"
     ]
    }
   ],
   "source": [
    "pdf_path = \"Validation_Papers/example.pdf\"\n",
    "text = extract_text_from_pdf(pdf_path)\n",
    "print(\"Number of characters in text\")\n",
    "print(len(text))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2d56e7f3",
   "metadata": {},
   "source": [
    "If you have a Latex file this works as well."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "1f153839",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "63029\n"
     ]
    }
   ],
   "source": [
    "text_file_path = 'Validation_Papers/example.tex'\n",
    "text = read_text_file(text_file_path)\n",
    "print(\"Number of characters in text\")\n",
    "print(len(text))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e142fb23",
   "metadata": {},
   "source": [
    "If you trust the format of the paper we can extract the title, abstract, and section text. You most likely need to manually add the abstract."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "7837fc05",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Title:\n",
      "\\documentclass{article}\n",
      "\n",
      "Abstract:\n",
      "None\n",
      "\n",
      "Section 1:\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "title = extract_title(text)\n",
    "abstract = extract_abstract(text)\n",
    "\n",
    "print(\"Title:\")\n",
    "print(title)\n",
    "print(\"\\nAbstract:\")\n",
    "print(abstract)\n",
    "\n",
    "section_1_text = extract_section(text, 1)\n",
    "print(\"\\nSection 1:\")\n",
    "print(section_1_text)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0be1a19c",
   "metadata": {},
   "source": [
    "In the next part we experiment with the OpenAI API. You need to provide a key of your own."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "cdce81cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "import openai\n",
    "import tiktoken\n",
    "\n",
    "def num_tokens_from_string(string: str, encoding_name = \"cl100k_base\") -> int:\n",
    "    \"\"\"Returns the number of tokens in a text string.\"\"\"\n",
    "    encoding = tiktoken.get_encoding(encoding_name)\n",
    "    num_tokens = len(encoding.encode(string))\n",
    "    return num_tokens\n",
    "\n",
    "# Provide your key here\n",
    "openai.api_key = \"\"\n",
    "\n",
    "use_model=\"gpt-3.5-turbo\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b6ec560f",
   "metadata": {},
   "source": [
    "First, let's calculate the number of tokens in our paper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ddb0d328",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "15247"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_tokens_from_string(text)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "51adb3ea",
   "metadata": {},
   "source": [
    "The GPT-4 API has a context size of 8k tokens. Roughly, one could nearly fit an entire NeurIPS formatted paper and expect a reasonable summary of the paper. The relevant challenges are that we also need a system prompt that communicates the current-objective to GPT. We want something simple, but effective so we'll go with a single-pass:\n",
    "\n",
    "1. GPT will take parts of the paper as input and take notes relevant to a system-prompt describing the task + abstract + rubric\n",
    "2. GPT will then organize these notes\n",
    "3. GPT will write a review using a system-prompt describing NeurIPS style reviewer guidelines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "6cfc0770",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "364\n",
      "337\n",
      "1638\n"
     ]
    }
   ],
   "source": [
    "# Load note_instructions\n",
    "print(\"Token lengths\")\n",
    "note_prompt = read_text_file(\"Note_Instruction.txt\")\n",
    "print(num_tokens_from_string(note_prompt))\n",
    "\n",
    "# Load note organization instructions\n",
    "organize_prompt = read_text_file(\"Organize_Notes.txt\")\n",
    "print(num_tokens_from_string(organize_prompt))\n",
    "\n",
    "# Load note organization instructions\n",
    "review_guidelines = read_text_file(\"NeurIPS_Guidelines.txt\")\n",
    "print(num_tokens_from_string(review_guidelines))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "22237e1f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create basic prompt scaffold\n",
    "def notes(text, prompt, current_notes = ''):\n",
    "    \n",
    "    response = openai.ChatCompletion.create(\n",
    "      model=use_model,\n",
    "      messages=[\n",
    "            {\"role\": \"system\", \"content\": \"You are a helpful assistant. \" + prompt},\n",
    "            {\"role\": \"user\", \"content\": \"\\n Text: \" + text + \"\\n Context: \" + current_notes + \"\\n Notes: \"},\n",
    "        ]\n",
    "    )\n",
    "\n",
    "    summary = response['choices'][0]['message']['content']\n",
    "    return summary\n",
    "\n",
    "# Create basic prompt scaffold\n",
    "def review(text, prompt):\n",
    "    \n",
    "    response = openai.ChatCompletion.create(\n",
    "      model=use_model,\n",
    "      messages=[\n",
    "            {\"role\": \"system\", \"content\": \"You are a helpful assistant. \" + prompt},\n",
    "            {\"role\": \"user\", \"content\": \"\\n Text: \" + text + \"\\n 1. Summary and contributions: \"},\n",
    "        ]\n",
    "    )\n",
    "\n",
    "    summary = response['choices'][0]['message']['content']\n",
    "    return summary"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "acc47d47",
   "metadata": {},
   "source": [
    "The code needs to be babysat becuase the API is inconsistent. If the code throws an error in the middle of aggregating notes you can comment out the \"all_notes\" variable and set the chunk variable to the current location."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "67ac3d73",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Length of paper: 63029\n",
      "Current Chunk: 3\n",
      "Progress: 24000 out of 63029\n",
      "Length of notes: 4072\n",
      "Current Chunk: 4\n",
      "Progress: 36000 out of 63029\n",
      "Length of notes: 5591\n",
      "Current Chunk: 5\n",
      "Progress: 48000 out of 63029\n",
      "Length of notes: 5625\n",
      "Current Chunk: 6\n",
      "Progress: 60000 out of 63029\n",
      "Length of notes: 7104\n"
     ]
    }
   ],
   "source": [
    "chunk = 1\n",
    "context_size = 12000\n",
    "#context_size = 15000\n",
    "exit = False\n",
    "all_notes = ''\n",
    "print(\"Length of paper: \" + str(len(text)))\n",
    "while not exit:\n",
    "    print(\"Current Chunk: \" + str(chunk))\n",
    "    print(\"Progress: \" + str((chunk-1)*context_size) + \" out of \" + str(len(text)))\n",
    "    print(\"Length of notes: \" + str(len(all_notes)))\n",
    "    if context_size*chunk >= len(text):\n",
    "        \n",
    "        paper_text = text[context_size*(chunk-1):]\n",
    "        #print(num_tokens_from_string(note_prompt + paper_text))\n",
    "        paper_notes = notes(paper_text, note_prompt)\n",
    "        #print(paper_notes)\n",
    "        exit = True\n",
    "    else:\n",
    "        paper_text = text[context_size*(chunk-1):context_size*chunk]\n",
    "        #print(num_tokens_from_string(note_prompt + paper_text))\n",
    "        paper_notes = notes(paper_text, note_prompt)\n",
    "        #print(paper_notes)\n",
    "        \n",
    "    chunk += 1\n",
    "    all_notes += paper_notes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d910094a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Organizing Notes\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'notes' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn [1], line 2\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mOrganizing Notes\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m----> 2\u001b[0m organized_notes \u001b[38;5;241m=\u001b[39m notes(all_notes, organize_prompt)\n\u001b[1;32m      3\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mGenerating Review\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m      4\u001b[0m review_text \u001b[38;5;241m=\u001b[39m review(abstract \u001b[38;5;241m+\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m+\u001b[39m organized_notes, review_guidelines)\n",
      "\u001b[0;31mNameError\u001b[0m: name 'notes' is not defined"
     ]
    }
   ],
   "source": [
    "print(\"Organizing Notes\")\n",
    "organized_notes = notes(all_notes, organize_prompt)\n",
    "print(\"Generating Review\")\n",
    "review_text = review(abstract + \"\\n\" + organized_notes, review_guidelines)\n",
    "print(review_text)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}