convert_fields_from_downloaded_dump_to_table.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "bdb40252",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Import libraries: \n",
    "import pandas as pd\n",
    "from bs4 import BeautifulSoup as soup\n",
    "import lxml.etree as ET\n",
    "import unicodedata\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "2ccbb664",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Load data from dump-file: \n",
    "with open(\"dataset_tutorial_eco.xml\", \"r\", encoding=\"utf-8\") as f:\n",
    "    result = f.readlines()\n",
    "content = \"\".join(result)\n",
    "\n",
    "#Parse data to xml: \n",
    "xml = soup(content, features=\"xml\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "cc2d0e39",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2828\n"
     ]
    }
   ],
   "source": [
    "#Find all bibliographic records in xml: \n",
    "records = xml.find_all('record', {'type':'Bibliographic'})\n",
    "\n",
    "#Print record at position 1 to double check that everything is fine:\n",
    "#print(records[1].prettify())\n",
    "\n",
    "print(len(records))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "fad2c888",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "100%|█████████████████████████████████████████████████████████████████████████████| 2828/2828 [00:20<00:00, 270.03it/s]\u001b[A"
     ]
    }
   ],
   "source": [
    "#Define function to parse each record and extract only the datafields needed: \n",
    "def parse_record(record):\n",
    "    \n",
    "    #ns = {\"marc\":\"http://www.loc.gov/MARC21/slim\"}\n",
    "    xml = ET.fromstring(unicodedata.normalize(\"NFC\", str(record)))\n",
    "      \n",
    "    #idn\n",
    "    idn = xml.xpath(\"controlfield[@tag = '001']\")\n",
    "    try:\n",
    "        idn = idn[0].text\n",
    "    except:\n",
    "        idn = 'fail'\n",
    "    \n",
    "    # title\n",
    "    title = xml.xpath(\"datafield[@tag = '245']/subfield[@code = 'a']\")\n",
    "    try:\n",
    "        title = title[0].text\n",
    "    except:\n",
    "        title = \"unknown\"\n",
    "         \n",
    "    # subtitle\n",
    "    subtitle = xml.xpath(\"datafield[@tag = '245']/subfield[@code = 'b']\")\n",
    "    try:\n",
    "        subtitle = subtitle[0].text\n",
    "    except:\n",
    "        subtitle = \"unknown\"   \n",
    " \n",
    "    #date:\n",
    "    date1 = xml.xpath(\"datafield[@tag = '264']/subfield[@code = 'c']\")\n",
    "    date2 = xml.xpath(\"controlfield[@tag = '008']\")\n",
    "        \n",
    "    if date1:\n",
    "        date = date1[0].text\n",
    "    elif date2: \n",
    "        date = date2[0].text  ## extract complete 008\n",
    "        date = date[7:11]     ## get only characters at positions 7-10\n",
    "    else:\n",
    "        date = \"none\"\n",
    "\n",
    "       \n",
    "    meta_dict = {\"idn\":idn, \"title\":title, \"subtitle\":subtitle, \"date\":date} #create dictionary\n",
    "    \n",
    "    return meta_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "406cea1f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████| 2828/2828 [00:22<00:00, 127.85it/s]\n",
      " 99%|████████████████████████████████████████████████████████████████████████████▌| 2812/2828 [00:09<00:00, 273.02it/s]"
     ]
    }
   ],
   "source": [
    "#Parse each record to function above: \n",
    "pbar = tqdm(total=len(records))\n",
    "\n",
    "result = []\n",
    "for record in records:\n",
    "    result.append(parse_record(record))\n",
    "    pbar.update()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "bd57104e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>idn</th>\n",
       "      <th>title</th>\n",
       "      <th>subtitle</th>\n",
       "      <th>date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1043718966</td>\n",
       "      <td>A cross-cultural study of motivational factors...</td>\n",
       "      <td>unknown</td>\n",
       "      <td>2010</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1019902175</td>\n",
       "      <td>A forecast evaluation of PCA-based adaptive ...</td>\n",
       "      <td>unknown</td>\n",
       "      <td>2010</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1012057232</td>\n",
       "      <td>A new perspective on social learning</td>\n",
       "      <td>unknown</td>\n",
       "      <td>2010</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1009567691</td>\n",
       "      <td>A reconsideration of full-cost pricing</td>\n",
       "      <td>methodological aspects of marginalism and theo...</td>\n",
       "      <td>2010</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1045290823</td>\n",
       "      <td>A study on the impact of mobile telecommunicat...</td>\n",
       "      <td>unknown</td>\n",
       "      <td>2010</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2823</th>\n",
       "      <td>120766099X</td>\n",
       "      <td>Modellierung der Zahlungsschwierigkeiten vonPr...</td>\n",
       "      <td>unknown</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2824</th>\n",
       "      <td>961697555</td>\n",
       "      <td>Object Warehouse</td>\n",
       "      <td>Konzeption der Basis objektorientierter Manage...</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2825</th>\n",
       "      <td>1035450941</td>\n",
       "      <td>Phasenfeldmodellierung mehrphasiger Erstarrung</td>\n",
       "      <td>unknown</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2826</th>\n",
       "      <td>1206191139</td>\n",
       "      <td>Reform der gemeinsamen Agrarpolitik und EU-Int...</td>\n",
       "      <td>unknown</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2827</th>\n",
       "      <td>995598835</td>\n",
       "      <td>Strategische Entscheidungen der europäischen L...</td>\n",
       "      <td>unknown</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2828 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             idn                                              title  \\\n",
       "0     1043718966  A cross-cultural study of motivational factors...   \n",
       "1     1019902175  A forecast evaluation of PCA-based adaptive ...   \n",
       "2     1012057232             A new perspective on social learning   \n",
       "3     1009567691           A reconsideration of full-cost pricing   \n",
       "4     1045290823  A study on the impact of mobile telecommunicat...   \n",
       "...          ...                                                ...   \n",
       "2823  120766099X  Modellierung der Zahlungsschwierigkeiten vonPr...   \n",
       "2824   961697555                                   Object Warehouse   \n",
       "2825  1035450941     Phasenfeldmodellierung mehrphasiger Erstarrung   \n",
       "2826  1206191139  Reform der gemeinsamen Agrarpolitik und EU-Int...   \n",
       "2827   995598835  Strategische Entscheidungen der europäischen L...   \n",
       "\n",
       "                                               subtitle  date  \n",
       "0                                               unknown  2010  \n",
       "1                                               unknown  2010  \n",
       "2                                               unknown  2010  \n",
       "3     methodological aspects of marginalism and theo...  2010  \n",
       "4                                               unknown  2010  \n",
       "...                                                 ...   ...  \n",
       "2823                                            unknown  2000  \n",
       "2824  Konzeption der Basis objektorientierter Manage...  2000  \n",
       "2825                                            unknown  2000  \n",
       "2826                                            unknown  2000  \n",
       "2827                                            unknown  2000  \n",
       "\n",
       "[2828 rows x 4 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Convert result to dataframe:\n",
    "df = pd.DataFrame(result)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "2dbeff0e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>idn</th>\n",
       "      <th>title</th>\n",
       "      <th>subtitle</th>\n",
       "      <th>date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2824</th>\n",
       "      <td>961697555</td>\n",
       "      <td>Object Warehouse</td>\n",
       "      <td>Konzeption der Basis objektorientierter Manage...</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            idn             title  \\\n",
       "2824  961697555  Object Warehouse   \n",
       "\n",
       "                                               subtitle  date  \n",
       "2824  Konzeption der Basis objektorientierter Manage...  2000  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#If needed: Search for a specific title in dataframe:\n",
    "df.query(\"title == 'Object Warehouse'\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "1193a547",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(\"Eco_small_dataset.csv\", encoding=\"utf-8\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8717c9a3",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8c0e1ada",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}